You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
53 lines
1.2 KiB
53 lines
1.2 KiB
#!/usr/bin/env ruby
|
|
|
|
require 'fileutils'
|
|
|
|
require 'bundler/setup'
|
|
require 'archive/tar/minitar'
|
|
require 'progressbar'
|
|
require 'typhoeus'
|
|
|
|
FTP_BASE = 'ftp://ftp.ncdc.noaa.gov/pub/data/gsod'
|
|
YEARS = %w(1950 1960 1970 1980 1990 2000 2010)
|
|
|
|
def main
|
|
include Archive::Tar
|
|
dest_dir = File.expand_path('../data', __FILE__)
|
|
Dir.chdir(dest_dir)
|
|
|
|
download_here_maybe?("#{FTP_BASE}/ish-history.csv")
|
|
|
|
YEARS.each do |year|
|
|
tarname = "gsod_#{year}.tar"
|
|
url = "#{FTP_BASE}/#{year}/#{tarname}"
|
|
dest = "#{dest_dir}/#{tarname}"
|
|
download_here_maybe?(url)
|
|
to_unpack = Minitar.open(dest, 'r').collect(&:full_name).select do |f|
|
|
f =~ /\.gz$/ && !File.exist?(f)
|
|
end
|
|
if !to_unpack.empty?
|
|
@progress = ProgressBar.new(File.basename(dest), to_unpack.length)
|
|
Minitar.unpack(dest, dest_dir, to_unpack) do |action,name,stats|
|
|
if action == :file_done
|
|
@progress.inc(1)
|
|
end
|
|
end
|
|
puts
|
|
end
|
|
end
|
|
end
|
|
|
|
def download_here_maybe?(url)
|
|
outfile = File.basename(url)
|
|
if !File.exist?(outfile)
|
|
File.open(outfile, 'w') do |f|
|
|
puts "Writing #{url} to #{outfile}"
|
|
f.write(Typhoeus::Request.get(url).body)
|
|
end
|
|
end
|
|
end
|
|
|
|
if $0 == __FILE__
|
|
main
|
|
end
|