box-o-sand/postgresql/tutorial/weather/import-data

#!/usr/bin/env ruby

require 'fileutils'

require 'bundler/setup'
require 'archive/tar/minitar'
require 'progressbar'
require 'typhoeus'

FTP_BASE = 'ftp://ftp.ncdc.noaa.gov/pub/data/gsod'
YEARS = %w(1950 1960 1970 1980 1990 2000 2010)

def main
  include Archive::Tar
  dest_dir = File.expand_path('../data', __FILE__)
  Dir.chdir(dest_dir)

  download_here_maybe?("#{FTP_BASE}/ish-history.csv")

  YEARS.each do |year|
    tarname = "gsod_#{year}.tar"
    url = "#{FTP_BASE}/#{year}/#{tarname}"
    dest = "#{dest_dir}/#{tarname}"
    download_here_maybe?(url)
    to_unpack = Minitar.open(dest, 'r').collect(&:full_name).select do |f|
      f =~ /\.gz$/ && !File.exist?(f)
    end
    if !to_unpack.empty?
      @progress = ProgressBar.new(File.basename(dest), to_unpack.length)
      Minitar.unpack(dest, dest_dir, to_unpack) do |action,name,stats|
        if action == :file_done
          @progress.inc(1)
        end
      end
      puts
    end
  end
end

def download_here_maybe?(url)
  outfile = File.basename(url)
  if !File.exist?(outfile)
    File.open(outfile, 'w') do |f|
      puts "Writing #{url} to #{outfile}"
      f.write(Typhoeus::Request.get(url).body)
    end
  end
end

if $0 == __FILE__
  main
end