Goofing around with minitar, typhoeus, and progressbar
for purposes of actually working with PostgreSQL and weather data.
This commit is contained in:
parent
65d5bae610
commit
d747f2d680
5
postgresql/tutorial/weather/Gemfile
Normal file
5
postgresql/tutorial/weather/Gemfile
Normal file
@ -0,0 +1,5 @@
|
||||
source :rubygems
|
||||
|
||||
gem 'archive-tar-minitar'
|
||||
gem 'progressbar'
|
||||
gem 'typhoeus'
|
18
postgresql/tutorial/weather/Gemfile.lock
Normal file
18
postgresql/tutorial/weather/Gemfile.lock
Normal file
@ -0,0 +1,18 @@
|
||||
GEM
|
||||
remote: http://rubygems.org/
|
||||
specs:
|
||||
archive-tar-minitar (0.5.2)
|
||||
ffi (1.1.0)
|
||||
mime-types (1.19)
|
||||
progressbar (0.9.0)
|
||||
typhoeus (0.4.2)
|
||||
ffi (~> 1.0)
|
||||
mime-types (~> 1.18)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
archive-tar-minitar
|
||||
progressbar
|
||||
typhoeus
|
@ -1,25 +1,52 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env ruby
|
||||
|
||||
FTP_BASE='ftp://ftp.ncdc.noaa.gov/pub/data/gsod'
|
||||
require 'fileutils'
|
||||
|
||||
pushd $(dirname $(readlink -f $0))
|
||||
require 'bundler/setup'
|
||||
require 'archive/tar/minitar'
|
||||
require 'progressbar'
|
||||
require 'typhoeus'
|
||||
|
||||
pushd ./data
|
||||
if [[ ! -f ish-history.csv ]]
|
||||
then
|
||||
curl -O $FTP_BASE/ish-history.csv
|
||||
fi
|
||||
FTP_BASE = 'ftp://ftp.ncdc.noaa.gov/pub/data/gsod'
|
||||
YEARS = %w(1950 1960 1970 1980 1990 2000 2010)
|
||||
|
||||
for year in 1950 1960 1970 1980 1990 2000 2010
|
||||
do
|
||||
tarname=gsod_$year.tar
|
||||
if [[ ! -f $tarname ]]
|
||||
then
|
||||
curl -O $FTP_BASE/$year/$tarname
|
||||
fi
|
||||
tar xf $tarname
|
||||
for gzfile in *-$year.op.gz
|
||||
do
|
||||
:
|
||||
done
|
||||
done
|
||||
def main
|
||||
include Archive::Tar
|
||||
dest_dir = File.expand_path('../data', __FILE__)
|
||||
Dir.chdir(dest_dir)
|
||||
|
||||
download_here_maybe?("#{FTP_BASE}/ish-history.csv")
|
||||
|
||||
YEARS.each do |year|
|
||||
tarname = "gsod_#{year}.tar"
|
||||
url = "#{FTP_BASE}/#{year}/#{tarname}"
|
||||
dest = "#{dest_dir}/#{tarname}"
|
||||
download_here_maybe?(url)
|
||||
to_unpack = Minitar.open(dest, 'r').collect(&:full_name).select do |f|
|
||||
f =~ /\.gz$/ && !File.exist?(f)
|
||||
end
|
||||
if !to_unpack.empty?
|
||||
@progress = ProgressBar.new(File.basename(dest), to_unpack.length)
|
||||
Minitar.unpack(dest, dest_dir, to_unpack) do |action,name,stats|
|
||||
if action == :file_done
|
||||
@progress.inc(1)
|
||||
end
|
||||
end
|
||||
puts
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def download_here_maybe?(url)
|
||||
outfile = File.basename(url)
|
||||
if !File.exist?(outfile)
|
||||
File.open(outfile, 'w') do |f|
|
||||
puts "Writing #{url} to #{outfile}"
|
||||
f.write(Typhoeus::Request.get(url).body)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if $0 == __FILE__
|
||||
main
|
||||
end
|
||||
|
Loading…
Reference in New Issue
Block a user