x/box-o-sand
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Cleaning, extracting, being a good person

Browse Source
This commit is contained in:
Dan Buch 2012-02-29 08:35:33 -05:00
parent 293006314b
commit 04182f3b80
1 changed files with 33 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
redis-with-ruby/earthquakes.rb
View File

@ -7,15 +7,14 @@ require 'open-uri'
class Earthquakes class Earthquakes
DATA_URL = 'http://earthquake.usgs.gov/earthquakes/catalogs/eqs7day-M1.txt' attr_accessor :id_set, :region_set, :magnitude_set, :data_url
attr_accessor :id_set, :region_set, :magnitude_set
def initialize def initialize
@redis = Redis.new @redis = Redis.new
@id_set = 'earthquake-ids' @id_set = 'earthquake-ids'
@region_set = 'earthquake-regions' @region_set = 'earthquake-regions'
@magnitude_set = 'earthquake-magnitudes' @magnitude_set = 'earthquake-magnitudes'
@data_url = 'http://earthquake.usgs.gov/earthquakes/catalogs/eqs7day-M1.txt'
@logger = Logger.new(STDOUT) @logger = Logger.new(STDOUT)
@logger.formatter = proc do |severity, datetime, progname, msg| @logger.formatter = proc do |severity, datetime, progname, msg|
@ -66,30 +65,44 @@ class Earthquakes
def load_data def load_data
data = CSV.parse( data = CSV.parse(
open(DATA_URL).read, :headers => true, :header_converters => [:symbol] open(@data_url).read, :headers => true, :header_converters => [:symbol]
) )
data.each do |row| data.each do |row|
@redis.multi do |r|
@logger.info("Adding earthquake #{row[:eqid]}") @logger.info("Adding earthquake #{row[:eqid]}")
id_key = with_prefix(row[:eqid].downcase, 'eqid') load_one_row(row)
r.sadd(@id_set, id_key)
row.headers.each do |header|
r.hset(id_key, header.downcase, row[header].downcase)
end
region_key = with_prefix(row[:region].downcase, 'eqregion')
r.sadd(region_key, id_key)
r.sadd(@region_set, region_key)
magnitude_key = with_prefix(row[:magnitude], 'eqmagnitude')
r.sadd(magnitude_key, id_key)
r.sadd(@magnitude_set, magnitude_key)
end
end end
end end
private private
def load_one_row(row)
@redis.multi do |redis_multi|
id_key = with_prefix(row[:eqid].downcase, 'eqid')
load_id_and_attrs_for_row(row, id_key, redis_multi)
load_region_for_row(row, id_key, redis_multi)
load_magnitude_for_row(row, id_key, redis_multi)
end
end
def load_id_and_attrs_for_row(row, id_key, redis_multi)
redis_multi.sadd(@id_set, id_key)
row.headers.each do |header|
redis_multi.hset(id_key, header.downcase, row[header].downcase)
end
end
def load_region_for_row(row, id_key, redis_multi)
region_key = with_prefix(row[:region].downcase, 'eqregion')
redis_multi.sadd(region_key, id_key)
redis_multi.sadd(@region_set, region_key)
end
def load_magnitude_for_row(row, id_key, redis_multi)
magnitude_key = with_prefix(row[:magnitude], 'eqmagnitude')
redis_multi.sadd(magnitude_key, id_key)
redis_multi.sadd(@magnitude_set, magnitude_key)
end
def with_prefix(key, prefix) def with_prefix(key, prefix)
if not /^#{prefix}:/.match(key) if not /^#{prefix}:/.match(key)
return "#{prefix}:#{key}" return "#{prefix}:#{key}"