Cleaning, extracting, being a good person
This commit is contained in:
parent
293006314b
commit
04182f3b80
@ -7,15 +7,14 @@ require 'open-uri'
|
||||
|
||||
|
||||
class Earthquakes
|
||||
DATA_URL = 'http://earthquake.usgs.gov/earthquakes/catalogs/eqs7day-M1.txt'
|
||||
|
||||
attr_accessor :id_set, :region_set, :magnitude_set
|
||||
attr_accessor :id_set, :region_set, :magnitude_set, :data_url
|
||||
|
||||
def initialize
|
||||
@redis = Redis.new
|
||||
@id_set = 'earthquake-ids'
|
||||
@region_set = 'earthquake-regions'
|
||||
@magnitude_set = 'earthquake-magnitudes'
|
||||
@data_url = 'http://earthquake.usgs.gov/earthquakes/catalogs/eqs7day-M1.txt'
|
||||
|
||||
@logger = Logger.new(STDOUT)
|
||||
@logger.formatter = proc do |severity, datetime, progname, msg|
|
||||
@ -66,30 +65,44 @@ class Earthquakes
|
||||
|
||||
def load_data
|
||||
data = CSV.parse(
|
||||
open(DATA_URL).read, :headers => true, :header_converters => [:symbol]
|
||||
open(@data_url).read, :headers => true, :header_converters => [:symbol]
|
||||
)
|
||||
|
||||
data.each do |row|
|
||||
@redis.multi do |r|
|
||||
@logger.info("Adding earthquake #{row[:eqid]}")
|
||||
id_key = with_prefix(row[:eqid].downcase, 'eqid')
|
||||
r.sadd(@id_set, id_key)
|
||||
row.headers.each do |header|
|
||||
r.hset(id_key, header.downcase, row[header].downcase)
|
||||
end
|
||||
|
||||
region_key = with_prefix(row[:region].downcase, 'eqregion')
|
||||
r.sadd(region_key, id_key)
|
||||
r.sadd(@region_set, region_key)
|
||||
|
||||
magnitude_key = with_prefix(row[:magnitude], 'eqmagnitude')
|
||||
r.sadd(magnitude_key, id_key)
|
||||
r.sadd(@magnitude_set, magnitude_key)
|
||||
end
|
||||
load_one_row(row)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def load_one_row(row)
|
||||
@redis.multi do |redis_multi|
|
||||
id_key = with_prefix(row[:eqid].downcase, 'eqid')
|
||||
load_id_and_attrs_for_row(row, id_key, redis_multi)
|
||||
load_region_for_row(row, id_key, redis_multi)
|
||||
load_magnitude_for_row(row, id_key, redis_multi)
|
||||
end
|
||||
end
|
||||
|
||||
def load_id_and_attrs_for_row(row, id_key, redis_multi)
|
||||
redis_multi.sadd(@id_set, id_key)
|
||||
row.headers.each do |header|
|
||||
redis_multi.hset(id_key, header.downcase, row[header].downcase)
|
||||
end
|
||||
end
|
||||
|
||||
def load_region_for_row(row, id_key, redis_multi)
|
||||
region_key = with_prefix(row[:region].downcase, 'eqregion')
|
||||
redis_multi.sadd(region_key, id_key)
|
||||
redis_multi.sadd(@region_set, region_key)
|
||||
end
|
||||
|
||||
def load_magnitude_for_row(row, id_key, redis_multi)
|
||||
magnitude_key = with_prefix(row[:magnitude], 'eqmagnitude')
|
||||
redis_multi.sadd(magnitude_key, id_key)
|
||||
redis_multi.sadd(@magnitude_set, magnitude_key)
|
||||
end
|
||||
|
||||
def with_prefix(key, prefix)
|
||||
if not /^#{prefix}:/.match(key)
|
||||
return "#{prefix}:#{key}"
|
||||
|
Loading…
Reference in New Issue
Block a user