62 lines
1.4 KiB
Ruby
62 lines
1.4 KiB
Ruby
|
require 'base64'
|
||
|
|
require 'logger'
|
||
|
|
require 'uri'
|
||
|
|
|
||
|
|
require 'nokogiri'
|
||
|
|
require 'typhoeus'
|
||
|
|
|
||
|
|
|
||
|
|
class GoogleMapLocationFetcher
|
||
|
|
attr_accessor :base_map_url, :log
|
||
|
|
|
||
|
|
def initialize
|
||
|
|
@base_map_url = [
|
||
|
|
'http://maps.googleapis.com/maps/api/staticmap',
|
||
|
|
'?zoom=12',
|
||
|
|
'&sensor=false',
|
||
|
|
'&size=512x512',
|
||
|
|
'&maptype=satellite',
|
||
|
|
].join('')
|
||
|
|
|
||
|
|
@log = Logger.new(
|
||
|
|
File.expand_path('../log/map-crawler.log', File.dirname(__FILE__))
|
||
|
|
)
|
||
|
|
@log.level = Logger::INFO
|
||
|
|
@log.formatter = lambda do |severity, time, prog, message|
|
||
|
|
"#{time} - #{severity} - #{message}\n"
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def self.mapdump_callback(location, image)
|
||
|
|
puts "Map '#{location}':"
|
||
|
|
puts Base64.encode64(image)
|
||
|
|
end
|
||
|
|
|
||
|
|
def fetch(locations, &callback)
|
||
|
|
callback ||= self.class.method(:mapdump_callback)
|
||
|
|
hydra = Typhoeus::Hydra.new(:initial_pool_size => 26)
|
||
|
|
|
||
|
|
locations.each do |location|
|
||
|
|
request = Typhoeus::Request.new(
|
||
|
|
"#{@base_map_url}¢er=#{URI.encode(location)}"
|
||
|
|
)
|
||
|
|
request.on_complete do |response|
|
||
|
|
handle_response(response, location, &callback)
|
||
|
|
end
|
||
|
|
|
||
|
|
hydra.queue(request)
|
||
|
|
end
|
||
|
|
|
||
|
|
hydra.run
|
||
|
|
end
|
||
|
|
|
||
|
|
def handle_response(response, location, &callback)
|
||
|
|
@log.info("Handling request at url #{response.effective_url}")
|
||
|
|
if response.success? and response.headers_hash[:content_type] =~ /image\/.*/
|
||
|
|
callback.call(location, response.body)
|
||
|
|
else
|
||
|
|
callback.call(location, '')
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|