require 'base64' require 'logger' require 'uri' require 'nokogiri' require 'typhoeus' class GoogleMapLocationFetcher attr_accessor :base_map_url, :log def initialize @base_map_url = [ 'http://maps.googleapis.com/maps/api/staticmap', '?zoom=15', '&sensor=false', '&size=512x512', '&maptype=satellite', ].join('') @log = Logger.new( File.expand_path('../log/map-crawler.log', File.dirname(__FILE__)) ) @log.level = Logger::INFO @log.formatter = lambda do |severity, time, prog, message| "#{time} - #{severity} - #{message}\n" end end def self.mapdump_callback(location, image) puts "Map '#{location}':" puts Base64.encode64(image) end def fetch(locations, &callback) callback ||= self.class.method(:mapdump_callback) hydra = Typhoeus::Hydra.new(:initial_pool_size => 26) locations.each do |location| request = Typhoeus::Request.new( "#{@base_map_url}¢er=#{URI.encode(location)}" ) request.on_complete do |response| handle_response(response, location, &callback) end hydra.queue(request) end hydra.run end def handle_response(response, location, &callback) @log.info("Handling request at url #{response.effective_url}") if response.success? and response.headers_hash[:content_type] =~ /image\/.*/ callback.call(location, response.body) else callback.call(location, '') end end end