From ee337ba7eeec0f68c674849d5481a86e7861e0d2 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Wed, 7 Mar 2012 00:11:05 -0500 Subject: [PATCH] It's been waaay too long since I last committed. Eesh. --- rails/map-mash/Gemfile | 2 + rails/map-mash/Gemfile.lock | 3 + .../app/controllers/mashes_controller.rb | 45 ++--- rails/map-mash/app/models/map.rb | 20 ++ .../config/initializers/01-settings.rb | 2 +- rails/map-mash/config/settings.yml | 5 + rails/map-mash/config/settings/default.yml | 8 - rails/map-mash/db/capital-cities.csv | 191 ++++++++++++++++++ .../db/migrate/20120304151123_create_maps.rb | 3 +- .../migrate/20120304164625_create_mashes.rb | 8 +- rails/map-mash/db/schema.rb | 11 +- .../lib/google_map_location_fetcher.rb | 61 ++++++ rails/map-mash/lib/map_crawler.rb | 49 ----- rails/map-mash/lib/tasks/maps.rake | 13 +- rails/map-mash/spec/lib/map_crawler_spec.rb | 19 -- 15 files changed, 323 insertions(+), 117 deletions(-) create mode 100644 rails/map-mash/config/settings.yml delete mode 100644 rails/map-mash/config/settings/default.yml create mode 100644 rails/map-mash/db/capital-cities.csv create mode 100644 rails/map-mash/lib/google_map_location_fetcher.rb delete mode 100644 rails/map-mash/lib/map_crawler.rb delete mode 100644 rails/map-mash/spec/lib/map_crawler_spec.rb diff --git a/rails/map-mash/Gemfile b/rails/map-mash/Gemfile index 34726d1..869f668 100644 --- a/rails/map-mash/Gemfile +++ b/rails/map-mash/Gemfile @@ -4,6 +4,7 @@ gem 'rails', '2.3.2' gem 'active_presenter', '1.2.1' gem 'activesupport', '2.3.2' gem 'awesome_print', '0.2.1', :require => 'ap' +gem 'fastercsv' gem 'haml' gem 'mc-settings' gem 'mongrel', '1.1.5' @@ -11,6 +12,7 @@ gem 'nokogiri', '1.4.3.1' gem 'rack', '1.2.1' gem 'rake', '0.8.7' gem 'rdoc' +gem 'redis' gem 'redis-session-store' gem 'responds_to_parent', '1.0.20091013' gem 'resque', '1.19.0' diff --git a/rails/map-mash/Gemfile.lock b/rails/map-mash/Gemfile.lock index 198d1e7..6a62041 100644 --- a/rails/map-mash/Gemfile.lock +++ b/rails/map-mash/Gemfile.lock @@ -31,6 +31,7 @@ GEM fakeredis (0.2.2) redis (~> 2.2.0) fakeweb (1.3.0) + fastercsv (1.5.3) fastthread (1.0.7) ffi (1.0.11) foreman (0.40.0) @@ -168,6 +169,7 @@ DEPENDENCIES fake_ftp (= 0.0.9) fakeredis (= 0.2.2) fakeweb (= 1.3.0) + fastercsv foreman guard guard-livereload @@ -183,6 +185,7 @@ DEPENDENCIES rake (= 0.8.7) rcov (= 0.9.9) rdoc + redis redis-session-store remarkable_activerecord (= 3.1.13) remarkable_rails (= 3.1.13) diff --git a/rails/map-mash/app/controllers/mashes_controller.rb b/rails/map-mash/app/controllers/mashes_controller.rb index bbf4e79..6fce0a8 100644 --- a/rails/map-mash/app/controllers/mashes_controller.rb +++ b/rails/map-mash/app/controllers/mashes_controller.rb @@ -1,44 +1,43 @@ class MashesController < ApplicationController - # GET /mashes - # GET /mashes.xml + def index @mashes = Mash.all respond_to do |format| - format.html # index.html.erb - format.xml { render :xml => @mashes } + format.html + format.xml { render :xml => @mashes } end end - # GET /mashes/1 - # GET /mashes/1.xml def show @mash = Mash.find(params[:id]) respond_to do |format| - format.html # show.html.erb - format.xml { render :xml => @mash } + format.html + format.xml { render :xml => @mash } end end - # GET /mashes/new - # GET /mashes/new.xml def new - @mash = Mash.new + map_a, map_b = Map.rand(2) + + @mash = Mash.new( + :requestor => request.remote_ip, + :map_a => map_a.id, + :map_b => map_b.id, + :winner => 0 + ) respond_to do |format| - format.html # new.html.erb - format.xml { render :xml => @mash } + format.html + format.xml { render :xml => @mash } end end - # GET /mashes/1/edit def edit @mash = Mash.find(params[:id]) end - # POST /mashes - # POST /mashes.xml def create @mash = Mash.new(params[:mash]) @@ -46,16 +45,14 @@ class MashesController < ApplicationController if @mash.save flash[:notice] = 'Mash was successfully created.' format.html { redirect_to(@mash) } - format.xml { render :xml => @mash, :status => :created, :location => @mash } + format.xml { render :xml => @mash, :status => :created, :location => @mash } else format.html { render :action => "new" } - format.xml { render :xml => @mash.errors, :status => :unprocessable_entity } + format.xml { render :xml => @mash.errors, :status => :unprocessable_entity } end end end - # PUT /mashes/1 - # PUT /mashes/1.xml def update @mash = Mash.find(params[:id]) @@ -63,23 +60,21 @@ class MashesController < ApplicationController if @mash.update_attributes(params[:mash]) flash[:notice] = 'Mash was successfully updated.' format.html { redirect_to(@mash) } - format.xml { head :ok } + format.xml { head :ok } else format.html { render :action => "edit" } - format.xml { render :xml => @mash.errors, :status => :unprocessable_entity } + format.xml { render :xml => @mash.errors, :status => :unprocessable_entity } end end end - # DELETE /mashes/1 - # DELETE /mashes/1.xml def destroy @mash = Mash.find(params[:id]) @mash.destroy respond_to do |format| format.html { redirect_to(mashes_url) } - format.xml { head :ok } + format.xml { head :ok } end end end diff --git a/rails/map-mash/app/models/map.rb b/rails/map-mash/app/models/map.rb index 1c285f3..651c768 100644 --- a/rails/map-mash/app/models/map.rb +++ b/rails/map-mash/app/models/map.rb @@ -1,5 +1,25 @@ +require 'fastercsv' + + class Map < ActiveRecord::Base def self.from_city_name(city_name) self.find_or_initialize_by_name(city_name).save! end + + def self.rand(count = 2) + self.find(:all, :order => 'RANDOM()', :limit => count) + end + + def self.import(csv_filename) + FasterCSV.parse(open(csv_filename), :headers => true, + :header_converters => [:downcase, :symbol]).each do |row| + map = self.find_or_initialize_by_name( + "#{row[:city]}, #{row[:country]}" + ) + map.save + if block_given? + yield map + end + end + end end diff --git a/rails/map-mash/config/initializers/01-settings.rb b/rails/map-mash/config/initializers/01-settings.rb index 05564e0..b3f0efe 100644 --- a/rails/map-mash/config/initializers/01-settings.rb +++ b/rails/map-mash/config/initializers/01-settings.rb @@ -2,5 +2,5 @@ require 'mc-settings' Setting.load( :path => Rails.root, - :files => ['config/settings/default.yml'] + :files => ['config/settings.yml'] ) diff --git a/rails/map-mash/config/settings.yml b/rails/map-mash/config/settings.yml new file mode 100644 index 0000000..82d93e5 --- /dev/null +++ b/rails/map-mash/config/settings.yml @@ -0,0 +1,5 @@ +resque_web: + port: 15678 + +redis: + port: 16379 diff --git a/rails/map-mash/config/settings/default.yml b/rails/map-mash/config/settings/default.yml deleted file mode 100644 index a8abf5d..0000000 --- a/rails/map-mash/config/settings/default.yml +++ /dev/null @@ -1,8 +0,0 @@ -resque_web: - port: 15678 - -redis: - port: 16379 - -map: - base_url: 'http://en.wikipedia.org/wiki/List_of_towns_and_cities_with_100,000_or_more_inhabitants/cityname:_{FIRST_LETTER}' diff --git a/rails/map-mash/db/capital-cities.csv b/rails/map-mash/db/capital-cities.csv new file mode 100644 index 0000000..2b9e72b --- /dev/null +++ b/rails/map-mash/db/capital-cities.csv @@ -0,0 +1,191 @@ +country,city +Afghanistan,Kabul +Albania,Tirane +Algeria,Algiers +Andorra,Andorra la Vella +Angola,Luanda +Antigua and Barbuda,St. John's +Argentina,Buenos Aires +Australia,Canberra +Austria,Vienna +Azerbaijan,Baku +Bahamas,Nassau +Bahrain,Manama +Bangladesh,Dhaka +Barbados,Bridgetown +Belarus,Minsk +Belgium,Brussels +Belize,Belmopan +Benin,Porto-Novo +Bhutan,Thimphu +Bolivia,Sucre +Bosnia and Herzegovina,Sarajevo +Botswana,Gaborone +Brazil,Brasilia +Brunei,Darussalam Bandar Seri Begawan +Bulgaria,Sofia +Burkina Faso,Ouagadougou +Burundi,Bujumbura +Cambodia,Phnom Penh +Cameroon,Yaounde +Canada,Ottawa +Cape Verde,Praia +Central African Republic,Bangui +Chad,N'Djamena +Chile,Santiago +China,Beijing +Colombia,Bogota +Comoros,Moroni +Congo,Brazzaville +Congo,Kinshasa +Costa Rica,San Jose +Cote d'Ivoire,Yamoussoukro +Croatia,Zagreb +Cuba,Havana +Cyprus,Nicosia +Czech Republic,Prague +Denmark,Copenhagen +Djibouti,Djibouti +Dominica,Roseau +Dominican Republic,Santo Domingo +Ecuador,Quito +Egypt,Cairo +El Salvador,San Salvador +Equatorial Guinea,Malabo +Eritrea,Asmara +Estonia,Tallinn +Ethiopia,Addis Ababa +Fiji,Suva +Finland,Helsinki +France,Paris +Gabon,Libreville +Gambia,Banjul +Georgia,Tbilisi +Germany,Berlin +Ghana,Accra +Greece,Athens +Grenada,St. George's +Guatemala,Guatemala City +Guinea,Conakry +Guinea-Bissau,Bissau +Guyana,Georgetown +Haiti,Port-au-Prince +Honduras,Tegucigalpa +Hungary,Budapest +Iceland,Reykjavik +India,New Delhi +Indonesia,Jakarta +Iran,Teheran +Iraq,Baghdad +Ireland,Dublin +Israel,Jerusalem +Italy,Rome +Jamaica,Kingston +Japan,Tokyo +Jordan,Amman +Kazakhstan,Astana +Kenya,Nairobi +Kiribati,South Tarawa +Korea,Pyongyang +Korea,Seoul +Kuwait,Kuwait City +Kyrgyzstan,Bishkek +Laos,Vientiane +Latvia,Riga +Lebanon,Beirut +Lesotho,Maseru +Liberia,Monrovia +Libya,Tripoli +Liechtenstein,Vaduz +Lithuania,Vilnius +Luxembourg,Luxembourg +Macedonia,Skopje +Madagascar,Antananarivo +Malawi,Lilongwe +Malaysia,Kuala Lumpur +Maldives,Male +Mali,Bamako +Malta,Valletta +Marshall Islands,Majuro +Mauritania,Nouakchott +Mauritius,Port Louis +Mexico,Mexico City +Micronesia,Palikir +Moldova,Chisinau +Monaco,Monaco +Mongolia,Ulan Bator +Morocco,Rabat +Mozambique,Maputo +Myanmar,Rangoon +Namibia,Windhoek +Nauru,Yaren +Nepal,Kathmandu +Netherlands,Amsterdam +New Zealand,Wellington +Nicaragua,Managua +Niger,Niamey +Nigeria,Abuja +Norway,Oslo +Oman,Muscat +Pakistan,Islamabad +Palau,Koror +Panama,Panama City +Papua New Guinea,Port Moresby +Paraguay,Asuncion +Peru,Lima +Philippines,Manila +Poland,Warsaw +Portugal,Lisbon +Qatar,Doha +Romania,Bucharest +Russian Federation,Moscow +Rwanda,Kigali +St. Kitts and Nevis,Basseterre +St. Lucia,Castries +St. Vincent and The Grenadines,Kingstown +Samoa,Apia +San Marino,San Marino +Sao Tome and Principe,Sao Tome +Saudi Arabia,Riyadh +Senegal,Dakar +Seychelles,Victoria +Sierra Leone,Freetown +Singapore,Singapore +Slovakia,Bratislava +Slovenia,Ljubljana +Solomon Islands,Honiara +Somalia,Mogadishu +South Africa,Pretoria +Spain,Madrid +Sri Lanka,Colombo +Sudan,Khartoum +Suriname,Paramaribo +Swaziland,Mbabane +Sweden,Stockholm +Switzerland,Bern +Syria,Damascus +Taiwan,Taipei +Tajikistan,Dushanbe +Tanzania,Dar es Salaam +Thailand,Bangkok +Togo,Lome +Tonga,Nuku'alofa +Trinidad and Tobago,Port-of-Spain +Tunisia,Tunis +Turkey,Ankara +Turkmenistan,Ashgabat +Tuvalu,Funafuti +Uganda,Kampala +Ukraine,Kiev +United Arab Emirates,Abu Dhabi +United Kingdom,London +United States,Washington +Uruguay,Montevideo +Uzbekistan,Tashkent +Vanuatu,Port Vila +Venezuela,Caracas +Vietnam,Hanoi +Western Sahara,El Aaiun +Yemen,Sana +Zambia,Lusaka +Zimbabwe,Harare diff --git a/rails/map-mash/db/migrate/20120304151123_create_maps.rb b/rails/map-mash/db/migrate/20120304151123_create_maps.rb index 5e01f18..2878d28 100644 --- a/rails/map-mash/db/migrate/20120304151123_create_maps.rb +++ b/rails/map-mash/db/migrate/20120304151123_create_maps.rb @@ -1,8 +1,7 @@ class CreateMaps < ActiveRecord::Migration def self.up create_table :maps do |t| - t.string :name - t.string :unique_hash + t.string :name, :null => false t.timestamps end diff --git a/rails/map-mash/db/migrate/20120304164625_create_mashes.rb b/rails/map-mash/db/migrate/20120304164625_create_mashes.rb index f4ac1bc..647b77a 100644 --- a/rails/map-mash/db/migrate/20120304164625_create_mashes.rb +++ b/rails/map-mash/db/migrate/20120304164625_create_mashes.rb @@ -1,10 +1,10 @@ class CreateMashes < ActiveRecord::Migration def self.up create_table :mashes do |t| - t.string :requester - t.integer :map_a - t.integer :map_b - t.integer :winner + t.string :requester, :null => false + t.integer :map_a, :null => false + t.integer :map_b, :null => false + t.integer :winner, :null => false t.timestamps end diff --git a/rails/map-mash/db/schema.rb b/rails/map-mash/db/schema.rb index 53c776b..288cd51 100644 --- a/rails/map-mash/db/schema.rb +++ b/rails/map-mash/db/schema.rb @@ -12,17 +12,16 @@ ActiveRecord::Schema.define(:version => 20120304164625) do create_table "maps", :force => true do |t| - t.string "name" - t.string "unique_hash" + t.string "name", :null => false t.datetime "created_at" t.datetime "updated_at" end create_table "mashes", :force => true do |t| - t.string "requester" - t.integer "map_a" - t.integer "map_b" - t.integer "winner" + t.string "requester", :null => false + t.integer "map_a", :null => false + t.integer "map_b", :null => false + t.integer "winner", :null => false t.datetime "created_at" t.datetime "updated_at" end diff --git a/rails/map-mash/lib/google_map_location_fetcher.rb b/rails/map-mash/lib/google_map_location_fetcher.rb new file mode 100644 index 0000000..54c3cb0 --- /dev/null +++ b/rails/map-mash/lib/google_map_location_fetcher.rb @@ -0,0 +1,61 @@ +require 'base64' +require 'logger' +require 'uri' + +require 'nokogiri' +require 'typhoeus' + + +class GoogleMapLocationFetcher + attr_accessor :base_map_url, :log + + def initialize + @base_map_url = [ + 'http://maps.googleapis.com/maps/api/staticmap', + '?zoom=15', + '&sensor=false', + '&size=512x512', + '&maptype=satellite', + ].join('') + + @log = Logger.new( + File.expand_path('../log/map-crawler.log', File.dirname(__FILE__)) + ) + @log.level = Logger::INFO + @log.formatter = lambda do |severity, time, prog, message| + "#{time} - #{severity} - #{message}\n" + end + end + + def self.mapdump_callback(location, image) + puts "Map '#{location}':" + puts Base64.encode64(image) + end + + def fetch(locations, &callback) + callback ||= self.class.method(:mapdump_callback) + hydra = Typhoeus::Hydra.new(:initial_pool_size => 26) + + locations.each do |location| + request = Typhoeus::Request.new( + "#{@base_map_url}¢er=#{URI.encode(location)}" + ) + request.on_complete do |response| + handle_response(response, location, &callback) + end + + hydra.queue(request) + end + + hydra.run + end + + def handle_response(response, location, &callback) + @log.info("Handling request at url #{response.effective_url}") + if response.success? and response.headers_hash[:content_type] =~ /image\/.*/ + callback.call(location, response.body) + else + callback.call(location, '') + end + end +end diff --git a/rails/map-mash/lib/map_crawler.rb b/rails/map-mash/lib/map_crawler.rb deleted file mode 100644 index 6e1248c..0000000 --- a/rails/map-mash/lib/map_crawler.rb +++ /dev/null @@ -1,49 +0,0 @@ -require 'logger' - -require 'nokogiri' -require 'typhoeus' - - -class MapCrawler - attr_accessor :base_map_url, :log, :request_pool - - def initialize(base_map_url) - @base_map_url = base_map_url - - @log = Logger.new( - File.expand_path('../log/map-crawler.log', File.dirname(__FILE__)) - ) - @log.level = Logger::INFO - @log.formatter = lambda do |severity, time, prog, message| - "#{time} - #{severity} - #{message}\n" - end - end - - def crawl(city_name_callback = nil) - city_name_callback ||= lambda { |n| puts n } - - hydra = Typhoeus::Hydra.new(:initial_pool_size => 26) - - ('A'..'Z').each do |letter| - letter_request = Typhoeus::Request.new( - @base_map_url.gsub(/\{FIRST_LETTER\}/, letter) - ) - letter_request.on_complete do |response| - handle_cities(response, city_name_callback) - end - - hydra.queue(letter_request) - end - - hydra.run - end - - def handle_cities(response, city_name_callback) - @log.info("Handling cities at url #{response.effective_url}") - doc = Nokogiri::HTML(response.body) - doc.css('div.mw-content-ltr ul')[3].css('li a').each do |anchor| - @log.info("Found city: #{anchor.text}") - city_name_callback.call(anchor.text.strip) - end - end -end diff --git a/rails/map-mash/lib/tasks/maps.rake b/rails/map-mash/lib/tasks/maps.rake index 3c25617..87aa02f 100644 --- a/rails/map-mash/lib/tasks/maps.rake +++ b/rails/map-mash/lib/tasks/maps.rake @@ -1,6 +1,13 @@ namespace :maps do - desc 'Index the maps!' - task :index => :environment do - MapCrawler.new(Setting.map(:base_url)).crawl(Map.method(:from_city_name)) + desc 'Seed the maps!' + task :seed => :environment do + require 'app/models/map' + + csv_filename = File.expand_path( + '../../db/capital-cities.csv', File.dirname(__FILE__) + ) + Map.import(csv_filename) do |map| + puts "Seeded map '#{map.name}'" + end end end diff --git a/rails/map-mash/spec/lib/map_crawler_spec.rb b/rails/map-mash/spec/lib/map_crawler_spec.rb deleted file mode 100644 index 3dd187a..0000000 --- a/rails/map-mash/spec/lib/map_crawler_spec.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'spec_helper' - - -describe MapCrawler do - let(:subject) { MapCrawler.new(Setting.map(:base_url)) } - - describe 'when crawling for actual maps', :integration => true do - it 'should increment the map count for each map found' do - map_count = 0 - count_increment = lambda do |n| - map_count += 1 - end - - expect do - subject.crawl(count_increment) - end.to change{ map_count }.by_at_least(26) - end - end -end