Merge pull request #58 from omnilord/reverse-geocode-address-cleanup

Reverse GeoCoding for models missing required address fields.

app/jobs/recode_geocoding_job.rb

@@ -0,0 +1,18 @@
+class RecodeGeocodingJob < ApplicationJob
+  queue_as :default
+
+  def perform(class_name = 'Shelter', *args)
+    case class_name.classify
+    when 'Shelter'
+      logger.info "Recoding #{args.empty? ? 'any invalid' : args.length} shelters"
+      shelters = Shelter.geo_recode(*args)
+      logger.info "#{shelters.length} shelters reverse geocoded"
+    when 'DistributionPoint'
+      logger.info "Recoding #{args.empty? ? 'any invalid' : args.length} distribution points"
+      shelters = DistributionPoint.geo_recode(*args)
+      logger.info "#{shelters.length} distribution points reverse geocoded"
+    else
+      logger.error "Invalid class name ('#{class_name}') passed to RecodeGeocoding job."
+    end
+  end
+end

app/models/concerns/geocodable.rb

@@ -0,0 +1,64 @@
+module Geocodable
+  extend ActiveSupport::Concern
+
+  #
+  # Geocoable Concern
+  #
+  # Adds geocoding to the model so that it can be managed appropriately.
+  #
+  # Requires the model to have database fields / columns:
+  #   latitude, longitude, address, city, state, zip, county
+
+  included do
+    after_commit :schedule_reverse_geocode
+
+    scope :incomplete_geocoding, -> { where(<<~SQL
+      county IS NULL OR TRIM(county) = '' OR
+      city IS NULL OR TRIM(city) = '' OR
+      state IS NULL OR TRIM(state) = '' OR
+      zip IS NULL OR TRIM(zip) = ''
+      SQL
+    ) }
+
+    geocoded_by :address
+    reverse_geocoded_by(:latitude, :longitude) do |obj, result|
+      geo = result.first
+      if geo
+        obj.county = geo.sub_state if obj.county.blank?
+        obj.city = geo.city if obj.city.blank?
+        obj.state = geo.state if obj.state.blank?
+        obj.zip = geo.postal_code if obj.zip.blank?
+        obj.address = geo.address
+      end
+      obj
+    end
+  end
+
+  class_methods do
+    def geo_recode(*ids)
+      list = ids.empty? ? incomplete_geocoding : where(id: ids)
+      list.each do |obj|
+        obj.recode_geofields
+        obj.save
+        sleep(0.1) # REVIEW: Is this unnecessary?
+      end
+    end
+  end
+
+  def reverse_geocode_needed?
+    county.blank? || city.blank? || zip.blank? || state.blank?
+  end
+
+  def recode_geofields(force = false)
+    reverse_geocode if force || reverse_geocode_needed?
+  end
+
+  def recode_geofields!(force = false)
+    recode_geofields(force)
+    save if changed?
+  end
+
+  def schedule_reverse_geocode
+    RecodeGeocodingJob.perform_later(self.class.name, self.id) if reverse_geocode_needed?
+  end
+end

app/models/distribution_point.rb

@@ -1,4 +1,6 @@
 class DistributionPoint < ApplicationRecord
+  include Geocodable
+
   default_scope { where(archived: false) }
 
   ColumnNames = %w[
@@ -42,8 +44,6 @@ class DistributionPoint < ApplicationRecord
   scope :outdated, ->(timing = 4.hours.ago) { where("updated_at < ?", timing) }
   scope :archived, -> { unscope(:where).where(archived: true) }
 
-  geocoded_by :address
-
   def self.to_csv
     attributes = %w[
       id updated_at active facility_name address city county state latitude longitude

app/models/shelter.rb

@@ -1,4 +1,6 @@
 class Shelter < ApplicationRecord
+  include Geocodable
+
   default_scope { where(active: true) }
 
   enum accepting: {
@@ -60,8 +62,6 @@ class Shelter < ApplicationRecord
   scope :outdated, ->(timing = 4.hours.ago) { where("updated_at < ?", timing) }
   scope :inactive, -> { unscope(:where).where(active: false) }
 
-  geocoded_by :address
-
   def self.to_csv
     attributes = %w[
 

lib/tasks/georecode.rake

@@ -0,0 +1,11 @@
+namespace :georecode do
+  desc "Rebuild the reverse geocoding fields (city, state, zip, county, address) on shelters from Google Places API"
+  task :shelters => :environment do |task, args|
+    RecodeGeocodingJob.perform_now('Shelter', *args.extras)
+  end
+
+  desc "Rebuild the reverse geocoding fields (city, state, zip, county, address) on Points of Distribution from Google Places API"
+  task :pods => :environment do |task, args|
+    RecodeGeocodingJob.perform_now('DistributionPoint', *args.extras)
+  end
+end

test/fixtures/files/google.geojson

@@ -0,0 +1,91 @@
+{
+  "address_components": [
+    {
+      "long_name": "149",
+      "short_name": "149",
+      "types": [
+        "street_number"
+      ]
+    },
+    {
+      "long_name": "9th Street",
+      "short_name": "9th St",
+      "types": [
+        "route"
+      ]
+    },
+    {
+      "long_name": "South of Market",
+      "short_name": "South of Market",
+      "types": [
+        "neighborhood",
+        "political"
+      ]
+    },
+    {
+      "long_name": "San Francisco",
+      "short_name": "SF",
+      "types": [
+        "locality",
+        "political"
+      ]
+    },
+    {
+      "long_name": "San Francisco County",
+      "short_name": "San Francisco County",
+      "types": [
+        "administrative_area_level_2",
+        "political"
+      ]
+    },
+    {
+      "long_name": "California",
+      "short_name": "CA",
+      "types": [
+        "administrative_area_level_1",
+        "political"
+      ]
+    },
+    {
+      "long_name": "United States",
+      "short_name": "US",
+      "types": [
+        "country",
+        "political"
+      ]
+    },
+    {
+      "long_name": "94103",
+      "short_name": "94103",
+      "types": [
+        "postal_code"
+      ]
+    }
+  ],
+  "formatted_address": "149 9th St, San Francisco, CA 94103, USA",
+  "geometry": {
+    "location": {
+      "lat": 37.7756906,
+      "lng": -122.4136764
+    },
+    "location_type": "ROOFTOP",
+    "viewport": {
+      "northeast": {
+        "lat": 37.77703958029149,
+        "lng": -122.4123274197085
+      },
+      "southwest": {
+        "lat": 37.7743416197085,
+        "lng": -122.4150253802915
+      }
+    }
+  },
+  "place_id": "ChIJgUEnJZ2AhYARdWgDnqcm4xM",
+  "plus_code": {
+    "compound_code": "QHGP+7G South of Market, San Francisco, CA, United States",
+    "global_code": "849VQHGP+7G"
+  },
+  "types": [
+    "street_address"
+  ]
+}

test/fixtures/shelters.yml

@@ -57,3 +57,26 @@ inactive:
   source: "https://t.co/REjWR4UR7Q"
   address_name: "Lone Star College North Harris 2700 W W Thorne Dr Houston TX"
   active: false
+
+incomplete1:
+  address: "155 9th Street, San Francisco, CA, 94103"
+  shelter: "Code for Amerca HQ"
+  accepting: "no"
+  active: true
+  updated_by: "Doostee"
+  notes: "This is not actually a shelter.  This is just a test."
+  longitude: -122.4136007
+  latitude: 37.7756917
+  source: "https://www.codeforamerica.org/contact-us"
+
+incomplete2:
+  address: "1101 Arch St, Philadelphia, PA 19107"
+  shelter: "Pennsylvania Convention Center"
+  county: "Philadelphia"
+  accepting: "no"
+  active: true
+  updated_by: "Doostee"
+  notes: "This is not actually a shelter.  This is just a test."
+  longitude: -122.4136007
+  latitude: 37.7756917
+  source: "https://www.codeforamerica.org/contact-us"

test/jobs/recode_geocoding_job_test.rb

@@ -0,0 +1,57 @@
+require 'test_helper'
+
+class RecodeGeocodingJobTest < ActiveJob::TestCase
+  def setup
+    @_original_lookup = Geocoder.config.lookup
+    Geocoder.configure(lookup: :test)
+    Geocoder::Lookup::Test.set_default_stub([mock_attributes])
+  end
+
+  def teardown
+    Geocoder::Lookup::Test.reset
+  end
+
+  test 'Shelters get recoded' do
+    assert_equal 3, Shelter.incomplete_geocoding.count, 'Invalid fixture data.'
+    RecodeGeocodingJob.perform_now('Shelter')
+    assert_equal 0, Shelter.incomplete_geocoding.count
+  end
+
+  test 'When an id is provider, only the referenced shelter gets updated' do
+    shelter = shelters(:incomplete1)
+    expectedCount = Shelter.incomplete_geocoding.count - 1
+    RecodeGeocodingJob.perform_now('Shelter', shelter.id)
+    assert_equal expectedCount, Shelter.incomplete_geocoding.count
+  end
+
+  test 'When a shelter is updated, all missing fields are filled' do
+    shelter = shelters(:incomplete1)
+    assert shelter.county.blank?
+    assert shelter.city.blank?
+    assert shelter.state.blank?
+    assert shelter.zip.blank?
+    RecodeGeocodingJob.perform_now('Shelter', shelter.id)
+    shelter.reload
+    refute shelter.county.blank?
+    refute shelter.city.blank?
+    refute shelter.state.blank?
+    refute shelter.zip.blank?
+  end
+
+private
+
+  def mock_attributes
+    @mock_attributes ||= {
+      'latitude'     => 37.7756906,
+      'longitude'    => -122.4136764,
+      'address'      => '149 9th St, San Francisco, CA 94103, USA',
+      'city'         => 'San Francisco',
+      'state'        => 'California',
+      'state_code'   => 'CA',
+      'postal_code'  => '94103',
+      'country'      => 'United States',
+      'country_code' => 'US',
+      'sub_state'    => 'San Francisco County'
+    }
+  end
+end

test/models/shelter_test.rb

@@ -3,7 +3,7 @@
 class ShelterTest < ActiveSupport::TestCase
 
   test "the fixtures work" do
-    assert_equal Shelter.count, 2
+    assert_equal Shelter.count, 4
   end
 
 end

test/test_helper.rb

@@ -15,10 +15,12 @@ class ActiveSupport::TestCase
         'latitude'     => 29.7496888,
         'longitude'    => -95.45794269999999,
         'address'      => '4800 Hallmark Dr, Houston, TX 77056, USA',
+        'city'         => 'Houston',
         'state'        => 'Texas',
         'state_code'   => 'TX',
         'country'      => 'United States',
-        'country_code' => 'US'
+        'country_code' => 'US',
+        'sub_state'    => 'Harris County'
       }
     ]
   )