Creating TestNominatim.py

.github/workflows/nominatim-docker-test.yml

@@ -0,0 +1,30 @@
+name: nominatim-docker-test
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  # pull_request:
+  #   branches:
+  #       - master
+
+  schedule:
+    # Run every Sunday at 4:05 am
+    - cron:  '5 4 * * 0'
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - name: Checkout
+    - uses: actions/checkout@v2
+
+    # Runs a single command using the runners shell
+    - name: Workflow test
+      run: echo Smoke test
+
+    - name: Test nominatim 
+      run: docker-compose -f ../../emission/integrationTests/docker-compose.yml up --exit-code-from web-server

bin/debug/save_ground_truth.py

@@ -32,6 +32,7 @@ def save_ct_list(args):
 
     parser_diary = subparsers.add_parser('diary', help='diary-based ground truth')
     parser_diary.add_argument("date", help="date to retrieve ground truth (YYYY-MM-DD)")
+    parser_diary.add_argument("file_name", help="file name to store the result to")
     parser_diary.set_defaults(func=save_diary)
 
     parser_obj_list = subparsers.add_parser('objects', help='download analysis objects directly')

emission/individual_tests/TestNominatim.py

@@ -0,0 +1,57 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+from builtins import object
+import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse
+import logging
+import json
+import emission.core.get_database as edb
+import emission.core.common as cm
+import unittest
+
+from emission.core.wrapper.trip_old import Coordinate
+
+import emission.net.ext_service.geocoder.nominatim as eco
+
+class NominatimTest(unittest.TestCase):
+
+    def test_make_url_geo(self):
+        expected_result = "http://nominatim.openstreetmap.org/search?q=Golden%2C+Colorado&format=json"
+        actual_result = eco.Geocoder.make_url_geo("Golden, Colorado")
+        self.assertEqual(expected_result, actual_result)
+
+    def test_get_json_geo(self):
+        expected_result = [{'place_id': 230592559, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 612835578, 'boundingbox': ['39.9041195', '39.914106', '-105.2365674', '-105.21353'], 'lat': '39.909856500000004', 'lon': '-105.22861864321705', 'display_name': 'NREL Flatiron Campus, Jefferson County, Colorado, United States', 'class': 'landuse', 'type': 'industrial', 'importance': 0.41000999999999993}, {'place_id': 165458796, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 241338672, 'boundingbox': ['39.7385155', '39.7391893', '-105.1743745', '-105.1721491'], 'lat': '39.738987699999996', 'lon': '-105.17326231002255', 'display_name': 'NREL Employee Parking Garage, Denver West Parkway, Pleasant View, West Pleasant View, Jefferson County, Colorado, 80419, United States', 'class': 'amenity', 'type': 'parking', 'importance': 0.21000999999999997}, {'place_id': 165006264, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 241338673, 'boundingbox': ['39.7385895', '39.7394212', '-105.1760441', '-105.1746968'], 'lat': '39.73896555', 'lon': '-105.17536976308806', 'display_name': 'NREL Parking Lot, Denver West Parkway, Pleasant View, West Pleasant View, Jefferson County, Colorado, 80419, United States', 'class': 'amenity', 'type': 'parking', 'importance': 0.21000999999999997}]
+        actual_result = eco.Geocoder.get_json_geo("NREL, Colorado")
+        self.assertEqual(expected_result, actual_result)
+
+    def test_geocode(self):
+        expected_result_lon = Coordinate(39.7546349, -105.220580).get_lon()
+        expected_result_lat = Coordinate(39.7546349, -105.220580).get_lat()
+        actual_result_lon = eco.Geocoder.geocode("Golden, Colorado").get_lon()
+        actual_result_lat = eco.Geocoder.geocode("Golden, Colorado").get_lat()
+        self.assertEqual(expected_result_lon, actual_result_lon)
+        self.assertEqual(expected_result_lat, actual_result_lat)
+
+
+    def test_make_url_reverse(self):
+        expected_result = "http://nominatim.openstreetmap.org/reverse?lat=39.7406821&lon=-105.168522&format=json"
+        actual_result = (eco.Geocoder.make_url_reverse(39.7406821, -105.1685220))
+        self.assertEqual(expected_result, actual_result)
+
+    def test_get_json_reverse(self):
+        expected_result = {'place_id': 151856645, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 193193531, 'lat': '39.74074185', 'lon': '-105.168658237606', 'display_name': 'Visitors Center, 15013, Denver West Parkway, Applewood, Jefferson County, Colorado, 80401, United States', 'address': {'building': 'Visitors Center', 'house_number': '15013', 'road': 'Denver West Parkway', 'village': 'Applewood', 'county': 'Jefferson County', 'state': 'Colorado', 'ISO3166-2-lvl4': 'US-CO', 'postcode': '80401', 'country': 'United States', 'country_code': 'us'}, 'boundingbox': ['39.7405892', '39.7409443', '-105.1687471', '-105.1683515']}
+        actual_result = eco.Geocoder.get_json_reverse(39.7406821, -105.1685220)
+        self.assertEqual(expected_result, actual_result)
+
+    def test_reverse_geocode(self):
+        expected_result = "Visitors Center, 15013, Denver West Parkway, Applewood, Jefferson County, Colorado, 80401, United States"
+        actual_result = eco.Geocoder.reverse_geocode(39.7406821, -105.1685220)
+        self.assertEqual(expected_result, actual_result)
+
+if __name__ == '__main__':
+    unittest.main()

emission/integrationTests/Dockerfile

@@ -0,0 +1,13 @@
+# python 3
+FROM ubuntu:latest
+
+RUN apt-get update
+RUN apt-get install -y curl
+
+# CHANGEME: Create the files that correspond to your configuration in the conf directory
+# COPY conf/net/auth/google_auth.json /usr/src/app/conf/net/auth/google_auth.json
+COPY start_integration_tests.sh /start_integration_tests.sh
+
+ENV NOMINATIM_QUERY_URL='http://localhost:8080'
+
+CMD ["/bin/bash", "/start_integration_tests.sh"]

emission/integrationTests/TestNominatimBeta.py

@@ -0,0 +1,30 @@
+import urllib.request, urllib.parse
+import json
+import unittest
+import os
+
+NOMINATIM_QUERY_URL = os.environ.get("NOMINATIM_QUERY_URL")
+print("query URL:", NOMINATIM_QUERY_URL)
+
+
+class TestReverseGeocode(unittest.TestCase):
+    def testCompareResult(self):
+        #Didn't use the query in nominatim.json because it would change how we query nominatim regularly. 
+        nominatim_reverse_query = NOMINATIM_QUERY_URL + "/reverse?"
+        params = {
+                "lat" : 41.831174, 
+                "lon" : -71.414907,
+                "format" : "json"
+            }
+        encoded_params = urllib.parse.urlencode(params)
+        url = nominatim_reverse_query + encoded_params
+        request = urllib.request.Request(url)
+        response = urllib.request.urlopen(request)
+        parsed_response = json.loads(response.read())
+        # ndn = nominatim display name
+        ndn = str(parsed_response.get("display_name"))
+        #expected display name is a string 
+        edn = "Rhode Island State Capitol Building, 82, Smith Street, Downtown, Providence, Providence County, 02903, United States"
+        self.assertEqual(ndn, edn)
+if __name__ == '__main__':
+    unittest.main()

emission/integrationTests/docker-compose.yml

@@ -0,0 +1,59 @@
+version: "3"
+services:
+  web-server:
+    build: integrationTests
+    depends_on:
+      - db
+      - nominatim
+    environment:
+      - DB_HOST=db
+      - WEB_SERVER_HOST=0.0.0.0
+    volumes:
+      # specify the host directory where the source code should live
+      # If this is ~/e-mission-server-docker, then you can edit the files at
+      # ~/e-mission-server-docker/src/e-mission-server/emission/...
+      # - CHANGEME:/src/
+      - ..:/src/e-mission-server
+    networks:
+       - emission
+  db:
+    image: mongo:4.4.0
+    ports:
+      - "1234:1234"
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+
+    #Volumes is the preferred way to persist data generated by a container. In this case we use a volume to persist the contents
+    #of the data base. Learn more about how to use volumes here: https://docs.docker.com/storage/volumes/
+    # And learn how to configure volumes in your compose file here: https://docs.docker.com/compose/compose-file/#volume-configuration-reference
+    volumes:
+      - mongo-data:/data/db
+    networks:
+       - emission
+  #adding section to incorporate nominatim server functionality
+  nominatim:
+    image: mediagis/nominatim
+    container_name: rhodeisland-nominatim
+    environment:
+      - PBF_URL=https://download.geofabrik.de/north-america/us/rhode-island-latest.osm.pbf
+      - REPLICATION_URL=https://download.geofabrik.de/north-america/us/rhode-island-updates/
+      - IMPORT_WIKIPEDIA=false
+    ports:
+      - "8080:8080"
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+    volumes:
+      - nominatim-data:/var/lib/postgresql/14/main
+    networks:
+      - emission
+
+networks:
+  emission:
+
+volumes:
+  mongo-data:
+  nominatim-data:

emission/integrationTests/start_integration_tests.sh

@@ -0,0 +1,24 @@
+# Run the tests in the docker environment
+# Using an automated install 
+cd /src/e-mission-server
+
+#set database URL using environment variable
+echo ${DB_HOST}
+if [ -z ${DB_HOST} ] ; then
+    local_host=`hostname -i`
+    sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
+else
+    sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf
+fi
+cat conf/storage/db.conf
+
+echo "Setting up conda..."
+source setup/setup_conda.sh Linux-x86_64
+
+echo "Setting up the test environment..."
+source setup/setup_tests.sh
+
+echo "Running tests..."
+source setup/activate_tests.sh
+
+./runIntegrationTests.sh

emission/net/ext_service/geocoder/nominatim.py

@@ -11,19 +11,12 @@
 import json
 
 from emission.core.wrapper.trip_old import Coordinate
-from pygeocoder import Geocoder as pyGeo  ## We fall back on this if we have to
-
-try:
-    googlemaps_key_file = open("conf/net/ext_service/googlemaps.json")
-    GOOGLE_MAPS_KEY = json.load(googlemaps_key_file)["api_key"]
-    googlemaps_key_file.close()
-except:
-    print("google maps key not configured, falling back to nominatim")
 
 try:
     nominatim_file = open("conf/net/ext_service/nominatim.json")
     NOMINATIM_QUERY_URL = json.load(nominatim_file)["query_url"]
     nominatim_file.close()
+
 except:
     print("nominatim not configured either, place decoding must happen on the client")
 
@@ -54,15 +47,10 @@ def get_json_geo(cls, address):
 
     @classmethod
     def geocode(cls, address):
-        # try:
-        #     jsn = cls.get_json_geo(address)
-        #     lat = float(jsn[0]["lat"])
-        #     lon = float(jsn[0]["lon"])
-        #     return Coordinate(lat, lon)
-        # except:
-        #     print "defaulting"
-        return _do_google_geo(address) # If we fail ask the gods
-
+        jsn = cls.get_json_geo(address)
+        lat = float(jsn[0]["lat"])
+        lon = float(jsn[0]["lon"])
+        return Coordinate(lat, lon)
 
     @classmethod
     def make_url_reverse(cls, lat, lon):
@@ -73,6 +61,7 @@ def make_url_reverse(cls, lat, lon):
         }
 
         query_url = NOMINATIM_QUERY_URL + "/reverse?"
+        print("=====Query_URL:", query_url)
         encoded_params = urllib.parse.urlencode(params)
         url = query_url + encoded_params
         logging.debug("For reverse geocoding, using URL %s" % url)
@@ -81,29 +70,15 @@ def make_url_reverse(cls, lat, lon):
     @classmethod
     def get_json_reverse(cls, lat, lng):
         request = urllib.request.Request(cls.make_url_reverse(lat, lng))
+        print("======request", request)
+        print(cls.make_url_reverse(lat, lng))
         response = urllib.request.urlopen(request)
         parsed_response = json.loads(response.read())
         logging.debug("parsed_response = %s" % parsed_response)
         return parsed_response
 
     @classmethod
     def reverse_geocode(cls, lat, lng):
-        # try:
-        #     jsn = cls.get_json_reverse(lat, lng)
-        #     address = jsn["display_name"]
-        #     return address
-
-        # except:
-        #     print "defaulting"
-        return _do_google_reverse(lat, lng) # Just in case
-
-## Failsafe section
-def _do_google_geo(address):
-    geo = pyGeo(GOOGLE_MAPS_KEY)
-    results = geo.geocode(address)
-    return Coordinate(results[0].coordinates[0], results[0].coordinates[1])
-
-def _do_google_reverse(lat, lng):
-    geo = pyGeo(GOOGLE_MAPS_KEY)
-    address = geo.reverse_geocode(lat, lng)
-    return address[0]
+        jsn = cls.get_json_reverse(lat, lng)
+        address = jsn["display_name"]
+        return address