Creating TestNominatim.py

.github/workflows/nominatim-docker-test.yml

@@ -0,0 +1,30 @@
+name: nominatim-docker-test
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  schedule:
+
+    # Run every Sunday at 4:05 am
+    - cron:  '5 4 * * 0'
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    # Runs a single command using the runners shell
+    - name: Workflow test
+      run: echo Smoke test
+
+      # Passes the geofabrik key into the docker-compose.yml file. 
+    - name: Test nominatim.py
+      run: GFBK_KEY=${{ secrets.GEOFABRIK_API }} docker-compose -f emission/integrationTests/docker-compose.yml up --exit-code-from web-server
+
+

README.md

@@ -11,10 +11,7 @@ backend server - the phone apps are available in the [e-mission-phone
 repo](https://github.com/amplab/e-mission-phone)
 
 
-- **Master** [![master:test-with-docker](https://github.com/e-mission/e-mission-server/workflows/test-with-docker/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Atest-with-docker) [![master:ubuntu-only-test-with-manual-install](https://github.com/e-mission/e-mission-server/workflows/ubuntu-only-test-with-manual-install/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Aubuntu-only-test-with-manual-install) [![master:osx-ubuntu-manual-install](https://github.com/e-mission/e-mission-server/workflows/osx-ubuntu-manual-install/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Aosx-ubuntu-manual-install)
-
-- **GIS branch:** [![master:ubuntu-only-test-with-manual-install](https://github.com/e-mission/e-mission-server/workflows/ubuntu-only-test-with-manual-install/badge.svg?branch=gis-based-mode-detection)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Agis-based-mode-detection+workflow%3Aubuntu-only-test-with-manual-install) [![osx-ubuntu-manual-install](https://github.com/e-mission/e-mission-server/workflows/osx-ubuntu-manual-install/badge.svg?branch=gis-based-mode-detection)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Agis-based-mode-detection+workflow%3Aosx-ubuntu-manual-install)
-
+- **Master** [![master:test-with-docker](https://github.com/e-mission/e-mission-server/workflows/test-with-docker/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Atest-with-docker) [![master:ubuntu-only-test-with-manual-install](https://github.com/e-mission/e-mission-server/workflows/ubuntu-only-test-with-manual-install/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Aubuntu-only-test-with-manual-install) [![master:osx-ubuntu-manual-install](https://github.com/e-mission/e-mission-server/workflows/osx-ubuntu-manual-install/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Aosx-ubuntu-manual-install) [![master:nominatim-docker-test](https://github.com/e-mission/e-mission-server/workflows/nominatim-docker-test/badge.svg)](https://github.com/e-mission/e-mission-server/actions/workflows/nominatim-docker-test.yml?query=branch%3Amaster)
 
 **Issues:** Since this repository is part of a larger project, all issues are tracked [in the central docs repository](https://github.com/e-mission/e-mission-docs/issues). If you have a question, [as suggested by the open source guide](https://opensource.guide/how-to-contribute/#communicating-effectively), please file an issue instead of sending an email. Since issues are public, other contributors can try to answer the question and benefit from the answer.
 

bin/debug/save_ground_truth.py

@@ -32,6 +32,7 @@ def save_ct_list(args):
 
     parser_diary = subparsers.add_parser('diary', help='diary-based ground truth')
     parser_diary.add_argument("date", help="date to retrieve ground truth (YYYY-MM-DD)")
+    parser_diary.add_argument("file_name", help="file name to store the result to")
     parser_diary.set_defaults(func=save_diary)
 
     parser_obj_list = subparsers.add_parser('objects', help='download analysis objects directly')

emission/individual_tests/TestNominatim.py

@@ -0,0 +1,120 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+import unittest
+import importlib
+import os
+from emission.core.wrapper.trip_old import Coordinate
+import requests
+import emission.core.wrapper.entry as ecwe
+import emission.analysis.intake.cleaning.clean_and_resample as clean
+import emission.net.ext_service.geocoder.nominatim as eco
+
+#Setting query URLs
+OPENSTREETMAP_QUERY_URL = os.environ.get("OPENSTREETMAP_QUERY_URL")
+GEOFABRIK_QUERY_URL = os.environ.get("GEOFABRIK_QUERY_URL")
+NOMINATIM_CONTAINER_URL = os.environ.get("NOMINATIM_CONTAINER_URL")
+
+class NominatimTest(unittest.TestCase):
+    maxDiff = None
+
+    def setUp(self):
+        #Creates a fake, cleaned place in Rhode Island
+        fake_id = "place_in_rhodeisland"
+        key = "segmentation/raw_place"
+        write_ts = 1694344333
+        data = {'source': 'FakeTripGenerator','location': {'type': 'Point', 'coordinates': [-71.4128343, 41.8239891]}}
+        fake_place = ecwe.Entry.create_fake_entry(fake_id, key, data, write_ts)
+        self.fake_place = fake_place
+
+    #When a nominatim service is called, we set the value of the NOMINATIM_QUERY_URL environment variable in nominatim.py and re-load the module.
+    def nominatim(service):
+        if service == "container":
+            os.environ["NOMINATIM_QUERY_URL"] = NOMINATIM_CONTAINER_URL
+            importlib.reload(eco)
+        elif service == "geofabrik":
+            os.environ["NOMINATIM_QUERY_URL"] = GEOFABRIK_QUERY_URL
+            importlib.reload(eco)
+        elif service == "OSM":
+            os.environ["NOMINATIM_QUERY_URL"] = OPENSTREETMAP_QUERY_URL
+            importlib.reload(eco)
+
+    #Basic query to check that OSM, the Rhode Island Container, and geofabrik are returning the same data.
+    def test_geofabrik_and_nominatim(self):
+        lat, lon = 41.8239891, -71.4128343
+        NominatimTest.nominatim("container")
+        container_result = eco.Geocoder.get_json_reverse(lat,lon)
+        NominatimTest.nominatim("OSM")
+        osm_result = eco.Geocoder.get_json_reverse(lat,lon)
+        NominatimTest.nominatim("geofabrik")
+        geofabrik_result = eco.Geocoder.get_json_reverse(lat,lon)
+        key_list = ['osm_id', 'boundingbox']
+        for k in key_list:
+            self.assertEqual(osm_result[k], geofabrik_result[k])
+            self.assertEqual(container_result[k], geofabrik_result[k])
+
+    #Checks the display name generated by get_filtered_place in clean_and_resample.py, which creates a cleaned place from the fake place
+    # and reverse geocodes with the coordinates.
+    def test_get_filtered_place(self):
+        fake_place_raw = self.fake_place
+        fake_place_data = clean.get_filtered_place(fake_place_raw).__getattr__("data")
+        actual_result = fake_place_data.__getattr__("display_name")
+        expected_result = "Dorrance Street, Providence"
+        self.assertEqual(expected_result, actual_result)
+
+    #Testing make_url_geo, which creates a query URL from the input string. 
+    def test_make_url_geo(self):
+        expected_result = GEOFABRIK_QUERY_URL + "/search?q=Providence%2C+Rhode+Island&format=json"
+        NominatimTest.nominatim("geofabrik")
+        actual_result = eco.Geocoder.make_url_geo("Providence, Rhode Island")
+        self.assertEqual(expected_result, actual_result)
+
+        #Testing make_url_reverse, which creates a query url from a lat and lon.
+    def test_make_url_reverse(self):
+        NominatimTest.nominatim("geofabrik")
+        lat, lon = 41.8239891, -71.4128343
+        expected_result = GEOFABRIK_QUERY_URL + (f"/reverse?lat={lat}&lon={lon}&format=json")
+        actual_result = (eco.Geocoder.make_url_reverse(lat, lon))
+        self.assertEqual(expected_result, actual_result)
+
+    #Testing get_json_geo, which passes in an address as a query. Compares three select k,v pairs in the results. 
+    def test_get_json_geo(self):
+        NominatimTest.nominatim("geofabrik")
+        expected_result = {'place_id': 132490, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 141567710, 'boundingbox': ['41.8325787', '41.8332278', '-71.4161848', '-71.4152064'], 'lat': '41.8330097', 'lon': '-71.41568124868104', 'display_name': 'State of Rhode Island Department of Administration, 1, Park Street, Downtown, Providence, Providence County, Rhode Island, 02908, United States', 'class': 'building', 'type': 'civic', 'importance': 1.75001}
+        actual_result = eco.Geocoder.get_json_geo("State of Rhode Island Department of Administration, 1, Park Street, Downtown, Providence, Providence County, 02908, United States")[0]
+        key_list = ['osm_id', 'boundingbox', 'display_name']
+        for k in key_list:
+            self.assertEqual(expected_result[k], actual_result[k])
+
+    #Testing the geocode function, which passes in an address and gets latitude and longitude.
+    # Test creates instance of coordinates using coordinate class. Getting lat and lon of the coordinate using get_lat and get_lon methods from the class.
+    def test_geocode(self):
+        NominatimTest.nominatim("geofabrik")
+        expected_result_lon = Coordinate(41.8239891, -71.4128343).get_lon()
+        expected_result_lat = Coordinate(41.8239891, -71.4128343).get_lat()
+        actual_result = eco.Geocoder.geocode("Providence, Rhode Island")
+        actual_result_lon = actual_result.get_lon()
+        actual_result_lat = actual_result.get_lat()
+        self.assertEqual(expected_result_lon, actual_result_lon)
+        self.assertEqual(expected_result_lat, actual_result_lat)
+
+    #Testing get_json_reverse, which reverse geocodes from a lat and lon. Tested result was modified to only look at the name returned with the coordinates, rather than the entire dictionary.
+    def test_get_json_reverse(self):
+        NominatimTest.nominatim("geofabrik")
+        expected_result = "Providence City Hall"
+        actual_result = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343)["display_name"].split(",")[0]
+        self.assertEqual(expected_result, actual_result)
+
+    #Testing reverse_geocode, which reverse geocodes from a lat and lon and returns only the display name. 
+    def test_reverse_geocode(self):
+        NominatimTest.nominatim("geofabrik")
+        expected_result = "Portugal Parkway, Fox Point, Providence, Providence County, Rhode Island, 02906, United States"
+        actual_result = eco.Geocoder.reverse_geocode(41.8174476, -71.3903767)
+        self.assertEqual(expected_result, actual_result)
+
+if __name__ == '__main__':
+    unittest.main()

emission/integrationTests/Dockerfile

@@ -0,0 +1,12 @@
+# python 3
+FROM ubuntu:latest
+
+RUN apt-get update
+RUN apt-get install -y curl
+
+# CHANGEME: Create the files that correspond to your configuration in the conf directory
+
+RUN echo "About to copy e-mission server code"
+COPY start_integration_tests.sh/ /start_integration_tests.sh
+
+CMD ["/bin/bash", "/start_integration_tests.sh"]

emission/integrationTests/docker-compose.yml

@@ -0,0 +1,58 @@
+version: "3"
+services:
+  web-server:
+    build: 
+      context: .
+    depends_on:
+      - db
+      - nominatim
+    environment:
+      - DB_HOST=db
+      - GFBK_KEY=$GFBK_KEY
+      - GEOFABRIK_QUERY_URL=https://geocoding.geofabrik.de/$GFBK_KEY
+      - OPENSTREETMAP_QUERY_URL=https://nominatim.openstreetmap.org
+      - NOMINATIM_CONTAINER_URL=http://rhodeisland-nominatim:8080
+
+    volumes:
+      # specify the host directory where the source code should live
+      # If this is ~/e-mission-server-docker, then you can edit the files at
+      # ~/e-mission-server-docker/src/e-mission-server/emission/...
+      # - CHANGEME:/src/
+      - ../..:/src/e-mission-server
+    networks:
+       - emission
+
+  db:
+    image: mongo:4.4.0
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+
+    #Volumes is the preferred way to persist data generated by a container. In this case we use a volume to persist the contents
+    #of the data base. Learn more about how to use volumes here: https://docs.docker.com/storage/volumes/
+    # And learn how to configure volumes in your compose file here: https://docs.docker.com/compose/compose-file/#volume-configuration-reference
+    volumes:
+      - mongo-data:/data/db
+    networks:
+       - emission
+  #adding section to incorporate nominatim server functionality
+  nominatim:
+    entrypoint: /app/start.sh
+    image: nataliejschultz/rhodeisland-image:4.0
+    container_name: rhodeisland-nominatim
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+    volumes:
+      - nominatim-data:/var/lib/postgresql/14/main
+    networks:
+      - emission
+
+networks:
+  emission:
+
+volumes:
+  mongo-data:
+  nominatim-data:

emission/integrationTests/start_integration_tests.sh

@@ -0,0 +1,28 @@
+# Run the tests in the docker environment
+# Using an automated install 
+cd /src/e-mission-server
+
+#set database URL using environment variable
+echo ${DB_HOST}
+if [ -z ${DB_HOST} ] ; then
+    local_host=`hostname -i`
+    sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
+else
+    sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf
+fi
+cat conf/storage/db.conf
+
+echo "Setting up conda..."
+source setup/setup_conda.sh Linux-x86_64
+
+echo "Setting up the test environment..."
+source setup/setup_tests.sh
+
+echo "Running tests..."
+source setup/activate_tests.sh
+
+echo "Adding permissions for the runIntegrationTests.sh script"
+chmod +x runIntegrationTests.sh
+echo "Permissions added for the runIntegrationTests.sh script"
+
+./runIntegrationTests.sh

emission/net/ext_service/geocoder/nominatim.py

@@ -9,26 +9,22 @@
 import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse
 import logging
 import json
+import os
 
 from emission.core.wrapper.trip_old import Coordinate
-from pygeocoder import Geocoder as pyGeo  ## We fall back on this if we have to
-
 try:
-    googlemaps_key_file = open("conf/net/ext_service/googlemaps.json")
-    GOOGLE_MAPS_KEY = json.load(googlemaps_key_file)["api_key"]
-    googlemaps_key_file.close()
-except:
-    print("google maps key not configured, falling back to nominatim")
+    NOMINATIM_QUERY_URL = os.environ.get("NOMINATIM_QUERY_URL")
+    logging.info(f"NOMINATIM_QUERY_URL: {NOMINATIM_QUERY_URL}")
+    print("Nominatim Query URL Configured:", NOMINATIM_QUERY_URL)
 
-try:
-    nominatim_file = open("conf/net/ext_service/nominatim.json")
-    NOMINATIM_QUERY_URL = json.load(nominatim_file)["query_url"]
-    nominatim_file.close()
+    if NOMINATIM_QUERY_URL is None:
+        raise Exception("Nominatim query url not configured")
 except:
-    print("nominatim not configured either, place decoding must happen on the client")
+    print("Nominatim URL not configured, place decoding must happen on the client")
 
 class Geocoder(object):
 
+
     def __init__(self):
         pass
 
@@ -38,7 +34,6 @@ def make_url_geo(cls, address):
             "q" : address,
             "format" : "json"
         }
-
         query_url = NOMINATIM_QUERY_URL + "/search?"
         encoded_params = urllib.parse.urlencode(params)
         url = query_url + encoded_params
@@ -54,15 +49,10 @@ def get_json_geo(cls, address):
 
     @classmethod
     def geocode(cls, address):
-        # try:
-        #     jsn = cls.get_json_geo(address)
-        #     lat = float(jsn[0]["lat"])
-        #     lon = float(jsn[0]["lon"])
-        #     return Coordinate(lat, lon)
-        # except:
-        #     print "defaulting"
-        return _do_google_geo(address) # If we fail ask the gods
-
+        jsn = cls.get_json_geo(address)
+        lat = float(jsn[0]["lat"])
+        lon = float(jsn[0]["lon"])
+        return Coordinate(lat, lon)
 
     @classmethod
     def make_url_reverse(cls, lat, lon):
@@ -71,7 +61,6 @@ def make_url_reverse(cls, lat, lon):
             "lon" : lon,
             "format" : "json"
         }
-
         query_url = NOMINATIM_QUERY_URL + "/reverse?"
         encoded_params = urllib.parse.urlencode(params)
         url = query_url + encoded_params
@@ -88,22 +77,6 @@ def get_json_reverse(cls, lat, lng):
 
     @classmethod
     def reverse_geocode(cls, lat, lng):
-        # try:
-        #     jsn = cls.get_json_reverse(lat, lng)
-        #     address = jsn["display_name"]
-        #     return address
-
-        # except:
-        #     print "defaulting"
-        return _do_google_reverse(lat, lng) # Just in case
-
-## Failsafe section
-def _do_google_geo(address):
-    geo = pyGeo(GOOGLE_MAPS_KEY)
-    results = geo.geocode(address)
-    return Coordinate(results[0].coordinates[0], results[0].coordinates[1])
-
-def _do_google_reverse(lat, lng):
-    geo = pyGeo(GOOGLE_MAPS_KEY)
-    address = geo.reverse_geocode(lat, lng)
-    return address[0]
+        jsn = cls.get_json_reverse(lat, lng)
+        address = jsn["display_name"]
+        return address

runIntegrationTests.sh

@@ -0,0 +1,4 @@
+set -e
+#commented out portion can be added back in once all of the integration tests start passing. For now, we just want to run the nominatim test. 
+# PYTHONPATH=. python -m unittest discover -s emission/integrationTests -p Test*;
+PYTHONPATH=. python -m unittest emission/individual_tests/TestNominatim.py

setup/docker-compose.tests.yml

@@ -1,14 +1,13 @@
 version: "3"
 services:
   web-server:
+  #builds from tests/dockerfile
     build: tests
     depends_on:
       - db
     environment:
       - DB_HOST=db
       - WEB_SERVER_HOST=0.0.0.0
-    ports:
-      - "8080:8080"
     volumes:
       # specify the host directory where the source code should live
       # If this is ~/e-mission-server-docker, then you can edit the files at