diff --git a/.github/workflows/nominatim-docker-test.yml b/.github/workflows/nominatim-docker-test.yml new file mode 100644 index 000000000..c684b1fde --- /dev/null +++ b/.github/workflows/nominatim-docker-test.yml @@ -0,0 +1,30 @@ +name: nominatim-docker-test + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + schedule: + + # Run every Sunday at 4:05 am + - cron: '5 4 * * 0' +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout + uses: actions/checkout@v2 + + # Runs a single command using the runners shell + - name: Workflow test + run: echo Smoke test + + # Passes the geofabrik key into the docker-compose.yml file. + - name: Test nominatim.py + run: GFBK_KEY=${{ secrets.GEOFABRIK_API }} docker-compose -f emission/integrationTests/docker-compose.yml up --exit-code-from web-server + + diff --git a/README.md b/README.md index f8fdfbd45..b82b3728d 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,7 @@ backend server - the phone apps are available in the [e-mission-phone repo](https://github.com/amplab/e-mission-phone) -- **Master** [![master:test-with-docker](https://github.com/e-mission/e-mission-server/workflows/test-with-docker/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Atest-with-docker) [![master:ubuntu-only-test-with-manual-install](https://github.com/e-mission/e-mission-server/workflows/ubuntu-only-test-with-manual-install/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Aubuntu-only-test-with-manual-install) [![master:osx-ubuntu-manual-install](https://github.com/e-mission/e-mission-server/workflows/osx-ubuntu-manual-install/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Aosx-ubuntu-manual-install) - -- **GIS branch:** [![master:ubuntu-only-test-with-manual-install](https://github.com/e-mission/e-mission-server/workflows/ubuntu-only-test-with-manual-install/badge.svg?branch=gis-based-mode-detection)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Agis-based-mode-detection+workflow%3Aubuntu-only-test-with-manual-install) [![osx-ubuntu-manual-install](https://github.com/e-mission/e-mission-server/workflows/osx-ubuntu-manual-install/badge.svg?branch=gis-based-mode-detection)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Agis-based-mode-detection+workflow%3Aosx-ubuntu-manual-install) - +- **Master** [![master:test-with-docker](https://github.com/e-mission/e-mission-server/workflows/test-with-docker/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Atest-with-docker) [![master:ubuntu-only-test-with-manual-install](https://github.com/e-mission/e-mission-server/workflows/ubuntu-only-test-with-manual-install/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Aubuntu-only-test-with-manual-install) [![master:osx-ubuntu-manual-install](https://github.com/e-mission/e-mission-server/workflows/osx-ubuntu-manual-install/badge.svg)](https://github.com/e-mission/e-mission-server/actions?query=branch%3Amaster+workflow%3Aosx-ubuntu-manual-install) [![master:nominatim-docker-test](https://github.com/e-mission/e-mission-server/workflows/nominatim-docker-test/badge.svg)](https://github.com/e-mission/e-mission-server/actions/workflows/nominatim-docker-test.yml?query=branch%3Amaster) **Issues:** Since this repository is part of a larger project, all issues are tracked [in the central docs repository](https://github.com/e-mission/e-mission-docs/issues). If you have a question, [as suggested by the open source guide](https://opensource.guide/how-to-contribute/#communicating-effectively), please file an issue instead of sending an email. Since issues are public, other contributors can try to answer the question and benefit from the answer. diff --git a/bin/debug/save_ground_truth.py b/bin/debug/save_ground_truth.py index 2a8f371b8..b8b95ab52 100644 --- a/bin/debug/save_ground_truth.py +++ b/bin/debug/save_ground_truth.py @@ -32,6 +32,7 @@ def save_ct_list(args): parser_diary = subparsers.add_parser('diary', help='diary-based ground truth') parser_diary.add_argument("date", help="date to retrieve ground truth (YYYY-MM-DD)") + parser_diary.add_argument("file_name", help="file name to store the result to") parser_diary.set_defaults(func=save_diary) parser_obj_list = subparsers.add_parser('objects', help='download analysis objects directly') diff --git a/emission/individual_tests/TestNominatim.py b/emission/individual_tests/TestNominatim.py new file mode 100644 index 000000000..a7f75b958 --- /dev/null +++ b/emission/individual_tests/TestNominatim.py @@ -0,0 +1,120 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +from builtins import * +import unittest +import importlib +import os +from emission.core.wrapper.trip_old import Coordinate +import requests +import emission.core.wrapper.entry as ecwe +import emission.analysis.intake.cleaning.clean_and_resample as clean +import emission.net.ext_service.geocoder.nominatim as eco + +#Setting query URLs +OPENSTREETMAP_QUERY_URL = os.environ.get("OPENSTREETMAP_QUERY_URL") +GEOFABRIK_QUERY_URL = os.environ.get("GEOFABRIK_QUERY_URL") +NOMINATIM_CONTAINER_URL = os.environ.get("NOMINATIM_CONTAINER_URL") + +class NominatimTest(unittest.TestCase): + maxDiff = None + + def setUp(self): + #Creates a fake, cleaned place in Rhode Island + fake_id = "place_in_rhodeisland" + key = "segmentation/raw_place" + write_ts = 1694344333 + data = {'source': 'FakeTripGenerator','location': {'type': 'Point', 'coordinates': [-71.4128343, 41.8239891]}} + fake_place = ecwe.Entry.create_fake_entry(fake_id, key, data, write_ts) + self.fake_place = fake_place + + #When a nominatim service is called, we set the value of the NOMINATIM_QUERY_URL environment variable in nominatim.py and re-load the module. + def nominatim(service): + if service == "container": + os.environ["NOMINATIM_QUERY_URL"] = NOMINATIM_CONTAINER_URL + importlib.reload(eco) + elif service == "geofabrik": + os.environ["NOMINATIM_QUERY_URL"] = GEOFABRIK_QUERY_URL + importlib.reload(eco) + elif service == "OSM": + os.environ["NOMINATIM_QUERY_URL"] = OPENSTREETMAP_QUERY_URL + importlib.reload(eco) + + #Basic query to check that OSM, the Rhode Island Container, and geofabrik are returning the same data. + def test_geofabrik_and_nominatim(self): + lat, lon = 41.8239891, -71.4128343 + NominatimTest.nominatim("container") + container_result = eco.Geocoder.get_json_reverse(lat,lon) + NominatimTest.nominatim("OSM") + osm_result = eco.Geocoder.get_json_reverse(lat,lon) + NominatimTest.nominatim("geofabrik") + geofabrik_result = eco.Geocoder.get_json_reverse(lat,lon) + key_list = ['osm_id', 'boundingbox'] + for k in key_list: + self.assertEqual(osm_result[k], geofabrik_result[k]) + self.assertEqual(container_result[k], geofabrik_result[k]) + + #Checks the display name generated by get_filtered_place in clean_and_resample.py, which creates a cleaned place from the fake place + # and reverse geocodes with the coordinates. + def test_get_filtered_place(self): + fake_place_raw = self.fake_place + fake_place_data = clean.get_filtered_place(fake_place_raw).__getattr__("data") + actual_result = fake_place_data.__getattr__("display_name") + expected_result = "Dorrance Street, Providence" + self.assertEqual(expected_result, actual_result) + + #Testing make_url_geo, which creates a query URL from the input string. + def test_make_url_geo(self): + expected_result = GEOFABRIK_QUERY_URL + "/search?q=Providence%2C+Rhode+Island&format=json" + NominatimTest.nominatim("geofabrik") + actual_result = eco.Geocoder.make_url_geo("Providence, Rhode Island") + self.assertEqual(expected_result, actual_result) + + #Testing make_url_reverse, which creates a query url from a lat and lon. + def test_make_url_reverse(self): + NominatimTest.nominatim("geofabrik") + lat, lon = 41.8239891, -71.4128343 + expected_result = GEOFABRIK_QUERY_URL + (f"/reverse?lat={lat}&lon={lon}&format=json") + actual_result = (eco.Geocoder.make_url_reverse(lat, lon)) + self.assertEqual(expected_result, actual_result) + + #Testing get_json_geo, which passes in an address as a query. Compares three select k,v pairs in the results. + def test_get_json_geo(self): + NominatimTest.nominatim("geofabrik") + expected_result = {'place_id': 132490, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 141567710, 'boundingbox': ['41.8325787', '41.8332278', '-71.4161848', '-71.4152064'], 'lat': '41.8330097', 'lon': '-71.41568124868104', 'display_name': 'State of Rhode Island Department of Administration, 1, Park Street, Downtown, Providence, Providence County, Rhode Island, 02908, United States', 'class': 'building', 'type': 'civic', 'importance': 1.75001} + actual_result = eco.Geocoder.get_json_geo("State of Rhode Island Department of Administration, 1, Park Street, Downtown, Providence, Providence County, 02908, United States")[0] + key_list = ['osm_id', 'boundingbox', 'display_name'] + for k in key_list: + self.assertEqual(expected_result[k], actual_result[k]) + + #Testing the geocode function, which passes in an address and gets latitude and longitude. + # Test creates instance of coordinates using coordinate class. Getting lat and lon of the coordinate using get_lat and get_lon methods from the class. + def test_geocode(self): + NominatimTest.nominatim("geofabrik") + expected_result_lon = Coordinate(41.8239891, -71.4128343).get_lon() + expected_result_lat = Coordinate(41.8239891, -71.4128343).get_lat() + actual_result = eco.Geocoder.geocode("Providence, Rhode Island") + actual_result_lon = actual_result.get_lon() + actual_result_lat = actual_result.get_lat() + self.assertEqual(expected_result_lon, actual_result_lon) + self.assertEqual(expected_result_lat, actual_result_lat) + + #Testing get_json_reverse, which reverse geocodes from a lat and lon. Tested result was modified to only look at the name returned with the coordinates, rather than the entire dictionary. + def test_get_json_reverse(self): + NominatimTest.nominatim("geofabrik") + expected_result = "Providence City Hall" + actual_result = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343)["display_name"].split(",")[0] + self.assertEqual(expected_result, actual_result) + + #Testing reverse_geocode, which reverse geocodes from a lat and lon and returns only the display name. + def test_reverse_geocode(self): + NominatimTest.nominatim("geofabrik") + expected_result = "Portugal Parkway, Fox Point, Providence, Providence County, Rhode Island, 02906, United States" + actual_result = eco.Geocoder.reverse_geocode(41.8174476, -71.3903767) + self.assertEqual(expected_result, actual_result) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/emission/integrationTests/Dockerfile b/emission/integrationTests/Dockerfile new file mode 100644 index 000000000..9bdc0484d --- /dev/null +++ b/emission/integrationTests/Dockerfile @@ -0,0 +1,12 @@ +# python 3 +FROM ubuntu:latest + +RUN apt-get update +RUN apt-get install -y curl + +# CHANGEME: Create the files that correspond to your configuration in the conf directory + +RUN echo "About to copy e-mission server code" +COPY start_integration_tests.sh/ /start_integration_tests.sh + +CMD ["/bin/bash", "/start_integration_tests.sh"] \ No newline at end of file diff --git a/emission/integrationTests/docker-compose.yml b/emission/integrationTests/docker-compose.yml new file mode 100644 index 000000000..6a9444dee --- /dev/null +++ b/emission/integrationTests/docker-compose.yml @@ -0,0 +1,58 @@ +version: "3" +services: + web-server: + build: + context: . + depends_on: + - db + - nominatim + environment: + - DB_HOST=db + - GFBK_KEY=$GFBK_KEY + - GEOFABRIK_QUERY_URL=https://geocoding.geofabrik.de/$GFBK_KEY + - OPENSTREETMAP_QUERY_URL=https://nominatim.openstreetmap.org + - NOMINATIM_CONTAINER_URL=http://rhodeisland-nominatim:8080 + + volumes: + # specify the host directory where the source code should live + # If this is ~/e-mission-server-docker, then you can edit the files at + # ~/e-mission-server-docker/src/e-mission-server/emission/... + # - CHANGEME:/src/ + - ../..:/src/e-mission-server + networks: + - emission + + db: + image: mongo:4.4.0 + deploy: + replicas: 1 + restart_policy: + condition: on-failure + + #Volumes is the preferred way to persist data generated by a container. In this case we use a volume to persist the contents + #of the data base. Learn more about how to use volumes here: https://docs.docker.com/storage/volumes/ + # And learn how to configure volumes in your compose file here: https://docs.docker.com/compose/compose-file/#volume-configuration-reference + volumes: + - mongo-data:/data/db + networks: + - emission + #adding section to incorporate nominatim server functionality + nominatim: + entrypoint: /app/start.sh + image: nataliejschultz/rhodeisland-image:4.0 + container_name: rhodeisland-nominatim + deploy: + replicas: 1 + restart_policy: + condition: on-failure + volumes: + - nominatim-data:/var/lib/postgresql/14/main + networks: + - emission + +networks: + emission: + +volumes: + mongo-data: + nominatim-data: \ No newline at end of file diff --git a/emission/integrationTests/start_integration_tests.sh b/emission/integrationTests/start_integration_tests.sh new file mode 100644 index 000000000..da2e30e5b --- /dev/null +++ b/emission/integrationTests/start_integration_tests.sh @@ -0,0 +1,28 @@ +# Run the tests in the docker environment +# Using an automated install +cd /src/e-mission-server + +#set database URL using environment variable +echo ${DB_HOST} +if [ -z ${DB_HOST} ] ; then + local_host=`hostname -i` + sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf +else + sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf +fi +cat conf/storage/db.conf + +echo "Setting up conda..." +source setup/setup_conda.sh Linux-x86_64 + +echo "Setting up the test environment..." +source setup/setup_tests.sh + +echo "Running tests..." +source setup/activate_tests.sh + +echo "Adding permissions for the runIntegrationTests.sh script" +chmod +x runIntegrationTests.sh +echo "Permissions added for the runIntegrationTests.sh script" + +./runIntegrationTests.sh \ No newline at end of file diff --git a/emission/net/ext_service/geocoder/nominatim.py b/emission/net/ext_service/geocoder/nominatim.py index a56b177b2..c5982478a 100644 --- a/emission/net/ext_service/geocoder/nominatim.py +++ b/emission/net/ext_service/geocoder/nominatim.py @@ -9,26 +9,22 @@ import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse import logging import json +import os from emission.core.wrapper.trip_old import Coordinate -from pygeocoder import Geocoder as pyGeo ## We fall back on this if we have to - try: - googlemaps_key_file = open("conf/net/ext_service/googlemaps.json") - GOOGLE_MAPS_KEY = json.load(googlemaps_key_file)["api_key"] - googlemaps_key_file.close() -except: - print("google maps key not configured, falling back to nominatim") + NOMINATIM_QUERY_URL = os.environ.get("NOMINATIM_QUERY_URL") + logging.info(f"NOMINATIM_QUERY_URL: {NOMINATIM_QUERY_URL}") + print("Nominatim Query URL Configured:", NOMINATIM_QUERY_URL) -try: - nominatim_file = open("conf/net/ext_service/nominatim.json") - NOMINATIM_QUERY_URL = json.load(nominatim_file)["query_url"] - nominatim_file.close() + if NOMINATIM_QUERY_URL is None: + raise Exception("Nominatim query url not configured") except: - print("nominatim not configured either, place decoding must happen on the client") + print("Nominatim URL not configured, place decoding must happen on the client") class Geocoder(object): + def __init__(self): pass @@ -38,7 +34,6 @@ def make_url_geo(cls, address): "q" : address, "format" : "json" } - query_url = NOMINATIM_QUERY_URL + "/search?" encoded_params = urllib.parse.urlencode(params) url = query_url + encoded_params @@ -54,15 +49,10 @@ def get_json_geo(cls, address): @classmethod def geocode(cls, address): - # try: - # jsn = cls.get_json_geo(address) - # lat = float(jsn[0]["lat"]) - # lon = float(jsn[0]["lon"]) - # return Coordinate(lat, lon) - # except: - # print "defaulting" - return _do_google_geo(address) # If we fail ask the gods - + jsn = cls.get_json_geo(address) + lat = float(jsn[0]["lat"]) + lon = float(jsn[0]["lon"]) + return Coordinate(lat, lon) @classmethod def make_url_reverse(cls, lat, lon): @@ -71,7 +61,6 @@ def make_url_reverse(cls, lat, lon): "lon" : lon, "format" : "json" } - query_url = NOMINATIM_QUERY_URL + "/reverse?" encoded_params = urllib.parse.urlencode(params) url = query_url + encoded_params @@ -88,22 +77,6 @@ def get_json_reverse(cls, lat, lng): @classmethod def reverse_geocode(cls, lat, lng): - # try: - # jsn = cls.get_json_reverse(lat, lng) - # address = jsn["display_name"] - # return address - - # except: - # print "defaulting" - return _do_google_reverse(lat, lng) # Just in case - -## Failsafe section -def _do_google_geo(address): - geo = pyGeo(GOOGLE_MAPS_KEY) - results = geo.geocode(address) - return Coordinate(results[0].coordinates[0], results[0].coordinates[1]) - -def _do_google_reverse(lat, lng): - geo = pyGeo(GOOGLE_MAPS_KEY) - address = geo.reverse_geocode(lat, lng) - return address[0] + jsn = cls.get_json_reverse(lat, lng) + address = jsn["display_name"] + return address \ No newline at end of file diff --git a/runIntegrationTests.sh b/runIntegrationTests.sh new file mode 100755 index 000000000..66d077243 --- /dev/null +++ b/runIntegrationTests.sh @@ -0,0 +1,4 @@ +set -e +#commented out portion can be added back in once all of the integration tests start passing. For now, we just want to run the nominatim test. +# PYTHONPATH=. python -m unittest discover -s emission/integrationTests -p Test*; +PYTHONPATH=. python -m unittest emission/individual_tests/TestNominatim.py diff --git a/setup/docker-compose.tests.yml b/setup/docker-compose.tests.yml index 0b6e93f9e..be7e63285 100644 --- a/setup/docker-compose.tests.yml +++ b/setup/docker-compose.tests.yml @@ -1,14 +1,13 @@ version: "3" services: web-server: + #builds from tests/dockerfile build: tests depends_on: - db environment: - DB_HOST=db - WEB_SERVER_HOST=0.0.0.0 - ports: - - "8080:8080" volumes: # specify the host directory where the source code should live # If this is ~/e-mission-server-docker, then you can edit the files at