-
Notifications
You must be signed in to change notification settings - Fork 119
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #929 from nataliejschultz/nominatim
Creating TestNominatim.py
- Loading branch information
Showing
10 changed files
with
270 additions
and
48 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| name: nominatim-docker-test | ||
|
|
||
| # Controls when the action will run. Triggers the workflow on push or pull request | ||
| # events but only for the master branch | ||
| on: | ||
| schedule: | ||
|
|
||
| # Run every Sunday at 4:05 am | ||
| - cron: '5 4 * * 0' | ||
| jobs: | ||
| # This workflow contains a single job called "build" | ||
| build: | ||
| # The type of runner that the job will run on | ||
| runs-on: ubuntu-latest | ||
|
|
||
| # Steps represent a sequence of tasks that will be executed as part of the job | ||
| steps: | ||
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | ||
| - name: Checkout | ||
| uses: actions/checkout@v2 | ||
|
|
||
| # Runs a single command using the runners shell | ||
| - name: Workflow test | ||
| run: echo Smoke test | ||
|
|
||
| # Passes the geofabrik key into the docker-compose.yml file. | ||
| - name: Test nominatim.py | ||
| run: GFBK_KEY=${{ secrets.GEOFABRIK_API }} docker-compose -f emission/integrationTests/docker-compose.yml up --exit-code-from web-server | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| from __future__ import unicode_literals | ||
| from __future__ import print_function | ||
| from __future__ import division | ||
| from __future__ import absolute_import | ||
| from future import standard_library | ||
| standard_library.install_aliases() | ||
| from builtins import * | ||
| import unittest | ||
| import importlib | ||
| import os | ||
| from emission.core.wrapper.trip_old import Coordinate | ||
| import requests | ||
| import emission.core.wrapper.entry as ecwe | ||
| import emission.analysis.intake.cleaning.clean_and_resample as clean | ||
| import emission.net.ext_service.geocoder.nominatim as eco | ||
|
|
||
| #Setting query URLs | ||
| OPENSTREETMAP_QUERY_URL = os.environ.get("OPENSTREETMAP_QUERY_URL") | ||
| GEOFABRIK_QUERY_URL = os.environ.get("GEOFABRIK_QUERY_URL") | ||
| NOMINATIM_CONTAINER_URL = os.environ.get("NOMINATIM_CONTAINER_URL") | ||
|
|
||
| class NominatimTest(unittest.TestCase): | ||
| maxDiff = None | ||
|
|
||
| def setUp(self): | ||
| #Creates a fake, cleaned place in Rhode Island | ||
| fake_id = "place_in_rhodeisland" | ||
| key = "segmentation/raw_place" | ||
| write_ts = 1694344333 | ||
| data = {'source': 'FakeTripGenerator','location': {'type': 'Point', 'coordinates': [-71.4128343, 41.8239891]}} | ||
| fake_place = ecwe.Entry.create_fake_entry(fake_id, key, data, write_ts) | ||
| self.fake_place = fake_place | ||
|
|
||
| #When a nominatim service is called, we set the value of the NOMINATIM_QUERY_URL environment variable in nominatim.py and re-load the module. | ||
| def nominatim(service): | ||
| if service == "container": | ||
| os.environ["NOMINATIM_QUERY_URL"] = NOMINATIM_CONTAINER_URL | ||
| importlib.reload(eco) | ||
| elif service == "geofabrik": | ||
| os.environ["NOMINATIM_QUERY_URL"] = GEOFABRIK_QUERY_URL | ||
| importlib.reload(eco) | ||
| elif service == "OSM": | ||
| os.environ["NOMINATIM_QUERY_URL"] = OPENSTREETMAP_QUERY_URL | ||
| importlib.reload(eco) | ||
|
|
||
| #Basic query to check that OSM, the Rhode Island Container, and geofabrik are returning the same data. | ||
| def test_geofabrik_and_nominatim(self): | ||
| lat, lon = 41.8239891, -71.4128343 | ||
| NominatimTest.nominatim("container") | ||
| container_result = eco.Geocoder.get_json_reverse(lat,lon) | ||
| NominatimTest.nominatim("OSM") | ||
| osm_result = eco.Geocoder.get_json_reverse(lat,lon) | ||
| NominatimTest.nominatim("geofabrik") | ||
| geofabrik_result = eco.Geocoder.get_json_reverse(lat,lon) | ||
| key_list = ['osm_id', 'boundingbox'] | ||
| for k in key_list: | ||
| self.assertEqual(osm_result[k], geofabrik_result[k]) | ||
| self.assertEqual(container_result[k], geofabrik_result[k]) | ||
|
|
||
| #Checks the display name generated by get_filtered_place in clean_and_resample.py, which creates a cleaned place from the fake place | ||
| # and reverse geocodes with the coordinates. | ||
| def test_get_filtered_place(self): | ||
| fake_place_raw = self.fake_place | ||
| fake_place_data = clean.get_filtered_place(fake_place_raw).__getattr__("data") | ||
| actual_result = fake_place_data.__getattr__("display_name") | ||
| expected_result = "Dorrance Street, Providence" | ||
| self.assertEqual(expected_result, actual_result) | ||
|
|
||
| #Testing make_url_geo, which creates a query URL from the input string. | ||
| def test_make_url_geo(self): | ||
| expected_result = GEOFABRIK_QUERY_URL + "/search?q=Providence%2C+Rhode+Island&format=json" | ||
| NominatimTest.nominatim("geofabrik") | ||
| actual_result = eco.Geocoder.make_url_geo("Providence, Rhode Island") | ||
| self.assertEqual(expected_result, actual_result) | ||
|
|
||
| #Testing make_url_reverse, which creates a query url from a lat and lon. | ||
| def test_make_url_reverse(self): | ||
| NominatimTest.nominatim("geofabrik") | ||
| lat, lon = 41.8239891, -71.4128343 | ||
| expected_result = GEOFABRIK_QUERY_URL + (f"/reverse?lat={lat}&lon={lon}&format=json") | ||
| actual_result = (eco.Geocoder.make_url_reverse(lat, lon)) | ||
| self.assertEqual(expected_result, actual_result) | ||
|
|
||
| #Testing get_json_geo, which passes in an address as a query. Compares three select k,v pairs in the results. | ||
| def test_get_json_geo(self): | ||
| NominatimTest.nominatim("geofabrik") | ||
| expected_result = {'place_id': 132490, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 141567710, 'boundingbox': ['41.8325787', '41.8332278', '-71.4161848', '-71.4152064'], 'lat': '41.8330097', 'lon': '-71.41568124868104', 'display_name': 'State of Rhode Island Department of Administration, 1, Park Street, Downtown, Providence, Providence County, Rhode Island, 02908, United States', 'class': 'building', 'type': 'civic', 'importance': 1.75001} | ||
| actual_result = eco.Geocoder.get_json_geo("State of Rhode Island Department of Administration, 1, Park Street, Downtown, Providence, Providence County, 02908, United States")[0] | ||
| key_list = ['osm_id', 'boundingbox', 'display_name'] | ||
| for k in key_list: | ||
| self.assertEqual(expected_result[k], actual_result[k]) | ||
|
|
||
| #Testing the geocode function, which passes in an address and gets latitude and longitude. | ||
| # Test creates instance of coordinates using coordinate class. Getting lat and lon of the coordinate using get_lat and get_lon methods from the class. | ||
| def test_geocode(self): | ||
| NominatimTest.nominatim("geofabrik") | ||
| expected_result_lon = Coordinate(41.8239891, -71.4128343).get_lon() | ||
| expected_result_lat = Coordinate(41.8239891, -71.4128343).get_lat() | ||
| actual_result = eco.Geocoder.geocode("Providence, Rhode Island") | ||
| actual_result_lon = actual_result.get_lon() | ||
| actual_result_lat = actual_result.get_lat() | ||
| self.assertEqual(expected_result_lon, actual_result_lon) | ||
| self.assertEqual(expected_result_lat, actual_result_lat) | ||
|
|
||
| #Testing get_json_reverse, which reverse geocodes from a lat and lon. Tested result was modified to only look at the name returned with the coordinates, rather than the entire dictionary. | ||
| def test_get_json_reverse(self): | ||
| NominatimTest.nominatim("geofabrik") | ||
| expected_result = "Providence City Hall" | ||
| actual_result = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343)["display_name"].split(",")[0] | ||
| self.assertEqual(expected_result, actual_result) | ||
|
|
||
| #Testing reverse_geocode, which reverse geocodes from a lat and lon and returns only the display name. | ||
| def test_reverse_geocode(self): | ||
| NominatimTest.nominatim("geofabrik") | ||
| expected_result = "Portugal Parkway, Fox Point, Providence, Providence County, Rhode Island, 02906, United States" | ||
| actual_result = eco.Geocoder.reverse_geocode(41.8174476, -71.3903767) | ||
| self.assertEqual(expected_result, actual_result) | ||
|
|
||
| if __name__ == '__main__': | ||
| unittest.main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| # python 3 | ||
| FROM ubuntu:latest | ||
|
|
||
| RUN apt-get update | ||
| RUN apt-get install -y curl | ||
|
|
||
| # CHANGEME: Create the files that correspond to your configuration in the conf directory | ||
|
|
||
| RUN echo "About to copy e-mission server code" | ||
| COPY start_integration_tests.sh/ /start_integration_tests.sh | ||
|
|
||
| CMD ["/bin/bash", "/start_integration_tests.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| version: "3" | ||
| services: | ||
| web-server: | ||
| build: | ||
| context: . | ||
| depends_on: | ||
| - db | ||
| - nominatim | ||
| environment: | ||
| - DB_HOST=db | ||
| - GFBK_KEY=$GFBK_KEY | ||
| - GEOFABRIK_QUERY_URL=https://geocoding.geofabrik.de/$GFBK_KEY | ||
| - OPENSTREETMAP_QUERY_URL=https://nominatim.openstreetmap.org | ||
| - NOMINATIM_CONTAINER_URL=http://rhodeisland-nominatim:8080 | ||
|
|
||
| volumes: | ||
| # specify the host directory where the source code should live | ||
| # If this is ~/e-mission-server-docker, then you can edit the files at | ||
| # ~/e-mission-server-docker/src/e-mission-server/emission/... | ||
| # - CHANGEME:/src/ | ||
| - ../..:/src/e-mission-server | ||
| networks: | ||
| - emission | ||
|
|
||
| db: | ||
| image: mongo:4.4.0 | ||
| deploy: | ||
| replicas: 1 | ||
| restart_policy: | ||
| condition: on-failure | ||
|
|
||
| #Volumes is the preferred way to persist data generated by a container. In this case we use a volume to persist the contents | ||
| #of the data base. Learn more about how to use volumes here: https://docs.docker.com/storage/volumes/ | ||
| # And learn how to configure volumes in your compose file here: https://docs.docker.com/compose/compose-file/#volume-configuration-reference | ||
| volumes: | ||
| - mongo-data:/data/db | ||
| networks: | ||
| - emission | ||
| #adding section to incorporate nominatim server functionality | ||
| nominatim: | ||
| entrypoint: /app/start.sh | ||
| image: nataliejschultz/rhodeisland-image:4.0 | ||
| container_name: rhodeisland-nominatim | ||
| deploy: | ||
| replicas: 1 | ||
| restart_policy: | ||
| condition: on-failure | ||
| volumes: | ||
| - nominatim-data:/var/lib/postgresql/14/main | ||
| networks: | ||
| - emission | ||
|
|
||
| networks: | ||
| emission: | ||
|
|
||
| volumes: | ||
| mongo-data: | ||
| nominatim-data: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| # Run the tests in the docker environment | ||
| # Using an automated install | ||
| cd /src/e-mission-server | ||
|
|
||
| #set database URL using environment variable | ||
| echo ${DB_HOST} | ||
| if [ -z ${DB_HOST} ] ; then | ||
| local_host=`hostname -i` | ||
| sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf | ||
| else | ||
| sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf | ||
| fi | ||
| cat conf/storage/db.conf | ||
|
|
||
| echo "Setting up conda..." | ||
| source setup/setup_conda.sh Linux-x86_64 | ||
|
|
||
| echo "Setting up the test environment..." | ||
| source setup/setup_tests.sh | ||
|
|
||
| echo "Running tests..." | ||
| source setup/activate_tests.sh | ||
|
|
||
| echo "Adding permissions for the runIntegrationTests.sh script" | ||
| chmod +x runIntegrationTests.sh | ||
| echo "Permissions added for the runIntegrationTests.sh script" | ||
|
|
||
| ./runIntegrationTests.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| set -e | ||
| #commented out portion can be added back in once all of the integration tests start passing. For now, we just want to run the nominatim test. | ||
| # PYTHONPATH=. python -m unittest discover -s emission/integrationTests -p Test*; | ||
| PYTHONPATH=. python -m unittest emission/individual_tests/TestNominatim.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters