Integration tests for Geofabrik Overpass API

.github/workflows/test-overpass.yml

@@ -0,0 +1,28 @@
+name: overpass-test
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  schedule:
+
+    # Run every Sunday at 6:05 am
+    - cron:  '5 6 * * 0'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    # Tests runner is functional
+    - name: Workflow test
+      run: echo Good morning!
+
+      # Runs TestOverpass.py to check for API functionality
+    - name: Test Overpass
+      env:
+        GEOFABRIK_OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
+      run: |
+        echo Testing overpass!
+        chmod +x emission/individual_tests/setup_and_test_overpass.sh
+        ./emission/individual_tests/setup_and_test_overpass.sh

emission/individual_tests/TestOverpass.py

@@ -0,0 +1,62 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+import unittest
+import os
+import requests
+import emission.net.ext_service.transit_matching.match_stops as enetm
+import logging
+
+#Set up query
+GEOFABRIK_OVERPASS_KEY = os.environ.get("GEOFABRIK_OVERPASS_KEY")
+
+#Sample loc1 = NREL East Gate
+loc1 = {'coordinates': [-105.16844103184974, 39.740428870224605]}
+#Sample loc2 = Denver Union Station
+loc2 = {'coordinates': [-105.00083982302972, 39.753710532185025]}
+#Sample loc3 = Grand Junction Train Station, CO
+loc3 = {'coordinates': [-108.57055213129632, 39.06472424640481]}
+
+class OverpassTest(unittest.TestCase):
+    def setUp(self):
+        # Un-comment the two lines below to print debug logs.
+        # loglevel = logging.DEBUG
+        # logging.basicConfig(level=loglevel)
+        sample_data = '[out:json][bbox];way[amenity=parking];out;&bbox=-122.1111238,37.4142118,-122.1055791,37.4187945'
+        call_base = 'api/interpreter?data='
+        self.public_url_base = 'https://lz4.overpass-api.de/'+ call_base + sample_data
+        self.gfbk_url_base = 'https://overpass.geofabrik.de/' + GEOFABRIK_OVERPASS_KEY + '/' + call_base + sample_data
+
+    def test_overpass(self):
+        r_gfbk = requests.get(self.gfbk_url_base)
+        r_public = requests.get(self.public_url_base)
+
+        if r_gfbk.status_code == 200 and r_public.status_code == 200:
+            print("requests successful!")
+            r_gfbk_len, r_public_len = len(r_gfbk.json()), len(r_public.json())
+            self.assertEqual(r_gfbk_len, r_public_len)
+        else:
+            print("status_gfbk", r_gfbk.status_code, type(r_gfbk.status_code), "status_public", r_public.status_code)
+
+    #Test utilizes the functions get_stops_near, get_public_transit_stops, and make_request_and_catch.  
+    def test_get_stops_near(self):
+        actual_result = enetm.get_stops_near(loc1, 150.0)[0]['routes'][0]['tags']
+        expected_result = {'from': 'National Renewable Energy Lab', 'name': 'RTD Route 125: Red Rocks College', 'network': 'RTD', 'network:wikidata': 'Q7309183', 'network:wikipedia': 'en:Regional Transportation District', 'operator': 'Regional Transportation District', 'public_transport:version': '1', 'ref': '125', 'route': 'bus', 'to': 'Red Rocks College', 'type': 'route'}
+        self.assertEqual(expected_result, actual_result)
+
+    #Get_stops_near generates two stops from the given coordinates.
+    # Get_predicted_transit_mode finds a common route between them (train).
+    def test_get_predicted_transit_mode(self):
+        stop1 = enetm.get_stops_near(loc2, 400.0)
+        stop2 = enetm.get_stops_near(loc3, 400.0)
+        actual_result = enetm.get_predicted_transit_mode(stop1, stop2)
+        expected_result = ['train', 'train']
+        self.assertEqual(actual_result, expected_result)
+
+if __name__ == '__main__':
+    unittest.main()
+

emission/individual_tests/setup_and_test_overpass.sh

@@ -0,0 +1,16 @@
+#Set up the testing environment
+# Using an automated install 
+
+echo ${DB_HOST}
+
+echo "Setting up conda..."
+source setup/setup_conda.sh Linux-x86_64
+
+echo "Setting up the test environment..."
+source setup/setup_tests.sh
+
+echo "Running tests..."
+source setup/activate_tests.sh
+
+set -e
+PYTHONPATH=. python -m unittest emission/individual_tests/TestOverpass.py

emission/net/ext_service/transit_matching/match_stops.py

@@ -3,24 +3,23 @@
 import requests
 import attrdict as ad
 import itertools
-import copy
+import os
 import time
 
 try:
-    config_file = open('conf/net/ext_service/overpass_server.json')
+    GEOFABRIK_OVERPASS_KEY = os.environ.get("GEOFABRIK_OVERPASS_KEY")
+    url = 'https://overpass.geofabrik.de/' + GEOFABRIK_OVERPASS_KEY + '/'
+    print("overpass configured")
 except:
-    print("overpass not configured, falling back to default overleaf.de")
-    config_file = open('conf/net/ext_service/overpass_server.json.sample')
+    print("overpass not configured, falling back to public overpass api")
+    url = "https://lz4.overpass-api.de/"
 
 try:
     query_file = open('conf/net/ext_service/overpass_transit_stops_query_template')
 except:
     print("transit stops query not configured, falling back to default")
     query_file = open('conf/net/ext_service/overpass_transit_stops_query_template.sample')
 
-config_data = json.load(config_file)
-url = config_data["url"]
-
 query_string = "".join(query_file.readlines())
 
 RETRY = -1