Merge branch 'master' of https://github.com/e-mission/e-mission-server …

…into Extend-Get-Store-Range

.github/workflows/test-with-manual-install.yml

@@ -70,4 +70,6 @@ jobs:
 
     - name: Teardown the test environment
       shell: bash -l {0}
-      run: source setup/teardown_tests.sh
+      run: |
+        source setup/activate_conda.sh
+        source setup/teardown_tests.sh

emission/analysis/config.py

@@ -1,17 +1,21 @@
 import json
 import os
 
+ANALYSIS_CONF_PATH = "conf/analysis/debug.conf.json"
+ANALYSIS_CONF_PROD_PATH = "conf/analysis/debug.conf.prod.json"
+ANALYSIS_CONF_DEV_PATH = "conf/analysis/debug.conf.dev.json"
+
 def get_config_data():
     try:
         print("Trying to open debug.conf.json")
-        config_file = open('conf/analysis/debug.conf.json')
+        config_file = open(ANALYSIS_CONF_PATH)
     except:
         if os.getenv("PROD_STAGE") == "TRUE":
             print("In production environment, config not overridden, using default production debug.conf")
-            config_file = open('conf/analysis/debug.conf.prod.json')
+            config_file = open(ANALYSIS_CONF_PROD_PATH)
         else:
             print("analysis.debug.conf.json not configured, falling back to sample, default configuration")
-            config_file = open('conf/analysis/debug.conf.dev.json')
+            config_file = open(ANALYSIS_CONF_DEV_PATH)
     ret_val = json.load(config_file)
     config_file.close()
     return ret_val

emission/net/ext_service/transit_matching/match_stops.py

@@ -15,12 +15,12 @@
     url = "https://lz4.overpass-api.de/"
 
 try:
-    query_file = open('conf/net/ext_service/overpass_transit_stops_query_template')
-except:
+    with open('conf/net/ext_service/overpass_transit_stops_query_template', 'r', encoding='UTF-8') as query_file:
+        query_string = "".join(query_file.readlines())
+except FileNotFoundError:
     print("transit stops query not configured, falling back to default")
-    query_file = open('conf/net/ext_service/overpass_transit_stops_query_template.sample')
-
-query_string = "".join(query_file.readlines())
+    with open('conf/net/ext_service/overpass_transit_stops_query_template.sample', 'r', encoding='UTF-8') as query_file:
+        query_string = "".join(query_file.readlines())
 
 RETRY = -1
 

emission/tests/analysisTests/intakeTests/TestFilterAccuracy.py

@@ -34,15 +34,14 @@ def setUp(self):
         import emission.core.get_database as edb
         import uuid
 
-        self.analysis_conf_path = \
-            etc.set_analysis_config("intake.cleaning.filter_accuracy.enable", True)
+        etc.set_analysis_config("intake.cleaning.filter_accuracy.enable", True)
         self.testUUID = None
 
     def tearDown(self):
         import emission.core.get_database as edb
         edb.get_timeseries_db().delete_many({"user_id": self.testUUID})
         edb.get_pipeline_state_db().delete_many({"user_id": self.testUUID})
-        os.remove(self.analysis_conf_path)
+        etc.clear_analysis_config()
 
     def checkSuccessfulRun(self):
         pipelineState = edb.get_pipeline_state_db().find_one({"user_id": self.testUUID,

emission/tests/analysisTests/intakeTests/TestPipelineRealData.py

@@ -60,8 +60,7 @@ class TestPipelineRealData(unittest.TestCase):
     def setUp(self):
         # Thanks to M&J for the number!
         np.random.seed(61297777)
-        self.analysis_conf_path = \
-            etc.set_analysis_config("analysis.result.section.key", "analysis/cleaned_section")
+        etc.set_analysis_config("analysis.result.section.key", "analysis/cleaned_section")
         logging.info("setUp complete")
 
     def tearDown(self):
@@ -76,8 +75,7 @@ def tearDown(self):
             # to determine whether to switch to a new implementation
             if not hasattr(self, "evaluation") or not self.evaluation:
                 self.clearRelatedDb()
-            if hasattr(self, "analysis_conf_path"):
-                os.remove(self.analysis_conf_path)
+            etc.clear_analysis_config()
             if hasattr(self, "seed_mode_path"):
                 os.remove(self.seed_mode_path)
             logging.info("tearDown complete")
@@ -744,8 +742,7 @@ def testJackUntrackedTimeMar12(self):
 
     def testJackUntrackedTimeMar12InferredSections(self):
         # Setup to use the inferred sections
-        self.analysis_conf_path = \
-            etc.set_analysis_config("analysis.result.section.key", "analysis/inferred_section")
+        etc.set_analysis_config("analysis.result.section.key", "analysis/inferred_section")
         # along with the proper random seed
         self.seed_mode_path = etc.copy_dummy_seed_for_inference()
         dataFile = "emission/tests/data/real_examples/jack_untracked_time_2023-03-12"

emission/tests/analysisTests/intakeTests/TestSectionSegmentation.py

@@ -41,8 +41,7 @@
 
 class TestSectionSegmentation(unittest.TestCase):
     def setUp(self):
-        self.analysis_conf_path = \
-            etc.set_analysis_config("intake.cleaning.filter_accuracy.enable", True)
+        etc.set_analysis_config("intake.cleaning.filter_accuracy.enable", True)
 
         etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-aug-27")
         self.androidUUID = self.testUUID
@@ -58,8 +57,7 @@ def setUp(self):
     def tearDown(self):
         if not hasattr(self, "evaluation") or not self.evaluation:
             self.clearRelatedDb()
-        if hasattr(self, "analysis_conf_path"):
-            os.remove(self.analysis_conf_path)
+        etc.clear_analysis_config()
 
     def clearRelatedDb(self):
         edb.get_timeseries_db().delete_many({"user_id": self.androidUUID})

emission/tests/analysisTests/intakeTests/TestTripSegmentation.py

@@ -36,8 +36,7 @@
 
 class TestTripSegmentation(unittest.TestCase):
     def setUp(self):
-        self.analysis_conf_path = \
-            etc.set_analysis_config("intake.cleaning.filter_accuracy.enable", True)
+        etc.set_analysis_config("intake.cleaning.filter_accuracy.enable", True)
 
         etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-aug-27")
         self.androidUUID = self.testUUID
@@ -51,7 +50,7 @@ def setUp(self):
         logging.debug("androidUUID = %s, iosUUID = %s" % (self.androidUUID, self.iosUUID))
 
     def tearDown(self):
-        os.remove(self.analysis_conf_path)
+        etc.clear_analysis_config()
         edb.get_timeseries_db().delete_many({"user_id": self.androidUUID}) 
         edb.get_timeseries_db().delete_many({"user_id": self.iosUUID})
         edb.get_pipeline_state_db().delete_many({"user_id": self.androidUUID})

emission/tests/common.py

@@ -266,25 +266,29 @@ def createDummyRequestEnviron(self, addl_headers, request_body):
     return test_environ
 
 def set_analysis_config(key, value):
+    """
+    Tests that call this in their setUp must call clear_analysis_config in their tearDown
+    """
     import emission.analysis.config as eac
     import shutil
 
-    analysis_conf_path = "conf/analysis/debug.conf.json"
-    shutil.copyfile("conf/analysis/debug.conf.dev.json",
-                    analysis_conf_path)
-    with open(analysis_conf_path) as fd:
+    shutil.copyfile(eac.ANALYSIS_CONF_DEV_PATH, eac.ANALYSIS_CONF_PATH)
+    with open(eac.ANALYSIS_CONF_PATH) as fd:
         curr_config = json.load(fd)
     curr_config[key] = value
-    with open(analysis_conf_path, "w") as fd:
+    with open(eac.ANALYSIS_CONF_PATH, "w") as fd:
         json.dump(curr_config, fd, indent=4)
-    logging.debug("Finished setting up %s" % analysis_conf_path)
-    with open(analysis_conf_path) as fd:
+    logging.debug("Finished setting up %s" % eac.ANALYSIS_CONF_PATH)
+    with open(eac.ANALYSIS_CONF_PATH) as fd:
         logging.debug("Current values are %s" % json.load(fd))
 
     eac.reload_config()
-
-    # Return this so that we can delete it in the teardown
-    return analysis_conf_path
+
+def clear_analysis_config():
+    import emission.analysis.config as eac
+    if os.path.exists(eac.ANALYSIS_CONF_PATH):
+        os.remove(eac.ANALYSIS_CONF_PATH)
+    eac.reload_config()
 
 def copy_dummy_seed_for_inference():
     import shutil

emission/tests/netTests/TestMetricsCleanedSections.py

@@ -23,8 +23,7 @@
 
 class TestMetrics(unittest.TestCase):
     def setUp(self):
-        self.analysis_conf_path = \
-            etc.set_analysis_config("analysis.result.section.key", "analysis/cleaned_section")
+        etc.set_analysis_config("analysis.result.section.key", "analysis/cleaned_section")
         etc.setupRealExample(self,
                              "emission/tests/data/real_examples/shankari_2015-aug-21")
         self.testUUID1 = self.testUUID
@@ -41,7 +40,7 @@ def setUp(self):
 
     def tearDown(self):
         self.clearRelatedDb()
-        os.remove(self.analysis_conf_path)
+        etc.clear_analysis_config()
 
     def clearRelatedDb(self):
         edb.get_timeseries_db().delete_many({"user_id": self.testUUID})

emission/tests/netTests/TestMetricsConfirmedTrips.py

@@ -17,8 +17,7 @@
 
 class TestMetrics(unittest.TestCase):
     def setUp(self):
-        self.analysis_conf_path = \
-            etc.set_analysis_config("analysis.result.section.key", "analysis/confirmed_trip")
+        etc.set_analysis_config("analysis.result.section.key", "analysis/confirmed_trip")
         self._loadDataFileAndInputs("emission/tests/data/real_examples/shankari_2016-06-20")
         self.testUUID1 = self.testUUID
         self._loadDataFileAndInputs("emission/tests/data/real_examples/shankari_2016-06-21")
@@ -39,7 +38,7 @@ def _loadDataFileAndInputs(self, dataFile):
 
     def tearDown(self):
         self.clearRelatedDb()
-        os.remove(self.analysis_conf_path)
+        etc.clear_analysis_config()
 
     def clearRelatedDb(self):
         edb.get_timeseries_db().delete_many({"user_id": self.testUUID1})