Redesign Build Deployment Process (External)

.docker/docker_start_script.sh

@@ -1,27 +1,4 @@
 #!/usr/bin/env bash
-#Configure web server
-
-# cd /usr/src/app/e-mission-server
-
-#set database URL using environment variable
-echo ${DB_HOST}
-if [ -z ${DB_HOST} ] ; then
-    local_host=`hostname -i`
-    jq --arg db_host "$local_host" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf
-else
-    jq --arg db_host "$DB_HOST" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf
-fi
-cat conf/storage/db.conf
-
-#set Web Server host using environment variable
-echo ${WEB_SERVER_HOST}
-if [ -z ${WEB_SERVER_HOST} ] ; then
-    local_host=`hostname -i`
-    sed "s_localhost_${local_host}_" conf/net/api/webserver.conf.sample > conf/net/api/webserver.conf
-else
-    sed "s_localhost_${WEB_SERVER_HOST}_" conf/net/api/webserver.conf.sample > conf/net/api/webserver.conf
-fi
-cat conf/net/api/webserver.conf
 
 if [ -z ${LIVERELOAD_SRC} ] ; then
     echo "Live reload disabled, "

.github/workflows/image_build_push.yml

@@ -1,29 +1,22 @@
-# This is a basic workflow to help you get started with Actions
-
 name: docker image
 
-# Controls when the action will run. Triggers the workflow on push or pull request 
-# events but only for the master branch
 on:
   push:
     branches: [ master, gis-based-mode-detection ]
 
-
-# Env variable
+#Dockerhub credentials are set as environment variables
 env:
   DOCKER_USER: ${{secrets.DOCKER_USER}}
   DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
   build:
-    # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
-    # Steps represent a sequence of tasks that will be executed as part of the job
+    outputs:
+      date: ${{ steps.date.outputs.date }}
+
     steps:
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v2
     - name: docker login
       run: | # log into docker hub account
@@ -46,3 +39,40 @@ jobs:
     - name: push docker image
       run: |
         docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }}
+
+    - name: Create a text file
+      run: |
+        echo ${{ steps.date.outputs.date }} > tag_file.txt
+        echo "Created tag text file"
+
+    - name: Upload Artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: docker-image-tag
+        path: tag_file.txt
+        overwrite: true
+
+  dispatch:
+    needs: build
+    runs-on: ubuntu-latest
+
+    env:
+      DOCKER_IMAGE_TAG: ${{ needs.build.outputs.date }}
+
+    strategy:
+      matrix:
+        repo: ['e-mission/op-admin-dashboard', 'e-mission/em-public-dashboard']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Trigger workflow in admin-dash, public-dash
+      # TODO: Create Fine-grained token with "Actions: write" permissions 
+      run: |
+        curl -L \
+          -X POST \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/${{ matrix.repo }}/actions/workflows/image_build_push.yml/dispatches \
+          -d '{"ref":"master", "inputs": {"docker_image_tag" : "${{ env.DOCKER_IMAGE_TAG }}"}}'

.gitignore

@@ -16,6 +16,8 @@ CFC_DataCollector/moves_collect.log
 webapp/www/lib
 conf/**/*.json
 !conf/**/*.schema.json
+!conf/analysis/debug.conf.dev.json
+!conf/analysis/debug.conf.prod.json
 
 *.ipynb_checkpoints*
 

Dockerfile

@@ -28,8 +28,8 @@ RUN chmod u+x ./.docker/setup_config.sh
 RUN bash -c "./.docker/setup_config.sh"
 
 # #declare environment variables
-ENV DB_HOST=''
-ENV WEB_SERVER_HOST=''
+ENV DB_HOST='db'
+ENV WEB_SERVER_HOST=0.0.0.0
 
 ENV LIVERELOAD_SRC=''
 ENV STUDY_CONFIG=''

conf/analysis/debug.conf.json.sample → conf/analysis/debug.conf.dev.json

@@ -10,5 +10,5 @@
     "section.startStopRadius": 150,
     "section.endStopRadius": 150,
     "analysis.result.section.key": "analysis/inferred_section",
-    "userinput.keylist": ["manual/mode_confirm", "manual/purpose_confirm", "manual/trip_user_input", "manual/place_user_input"]
+    "userinput.keylist": ["manual/mode_confirm", "manual/purpose_confirm", "manual/replaced_mode", "manual/trip_user_input", "manual/place_user_input"]
 }

conf/analysis/debug.conf.prod.json

@@ -0,0 +1,14 @@
+{
+    "intake.segmentation.section_segmentation.sectionValidityAssertions": true,
+    "intake.cleaning.clean_and_resample.speedDistanceAssertions": false,
+    "intake.cleaning.clean_and_resample.sectionValidityAssertions": false,
+    "intake.cleaning.filter_accuracy.enable": false,
+    "classification.inference.mode.useAdvancedFeatureIndices": true,
+    "classification.inference.mode.useBusTrainFeatureIndices": true,
+    "classification.validityAssertions": true,
+    "output.conversion.validityAssertions": true,
+    "section.startStopRadius": 150,
+    "section.endStopRadius": 150,
+    "analysis.result.section.key": "analysis/inferred_section",
+    "userinput.keylist": ["manual/mode_confirm", "manual/purpose_confirm", "manual/replaced_mode", "manual/trip_user_input", "manual/place_user_input"]
+}

emission/analysis/config.py

@@ -1,11 +1,17 @@
 import json
+import os
 
 def get_config_data():
     try:
+        print("Trying to open debug.conf.json")
         config_file = open('conf/analysis/debug.conf.json')
     except:
-        print("analysis.debug.conf.json not configured, falling back to sample, default configuration")
-        config_file = open('conf/analysis/debug.conf.json.sample')
+        if os.getenv("PROD_STAGE") == "TRUE":
+            print("In production environment, opening internal debug.conf")
+            config_file = open('conf/analysis/debug.conf.prod.json')
+        else:
+            print("analysis.debug.conf.json not configured, falling back to sample, default configuration")
+            config_file = open('conf/analysis/debug.conf.dev.json')
     ret_val = json.load(config_file)
     config_file.close()
     return ret_val

emission/core/config.py

@@ -0,0 +1,32 @@
+import json
+import logging
+import os
+
+def get_config_data_from_env():
+    config_data_env = {
+        "timeseries": {
+            "url": os.getenv('DB_HOST', "localhost"),
+            "result_limit": os.getenv('DB_TS_RESULT_LIMIT', 250000)
+        }
+    }
+    return config_data_env
+
+def get_config_data():
+    try:
+        config_file = open('conf/storage/db.conf')
+        ret_val = json.load(config_file)
+        config_file.close()
+    except:
+        ret_val = get_config_data_from_env()
+        if ret_val["timeseries"]["url"] == "localhost":
+            print("storage not configured, falling back to sample, default configuration")
+    return ret_val
+
+config_data = get_config_data()
+
+def get_config():
+    return config_data
+
+def reload_config():
+    global config_data
+    config_data = get_config_data()

emission/core/get_database.py

@@ -10,16 +10,10 @@
 import os
 import json
 
-try:
-    config_file = open('conf/storage/db.conf')
-except:
-    print("storage not configured, falling back to sample, default configuration")
-    config_file = open('conf/storage/db.conf.sample')
+import emission.core.config as ecc
 
-config_data = json.load(config_file)
-url = config_data["timeseries"]["url"]
-result_limit = config_data["timeseries"]["result_limit"]
-config_file.close()
+url = ecc.get_config()["timeseries"]["url"]
+result_limit = ecc.get_config()["timeseries"]["result_limit"]
 
 try:
     parsed=pymongo.uri_parser.parse_uri(url)

emission/integrationTests/start_integration_tests.sh

@@ -2,15 +2,7 @@
 # Using an automated install 
 cd /src/e-mission-server
 
-#set database URL using environment variable
 echo ${DB_HOST}
-if [ -z ${DB_HOST} ] ; then
-    local_host=`hostname -i`
-    sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
-else
-    sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf
-fi
-cat conf/storage/db.conf
 
 echo "Setting up conda..."
 source setup/setup_conda.sh Linux-x86_64
@@ -25,4 +17,4 @@ echo "Adding permissions for the runIntegrationTests.sh script"
 chmod +x runIntegrationTests.sh
 echo "Permissions added for the runIntegrationTests.sh script"
 
-./runIntegrationTests.sh
+./runIntegrationTests.sh

emission/integrationTests/storageTests/TestMongodbAuth.py

@@ -47,10 +47,19 @@ def setUp(self):
         self.uuid = uuid.uuid4()
         self.testUserId = self.uuid
         self.db_conf_file = "conf/storage/db.conf"
+        self.originalDBEnvVars = {}
         self.createAdmin()
 
     def tearDown(self):
         self.admin_auth.command({"dropAllUsersFromDatabase": 1})
+        logging.debug("Deleting test db environment variables")
+        for env_var_name, env_var_value in self.testModifiedEnvVars.items():
+            del os.environ[env_var_name]
+		# Restoring original db environment variables
+        for env_var_name, env_var_value in self.originalDBEnvVars.items():
+            os.environ[env_var_name] = env_var_value
+        logging.debug("Finished restoring original db environment variables")
+        logging.debug("Restored original values are = %s" % self.originalDBEnvVars)
         try:
             os.remove(self.db_conf_file)
         except FileNotFoundError as e:
@@ -67,14 +76,20 @@ def createAdmin(self):
         self.admin_auth = pymongo.MongoClient(self.getURL(self.test_username, self.test_password)).admin
 
     def configureDB(self, url):
-        config = {
-            "timeseries": {
-                "url": url,
-                "result_limit": 250000
-            }
+        self.testModifiedEnvVars = {
+            'DB_HOST' : url
         }
-        with open(self.db_conf_file, "w") as fp:
-            json.dump(config, fp, indent=4)
+
+        for env_var_name, env_var_value in self.testModifiedEnvVars.items():
+            if os.getenv(env_var_name) is not None:
+                # Storing original db environment variables before modification
+                self.originalDBEnvVars[env_var_name] = os.getenv(env_var_name)
+            # Setting db environment variables with test values
+            os.environ[env_var_name] = env_var_value
+
+        logging.debug("Finished setting up test db environment variables")
+        logging.debug("Current original values are = %s" % self.originalDBEnvVars)
+        logging.debug("Current modified values are = %s" % self.testModifiedEnvVars)
 
     def getURL(self, username, password, dbname="admin"):
         return "mongodb://%s:%s@localhost/%s?authSource=admin&authMechanism=SCRAM-SHA-1" % (username, password, dbname)

emission/net/api/cfc_webapp.py

@@ -51,27 +51,17 @@
 import emission.storage.timeseries.cache_series as esdc
 import emission.core.timer as ect
 import emission.core.get_database as edb
+import emission.net.api.config as enac
 
-try:
-    config_file = open('conf/net/api/webserver.conf')
-except:
-    logging.debug("webserver not configured, falling back to sample, default configuration")
-    config_file = open('conf/net/api/webserver.conf.sample')
-
-OPENPATH_URL="https://www.nrel.gov/transportation/openpath.html"
 STUDY_CONFIG = os.getenv('STUDY_CONFIG', "stage-program")
-
-config_data = json.load(config_file)
-config_file.close()
-static_path = config_data["paths"]["static_path"]
-python_path = config_data["paths"]["python_path"]
-server_host = config_data["server"]["host"]
-server_port = config_data["server"]["port"]
-socket_timeout = config_data["server"]["timeout"]
-log_base_dir = config_data["paths"]["log_base_dir"]
-auth_method = config_data["server"]["auth"]
-aggregate_call_auth = config_data["server"]["aggregate_call_auth"]
-not_found_redirect = config_data["paths"].get("404_redirect", OPENPATH_URL)
+enac.reload_config()
+static_path = enac.get_config()["static_path"]
+server_host = enac.get_config()["server_host"]
+server_port = enac.get_config()["server_port"]
+socket_timeout = enac.get_config()["socket_timeout"]
+auth_method = enac.get_config()["auth_method"]
+aggregate_call_auth = enac.get_config()["aggregate_call_auth"]
+not_found_redirect = enac.get_config()["not_found_redirect"]
 
 BaseRequest.MEMFILE_MAX = 1024 * 1024 * 1024 # Allow the request size to be 1G
 # to accomodate large section sizes

emission/net/api/config.py

@@ -0,0 +1,36 @@
+import json
+import logging
+import os
+
+def get_config_data_from_env():
+    config_data_env = {
+        "static_path": os.getenv('WEB_SERVER_STATIC_PATH', "webapp/www/"),
+        "server_host": os.getenv('WEB_SERVER_HOST', "0.0.0.0"),
+        "server_port": os.getenv('WEB_SERVER_PORT', "8080"),
+        "socket_timeout": os.getenv('WEB_SERVER_TIMEOUT', "3600"),
+        "auth_method": os.getenv('WEB_SERVER_AUTH', "skip"),
+        "aggregate_call_auth": os.getenv('WEB_SERVER_AGGREGATE_CALL_AUTH', "no_auth"),
+        "not_found_redirect": os.getenv('WEB_SERVER_REDIRECT_URL', "https://www.nrel.gov/transportation/openpath.html")
+    }
+    return config_data_env
+
+
+def get_config_data():
+    try:
+        config_file = open('conf/net/api/webserver.conf')
+        ret_val = json.load(config_file)
+        config_file.close()
+    except:
+        # if check_unset_env_vars():
+        logging.debug("webserver not configured, falling back to sample, default configuration")
+        ret_val = get_config_data_from_env()
+    return ret_val
+
+config_data = get_config_data()
+
+def get_config():
+    return config_data
+
+def reload_config():
+    global config_data
+    config_data = get_config_data()

emission/net/auth/secret.py

@@ -4,7 +4,11 @@
 
 class SecretMethod(object):
     def __init__(self):
-        key_file = open('conf/net/auth/secret_list.json')
+        try:
+            key_file = open('conf/net/auth/secret_list.json')
+        except:
+            print("secret_list.json not configured, falling back to sample, default configuration")
+            key_file = open('conf/net/auth/secret_list.json.sample')
         key_data = json.load(key_file)
         key_file.close()
         self.client_secret_list = key_data["client_secret_list"]