cleaning & batch settings update (#20)

surfriderfoundationeurope · Jun 7, 2022 · 9df05af · 9df05af
1 parent b25a0d2
commit 9df05af
Show file tree

Hide file tree

Showing 39 changed files with 5 additions and 2,545 deletions.
diff --git a/src/batch/airflow/dag_trigger_batch_etl.py b/src/batch/airflow/dag_trigger_batch_etl.py
diff --git a/src/batch/airflow/dag_trigger_batch_etl_all.py b/src/batch/airflow/dag_trigger_batch_etl_all.py
@@ -42,10 +42,10 @@
         AKS_RG = os.getenv('AKS_RG')
         AKS_CLUSTER = os.getenv('AKS_CLUSTER')
         AKS_NODEPOOL = os.getenv('AKS_NODEPOOL')
-        AI_DEV_URL = os.getenv('AI_DEV_URL')
+        AI_URL = os.getenv('AI_URL')
         VIDEO_TEST_NAME = os.getenv('VIDEO_TEST_NAME')
         VIDEO_TEST_URL = os.getenv('VIDEO_TEST_URL')
-        ETL_DEV_URL = os.getenv('ETL_DEV_URL')
+        ETL_URL = os.getenv('ETL_URL')
 
         # PG connection
         pg_conn_string = get_pg_connection_string()
@@ -99,7 +99,7 @@ def wait_surfnet():
             ai_ready = False
             while (ai_ready == False):
                 try:
-                    response = requests.get(AI_DEV_URL+':8000')
+                    response = requests.get(AI_URL+':8000')
                     output = [response.status_code][0]
                     if output == 200:
                         ai_ready = True
@@ -134,7 +134,7 @@ def post_video_surfnet(video_name,ai_url):
             output = [response._content]
             return str(output)
 
-        post_surfnet_video_op = post_video_surfnet(VIDEO_TEST_NAME,AI_DEV_URL)
+        post_surfnet_video_op = post_video_surfnet(VIDEO_TEST_NAME,AI_URL)
         # [END post_surfnet_video]
 
         # [START trigger_batch_etl]
@@ -162,7 +162,7 @@ def trigger_batch_etl(cursor):
                 blob_name = notprocessed_media['name']
                 container = notprocessed_media['container']
                 logid = notprocessed_media['logid']
-                url = f'{ETL_DEV_URL}:80/api/etlHttpTrigger?container={container}&blob={blob_name}&prediction=ai&source={container}&target=postgre&aiurl={AI_DEV_URL}&logid={logid}'
+                url = f'{ETL_URL}:80/api/etlHttpTrigger?container={container}&blob={blob_name}&prediction=ai&source={container}&target=postgre&aiurl={AI_URL}&logid={logid}'
                 response = requests.get(url)
                 if not response.ok:
                     print(f'Request to ETL failed wih reason {response.reason}.')

diff --git a/src/batch/cronjob/hello.yaml b/src/batch/cronjob/hello.yaml
diff --git a/src/realtime/etlAPIs/.gitignore → src/realtime/etlAPI/.gitignore b/src/realtime/etlAPIs/.gitignore → src/realtime/etlAPI/.gitignore
diff --git a/src/realtime/etlAPIsVM/Dockerfile → src/realtime/etlAPI/Dockerfile b/src/realtime/etlAPIsVM/Dockerfile → src/realtime/etlAPI/Dockerfile
diff --git a/...ealtime/etlAPIsVM/etlHttpTrigger/Makefile → src/realtime/etlAPI/etlHttpTrigger/Makefile b/...ealtime/etlAPIsVM/etlHttpTrigger/Makefile → src/realtime/etlAPI/etlHttpTrigger/Makefile
diff --git a/...time/etlAPIsVM/etlHttpTrigger/__init__.py → ...ealtime/etlAPI/etlHttpTrigger/__init__.py b/...time/etlAPIsVM/etlHttpTrigger/__init__.py → ...ealtime/etlAPI/etlHttpTrigger/__init__.py
diff --git a/...me/etlAPIsVM/etlHttpTrigger/function.json → ...ltime/etlAPI/etlHttpTrigger/function.json b/...me/etlAPIsVM/etlHttpTrigger/function.json → ...ltime/etlAPI/etlHttpTrigger/function.json
diff --git a/...me/etlAPIs/etlHttpTrigger/prediction.json → ...ime/etlAPI/etlHttpTrigger/prediction.json b/...me/etlAPIs/etlHttpTrigger/prediction.json → ...ime/etlAPI/etlHttpTrigger/prediction.json
diff --git a/...time/etlAPIsVM/etlHttpTrigger/utils/ai.py → ...ealtime/etlAPI/etlHttpTrigger/utils/ai.py b/...time/etlAPIsVM/etlHttpTrigger/utils/ai.py → ...ealtime/etlAPI/etlHttpTrigger/utils/ai.py
diff --git a/...time/etlAPIs/etlHttpTrigger/utils/blob.py → ...ltime/etlAPI/etlHttpTrigger/utils/blob.py b/...time/etlAPIs/etlHttpTrigger/utils/blob.py → ...ltime/etlAPI/etlHttpTrigger/utils/blob.py
diff --git a/...tlAPIs/etlHttpTrigger/utils/exceptions.py → ...etlAPI/etlHttpTrigger/utils/exceptions.py b/...tlAPIs/etlHttpTrigger/utils/exceptions.py → ...etlAPI/etlHttpTrigger/utils/exceptions.py
diff --git a/...ime/etlAPIsVM/etlHttpTrigger/utils/gps.py → ...altime/etlAPI/etlHttpTrigger/utils/gps.py b/...ime/etlAPIsVM/etlHttpTrigger/utils/gps.py → ...altime/etlAPI/etlHttpTrigger/utils/gps.py
diff --git a/...ime/etlAPIs/etlHttpTrigger/utils/media.py → ...time/etlAPI/etlHttpTrigger/utils/media.py b/...ime/etlAPIs/etlHttpTrigger/utils/media.py → ...time/etlAPI/etlHttpTrigger/utils/media.py
diff --git a/...etlAPIsVM/etlHttpTrigger/utils/postgre.py → ...me/etlAPI/etlHttpTrigger/utils/postgre.py b/...etlAPIsVM/etlHttpTrigger/utils/postgre.py → ...me/etlAPI/etlHttpTrigger/utils/postgre.py
diff --git a/src/realtime/etlAPIsVM/host.json → src/realtime/etlAPI/host.json b/src/realtime/etlAPIsVM/host.json → src/realtime/etlAPI/host.json
diff --git a/src/realtime/etlAPIsVM/requirements.txt → src/realtime/etlAPI/requirements.txt b/src/realtime/etlAPIsVM/requirements.txt → src/realtime/etlAPI/requirements.txt
diff --git a/src/realtime/etlAPIs/Dockerfile b/src/realtime/etlAPIs/Dockerfile