workflows: fix restart workflows issue

* ref: cern-sis/issues-inspire/issues/647
DonHaul · Dec 20, 2024 · b8598e0 · b8598e0
1 parent 9e78629
commit b8598e0
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 85 deletions.
diff --git a/backoffice/backoffice/authors/airflow_utils.py b/backoffice/backoffice/authors/airflow_utils.py
@@ -159,15 +159,18 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None):
 
     :param workflow_id: workflow_id  for dags that should be restarted
     :param workflow_type: type of workflow the will be restarted
-    :param params: parameters of new dag execution
+    :param params: parameters of new dag execution, if not provided will be fetched from the workflow
     :returns: request response
     """
+    conf = params if params else fetch_conf_workflow_dag(workflow_id, workflow_type)
 
-    data = fetch_data_workflow_dag(workflow_id, workflow_type)
     delete_workflow_dag_runs(workflow_id, workflow_type)
 
     return trigger_airflow_dag(
-        WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params or data
+        WORKFLOW_DAGS[workflow_type].initialize,
+        str(workflow_id),
+        workflow=conf.get("workflow"),
+        extra_data=conf.get("data"),
     )
 
 
@@ -183,7 +186,7 @@ def delete_workflow_dag_runs(workflow_id, workflow_type):
         delete_workflow_dag(dag_id, str(workflow_id))
 
 
-def fetch_data_workflow_dag(workflow_id, workflow_type):
+def fetch_conf_workflow_dag(workflow_id, workflow_type):
     """Fetches Data that the workflow ran with
 
     :param workflow_id: workflow_id for dag to get data of
@@ -194,4 +197,4 @@ def fetch_data_workflow_dag(workflow_id, workflow_type):
     executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type)
 
     _, dag = next(iter(executed_dags_for_workflow.items()))
-    return dag["conf"].get("data")
+    return dag["conf"]
diff --git a/...owUtils.test_fetch_data_workflow_dag.yaml → ...owUtils.test_fetch_conf_workflow_dag.yaml b/...owUtils.test_fetch_data_workflow_dag.yaml → ...owUtils.test_fetch_conf_workflow_dag.yaml
@@ -1,7 +1,8 @@
 interactions:
 - request:
     body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id":
-      "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}}}'
+      "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}, "workflow":
+      {"id": "id"}}}'
     headers:
       Accept:
       - '*/*'
@@ -10,33 +11,33 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '145'
+      - '171'
       Content-Type:
       - application/json
     method: POST
     uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns
   response:
     body:
       string: "{\n  \"conf\": {\n    \"data\": {\n      \"test\": \"test\"\n    },\n
-        \   \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n  },\n  \"dag_id\":
-        \"author_create_initialization_dag\",\n  \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n
-        \ \"data_interval_end\": \"2024-10-18T11:54:25.361392+00:00\",\n  \"data_interval_start\":
-        \"2024-10-18T11:54:25.361392+00:00\",\n  \"end_date\": null,\n  \"execution_date\":
-        \"2024-10-18T11:54:25.361392+00:00\",\n  \"external_trigger\": true,\n  \"last_scheduling_decision\":
-        null,\n  \"logical_date\": \"2024-10-18T11:54:25.361392+00:00\",\n  \"note\":
-        null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n  \"state\":
-        \"queued\"\n}\n"
+        \   \"workflow\": {\n      \"id\": \"id\"\n    },\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
+        \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-12-20T11:02:49.477410+00:00\",\n
+        \ \"data_interval_start\": \"2024-12-20T11:02:49.477410+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-12-20T11:02:49.477410+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-12-20T11:02:49.477410+00:00\",\n
+        \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
+        \ \"state\": \"queued\"\n}\n"
     headers:
       Cache-Control:
       - no-store
       Connection:
       - close
       Content-Length:
-      - '621'
+      - '663'
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:25 GMT
+      - Fri, 20 Dec 2024 11:02:49 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -58,25 +59,25 @@ interactions:
   response:
     body:
       string: "{\n  \"conf\": {\n    \"data\": {\n      \"test\": \"test\"\n    },\n
-        \   \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n  },\n  \"dag_id\":
-        \"author_create_initialization_dag\",\n  \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n
-        \ \"data_interval_end\": \"2024-10-18T11:54:25.361392+00:00\",\n  \"data_interval_start\":
-        \"2024-10-18T11:54:25.361392+00:00\",\n  \"end_date\": null,\n  \"execution_date\":
-        \"2024-10-18T11:54:25.361392+00:00\",\n  \"external_trigger\": true,\n  \"last_scheduling_decision\":
-        null,\n  \"logical_date\": \"2024-10-18T11:54:25.361392+00:00\",\n  \"note\":
-        null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n  \"state\":
-        \"queued\"\n}\n"
+        \   \"workflow\": {\n      \"id\": \"id\"\n    },\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
+        \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-12-20T11:02:49.477410+00:00\",\n
+        \ \"data_interval_start\": \"2024-12-20T11:02:49.477410+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-12-20T11:02:49.477410+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-12-20T11:02:49.477410+00:00\",\n
+        \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
+        \ \"state\": \"queued\"\n}\n"
     headers:
       Cache-Control:
       - no-store
       Connection:
       - close
       Content-Length:
-      - '621'
+      - '663'
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:25 GMT
+      - Fri, 20 Dec 2024 11:02:49 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -110,7 +111,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Fri, 18 Oct 2024 11:54:25 GMT
+      - Fri, 20 Dec 2024 11:02:49 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -144,7 +145,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Fri, 18 Oct 2024 11:54:25 GMT
+      - Fri, 20 Dec 2024 11:02:49 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -176,7 +177,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:25 GMT
+      - Fri, 20 Dec 2024 11:02:49 GMT
       Server:
       - gunicorn
       X-Robots-Tag:

diff --git a/...ffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml b/...ffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml
@@ -1,7 +1,8 @@
 interactions:
 - request:
     body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id":
-      "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}}}'
+      "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}, "workflow":
+      {"id": "id"}}}'
     headers:
       Accept:
       - '*/*'
@@ -10,33 +11,33 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '145'
+      - '171'
       Content-Type:
       - application/json
     method: POST
     uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns
   response:
     body:
       string: "{\n  \"conf\": {\n    \"data\": {\n      \"test\": \"test\"\n    },\n
-        \   \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n  },\n  \"dag_id\":
-        \"author_create_initialization_dag\",\n  \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n
-        \ \"data_interval_end\": \"2024-10-18T11:54:49.928543+00:00\",\n  \"data_interval_start\":
-        \"2024-10-18T11:54:49.928543+00:00\",\n  \"end_date\": null,\n  \"execution_date\":
-        \"2024-10-18T11:54:49.928543+00:00\",\n  \"external_trigger\": true,\n  \"last_scheduling_decision\":
-        null,\n  \"logical_date\": \"2024-10-18T11:54:49.928543+00:00\",\n  \"note\":
-        null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n  \"state\":
-        \"queued\"\n}\n"
+        \   \"workflow\": {\n      \"id\": \"id\"\n    },\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
+        \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-12-20T10:59:09.785487+00:00\",\n
+        \ \"data_interval_start\": \"2024-12-20T10:59:09.785487+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-12-20T10:59:09.785487+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-12-20T10:59:09.785487+00:00\",\n
+        \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
+        \ \"state\": \"queued\"\n}\n"
     headers:
       Cache-Control:
       - no-store
       Connection:
       - close
       Content-Length:
-      - '621'
+      - '663'
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:49 GMT
+      - Fri, 20 Dec 2024 10:59:09 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -58,25 +59,25 @@ interactions:
   response:
     body:
       string: "{\n  \"conf\": {\n    \"data\": {\n      \"test\": \"test\"\n    },\n
-        \   \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n  },\n  \"dag_id\":
-        \"author_create_initialization_dag\",\n  \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n
-        \ \"data_interval_end\": \"2024-10-18T11:54:49.928543+00:00\",\n  \"data_interval_start\":
-        \"2024-10-18T11:54:49.928543+00:00\",\n  \"end_date\": null,\n  \"execution_date\":
-        \"2024-10-18T11:54:49.928543+00:00\",\n  \"external_trigger\": true,\n  \"last_scheduling_decision\":
-        null,\n  \"logical_date\": \"2024-10-18T11:54:49.928543+00:00\",\n  \"note\":
-        null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n  \"state\":
-        \"queued\"\n}\n"
+        \   \"workflow\": {\n      \"id\": \"id\"\n    },\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
+        \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-12-20T10:59:09.785487+00:00\",\n
+        \ \"data_interval_start\": \"2024-12-20T10:59:09.785487+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-12-20T10:59:09.785487+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-12-20T10:59:09.785487+00:00\",\n
+        \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
+        \ \"state\": \"queued\"\n}\n"
     headers:
       Cache-Control:
       - no-store
       Connection:
       - close
       Content-Length:
-      - '621'
+      - '663'
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:09 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -110,7 +111,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:09 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -144,7 +145,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:09 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -166,25 +167,25 @@ interactions:
   response:
     body:
       string: "{\n  \"conf\": {\n    \"data\": {\n      \"test\": \"test\"\n    },\n
-        \   \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n  },\n  \"dag_id\":
-        \"author_create_initialization_dag\",\n  \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n
-        \ \"data_interval_end\": \"2024-10-18T11:54:49.928543+00:00\",\n  \"data_interval_start\":
-        \"2024-10-18T11:54:49.928543+00:00\",\n  \"end_date\": null,\n  \"execution_date\":
-        \"2024-10-18T11:54:49.928543+00:00\",\n  \"external_trigger\": true,\n  \"last_scheduling_decision\":
-        null,\n  \"logical_date\": \"2024-10-18T11:54:49.928543+00:00\",\n  \"note\":
-        null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n  \"state\":
-        \"queued\"\n}\n"
+        \   \"workflow\": {\n      \"id\": \"id\"\n    },\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
+        \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-12-20T10:59:09.785487+00:00\",\n
+        \ \"data_interval_start\": \"2024-12-20T10:59:09.785487+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-12-20T10:59:09.785487+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-12-20T10:59:09.785487+00:00\",\n
+        \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
+        \ \"state\": \"queued\"\n}\n"
     headers:
       Cache-Control:
       - no-store
       Connection:
       - close
       Content-Length:
-      - '621'
+      - '663'
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:10 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -218,7 +219,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:10 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -252,7 +253,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:10 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -284,7 +285,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:10 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -294,7 +295,8 @@ interactions:
       message: NO CONTENT
 - request:
     body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id":
-      "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}}}'
+      "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}, "workflow":
+      {"id": "id"}}}'
     headers:
       Accept:
       - '*/*'
@@ -303,33 +305,33 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '145'
+      - '171'
       Content-Type:
       - application/json
     method: POST
     uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns
   response:
     body:
       string: "{\n  \"conf\": {\n    \"data\": {\n      \"test\": \"test\"\n    },\n
-        \   \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n  },\n  \"dag_id\":
-        \"author_create_initialization_dag\",\n  \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n
-        \ \"data_interval_end\": \"2024-10-18T11:54:50.810267+00:00\",\n  \"data_interval_start\":
-        \"2024-10-18T11:54:50.810267+00:00\",\n  \"end_date\": null,\n  \"execution_date\":
-        \"2024-10-18T11:54:50.810267+00:00\",\n  \"external_trigger\": true,\n  \"last_scheduling_decision\":
-        null,\n  \"logical_date\": \"2024-10-18T11:54:50.810267+00:00\",\n  \"note\":
-        null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n  \"state\":
-        \"queued\"\n}\n"
+        \   \"workflow\": {\n      \"id\": \"id\"\n    },\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
+        \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-12-20T10:59:10.350095+00:00\",\n
+        \ \"data_interval_start\": \"2024-12-20T10:59:10.350095+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-12-20T10:59:10.350095+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-12-20T10:59:10.350095+00:00\",\n
+        \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
+        \ \"state\": \"queued\"\n}\n"
     headers:
       Cache-Control:
       - no-store
       Connection:
       - close
       Content-Length:
-      - '621'
+      - '663'
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:10 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -361,7 +363,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 18 Oct 2024 11:54:50 GMT
+      - Fri, 20 Dec 2024 10:59:10 GMT
       Server:
       - gunicorn
       X-Robots-Tag: