From 8157b87386e1460af61302260b033091b2c1f750 Mon Sep 17 00:00:00 2001 From: DonHaul Date: Tue, 27 Aug 2024 10:46:12 +0200 Subject: [PATCH] backoffice: fix restart with params * ref: cern-sis/issues-inspire/issues/545 --- .../backoffice/workflows/airflow_utils.py | 10 +- ...kflowViewSet.test_restart_full_dagrun.yaml | 107 ++++++++++-------- .../backoffice/workflows/tests/test_views.py | 10 +- 3 files changed, 70 insertions(+), 57 deletions(-) diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py index 5f88b9e7..4dbf8053 100644 --- a/backoffice/backoffice/workflows/airflow_utils.py +++ b/backoffice/backoffice/workflows/airflow_utils.py @@ -156,6 +156,7 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): :param params: parameters of new dag execution :returns: request response """ + delete_workflow_dag_runs(workflow_id, workflow_type) return trigger_airflow_dag( @@ -163,7 +164,7 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): ) -def delete_workflow_dag_runs(workflow_id, workflow_type): +def delete_workflow_dag_runs(workflow_id, workflow_type, params=None): """Deletes runs of a given workflow. :param workflow_id: workflow_id for dags that should be restarted @@ -171,8 +172,13 @@ def delete_workflow_dag_runs(workflow_id, workflow_type): """ executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type) - for dag_id in executed_dags_for_workflow: + data = None + + for dag_id, dag_data in executed_dags_for_workflow.items(): delete_workflow_dag(dag_id, str(workflow_id)) + # fetch data from previous drag run + if data is None: + data = dag_data["conf"].get("data") return JsonResponse( data={"message": f"Dag runs for worfklow {workflow_id} have been deleted"} diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml index 9cfc7cdc..009d219c 100644 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml @@ -1,7 +1,7 @@ interactions: - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' + "00000000-0000-0000-0000-000000000000", "data": {"test": "test"}}}' headers: Accept: - '*/*' @@ -10,30 +10,33 @@ interactions: Connection: - keep-alive Content-Length: - - '119' + - '145' Content-Type: - application/json method: POST uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:28.743346+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:28.743346+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:42.301248+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:42.301248+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -50,29 +53,30 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:28.743346+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:28.743346+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:42.301248+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:42.301248+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -89,16 +93,16 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.9.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: @@ -106,7 +110,7 @@ interactions: Content-Type: - application/problem+json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -123,16 +127,16 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.9.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: @@ -140,7 +144,7 @@ interactions: Content-Type: - application/problem+json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -159,20 +163,20 @@ interactions: - keep-alive Content-Length: - '0' - Content-Type: - - application/json method: DELETE uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: '' headers: + Cache-Control: + - no-store Connection: - close Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:49 GMT Server: - gunicorn X-Robots-Tag: @@ -182,7 +186,7 @@ interactions: message: NO CONTENT - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' + "00000000-0000-0000-0000-000000000000", "data": {"test": "test"}}}' headers: Accept: - '*/*' @@ -191,30 +195,33 @@ interactions: Connection: - keep-alive Content-Length: - - '119' + - '145' Content-Type: - application/json method: POST uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:29.037550+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:29.037550+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:29.037550+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:29.037550+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:49.946419+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:49.946419+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:49.946419+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:49.946419+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:29 GMT + - Wed, 21 Aug 2024 13:20:49 GMT Server: - gunicorn X-Robots-Tag: @@ -233,20 +240,20 @@ interactions: - keep-alive Content-Length: - '0' - Content-Type: - - application/json method: DELETE uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: '' headers: + Cache-Control: + - no-store Connection: - close Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:29 GMT + - Wed, 21 Aug 2024 13:20:56 GMT Server: - gunicorn X-Robots-Tag: diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index a9df1599..d3d7c979 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -325,7 +325,7 @@ def setUp(self): super().setUp() self.workflow = Workflow.objects.create( - data={}, + data={"test": "test"}, status="running", core=True, is_update=False, @@ -419,8 +419,8 @@ def test_restart_full_dagrun(self): kwargs={"pk": self.workflow.id}, ) response = self.api_client.post(url) - self.assertEqual(response.status_code, 200) + self.assertIn("test", response.json()["conf"]["data"]) @pytest.mark.vcr() def test_restart_a_task(self): @@ -429,8 +429,9 @@ def test_restart_a_task(self): "api:workflows-authors-restart", kwargs={"pk": self.workflow.id}, ) + response = self.api_client.post( - url, json={"task_ids": ["set_workflow_status_to_running"]} + url, format="json", data={"restart_current_task": True} ) self.assertEqual(response.status_code, 200) @@ -441,9 +442,8 @@ def test_restart_with_params(self): "api:workflows-authors-restart", kwargs={"pk": self.workflow.id}, ) - response = self.api_client.post( - url, json={"params": {"workflow_id": self.workflow.id}} + url, format="json", data={"params": {"workflow_id": self.workflow.id}} ) self.assertEqual(response.status_code, 200)