Skip to content

Commit

Permalink
author update: airflow dag fixes
Browse files Browse the repository at this point in the history 
* ref: cern-sis/issues-inspire/issues/517

destroy draft

destroy draft
  • Loading branch information
@DonHaul
DonHaul committed Aug 19, 2024
1 parent ed845d4 commit 6bebc16
Show file tree
Hide file tree
Showing 8 changed files with 648 additions and 12 deletions.
17 changes: 15 additions & 2 deletions backoffice/backoffice/workflows/airflow_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,11 +156,24 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None):
:param params: parameters of new dag execution
:returns: request response
"""
delete_workflow_dag_runs(workflow_id, workflow_type)

return trigger_airflow_dag(
WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params
)


def delete_workflow_dag_runs(workflow_id, workflow_type):
"""Deletes runs of a given workflow.
:param workflow_id: workflow_id for dags that should be restarted
:param workflow_type: type of workflow the will be restarted
"""
executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type)

for dag_id in executed_dags_for_workflow:
delete_workflow_dag(dag_id, str(workflow_id))

return trigger_airflow_dag(
WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params
return JsonResponse(
data={"message": f"Dag runs for worfklow {workflow_id} have been deleted"}
)
8 changes: 6 additions & 2 deletions backoffice/backoffice/workflows/api/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ def get_queryset(self):
return self.queryset.filter(status__status=status)
return self.queryset

def perform_destroy(self, instance):
airflow_utils.delete_workflow_dag_runs(instance.id, instance.workflow_type)
super().perform_destroy(instance)


class WorkflowTicketViewSet(viewsets.ViewSet):
def retrieve(self, request, *args, **kwargs):
Expand Down Expand Up @@ -104,8 +108,8 @@ class AuthorWorkflowViewSet(viewsets.ViewSet):
serializer_class = WorkflowAuthorSerializer

@extend_schema(
summary="Create a New Author",
description="Creates a new author, launches the required airflow dags.",
summary="Create/Update an Author",
description="Creates/Updates an author, launches the required airflow dags.",
request=serializer_class,
)
def create(self, request):
Expand Down
209 changes: 209 additions & 0 deletions .../backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
interactions:
- request:
body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id":
"00000000-0000-0000-0000-000000000001"}}'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '119'
Content-Type:
- application/json
method: POST
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns
response:
body:
string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
\ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\":
\"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-08-19T11:39:24.438383+00:00\",\n
\ \"data_interval_start\": \"2024-08-19T11:39:24.438383+00:00\",\n \"end_date\":
null,\n \"execution_date\": \"2024-08-19T11:39:24.438383+00:00\",\n \"external_trigger\":
true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T11:39:24.438383+00:00\",\n
\ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n
\ \"state\": \"queued\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '579'
Content-Type:
- application/json
Date:
- Mon, 19 Aug 2024 11:39:24 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 200
message: OK
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
method: GET
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001
response:
body:
string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
\ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\":
\"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-08-19T11:39:24.438383+00:00\",\n
\ \"data_interval_start\": \"2024-08-19T11:39:24.438383+00:00\",\n \"end_date\":
null,\n \"execution_date\": \"2024-08-19T11:39:24.438383+00:00\",\n \"external_trigger\":
true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T11:39:24.438383+00:00\",\n
\ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n
\ \"state\": \"queued\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '579'
Content-Type:
- application/json
Date:
- Mon, 19 Aug 2024 11:39:24 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 200
message: OK
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
method: GET
uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000001
response:
body:
string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag'
and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\":
404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '294'
Content-Type:
- application/problem+json
Date:
- Mon, 19 Aug 2024 11:39:24 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 404
message: NOT FOUND
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
method: GET
uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000001
response:
body:
string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag'
and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\":
404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '294'
Content-Type:
- application/problem+json
Date:
- Mon, 19 Aug 2024 11:39:24 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 404
message: NOT FOUND
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '0'
method: DELETE
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001
response:
body:
string: ''
headers:
Connection:
- close
Content-Type:
- application/json
Date:
- Mon, 19 Aug 2024 11:39:24 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 204
message: NO CONTENT
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '0'
method: DELETE
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001
response:
body:
string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag'
and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\":
404,\n \"title\": \"Not Found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '293'
Content-Type:
- application/problem+json
Date:
- Mon, 19 Aug 2024 11:39:24 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 404
message: NOT FOUND
version: 1
Loading

0 comments on commit 6bebc16

Please sign in to comment.