trying bashoperator

lsst-uk · Jun 27, 2024 · 97aa3d7 · 97aa3d7
1 parent 74423b3
commit 97aa3d7
Showing 1 changed file with 6 additions and 7 deletions.
diff --git a/echo-side/dags/monitor.py b/echo-side/dags/monitor.py
@@ -1,5 +1,6 @@
 from airflow import DAG
-from airflow.operators.docker_operator import DockerOperator
+# from airflow.operators.docker_operator import DockerOperator
+from airflow.operators.bash_operator import BashOperator
 from airflow.utils.dates import days_ago
 from airflow.models import Variable
 from datetime import timedelta
@@ -21,16 +22,14 @@
     catchup=False,
 )
 
-# DockerOperator to run the container
-list_csv_files = DockerOperator(
+# BashOperator to run the script
+list_csv_files = BashOperator(
     task_id='list_csv_files',
-    image='ghcr.io/lsst-uk/csd3-echo-somerville',
-    command='python csd3-echo-somerville/scripts/list_backup_csvs.py --bucket_name LSST-IR-FUSION-Butlers',
-    environment={
+    bash_command='python csd3-echo-somerville/scripts/list_backup_csvs.py --bucket_name LSST-IR-FUSION-Butlers',
+    env={
         'ECHO_S3_ACCESS_KEY': Variable.get("ECHO_S3_ACCESS_KEY"),
         'ECHO_S3_SECRET_KEY': Variable.get("ECHO_S3_SECRET_KEY"),
     },
-    auto_remove=True,
     dag=dag,
 )