From d6ac7959614429db8f616c425416cc1bb3a53e9a Mon Sep 17 00:00:00 2001 From: Diego Quintana Date: Mon, 6 Nov 2023 13:12:02 +0100 Subject: [PATCH] change: update docker connection id and image registry at dags (#17) --- dags/calcul_potencia_optima_dag.py | 6 ++---- dags/erppeek_get_pilotatge_kpis_dag.py | 4 ++-- dags/hs_get_conversations_dag.py | 4 ++-- dags/hs_get_tags_dag.py | 6 +++--- dags/hs_mailbox_reports_dag.py | 4 ++-- dags/hs_transform_conversations_dag.py | 4 ++-- dags/meff_update_closing_prices_dag.py | 12 ++++++------ dags/odoo_get_hr_employee_dag.py | 4 ++-- dags/omie_price_hour_dag.py | 4 ++-- dags/tomatic_calls_stats_dag.py | 4 ++-- 10 files changed, 25 insertions(+), 27 deletions(-) diff --git a/dags/calcul_potencia_optima_dag.py b/dags/calcul_potencia_optima_dag.py index a8a1861..202a714 100644 --- a/dags/calcul_potencia_optima_dag.py +++ b/dags/calcul_potencia_optima_dag.py @@ -47,13 +47,12 @@ def get_random_moll() -> str: sampled_moll = get_random_moll() - get_conversations_task = DockerOperator( api_version="auto", task_id="calcul_potencia_optima", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.erp.calcul_potencia_optima --dbapi "{{ var.value.erp_prod_sp2_uri_ro }}" --dbapi-dades "{{ var.value.dades_prod_db }}"', @@ -64,4 +63,3 @@ def get_random_moll() -> str: retrieve_output=True, trigger_rule="none_failed", ) - diff --git a/dags/erppeek_get_pilotatge_kpis_dag.py b/dags/erppeek_get_pilotatge_kpis_dag.py index acc4ac3..c274903 100644 --- a/dags/erppeek_get_pilotatge_kpis_dag.py +++ b/dags/erppeek_get_pilotatge_kpis_dag.py @@ -51,9 +51,9 @@ def get_random_moll() -> str: get_conversations_task = DockerOperator( api_version="auto", task_id="erppeek_get_pilotatge_kpis_daily", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.erppeek.filtered_models_single_kpis "{{ var.value.dades_prod_db }}" "daily" \ diff --git a/dags/hs_get_conversations_dag.py b/dags/hs_get_conversations_dag.py index 9f6b3c3..2beea67 100644 --- a/dags/hs_get_conversations_dag.py +++ b/dags/hs_get_conversations_dag.py @@ -51,9 +51,9 @@ def get_random_moll() -> str: get_conversations_task = DockerOperator( api_version="auto", task_id="hs_get_conversations", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.helpscout.hs_get_conversations "{{ data_interval_start }}" "{{ data_interval_end }}" \ diff --git a/dags/hs_get_tags_dag.py b/dags/hs_get_tags_dag.py index 2ed4f44..9bd9db7 100644 --- a/dags/hs_get_tags_dag.py +++ b/dags/hs_get_tags_dag.py @@ -22,7 +22,7 @@ def get_random_moll() -> str: "retries": 5, "retry_delay": timedelta(minutes=5), } - + nfs_config = { "type": "nfs", "o": f"addr={addr},nfsvers=4", @@ -51,9 +51,9 @@ def get_random_moll() -> str: get_tags_task = DockerOperator( api_version="auto", task_id="hs_get_tags", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.helpscout.hs_get_tags "{{ data_interval_start }}" "{{ data_interval_end }}" \ diff --git a/dags/hs_mailbox_reports_dag.py b/dags/hs_mailbox_reports_dag.py index 5b29a80..73bc6d9 100644 --- a/dags/hs_mailbox_reports_dag.py +++ b/dags/hs_mailbox_reports_dag.py @@ -50,9 +50,9 @@ def get_random_moll() -> str: get_conversations_task = DockerOperator( api_version="auto", task_id="hs_mailbox_reports", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.helpscout.hs_mailbox_reports update-hs-kpis-pilotatge "{{ data_interval_start }}" "{{ data_interval_end }}" \ diff --git a/dags/hs_transform_conversations_dag.py b/dags/hs_transform_conversations_dag.py index f79f4ee..64d503d 100644 --- a/dags/hs_transform_conversations_dag.py +++ b/dags/hs_transform_conversations_dag.py @@ -59,9 +59,9 @@ def get_random_moll() -> str: transform_conversations_task = DockerOperator( api_version="auto", task_id="hs_transform_conversations", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m pipelines.hs_transform_conversations "{{ data_interval_start }}" "{{ data_interval_end }}" \ diff --git a/dags/meff_update_closing_prices_dag.py b/dags/meff_update_closing_prices_dag.py index 32771ec..f97622d 100644 --- a/dags/meff_update_closing_prices_dag.py +++ b/dags/meff_update_closing_prices_dag.py @@ -51,9 +51,9 @@ def get_random_moll() -> str: meff_update_closing_prices_task = DockerOperator( api_version="auto", task_id="meff_update_closing_prices", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.meff.meff_update_closing_prices "{{ var.value.puppis_prod_db}}"', @@ -83,9 +83,9 @@ def get_random_moll() -> str: meff_slice_day_closing_prices_task = DockerOperator( api_version="auto", task_id="meff_slice_day_closing_prices", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m pipelines.meff_closing_prices_day_slice "{{ var.value.puppis_prod_db}}"', @@ -100,9 +100,9 @@ def get_random_moll() -> str: meff_slice_month_closing_prices_task = DockerOperator( api_version="auto", task_id="meff_slice_month_closing_prices", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m pipelines.meff_closing_prices_month_slice "{{ var.value.puppis_prod_db}}"', diff --git a/dags/odoo_get_hr_employee_dag.py b/dags/odoo_get_hr_employee_dag.py index 8688673..6117a78 100644 --- a/dags/odoo_get_hr_employee_dag.py +++ b/dags/odoo_get_hr_employee_dag.py @@ -50,9 +50,9 @@ def get_random_moll() -> str: get_hr_employee_task = DockerOperator( api_version="auto", task_id="odoo_get_hr_employee", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.odoo.hr_employees "{{ var.value.odoo_dbapi}}" "{{ var.value.puppis_prod_db}}" "{{ dag_run.start_date }}"', diff --git a/dags/omie_price_hour_dag.py b/dags/omie_price_hour_dag.py index 5086c6d..6e80170 100644 --- a/dags/omie_price_hour_dag.py +++ b/dags/omie_price_hour_dag.py @@ -50,9 +50,9 @@ def get_random_moll() -> str: get_hr_employee_task = DockerOperator( api_version="auto", task_id="omie_get_price_hour", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.omie.omie_update_last_hour_price "{{ var.value.puppis_prod_db}}"', diff --git a/dags/tomatic_calls_stats_dag.py b/dags/tomatic_calls_stats_dag.py index 7e64954..e33ca42 100644 --- a/dags/tomatic_calls_stats_dag.py +++ b/dags/tomatic_calls_stats_dag.py @@ -51,9 +51,9 @@ def get_random_moll() -> str: tomatic_calls_stats_task = DockerOperator( api_version="auto", task_id="tomatic_calls_stats", - docker_conn_id="somenergia_registry", + docker_conn_id="somenergia_harbor_dades_registry", image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m datasources.tomatic.tomatic_stats "{{ var.value.dades_prod_db }}" prod "{{ var.value.tomatera_user }}" \