WIP

Ferlab-Ste-Justine · Mar 8, 2024 · 5794f88 · 5794f88
1 parent b23ef58
commit 5794f88
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 18 deletions.
diff --git a/dags/etl_import_fhir.py b/dags/etl_import_fhir.py
@@ -1,10 +1,24 @@
 from datetime import datetime
 
 from airflow import DAG
-from airflow.models.param import Param
-
-from lib.config import fhir_import_config
-from lib.operators.fhir_import import FhirCsvOperator
+from airflow.models import Param, Variable
+
+from lib.config import fhir_url, keycloak_client_secret_name, keycloak_url, aws_secret_name, aws_secret_access_key, aws_secret_secret_key, clinical_data_bucket, file_import_bucket, kube_config
+from lib.operators.fhir_import import FhirCsvOperator, FhirCsvConfig
+
+fhir_import_config = FhirCsvConfig(
+    fhir_url=fhir_url,
+    keycloak_client_secret_name = keycloak_client_secret_name,
+    keycloak_url=keycloak_url,
+    aws_credentials_secret_name= aws_secret_name,
+    aws_credentials_secret_access_key=aws_secret_access_key,
+    aws_credentials_secret_secret_key=aws_secret_secret_key,
+    clinical_data_bucket_name=clinical_data_bucket,
+    file_import_bucket_name=file_import_bucket,
+    id_service_url=Variable.get('id_service_url'),
+    kube_config=kube_config,
+    image=Variable.get('fhir_import_image') 
+).args("bio/ferlab/cqdg/etl/FhirImport")
 
 with DAG(
     dag_id='etl_import_fhir',

diff --git a/dags/lib/config.py b/dags/lib/config.py
@@ -29,6 +29,8 @@ class Env:
 clinical_data_bucket = Variable.get('clinical_data_bucket')
 file_import_bucket = Variable.get('file_import_bucket')
 
+keycloak_client_secret_name = 'keycloak-client-system-credentials'
+
 default_params = {
     'study_id': Param('ST0000017', type='string'),
     'project': Param('cqdg', type='string'),
@@ -157,18 +159,6 @@ class Env:
 etl_variant_normalized_config = etl_variant_config \
     .with_spark_class('bio.ferlab.etl.normalized.RunNormalizedGenomic')
 
-keycloak_client_secret_name = 'keycloak-client-system-credentials'
 
-fhir_import_config = FhirCsvConfig(
-    fhir_url=fhir_url,
-    keycloak_client_secret_name = keycloak_client_secret_name,
-    keycloak_url=keycloak_url,
-    aws_credentials_secret_name= aws_secret_name,
-    aws_credentials_secret_access_key=aws_secret_access_key,
-    aws_credentials_secret_secret_key=aws_secret_secret_key,
-    clinical_data_bucket_name=clinical_data_bucket,
-    file_import_bucket_name=file_import_bucket,
-    id_service_url=Variable.get('id_service_url'),
-    kube_config=kube_config,
-    image=Variable.get('fhir_import_image') 
-).args("bio/ferlab/cqdg/etl/FhirImport")
+
+