WIP

Ferlab-Ste-Justine · Mar 10, 2024 · 10941dc · 10941dc
1 parent 46dc71b
commit 10941dc
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/dags/etl_normalize_variants.py b/dags/etl_normalize_variants.py
@@ -15,10 +15,10 @@
                 '--study-id', study_id,
                 '--study-code', '{{ params.study_code }}' 
 ) \
-.with_spark_jar('s3a://cqdg-qa-app-datalake/jars/variant-task.jar') \
+.with_spark_jar('https://github.com/Ferlab-Ste-Justine/etl-cqdg-portal/releases/download/v2.21.5/variant-task.jar') \
 .add_spark_conf({
-    'spark.jars.packages': 'io.delta:delta-core_2.12:2.3.0,io.projectglow:glow-spark3_2.12:1.2.1',
-    'spark.jars.excludes':'org.apache.hadoop:hadoop-client',
+    # 'spark.jars.packages': 'io.delta:delta-core_2.12:2.3.0,io.projectglow:glow-spark3_2.12:1.2.1',
+    # 'spark.jars.excludes':'org.apache.hadoop:hadoop-client',
     'spark.kubernetes.container.image': 'ferlabcrsj/spark:469f2fc61f06fcfa73c1480a5e5f73f59768152d',
     # 'spark.kubernetes.file.upload.path': f's3a://{datalake_bucket}/dependencies'    
 })