diff --git a/dags/etl_normalize_variants.py b/dags/etl_normalize_variants.py index 48bd90e..8b87961 100644 --- a/dags/etl_normalize_variants.py +++ b/dags/etl_normalize_variants.py @@ -15,10 +15,10 @@ '--study-id', study_id, '--study-code', '{{ params.study_code }}' ) \ -.with_spark_jar('s3a://cqdg-qa-app-datalake/jars/variant-task.jar') \ +.with_spark_jar('https://github.com/Ferlab-Ste-Justine/etl-cqdg-portal/releases/download/v2.21.5/variant-task.jar') \ .add_spark_conf({ - 'spark.jars.packages': 'io.delta:delta-core_2.12:2.3.0,io.projectglow:glow-spark3_2.12:1.2.1', - 'spark.jars.excludes':'org.apache.hadoop:hadoop-client', + # 'spark.jars.packages': 'io.delta:delta-core_2.12:2.3.0,io.projectglow:glow-spark3_2.12:1.2.1', + # 'spark.jars.excludes':'org.apache.hadoop:hadoop-client', 'spark.kubernetes.container.image': 'ferlabcrsj/spark:469f2fc61f06fcfa73c1480a5e5f73f59768152d', # 'spark.kubernetes.file.upload.path': f's3a://{datalake_bucket}/dependencies' })