WIP

Ferlab-Ste-Justine · Mar 11, 2024 · 353fbef · 353fbef
1 parent 360b559
commit 353fbef
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/dags/etl_normalize_variants.py b/dags/etl_normalize_variants.py
@@ -45,6 +45,7 @@
 .with_spark_jar('http://repository-gateway/jars/variant-task.jar') \
 .add_spark_conf({
     # 'spark.jars.packages': 'io.delta:delta-core_2.12:2.3.0',
+    'spark.jars.packages': 'org.apache.hadoop:hadoop-aws:3.3.2',
     # 'spark.jars.packages': 'io.delta:delta-core_2.12:2.1.1,io.projectglow:glow-spark3_2.12:1.2.1',
     # 'spark.jars.excludes':'org.apache.hadoop:hadoop-client',
     # 'spark.kubernetes.container.image': 'ferlabcrsj/spark:469f2fc61f06fcfa73c1480a5e5f73f59768152d',