fixt: CQDG-642 Upgrade delta lake for spark 3.5.1

Ferlab-Ste-Justine · Mar 14, 2024 · bd96586 · bd96586
1 parent e0cf13b
commit bd96586
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/dags/lib/config.py b/dags/lib/config.py
@@ -141,7 +141,7 @@ class Env:
     spark_configs=[spark_default_conf],
     image=Variable.get('etl_image'),
     kube_config=kube_config
-).add_packages('org.apache.hadoop:hadoop-aws:3.3.4', 'io.delta:delta-core_2.12:2.4.0')
+).add_packages('org.apache.hadoop:hadoop-aws:3.3.4', 'io.delta:delta-spark_2.12:3.1.0')
 
 etl_index_config = etl_base_config \
     .add_spark_conf(spark_small_conf, spark_index_conf) \