From 353fbef6b7e851ba2aad48e35e9ca9c2990ea6c3 Mon Sep 17 00:00:00 2001 From: jecos Date: Mon, 11 Mar 2024 13:49:34 -0400 Subject: [PATCH] WIP --- dags/etl_normalize_variants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dags/etl_normalize_variants.py b/dags/etl_normalize_variants.py index f976871..c0dc3d6 100644 --- a/dags/etl_normalize_variants.py +++ b/dags/etl_normalize_variants.py @@ -45,6 +45,7 @@ .with_spark_jar('http://repository-gateway/jars/variant-task.jar') \ .add_spark_conf({ # 'spark.jars.packages': 'io.delta:delta-core_2.12:2.3.0', + 'spark.jars.packages': 'org.apache.hadoop:hadoop-aws:3.3.2', # 'spark.jars.packages': 'io.delta:delta-core_2.12:2.1.1,io.projectglow:glow-spark3_2.12:1.2.1', # 'spark.jars.excludes':'org.apache.hadoop:hadoop-client', # 'spark.kubernetes.container.image': 'ferlabcrsj/spark:469f2fc61f06fcfa73c1480a5e5f73f59768152d',