Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
jecos committed Mar 8, 2024
1 parent 8da149d commit 2e965ce
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions dags/etl_normalize_variants.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@
'--study-id', study_id,
'--study-code', '{{ params.study_code }}'
) \
.with_spark_jar('file:///app/variant-task.jar') \
.add_spark_conf({
'spark.jars.packages': 'org.apache.hadoop:hadoop-aws:3.3.2,io.delta:delta-core_2.12:2.3.0,io.projectglow:glow-spark3_2.12:1.2.1',
'spark.kubernetes.container.image': 'apache/spark:3.3.3'
})
'spark.kubernetes.container.image': 'apache/spark:3.3.3',
'spark.jars.excludes': 'org.apache.httpcomponents:httpcore,org.apache.httpcomponents:httpclientorg.apache.hadoop:hadoop-client'
})
def normalize_variant_operator(name):
return normalized_etl.prepend_args(name).operator(
task_id=f'normalize-{name}',
Expand Down

0 comments on commit 2e965ce

Please sign in to comment.