From 26eaaea86b7665a2f6200678e4ffecd251f1ab56 Mon Sep 17 00:00:00 2001 From: Tianhao-Gu Date: Thu, 23 May 2024 19:04:49 -0500 Subject: [PATCH] disable hive concurrency support --- src/spark/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spark/utils.py b/src/spark/utils.py index c050b4e..21933f2 100644 --- a/src/spark/utils.py +++ b/src/spark/utils.py @@ -47,7 +47,8 @@ def _get_delta_lake_conf(jars_str: str) -> dict: "spark.hadoop.fs.s3a.secret.key": os.environ.get("MINIO_SECRET_KEY"), "spark.hadoop.fs.s3a.path.style.access": "true", "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem", - "spark.sql.catalogImplementation": "hive" + "spark.sql.catalogImplementation": "hive", + "spark.hadoop.hive.support.concurrency": "false" # Prevent Hive metastore locks }