diff --git a/src/spark/utils.py b/src/spark/utils.py index c050b4e..21933f2 100644 --- a/src/spark/utils.py +++ b/src/spark/utils.py @@ -47,7 +47,8 @@ def _get_delta_lake_conf(jars_str: str) -> dict: "spark.hadoop.fs.s3a.secret.key": os.environ.get("MINIO_SECRET_KEY"), "spark.hadoop.fs.s3a.path.style.access": "true", "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem", - "spark.sql.catalogImplementation": "hive" + "spark.sql.catalogImplementation": "hive", + "spark.hadoop.hive.support.concurrency": "false" # Prevent Hive metastore locks }