chore: upgrade the spark version in the integration to 3.2.1

dbt-labs · May 7, 2022 · 0b30719 · 0b30719
1 parent 7b92fab
commit 0b30719
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 10 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,19 +8,20 @@ jobs:
       - image: fishtownanalytics/test-container:10
     steps:
       - checkout
-      - run: tox -e flake8,unit
+      - run: tox -e unit
 
   integration-spark-session:
     environment:
       DBT_INVOCATION_ENV: circle
     docker:
-      - image: godatadriven/pyspark:3.1
+      - image: bitnami/spark:3.2.1
+        user: root
     steps:
       - checkout
       - run: apt-get update
-      - run: python3 -m pip install --upgrade pip
       - run: apt-get install -y git gcc g++ unixodbc-dev libsasl2-dev
-      - run: python3 -m pip install tox
+      - run: python3 -m pip install --upgrade pip
+      - run: python3 -m pip install flake8,tox,pyspark==3.2.1
       - run:
           name: Run integration tests
           command: tox -e integration-spark-session
@@ -33,7 +34,7 @@ jobs:
       DBT_INVOCATION_ENV: circle
     docker:
       - image: fishtownanalytics/test-container:10
-      - image: godatadriven/spark:2
+      - image: godatadriven/spark:3.1
         environment:
           WAIT_FOR: localhost:5432
         command: >
@@ -44,7 +45,7 @@ jobs:
           --conf spark.hadoop.javax.jdo.option.ConnectionPassword=dbt
           --conf spark.hadoop.javax.jdo.option.ConnectionDriverName=org.postgresql.Driver
           --conf spark.serializer=org.apache.spark.serializer.KryoSerializer
-          --conf spark.jars.packages=org.apache.hudi:hudi-spark-bundle_2.11:0.9.0
+          --conf spark.jars.packages=org.apache.hudi:hudi-spark3.1-bundle_2.12:0.11.0
           --conf spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension
           --conf spark.driver.userClassPathFirst=true
           --conf spark.hadoop.datanucleus.autoCreateTables=true

diff --git a/dbt/adapters/spark/impl.py b/dbt/adapters/spark/impl.py
@@ -1,3 +1,4 @@
+import json
 import re
 from concurrent.futures import Future
 from dataclasses import dataclass
@@ -149,8 +150,6 @@ def list_relations_without_caching(
         relations = []
         view_names = views.columns["viewName"].values()
 
-        raise Exception(f"tbl: {tables.print_json()}")
-
         for tbl in tables:
             rel_type = RelationType('view' if tbl['tableName'] in view_names else 'table')
             _schema = tbl['namespace'] if 'namespace' in tbl else tbl['database']

diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql
@@ -179,15 +179,13 @@
   {% call statement('list_tables_without_caching', fetch_result=True) -%}
     show tables in {{ relation.schema }}
   {% endcall %}
-
   {% do return(load_result('list_tables_without_caching').table) %}
 {% endmacro %}
 
 {% macro spark__list_views_without_caching(relation) %}
   {% call statement('list_views_without_caching', fetch_result=True) -%}
     show views in {{ relation.schema }}
   {% endcall %}
-
   {% do return(load_result('list_views_without_caching').table) %}
 {% endmacro %}