Adding support for Spark 3.5 (awslabs#210)

* Adding support for Spark 3.5

Skipping Spark 3.4 for now, due to dependency issue with Breeze. Will revisit in future PR.

* Added spark 3.5 to workflow

.github/workflows/base.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        PYSPARK_VERSION: ["3.1.3", "3.2", "3.3"]
+        PYSPARK_VERSION: ["3.1.3", "3.2", "3.3", "3.5"]
 
     steps:
       - uses: actions/checkout@v3

Dockerfile

@@ -22,8 +22,8 @@ COPY poetry.lock /python-deequ
 WORKDIR python-deequ
 
 RUN poetry install -vvv
-RUN poetry add pyspark==3.3 -vvv
+RUN poetry add pyspark==3.5.0 -vvv
 
-ENV SPARK_VERSION=3.3
+ENV SPARK_VERSION=3.5
 COPY . /python-deequ
 CMD poetry run python -m pytest -s tests

pydeequ/configs.py

@@ -5,6 +5,7 @@
 
 
 SPARK_TO_DEEQU_COORD_MAPPING = {
+    "3.5": "com.amazon.deequ:deequ:2.0.7-spark-3.5",
     "3.3": "com.amazon.deequ:deequ:2.0.7-spark-3.3",
     "3.2": "com.amazon.deequ:deequ:2.0.7-spark-3.2",
     "3.1": "com.amazon.deequ:deequ:2.0.7-spark-3.1"

pyproject.toml

@@ -28,7 +28,7 @@ classifiers = [
 
 
 [tool.poetry.dependencies]
-python = ">=3.7,<4"
+python = ">=3.8,<4"
 numpy = ">=1.14.1"
 pandas = ">=0.23.0"
 pyspark = { version = ">=2.4.7, <3.3.0", optional = true }