python 3.11 support (#263)

* adding 3.11 to actions * bumping spark versions * excluding spark matrix * updating extras * adding back exclude * adding in python 3.12 * adding spark support matrix, remove 3.12 * adding spark support matrix, remove 3.12
capitalone · Feb 2, 2024 · e105d84 · e105d84
1 parent 5d422c0
commit e105d84
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 7 deletions.
diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml
@@ -19,8 +19,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10']
-        spark-version: [3.1.3, 3.2.3, 3.3.1, 3.4.0, 3.5.0]
+        python-version: [3.8, 3.9, '3.10', '3.11']
+        spark-version: [3.1.3, 3.2.4, 3.3.4, 3.4.2, 3.5.0]
+        exclude:
+          - python-version: '3.11'
+            spark-version: 3.1.3
+          - python-version: '3.11'
+            spark-version: 3.2.4
+          - python-version: '3.11'
+            spark-version: 3.3.4
     env:
       PYTHON_VERSION: ${{ matrix.python-version }}
       SPARK_VERSION: ${{ matrix.spark-version }}
@@ -55,7 +62,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10']
+        python-version: [3.8, 3.9, '3.10', '3.11']
     env:
       PYTHON_VERSION: ${{ matrix.python-version }}
 
@@ -81,7 +88,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10']
+        python-version: [3.8, 3.9, '3.10', '3.11']
     env:
       PYTHON_VERSION: ${{ matrix.python-version }}
 

diff --git a/README.md b/README.md
@@ -38,6 +38,22 @@ pip install datacompy[ray]
 
 ```
 
+### In-scope Spark versions
+Different versions of Spark play nicely with only certain versions of Python below is a matrix of what we test with
+
+|             | Spark 3.1.3  | Spark 3.2.3 | Spark 3.3.4 | Spark 3.4.4 | Spark 3.5.0 |
+|-------------|--------------|-------------|-------------|-------------|-------------|
+| Python 3.8  | ✅           | ✅           | ✅           | ✅          | ✅          |
+| Python 3.9  | ✅           | ✅           | ✅           | ✅          | ✅          |
+| Python 3.10 | ✅           | ✅           | ✅           | ✅          | ✅          |
+| Python 3.11 | ❌           | ❌           | ❌           | ✅          | ✅          |
+| Python 3.12 | ❌           | ❌           | ❌           | ❌          | ❌          |
+
+
+:::{note}
+At the current time Python ``3.12`` is not supported by Spark and also Ray within Fugue.
+:::
+
 ## Supported backends
 
 - Pandas: ([See documentation](https://capitalone.github.io/datacompy/pandas_usage.html))

diff --git a/pyproject.toml b/pyproject.toml
@@ -28,6 +28,7 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
 ]
 
 dynamic = ["version"]
@@ -59,8 +60,11 @@ python-tag = "py3"
 
 [project.optional-dependencies]
 duckdb = ["fugue[duckdb]"]
-polars = ["fugue[polars]"]
-spark = ["fugue[spark]"]
+polars = ["polars"]
+spark = [
+    "pyspark>=3.1.1; python_version < '3.11'",
+    "pyspark>=3.4; python_version >= '3.11'",
+]
 dask = ["fugue[dask]"]
 ray = ["fugue[ray]"]
 docs = [
@@ -77,7 +81,7 @@ tests-spark = [
     "pytest",
     "pytest-cov",
     "pytest-spark",
-    "fugue[spark]",
+    "spark",
 ]
 qa = [
     "pre-commit",