diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index b5ec2ea2..a8390f0e 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -19,8 +19,15 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, '3.10'] - spark-version: [3.1.3, 3.2.3, 3.3.1, 3.4.0, 3.5.0] + python-version: [3.8, 3.9, '3.10', '3.11'] + spark-version: [3.1.3, 3.2.4, 3.3.4, 3.4.2, 3.5.0] + exclude: + - python-version: '3.11' + spark-version: 3.1.3 + - python-version: '3.11' + spark-version: 3.2.4 + - python-version: '3.11' + spark-version: 3.3.4 env: PYTHON_VERSION: ${{ matrix.python-version }} SPARK_VERSION: ${{ matrix.spark-version }} @@ -55,7 +62,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, '3.10'] + python-version: [3.8, 3.9, '3.10', '3.11'] env: PYTHON_VERSION: ${{ matrix.python-version }} @@ -81,7 +88,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, '3.10'] + python-version: [3.8, 3.9, '3.10', '3.11'] env: PYTHON_VERSION: ${{ matrix.python-version }} diff --git a/README.md b/README.md index d1be5efa..4dd0c869 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,22 @@ pip install datacompy[ray] ``` +### In-scope Spark versions +Different versions of Spark play nicely with only certain versions of Python below is a matrix of what we test with + +| | Spark 3.1.3 | Spark 3.2.3 | Spark 3.3.4 | Spark 3.4.4 | Spark 3.5.0 | +|-------------|--------------|-------------|-------------|-------------|-------------| +| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.9 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.10 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.11 | ❌ | ❌ | ❌ | ✅ | ✅ | +| Python 3.12 | ❌ | ❌ | ❌ | ❌ | ❌ | + + +:::{note} +At the current time Python ``3.12`` is not supported by Spark and also Ray within Fugue. +::: + ## Supported backends - Pandas: ([See documentation](https://capitalone.github.io/datacompy/pandas_usage.html)) diff --git a/pyproject.toml b/pyproject.toml index 21bcefd1..43b27c63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] dynamic = ["version"] @@ -59,8 +60,11 @@ python-tag = "py3" [project.optional-dependencies] duckdb = ["fugue[duckdb]"] -polars = ["fugue[polars]"] -spark = ["fugue[spark]"] +polars = ["polars"] +spark = [ + "pyspark>=3.1.1; python_version < '3.11'", + "pyspark>=3.4; python_version >= '3.11'", +] dask = ["fugue[dask]"] ray = ["fugue[ray]"] docs = [ @@ -77,7 +81,7 @@ tests-spark = [ "pytest", "pytest-cov", "pytest-spark", - "fugue[spark]", + "spark", ] qa = [ "pre-commit",