Fix Selective checks for Task SDK (apache#43185)

Task SDK tests were running as part of "DB tests" with other Core files & providers, this PR changes it so they run separately.
It also adds separate mypy checks for it.

This commit fixes several other issues to allow running Task SDK  tests separately.

.github/workflows/ci.yml

@@ -99,7 +99,7 @@ jobs:
  ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
  prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
  docs-build: ${{ steps.selective-checks.outputs.docs-build }}
- mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }}
+ mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
  needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
  needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
  needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
@@ -298,7 +298,7 @@ jobs:
  runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }}
  image-tag: ${{ needs.build-info.outputs.image-tag }}
  needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
- mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
+ mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
  python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
  branch: ${{ needs.build-info.outputs.default-branch }}
  canary-run: ${{ needs.build-info.outputs.canary-run }}
@@ -644,6 +644,24 @@ jobs:
  ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
  needs.build-info.outputs.needs-helm-tests == 'true')
 
+ tests-task-sdk:
+ name: "Task SDK tests"
+ uses: ./.github/workflows/task-sdk-tests.yml
+ needs: [build-info, wait-for-ci-images]
+ permissions:
+ contents: read
+ packages: read
+ secrets: inherit
+ with:
+ runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }}
+ if: >
+ ( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
+ needs.build-info.outputs.run-tests == 'true')
+
  finalize-tests:
  name: Finalize tests
  permissions:

.github/workflows/static-checks-mypy-docs.yml

@@ -36,7 +36,7 @@ on: # yamllint disable-line rule:truthy
  description: "Whether to run mypy checks (true/false)"
  required: true
  type: string
- mypy-folders:
+ mypy-checks:
  description: "List of folders to run mypy checks on"
  required: false
  type: string
@@ -148,7 +148,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- mypy-folder: ${{ fromJSON(inputs.mypy-folders) }}
+ mypy-check: ${{ fromJSON(inputs.mypy-checks) }}
  env:
  PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
  IMAGE_TAG: "${{ inputs.image-tag }}"
@@ -166,10 +166,10 @@ jobs:
  - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
  uses: ./.github/actions/prepare_breeze_and_image
  id: breeze
- - name: "MyPy checks for ${{ matrix.mypy-folder }}"
+ - name: "MyPy checks for ${{ matrix.mypy-check }}"
  run: |
  pip install pre-commit
- pre-commit run --color always --verbose --hook-stage manual mypy-${{matrix.mypy-folder}} --all-files
+ pre-commit run --color always --verbose --hook-stage manual ${{matrix.mypy-check}} --all-files
  env:
  VERBOSE: "false"
  COLUMNS: "250"

.github/workflows/task-sdk-tests.yml

@@ -28,10 +28,6 @@ on: # yamllint disable-line rule:truthy
  description: "Tag to set for the image"
  required: true
  type: string
- canary-run:
- description: "Whether this is a canary run"
- required: true
- type: string
  default-python-version:
  description: "Which version of python should be used by default"
  required: true
@@ -40,6 +36,10 @@ on: # yamllint disable-line rule:truthy
  description: "JSON-formatted array of Python versions to build images from"
  required: true
  type: string
+ run-task-sdk-tests:
+ description: "Whether to run Task SDK tests or not (true/false)"
+ required: true
+ type: string
 jobs:
  task-sdk-tests:
  timeout-minutes: 80
@@ -58,7 +58,6 @@ jobs:
  PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
  VERBOSE: "true"
  CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
- if: inputs.run-task-sdk-tests == 'true'
  steps:
  - name: "Cleanup repo"
  shell: bash
@@ -81,11 +80,7 @@ jobs:
  pipx uninstall twine || true
  pipx install twine && twine check dist/*.whl
  - name: >
- Run provider unit tests on
- Airflow Task SDK:Python ${{ matrix.python-version }}
- if: matrix.run-tests == 'true'
+ Run unit tests for Airflow Task SDK:Python ${{ matrix.python-version }}
  run: >
  breeze testing tests --run-in-parallel
  --parallel-test-types TaskSDK
- --use-packages-from-dist
- --package-format wheel

.pre-commit-config.yaml

@@ -1291,6 +1291,7 @@ repos:
  ^.*/.*_vendor/ |
  ^airflow/migrations |
  ^providers/ |
+ ^task_sdk/ |
  ^dev |
  ^scripts |
  ^docs |
@@ -1343,6 +1344,23 @@ repos:
  files: ^.*\.py$
  require_serial: true
  additional_dependencies: ['rich>=12.4.4']
+ - id: mypy-task-sdk
+ name: Run mypy for Task SDK
+ language: python
+ entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages
+ files: ^task_sdk/src/airflow/sdk/.*\.py$|^task_sdk/tests//.*\.py$
+ exclude: ^.*/.*_vendor/
+ require_serial: true
+ additional_dependencies: ['rich>=12.4.4']
+ - id: mypy-task-sdk
+ stages: ['manual']
+ name: Run mypy for Task SDK (manual)
+ language: python
+ entry: ./scripts/ci/pre_commit/mypy_folder.py task_sdk/src/airflow/sdk
+ pass_filenames: false
+ files: ^.*\.py$
+ require_serial: true
+ additional_dependencies: ['rich>=12.4.4']
  - id: check-provider-yaml-valid
  name: Validate provider.yaml files
  entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py

Dockerfile.ci

@@ -1385,6 +1385,7 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
 COPY pyproject.toml ${AIRFLOW_SOURCES}/pyproject.toml
 COPY providers/pyproject.toml ${AIRFLOW_SOURCES}/providers/pyproject.toml
 COPY task_sdk/pyproject.toml ${AIRFLOW_SOURCES}/task_sdk/pyproject.toml
+COPY task_sdk/README.md ${AIRFLOW_SOURCES}/task_sdk/README.md
 COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/
 COPY tests_common/ ${AIRFLOW_SOURCES}/tests_common/
 COPY generated/* ${AIRFLOW_SOURCES}/generated/

contributing-docs/08_static_code_checks.rst

@@ -325,6 +325,9 @@ require Breeze Docker image to be built locally.
 | mypy-providers | * Run mypy for providers | * |
 | | * Run mypy for providers (manual) | |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
+| mypy-task-sdk | * Run mypy for Task SDK | * |
+| | * Run mypy for Task SDK (manual) | |
++-----------------------------------------------------------+--------------------------------------------------------+---------+
 | pretty-format-json | Format JSON files | |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | pylint | pylint | |

dev/breeze/doc/ci/04_selective_checks.md

@@ -203,7 +203,7 @@ Github Actions to pass the list of parameters to a command to execute
 | kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.9-v1.25.2 3.9-v1.26.4 | * |
 | kubernetes-versions | All Kubernetes versions to use for tests as JSON array | ['v1.25.2'] | |
 | kubernetes-versions-list-as-string | All Kubernetes versions to use for tests as space-separated string | v1.25.2 | * |
-| mypy-folders | List of folders to be considered for mypy | [] | |
+| mypy-checks | List of folders to be considered for mypy | [] | |
 | mysql-exclude | Which versions of MySQL to exclude for tests as JSON array | [] | |
 | mysql-versions | Which versions of MySQL to use for tests as JSON array | ['5.7'] | |
 | needs-api-codegen | Whether "api-codegen" are needed to run ("true"/"false") | true | |

dev/breeze/doc/images/output_shell.txt

@@ -1 +1 @@
-7b382a009f0280b761743a0746739b80
+fd70e0f17940f32fbc0579e8f77fc6c4