[FEA] Add third-party library integration testing of cudf.pandas to cudf

.github/workflows/nightly.yaml

@@ -0,0 +1,44 @@
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+name: cudf-pandas-integration test on default branch (nightly / manually)
+
+on:
+  workflow_dispatch:
+    # The below exists in alignment with rest of RAPIDS nightly pipeline. They are currently unused.
+    inputs:
+      branch:
+        required: true
+        type: string
+      date:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Extract libraries from dependencies.yaml
+        id: extractlib
+        run: |
+          LIBS=$(python/cudf/cudf_pandas_tests/third_party_integration_tests/ci/extract_lib.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml)
+          echo "LIBS=${LIBS}" >> $GITHUB_ENV
+
+      - name: Run integration tests
+        run: |
+          for lib in ${{ env.LIBS }}; do
+            echo "Running tests for $lib"
+            CUDA_MAJOR=$(if [ "$lib" = "tensorflow" ]; then echo "11"; else echo "12"; fi)
+            python/cudf/cudf_pandas_tests/third_party_integration_tests/ci/test.sh $lib $CUDA_MAJOR
+          done
+        env:
+          LIBS: ${{ env.LIBS }}
+    secrets: inherit

python/cudf/cudf_pandas_tests/third_party_integration_tests/ci/ci_run_library_tests.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+cleanup() {
+    rm tests/results-*.pickle
+}
+
+trap cleanup EXIT
+
+runtest_gold() {
+    local lib=$1
+    local test_keys=${@:2}
+
+    pytest \
+    -v \
+    --continue-on-collection-errors \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-${lib}-gold.xml" \
+    --numprocesses=${NUM_PROCESSES} \
+    --dist=worksteal \
+    ${TEST_DIR}/test_${lib}*.py \
+    ${test_keys}
+}
+
+runtest_cudf_pandas() {
+    local lib=$1
+    local test_keys=${@:2}
+
+    pytest \
+    -p cudf.pandas \
+    -v \
+    --continue-on-collection-errors \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-${lib}-cudf-pandas.xml" \
+    --numprocesses=${NUM_PROCESSES} \
+    --dist=worksteal \
+    ${TEST_DIR}/test_${lib}*.py \
+    ${test_keys}
+}
+
+main() {
+    local lib=$1
+    local test_keys=${@:2}
+
+    # generation phase
+    runtest_gold ${lib} ${test_keys}
+    runtest_cudf_pandas ${lib} ${test_keys}
+
+    # assertion phase
+    pytest \
+    --compare \
+    -p cudf.pandas \
+    -v \
+    --continue-on-collection-errors \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-${lib}-assertion.xml" \
+    --numprocesses=${NUM_PROCESSES} \
+    --dist=worksteal \
+    ${TEST_DIR}/test_${lib}*.py \
+    ${test_keys}
+}
+
+main $@

python/cudf/cudf_pandas_tests/third_party_integration_tests/ci/extract_lib.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+write_output() {
+  local key="$1"
+  local value="$2"
+  echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}"
+}
+
+extract_lib_from_dependencies_yaml() {
+    local file=$1
+    # Parse all keys in dependencies.yaml under the "files" section,
+    # extract all the keys that starts with "test_", and extract the
+    # rest
+    local extracted_libs="$(yq -o json $file | jq -rc '.files | with_entries( select(.key | contains("test_")) ) | keys | map(sub("^test_"; ""))')"
+    echo $extracted_libs
+    write_output "LIBS" $extracted_libs
+}
+
+
+main() {
+    local dependencies_yaml="$1"
+    extract_lib_from_dependencies_yaml "$dependencies_yaml"
+}
+
+main "$@"

python/cudf/cudf_pandas_tests/third_party_integration_tests/ci/test.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+# Common setup steps shared by Python test jobs
+
+LIB=$1
+
+set -euo pipefail
+
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-logger "Generate Python testing dependencies"
+rapids-dependency-file-generator \
+  --output conda \
+  --file-key test_${LIB} \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --yes -f env.yaml -n test
+
+# Temporarily allow unbound variables for conda activation.
+set +u
+conda activate test
+set -u
+
+RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
+mkdir -p "${RAPIDS_TESTS_DIR}"
+
+repo_root=$(git rev-parse --show-toplevel)
+TEST_DIR=${repo_root}/tests
+
+rapids-print-env
+
+rapids-logger "Check GPU usage"
+nvidia-smi
+
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
+rapids-logger "pytest ${LIB}"
+
+NUM_PROCESSES=8
+serial_libraries=(
+    "tensorflow"
+)
+for serial_library in "${serial_libraries[@]}"; do
+    if [ "${LIB}" = "${serial_library}" ]; then
+        NUM_PROCESSES=1
+    fi
+done
+
+RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR} TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/ci_run_library_tests.sh ${LIB}
+
+rapids-logger "Test script exiting with value: ${EXITCODE}"
+exit ${EXITCODE}