From 259a5b9e009d6075ba19a313e84fb76378583811 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 3 Sep 2024 14:19:16 -0700 Subject: [PATCH 1/3] improvement/cudf-pandas-integration-tests --- .github/workflows/pr.yaml | 12 ++++++++++ ..._library_tests.sh => run-library-tests.sh} | 24 +++++++------------ .../third-party-integration/test.sh | 2 +- 3 files changed, 22 insertions(+), 16 deletions(-) rename ci/cudf_pandas_scripts/third-party-integration/{ci_run_library_tests.sh => run-library-tests.sh} (69%) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8730804e8b6..d4d5e3a940e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -36,6 +36,7 @@ jobs: - unit-tests-cudf-pandas - pandas-tests - pandas-tests-diff + - third-party-integration-tests-cudf-pandas secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 if: always() @@ -304,3 +305,14 @@ jobs: node_type: cpu4 build_type: pull-request run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh" + third-party-integration-tests-cudf-pandas: + needs: wheel-build-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: | + ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml \ No newline at end of file diff --git a/ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh b/ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh similarity index 69% rename from ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh rename to ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh index 54a56508cdc..d44d25d658c 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh @@ -9,23 +9,17 @@ cleanup() { trap cleanup EXIT -runtest_gold() { +runtest() { local lib=$1 + local mode=$2 - pytest \ - -v \ - --continue-on-collection-errors \ - --cache-clear \ - --numprocesses=${NUM_PROCESSES} \ - --dist=worksteal \ - ${TEST_DIR}/test_${lib}*.py -} - -runtest_cudf_pandas() { - local lib=$1 + local plugin="" + if [ "$mode" = "cudf" ]; then + plugin="-p cudf.pandas" + fi pytest \ - -p cudf.pandas \ + $plugin \ -v \ --continue-on-collection-errors \ --cache-clear \ @@ -38,8 +32,8 @@ main() { local lib=$1 # generation phase - runtest_gold ${lib} - runtest_cudf_pandas ${lib} + runtest ${lib} "gold" + runtest ${lib} "cudf" # assertion phase pytest \ diff --git a/ci/cudf_pandas_scripts/third-party-integration/test.sh b/ci/cudf_pandas_scripts/third-party-integration/test.sh index 89b28c30e39..f8ddbaba0f3 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/test.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/test.sh @@ -72,7 +72,7 @@ main() { fi done - TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${lib} + TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh ${lib} rapids-logger "Test script exiting with value: ${EXITCODE}" done From a729a28abd7c65a34852c4cc62e1075cc625d3b5 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 3 Sep 2024 14:20:39 -0700 Subject: [PATCH 2/3] add blank line --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d4d5e3a940e..fe07b61d39e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -315,4 +315,4 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: | - ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml \ No newline at end of file + ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml From 45f6172496357e36c5379c81d89e41e9de13793d Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 3 Sep 2024 17:29:41 -0700 Subject: [PATCH 3/3] remove job from pr workflow --- .github/workflows/pr.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fe07b61d39e..8730804e8b6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -36,7 +36,6 @@ jobs: - unit-tests-cudf-pandas - pandas-tests - pandas-tests-diff - - third-party-integration-tests-cudf-pandas secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 if: always() @@ -305,14 +304,3 @@ jobs: node_type: cpu4 build_type: pull-request run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh" - third-party-integration-tests-cudf-pandas: - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: | - ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml