[FEA] Add third-party library integration testing of cudf.pandas to cudf

.github/workflows/pr.yaml

@@ -36,6 +36,7 @@ jobs:
       - unit-tests-cudf-pandas
       - pandas-tests
       - pandas-tests-diff
+      - integration-tests
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: always()
@@ -296,3 +297,22 @@ jobs:
         node_type: cpu4
         build_type: pull-request
         run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh"
+  third-party-integration-tests-cudf-pandas:
+    needs: wheel-build-cudf
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      node_type: "gpu-v100-latest-1"
+      arch: "amd64"
+      container_image: "rapidsai/ci-conda:latest"
+      run_script: |
+        LIBS=$(ci/cudf_pandas_scripts/third-party-integration/extract_lib.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml)
+        LIBS=${LIBS#[}
+        LIBS=${LIBS%]}
+        for lib in ${LIBS//,/ }; do
+          lib=$(echo "$lib" | tr -d '""')
+          echo "Running tests for $lib"
+          CUDA_MAJOR=$(if [ "$lib" = "tensorflow" ]; then echo "11"; else echo "12"; fi)
+          ci/cudf_pandas_scripts/third-party-integration/test.sh $lib $CUDA_MAJOR
+        done

.github/workflows/test.yaml

@@ -124,3 +124,22 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       script: ci/cudf_pandas_scripts/run_tests.sh
+  integration-tests:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      container_image: "rapidsai/ci-conda:latest"
+      run_script: |
+        LIBS=$(ci/cudf_pandas_scripts/third-party-integration/extract_lib.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml)
+        LIBS=${LIBS#[}
+        LIBS=${LIBS%]}
+        for lib in ${LIBS//,/ }; do
+          lib=$(echo "$lib" | tr -d '""')
+          echo "Running tests for $lib"
+          CUDA_MAJOR=$(if [ "$lib" = "tensorflow" ]; then echo "11"; else echo "12"; fi)
+          ci/cudf_pandas_scripts/third-party-integration/test.sh $lib $CUDA_MAJOR
+        done

ci/cudf_pandas_scripts/run_tests.sh

@@ -62,6 +62,7 @@ else
 fi
 
 python -m pytest -p cudf.pandas \
+    --ignore=./python/cudf/cudf_pandas_tests/third_party_integration_tests/ \
     --cov-config=./python/cudf/.coveragerc \
     --cov=cudf \
     --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-pandas-coverage.xml" \
@@ -77,6 +78,7 @@ for version in "${versions[@]}"; do
     echo "Installing pandas version: ${version}"
     python -m pip install "numpy>=1.23,<2.0a0" "pandas==${version}"
     python -m pytest -p cudf.pandas \
+    --ignore=./python/cudf/cudf_pandas_tests/third_party_integration_tests/ \
     --cov-config=./python/cudf/.coveragerc \
     --cov=cudf \
     --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-pandas-coverage.xml" \

ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+cleanup() {
+    rm ${TEST_DIR}/results-*.pickle
+}
+
+trap cleanup EXIT
+
+runtest_gold() {
+    local lib=$1
+    local test_keys=${@:2}
+
+    pytest \
+    -v \
+    --continue-on-collection-errors \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-${lib}-gold.xml" \
+    --numprocesses=${NUM_PROCESSES} \
+    --dist=worksteal \
+    ${TEST_DIR}/test_${lib}*.py \
+    ${test_keys}
+}
+
+runtest_cudf_pandas() {
+    local lib=$1
+    local test_keys=${@:2}
+
+    pytest \
+    -p cudf.pandas \
+    -v \
+    --continue-on-collection-errors \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-${lib}-cudf-pandas.xml" \
+    --numprocesses=${NUM_PROCESSES} \
+    --dist=worksteal \
+    ${TEST_DIR}/test_${lib}*.py \
+    ${test_keys}
+}
+
+main() {
+    local lib=$1
+    local test_keys=${@:2}
+
+    # generation phase
+    runtest_gold ${lib} ${test_keys}
+    runtest_cudf_pandas ${lib} ${test_keys}
+
+    # assertion phase
+    pytest \
+    --compare \
+    -p cudf.pandas \
+    -v \
+    --continue-on-collection-errors \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-${lib}-assertion.xml" \
+    --numprocesses=${NUM_PROCESSES} \
+    --dist=worksteal \
+    ${TEST_DIR}/test_${lib}*.py \
+    ${test_keys}
+}
+
+main $@

ci/cudf_pandas_scripts/third-party-integration/extract_lib.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+write_output() {
+  local key="$1"
+  local value="$2"
+  echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}"
+}
+
+extract_lib_from_dependencies_yaml() {
+    local file=$1
+    # Parse all keys in dependencies.yaml under the "files" section,
+    # extract all the keys that starts with "test_", and extract the
+    # rest
+    local extracted_libs="$(yq -o json $file | jq -rc '.files | with_entries( select(.key | contains("test_")) ) | keys | map(sub("^test_"; ""))')"
+    echo $extracted_libs
+}
+
+
+main() {
+    local dependencies_yaml="$1"
+    extract_lib_from_dependencies_yaml "$dependencies_yaml"
+}
+
+main "$@"

ci/cudf_pandas_scripts/third-party-integration/test.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+# Common setup steps shared by Python test jobs
+
+LIB=$1
+
+set -euo pipefail
+
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-logger "Generate Python testing dependencies"
+rapids-dependency-file-generator \
+  --config "python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml" \
+  --output conda \
+  --file-key test_${LIB} \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --yes -f env.yaml -n test
+
+# Temporarily allow unbound variables for conda activation.
+set +u
+conda activate test
+set -u
+
+RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
+
+mkdir -p "${RAPIDS_TESTS_DIR}"
+
+repo_root=$(git rev-parse --show-toplevel)
+TEST_DIR=${repo_root}/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests
+
+rapids-print-env
+
+rapids-logger "Check GPU usage"
+nvidia-smi
+
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
+rapids-logger "pytest ${LIB}"
+
+NUM_PROCESSES=8
+serial_libraries=(
+    "tensorflow"
+)
+for serial_library in "${serial_libraries[@]}"; do
+    if [ "${LIB}" = "${serial_library}" ]; then
+        NUM_PROCESSES=1
+    fi
+done
+
+RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR} TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${LIB}
+
+rapids-logger "Test script exiting with value: ${EXITCODE}"
+exit ${EXITCODE}