remove extract_lib.sh

.github/workflows/pr.yaml

@@ -307,4 +307,4 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: |
-        ci/cudf_pandas_scripts/third-party-integration/test_new.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml
+        ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml

.github/workflows/test.yaml

@@ -134,4 +134,4 @@ jobs:
       sha: ${{ inputs.sha }}
       container_image: "rapidsai/ci-conda:latest"
       run_script: |
-        ci/cudf_pandas_scripts/third-party-integration/test_new.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml
+        ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml

ci/cudf_pandas_scripts/third-party-integration/test.sh

@@ -3,55 +3,85 @@
 
 # Common setup steps shared by Python test jobs
 
-LIB=$1
-
 set -euo pipefail
 
-. /opt/conda/etc/profile.d/conda.sh
+write_output() {
+  local key="$1"
+  local value="$2"
+  echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}"
+}
+
+extract_lib_from_dependencies_yaml() {
+    local file=$1
+    # Parse all keys in dependencies.yaml under the "files" section,
+    # extract all the keys that start with "test_", and extract the rest
+    local extracted_libs="$(yq -o json $file | jq -rc '.files | with_entries(select(.key | contains("test_"))) | keys | map(sub("^test_"; ""))')"
+    echo $extracted_libs
+}
+
+main() {
+    local dependencies_yaml="$1"
+
+    LIBS=$(extract_lib_from_dependencies_yaml "$dependencies_yaml")
+    LIBS=${LIBS#[}
+    LIBS=${LIBS%]}
+
+    for lib in ${LIBS//,/ }; do
+        lib=$(echo "$lib" | tr -d '""')
+        echo "Running tests for library $lib"
+
+        CUDA_MAJOR=$(if [ "$lib" = "tensorflow" ]; then echo "11"; else echo "12"; fi)
+
+        . /opt/conda/etc/profile.d/conda.sh
+
+        rapids-logger "Generate Python testing dependencies"
+        rapids-dependency-file-generator \
+          --config "$dependencies_yaml" \
+          --output conda \
+          --file-key test_${lib} \
+          --matrix "cuda=${CUDA_MAJOR};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+        rapids-mamba-retry env create --yes -f env.yaml -n test
 
-rapids-logger "Generate Python testing dependencies"
-rapids-dependency-file-generator \
-  --config "python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml" \
-  --output conda \
-  --file-key test_${LIB} \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+        # Temporarily allow unbound variables for conda activation.
+        set +u
+        conda activate test
+        set -u
 
-rapids-mamba-retry env create --yes -f env.yaml -n test
+        RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
 
-# Temporarily allow unbound variables for conda activation.
-set +u
-conda activate test
-set -u
+        mkdir -p "${RAPIDS_TESTS_DIR}"
 
-RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
+        repo_root=$(git rev-parse --show-toplevel)
+        TEST_DIR=${repo_root}/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests
 
-mkdir -p "${RAPIDS_TESTS_DIR}"
+        rapids-print-env
 
-repo_root=$(git rev-parse --show-toplevel)
-TEST_DIR=${repo_root}/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests
+        rapids-logger "Check GPU usage"
+        nvidia-smi
 
-rapids-print-env
+        EXITCODE=0
+        trap "EXITCODE=1" ERR
+        set +e
 
-rapids-logger "Check GPU usage"
-nvidia-smi
+        rapids-logger "pytest ${lib}"
 
-EXITCODE=0
-trap "EXITCODE=1" ERR
-set +e
+        NUM_PROCESSES=8
+        serial_libraries=(
+            "tensorflow"
+        )
+        for serial_library in "${serial_libraries[@]}"; do
+            if [ "${lib}" = "${serial_library}" ]; then
+                NUM_PROCESSES=1
+            fi
+        done
 
-rapids-logger "pytest ${LIB}"
+        RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR} TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${lib}
 
-NUM_PROCESSES=8
-serial_libraries=(
-    "tensorflow"
-)
-for serial_library in "${serial_libraries[@]}"; do
-    if [ "${LIB}" = "${serial_library}" ]; then
-        NUM_PROCESSES=1
-    fi
-done
+        rapids-logger "Test script exiting with value: ${EXITCODE}"
+    done
 
-RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR} TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${LIB}
+    exit ${EXITCODE}
+}
 
-rapids-logger "Test script exiting with value: ${EXITCODE}"
-exit ${EXITCODE}
+main "$@"