From 609313c0117a5cd76589696efc4839f0d15100e2 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 28 Aug 2024 06:46:03 -0700 Subject: [PATCH] remove extract_lib.sh --- .github/workflows/pr.yaml | 2 +- .github/workflows/test.yaml | 2 +- .../third-party-integration/extract_lib.sh | 27 ----- .../third-party-integration/test.sh | 104 +++++++++++------- .../third-party-integration/test_new.sh | 87 --------------- 5 files changed, 69 insertions(+), 153 deletions(-) delete mode 100755 ci/cudf_pandas_scripts/third-party-integration/extract_lib.sh delete mode 100644 ci/cudf_pandas_scripts/third-party-integration/test_new.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 41c71d30a0c..d55207dd0dd 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -307,4 +307,4 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: | - ci/cudf_pandas_scripts/third-party-integration/test_new.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml + ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f499e55a713..2c68f2861bb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -134,4 +134,4 @@ jobs: sha: ${{ inputs.sha }} container_image: "rapidsai/ci-conda:latest" run_script: | - ci/cudf_pandas_scripts/third-party-integration/test_new.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml + ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml diff --git a/ci/cudf_pandas_scripts/third-party-integration/extract_lib.sh b/ci/cudf_pandas_scripts/third-party-integration/extract_lib.sh deleted file mode 100755 index 67ec5c773bc..00000000000 --- a/ci/cudf_pandas_scripts/third-party-integration/extract_lib.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -write_output() { - local key="$1" - local value="$2" - echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}" -} - -extract_lib_from_dependencies_yaml() { - local file=$1 - # Parse all keys in dependencies.yaml under the "files" section, - # extract all the keys that starts with "test_", and extract the - # rest - local extracted_libs="$(yq -o json $file | jq -rc '.files | with_entries( select(.key | contains("test_")) ) | keys | map(sub("^test_"; ""))')" - echo $extracted_libs -} - - -main() { - local dependencies_yaml="$1" - extract_lib_from_dependencies_yaml "$dependencies_yaml" -} - -main "$@" diff --git a/ci/cudf_pandas_scripts/third-party-integration/test.sh b/ci/cudf_pandas_scripts/third-party-integration/test.sh index 6c1c7efe489..61370a1dfdb 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/test.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/test.sh @@ -3,55 +3,85 @@ # Common setup steps shared by Python test jobs -LIB=$1 - set -euo pipefail -. /opt/conda/etc/profile.d/conda.sh +write_output() { + local key="$1" + local value="$2" + echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}" +} + +extract_lib_from_dependencies_yaml() { + local file=$1 + # Parse all keys in dependencies.yaml under the "files" section, + # extract all the keys that start with "test_", and extract the rest + local extracted_libs="$(yq -o json $file | jq -rc '.files | with_entries(select(.key | contains("test_"))) | keys | map(sub("^test_"; ""))')" + echo $extracted_libs +} + +main() { + local dependencies_yaml="$1" + + LIBS=$(extract_lib_from_dependencies_yaml "$dependencies_yaml") + LIBS=${LIBS#[} + LIBS=${LIBS%]} + + for lib in ${LIBS//,/ }; do + lib=$(echo "$lib" | tr -d '""') + echo "Running tests for library $lib" + + CUDA_MAJOR=$(if [ "$lib" = "tensorflow" ]; then echo "11"; else echo "12"; fi) + + . /opt/conda/etc/profile.d/conda.sh + + rapids-logger "Generate Python testing dependencies" + rapids-dependency-file-generator \ + --config "$dependencies_yaml" \ + --output conda \ + --file-key test_${lib} \ + --matrix "cuda=${CUDA_MAJOR};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + + rapids-mamba-retry env create --yes -f env.yaml -n test -rapids-logger "Generate Python testing dependencies" -rapids-dependency-file-generator \ - --config "python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml" \ - --output conda \ - --file-key test_${LIB} \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + # Temporarily allow unbound variables for conda activation. + set +u + conda activate test + set -u -rapids-mamba-retry env create --yes -f env.yaml -n test + RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} -# Temporarily allow unbound variables for conda activation. -set +u -conda activate test -set -u + mkdir -p "${RAPIDS_TESTS_DIR}" -RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} + repo_root=$(git rev-parse --show-toplevel) + TEST_DIR=${repo_root}/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests -mkdir -p "${RAPIDS_TESTS_DIR}" + rapids-print-env -repo_root=$(git rev-parse --show-toplevel) -TEST_DIR=${repo_root}/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests + rapids-logger "Check GPU usage" + nvidia-smi -rapids-print-env + EXITCODE=0 + trap "EXITCODE=1" ERR + set +e -rapids-logger "Check GPU usage" -nvidia-smi + rapids-logger "pytest ${lib}" -EXITCODE=0 -trap "EXITCODE=1" ERR -set +e + NUM_PROCESSES=8 + serial_libraries=( + "tensorflow" + ) + for serial_library in "${serial_libraries[@]}"; do + if [ "${lib}" = "${serial_library}" ]; then + NUM_PROCESSES=1 + fi + done -rapids-logger "pytest ${LIB}" + RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR} TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${lib} -NUM_PROCESSES=8 -serial_libraries=( - "tensorflow" -) -for serial_library in "${serial_libraries[@]}"; do - if [ "${LIB}" = "${serial_library}" ]; then - NUM_PROCESSES=1 - fi -done + rapids-logger "Test script exiting with value: ${EXITCODE}" + done -RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR} TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${LIB} + exit ${EXITCODE} +} -rapids-logger "Test script exiting with value: ${EXITCODE}" -exit ${EXITCODE} +main "$@" diff --git a/ci/cudf_pandas_scripts/third-party-integration/test_new.sh b/ci/cudf_pandas_scripts/third-party-integration/test_new.sh deleted file mode 100644 index 61370a1dfdb..00000000000 --- a/ci/cudf_pandas_scripts/third-party-integration/test_new.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -# Common setup steps shared by Python test jobs - -set -euo pipefail - -write_output() { - local key="$1" - local value="$2" - echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}" -} - -extract_lib_from_dependencies_yaml() { - local file=$1 - # Parse all keys in dependencies.yaml under the "files" section, - # extract all the keys that start with "test_", and extract the rest - local extracted_libs="$(yq -o json $file | jq -rc '.files | with_entries(select(.key | contains("test_"))) | keys | map(sub("^test_"; ""))')" - echo $extracted_libs -} - -main() { - local dependencies_yaml="$1" - - LIBS=$(extract_lib_from_dependencies_yaml "$dependencies_yaml") - LIBS=${LIBS#[} - LIBS=${LIBS%]} - - for lib in ${LIBS//,/ }; do - lib=$(echo "$lib" | tr -d '""') - echo "Running tests for library $lib" - - CUDA_MAJOR=$(if [ "$lib" = "tensorflow" ]; then echo "11"; else echo "12"; fi) - - . /opt/conda/etc/profile.d/conda.sh - - rapids-logger "Generate Python testing dependencies" - rapids-dependency-file-generator \ - --config "$dependencies_yaml" \ - --output conda \ - --file-key test_${lib} \ - --matrix "cuda=${CUDA_MAJOR};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml - - rapids-mamba-retry env create --yes -f env.yaml -n test - - # Temporarily allow unbound variables for conda activation. - set +u - conda activate test - set -u - - RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} - - mkdir -p "${RAPIDS_TESTS_DIR}" - - repo_root=$(git rev-parse --show-toplevel) - TEST_DIR=${repo_root}/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests - - rapids-print-env - - rapids-logger "Check GPU usage" - nvidia-smi - - EXITCODE=0 - trap "EXITCODE=1" ERR - set +e - - rapids-logger "pytest ${lib}" - - NUM_PROCESSES=8 - serial_libraries=( - "tensorflow" - ) - for serial_library in "${serial_libraries[@]}"; do - if [ "${lib}" = "${serial_library}" ]; then - NUM_PROCESSES=1 - fi - done - - RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR} TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${lib} - - rapids-logger "Test script exiting with value: ${EXITCODE}" - done - - exit ${EXITCODE} -} - -main "$@"