From 4743b62e10d2c5c7507e4d02a26c6dbb7bec4297 Mon Sep 17 00:00:00 2001 From: Mark Harris Date: Thu, 9 Nov 2023 11:39:00 +0000 Subject: [PATCH] Fix artifact path and make a utility function --- ci/build_cpp.sh | 24 ++++-------------------- ci/build_docs.sh | 24 +++++------------------- ci/build_python.sh | 24 +++++------------------- ci/test_cpp.sh | 26 ++++++-------------------- ci/test_notebooks.sh | 24 +++++------------------- ci/test_python.sh | 26 ++++++-------------------- ci/utils/rapids-pr-artifact-path.sh | 0 7 files changed, 31 insertions(+), 117 deletions(-) create mode 100644 ci/utils/rapids-pr-artifact-path.sh diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 1a2749bf3..d48536f9f 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,26 +7,10 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" - -rapids_repo_pr_artifact_channel () { - local repo=$1 - local pr=$2 - local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) - - if [[ $3 == "cpp" ]] - then - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - else - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) - fi -} - -LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) -LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) - -echo "LIBRMM_CHANNEL == " ${LIBRMM_CHANNEL} -echo "LIBCUDF_CHANNEL == " ${LIBCUDF_CHANNEL} +ARTIFACT="$(realpath "$(dirname "$0")/utils/rapids-pr-artifact-path.sh")" + +LIBRMM_CHANNEL=$(${ARTIFACT} rmm 1095 cpp) +LIBCUDF_CHANNEL=$(${ARTIFACT} cudf 14365 cpp) rapids-print-env diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 8b660b0c4..380f7084e 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -20,26 +20,12 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') +ARTIFACT="$(realpath "$(dirname "$0")/utils/rapids-pr-artifact-path.sh")" -rapids_repo_pr_artifact_channel () { - local repo=$1 - local pr=$2 - local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) - - if [[ $3 == "cpp" ]] - then - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - else - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) - fi -} - -LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) -RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) -LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) -CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) +LIBRMM_CHANNEL=$(${ARTIFACT} rmm 1095 cpp) +RMM_CHANNEL=$(${ARTIFACT} rmm 1095 python) +LIBCUDF_CHANNEL=$(${ARTIFACT} cudf 14365 cpp) +CUDF_CHANNEL=$(${ARTIFACT} cudf 14365 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ diff --git a/ci/build_python.sh b/ci/build_python.sh index fed355be1..ddd160148 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -19,28 +19,14 @@ for package_name in cuspatial cuproj; do sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/${package_name}/_version.py" done -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') - CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -rapids_repo_pr_artifact_channel () { - local repo=$1 - local pr=$2 - local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) - - if [[ $3 == "cpp" ]] - then - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - else - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) - fi -} +ARTIFACT="$(realpath "$(dirname "$0")/utils/rapids-pr-artifact-path.sh")" -LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) -RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) -LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) -CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) +LIBRMM_CHANNEL=$(${ARTIFACT} rmm 1095 cpp) +RMM_CHANNEL=$(${ARTIFACT} rmm 1095 python) +LIBCUDF_CHANNEL=$(${ARTIFACT} cudf 14365 cpp) +CUDF_CHANNEL=$(${ARTIFACT} cudf 14365 python) rapids-logger "Begin py build cuSpatial" diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index d17639ace..ce0d9a93b 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -18,28 +18,14 @@ set +u conda activate test set -u -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') - CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -rapids_repo_pr_artifact_channel () { - local repo=$1 - local pr=$2 - local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) - - if [[ $3 == "cpp" ]] - then - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - else - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) - fi -} - -LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) -RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) -LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) -CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) +ARTIFACT="$(realpath "$(dirname "$0")/utils/rapids-pr-artifact-path.sh")" + +LIBRMM_CHANNEL=$(${ARTIFACT} rmm 1095 cpp) +RMM_CHANNEL=$(${ARTIFACT} rmm 1095 python) +LIBCUDF_CHANNEL=$(${ARTIFACT} cudf 14365 cpp) +CUDF_CHANNEL=$(${ARTIFACT} cudf 14365 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index efa52095d..b3b227ba3 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -24,26 +24,12 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') +ARTIFACT="$(realpath "$(dirname "$0")/utils/rapids-pr-artifact-path.sh")" -rapids_repo_pr_artifact_channel () { - local repo=$1 - local pr=$2 - local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) - - if [[ $3 == "cpp" ]] - then - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - else - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) - fi -} - -LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) -RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) -LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) -CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) +LIBRMM_CHANNEL=$(${ARTIFACT} rmm 1095 cpp) +RMM_CHANNEL=$(${ARTIFACT} rmm 1095 python) +LIBCUDF_CHANNEL=$(${ARTIFACT} cudf 14365 cpp) +CUDF_CHANNEL=$(${ARTIFACT} cudf 14365 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ diff --git a/ci/test_python.sh b/ci/test_python.sh index 78bb07baf..1ee488922 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -22,26 +22,12 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') - -rapids_repo_pr_artifact_channel () { - local repo=$1 - local pr=$2 - local commit=$(git ls-remote https://github.com/rapidsai/${repo}.git refs/heads/pull-request/${pr} | cut -c1-7) - - if [[ $3 == "cpp" ]] - then - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - else - echo $(rapids-get-artifact ci/${repo}/pull-request/${pr}/${commit}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) - fi -} - -LIBRMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 cpp) -RMM_CHANNEL=$(rapids_repo_pr_artifact_channel rmm 1095 python) -LIBCUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 cpp) -CUDF_CHANNEL=$(rapids_repo_pr_artifact_channel cudf 14365 python) +ARTIFACT="$(realpath "$(dirname "$0")/utils/rapids-pr-artifact-path.sh")" + +LIBRMM_CHANNEL=$(${ARTIFACT} rmm 1095 cpp) +RMM_CHANNEL=$(${ARTIFACT} rmm 1095 python) +LIBCUDF_CHANNEL=$(${ARTIFACT} cudf 14365 cpp) +CUDF_CHANNEL=$(${ARTIFACT} cudf 14365 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} diff --git a/ci/utils/rapids-pr-artifact-path.sh b/ci/utils/rapids-pr-artifact-path.sh new file mode 100644 index 000000000..e69de29bb