From 73ebcaf448e7b7223710c3df21260725e7e1e40b Mon Sep 17 00:00:00 2001 From: Mark Harris <783069+harrism@users.noreply.github.com> Date: Tue, 7 Nov 2023 01:38:18 +0000 Subject: [PATCH 1/3] Test CI with artifacts from rapidsai/rmm#1095 --- ci/build_cpp.sh | 13 ++++++++++++- ci/build_python.sh | 23 ++++++++++++++++++++++- ci/test_cpp.sh | 11 +++++++++++ ci/test_notebooks.sh | 10 ++++++++++ ci/test_python.sh | 14 +++++++++++++- 5 files changed, 68 insertions(+), 3 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index d0d13f99448..804fca9ca95 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,12 +7,23 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja +REPO="rmm" +PR_NUMBER="1095" +COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') +LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + rapids-print-env version=$(rapids-generate-version) rapids-logger "Begin cpp build" -RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcugraph +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + conda/recipes/libcugraph rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 90a40c539ff..0915a03d56a 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,6 +11,15 @@ rapids-print-env CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +REPO="rmm" +PR_NUMBER="1095" +COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') +LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + + version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) export RAPIDS_PACKAGE_VERSION=${version} @@ -19,7 +28,7 @@ echo "${version}" > VERSION rapids-logger "Begin py build" package_dir="python" -for package_name in pylibcugraph cugraph nx-cugraph cugraph-pyg cugraph-dgl; do +for package_name in pylibcugraph cugraph nx-cugraph cugraph-pyg cugraph-dgl; do underscore_package_name=$(echo "${package_name}" | tr "-" "_") sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" done @@ -29,12 +38,16 @@ done rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ conda/recipes/pylibcugraph rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ conda/recipes/cugraph # NOTE: nothing in nx-cugraph is CUDA-specific, but it is built on each CUDA @@ -45,6 +58,8 @@ rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ conda/recipes/nx-cugraph # NOTE: nothing in the cugraph-service packages are CUDA-specific, but they are @@ -59,6 +74,8 @@ rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ conda/recipes/cugraph-service RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" @@ -72,6 +89,8 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then --channel pyg \ --channel pytorch \ --channel pytorch-nightly \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ conda/recipes/cugraph-pyg # Only CUDA 11 is supported right now due to PyTorch requirement. @@ -82,6 +101,8 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then --channel dglteam \ --channel pytorch \ --channel pytorch-nightly \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ conda/recipes/cugraph-dgl fi diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 95bc55c212a..5c86138599d 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -19,6 +19,15 @@ conda activate test set -u CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) + +REPO="rmm" +PR_NUMBER="1095" +COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') +LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -26,6 +35,8 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ libcugraph libcugraph_etl libcugraph-tests rapids-logger "Check GPU usage" diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 653f23f947d..d067d4e18fd 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -24,9 +24,19 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +REPO="rmm" +PR_NUMBER="1095" +COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') +LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ libcugraph pylibcugraph cugraph NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" diff --git a/ci/test_python.sh b/ci/test_python.sh index 1690ce2f15b..60cc5a54f1f 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -22,6 +22,14 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +REPO="rmm" +PR_NUMBER="1095" +COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') +LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) + RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" @@ -31,6 +39,8 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ libcugraph \ pylibcugraph \ cugraph \ @@ -206,13 +216,15 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then 'pyg=2.3' \ 'pytorch=2.0.0' \ 'pytorch-cuda=11.8' - + # Install pyg dependencies (which requires pip) pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.0.0+cu118.html rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ libcugraph \ pylibcugraph \ pylibcugraphops \ From aeee8732d895622bfe64321d07a1f16208cff1ad Mon Sep 17 00:00:00 2001 From: Mark Harris <783069+harrism@users.noreply.github.com> Date: Thu, 16 Nov 2023 05:48:41 +0000 Subject: [PATCH 2/3] Update to use gha tool and rerun --- ci/build_cpp.sh | 10 +++------- ci/build_docs.sh | 10 ++++++++++ ci/build_python.sh | 36 ++++++++++++++++++++++++++++-------- ci/test_cpp.sh | 14 +++++++------- ci/test_notebooks.sh | 14 +++++++------- ci/test_python.sh | 30 +++++++++++++++++++++++------- 6 files changed, 78 insertions(+), 36 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 804fca9ca95..690c8fa7eaf 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,13 +7,9 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) rapids-print-env diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 3f97f652d41..3cb21f35607 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -20,9 +20,19 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) + rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ libcugraph \ pylibcugraph \ cugraph \ diff --git a/ci/build_python.sh b/ci/build_python.sh index 0915a03d56a..05a8be002eb 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,14 +11,10 @@ rapids-print-env CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) - +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python) version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) @@ -40,6 +36,10 @@ rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/pylibcugraph rapids-conda-retry mambabuild \ @@ -48,6 +48,10 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/cugraph # NOTE: nothing in nx-cugraph is CUDA-specific, but it is built on each CUDA @@ -60,6 +64,10 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/nx-cugraph # NOTE: nothing in the cugraph-service packages are CUDA-specific, but they are @@ -76,6 +84,10 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/cugraph-service RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" @@ -91,6 +103,10 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then --channel pytorch-nightly \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/cugraph-pyg # Only CUDA 11 is supported right now due to PyTorch requirement. @@ -103,6 +119,10 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then --channel pytorch-nightly \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/cugraph-dgl fi diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 5c86138599d..1505ca595d2 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -20,13 +20,9 @@ set -u CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -37,6 +33,10 @@ rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ libcugraph libcugraph_etl libcugraph-tests rapids-logger "Check GPU usage" diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index d067d4e18fd..357af8b9841 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -24,19 +24,19 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ libcugraph pylibcugraph cugraph NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" diff --git a/ci/test_python.sh b/ci/test_python.sh index 60cc5a54f1f..c7e4a127ad7 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -22,13 +22,9 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -REPO="rmm" -PR_NUMBER="1095" -COMMIT=$(git ls-remote https://github.com/rapidsai/${REPO}.git refs/heads/pull-request/${PR_NUMBER} | cut -c1-7) -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -PYTHON_MINOR_VERSION=$(python --version | sed -E 's/Python [0-9]+\.([0-9]+)\.[0-9]+/\1/g') -LIBRMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -RMM_CHANNEL=$(rapids-get-artifact ci/${REPO}/pull-request/${PR_NUMBER}/${COMMIT}/rmm_conda_python_cuda${RAPIDS_CUDA_MAJOR}_3${PYTHON_MINOR_VERSION}_$(arch).tar.gz) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} @@ -41,6 +37,10 @@ rapids-mamba-retry install \ --channel "${PYTHON_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ libcugraph \ pylibcugraph \ cugraph \ @@ -162,6 +162,12 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then --channel pytorch-nightly \ --channel dglteam/label/cu118 \ --channel nvidia \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ libcugraph \ pylibcugraph \ pylibcugraphops \ @@ -213,6 +219,12 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then --channel pyg \ --channel pytorch \ --channel nvidia \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ 'pyg=2.3' \ 'pytorch=2.0.0' \ 'pytorch-cuda=11.8' @@ -225,6 +237,10 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then --channel "${PYTHON_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ libcugraph \ pylibcugraph \ pylibcugraphops \ From 867a1a1dd95bbd822f3e116ea08a600641beda36 Mon Sep 17 00:00:00 2001 From: Mark Harris <783069+harrism@users.noreply.github.com> Date: Thu, 16 Nov 2023 05:57:57 +0000 Subject: [PATCH 3/3] Fix scripts --- ci/build_cpp.sh | 3 ++- ci/build_docs.sh | 3 +++ ci/build_python.sh | 2 ++ ci/test_cpp.sh | 3 --- ci/test_notebooks.sh | 3 +++ ci/test_python.sh | 3 +++ 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 690c8fa7eaf..029aea22540 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -19,7 +19,8 @@ rapids-logger "Begin cpp build" RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ conda/recipes/libcugraph rapids-upload-conda-to-s3 cpp diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 3cb21f35607..ed8019a0647 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -21,8 +21,11 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 python) LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 python) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ diff --git a/ci/build_python.sh b/ci/build_python.sh index 05a8be002eb..57860b8c837 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -12,7 +12,9 @@ rapids-print-env CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 python) LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 python) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python) diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 1505ca595d2..b3bc7a0bafb 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -32,11 +32,8 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ --channel "${LIBCUDF_CHANNEL}" \ - --channel "${CUDF_CHANNEL}" \ --channel "${LIBRAFT_CHANNEL}" \ - --channel "${RAFT_CHANNEL}" \ libcugraph libcugraph_etl libcugraph-tests rapids-logger "Check GPU usage" diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 357af8b9841..5f8e2a4d0d2 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -25,8 +25,11 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 python) LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 python) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ diff --git a/ci/test_python.sh b/ci/test_python.sh index c7e4a127ad7..af36913b03b 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -23,8 +23,11 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 python) LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 python) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}