From e90d2769e46f9ca1af3b1b9c4eadb97df8f63393 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 15 Dec 2023 20:37:05 -0600 Subject: [PATCH] Use upstream artifacts. --- ci/build_cpp.sh | 10 +++++++++- ci/build_python.sh | 38 +++++++++++++++++++++++++++++++++++++- ci/test_cpp.sh | 7 +++++++ ci/test_python.sh | 26 +++++++++++++++++++++++++- 4 files changed, 78 insertions(+), 3 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index d0d13f99448..6b01d6fade8 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -9,10 +9,18 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1404 cpp) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14576 cpp) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2049 cpp) + version=$(rapids-generate-version) rapids-logger "Begin cpp build" -RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcugraph +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + conda/recipes/libcugraph rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 90a40c539ff..12f71a2c449 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -10,6 +10,12 @@ export CMAKE_GENERATOR=Ninja rapids-print-env CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1404 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1404 python) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14576 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14576 python) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2049 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2049 python) version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) @@ -19,7 +25,7 @@ echo "${version}" > VERSION rapids-logger "Begin py build" package_dir="python" -for package_name in pylibcugraph cugraph nx-cugraph cugraph-pyg cugraph-dgl; do +for package_name in pylibcugraph cugraph nx-cugraph cugraph-pyg cugraph-dgl; do underscore_package_name=$(echo "${package_name}" | tr "-" "_") sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" done @@ -29,11 +35,23 @@ done rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/pylibcugraph rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cugraph @@ -44,6 +62,12 @@ rapids-conda-retry mambabuild \ rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/nx-cugraph @@ -68,6 +92,12 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ --channel pyg \ --channel pytorch \ @@ -78,6 +108,12 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ --channel dglteam \ --channel pytorch \ diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 95bc55c212a..1cd3e53c696 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -19,6 +19,10 @@ conda activate test set -u CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1404 cpp) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14576 cpp) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2049 cpp) + RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -26,6 +30,9 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ libcugraph libcugraph_etl libcugraph-tests rapids-logger "Check GPU usage" diff --git a/ci/test_python.sh b/ci/test_python.sh index d6e92e8d1a5..9b2c786888c 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -21,6 +21,12 @@ set -u rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1404 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1404 python) +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14576 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14576 python) +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2049 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2049 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} @@ -31,6 +37,12 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ libcugraph \ pylibcugraph \ cugraph \ @@ -148,6 +160,12 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ --channel pytorch \ --channel pytorch-nightly \ --channel dglteam/label/cu118 \ @@ -201,6 +219,12 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ --channel pytorch \ --channel nvidia \ --channel pyg \ @@ -208,7 +232,7 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then "cugraph-pyg" \ "pytorch>=2.0,<2.1" \ "pytorch-cuda=11.8" - + # Install pyg dependencies (which requires pip) pip install \ pyg_lib \