kick off a build

rapidsai · Sep 19, 2024 · 8f918bd · 8f918bd
1 parent 1a89935
commit 8f918bd
Show file tree

Hide file tree

Showing 10 changed files with 154 additions and 145 deletions.
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -12,103 +12,103 @@ concurrency:
 jobs:
   pr-builder:
     needs:
-      - checks
+      # - checks
       - conda-cpp-build
-      - conda-cpp-tests
-      - conda-cpp-checks
+      # - conda-cpp-tests
+      # - conda-cpp-checks
       - conda-python-build
-      - conda-python-tests
-      - docs-build
+      # - conda-python-tests
+      # - docs-build
       - wheel-build-pylibraft
-      - wheel-tests-pylibraft
+      # - wheel-tests-pylibraft
       - wheel-build-raft-dask
-      - wheel-tests-raft-dask
+      # - wheel-tests-raft-dask
       - devcontainer
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-  checks:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      enable_check_generated_files: false
+  # checks:
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     enable_check_generated_files: false
   conda-cpp-build:
-    needs: checks
+    # needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       node_type: cpu16
-  conda-cpp-tests:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-  conda-cpp-checks:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      enable_check_symbols: true
-      symbol_exclusions: raft_cutlass
+  # conda-cpp-tests:
+  #   needs: conda-cpp-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  # conda-cpp-checks:
+  #   needs: conda-cpp-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     enable_check_symbols: true
+  #     symbol_exclusions: raft_cutlass
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
-  conda-python-tests:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-  docs-build:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/build_docs.sh"
+  # conda-python-tests:
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  # docs-build:
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:latest"
+  #     run_script: "ci/build_docs.sh"
   wheel-build-pylibraft:
-    needs: checks
+    # needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_pylibraft.sh
-  wheel-tests-pylibraft:
-    needs: wheel-build-pylibraft
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_pylibraft.sh
+  # wheel-tests-pylibraft:
+  #   needs: wheel-build-pylibraft
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/test_wheel_pylibraft.sh
   wheel-build-raft-dask:
     needs: wheel-tests-pylibraft
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: "ci/build_wheel_raft_dask.sh"
-  wheel-tests-raft-dask:
-    needs: wheel-build-raft-dask
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_raft_dask.sh
-  devcontainer:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      arch: '["amd64"]'
-      cuda: '["12.5"]'
-      build_command: |
-        sccache -z;
-        build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose;
-        sccache -s;
+  # wheel-tests-raft-dask:
+  #   needs: wheel-build-raft-dask
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/test_wheel_raft_dask.sh
+  # devcontainer:
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     arch: '["amd64"]'
+  #     cuda: '["12.5"]'
+  #     build_command: |
+  #       sccache -z;
+  #       build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose;
+  #       sccache -s;
diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -11,18 +11,13 @@ source rapids-date-string
 
 export CMAKE_GENERATOR=Ninja
 
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
+source ./ci/use_conda_packages_from_prs.sh
+
 rapids-print-env
 
 rapids-logger "Begin cpp build"
 
-LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp)
-RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python)
-UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp)
-
-RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild \
-    --channel "${LIBRMM_CHANNEL}" \
-    --channel "${RMM_CHANNEL}" \
-    --channel "${UCXX_CHANNEL}" \
-    conda/recipes/libraft
+RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libraft
 
 rapids-upload-conda-to-s3 cpp
diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -11,16 +11,15 @@ source rapids-date-string
 
 export CMAKE_GENERATOR=Ninja
 
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
+source ./ci/use_conda_packages_from_prs.sh
+
 rapids-print-env
 
 rapids-logger "Begin py build"
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
-LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp)
-RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python)
-UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp)
-
 version=$(rapids-generate-version)
 git_commit=$(git rev-parse HEAD)
 export RAPIDS_PACKAGE_VERSION=${version}
@@ -31,28 +30,19 @@ echo "${version}" > VERSION
 rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
-  --channel "${LIBRMM_CHANNEL}" \
-  --channel "${RMM_CHANNEL}" \
-  --channel "${UCXX_CHANNEL}" \
   conda/recipes/pylibraft
 
 rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  --channel "${LIBRMM_CHANNEL}" \
-  --channel "${RMM_CHANNEL}" \
-  --channel "${UCXX_CHANNEL}" \
   conda/recipes/raft-dask
 
 # Build ann-bench for each cuda and python version
 rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  --channel "${LIBRMM_CHANNEL}" \
-  --channel "${RMM_CHANNEL}" \
-  --channel "${UCXX_CHANNEL}" \
   conda/recipes/raft-ann-bench
 
 # Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python
@@ -63,9 +53,6 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  --channel "${LIBRMM_CHANNEL}" \
-  --channel "${RMM_CHANNEL}" \
-  --channel "${UCXX_CHANNEL}" \
   conda/recipes/raft-ann-bench-cpu
 fi
 

diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -18,6 +18,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 rapids-generate-version > VERSION
 
+source ./ci/use_wheels_from_prs.sh
+
 cd "${package_dir}"
 
 case "${RAPIDS_CUDA_VERSION}" in
@@ -39,18 +41,6 @@ case "${RAPIDS_CUDA_VERSION}" in
   ;;
 esac
 
-LIBRMM_CHANNEL=$(
-  RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp
-)
-RMM_CHANNEL=$(
-  RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python
-)
-
-echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${RMM_CHANNEL}/rmm*.whl)" > /tmp/constraints.txt
-echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRMM_CHANNEL}/librmm*.whl)" >> /tmp/constraints.txt
-echo "" >> /tmp/constraints.txt
-export PIP_CONSTRAINT=/tmp/constraints.txt
-
 # Hardcode the output dir
 python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check
 

diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -5,20 +5,14 @@ set -euo pipefail
 
 # Support invoking test_cpp.sh outside the script directory
 cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
+source ./ci/use_conda_packages_from_prs.sh
 
 . /opt/conda/etc/profile.d/conda.sh
 
-LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp)
-RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python)
-UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp)
-
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_cpp \
-  --prepend-channel "${LIBRMM_CHANNEL}" \
-  --prepend-channel "${RMM_CHANNEL}" \
-  --prepend-channel "${UCXX_CHANNEL}" \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
@@ -36,7 +30,6 @@ rapids-print-env
 
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
-  --channel "${UCXX_CHANNEL}" \
   libraft-headers libraft libraft-tests
 
 rapids-logger "Check GPU usage"

diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -5,20 +5,14 @@ set -euo pipefail
 
 # Support invoking test_python.sh outside the script directory
 cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
+source ./ci/use_conda_packages_from_prs.sh
 
 . /opt/conda/etc/profile.d/conda.sh
 
-LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp)
-RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python)
-UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp)
-
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_python \
-  --prepend-channel "${LIBRMM_CHANNEL}" \
-  --prepend-channel "${RMM_CHANNEL}" \
-  --prepend-channel "${UCXX_CHANNEL}" \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
@@ -41,9 +35,6 @@ rapids-print-env
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
-  --channel "${LIBRMM_CHANNEL}" \
-  --channel "${RMM_CHANNEL}" \
-  --channel "${UCXX_CHANNEL}" \
   libraft libraft-headers pylibraft raft-dask
 
 rapids-logger "Check GPU usage"

diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh
@@ -7,17 +7,7 @@ mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
-LIBRMM_CHANNEL=$(
-  RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp
-)
-RMM_CHANNEL=$(
-  RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python
-)
-
-echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${RMM_CHANNEL}/rmm*.whl)" > /tmp/constraints.txt
-echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRMM_CHANNEL}/librmm*.whl)" >> /tmp/constraints.txt
-echo "" >> /tmp/constraints.txt
-export PIP_CONSTRAINT=/tmp/constraints.txt
+source ./ci/use_wheels_from_prs.sh
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install $(echo ./dist/pylibraft*.whl)[test]

diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh
@@ -7,17 +7,7 @@ mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
-LIBRMM_CHANNEL=$(
-  RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp
-)
-RMM_CHANNEL=$(
-  RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python
-)
-
-echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${RMM_CHANNEL}/rmm*.whl)" > /tmp/constraints.txt
-echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRMM_CHANNEL}/librmm*.whl)" >> /tmp/constraints.txt
-echo "" >> /tmp/constraints.txt
-export PIP_CONSTRAINT=/tmp/constraints.txt
+source ./ci/use_wheels_from_prs.sh
 
 # Download the pylibraft built in the previous step
 RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep

diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp)
+RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python)
+
+CUDF_CPP_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp)
+CUDF_PYTHON_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python)
+
+UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp)
+
+conda config --system --add channels "${LIBRMM_CHANNEL}"
+conda config --system --add channels "${RMM_CHANNEL}"
+conda config --system --add channels "${CUDF_CPP_CHANNEL}"
+conda config --system --add channels "${CUDF_PYTHON_CHANNEL}"
+conda config --system --add channels "${UCXX_CHANNEL}"