From 8f918bd31325ba35ff120d603a4112920512d512 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 18 Sep 2024 21:16:47 -0500 Subject: [PATCH] kick off a build --- .github/workflows/pr.yaml | 136 +++++++++++++++--------------- ci/build_cpp.sh | 13 +-- ci/build_python.sh | 19 +---- ci/build_wheel.sh | 14 +-- ci/test_cpp.sh | 9 +- ci/test_python.sh | 11 +-- ci/test_wheel_pylibraft.sh | 12 +-- ci/test_wheel_raft_dask.sh | 12 +-- ci/use_conda_packages_from_prs.sh | 15 ++++ ci/use_wheels_from_prs.sh | 58 +++++++++++++ 10 files changed, 154 insertions(+), 145 deletions(-) create mode 100755 ci/use_conda_packages_from_prs.sh create mode 100755 ci/use_wheels_from_prs.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 381ca6b378..03b38ff6d0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,82 +12,82 @@ concurrency: jobs: pr-builder: needs: - - checks + # - checks - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks + # - conda-cpp-tests + # - conda-cpp-checks - conda-python-build - - conda-python-tests - - docs-build + # - conda-python-tests + # - docs-build - wheel-build-pylibraft - - wheel-tests-pylibraft + # - wheel-tests-pylibraft - wheel-build-raft-dask - - wheel-tests-raft-dask + # - wheel-tests-raft-dask - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 - with: - enable_check_generated_files: false + # checks: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + # with: + # enable_check_generated_files: false conda-cpp-build: - needs: checks + # needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request node_type: cpu16 - conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: raft_cutlass + # conda-cpp-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + # with: + # build_type: pull-request + # conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + # with: + # build_type: pull-request + # enable_check_symbols: true + # symbol_exclusions: raft_cutlass conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request - conda-python-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" + # conda-python-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + # with: + # build_type: pull-request + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/build_docs.sh" wheel-build-pylibraft: - needs: checks + # needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh - wheel-tests-pylibraft: - needs: wheel-build-pylibraft - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - with: - build_type: pull-request - script: ci/test_wheel_pylibraft.sh + # wheel-tests-pylibraft: + # needs: wheel-build-pylibraft + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + # with: + # build_type: pull-request + # script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit @@ -95,20 +95,20 @@ jobs: with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" - wheel-tests-raft-dask: - needs: wheel-build-raft-dask - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - with: - build_type: pull-request - script: ci/test_wheel_raft_dask.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; - sccache -s; + # wheel-tests-raft-dask: + # needs: wheel-build-raft-dask + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + # with: + # build_type: pull-request + # script: ci/test_wheel_raft_dask.sh + # devcontainer: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + # with: + # arch: '["amd64"]' + # cuda: '["12.5"]' + # build_command: | + # sccache -z; + # build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; + # sccache -s; diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 9dce4cee56..b11a99bf56 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -11,18 +11,13 @@ source rapids-date-string export CMAKE_GENERATOR=Ninja +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +source ./ci/use_conda_packages_from_prs.sh + rapids-print-env rapids-logger "Begin cpp build" -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - -RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ - conda/recipes/libraft +RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libraft rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 40902bac3a..c54e998935 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,16 +11,15 @@ source rapids-date-string export CMAKE_GENERATOR=Ninja +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +source ./ci/use_conda_packages_from_prs.sh + rapids-print-env rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) export RAPIDS_PACKAGE_VERSION=${version} @@ -31,18 +30,12 @@ echo "${version}" > VERSION rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ conda/recipes/pylibraft rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ conda/recipes/raft-dask # Build ann-bench for each cuda and python version @@ -50,9 +43,6 @@ rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ conda/recipes/raft-ann-bench # Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python @@ -63,9 +53,6 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ conda/recipes/raft-ann-bench-cpu fi diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 99f22fdd9b..4628afc973 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -18,6 +18,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" rapids-generate-version > VERSION +source ./ci/use_wheels_from_prs.sh + cd "${package_dir}" case "${RAPIDS_CUDA_VERSION}" in @@ -39,18 +41,6 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac -LIBRMM_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp -) -RMM_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python -) - -echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${RMM_CHANNEL}/rmm*.whl)" > /tmp/constraints.txt -echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRMM_CHANNEL}/librmm*.whl)" >> /tmp/constraints.txt -echo "" >> /tmp/constraints.txt -export PIP_CONSTRAINT=/tmp/constraints.txt - # Hardcode the output dir python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index be990c026a..e819bf61d6 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -5,20 +5,14 @@ set -euo pipefail # Support invoking test_cpp.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +source ./ci/use_conda_packages_from_prs.sh . /opt/conda/etc/profile.d/conda.sh -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ - --prepend-channel "${LIBRMM_CHANNEL}" \ - --prepend-channel "${RMM_CHANNEL}" \ - --prepend-channel "${UCXX_CHANNEL}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test @@ -36,7 +30,6 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ libraft-headers libraft libraft-tests rapids-logger "Check GPU usage" diff --git a/ci/test_python.sh b/ci/test_python.sh index 34c81b04d6..cd94103498 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -5,20 +5,14 @@ set -euo pipefail # Support invoking test_python.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +source ./ci/use_conda_packages_from_prs.sh . /opt/conda/etc/profile.d/conda.sh -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_python \ - --prepend-channel "${LIBRMM_CHANNEL}" \ - --prepend-channel "${RMM_CHANNEL}" \ - --prepend-channel "${UCXX_CHANNEL}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test @@ -41,9 +35,6 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ libraft libraft-headers pylibraft raft-dask rapids-logger "Check GPU usage" diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 9ac1db3dfe..3fd4225526 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -7,17 +7,7 @@ mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -LIBRMM_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp -) -RMM_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python -) - -echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${RMM_CHANNEL}/rmm*.whl)" > /tmp/constraints.txt -echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRMM_CHANNEL}/librmm*.whl)" >> /tmp/constraints.txt -echo "" >> /tmp/constraints.txt -export PIP_CONSTRAINT=/tmp/constraints.txt +source ./ci/use_wheels_from_prs.sh # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index 7fbacaa306..6ce87bceb5 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -7,17 +7,7 @@ mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -LIBRMM_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp -) -RMM_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python -) - -echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${RMM_CHANNEL}/rmm*.whl)" > /tmp/constraints.txt -echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRMM_CHANNEL}/librmm*.whl)" >> /tmp/constraints.txt -echo "" >> /tmp/constraints.txt -export PIP_CONSTRAINT=/tmp/constraints.txt +source ./ci/use_wheels_from_prs.sh # Download the pylibraft built in the previous step RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh new file mode 100755 index 0000000000..1fee3fcb48 --- /dev/null +++ b/ci/use_conda_packages_from_prs.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) + +CUDF_CPP_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) +CUDF_PYTHON_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) + +UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) + +conda config --system --add channels "${LIBRMM_CHANNEL}" +conda config --system --add channels "${RMM_CHANNEL}" +conda config --system --add channels "${CUDF_CPP_CHANNEL}" +conda config --system --add channels "${CUDF_PYTHON_CHANNEL}" +conda config --system --add channels "${UCXX_CHANNEL}" diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh new file mode 100755 index 0000000000..52e3e1d485 --- /dev/null +++ b/ci/use_wheels_from_prs.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +LIBRMM_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp +) +RMM_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python +) + +UCXX_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python +) +LIBUCXX_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=libucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 cpp +) +DISTRIBUTED_UCXX_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python +) + +CUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=cudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python +) +LIBCUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=libcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 cpp +) +PYLIBCUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=pylibcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python +) +DASK_CUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=dask_cudf_${RAPIDS_PY_CUDA_SUFFIX} \ + RAPIDS_PY_WHEEL_PURE=1 \ + rapids-get-pr-wheel-artifact cudf 16806 python +) + +# RAFT_DASK_CHANNEL=$( +# RAPIDS_PY_WHEEL_NAME=raft_dask_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python +# ) +# PYLIBRAFT_CHANNEL=$( +# RAPIDS_PY_WHEEL_NAME=pylibraft_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python +# ) + +cat > /tmp/constraints.txt <