diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 05ee9412..8ef3c48c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,18 +12,18 @@ concurrency: jobs: pr-builder: needs: - - checks + # - checks - conda-python-build - conda-python-tests - conda-notebook-tests - - docs-build + # - docs-build - wheel-build - wheel-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + # checks: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 conda-python-build: needs: checks secrets: inherit @@ -47,16 +47,16 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/build_docs.sh" wheel-build: needs: checks secrets: inherit @@ -66,12 +66,12 @@ jobs: script: ci/build_wheel.sh # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - wheel-tests: - needs: wheel-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - with: - build_type: pull-request - script: ci/test_wheel.sh - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] + # wheel-tests: + # needs: wheel-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + # with: + # build_type: pull-request + # script: ci/test_wheel.sh + # # This selects "ARCH=amd64 + the latest supported Python + CUDA". + # matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] diff --git a/ci/build_python.sh b/ci/build_python.sh index 9c0bbc83..1c1d7747 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -9,6 +9,8 @@ source rapids-configure-sccache source rapids-date-string +source ./ci/use_conda_packages_from_prs.sh + rapids-print-env rapids-generate-version > ./VERSION diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 78e2bed2..87595e0f 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -13,6 +13,8 @@ rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +source ./ci/use_wheels_from_prs.sh + cd "${package_dir}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 730129f0..389c63c6 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -4,6 +4,7 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +source ./ci/use_conda_packages_from_prs.sh rapids-logger "Generate notebook testing dependencies" rapids-dependency-file-generator \ diff --git a/ci/test_python.sh b/ci/test_python.sh index 0e7c1f20..94314ca2 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -4,6 +4,7 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +source ./ci/use_conda_packages_from_prs.sh rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index c1253fe9..2b775ff4 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -6,6 +6,8 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cuxfilter_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist +source ./ci/use_wheels_from_prs.sh + # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/cuxfilter*.whl)[test] diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh new file mode 100755 index 00000000..51edd6ad --- /dev/null +++ b/ci/use_conda_packages_from_prs.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) + +CUDF_CPP_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) +CUDF_PYTHON_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) + +# UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) + +# LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) +# RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) + +LIBCUSPATIAL_CHANNEL=$(rapids-get-pr-conda-artifact cuspatial 1441 cpp) +CUSPATIAL_CHANNEL=$(rapids-get-pr-conda-artifact cuspatial 1441 python) + +conda config --system --add channels "${LIBRMM_CHANNEL}" +conda config --system --add channels "${RMM_CHANNEL}" +conda config --system --add channels "${CUDF_CPP_CHANNEL}" +conda config --system --add channels "${CUDF_PYTHON_CHANNEL}" +# conda config --system --add channels "${UCXX_CHANNEL}" +# conda config --system --add channels "${LIBRAFT_CHANNEL}" +# conda config --system --add channels "${RAFT_CHANNEL}" +conda config --system --add channels "${LIBCUSPATIAL_CHANNEL}" +conda config --system --add channels "${CUSPATIAL_CHANNEL}" diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh new file mode 100755 index 00000000..3fa22a2f --- /dev/null +++ b/ci/use_wheels_from_prs.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +LIBRMM_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp +) +RMM_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python +) + +# UCXX_CHANNEL=$( +# RAPIDS_PY_WHEEL_NAME=ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python +# ) +# LIBUCXX_CHANNEL=$( +# RAPIDS_PY_WHEEL_NAME=libucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 cpp +# ) +# DISTRIBUTED_UCXX_CHANNEL=$( +# RAPIDS_PY_WHEEL_NAME=distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python +# ) + +CUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=cudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python +) +LIBCUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=libcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 cpp +) +PYLIBCUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=pylibcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python +) +DASK_CUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=dask_cudf_${RAPIDS_PY_CUDA_SUFFIX} \ + RAPIDS_PY_WHEEL_PURE=1 \ + rapids-get-pr-wheel-artifact cudf 16806 python +) + +# RAFT_DASK_CHANNEL=$( +# RAPIDS_PY_WHEEL_NAME=raft_dask_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python +# ) +# PYLIBRAFT_CHANNEL=$( +# RAPIDS_PY_WHEEL_NAME=pylibraft_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python +# ) + +CUSPATIAL_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=cuspatial_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cuspatial 1441 python +) +CUPROJ_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=cuproj_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cuproj 1441 python +) +LIBCUSPATIAL_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=libcuspatial_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cuspatial 1441 cpp +) + +cat > /tmp/constraints.txt <