test with cuspatial PR artifacts

.github/workflows/pr.yaml

@@ -12,18 +12,18 @@ concurrency:
 jobs:
   pr-builder:
     needs:
-      - checks
+      # - checks
       - conda-python-build
       - conda-python-tests
       - conda-notebook-tests
-      - docs-build
+      # - docs-build
       - wheel-build
       - wheel-tests
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-  checks:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  # checks:
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
   conda-python-build:
     needs: checks
     secrets: inherit
@@ -47,16 +47,16 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/test_notebooks.sh"
-  docs-build:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/build_docs.sh"
+  # docs-build:
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:latest"
+  #     run_script: "ci/build_docs.sh"
   wheel-build:
     needs: checks
     secrets: inherit
@@ -66,12 +66,12 @@ jobs:
       script: ci/build_wheel.sh
       # This selects "ARCH=amd64 + the latest supported Python + CUDA".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
-  wheel-tests:
-    needs: wheel-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/test_wheel.sh
-      # This selects "ARCH=amd64 + the latest supported Python + CUDA".
-      matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.]
+  # wheel-tests:
+  #   needs: wheel-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/test_wheel.sh
+  #     # This selects "ARCH=amd64 + the latest supported Python + CUDA".
+  #     matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.]

ci/build_python.sh

@@ -9,6 +9,8 @@ source rapids-configure-sccache
 
 source rapids-date-string
 
+source ./ci/use_conda_packages_from_prs.sh
+
 rapids-print-env
 
 rapids-generate-version > ./VERSION

ci/build_wheel.sh

@@ -13,6 +13,8 @@ rapids-generate-version > ./VERSION
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
+source ./ci/use_wheels_from_prs.sh
+
 cd "${package_dir}"
 
 python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check

ci/test_notebooks.sh

@@ -4,6 +4,7 @@
 set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
+source ./ci/use_conda_packages_from_prs.sh
 
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \

ci/test_python.sh

@@ -4,6 +4,7 @@
 set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
+source ./ci/use_conda_packages_from_prs.sh
 
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \

ci/test_wheel.sh

@@ -6,6 +6,8 @@ set -eou pipefail
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="cuxfilter_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
 
+source ./ci/use_wheels_from_prs.sh
+
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install $(echo ./dist/cuxfilter*.whl)[test]
 

ci/use_conda_packages_from_prs.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp)
+RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python)
+
+CUDF_CPP_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp)
+CUDF_PYTHON_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python)
+
+# UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp)
+
+# LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp)
+# RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python)
+
+LIBCUSPATIAL_CHANNEL=$(rapids-get-pr-conda-artifact cuspatial 1441 cpp)
+CUSPATIAL_CHANNEL=$(rapids-get-pr-conda-artifact cuspatial 1441 python)
+
+conda config --system --add channels "${LIBRMM_CHANNEL}"
+conda config --system --add channels "${RMM_CHANNEL}"
+conda config --system --add channels "${CUDF_CPP_CHANNEL}"
+conda config --system --add channels "${CUDF_PYTHON_CHANNEL}"
+# conda config --system --add channels "${UCXX_CHANNEL}"
+# conda config --system --add channels "${LIBRAFT_CHANNEL}"
+# conda config --system --add channels "${RAFT_CHANNEL}"
+conda config --system --add channels "${LIBCUSPATIAL_CHANNEL}"
+conda config --system --add channels "${CUSPATIAL_CHANNEL}"

ci/use_wheels_from_prs.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+LIBRMM_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp
+)
+RMM_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python
+)
+
+# UCXX_CHANNEL=$(
+#   RAPIDS_PY_WHEEL_NAME=ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python
+# )
+# LIBUCXX_CHANNEL=$(
+#   RAPIDS_PY_WHEEL_NAME=libucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 cpp
+# )
+# DISTRIBUTED_UCXX_CHANNEL=$(
+#   RAPIDS_PY_WHEEL_NAME=distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python
+# )
+
+CUDF_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=cudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python
+)
+LIBCUDF_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=libcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 cpp
+)
+PYLIBCUDF_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=pylibcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python
+)
+DASK_CUDF_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=dask_cudf_${RAPIDS_PY_CUDA_SUFFIX} \
+  RAPIDS_PY_WHEEL_PURE=1 \
+    rapids-get-pr-wheel-artifact cudf 16806 python
+)
+
+# RAFT_DASK_CHANNEL=$(
+#   RAPIDS_PY_WHEEL_NAME=raft_dask_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python
+# )
+# PYLIBRAFT_CHANNEL=$(
+#   RAPIDS_PY_WHEEL_NAME=pylibraft_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python
+# )
+
+CUSPATIAL_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=cuspatial_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cuspatial 1441 python
+)
+CUPROJ_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=cuproj_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cuproj 1441 python
+)
+LIBCUSPATIAL_CHANNEL=$(
+  RAPIDS_PY_WHEEL_NAME=libcuspatial_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cuspatial 1441 cpp
+)
+
+cat > /tmp/constraints.txt <<EOF
+librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBRMM_CHANNEL}/librmm_*.whl)
+rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${RMM_CHANNEL}/rmm_*.whl)
+cudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CUDF_CHANNEL}/cudf_*.whl)
+libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_CHANNEL}/libcudf_*.whl)
+pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_CHANNEL}/pylibcudf_*.whl)
+dask-cudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${DASK_CUDF_CHANNEL}/dask_cudf_*.whl)
+cuspatial-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CUSPATIAL_CHANNEL}/cuspatial_*.whl)
+libcuspatial-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUSPATIAL_CHANNEL}/libcuspatial_*.whl)
+cuproj-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CUPROJ_CHANNEL}/cuproj_*.whl)
+EOF
+
+# ucxx-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${UCXX_CHANNEL}/ucxx_*.whl)
+# libucxx-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBUCXX_CHANNEL}/libucxx_*.whl)
+# distributed-ucxx-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${DISTRIBUTED_UCXX_CHANNEL}/distributed_ucxx_*.whl)
+# raft-dask-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${RAFT_DASK_CHANNEL}/raft_dask_*.whl)
+# pylibraft-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBRAFT_CHANNEL}/pylibraft_*.whl)
+export PIP_CONSTRAINT=/tmp/constraints.txt