build wheels without build isolation

.github/workflows/pr.yaml

@@ -12,132 +12,14 @@ concurrency:
 jobs:
   pr-builder:
     needs:
-      - changed-files
-      - checks
-      - conda-cpp-build
-      - conda-cpp-tests
-      - conda-cpp-checks
-      - conda-notebook-tests
-      - conda-python-build
-      - conda-python-tests
-      - docs-build
       - wheel-build-pylibcugraph
-      - wheel-tests-pylibcugraph
       - wheel-build-cugraph
-      - wheel-tests-cugraph
       - wheel-build-nx-cugraph
-      - wheel-tests-nx-cugraph
       - wheel-build-cugraph-dgl
-      - wheel-tests-cugraph-dgl
       - wheel-build-cugraph-pyg
-      - wheel-tests-cugraph-pyg
       - wheel-build-cugraph-equivariant
-      - wheel-tests-cugraph-equivariant
-      - devcontainer
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: always()
-    with:
-      needs: ${{ toJSON(needs) }}
-  changed-files:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      files_yaml: |
-        test_cpp:
-          - '**'
-          - '!.devcontainers/**'
-          - '!CONTRIBUTING.md'
-          - '!README.md'
-          - '!docs/**'
-          - '!img/**'
-          - '!mg_utils/**'
-          - '!notebooks/**'
-          - '!python/**'
-          - '!readme_pages/**'
-          # TODO: Remove this before merging
-          - '!.github/**'
-        test_notebooks:
-          - '**'
-          - '!.devcontainers/**'
-          - '!CONTRIBUTING.md'
-          - '!README.md'
-          - '!docs/**'
-          # TODO: Remove this before merging
-          - '!.github/**'
-        test_python:
-          - '**'
-          - '!.devcontainers/**'
-          - '!CONTRIBUTING.md'
-          - '!README.md'
-          - '!docs/**'
-          - '!img/**'
-          - '!notebooks/**'
-          # TODO: Remove this before merging
-          - '!.github/**'
-  checks:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      enable_check_generated_files: false
-  conda-cpp-build:
-    needs: checks
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      node_type: cpu32
-  conda-cpp-tests:
-    needs: [conda-cpp-build, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
-    with:
-      build_type: pull-request
-  conda-cpp-checks:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      enable_check_symbols: true
-      symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel)
-  conda-python-build:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-  conda-python-tests:
-    needs: [conda-python-build, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-  conda-notebook-tests:
-    needs: [conda-python-build, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
-      run_script: "ci/test_notebooks.sh"
-  docs-build:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
-      run_script: "ci/build_docs.sh"
   wheel-build-pylibcugraph:
-    needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
@@ -147,16 +29,8 @@ jobs:
       extra-repo-sha: branch-24.12
       extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
       node_type: cpu32
-  wheel-tests-pylibcugraph:
-    needs: [wheel-build-pylibcugraph, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_pylibcugraph.sh
   wheel-build-cugraph:
-    needs: wheel-tests-pylibcugraph
+    needs: wheel-build-pylibcugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
@@ -165,85 +39,30 @@ jobs:
       extra-repo: rapidsai/cugraph-ops
       extra-repo-sha: branch-24.12
       extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
-  wheel-tests-cugraph:
-    needs: [wheel-build-cugraph, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_cugraph.sh
   wheel-build-nx-cugraph:
-    needs: wheel-tests-pylibcugraph
+    needs: wheel-build-cugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_nx-cugraph.sh
-  wheel-tests-nx-cugraph:
-    needs: [wheel-build-nx-cugraph, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_nx-cugraph.sh
   wheel-build-cugraph-dgl:
-    needs: wheel-tests-cugraph
+    needs: wheel-build-cugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_cugraph-dgl.sh
-  wheel-tests-cugraph-dgl:
-    needs: [wheel-build-cugraph-dgl, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_cugraph-dgl.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
   wheel-build-cugraph-pyg:
-    needs: wheel-tests-cugraph
+    needs: wheel-build-cugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_cugraph-pyg.sh
-  wheel-tests-cugraph-pyg:
-    needs: [wheel-build-cugraph-pyg, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_cugraph-pyg.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
   wheel-build-cugraph-equivariant:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_cugraph-equivariant.sh
-  wheel-tests-cugraph-equivariant:
-    needs: [wheel-build-cugraph-equivariant, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_cugraph-equivariant.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
-  devcontainer:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      arch: '["amd64"]'
-      cuda: '["12.5"]'
-      node_type: cpu32
-      extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
-      build_command: |
-        sccache -z;
-        build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON;
-        sccache -s;

ci/build_wheel.sh

@@ -11,18 +11,52 @@ source rapids-date-string
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
+touch /tmp/requirements-build.txt
+
+if [[ "${package_name}" == "cuspatial" ]]; then
+    # Download the pylibcugraph wheel built in the previous step and ensure
+    # it's installed in the build environment.
+    PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcugraph_dist)
+    echo ${PYLIBCUGRAPH_WHEELHOUSE}/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl >> /tmp/requirements-build.txt
+fi
+
+rapids-logger "Generating build requirements"
+declare -r matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true"
+
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key "py_build_${package_name/-/_}" \
+  --matrix "${matrix_selectors}" \
+| tee -a /tmp/requirements-build.txt
+
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key "py_rapids_build_${package_name/-/_}" \
+  --matrix "${matrix_selectors}" \
+| tee -a /tmp/requirements-build.txt
+
+rapids-logger "Installing build requirements"
+python -m pip install \
+    -v \
+    --prefer-binary \
+    -r /tmp/requirements-build.txt
+
 rapids-generate-version > ./VERSION
 
 cd "${package_dir}"
 
+rapids-logger "Building '${package_name}' wheel"
 python -m pip wheel \
     -w dist \
     -v \
+    --no-build-isolation \
     --no-deps \
     --disable-pip-version-check \
     --extra-index-url https://pypi.nvidia.com \
     .
 
+sccache --show-adv-stats
+
 # pure-python packages should be marked as pure, and not have auditwheel run on them.
 if [[ ${package_name} == "nx-cugraph" ]] || \
    [[ ${package_name} == "cugraph-dgl" ]] || \

ci/build_wheel_cugraph.sh

@@ -5,18 +5,6 @@ set -euo pipefail
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download the pylibcugraph wheel built in the previous step and make it
-# available for pip to find.
-#
-# ensure 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run
-#
-# using env variable PIP_CONSTRAINT is necessary to ensure the constraints
-# are used when creating the isolated build environment
-CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcugraph_dist)
-
-echo "pylibcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/pylibcugraph_*.whl)" > ./constraints.txt
-export PIP_CONSTRAINT="${PWD}/constraints.txt"
-
 PARALLEL_LEVEL=$(python -c \
   "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))")
 case "${RAPIDS_CUDA_VERSION}" in

dependencies.yaml

@@ -158,6 +158,13 @@ files:
     includes:
       - python_build_rapids
       - python_build_wheel
+  py_rapids_build_nx_cugraph:
+    output: pyproject
+    pyproject_dir: python/nx-cugraph
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes: []
   py_run_nx_cugraph:
     output: pyproject
     pyproject_dir: python/nx-cugraph
@@ -184,6 +191,13 @@ files:
     includes:
       - python_build_rapids
       - python_build_wheel
+  py_rapids_build_cugraph_dgl:
+    output: pyproject
+    pyproject_dir: python/cugraph-dgl
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes: []
   py_run_cugraph_dgl:
     output: pyproject
     pyproject_dir: python/cugraph-dgl
@@ -211,6 +225,13 @@ files:
     includes:
       - python_build_rapids
       - python_build_wheel
+  py_rapids_build_cugraph_pyg:
+    output: pyproject
+    pyproject_dir: python/cugraph-pyg
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes: []
   py_run_cugraph_pyg:
     output: pyproject
     pyproject_dir: python/cugraph-pyg
@@ -238,6 +259,13 @@ files:
     includes:
       - python_build_rapids
       - python_build_wheel
+  py_rapids_build_cugraph_equivariant:
+    output: pyproject
+    pyproject_dir: python/cugraph-equivariant
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes: []
   py_run_cugraph_equivariant:
     output: pyproject
     pyproject_dir: python/cugraph-equivariant
@@ -262,6 +290,13 @@ files:
     includes:
       - python_build_rapids
       - python_build_wheel
+  py_rapids_build_cugraph_service_client:
+    output: pyproject
+    pyproject_dir: python/cugraph-service/client
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes: []
   py_run_cugraph_service_client:
     output: pyproject
     pyproject_dir: python/cugraph-service/client
@@ -277,6 +312,13 @@ files:
     includes:
       - python_build_rapids
       - python_build_wheel
+  py_rapids_build_cugraph_service_server:
+    output: pyproject
+    pyproject_dir: python/cugraph-service/server
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes: []
   py_run_cugraph_service_server:
     output: pyproject
     pyproject_dir: python/cugraph-service/server