add PR CI for cugraph-pyg and cugraph-dgl (#59)

Another steps towards completing the work started in #53 Fixes #15 Contributes to rapidsai/build-planning#111 Proposes changes to get CI running on pull requests for `cugraph-pyg` and `cugraph-dgl` ## Notes for Reviewers Workflows for nightly builds and publishing nightly packages are intentionally not included here. See #58 (comment) Notebook tests are intentionally not added here... they'll be added in the next PR. Pulls in changes from these other upstream PRs that had not been ported over to this repo: * rapidsai/cugraph#4690 * rapidsai/cugraph#4393 Authors: - James Lamb (https://github.com/jameslamb) - Alex Barghi (https://github.com/alexbarghi-nv) Approvers: - Alex Barghi (https://github.com/alexbarghi-nv) - Bradley Dice (https://github.com/bdice) URL: #59
rapidsai · Oct 30, 2024 · 31ee98f · 31ee98f
1 parent 25d1b55
commit 31ee98f
Show file tree

Hide file tree

Showing 43 changed files with 460 additions and 211 deletions.
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -20,6 +20,10 @@ jobs:
       - conda-python-tests
       - wheel-build-pylibwholegraph
       - wheel-tests-pylibwholegraph
+      - wheel-build-cugraph-dgl
+      - wheel-tests-cugraph-dgl
+      - wheel-build-cugraph-pyg
+      - wheel-tests-cugraph-pyg
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: always()
@@ -104,3 +108,35 @@ jobs:
       build_type: pull-request
       script: ci/test_wheel_pylibwholegraph.sh
       matrix_filter: map(select(.ARCH == "amd64"))
+  wheel-build-cugraph-dgl:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_cugraph-dgl.sh
+  wheel-tests-cugraph-dgl:
+    needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-dgl, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_cugraph-dgl.sh
+      matrix_filter: map(select(.ARCH == "amd64"))
+  wheel-build-cugraph-pyg:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_cugraph-pyg.sh
+  wheel-tests-cugraph-pyg:
+    needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-pyg, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_cugraph-pyg.sh
+      matrix_filter: map(select(.ARCH == "amd64"))
diff --git a/.gitignore b/.gitignore
@@ -70,9 +70,10 @@ cpp/thirdparty/googletest/
 *.iws
 
 ## Datasets
-datasets/*
+datasets/
 !datasets/cyber.csv
 !datasets/get_test_data.sh
+!datasets/karate.csv
 !datasets/karate-data.csv
 !datasets/karate_undirected.csv
 !datasets/karate-disjoint.csv

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -54,7 +54,7 @@ repos:
               setup[.]cfg$
       - id: verify-alpha-spec
   - repo: https://github.com/rapidsai/dependency-file-generator
-    rev: v1.15.1
+    rev: v1.16.0
     hooks:
         - id: rapids-dependency-file-generator
           args: ["--clean"]
diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -17,6 +17,10 @@ version=$(rapids-generate-version)
 
 rapids-logger "Begin cpp build"
 
+sccache --zero-stats
+
 RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libwholegraph
 
+sccache --show-adv-stats
+
 rapids-upload-conda-to-s3 cpp
diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -17,6 +17,8 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 rapids-generate-version > ./VERSION
 
+sccache --zero-stats
+
 # TODO: Remove `--no-test` flags once importing on a CPU
 # node works correctly
 rapids-logger "Begin pylibwholegraph build"
@@ -25,4 +27,16 @@ RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/pylibwholegraph
 
+sccache --show-adv-stats
+
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  conda/recipes/cugraph-pyg
+
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  conda/recipes/cugraph-dgl
+
 rapids-upload-conda-to-s3 python
diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -32,6 +32,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 cd "${package_dir}"
 
+sccache --zero-stats
+
 rapids-logger "Building '${package_name}' wheel"
 python -m pip wheel \
     -w dist \

diff --git a/ci/run_cugraph_dgl_pytests.sh b/ci/run_cugraph_dgl_pytests.sh
@@ -4,6 +4,6 @@
 set -euo pipefail
 
 # Support invoking run_cugraph_dgl_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-dgl/tests
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-dgl/cugraph_dgl
 
 pytest --cache-clear --ignore=mg "$@" .
diff --git a/ci/run_cugraph_pyg_pytests.sh b/ci/run_cugraph_pyg_pytests.sh
@@ -9,7 +9,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-pyg/cugraph_
 pytest --cache-clear --benchmark-disable "$@" .
 
 # Used to skip certain examples in CI due to memory limitations
-export CI_RUN=1
+export CI=true
 
 # Test examples
 for e in "$(pwd)"/examples/*.py; do

diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh
@@ -4,24 +4,15 @@
 set -eoxu pipefail
 
 package_name="cugraph-dgl"
-package_dir="python/cugraph-dgl"
-
-python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download wheels built during this job.
-RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-python -m pip install ./local-deps/*.whl
-
-# use 'ls' to expand wildcard before adding `[extra]` requires for pip
+# Download the pylibwholegraph and cugraph-dgl built in the previous step
+RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
-# pip creates wheels using python package names
-python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
-
 
+# determine pytorch and DGL sources
 PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')"
 PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2}
 if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then
@@ -30,20 +21,19 @@ else
   PYTORCH_CUDA_VER=$PKG_CUDA_VER
 fi
 PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}"
-DGL_URL="https://data.dgl.ai/wheels/cu${PYTORCH_CUDA_VER}/repo.html"
-
-# Starting from 2.2, PyTorch wheels depend on nvidia-nccl-cuxx>=2.19 wheel and
-# dynamically link to NCCL. RAPIDS CUDA 11 CI images have an older NCCL version that
-# might shadow the newer NCCL required by PyTorch during import (when importing
-# `cupy` before `torch`).
-if [[ "${NCCL_VERSION}" < "2.19" ]]; then
-  PYTORCH_VER="2.1.0"
-else
-  PYTORCH_VER="2.3.0"
-fi
-
-rapids-logger "Installing PyTorch and DGL"
-rapids-retry python -m pip install "torch==${PYTORCH_VER}" --index-url ${PYTORCH_URL}
-rapids-retry python -m pip install dgl==2.0.0 --find-links ${DGL_URL}
-
-python -m pytest python/cugraph-dgl/tests
+DGL_URL="https://data.dgl.ai/wheels/torch-2.3/cu${PYTORCH_CUDA_VER}/repo.html"
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    -v \
+    --extra-index-url "${PYTORCH_URL}" \
+    --find-links "${DGL_URL}" \
+    "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+    'dgl==2.4.0' \
+    'torch>=2.0,<2.4.0a0'
+
+# RAPIDS_DATASET_ROOT_DIR is used by test scripts
+export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
+
+python -m pytest python/cugraph-dgl/cugraph_dgl/tests
diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh
@@ -4,46 +4,33 @@
 set -eoxu pipefail
 
 package_name="cugraph-pyg"
-package_dir="python/cugraph-pyg"
-
-python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download wheels built during this job.
-RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-python -m pip install ./local-deps/*.whl
-
-# use 'ls' to expand wildcard before adding `[extra]` requires for pip
+# Download the pylibwholegraph and cugraph-pyg built in the previous step
+RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
-# pip creates wheels using python package names
-python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
-
-# RAPIDS_DATASET_ROOT_DIR is used by test scripts
-export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
-
-# Used to skip certain examples in CI due to memory limitations
-export CI_RUN=1
 
+# determine pytorch and pyg sources
 if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then
   PYTORCH_URL="https://download.pytorch.org/whl/cu118"
-  PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html"
+  PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html"
 else
   PYTORCH_URL="https://download.pytorch.org/whl/cu121"
-  PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html"
+  PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html"
 fi
-rapids-logger "Installing PyTorch and PyG dependencies"
-rapids-retry python -m pip install torch==2.1.0 --index-url ${PYTORCH_URL}
-rapids-retry python -m pip install "torch-geometric>=2.5,<2.6"
-rapids-retry python -m pip install \
-  ogb \
-  pyg_lib \
-  torch_scatter \
-  torch_sparse \
-  tensordict \
-  -f ${PYG_URL}
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    -v \
+    --extra-index-url "${PYTORCH_URL}" \
+    --find-links "${PYG_URL}" \
+    "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
+
+# RAPIDS_DATASET_ROOT_DIR is used by test scripts
+export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
 
 rapids-logger "pytest cugraph-pyg (single GPU)"
 pushd python/cugraph-pyg/cugraph_pyg

diff --git a/ci/test_wheel_pylibwholegraph.sh b/ci/test_wheel_pylibwholegraph.sh
@@ -25,13 +25,8 @@ mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
 rapids-logger "Installing Packages"
 rapids-retry python -m pip install \
     --extra-index-url ${INDEX_URL} \
-    "$(echo ./dist/pylibwholegraph*.whl)[test]"
-
-# install torch separately, to be sure we get a CUDA build
-python -m pip install \
-  --index-url "${INDEX_URL}" \
-  -v \
-  'torch>=2.0,<2.4.0a0'
+    "$(echo ./dist/pylibwholegraph*.whl)[test]" \
+    'torch>=2.0,<2.4.0a0'
 
 rapids-logger "pytest pylibwholegraph"
 cd python/pylibwholegraph/pylibwholegraph/tests

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -4,7 +4,6 @@ channels:
 - rapidsai
 - rapidsai-nightly
 - dask/label/dev
-- pyg
 - conda-forge
 - nvidia
 dependencies:
@@ -18,7 +17,7 @@ dependencies:
 - cupy>=12.0.0
 - cython>=3.0.0
 - dask-cudf==24.12.*,>=0.0.0a0
-- dglteam/label/th21_cu118::dgl
+- dglteam/label/th23_cu118::dgl>=2.4.0.th23.cu*
 - doxygen
 - graphviz
 - ipython
@@ -33,7 +32,6 @@ dependencies:
 - pre-commit
 - pydantic
 - pydata-sphinx-theme
-- pyg::pyg
 - pylibcugraphops==24.12.*,>=0.0.0a0
 - pylibraft==24.12.*,>=0.0.0a0
 - pytest
@@ -42,16 +40,20 @@ dependencies:
 - pytest-forked
 - pytest-xdist
 - pytorch-cuda=11.8
-- pytorch::pytorch>=2.0,<2.4.0a0
+- pytorch::pytorch>=2.3,<2.4.0a0
+- pytorch_geometric>=2.5,<2.6
 - raft-dask==24.12.*,>=0.0.0a0
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
 - rmm==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - scipy
+- setuptools>=61.0.0
 - sphinx-copybutton
 - sphinx-markdown-tables
 - sphinx<6
 - sphinxcontrib-websupport
 - torchdata
 - wget
+- wheel
 name: all_cuda-118_arch-x86_64
diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml
@@ -4,7 +4,6 @@ channels:
 - rapidsai
 - rapidsai-nightly
 - dask/label/dev
-- pyg
 - conda-forge
 - nvidia
 dependencies:
@@ -19,7 +18,7 @@ dependencies:
 - cupy>=12.0.0
 - cython>=3.0.0
 - dask-cudf==24.12.*,>=0.0.0a0
-- dglteam/label/th21_cu121::dgl
+- dglteam/label/th23_cu121::dgl>=2.4.0.th23.cu*
 - doxygen
 - graphviz
 - ipython
@@ -38,7 +37,6 @@ dependencies:
 - pre-commit
 - pydantic
 - pydata-sphinx-theme
-- pyg::pyg
 - pylibcugraphops==24.12.*,>=0.0.0a0
 - pylibraft==24.12.*,>=0.0.0a0
 - pytest
@@ -47,16 +45,20 @@ dependencies:
 - pytest-forked
 - pytest-xdist
 - pytorch-cuda=12.1
-- pytorch::pytorch>=2.0,<2.4.0a0
+- pytorch::pytorch>=2.3,<2.4.0a0
+- pytorch_geometric>=2.5,<2.6
 - raft-dask==24.12.*,>=0.0.0a0
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
 - rmm==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - scipy
+- setuptools>=61.0.0
 - sphinx-copybutton
 - sphinx-markdown-tables
 - sphinx<6
 - sphinxcontrib-websupport
 - torchdata
 - wget
+- wheel
 name: all_cuda-121_arch-x86_64