add full CI for wholegraph

.github/workflows/build.yaml

@@ -0,0 +1,75 @@
+name: build
+
+on:
+  push:
+    branches:
+      - "branch-*"
+    tags:
+      - v[0-9][0-9].[0-9][0-9].[0-9][0-9]
+  workflow_dispatch:
+    inputs:
+      branch:
+        required: true
+        type: string
+      date:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+      build_type:
+        type: string
+        default: nightly
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  cpp-build:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      node_type: cpu32
+  python-build:
+    needs: [cpp-build]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+  upload-conda:
+    needs: [cpp-build, python-build]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+  wheel-build-pylibwholegraph:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_pylibwholegraph.sh
+  wheel-publish-pylibwholegraph:
+    needs: wheel-build-pylibwholegraph
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: pylibwholegraph
+

.github/workflows/pr.yaml

@@ -14,6 +14,13 @@ jobs:
     needs:
       - changed-files
       - checks
+      - conda-cpp-build
+      - conda-cpp-tests
+      - conda-notebook-tests
+      - conda-python-build
+      - conda-python-tests
+      - wheel-build-pylibwholegraph
+      - wheel-tests-pylibwholegraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: always()
@@ -54,3 +61,47 @@ jobs:
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       enable_check_generated_files: false
+  conda-cpp-build:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      node_type: cpu32
+  conda-cpp-tests:
+    needs: [conda-cpp-build, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
+    with:
+      build_type: pull-request
+  conda-python-build:
+    needs: conda-cpp-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+  conda-python-tests:
+    needs: [conda-python-build, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+  wheel-build-pylibwholegraph:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_pylibwholegraph.sh
+      node_type: cpu32
+  wheel-tests-pylibwholegraph:
+    needs: [wheel-build-pylibwholegraph, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_pylibwholegraph.sh
+      matrix_filter: map(select(.ARCH == "amd64"))

.github/workflows/test.yaml

@@ -0,0 +1,43 @@
+name: test
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        required: true
+        type: string
+      date:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+
+jobs:
+  conda-cpp-tests:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+  conda-python-tests:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+  wheel-tests-pylibwholegraph:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      script: ci/test_wheel_pylibwholegraph.sh
+      matrix_filter: map(select(.ARCH == "amd64"))
+

ci/build_cpp.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+rapids-configure-conda-channels
+
+source rapids-configure-sccache
+
+source rapids-date-string
+
+export CMAKE_GENERATOR=Ninja
+
+rapids-print-env
+
+version=$(rapids-generate-version)
+
+rapids-logger "Begin cpp build"
+
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libwholegraph
+
+rapids-upload-conda-to-s3 cpp

ci/build_python.sh

@@ -15,41 +15,14 @@ rapids-print-env
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
-export RAPIDS_PACKAGE_VERSION=${version}
-echo "${version}" > VERSION
-
-rapids-logger "Begin py build"
-
-package_dir="python"
-for package_name in cugraph-pyg cugraph-dgl; do
-  underscore_package_name=$(echo "${package_name}" | tr "-" "_")
-  sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py"
-done
-
-RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
-
-if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
-  # Only CUDA 11 is supported right now due to PyTorch requirement.
-  rapids-conda-retry mambabuild \
-    --no-test \
-    --channel "${CPP_CHANNEL}" \
-    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-    --channel pyg \
-    --channel pytorch \
-    --channel pytorch-nightly \
-    conda/recipes/cugraph-pyg
-
-  # Only CUDA 11 is supported right now due to PyTorch requirement.
-  rapids-conda-retry mambabuild \
-    --no-test \
-    --channel "${CPP_CHANNEL}" \
-    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-    --channel dglteam \
-    --channel pytorch \
-    --channel pytorch-nightly \
-    conda/recipes/cugraph-dgl
-fi
+rapids-generate-version > ./VERSION
+
+# TODO: Remove `--no-test` flags once importing on a CPU
+# node works correctly
+rapids-logger "Begin pylibwholegraph build"
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  conda/recipes/pylibwholegraph
 
 rapids-upload-conda-to-s3 python

ci/run_ctests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+# Support customizing the ctests' install location
+cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libwholegraph/"
+
+find . -type f -executable -print0 | xargs -0 -r -t -n1 -P1 sh -c 'exec "$0"';

ci/run_pylibwholegraph_pytests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+# Support invoking run_pytests.sh outside the script directory
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/pylibwholegraph/pylibwholegraph/
+
+pytest --cache-clear --forked --import-mode=append "$@" tests

ci/test_cpp.sh

@@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
@@ -22,30 +24,22 @@ conda activate test
 set -u
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
 
 rapids-print-env
 
 rapids-mamba-retry install \
-    --channel "${CPP_CHANNEL}" \
-    libcugraph libcugraph_etl libcugraph-tests
+  --channel "${CPP_CHANNEL}" \
+  "libwholegraph=${RAPIDS_VERSION}" \
+  "libwholegraph-tests=${RAPIDS_VERSION}"
 
 rapids-logger "Check GPU usage"
 nvidia-smi
 
-# RAPIDS_DATASET_ROOT_DIR is used by test scripts
-export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
-pushd "${RAPIDS_DATASET_ROOT_DIR}"
-./get_test_data.sh --subset
-popd
-
-export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/
-
-# Run libcugraph gtests from libcugraph-tests package
-rapids-logger "Run gtests"
-./ci/run_ctests.sh -j10 && EXITCODE=$? || EXITCODE=$?;
+# Run libwholegraph tests from libwholegraph-tests package
+rapids-logger "Run tests"
+./ci/run_ctests.sh && EXITCODE=$? || EXITCODE=$?
 
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}

ci/test_wheel_pylibwholegraph.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -e          # abort the script on error
+set -o pipefail # piped commands propagate their error
+set -E          # ERR traps are inherited by subcommands
+
+mkdir -p ./dist
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+
+# determine pytorch source
+PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')"
+PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2}
+if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then
+  INDEX_URL="https://download.pytorch.org/whl/cu121"
+else
+  INDEX_URL="https://download.pytorch.org/whl/cu${PKG_CUDA_VER}"
+fi
+RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
+RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
+mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+rapids-logger "Installing Packages"
+rapids-retry python -m pip install \
+    --extra-index-url ${INDEX_URL} \
+    "$(echo ./dist/pylibwholegraph*.whl)[test]"
+
+rapids-logger "pytest pylibwholegraph"
+cd python/pylibwholegraph/pylibwholegraph/tests
+python -m pytest \
+  --cache-clear \
+  --forked \
+  --import-mode=append \
+  .

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -39,6 +39,7 @@ dependencies:
 - pytest
 - pytest-benchmark
 - pytest-cov
+- pytest-forked
 - pytest-xdist
 - pytorch-cuda=11.8
 - pytorch::pytorch>=2.0,<2.2.0a0
@@ -47,12 +48,10 @@ dependencies:
 - rmm==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - scipy
-- setuptools>=61.0.0
 - sphinx-copybutton
 - sphinx-markdown-tables
 - sphinx<6
 - sphinxcontrib-websupport
 - torchdata
 - wget
-- wheel
 name: all_cuda-118_arch-x86_64