add notebook tests, build.sh args

.github/workflows/pr.yaml

@@ -15,15 +15,16 @@ jobs:
       - changed-files
       - checks
       - conda-cpp-build
-      - conda-cpp-tests
+      # - conda-cpp-tests
       - conda-python-build
       - conda-python-tests
-      - wheel-build-pylibwholegraph
-      - wheel-tests-pylibwholegraph
-      - wheel-build-cugraph-dgl
-      - wheel-tests-cugraph-dgl
-      - wheel-build-cugraph-pyg
-      - wheel-tests-cugraph-pyg
+      - conda-notebook-tests
+      # - wheel-build-pylibwholegraph
+      # - wheel-tests-pylibwholegraph
+      # - wheel-build-cugraph-dgl
+      # - wheel-tests-cugraph-dgl
+      # - wheel-build-cugraph-pyg
+      # - wheel-tests-cugraph-pyg
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: always()
@@ -51,6 +52,7 @@ jobs:
           - '!CONTRIBUTING.md'
           - '!README.md'
           - '!docs/**'
+          - '!readme_pages/**'
         test_python:
           - '**'
           - '!.devcontainers/**'
@@ -59,6 +61,7 @@ jobs:
           - '!docs/**'
           - '!img/**'
           - '!notebooks/**'
+          - '!readme_pages/**'
   checks:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -70,71 +73,82 @@ jobs:
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
-  conda-cpp-tests:
-    needs: [conda-cpp-build, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
-    with:
-      build_type: pull-request
+  # conda-cpp-tests:
+  #   needs: [conda-cpp-build, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
+  #   with:
+  #     build_type: pull-request
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
-  conda-python-tests:
+  conda-notebook-tests:
     needs: [conda-python-build, changed-files]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-  wheel-build-pylibwholegraph:
-    needs: checks
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/build_wheel_pylibwholegraph.sh
-  wheel-tests-pylibwholegraph:
-    needs: [wheel-build-pylibwholegraph, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
     with:
       build_type: pull-request
-      script: ci/test_wheel_pylibwholegraph.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
-  wheel-build-cugraph-dgl:
-    needs: checks
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/build_wheel_cugraph-dgl.sh
-  wheel-tests-cugraph-dgl:
-    needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-dgl, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_cugraph-dgl.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
-  wheel-build-cugraph-pyg:
-    needs: checks
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/build_wheel_cugraph-pyg.sh
-  wheel-tests-cugraph-pyg:
-    needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-pyg, changed-files]
+      node_type: "gpu-v100-latest-1"
+      arch: "amd64"
+      container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
+      run_script: "ci/test_notebooks.sh"
+  conda-python-tests:
+    needs: [conda-python-build, changed-files]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-test[email protected]
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests[email protected]
     if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
     with:
       build_type: pull-request
-      script: ci/test_wheel_cugraph-pyg.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
+  # wheel-build-pylibwholegraph:
+  #   needs: checks
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/build_wheel_pylibwholegraph.sh
+  # wheel-tests-pylibwholegraph:
+  #   needs: [wheel-build-pylibwholegraph, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/test_wheel_pylibwholegraph.sh
+  #     matrix_filter: map(select(.ARCH == "amd64"))
+  # wheel-build-cugraph-dgl:
+  #   needs: checks
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/build_wheel_cugraph-dgl.sh
+  # wheel-tests-cugraph-dgl:
+  #   needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-dgl, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/test_wheel_cugraph-dgl.sh
+  #     matrix_filter: map(select(.ARCH == "amd64"))
+  # wheel-build-cugraph-pyg:
+  #   needs: checks
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/build_wheel_cugraph-pyg.sh
+  # wheel-tests-cugraph-pyg:
+  #   needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-pyg, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+  #   with:
+  #     build_type: pull-request
+  #     script: ci/test_wheel_cugraph-pyg.sh
+  #     matrix_filter: map(select(.ARCH == "amd64"))

.github/workflows/test.yaml

@@ -22,6 +22,16 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  conda-notebook-tests:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
+    with:
+      build_type: pull-request
+      node_type: "gpu-v100-latest-1"
+      arch: "amd64"
+      container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.12"
+      run_script: "ci/test_notebooks.sh"
   conda-python-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]

.gitignore

@@ -15,7 +15,7 @@ __pycache__
 DartConfiguration.tcl
 .DS_Store
 rmm_log.txt
-.ruff_cache
+.ruff_cache/
 
 # Unit test / coverage reports
 htmlcov/

build.sh

@@ -29,12 +29,14 @@ VALIDARGS="
    pylibwholegraph
    libwholegraph
    tests
+   benchmarks
    all
    -v
    -g
    -n
    --pydevelop
    --allgpuarch
+   --compile-cmd
    --clean
    -h
    --help
@@ -49,13 +51,16 @@ HELP="$0 [<target> ...] [<flag> ...]
    pylibwholegraph            - build the pylibwholegraph Python package
    libwholegraph              - build the libwholegraph library
    tests                      - build the C++ tests
+   benchmarks                 - build benchmarks
    all                        - build everything
  and <flag> is:
    -v                         - verbose build mode
    -g                         - build for debug
    -n                         - do not install after a successful build (does not affect Python packages)
    --pydevelop                - install the Python packages in editable mode
    --allgpuarch               - build for all supported GPU architectures
+   --enable-nvshmem            - build with nvshmem support (beta).
+   --compile-cmd               - only output compile commands (invoke CMake without build)
    --clean                    - clean an individual target (note: to do a complete rebuild, use the clean target described above)
    -h                         - print this text
 
@@ -140,11 +145,22 @@ if hasArg --pydevelop; then
     PYTHON_ARGS_FOR_INSTALL="${PYTHON_ARGS_FOR_INSTALL} -e"
 fi
 
+if hasArg --enable-nvshmem; then
+    BUILD_WITH_NVSHMEM=ON
+else
+    BUILD_WITH_NVSHMEM=OFF
+fi
 if hasArg tests; then
     BUILD_TESTS=ON
 else
     BUILD_TESTS=OFF
 fi
+if hasArg benchmarks; then
+    BUILD_BENCHMARKS=ON
+else
+    BUILD_BENCHMARKS=OFF
+fi
+
 
 # If clean or uninstall targets given, run them prior to any other steps
 if hasArg uninstall; then
@@ -250,7 +266,7 @@ if hasArg cugraph-pyg || buildDefault || hasArg all; then
     fi
 fi
 
-# Install the cugraph-dgl extensions for DGL
+# Build and install the cugraph-dgl Python package
 if hasArg cugraph-dgl || buildDefault ||hasArg all; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph-dgl

ci/build_cpp.sh

@@ -13,7 +13,11 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
-version=$(rapids-generate-version)
+# TODO: revert this once we start publishing nightly packages
+#       from the 'cugraph-gnn' repo and stop publishing them from
+#       the 'cugraph' / 'wholegraph' repos
+#version=$(rapids-generate-version)
+version="24.12.00a1000"
 
 rapids-logger "Begin cpp build"
 

ci/build_python.sh

@@ -15,7 +15,11 @@ rapids-print-env
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
-rapids-generate-version > ./VERSION
+# TODO: revert this once we start publishing nightly packages
+#       from the 'cugraph-gnn' repo and stop publishing them from
+#       the 'cugraph' / 'wholegraph' repos
+# rapids-generate-version > ./VERSION
+echo "24.12.00a1000" > ./VERSION
 
 sccache --zero-stats
 

ci/test_notebooks.sh

@@ -5,6 +5,8 @@ set -Eeuo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
@@ -24,18 +26,20 @@ rapids-logger "Downloading artifacts from previous jobs"
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
+# TODO: remove the '>=24.12.00a1000' once we start publishing nightly packages
+#       from the 'cugraph-gnn' repo and stop publishing them from
+#       the 'cugraph' / 'wholegraph' repos
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
-  libcugraph pylibcugraph cugraph
+  --channel dglteam/label/th23_cu118 \
+  "cugraph-dgl=${RAPIDS_VERSION},>=24.12.00a1000"
 
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 NOTEBOOK_LIST="$(realpath "$(dirname "$0")/notebook_list.py")"
 EXITCODE=0
 trap "EXITCODE=1" ERR
 
-
-pushd notebooks
 TOPLEVEL_NB_FOLDERS="$(find . -name "*.ipynb" | cut -d'/' -f2 | sort -u)"
 set +e
 # Always run nbtest in all TOPLEVEL_NB_FOLDERS, set EXITCODE to failure