From 6ad8cf8a59b8124dae1c9266945b137f295626d8 Mon Sep 17 00:00:00 2001 From: Brad Rees <34135411+BradReesWork@users.noreply.github.com> Date: Thu, 21 Nov 2024 22:25:27 -0500 Subject: [PATCH 1/3] Remove cugraph-equivariance (#4762) The cugraph-equivariant package was moved into cuEquivariance a few releases ago. This PR removes the code and build processes. Authors: - Brad Rees (https://github.com/BradReesWork) - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) - Tingyu Wang (https://github.com/tingyu66) - Xiaoyun Wang (https://github.com/wangxiaoyunNV) - Alex Barghi (https://github.com/alexbarghi-nv) - Chuck Hastings (https://github.com/ChuckHastings) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4762 --- .github/workflows/build.yaml | 19 -- .github/workflows/pr.yaml | 17 -- .github/workflows/test.yaml | 10 - build.sh | 14 +- ci/build_python.sh | 5 - ci/build_wheel.sh | 43 ++- ci/build_wheel_cugraph-equivariant.sh | 9 - ci/run_cugraph_equivariant_pytests.sh | 9 - ci/test_python.sh | 37 --- ci/test_wheel_cugraph-equivariant.sh | 32 -- conda/recipes/cugraph-equivariant/build.sh | 7 - conda/recipes/cugraph-equivariant/meta.yaml | 39 --- cpp/tests/community/balanced_edge_test.cpp | 2 +- dependencies.yaml | 24 -- .../source/installation/getting_cugraph.md | 2 - python/cugraph-equivariant/LICENSE | 1 - python/cugraph-equivariant/README.md | 5 - .../cugraph_equivariant/VERSION | 1 - .../cugraph_equivariant/__init__.py | 14 - .../cugraph_equivariant/_version.py | 35 --- .../cugraph_equivariant/nn/__init__.py | 21 -- .../nn/tensor_product_conv.py | 283 ------------------ .../cugraph_equivariant/tests/conftest.py | 39 --- .../cugraph_equivariant/tests/pytest.ini | 4 - .../cugraph_equivariant/tests/test_scatter.py | 40 --- .../tests/test_tensor_product_conv.py | 136 --------- .../cugraph_equivariant/tests/test_version.py | 12 - .../cugraph_equivariant/utils/__init__.py | 18 -- .../cugraph_equivariant/utils/scatter.py | 42 --- python/cugraph-equivariant/pyproject.toml | 80 ----- python/cugraph-equivariant/setup.py | 20 -- 31 files changed, 21 insertions(+), 999 deletions(-) delete mode 100755 ci/build_wheel_cugraph-equivariant.sh delete mode 100755 ci/run_cugraph_equivariant_pytests.sh delete mode 100755 ci/test_wheel_cugraph-equivariant.sh delete mode 100644 conda/recipes/cugraph-equivariant/build.sh delete mode 100644 conda/recipes/cugraph-equivariant/meta.yaml delete mode 120000 python/cugraph-equivariant/LICENSE delete mode 100644 python/cugraph-equivariant/README.md delete mode 120000 python/cugraph-equivariant/cugraph_equivariant/VERSION delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/__init__.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/_version.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/nn/__init__.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/nn/tensor_product_conv.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/tests/conftest.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/tests/pytest.ini delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/tests/test_scatter.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/tests/test_tensor_product_conv.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/tests/test_version.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/utils/__init__.py delete mode 100644 python/cugraph-equivariant/cugraph_equivariant/utils/scatter.py delete mode 100644 python/cugraph-equivariant/pyproject.toml delete mode 100644 python/cugraph-equivariant/setup.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9a055d1dc51..2962b4438fe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -113,22 +113,3 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cugraph - wheel-build-cugraph-equivariant: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - script: ci/build_wheel_cugraph-equivariant.sh - wheel-publish-cugraph-equivariant: - needs: wheel-build-cugraph-equivariant - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - package-name: cugraph-equivariant diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bac56fec9f5..e82342dfd94 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,8 +25,6 @@ jobs: - wheel-tests-pylibcugraph - wheel-build-cugraph - wheel-tests-cugraph - - wheel-build-cugraph-equivariant - - wheel-tests-cugraph-equivariant - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 @@ -161,21 +159,6 @@ jobs: with: build_type: pull-request script: ci/test_wheel_cugraph.sh - wheel-build-cugraph-equivariant: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 - with: - build_type: pull-request - script: ci/build_wheel_cugraph-equivariant.sh - wheel-tests-cugraph-equivariant: - needs: [wheel-build-cugraph-equivariant, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph-equivariant.sh - matrix_filter: map(select(.ARCH == "amd64")) devcontainer: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 245ac018cec..72f76904f4b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -58,13 +58,3 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_cugraph.sh - wheel-tests-cugraph-equivariant: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: ci/test_wheel_cugraph-equivariant.sh - matrix_filter: map(select(.ARCH == "amd64")) diff --git a/build.sh b/build.sh index 1ab98fe4378..398582c74c0 100755 --- a/build.sh +++ b/build.sh @@ -31,7 +31,6 @@ VALIDARGS=" cugraph-service cugraph-pyg cugraph-dgl - cugraph-equivariant cpp-mgtests cpp-mtmgtests docs @@ -59,7 +58,6 @@ HELP="$0 [ ...] [ ...] cugraph-service - build the cugraph-service_client and cugraph-service_server Python package cugraph-pyg - build the cugraph-pyg Python package cugraph-dgl - build the cugraph-dgl extensions for DGL - cugraph-equivariant - build the cugraph-equivariant Python package cpp-mgtests - build libcugraph and libcugraph_etl MG tests. Builds MPI communicator, adding MPI as a dependency. cpp-mtmgtests - build libcugraph MTMG tests. Adds UCX as a dependency (temporary). docs - build the docs @@ -209,8 +207,7 @@ if hasArg uninstall; then # FIXME: if multiple versions of these packages are installed, this only # removes the latest one and leaves the others installed. build.sh uninstall # can be run multiple times to remove all of them, but that is not obvious. - pip uninstall -y pylibcugraph cugraph cugraph-service-client cugraph-service-server \ - cugraph-dgl cugraph-pyg cugraph-equivariant + pip uninstall -y pylibcugraph cugraph cugraph-service-client cugraph-service-server fi if hasArg clean; then @@ -346,15 +343,6 @@ if hasArg cugraph-dgl || hasArg all; then fi fi -# Build and install the cugraph-equivariant Python package -if hasArg cugraph-equivariant || hasArg all; then - if hasArg --clean; then - cleanPythonDir ${REPODIR}/python/cugraph-equivariant - else - python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-equivariant - fi -fi - # Build the docs if hasArg docs || hasArg all; then if [ ! -d ${LIBCUGRAPH_BUILD_DIR} ]; then diff --git a/ci/build_python.sh b/ci/build_python.sh index 9a8f1227488..eab41f63da0 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -50,9 +50,4 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cugraph-service -rapids-conda-retry mambabuild \ - --no-test \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cugraph-equivariant - rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 3c89d63538c..9a77e6b3021 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -29,27 +29,22 @@ python -m pip wheel \ sccache --show-adv-stats -# pure-python packages should be marked as pure, and not have auditwheel run on them. -if [[ ${package_name} == "cugraph-equivariant" ]]; then - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist -else - case "${RAPIDS_CUDA_VERSION}" in - 12.*) - EXCLUDE_ARGS=( - --exclude "libcublas.so.12" - --exclude "libcublasLt.so.12" - --exclude "libcurand.so.10" - --exclude "libcusolver.so.11" - --exclude "libcusparse.so.12" - --exclude "libnvJitLink.so.12" - ) - ;; - 11.*) - EXCLUDE_ARGS=() - ;; - esac - - mkdir -p final_dist - python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist -fi +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXCLUDE_ARGS=( + --exclude "libcublas.so.12" + --exclude "libcublasLt.so.12" + --exclude "libcurand.so.10" + --exclude "libcusolver.so.11" + --exclude "libcusparse.so.12" + --exclude "libnvJitLink.so.12" + ) + ;; + 11.*) + EXCLUDE_ARGS=() + ;; +esac + +mkdir -p final_dist +python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist diff --git a/ci/build_wheel_cugraph-equivariant.sh b/ci/build_wheel_cugraph-equivariant.sh deleted file mode 100755 index 2f270422f84..00000000000 --- a/ci/build_wheel_cugraph-equivariant.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -package_dir="python/cugraph-equivariant" - -./ci/build_wheel.sh cugraph-equivariant ${package_dir} -./ci/validate_wheel.sh ${package_dir} dist diff --git a/ci/run_cugraph_equivariant_pytests.sh b/ci/run_cugraph_equivariant_pytests.sh deleted file mode 100755 index 5d5a5fb05c2..00000000000 --- a/ci/run_cugraph_equivariant_pytests.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -# Support invoking run_cugraph_equivariant_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-equivariant/cugraph_equivariant - -pytest --cache-clear "$@" . diff --git a/ci/test_python.sh b/ci/test_python.sh index a3a177dcfc6..646b61805cc 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -100,42 +100,5 @@ rapids-logger "pytest cugraph-service (single GPU)" --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-service-coverage.xml" \ --cov-report=term -# test cugraph-equivariant -if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_equivariant - set +u - conda activate test_cugraph_equivariant - set -u - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - --channel conda-forge \ - --channel nvidia \ - "cugraph-equivariant=${RAPIDS_VERSION_MAJOR_MINOR}.*" - pip install e3nn==0.5.1 - - rapids-print-env - - rapids-logger "pytest cugraph-equivariant" - ./ci/run_cugraph_equivariant_pytests.sh \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-equivariant.xml" \ - --cov-config=../../.coveragerc \ - --cov=cugraph_equivariant \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-equivariant-coverage.xml" \ - --cov-report=term - - # Reactivate the test environment back - set +u - conda deactivate - conda activate test - set -u - else - rapids-logger "skipping cugraph-equivariant pytest on ARM64" - fi -else - rapids-logger "skipping cugraph-equivariant pytest on CUDA!=11.8" -fi - rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_wheel_cugraph-equivariant.sh b/ci/test_wheel_cugraph-equivariant.sh deleted file mode 100755 index 3be1d578964..00000000000 --- a/ci/test_wheel_cugraph-equivariant.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -eoxu pipefail - -package_name="cugraph-equivariant" - -mkdir -p ./dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# Download the cugraph-equivariant built in the previous step -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist - -# determine pytorch source -PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" -PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} -if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then - PYTORCH_CUDA_VER="121" -else - PYTORCH_CUDA_VER=$PKG_CUDA_VER -fi -PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}" - -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install \ - -v \ - --extra-index-url "${PYTORCH_URL}" \ - "$(echo ./dist/cugraph_equivariant_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ - 'e3nn' \ - 'torch>=2.3.0,<2.4' - -python -m pytest python/cugraph-equivariant/cugraph_equivariant/tests diff --git a/conda/recipes/cugraph-equivariant/build.sh b/conda/recipes/cugraph-equivariant/build.sh deleted file mode 100644 index f0ff1688b55..00000000000 --- a/conda/recipes/cugraph-equivariant/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2024, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory - -./build.sh cugraph-equivariant diff --git a/conda/recipes/cugraph-equivariant/meta.yaml b/conda/recipes/cugraph-equivariant/meta.yaml deleted file mode 100644 index 9dc9d51fa48..00000000000 --- a/conda/recipes/cugraph-equivariant/meta.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: cugraph-equivariant - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - -requirements: - host: - - python - - rapids-build-backend>=0.3.1,<0.4.0.dev0 - - setuptools>=61.0.0 - run: - - pylibcugraphops ={{ minor_version }} - - python - -tests: - imports: - - cugraph_equivariant - -about: - home: https://rapids.ai/ - dev_url: https://github.com/rapidsai/cugraph - license: Apache-2.0 - license_file: ../../../LICENSE - summary: GPU-accelerated equivariant convolutional layers. diff --git a/cpp/tests/community/balanced_edge_test.cpp b/cpp/tests/community/balanced_edge_test.cpp index c4488dc9b9e..614a4ee4190 100644 --- a/cpp/tests/community/balanced_edge_test.cpp +++ b/cpp/tests/community/balanced_edge_test.cpp @@ -15,7 +15,7 @@ #include #include -TEST(balanced_edge, success) +TEST(balanced_edge, DISABLED_success) { std::vector off_h = {0, 16, 25, 35, 41, 44, 48, 52, 56, 61, 63, 66, 67, 69, 74, 76, 78, 80, 82, 84, 87, 89, 91, 93, diff --git a/dependencies.yaml b/dependencies.yaml index 208b3bedd63..6aa1b86aa60 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -202,30 +202,6 @@ files: - test_python_common - depends_on_pylibwholegraph - depends_on_pytorch - py_build_cugraph_equivariant: - output: pyproject - pyproject_dir: python/cugraph-equivariant - extras: - table: build-system - includes: - - python_build_rapids - - python_build_wheel - py_run_cugraph_equivariant: - output: pyproject - pyproject_dir: python/cugraph-equivariant - extras: - table: project - includes: - # Deprecate pylibcugraphops - - depends_on_pylibcugraphops - py_test_cugraph_equivariant: - output: pyproject - pyproject_dir: python/cugraph-equivariant - extras: - table: project.optional-dependencies - key: test - includes: - - test_python_common py_build_cugraph_service_client: output: pyproject pyproject_dir: python/cugraph-service/client diff --git a/docs/cugraph/source/installation/getting_cugraph.md b/docs/cugraph/source/installation/getting_cugraph.md index 01bc9e379c9..0c553acf964 100644 --- a/docs/cugraph/source/installation/getting_cugraph.md +++ b/docs/cugraph/source/installation/getting_cugraph.md @@ -31,7 +31,6 @@ cuGraph Conda packages * cugraph-service-server * cugraph-dgl * cugraph-pyg - * cugraph-equivariant * nx-cugraph Replace the package name in the example below to the one you want to install. @@ -61,7 +60,6 @@ Replace `-cu12` with `-cu11` for packages supporting CUDA 11. Also available: * cugraph-dgl-cu12 * cugraph-pyg-cu12 - * cugraph-equivariant-cu12 * nx-cugraph-cu12
diff --git a/python/cugraph-equivariant/LICENSE b/python/cugraph-equivariant/LICENSE deleted file mode 120000 index 30cff7403da..00000000000 --- a/python/cugraph-equivariant/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../LICENSE \ No newline at end of file diff --git a/python/cugraph-equivariant/README.md b/python/cugraph-equivariant/README.md deleted file mode 100644 index d5de8852709..00000000000 --- a/python/cugraph-equivariant/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# cugraph-equivariant - -## Description - -cugraph-equivariant library provides fast symmetry-preserving (equivariant) operations and convolutional layers, to accelerate the equivariant neural networks in drug discovery and other domains. diff --git a/python/cugraph-equivariant/cugraph_equivariant/VERSION b/python/cugraph-equivariant/cugraph_equivariant/VERSION deleted file mode 120000 index d62dc733efd..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/VERSION +++ /dev/null @@ -1 +0,0 @@ -../../../VERSION \ No newline at end of file diff --git a/python/cugraph-equivariant/cugraph_equivariant/__init__.py b/python/cugraph-equivariant/cugraph_equivariant/__init__.py deleted file mode 100644 index 20507bd9329..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cugraph_equivariant._version import __git_commit__, __version__ diff --git a/python/cugraph-equivariant/cugraph_equivariant/_version.py b/python/cugraph-equivariant/cugraph_equivariant/_version.py deleted file mode 100644 index 940ebac74f7..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/_version.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib.resources - -# Read VERSION file from the module that is symlinked to VERSION file -# in the root of the repo at build time or copied to the module at -# installation. VERSION is a separate file that allows CI build-time scripts -# to update version info (including commit hashes) without modifying -# source files. -__version__ = ( - importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() -) -try: - __git_commit__ = ( - importlib.resources.files(__package__) - .joinpath("GIT_COMMIT") - .read_text() - .strip() - ) -except FileNotFoundError: - __git_commit__ = "" - -__all__ = ["__git_commit__", "__version__"] diff --git a/python/cugraph-equivariant/cugraph_equivariant/nn/__init__.py b/python/cugraph-equivariant/cugraph_equivariant/nn/__init__.py deleted file mode 100644 index 8f4d8de0042..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/nn/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .tensor_product_conv import FullyConnectedTensorProductConv - -DiffDockTensorProductConv = FullyConnectedTensorProductConv - -__all__ = [ - "FullyConnectedTensorProductConv", - "DiffDockTensorProductConv", -] diff --git a/python/cugraph-equivariant/cugraph_equivariant/nn/tensor_product_conv.py b/python/cugraph-equivariant/cugraph_equivariant/nn/tensor_product_conv.py deleted file mode 100644 index 923edbfc44a..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/nn/tensor_product_conv.py +++ /dev/null @@ -1,283 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Sequence, Union, NamedTuple - -import torch -from torch import nn -from e3nn import o3 -from e3nn.nn import BatchNorm - -from cugraph_equivariant.utils import scatter_reduce - -try: - from pylibcugraphops.pytorch.operators import FusedFullyConnectedTensorProduct -except ImportError as exc: - raise RuntimeError( - "FullyConnectedTensorProductConv is no longer supported in " - "cugraph-equivariant starting from version 24.08. It will be migrated " - "to the new `cuequivariance` package. Please use 24.06 release for the " - "legacy interface." - ) from exc - - -class Graph(NamedTuple): - edge_index: torch.Tensor - size: tuple[int, int] - - -class FullyConnectedTensorProductConv(nn.Module): - r"""Message passing layer for tensor products in DiffDock-like architectures. - The left operand of tensor product is the spherical harmonic representation - of edge vector; the right operand consists of node features in irreps. - - .. math:: - \sum_{b \in \mathcal{N}_a} Y\left(\hat{r}_{a b}\right) - \otimes_{\psi_{a b}} \mathbf{h}_b - - where the path weights :math:`\psi_{a b}` can be constructed from edge - embeddings and scalar features using an MLP: - - .. math:: - \psi_{a b} = \operatorname{MLP} - \left(e_{a b}, \mathbf{h}_a^0, \mathbf{h}_b^0\right) - - Users have the option to either directly input the weights or provide the - MLP parameters and scalar features from edges and nodes. - - Parameters - ---------- - in_irreps : e3nn.o3.Irreps - Irreps for the input node features. - - sh_irreps : e3nn.o3.Irreps - Irreps for the spherical harmonic representations of edge vectors. - - out_irreps : e3nn.o3.Irreps - Irreps for the output. - - batch_norm : bool, optional (default=True) - If true, batch normalization is applied. - - mlp_channels : sequence of ints, optional (default=None) - A sequence of integers defining number of neurons in each layer in MLP - before the output layer. If `None`, no MLP will be added. The input layer - contains edge embeddings and node scalar features. - - mlp_activation : nn.Module or sequence of nn.Module, optional (default=nn.GELU()) - A sequence of functions to be applied in between linear layers in MLP, - e.g., `nn.Sequential(nn.ReLU(), nn.Dropout(0.4))`. - - e3nn_compat_mode: bool, optional (default=False) - cugraph-ops and e3nn use different memory layout for Irreps-tensors. - The last (fastest moving) dimension is num_channels for cugraph-ops and - ir.dim for e3nn. When enabled, the input and output of this layer will - follow e3nn's memory layout. - - Examples - -------- - Case 1: MLP with the input layer having 6 channels and 2 hidden layers - having 16 channels. edge_emb.size(1) must match the size of the input layer: 6 - - >>> conv1 = FullyConnectedTensorProductConv(in_irreps, sh_irreps, out_irreps, - >>> mlp_channels=[6, 16, 16], mlp_activation=nn.ReLU()).cuda() - >>> out = conv1(src_features, edge_sh, edge_emb, graph) - - Case 2: If `edge_emb` is constructed by concatenating scalar features from - edges, sources and destinations, as in DiffDock, the layer can accept each - scalar component separately: - - >>> conv2 = FullyConnectedTensorProductConv(in_irreps, sh_irreps, out_irreps, - >>> mlp_channels=[6, 16, 16], mlp_activation=nn.ReLU()).cuda() - >>> out = conv2(src_features, edge_sh, edge_scalars, graph, - >>> src_scalars=src_scalars, dst_scalars=dst_scalars) - - This allows a smaller GEMM in the first MLP layer by performing GEMM on each - component before indexing. The first-layer weights are split into sections - for edges, sources and destinations, in that order.This is equivalent to - - >>> src, dst = graph.edge_index - >>> edge_emb = torch.hstack((edge_scalars, src_scalars[src], dst_scalars[dst])) - >>> out = conv2(src_features, edge_sh, edge_emb, graph) - - Case 3: No MLP, `edge_emb` will be directly used as the tensor product weights: - - >>> conv3 = FullyConnectedTensorProductConv(in_irreps, sh_irreps, out_irreps, - >>> mlp_channels=None).cuda() - >>> out = conv3(src_features, edge_sh, edge_emb, graph) - - """ - - def __init__( - self, - in_irreps: o3.Irreps, - sh_irreps: o3.Irreps, - out_irreps: o3.Irreps, - batch_norm: bool = True, - mlp_channels: Optional[Sequence[int]] = None, - mlp_activation: Union[nn.Module, Sequence[nn.Module]] = nn.GELU(), - e3nn_compat_mode: bool = False, - ): - super().__init__() - self.in_irreps = in_irreps - self.out_irreps = out_irreps - self.sh_irreps = sh_irreps - self.e3nn_compat_mode = e3nn_compat_mode - - self.tp = FusedFullyConnectedTensorProduct( - in_irreps, sh_irreps, out_irreps, e3nn_compat_mode=e3nn_compat_mode - ) - - self.batch_norm = BatchNorm(out_irreps) if batch_norm else None - - if mlp_activation is None: - mlp_activation = [] - elif hasattr(mlp_activation, "__len__") and hasattr( - mlp_activation, "__getitem__" - ): - mlp_activation = list(mlp_activation) - else: - mlp_activation = [mlp_activation] - - if mlp_channels is not None: - dims = list(mlp_channels) + [self.tp.weight_numel] - mlp = [] - for i in range(len(dims) - 1): - mlp.append(nn.Linear(dims[i], dims[i + 1])) - if i != len(dims) - 2: - mlp.extend(mlp_activation) - self.mlp = nn.Sequential(*mlp) - else: - self.mlp = None - - def forward( - self, - src_features: torch.Tensor, - edge_sh: torch.Tensor, - edge_emb: torch.Tensor, - graph: tuple[torch.Tensor, tuple[int, int]], - src_scalars: Optional[torch.Tensor] = None, - dst_scalars: Optional[torch.Tensor] = None, - reduce: str = "mean", - edge_envelope: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Forward pass. - - Parameters - ---------- - src_features : torch.Tensor - Source node features. - Shape: (num_src_nodes, in_irreps.dim) - - edge_sh : torch.Tensor - The spherical harmonic representations of the edge vectors. - Shape: (num_edges, sh_irreps.dim) - - edge_emb: torch.Tensor - Edge embeddings that are fed into MLPs to generate tensor product weights. - Shape: (num_edges, dim), where `dim` should be: - - `tp.weight_numel` when the layer does not contain MLPs. - - num_edge_scalars, when scalar features from edges, sources and - destinations are passed in separately. - - graph : tuple - A tuple that stores the graph information, with the first element being - the adjacency matrix in COO, and the second element being its shape: - (num_src_nodes, num_dst_nodes). - - src_scalars: torch.Tensor, optional - Scalar features of source nodes. See examples for usage. - Shape: (num_src_nodes, num_src_scalars) - - dst_scalars: torch.Tensor, optional - Scalar features of destination nodes. See examples for usage. - Shape: (num_dst_nodes, num_dst_scalars) - - reduce : str, optional (default="mean") - Reduction operator. Choose between "mean" and "sum". - - edge_envelope: torch.Tensor, optional - Typically used as attenuation factors to fade out messages coming - from nodes close to the cutoff distance used to create the graph. - This is important to make the model smooth to the changes in node's - coordinates. - Shape: (num_edges,) - - Returns - ------- - torch.Tensor - Output node features. - Shape: (num_dst_nodes, out_irreps.dim) - """ - edge_emb_size = edge_emb.size(-1) - src_scalars_size = 0 if src_scalars is None else src_scalars.size(-1) - dst_scalars_size = 0 if dst_scalars is None else dst_scalars.size(-1) - - if self.mlp is None: - if self.tp.weight_numel != edge_emb_size: - raise RuntimeError( - f"When MLP is not present, edge_emb's last dimension must " - f"equal tp.weight_numel (but got {edge_emb_size} and " - f"{self.tp.weight_numel})" - ) - else: - total_size = edge_emb_size + src_scalars_size + dst_scalars_size - if self.mlp[0].in_features != total_size: - raise RuntimeError( - f"The size of MLP's input layer ({self.mlp[0].in_features}) " - f"does not match the total number of scalar features from " - f"edge_emb, src_scalars and dst_scalars ({total_size})" - ) - - if reduce not in ["mean", "sum"]: - raise RuntimeError( - f"reduce argument must be either 'mean' or 'sum', got {reduce}." - ) - - (src, dst), (num_src_nodes, num_dst_nodes) = graph - - if self.mlp is not None: - if src_scalars is None and dst_scalars is None: - tp_weights = self.mlp(edge_emb) - else: - w_edge, w_src, w_dst = torch.split( - self.mlp[0].weight, - (edge_emb_size, src_scalars_size, dst_scalars_size), - dim=-1, - ) - tp_weights = edge_emb @ w_edge.T + self.mlp[0].bias - - if src_scalars is not None: - tp_weights += (src_scalars @ w_src.T)[src] - - if dst_scalars is not None: - tp_weights += (dst_scalars @ w_dst.T)[dst] - - tp_weights = self.mlp[1:](tp_weights) - else: - tp_weights = edge_emb - - out = self.tp(src_features[src], edge_sh, tp_weights) - - if edge_envelope is not None: - out = out * edge_envelope.view(-1, 1) - - dtype = out.dtype - out = scatter_reduce( - out.float(), dst, dim=0, dim_size=num_dst_nodes, reduce=reduce - ).to(dtype) - - if self.batch_norm: - out = self.batch_norm(out) - - return out diff --git a/python/cugraph-equivariant/cugraph_equivariant/tests/conftest.py b/python/cugraph-equivariant/cugraph_equivariant/tests/conftest.py deleted file mode 100644 index 806e03e6d76..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/tests/conftest.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import torch - - -@pytest.fixture -def example_scatter_data(): - src_feat = torch.Tensor([3, 1, 0, 1, 1, 2]) - dst_indices = torch.Tensor([0, 1, 2, 2, 3, 1]) - - results = { - "sum": torch.Tensor([3.0, 3.0, 1.0, 1.0]), - "mean": torch.Tensor([3.0, 1.5, 0.5, 1.0]), - "prod": torch.Tensor([3.0, 2.0, 0.0, 1.0]), - "amax": torch.Tensor([3.0, 2.0, 1.0, 1.0]), - "amin": torch.Tensor([3.0, 1.0, 0.0, 1.0]), - } - - return src_feat, dst_indices, results - - -@pytest.fixture -def empty_scatter_data(): - src_feat = torch.empty((0, 41)) - dst_indices = torch.empty((0,)) - - return src_feat, dst_indices diff --git a/python/cugraph-equivariant/cugraph_equivariant/tests/pytest.ini b/python/cugraph-equivariant/cugraph_equivariant/tests/pytest.ini deleted file mode 100644 index 7b0a9f29fb1..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/tests/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. - -[pytest] -addopts = --tb=native diff --git a/python/cugraph-equivariant/cugraph_equivariant/tests/test_scatter.py b/python/cugraph-equivariant/cugraph_equivariant/tests/test_scatter.py deleted file mode 100644 index d28a32edcb1..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/tests/test_scatter.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import torch -from cugraph_equivariant.utils import scatter_reduce - - -@pytest.mark.parametrize("reduce", ["sum", "mean", "prod", "amax", "amin"]) -def test_scatter_reduce(example_scatter_data, reduce): - device = torch.device("cuda") - src, index, out_true = example_scatter_data - src = src.to(device) - index = index.to(device) - - out = scatter_reduce(src, index, dim=0, dim_size=None, reduce=reduce) - - assert torch.allclose(out.cpu(), out_true[reduce]) - - -def test_scatter_reduce_empty(empty_scatter_data): - device = torch.device("cuda") - src, index = empty_scatter_data - src = src.to(device) - index = index.to(device) - - out = scatter_reduce(src, index, dim=0, dim_size=None) - - assert out.numel() == 0 - assert out.size(1) == src.size(1) diff --git a/python/cugraph-equivariant/cugraph_equivariant/tests/test_tensor_product_conv.py b/python/cugraph-equivariant/cugraph_equivariant/tests/test_tensor_product_conv.py deleted file mode 100644 index ce325c47aa0..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/tests/test_tensor_product_conv.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -try: - from cugraph_equivariant.nn import FullyConnectedTensorProductConv -except RuntimeError: - pytest.skip( - "Migrated to cuequivariance package starting from 24.08.", - allow_module_level=True, - ) - -import torch -from torch import nn -from e3nn import o3 -from cugraph_equivariant.nn.tensor_product_conv import Graph - -device = torch.device("cuda") - - -def create_random_graph( - num_src_nodes, - num_dst_nodes, - num_edges, - dtype=None, - device=None, -): - row = torch.randint(num_src_nodes, (num_edges,), dtype=dtype, device=device) - col = torch.randint(num_dst_nodes, (num_edges,), dtype=dtype, device=device) - edge_index = torch.stack([row, col], dim=0) - - return Graph(edge_index, (num_src_nodes, num_dst_nodes)) - - -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("e3nn_compat_mode", [True, False]) -@pytest.mark.parametrize("batch_norm", [True, False]) -@pytest.mark.parametrize( - "mlp_channels, mlp_activation, scalar_sizes", - [ - [(30, 8, 8), nn.Sequential(nn.Dropout(0.3), nn.ReLU()), (15, 15, 0)], - [(7,), nn.GELU(), (2, 3, 2)], - [None, None, None], - ], -) -def test_tensor_product_conv_equivariance( - mlp_channels, mlp_activation, scalar_sizes, batch_norm, e3nn_compat_mode, dtype -): - torch.manual_seed(12345) - to_kwargs = {"device": device, "dtype": dtype} - - in_irreps = o3.Irreps("10x0e + 10x1e") - out_irreps = o3.Irreps("20x0e + 10x1e") - sh_irreps = o3.Irreps.spherical_harmonics(lmax=2) - - tp_conv = FullyConnectedTensorProductConv( - in_irreps=in_irreps, - sh_irreps=sh_irreps, - out_irreps=out_irreps, - mlp_channels=mlp_channels, - mlp_activation=mlp_activation, - batch_norm=batch_norm, - e3nn_compat_mode=e3nn_compat_mode, - ).to(**to_kwargs) - - num_src_nodes, num_dst_nodes = 9, 7 - num_edges = 40 - graph = create_random_graph(num_src_nodes, num_dst_nodes, num_edges, device=device) - - edge_sh = torch.randn(num_edges, sh_irreps.dim, **to_kwargs) - src_features = torch.randn(num_src_nodes, in_irreps.dim, **to_kwargs) - - rot = o3.rand_matrix() - D_in = tp_conv.in_irreps.D_from_matrix(rot).to(**to_kwargs) - D_sh = tp_conv.sh_irreps.D_from_matrix(rot).to(**to_kwargs) - D_out = tp_conv.out_irreps.D_from_matrix(rot).to(**to_kwargs) - - if mlp_channels is None: - edge_emb = torch.randn(num_edges, tp_conv.tp.weight_numel, **to_kwargs) - src_scalars = dst_scalars = None - else: - if scalar_sizes: - edge_emb = torch.randn(num_edges, scalar_sizes[0], **to_kwargs) - src_scalars = ( - None - if scalar_sizes[1] == 0 - else torch.randn(num_src_nodes, scalar_sizes[1], **to_kwargs) - ) - dst_scalars = ( - None - if scalar_sizes[2] == 0 - else torch.randn(num_dst_nodes, scalar_sizes[2], **to_kwargs) - ) - else: - edge_emb = torch.randn(num_edges, tp_conv.mlp[0].in_features, **to_kwargs) - src_scalars = dst_scalars = None - - # rotate before - torch.manual_seed(12345) - out_before = tp_conv( - src_features=src_features @ D_in.T, - edge_sh=edge_sh @ D_sh.T, - edge_emb=edge_emb, - graph=graph, - src_scalars=src_scalars, - dst_scalars=dst_scalars, - ) - - # rotate after - torch.manual_seed(12345) - out_after = ( - tp_conv( - src_features=src_features, - edge_sh=edge_sh, - edge_emb=edge_emb, - graph=graph, - src_scalars=src_scalars, - dst_scalars=dst_scalars, - ) - @ D_out.T - ) - - atol = 1e-3 if dtype == torch.float32 else 1e-1 - if e3nn_compat_mode: - assert torch.allclose(out_before, out_after, rtol=1e-4, atol=atol) diff --git a/python/cugraph-equivariant/cugraph_equivariant/tests/test_version.py b/python/cugraph-equivariant/cugraph_equivariant/tests/test_version.py deleted file mode 100644 index e8b484fe16c..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/tests/test_version.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. - -import cugraph_equivariant - - -def test_version_constants_are_populated(): - # __git_commit__ will only be non-empty in a built distribution - assert isinstance(cugraph_equivariant.__git_commit__, str) - - # __version__ should always be non-empty - assert isinstance(cugraph_equivariant.__version__, str) - assert len(cugraph_equivariant.__version__) > 0 diff --git a/python/cugraph-equivariant/cugraph_equivariant/utils/__init__.py b/python/cugraph-equivariant/cugraph_equivariant/utils/__init__.py deleted file mode 100644 index b4acfe8d090..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/utils/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .scatter import scatter_reduce - -__all__ = [ - "scatter_reduce", -] diff --git a/python/cugraph-equivariant/cugraph_equivariant/utils/scatter.py b/python/cugraph-equivariant/cugraph_equivariant/utils/scatter.py deleted file mode 100644 index 909fbc99365..00000000000 --- a/python/cugraph-equivariant/cugraph_equivariant/utils/scatter.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional - -import torch - - -def broadcast(src: torch.Tensor, ref: torch.Tensor, dim: int) -> torch.Tensor: - size = ((1,) * dim) + (-1,) + ((1,) * (ref.dim() - dim - 1)) - return src.view(size).expand_as(ref) - - -def scatter_reduce( - src: torch.Tensor, - index: torch.Tensor, - dim: int = 0, - dim_size: Optional[int] = None, # value of out.size(dim) - reduce: str = "sum", # "sum", "prod", "mean", "amax", "amin" -): - # scatter() expects index to be int64 - index = broadcast(index, src, dim).to(torch.int64) - - size = list(src.size()) - - if dim_size is not None: - size[dim] = dim_size - else: - size[dim] = 0 if index.numel() == 0 else int(index.max()) + 1 - - out = torch.zeros(size, dtype=src.dtype, device=src.device) - return out.scatter_reduce_(dim, index, src, reduce, include_self=False) diff --git a/python/cugraph-equivariant/pyproject.toml b/python/cugraph-equivariant/pyproject.toml deleted file mode 100644 index ffaeb4f9c34..00000000000 --- a/python/cugraph-equivariant/pyproject.toml +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[build-system] -requires = [ - "rapids-build-backend>=0.3.1,<0.4.0.dev0", - "setuptools>=61.0.0", - "wheel", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -build-backend = "rapids_build_backend.build" - -[project] -name = "cugraph-equivariant" -dynamic = ["version"] -description = "Fast GPU-based equivariant operations and convolutional layers." -readme = { file = "README.md", content-type = "text/markdown" } -authors = [ - { name = "NVIDIA Corporation" }, -] -license = { text = "Apache 2.0" } -requires-python = ">=3.10" -classifiers = [ - "Intended Audience :: Developers", - "Programming Language :: Python", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", -] -dependencies = [ - "pylibcugraphops==24.12.*,>=0.0.0a0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - -[project.urls] -Homepage = "https://github.com/rapidsai/cugraph" -Documentation = "https://docs.rapids.ai/api/cugraph/stable/api_docs/cugraph-ops/" - -[project.optional-dependencies] -test = [ - "pandas", - "pytest", - "pytest-benchmark", - "pytest-cov", - "pytest-xdist", - "scipy", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - -[tool.setuptools] -license-files = ["LICENSE"] - -[tool.setuptools.dynamic] -version = {file = "cugraph_equivariant/VERSION"} - -[tool.setuptools.packages.find] -include = [ - "cugraph_equivariant*", - "cugraph_equivariant.*", -] - -[tool.rapids-build-backend] -build-backend = "setuptools.build_meta" -dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true" - -[tool.pydistcheck] -select = [ - "distro-too-large-compressed", -] - -# PyPI limit is 100 MiB, fail CI before we get too close to that -max_allowed_size_compressed = '75M' diff --git a/python/cugraph-equivariant/setup.py b/python/cugraph-equivariant/setup.py deleted file mode 100644 index acd0df3f717..00000000000 --- a/python/cugraph-equivariant/setup.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import find_packages, setup - -if __name__ == "__main__": - packages = find_packages(include=["cugraph_equivariant*"]) - setup( - package_data={key: ["VERSION"] for key in packages}, - ) From 290d5d4a0f6f68f228f6abfe758c8ac2760c97fc Mon Sep 17 00:00:00 2001 From: jakirkham Date: Fri, 22 Nov 2024 04:29:36 -0800 Subject: [PATCH 2/3] Relax PyTorch upper bound (allowing 2.4) (#4703) As the issue around PyTorch being built without NumPy was fixed in conda-forge, we can now relax these upper bounds to allow PyTorch 2.4. xref: https://github.com/conda-forge/pytorch-cpu-feedstock/issues/254 xref: https://github.com/conda-forge/pytorch-cpu-feedstock/issues/266 xref: https://github.com/rapidsai/cugraph/pull/4615 Authors: - https://github.com/jakirkham - Alex Barghi (https://github.com/alexbarghi-nv) Approvers: - Alex Barghi (https://github.com/alexbarghi-nv) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cugraph/pull/4703 --- ci/build_docs.sh | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/cugraph-dgl/meta.yaml | 2 +- conda/recipes/cugraph-pyg/meta.yaml | 2 +- dependencies.yaml | 6 ++---- python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml | 2 +- python/cugraph-dgl/pyproject.toml | 2 +- python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml | 2 +- python/cugraph-pyg/pyproject.toml | 2 +- 10 files changed, 11 insertions(+), 13 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 01c573c96ca..2d7e90da8d0 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -48,7 +48,7 @@ rapids-mamba-retry install \ "libcugraph_etl=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ "pylibcugraphops=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "pytorch>=2.3,<2.4" \ + 'pytorch>=2.3' \ "cuda-version=${CONDA_CUDA_VERSION}" export RAPIDS_DOCS_DIR="$(mktemp -d)" diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index e4269707168..ec4c23541f9 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -54,7 +54,7 @@ dependencies: - pytest-cov - pytest-xdist - python-louvain -- pytorch>=2.3,<2.4.0a0 +- pytorch>=2.3 - raft-dask==24.12.*,>=0.0.0a0 - rapids-build-backend>=0.3.1,<0.4.0.dev0 - rapids-dask-dependency==24.12.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index eb2625b9d50..de0507c7c22 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -59,7 +59,7 @@ dependencies: - pytest-cov - pytest-xdist - python-louvain -- pytorch>=2.3,<2.4.0a0 +- pytorch>=2.3 - raft-dask==24.12.*,>=0.0.0a0 - rapids-build-backend>=0.3.1,<0.4.0.dev0 - rapids-dask-dependency==24.12.*,>=0.0.0a0 diff --git a/conda/recipes/cugraph-dgl/meta.yaml b/conda/recipes/cugraph-dgl/meta.yaml index 0383fc8adf8..ed449b5982a 100644 --- a/conda/recipes/cugraph-dgl/meta.yaml +++ b/conda/recipes/cugraph-dgl/meta.yaml @@ -31,7 +31,7 @@ requirements: - pylibcugraphops ={{ minor_version }} - tensordict >=0.1.2 - python - - pytorch >=2.3,<2.4.0a0 + - pytorch >=2.3 - cupy >=12.0.0 tests: diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 7d3e503e23a..b44614baa9f 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -31,7 +31,7 @@ requirements: - numba >=0.57 - numpy >=1.23,<3.0a0 - python - - pytorch >=2.3,<2.4.0a0 + - pytorch >=2.3 - cupy >=12.0.0 - cugraph ={{ version }} - pylibcugraphops ={{ minor_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 6aa1b86aa60..f959f4089f5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -630,9 +630,7 @@ dependencies: - output_types: [conda] packages: - *cugraph_unsuffixed - # ceiling could be removed when this is fixed: - # https://github.com/conda-forge/pytorch-cpu-feedstock/issues/254 - - &pytorch_conda pytorch>=2.3,<2.4.0a0 + - &pytorch_conda pytorch>=2.3 - pytorch-cuda==11.8 - &tensordict tensordict>=0.1.2 - dgl>=2.4.0.cu* @@ -670,7 +668,7 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - &pytorch_pip torch>=2.3,<2.4.0a0 + - &pytorch_pip torch>=2.3 - *tensordict - matrix: {cuda: "11.*"} packages: diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index 174012b8f8c..85c85c2043a 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -18,7 +18,7 @@ dependencies: - pytest-cov - pytest-xdist - pytorch-cuda==11.8 -- pytorch>=2.3,<2.4.0a0 +- pytorch>=2.3 - scipy - tensordict>=0.1.2 name: cugraph_dgl_dev_cuda-118 diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index e3e12216ac7..af9e91a988e 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -40,7 +40,7 @@ test = [ "pytest-xdist", "scipy", "tensordict>=0.1.2", - "torch>=2.3,<2.4.0a0", + "torch>=2.3", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index 4778ff0eaf6..5fbd947965f 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -17,7 +17,7 @@ dependencies: - pytest-cov - pytest-xdist - pytorch-cuda==11.8 -- pytorch>=2.3,<2.4.0a0 +- pytorch>=2.3 - pytorch_geometric>=2.5,<2.6 - scipy - tensordict>=0.1.2 diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index e157f36f8f6..a30cd375635 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -49,7 +49,7 @@ test = [ "pytest-xdist", "scipy", "tensordict>=0.1.2", - "torch>=2.3,<2.4.0a0", + "torch>=2.3", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.setuptools] From d3b80a2d5baf4bd910d211ef2b9825fb29101302 Mon Sep 17 00:00:00 2001 From: Joseph Nke <76006812+jnke2016@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:40:21 +0100 Subject: [PATCH 3/3] extract the edgelist from the graph (#4750) This PR exposes the C++ function decompress_to_edgelist to the C, PLC and Python API. This will enable the extraction of the edgelist from a graph which is currently not supported. It also removes the deprecated parameter `legacy_renum_only` Authors: - Joseph Nke (https://github.com/jnke2016) Approvers: - Rick Ratzel (https://github.com/rlratzel) - Chuck Hastings (https://github.com/ChuckHastings) - Seunghwa Kang (https://github.com/seunghwak) URL: https://github.com/rapidsai/cugraph/pull/4750 --- cpp/CMakeLists.txt | 2 + cpp/include/cugraph_c/graph_functions.h | 102 +++++++++++ cpp/src/c_api/decompress_to_edgelist.cpp | 137 ++++++++++++++ cpp/src/c_api/edgelist.cpp | 83 +++++++++ cpp/src/c_api/edgelist.hpp | 34 ++++ .../cugraph/structure/graph_classes.py | 33 ---- .../simpleDistributedGraph.py | 99 ++++++++-- .../graph_implementation/simpleGraph.py | 75 ++++++-- .../cugraph/cugraph/structure/number_map.py | 27 +-- .../cugraph/tests/structure/test_graph.py | 52 ++++++ .../cugraph/tests/structure/test_graph_mg.py | 54 ++++++ .../pylibcugraph/pylibcugraph/CMakeLists.txt | 1 + python/pylibcugraph/pylibcugraph/__init__.py | 2 + .../_cugraph_c/graph_functions.pxd | 65 ++++++- .../pylibcugraph/decompress_to_edgelist.pyx | 169 ++++++++++++++++++ 15 files changed, 840 insertions(+), 95 deletions(-) create mode 100644 cpp/src/c_api/decompress_to_edgelist.cpp create mode 100644 cpp/src/c_api/edgelist.cpp create mode 100644 cpp/src/c_api/edgelist.hpp create mode 100644 python/pylibcugraph/pylibcugraph/decompress_to_edgelist.pyx diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 27e1999cb75..65772b4f5dd 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -538,6 +538,8 @@ add_library(cugraph_c src/c_api/weakly_connected_components.cpp src/c_api/strongly_connected_components.cpp src/c_api/allgather.cpp + src/c_api/decompress_to_edgelist.cpp + src/c_api/edgelist.cpp ) add_library(cugraph::cugraph_c ALIAS cugraph_c) diff --git a/cpp/include/cugraph_c/graph_functions.h b/cpp/include/cugraph_c/graph_functions.h index ff7e439232a..964b2f2c8d6 100644 --- a/cpp/include/cugraph_c/graph_functions.h +++ b/cpp/include/cugraph_c/graph_functions.h @@ -104,6 +104,8 @@ cugraph_error_code_t cugraph_two_hop_neighbors( /** * @brief Opaque induced subgraph type + * + * @deprecated This API will be deleted, use cugraph_edgelist_t */ typedef struct { int32_t align_; @@ -112,6 +114,8 @@ typedef struct { /** * @brief Get the source vertex ids * + * @deprecated This API will be deleted, use cugraph_edgelist_get_sources + * * @param [in] induced_subgraph Opaque pointer to induced subgraph * @return type erased array view of source vertex ids */ @@ -121,6 +125,8 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_sources( /** * @brief Get the destination vertex ids * + * @deprecated This API will be deleted, use cugraph_edgelist_get_destinations + * * @param [in] induced_subgraph Opaque pointer to induced subgraph * @return type erased array view of destination vertex ids */ @@ -130,6 +136,8 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_destinatio /** * @brief Get the edge weights * + * @deprecated This API will be deleted, use cugraph_edgelist_get_edge_weights + * * @param [in] induced_subgraph Opaque pointer to induced subgraph * @return type erased array view of edge weights */ @@ -139,6 +147,8 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_weigh /** * @brief Get the edge ids * + * @deprecated This API will be deleted, use cugraph_edgelist_get_edge_ids + * * @param [in] induced_subgraph Opaque pointer to induced subgraph * @return type erased array view of edge ids */ @@ -148,6 +158,8 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_ids( /** * @brief Get the edge types * + * @deprecated This API will be deleted, use cugraph_edgelist_get_edge_type_ids + * * @param [in] induced_subgraph Opaque pointer to induced subgraph * @return type erased array view of edge types */ @@ -157,6 +169,8 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_type_ /** * @brief Get the subgraph offsets * + * @deprecated This API will be deleted, use cugraph_edgelist_get_edge_offsets + * * @param [in] induced_subgraph Opaque pointer to induced subgraph * @return type erased array view of subgraph identifiers */ @@ -166,6 +180,8 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_subgraph_o /** * @brief Free induced subgraph * + * @deprecated This API will be deleted, use cugraph_edgelist_free + * * @param [in] induced subgraph Opaque pointer to induced subgraph */ void cugraph_induced_subgraph_result_free(cugraph_induced_subgraph_result_t* induced_subgraph); @@ -361,6 +377,92 @@ cugraph_type_erased_device_array_view_t* cugraph_degrees_result_get_out_degrees( */ void cugraph_degrees_result_free(cugraph_degrees_result_t* degrees_result); +/** + * @brief Opaque edgelist type + * + */ +typedef struct { + int32_t align_; +} cugraph_edgelist_t; + +/** + * @brief Get the source vertex ids + * + * @param [in] edgelist Opaque pointer to edgelist + * @return type erased array view of source vertex ids + */ +cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_sources(cugraph_edgelist_t* edgelist); + +/** + * @brief Get the destination vertex ids + * + * @param [in] edgelist Opaque pointer to edgelist + * @return type erased array view of destination vertex ids + */ +cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_destinations( + cugraph_edgelist_t* edgelist); + +/** + * @brief Get the edge weights + * + * @param [in] edgelist Opaque pointer to edgelist + * @return type erased array view of edge weights + */ +cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_edge_weights( + cugraph_edgelist_t* edgelist); + +/** + * @brief Get the edge ids + * + * @param [in] edgelist Opaque pointer to edgelist + * @return type erased array view of edge ids + */ +cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_edge_ids( + cugraph_edgelist_t* edgelist); + +/** + * @brief Get the edge types + * + * @param [in] edgelist Opaque pointer to edgelist + * @return type erased array view of edge types + */ +cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_edge_type_ids( + cugraph_edgelist_t* edgelist); + +/** + * @brief Get the edge offsets + * + * @param [in] edgelist Opaque pointer to edgelist + * @return type erased array view of subgraph identifiers + */ +cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_edge_offsets( + cugraph_edgelist_t* edgelist); + +/** + * @brief Free edgelist + * + * @param [in] edgelist Opaque pointer to edgelist + */ +void cugraph_edgelist_free(cugraph_edgelist_t* edgelist); + +/** + * @brief Construct the edge list from the graph view object. + * + * @param [in] handle Handle for accessing resources + * @param [in] graph Graph to operate on + * @param [in] do_expensive_check A flag to run expensive checks for input arguments (if set to + * true) + * @param [out] result Opaque pointer to edgelist + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_decompress_to_edgelist(const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + bool_t do_expensive_check, + cugraph_edgelist_t** result, + cugraph_error_t** error); + #ifdef __cplusplus } #endif diff --git a/cpp/src/c_api/decompress_to_edgelist.cpp b/cpp/src/c_api/decompress_to_edgelist.cpp new file mode 100644 index 00000000000..75bf0c0fd60 --- /dev/null +++ b/cpp/src/c_api/decompress_to_edgelist.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "c_api/abstract_functor.hpp" +#include "c_api/core_result.hpp" +#include "c_api/edgelist.hpp" +#include "c_api/graph.hpp" +#include "c_api/resource_handle.hpp" +#include "c_api/utils.hpp" + +#include + +#include +#include +#include +#include + +#include + +namespace { + +struct decompress_to_edgelist_functor : public cugraph::c_api::abstract_functor { + raft::handle_t const& handle_; + cugraph::c_api::cugraph_graph_t* graph_{}; + + cugraph::c_api::cugraph_core_result_t const* core_result_{}; + bool do_expensive_check_{}; + cugraph::c_api::cugraph_edgelist_t* result_{}; + + decompress_to_edgelist_functor(cugraph_resource_handle_t const* handle, + cugraph_graph_t* graph, + bool do_expensive_check) + : abstract_functor(), + handle_(*reinterpret_cast(handle)->handle_), + graph_(reinterpret_cast(graph)), + do_expensive_check_(do_expensive_check) + { + } + + template + void operator()() + { + if constexpr (!cugraph::is_candidate::value) { + unsupported(); + } else { + if constexpr (store_transposed) { + error_code_ = cugraph::c_api:: + transpose_storage( + handle_, graph_, error_.get()); + if (error_code_ != CUGRAPH_SUCCESS) return; + } + + auto graph = + reinterpret_cast*>( + graph_->graph_); + + auto graph_view = graph->view(); + + auto edge_weights = reinterpret_cast, + weight_t>*>(graph_->edge_weights_); + + auto edge_ids = reinterpret_cast, + edge_t>*>(graph_->edge_ids_); + + auto edge_types = reinterpret_cast, + edge_type_type_t>*>(graph_->edge_types_); + + auto number_map = reinterpret_cast*>(graph_->number_map_); + + auto [result_src, result_dst, result_wgt, result_edge_id, result_edge_type] = + cugraph::decompress_to_edgelist( + handle_, + graph_view, + (edge_weights != nullptr) ? std::make_optional(edge_weights->view()) : std::nullopt, + (edge_ids != nullptr) ? std::make_optional(edge_ids->view()) : std::nullopt, + (edge_types != nullptr) ? std::make_optional(edge_types->view()) : std::nullopt, + (number_map != nullptr) ? std::make_optional>( + number_map->data(), number_map->size()) + : std::nullopt, + do_expensive_check_); + + result_ = new cugraph::c_api::cugraph_edgelist_t{ + new cugraph::c_api::cugraph_type_erased_device_array_t(result_src, graph_->vertex_type_), + new cugraph::c_api::cugraph_type_erased_device_array_t(result_dst, graph_->vertex_type_), + result_wgt ? new cugraph::c_api::cugraph_type_erased_device_array_t(*result_wgt, + graph_->weight_type_) + : NULL, + result_edge_id ? new cugraph::c_api::cugraph_type_erased_device_array_t(*result_edge_id, + graph_->edge_type_) + : NULL, + result_edge_type ? new cugraph::c_api::cugraph_type_erased_device_array_t( + *result_edge_type, graph_->edge_type_id_type_) + : NULL, + NULL}; + } + } +}; + +} // namespace + +extern "C" cugraph_error_code_t cugraph_decompress_to_edgelist( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + bool_t do_expensive_check, + cugraph_edgelist_t** result, + cugraph_error_t** error) +{ + decompress_to_edgelist_functor functor(handle, graph, do_expensive_check); + + return cugraph::c_api::run_algorithm(graph, functor, result, error); +} diff --git a/cpp/src/c_api/edgelist.cpp b/cpp/src/c_api/edgelist.cpp new file mode 100644 index 00000000000..640b2bf2853 --- /dev/null +++ b/cpp/src/c_api/edgelist.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "c_api/edgelist.hpp" + +#include + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_sources( + cugraph_edgelist_t* edgelist) +{ + auto internal_pointer = reinterpret_cast(edgelist); + return reinterpret_cast(internal_pointer->src_->view()); +} + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_destinations( + cugraph_edgelist_t* edgelist) +{ + auto internal_pointer = reinterpret_cast(edgelist); + return reinterpret_cast(internal_pointer->dst_->view()); +} + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_edge_weights( + cugraph_edgelist_t* edgelist) +{ + auto internal_pointer = reinterpret_cast(edgelist); + return (internal_pointer->wgt_ == nullptr) + ? NULL + : reinterpret_cast( + internal_pointer->wgt_->view()); +} + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_edge_ids( + cugraph_edgelist_t* edgelist) +{ + auto internal_pointer = reinterpret_cast(edgelist); + return (internal_pointer->edge_ids_ == nullptr) + ? NULL + : reinterpret_cast( + internal_pointer->edge_ids_->view()); +} + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_edge_type_ids( + cugraph_edgelist_t* edgelist) +{ + auto internal_pointer = reinterpret_cast(edgelist); + return (internal_pointer->edge_type_ids_ == nullptr) + ? NULL + : reinterpret_cast( + internal_pointer->edge_type_ids_->view()); +} + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_edgelist_get_edge_offsets( + cugraph_edgelist_t* edgelist) +{ + auto internal_pointer = reinterpret_cast(edgelist); + return reinterpret_cast( + internal_pointer->subgraph_offsets_->view()); +} + +extern "C" void cugraph_edgelist_free(cugraph_edgelist_t* edgelist) +{ + auto internal_pointer = reinterpret_cast(edgelist); + delete internal_pointer->src_; + delete internal_pointer->dst_; + delete internal_pointer->wgt_; + delete internal_pointer->edge_ids_; + delete internal_pointer->edge_type_ids_; + delete internal_pointer->subgraph_offsets_; + delete internal_pointer; +} diff --git a/cpp/src/c_api/edgelist.hpp b/cpp/src/c_api/edgelist.hpp new file mode 100644 index 00000000000..bc0f2d337f1 --- /dev/null +++ b/cpp/src/c_api/edgelist.hpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "c_api/array.hpp" + +namespace cugraph { +namespace c_api { + +struct cugraph_edgelist_t { + cugraph_type_erased_device_array_t* src_{}; + cugraph_type_erased_device_array_t* dst_{}; + cugraph_type_erased_device_array_t* wgt_{}; + cugraph_type_erased_device_array_t* edge_ids_{}; + cugraph_type_erased_device_array_t* edge_type_ids_{}; + cugraph_type_erased_device_array_t* subgraph_offsets_{}; +}; + +} // namespace c_api +} // namespace cugraph diff --git a/python/cugraph/cugraph/structure/graph_classes.py b/python/cugraph/cugraph/structure/graph_classes.py index 84234f7e904..90f809fa6c1 100644 --- a/python/cugraph/cugraph/structure/graph_classes.py +++ b/python/cugraph/cugraph/structure/graph_classes.py @@ -115,7 +115,6 @@ def from_cudf_edgelist( edge_type=None, renumber=True, store_transposed=False, - legacy_renum_only=False, symmetrize=None, ): """ @@ -168,13 +167,6 @@ def from_cudf_edgelist( If True, stores the transpose of the adjacency matrix. Required for certain algorithms. - legacy_renum_only : bool, optional (default=False) - If True, skips the C++ renumbering step. Must be true for - pylibcugraph algorithms. Must be false for algorithms - not yet converted to the pylibcugraph C API. - - This parameter is deprecated and will be removed. - symmetrize: bool, optional (default=None) If True, symmetrize the edge list for an undirected graph. Setting this flag to True for a directed graph returns an error. The default @@ -210,7 +202,6 @@ def from_cudf_edgelist( edge_type=edge_type, renumber=renumber, store_transposed=store_transposed, - legacy_renum_only=legacy_renum_only, symmetrize=symmetrize, ) @@ -306,7 +297,6 @@ def from_dask_cudf_edgelist( edge_type=None, renumber=True, store_transposed=False, - legacy_renum_only=False, ): """ Initializes the distributed graph from the dask_cudf.DataFrame @@ -353,13 +343,6 @@ def from_dask_cudf_edgelist( If True, stores the transpose of the adjacency matrix. Required for certain algorithms. - legacy_renum_only : bool, optional (default=False) - If True, skips the C++ renumbering step. Must be true for - pylibcugraph algorithms. Must be false for algorithms - not yet converted to the pylibcugraph C API. - - This parameter is deprecated and will be removed. - """ if self._Impl is None: @@ -378,7 +361,6 @@ def from_dask_cudf_edgelist( edge_type=edge_type, renumber=renumber, store_transposed=store_transposed, - legacy_renum_only=legacy_renum_only, ) # Move to Compat Module @@ -869,7 +851,6 @@ def from_cudf_edgelist( edge_attr=None, renumber=True, store_transposed=False, - legacy_renum_only=False, ): """ Initialize a graph from the edge list. It is an error to call this @@ -909,13 +890,6 @@ def from_cudf_edgelist( If True, stores the transpose of the adjacency matrix. Required for certain algorithms. - legacy_renum_only : bool, optional (default=False) - If True, skips the C++ renumbering step. Must be true for - pylibcugraph algorithms. Must be false for algorithms - not yet converted to the pylibcugraph C API. - - This parameter is deprecated and will be removed. - Examples -------- >>> df = cudf.read_csv(datasets_path / 'karate.csv', delimiter=' ', @@ -945,7 +919,6 @@ def from_dask_cudf_edgelist( edge_attr=None, renumber=True, store_transposed=False, - legacy_renum_only=False, ): """ Initializes the distributed graph from the dask_cudf.DataFrame @@ -980,12 +953,6 @@ def from_dask_cudf_edgelist( If True, stores the transpose of the adjacency matrix. Required for certain algorithms. - legacy_renum_only : bool, optional (default=False) - If True, skips the C++ renumbering step. Must be true for - pylibcugraph algorithms. Must be false for algorithms - not yet converted to the pylibcugraph C API. - - This parameter is deprecated and will be removed. """ raise TypeError("Distributed N-partite graph not supported") diff --git a/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py b/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py index 83dad234287..ced72a6bbe2 100644 --- a/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py +++ b/python/cugraph/cugraph/structure/graph_implementation/simpleDistributedGraph.py @@ -14,6 +14,7 @@ import gc from typing import Union, Iterable import warnings +from typing import Tuple import cudf import cupy as cp @@ -31,6 +32,7 @@ degrees as pylibcugraph_degrees, in_degrees as pylibcugraph_in_degrees, out_degrees as pylibcugraph_out_degrees, + decompress_to_edgelist as pylibcugraph_decompress_to_edgelist, ) from cugraph.structure.number_map import NumberMap @@ -172,7 +174,6 @@ def __from_edgelist( edge_type=None, renumber=True, store_transposed=False, - legacy_renum_only=False, symmetrize=None, ): if not isinstance(input_ddf, dask_cudf.DataFrame): @@ -333,9 +334,7 @@ def __from_edgelist( # the edgelist_df and not do any renumbering. # C++ renumbering is enabled by default for algorithms that # support it (but only called if renumbering is on) - self.compute_renumber_edge_list( - transposed=store_transposed, legacy_renum_only=legacy_renum_only - ) + self.compute_renumber_edge_list(transposed=store_transposed) if renumber is False: self.properties.renumbered = False @@ -979,6 +978,84 @@ def convert_to_cudf(cp_arrays): return ddf + def decompress_to_edgelist( + self, return_unrenumbered_edgelist: bool = True + ) -> dask_cudf.DataFrame: + """ + Extract a the edgelist from a graph. + + Parameters + ---------- + return_unrenumbered_edgelist : bool (default=True) + Flag determining whether to return the original + input edgelist if 'True' or the renumbered one + of 'False' and the edgelist was renumbered. + + Returns + ------- + df : dask_cudf.cudf.DataFrame + Distributed GPU data frame containing all induced sources identifiers, + destination identifiers, and if applicable edge weights, edge ids and + edge types + """ + + # Initialize dask client + client = default_client() + + do_expensive_check = False + + def _call_decompress_to_edgelist( + sID: bytes, + mg_graph_x, + do_expensive_check: bool, + ) -> Tuple[cp.ndarray, cp.ndarray, cp.ndarray, cp.ndarray]: + return pylibcugraph_decompress_to_edgelist( + resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()), + graph=mg_graph_x, + do_expensive_check=do_expensive_check, + ) + + result = [ + client.submit( + _call_decompress_to_edgelist, + Comms.get_session_id(), + self._plc_graph[w], + do_expensive_check, + ) + for w in Comms.get_workers() + ] + wait(result) + + def convert_to_cudf(cp_arrays: cp.ndarray) -> cudf.DataFrame: + cp_src, cp_dst, cp_weight, cp_edge_ids, cp_edge_type_ids = cp_arrays + + df = cudf.DataFrame() + df["src"] = cp_src + df["dst"] = cp_dst + if cp_weight is not None: + df["weight"] = cp_weight + if cp_edge_ids is not None: + df["edge_ids"] = cp_edge_ids + if cp_edge_type_ids is not None: + df["edge_type_ids"] = cp_edge_type_ids + + return df + + cudf_result = [ + client.submit(convert_to_cudf, cp_arrays) for cp_arrays in result + ] + + wait(cudf_result) + + ddf = dask_cudf.from_delayed(cudf_result).persist() + wait(ddf) + + if self.properties.renumbered and return_unrenumbered_edgelist: + ddf = self.renumber_map.unrenumber(ddf, "src") + ddf = self.renumber_map.unrenumber(ddf, "dst") + + return ddf + def select_random_vertices( self, random_state: int = None, num_vertices: int = None ) -> Union[dask_cudf.Series, dask_cudf.DataFrame]: @@ -1214,7 +1291,7 @@ def neighbors(self, n): ddf = self.edgelist.edgelist_df return ddf[ddf["src"] == n]["dst"].reset_index(drop=True) - def compute_renumber_edge_list(self, transposed=False, legacy_renum_only=False): + def compute_renumber_edge_list(self, transposed=False): """ Compute a renumbered edge list This function works in the MNMG pipeline and will transform @@ -1237,20 +1314,9 @@ def compute_renumber_edge_list(self, transposed=False, legacy_renum_only=False): structure. If False, renumber with the intent to make a CSR-like structure. Defaults to False. - legacy_renum_only : (optional) bool - if True, The C++ renumbering will not be triggered. - This parameter is added for new algos following the - C/Pylibcugraph path - This parameter is deprecated and will be removed. """ - if legacy_renum_only: - warning_msg = ( - "The parameter 'legacy_renum_only' is deprecated and will be removed." - ) - warnings.warn(warning_msg, DeprecationWarning) - if not self.properties.renumber: self.edgelist = self.EdgeList(self.input_df) self.renumber_map = None @@ -1269,7 +1335,6 @@ def compute_renumber_edge_list(self, transposed=False, legacy_renum_only=False): self.source_columns, self.destination_columns, store_transposed=transposed, - legacy_renum_only=legacy_renum_only, ) self.edgelist = self.EdgeList(renumbered_ddf) diff --git a/python/cugraph/cugraph/structure/graph_implementation/simpleGraph.py b/python/cugraph/cugraph/structure/graph_implementation/simpleGraph.py index 858b114ebdc..4523b7f13b8 100644 --- a/python/cugraph/cugraph/structure/graph_implementation/simpleGraph.py +++ b/python/cugraph/cugraph/structure/graph_implementation/simpleGraph.py @@ -14,6 +14,7 @@ from cugraph.structure import graph_primtypes_wrapper from cugraph.structure.replicate_edgelist import replicate_cudf_dataframe from cugraph.structure.symmetrize import symmetrize as symmetrize_df +from pylibcugraph import decompress_to_edgelist as pylibcugraph_decompress_to_edgelist from cugraph.structure.number_map import NumberMap import cugraph.dask.common.mg_utils as mg_utils import cudf @@ -132,17 +133,9 @@ def __from_edgelist( edge_id=None, edge_type=None, renumber=True, - legacy_renum_only=False, store_transposed=False, symmetrize=None, ): - if legacy_renum_only: - warning_msg = ( - "The parameter 'legacy_renum_only' is deprecated and will be removed." - ) - warnings.warn( - warning_msg, - ) if self.properties.directed and symmetrize: raise ValueError( @@ -266,11 +259,7 @@ def __from_edgelist( if renumber: # FIXME: Should SG do lazy evaluation like MG? elist, renumber_map = NumberMap.renumber( - elist, - source, - destination, - store_transposed=False, - legacy_renum_only=legacy_renum_only, + elist, source, destination, store_transposed=False ) source = renumber_map.renumbered_src_col_name destination = renumber_map.renumbered_dst_col_name @@ -312,6 +301,8 @@ def __from_edgelist( # FIXME: if the user calls self.edgelist.edgelist_df after creating a # symmetric graph, return the symmetric edgelist? + # FIXME: For better memory footprint, avoid storing this edgelist and instead + # call decompress_to_edgelist to extract the edgelist from the graph self.edgelist = simpleGraphImpl.EdgeList( elist[source], elist[destination], value_col ) @@ -804,6 +795,64 @@ def get_two_hop_neighbors(self, start_vertices=None): return df + def decompress_to_edgelist( + self, return_unrenumbered_edgelist: bool = True + ) -> cudf.DataFrame: + """ + Extract a the edgelist from a graph. + + Parameters + ---------- + return_unrenumbered_edgelist : bool (default=True) + Flag determining whether to return the original input edgelist + if 'True' or the renumbered one of 'False' and the edgelist was + renumbered. + + Returns + ------- + + df : cudf.DataFrame + GPU data frame containing all sources identifiers, + destination identifiers and if applicable edge weights, edge ids and + edge types + + Examples + -------- + >>> from cugraph.datasets import karate + >>> G = karate.get_graph(download=True) + >>> edgelist = G.decompress_to_edgelist() + + """ + + do_expensive_check = False + ( + source, + destination, + weight, + edge_ids, + edge_type_ids, + ) = pylibcugraph_decompress_to_edgelist( + resource_handle=ResourceHandle(), + graph=self._plc_graph, + do_expensive_check=do_expensive_check, + ) + + df = cudf.DataFrame() + df["src"] = source + df["dst"] = destination + if weight is not None: + df["weight"] = weight + if edge_ids is not None: + df["edge_ids"] = edge_ids + if edge_type_ids is not None: + df["edge_type_ids"] = edge_type_ids + + if self.properties.renumbered and return_unrenumbered_edgelist: + df, _ = self.renumber_map.unrenumber(df, "src", get_column_names=True) + df, _ = self.renumber_map.unrenumber(df, "dst", get_column_names=True) + + return df + def select_random_vertices( self, random_state: int = None, diff --git a/python/cugraph/cugraph/structure/number_map.py b/python/cugraph/cugraph/structure/number_map.py index b0118fee960..39738daff36 100644 --- a/python/cugraph/cugraph/structure/number_map.py +++ b/python/cugraph/cugraph/structure/number_map.py @@ -18,7 +18,6 @@ import dask_cudf import numpy as np import cudf -import warnings class NumberMap: @@ -462,12 +461,7 @@ def from_internal_vertex_id( @staticmethod def renumber_and_segment( - df, - src_col_names, - dst_col_names, - preserve_order=False, - store_transposed=False, - legacy_renum_only=False, + df, src_col_names, dst_col_names, preserve_order=False, store_transposed=False ): """ Given an input dataframe with its column names, this function returns the @@ -475,11 +469,6 @@ def renumber_and_segment( to external vertex IDs. the parameter 'preserve_order' ensures that the order of the edges is preserved during renumbering. """ - if legacy_renum_only: - warning_msg = ( - "The parameter 'legacy_renum_only' is deprecated and will be removed." - ) - warnings.warn(warning_msg, DeprecationWarning) renumbered = False @@ -584,20 +573,10 @@ def renumber_and_segment( @staticmethod def renumber( - df, - src_col_names, - dst_col_names, - preserve_order=False, - store_transposed=False, - legacy_renum_only=False, + df, src_col_names, dst_col_names, preserve_order=False, store_transposed=False ): return NumberMap.renumber_and_segment( - df, - src_col_names, - dst_col_names, - preserve_order, - store_transposed, - legacy_renum_only, + df, src_col_names, dst_col_names, preserve_order, store_transposed )[0:2] def unrenumber(self, df, column_name, preserve_order=False, get_column_names=False): diff --git a/python/cugraph/cugraph/tests/structure/test_graph.py b/python/cugraph/cugraph/tests/structure/test_graph.py index b3e517100e1..6fcfef726b1 100644 --- a/python/cugraph/cugraph/tests/structure/test_graph.py +++ b/python/cugraph/cugraph/tests/structure/test_graph.py @@ -179,6 +179,58 @@ def test_add_edge_list_to_adj_list(graph_file): assert values_cu is None +@pytest.mark.sg +@pytest.mark.parametrize("graph_file", utils.DATASETS) +@pytest.mark.parametrize("is_directed", [True, False]) +@pytest.mark.parametrize("renumber", [True, False]) +def test_decompress_to_edgelist(graph_file, is_directed, renumber): + input_df = utils.read_csv_file(graph_file) + input_df = input_df.rename(columns={"0": "src", "1": "dst", "2": "weight"}) + + G = cugraph.Graph(directed=is_directed) + input_df_ = cudf.DataFrame() + if renumber: + input_df_["src_0"] = cudf.Series(input_df["src"]) + input_df_["dst_0"] = cudf.Series(input_df["dst"]) + input_df_["weight"] = cudf.Series(input_df["weight"]) + input_df_["src_1"] = input_df_["src_0"] + 1000 + input_df_["dst_1"] = input_df_["dst_0"] + 1000 + + input_df = input_df_ + source = ["src_0", "src_1"] + destination = ["dst_0", "dst_1"] + else: + source = "src" + destination = "dst" + + G.from_cudf_edgelist( + input_df, source=source, destination=destination, weight="weight", renumber=True + ) + + extracted_df = G.decompress_to_edgelist(return_unrenumbered_edgelist=True) + + if renumber: + extracted_df = extracted_df.rename( + columns={ + "0_src": "src_0", + "1_src": "src_1", + "0_dst": "dst_0", + "1_dst": "dst_1", + } + ) + extracted_df = extracted_df.sort_values( + ["src_0", "src_1", "dst_0", "dst_1"] + ).reset_index(drop=True) + input_df = input_df.sort_values( + ["src_0", "src_1", "dst_0", "dst_1"] + ).reset_index(drop=True) + else: + extracted_df = extracted_df.sort_values(["src", "dst"]).reset_index(drop=True) + input_df = input_df.sort_values(["src", "dst"]).reset_index(drop=True) + + assert_frame_equal(input_df, extracted_df, check_dtype=False, check_like=True) + + # Test @pytest.mark.sg @pytest.mark.parametrize("graph_file", utils.DATASETS) diff --git a/python/cugraph/cugraph/tests/structure/test_graph_mg.py b/python/cugraph/cugraph/tests/structure/test_graph_mg.py index f2cc1583f93..e5eeb0f653b 100644 --- a/python/cugraph/cugraph/tests/structure/test_graph_mg.py +++ b/python/cugraph/cugraph/tests/structure/test_graph_mg.py @@ -420,3 +420,57 @@ def test_graph_creation_properties(dask_client, graph_file, directed, renumber): assert sG.number_of_nodes() == mG.number_of_nodes() assert sG.number_of_edges() == mG.number_of_edges() assert_frame_equal(sG_edgelist_view, mG_edgelist_view, check_dtype=False) + + +@pytest.mark.parametrize("directed", [True, False]) +@pytest.mark.parametrize("renumber", [True, False]) +@pytest.mark.parametrize("graph_file", datasets) +def test_decompress_to_edgelist(dask_client, graph_file, directed, renumber): + input_df = utils.read_csv_file(graph_file) + input_df = input_df.rename(columns={"0": "src", "1": "dst", "2": "weight"}) + + G = cugraph.Graph(directed=directed) + input_df_ = cudf.DataFrame() + if renumber: + input_df_["src_0"] = cudf.Series(input_df["src"]) + input_df_["dst_0"] = cudf.Series(input_df["dst"]) + input_df_["weight"] = cudf.Series(input_df["weight"]) + input_df_["src_1"] = input_df_["src_0"] + 1000 + input_df_["dst_1"] = input_df_["dst_0"] + 1000 + + input_df = input_df_ + source = ["src_0", "src_1"] + destination = ["dst_0", "dst_1"] + else: + source = "src" + destination = "dst" + num_workers = len(Comms.get_workers()) + + input_ddf = dask_cudf.from_cudf(input_df, npartitions=num_workers) + + G = cugraph.Graph(directed=True) + G.from_dask_cudf_edgelist( + input_ddf, source=source, destination=destination, weight="weight" + ) + + extracted_df = ( + G.decompress_to_edgelist(return_unrenumbered_edgelist=True) + .compute() + .reset_index(drop=True) + ) + + if renumber: + extracted_df = extracted_df.rename( + columns={ + "0_src": "src_0", + "1_src": "src_1", + "0_dst": "dst_0", + "1_dst": "dst_1", + } + ) + extracted_df = extracted_df.sort_values( + ["src_0", "src_1", "dst_0", "dst_1"] + ).reset_index(drop=True) + input_df = input_df.sort_values( + ["src_0", "src_1", "dst_0", "dst_1"] + ).reset_index(drop=True) diff --git a/python/pylibcugraph/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/CMakeLists.txt index fb46030bc56..fe7c4b64aa5 100644 --- a/python/pylibcugraph/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/CMakeLists.txt @@ -70,6 +70,7 @@ set(cython_sources homogeneous_biased_neighbor_sample.pyx homogeneous_uniform_neighbor_sample.pyx edge_id_lookup_table.pyx + decompress_to_edgelist.pyx ) set(linked_libraries cugraph::cugraph;cugraph::cugraph_c) diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 5aa351f9ce1..9047144c13a 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -126,6 +126,8 @@ from pylibcugraph.degrees import in_degrees, out_degrees, degrees +from pylibcugraph.decompress_to_edgelist import decompress_to_edgelist + from pylibcugraph import exceptions diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd index b8f16cb94c8..b27a7230a13 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd @@ -122,41 +122,41 @@ cdef extern from "cugraph_c/graph_functions.h": ########################################################################### # induced_subgraph - ctypedef struct cugraph_induced_subgraph_result_t: + ctypedef struct cugraph_induced_subgraph_result_t: # Deprecated pass cdef cugraph_type_erased_device_array_view_t* \ - cugraph_induced_subgraph_get_sources( + cugraph_induced_subgraph_get_sources( # Deprecated cugraph_induced_subgraph_result_t* induced_subgraph ) cdef cugraph_type_erased_device_array_view_t* \ - cugraph_induced_subgraph_get_destinations( + cugraph_induced_subgraph_get_destinations( # Deprecated cugraph_induced_subgraph_result_t* induced_subgraph ) cdef cugraph_type_erased_device_array_view_t* \ - cugraph_induced_subgraph_get_edge_weights( + cugraph_induced_subgraph_get_edge_weights( # Deprecated cugraph_induced_subgraph_result_t* induced_subgraph ) cdef cugraph_type_erased_device_array_view_t* \ - cugraph_induced_subgraph_get_edge_ids( + cugraph_induced_subgraph_get_edge_ids( # Deprecated cugraph_induced_subgraph_result_t* induced_subgraph ) cdef cugraph_type_erased_device_array_view_t* \ - cugraph_induced_subgraph_get_edge_type_ids( + cugraph_induced_subgraph_get_edge_type_ids( # Deprecated cugraph_induced_subgraph_result_t* induced_subgraph ) cdef cugraph_type_erased_device_array_view_t* \ - cugraph_induced_subgraph_get_subgraph_offsets( + cugraph_induced_subgraph_get_subgraph_offsets( # Deprecated cugraph_induced_subgraph_result_t* induced_subgraph ) cdef void \ - cugraph_induced_subgraph_result_free( + cugraph_induced_subgraph_result_free( # Deprecated cugraph_induced_subgraph_result_t* induced_subgraph ) @@ -250,3 +250,52 @@ cdef extern from "cugraph_c/graph_functions.h": cugraph_degrees_result_free( cugraph_degrees_result_t* degrees_result ) + + ########################################################################### + # decompress to edgelist + ctypedef struct cugraph_edgelist_t: + pass + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_edgelist_get_sources( + cugraph_edgelist_t* edgelist + ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_edgelist_get_destinations( + cugraph_edgelist_t* edgelist + ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_edgelist_get_edge_weights( + cugraph_edgelist_t* edgelist + ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_edgelist_get_edge_ids( + cugraph_edgelist_t* edgelist + ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_edgelist_get_edge_type_ids( + cugraph_edgelist_t* edgelist + ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_edgelist_get_edge_offsets( + cugraph_edgelist_t* edgelist + ) + + cdef void \ + cugraph_edgelist_free( + cugraph_edgelist_t* edgelist + ) + + cdef cugraph_error_code_t \ + cugraph_decompress_to_edgelist( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + bool_t do_expensive_check, + cugraph_edgelist_t** result, + cugraph_error_t** error + ) diff --git a/python/pylibcugraph/pylibcugraph/decompress_to_edgelist.pyx b/python/pylibcugraph/pylibcugraph/decompress_to_edgelist.pyx new file mode 100644 index 00000000000..58c29940aba --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/decompress_to_edgelist.pyx @@ -0,0 +1,169 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Have cython use python 3 syntax +# cython: language_level = 3 + + +from pylibcugraph._cugraph_c.types cimport ( + bool_t, +) +from pylibcugraph._cugraph_c.resource_handle cimport ( + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c.error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c.array cimport ( + cugraph_type_erased_device_array_view_t, +) +from pylibcugraph._cugraph_c.graph cimport ( + cugraph_graph_t, +) +from pylibcugraph._cugraph_c.graph_functions cimport ( + cugraph_edgelist_t, + cugraph_decompress_to_edgelist, + cugraph_edgelist_get_sources, + cugraph_edgelist_get_destinations, + cugraph_edgelist_get_edge_weights, + cugraph_edgelist_get_edge_ids, + cugraph_edgelist_get_edge_type_ids, + cugraph_edgelist_get_edge_offsets, + cugraph_edgelist_free, +) + +from pylibcugraph.resource_handle cimport ( + ResourceHandle, +) +from pylibcugraph.graphs cimport ( + _GPUGraph, +) +from pylibcugraph.utils cimport ( + assert_success, + copy_to_cupy_array, + create_cugraph_type_erased_device_array_view_from_py_obj, +) + + +def decompress_to_edgelist(ResourceHandle resource_handle, + _GPUGraph graph, + bool_t do_expensive_check): + """ + Extract a the edgelist from a graph + + Parameters + ---------- + resource_handle : ResourceHandle + Handle to the underlying device resources needed for referencing data + and running algorithms. + + graph : SGGraph or MGGraph + The input graph. + + do_expensive_check : bool_t + If True, performs more extensive tests on the inputs to ensure + validitity, at the expense of increased run time. + + Returns + ------- + A tuple of device arrays containing the sources, destinations and if applicable + edge_weights, edge_ids and/or edge_type_ids. + + Examples + -------- + >>> import pylibcugraph, cupy, numpy + >>> srcs = cupy.asarray([0, 1, 1, 2, 2, 2, 3, 4], dtype=numpy.int32) + >>> dsts = cupy.asarray([1, 3, 4, 0, 1, 3, 5, 5], dtype=numpy.int32) + >>> weights = cupy.asarray( + ... [0.1, 2.1, 1.1, 5.1, 3.1, 4.1, 7.2, 3.2], dtype=numpy.float32) + >>> resource_handle = pylibcugraph.ResourceHandle() + >>> graph_props = pylibcugraph.GraphProperties( + ... is_symmetric=False, is_multigraph=False) + >>> G = pylibcugraph.SGGraph( + ... resource_handle, graph_props, srcs, dsts, weight_array=weights, + ... store_transposed=False, renumber=False, do_expensive_check=False) + >>> (sources, destinations, edge_weights, _, _) = + ... pylibcugraph.decompress_to_edgelist( + ... resource_handle, G, False) + >>> sources + [0, 1, 1, 2, 2, 2, 3, 4] + >>> destinations + [1, 3, 4, 0, 1, 3, 5, 5] + >>> edge_weights + [0.1, 2.1, 1.1, 5.1, 3.1, 4.1, 7.2, 3.2] + """ + + cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ + resource_handle.c_resource_handle_ptr + cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr + cdef cugraph_edgelist_t* result_ptr + cdef cugraph_error_code_t error_code + cdef cugraph_error_t* error_ptr + + error_code = cugraph_decompress_to_edgelist(c_resource_handle_ptr, + c_graph_ptr, + do_expensive_check, + &result_ptr, + &error_ptr) + assert_success(error_code, error_ptr, "cugraph_decompress_to_edgelist") + + # Extract individual device array pointers from result and copy to cupy + # arrays for returning. + cdef cugraph_type_erased_device_array_view_t* sources_ptr = \ + cugraph_edgelist_get_sources(result_ptr) + cdef cugraph_type_erased_device_array_view_t* destinations_ptr = \ + cugraph_edgelist_get_destinations(result_ptr) + cdef cugraph_type_erased_device_array_view_t* edge_weights_ptr = \ + cugraph_edgelist_get_edge_weights(result_ptr) + + cdef cugraph_type_erased_device_array_view_t* edge_ids_ptr = \ + cugraph_edgelist_get_edge_ids(result_ptr) + cdef cugraph_type_erased_device_array_view_t* edge_type_ids_ptr = \ + cugraph_edgelist_get_edge_type_ids(result_ptr) + + + """ + cdef cugraph_type_erased_device_array_view_t* subgraph_offsets_ptr = \ + cugraph_edgelist_get_edge_offsets(result_ptr) + """ + + # FIXME: Get ownership of the result data instead of performing a copy + # for perfomance improvement + cupy_edge_weights = None + cupy_edge_ids = None + cupy_edge_type_ids = None + cupy_sources = copy_to_cupy_array( + c_resource_handle_ptr, sources_ptr) + cupy_destinations = copy_to_cupy_array( + c_resource_handle_ptr, destinations_ptr) + if edge_weights_ptr != NULL: + cupy_edge_weights = copy_to_cupy_array( + c_resource_handle_ptr, edge_weights_ptr) + if edge_ids_ptr != NULL: + cupy_edge_ids = copy_to_cupy_array( + c_resource_handle_ptr, edge_ids_ptr) + if edge_type_ids_ptr != NULL: + cupy_edge_type_ids = copy_to_cupy_array( + c_resource_handle_ptr, edge_type_ids_ptr) + + """ + cupy_subgraph_offsets = copy_to_cupy_array( + c_resource_handle_ptr, subgraph_offsets_ptr) + """ + + # Free pointer + cugraph_edgelist_free(result_ptr) + + return (cupy_sources, cupy_destinations, + cupy_edge_weights, cupy_edge_ids, cupy_edge_type_ids)