diff --git a/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh b/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh index a1d32474e5f..3059e3d4bdf 100755 --- a/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh +++ b/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh @@ -30,7 +30,7 @@ algos=" datasets=" netscience email_Eu_core - cit_patents + cit-patents hollywood soc-livejournal " diff --git a/ci/run_nx_cugraph_pytests.sh b/ci/run_nx_cugraph_pytests.sh index b0caffd0a0f..0e309d1e2d4 100755 --- a/ci/run_nx_cugraph_pytests.sh +++ b/ci/run_nx_cugraph_pytests.sh @@ -6,4 +6,5 @@ set -euo pipefail # Support invoking run_nx_cugraph_pytests.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/nx-cugraph/nx_cugraph -pytest --capture=no --cache-clear --benchmark-disable "$@" tests +NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest --capture=no --cache-clear --benchmark-disable "$@" tests +NX_CUGRAPH_USE_COMPAT_GRAPHS=True pytest --capture=no --cache-clear --benchmark-disable "$@" tests diff --git a/ci/test_python.sh b/ci/test_python.sh index 293be07d85e..d5a578c2c43 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -108,7 +108,7 @@ echo "nx-cugraph coverage from networkx tests: $_coverage" echo $_coverage | awk '{ if ($NF == "0.0%") exit 1 }' # Ensure all algorithms were called by comparing covered lines to function lines. # Run our tests again (they're fast enough) to add their coverage, then create coverage.json -pytest \ +NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest \ --pyargs nx_cugraph \ --config-file=../pyproject.toml \ --cov-config=../pyproject.toml \ diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 158704e08d1..e3690dfde6e 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -37,6 +37,7 @@ else DASK_DISTRIBUTED__SCHEDULER__WORKER_TTL="1000s" \ DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT="1000s" \ DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT="1000s" \ + NX_CUGRAPH_USE_COMPAT_GRAPHS=False \ python -m pytest \ -v \ --import-mode=append \ diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 55932298e8e..30cba4577b9 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -33,7 +33,7 @@ dependencies: - libraft==24.10.*,>=0.0.0a0 - librmm==24.10.*,>=0.0.0a0 - nbsphinx -- nccl>=2.9.9 +- nccl>=2.18.1.1 - networkx>=2.5.1 - networkx>=3.0 - ninja @@ -44,7 +44,6 @@ dependencies: - nvcc_linux-64=11.8 - ogb - openmpi -- packaging>=21 - pandas - pre-commit - pydantic diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 38765dc8f0f..1789b2dd410 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -39,7 +39,7 @@ dependencies: - libraft==24.10.*,>=0.0.0a0 - librmm==24.10.*,>=0.0.0a0 - nbsphinx -- nccl>=2.9.9 +- nccl>=2.18.1.1 - networkx>=2.5.1 - networkx>=3.0 - ninja @@ -49,7 +49,6 @@ dependencies: - numpydoc - ogb - openmpi -- packaging>=21 - pandas - pre-commit - pydantic diff --git a/conda/recipes/libcugraph/conda_build_config.yaml b/conda/recipes/libcugraph/conda_build_config.yaml index 26aa428d7f5..6b50d0aad23 100644 --- a/conda/recipes/libcugraph/conda_build_config.yaml +++ b/conda/recipes/libcugraph/conda_build_config.yaml @@ -17,7 +17,7 @@ doxygen_version: - ">=1.8.11" nccl_version: - - ">=2.9.9" + - ">=2.18.1.1" c_stdlib: - sysroot diff --git a/cpp/include/cugraph/mtmg/instance_manager.hpp b/cpp/include/cugraph/mtmg/instance_manager.hpp index a2111804997..759635b4a34 100644 --- a/cpp/include/cugraph/mtmg/instance_manager.hpp +++ b/cpp/include/cugraph/mtmg/instance_manager.hpp @@ -20,6 +20,8 @@ #include +#include + #include namespace cugraph { diff --git a/cpp/include/cugraph/mtmg/resource_manager.hpp b/cpp/include/cugraph/mtmg/resource_manager.hpp index a9e4b81f894..e9d25c4576b 100644 --- a/cpp/include/cugraph/mtmg/resource_manager.hpp +++ b/cpp/include/cugraph/mtmg/resource_manager.hpp @@ -27,6 +27,8 @@ #include #include +#include + #include namespace cugraph { diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 09b1431e33b..3752e823659 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -220,6 +220,7 @@ function(ConfigureTestMG CMAKE_TEST_NAME) GPUS ${GPU_COUNT} PERCENT 100 INSTALL_COMPONENT_SET testing_mg + INSTALL_TARGET ${CMAKE_TEST_NAME} ) set_tests_properties(${CMAKE_TEST_NAME} PROPERTIES LABELS "CUGRAPH_MG") @@ -302,6 +303,7 @@ function(ConfigureCTestMG CMAKE_TEST_NAME) GPUS ${GPU_COUNT} PERCENT 100 INSTALL_COMPONENT_SET testing_mg + INSTALL_TARGET ${CMAKE_TEST_NAME} ) set_tests_properties(${CMAKE_TEST_NAME} PROPERTIES LABELS "CUGRAPH_C_MG") diff --git a/cpp/tests/mtmg/multi_node_threaded_test.cu b/cpp/tests/mtmg/multi_node_threaded_test.cu index 06ccd4a7fa1..374c432aac5 100644 --- a/cpp/tests/mtmg/multi_node_threaded_test.cu +++ b/cpp/tests/mtmg/multi_node_threaded_test.cu @@ -39,6 +39,7 @@ #include #include +#include #include #include diff --git a/dependencies.yaml b/dependencies.yaml index b498170ba9e..be9378ed1ce 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -530,7 +530,7 @@ dependencies: - aiohttp - fsspec>=0.6.0 - requests - - nccl>=2.9.9 + - nccl>=2.18.1.1 - ucx-proc=*=gpu - &ucx_py_unsuffixed ucx-py==0.40.*,>=0.0.0a0 - output_types: pyproject @@ -688,7 +688,6 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - packaging>=21 # not needed by nx-cugraph tests, but is required for running networkx tests - pytest-mpl cugraph_dgl_dev: diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index 41c18c27ecf..428d266dd2e 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -22,6 +22,7 @@ $ python _nx_cugraph/__init__.py """ +import os from _nx_cugraph._version import __version__ @@ -293,12 +294,20 @@ def get_info(): for key in info_keys: del d[key] + + d["default_config"] = { + "use_compat_graphs": os.environ.get("NX_CUGRAPH_USE_COMPAT_GRAPHS", "true") + .strip() + .lower() + == "true", + } return d -def _check_networkx_version(): - import warnings +def _check_networkx_version() -> tuple[int, int]: + """Check the version of networkx and return ``(major, minor)`` version tuple.""" import re + import warnings import networkx as nx @@ -321,6 +330,10 @@ def _check_networkx_version(): f"{nx.__version__}. Please upgrade (or fix) your Python environment." ) + nxver_major = int(version_major) + nxver_minor = int(re.match(r"^\d+", version_minor).group()) + return (nxver_major, nxver_minor) + if __name__ == "__main__": from pathlib import Path diff --git a/python/nx-cugraph/lint.yaml b/python/nx-cugraph/lint.yaml index b2184a185c4..dab2ea70ef1 100644 --- a/python/nx-cugraph/lint.yaml +++ b/python/nx-cugraph/lint.yaml @@ -26,7 +26,7 @@ repos: - id: mixed-line-ending - id: trailing-whitespace - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.18 + rev: v0.19 hooks: - id: validate-pyproject name: Validate pyproject.toml @@ -40,29 +40,29 @@ repos: hooks: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.17.0 hooks: - id: pyupgrade args: [--py310-plus] - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black # - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.4 + rev: v0.6.7 hooks: - id: ruff args: [--fix-only, --show-fixes] # --unsafe-fixes] - repo: https://github.com/PyCQA/flake8 - rev: 7.1.0 + rev: 7.1.1 hooks: - id: flake8 args: ['--per-file-ignores=_nx_cugraph/__init__.py:E501', '--extend-ignore=B020,SIM105'] # Why is this necessary? additional_dependencies: &flake8_dependencies # These versions need updated manually - - flake8==7.1.0 - - flake8-bugbear==24.4.26 + - flake8==7.1.1 + - flake8-bugbear==24.8.19 - flake8-simplify==0.21.0 - repo: https://github.com/asottile/yesqa rev: v1.5.0 @@ -77,7 +77,7 @@ repos: additional_dependencies: [tomli] files: ^(nx_cugraph|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.4 + rev: v0.6.7 hooks: - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/python/nx-cugraph/nx_cugraph/__init__.py b/python/nx-cugraph/nx_cugraph/__init__.py index 542256fa781..4404e57f645 100644 --- a/python/nx-cugraph/nx_cugraph/__init__.py +++ b/python/nx-cugraph/nx_cugraph/__init__.py @@ -12,6 +12,11 @@ # limitations under the License. from networkx.exception import * +from _nx_cugraph._version import __git_commit__, __version__ +from _nx_cugraph import _check_networkx_version + +_nxver: tuple[int, int] = _check_networkx_version() + from . import utils from . import classes @@ -32,7 +37,10 @@ from . import algorithms from .algorithms import * -from _nx_cugraph._version import __git_commit__, __version__ -from _nx_cugraph import _check_networkx_version +from .interface import BackendInterface -_check_networkx_version() +BackendInterface.Graph = classes.Graph +BackendInterface.DiGraph = classes.DiGraph +BackendInterface.MultiGraph = classes.MultiGraph +BackendInterface.MultiDiGraph = classes.MultiDiGraph +del BackendInterface diff --git a/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py index 60276b7d41b..214970235c6 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py @@ -16,6 +16,7 @@ import networkx as nx import numpy as np +from nx_cugraph import _nxver from nx_cugraph.generators._utils import _create_using_class, _number_and_nodes from nx_cugraph.utils import index_dtype, networkx_algorithm @@ -48,7 +49,7 @@ def complete_bipartite_graph(n1, n2, create_using=None): nodes.extend(range(n2) if nodes2 is None else nodes2) if len(set(nodes)) != len(nodes): raise nx.NetworkXError("Inputs n1 and n2 must contain distinct nodes") - if nx.__version__[:3] <= "3.3": + if _nxver <= (3, 3): name = f"complete_bipartite_graph({orig_n1}, {orig_n2})" else: name = f"complete_bipartite_graph({n1}, {n2})" diff --git a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py index ea1318060e0..52c512c454d 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py @@ -12,9 +12,9 @@ # limitations under the License. import warnings -import networkx as nx import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( _dtype_param, @@ -27,7 +27,7 @@ __all__ = ["louvain_communities"] # max_level argument was added to NetworkX 3.3 -if nx.__version__[:3] <= "3.2": +if _nxver <= (3, 2): _max_level_param = { "max_level : int, optional": ( "Upper limit of the number of macro-iterations (max: 500)." @@ -81,7 +81,7 @@ def _louvain_communities( node_ids, clusters, modularity = plc.louvain( resource_handle=plc.ResourceHandle(), graph=G._get_plc_graph(weight, 1, dtype), - max_level=max_level, # TODO: add this parameter to NetworkX + max_level=max_level, threshold=threshold, resolution=resolution, do_expensive_check=False, diff --git a/python/nx-cugraph/nx_cugraph/algorithms/core.py b/python/nx-cugraph/nx_cugraph/algorithms/core.py index 8eb9a9946e7..e69ee88a17c 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/core.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/core.py @@ -15,6 +15,7 @@ import pylibcugraph as plc import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( _get_int_dtype, @@ -58,9 +59,12 @@ def _(G): @networkx_algorithm(is_incomplete=True, version_added="23.12", _plc="k_truss_subgraph") def k_truss(G, k): if is_nx := isinstance(G, nx.Graph): + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + is_compat_graph = False if nxcg.number_of_selfloops(G) > 0: - if nx.__version__[:3] <= "3.2": + if _nxver <= (3, 2): exc_class = nx.NetworkXError else: exc_class = nx.NetworkXNotImplemented @@ -128,6 +132,7 @@ def k_truss(G, k): node_values, node_masks, key_to_id=key_to_id, + use_compat_graph=is_compat_graph, ) new_graph.graph.update(G.graph) return new_graph diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py index e529b83ab1a..cc59fd5eb64 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py @@ -15,6 +15,7 @@ import numpy as np import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import ( _dtype_param, @@ -53,7 +54,7 @@ def hits( if nstart is not None: nstart = G._dict_to_nodearray(nstart, 0, dtype) if max_iter <= 0: - if nx.__version__[:3] <= "3.2": + if _nxver <= (3, 2): raise ValueError("`maxiter` must be a positive integer.") raise nx.PowerIterationFailedConvergence(max_iter) try: diff --git a/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py b/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py index f53b3458949..75dc5fbc706 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py @@ -23,6 +23,7 @@ @networkx_algorithm(version_added="24.02") def complement(G): + is_compat_graph = isinstance(G, nxcg.Graph) G = _to_graph(G) N = G._N # Upcast to int64 so indices don't overflow. @@ -43,6 +44,7 @@ def complement(G): src_indices.astype(index_dtype), dst_indices.astype(index_dtype), key_to_id=G.key_to_id, + use_compat_graph=is_compat_graph, ) @@ -51,10 +53,16 @@ def reverse(G, copy=True): if not G.is_directed(): raise nx.NetworkXError("Cannot reverse an undirected graph.") if isinstance(G, nx.Graph): - if not copy: + is_compat_graph = isinstance(G, nxcg.Graph) + if not copy and not is_compat_graph: raise RuntimeError( "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.reverse`" ) G = nxcg.from_networkx(G, preserve_all_attrs=True) - return G.reverse(copy=copy) + else: + is_compat_graph = False + rv = G.reverse(copy=copy) + if is_compat_graph: + return rv._to_compat_graph() + return rv diff --git a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py index 7d6d77f34a4..ab3c7214303 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py @@ -14,6 +14,7 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _dtype_param, _get_float_dtype, networkx_algorithm @@ -57,7 +58,7 @@ def shortest_path( paths = nxcg.all_pairs_dijkstra_path(G, weight=weight, dtype=dtype) else: # method == 'bellman-ford': paths = nxcg.all_pairs_bellman_ford_path(G, weight=weight, dtype=dtype) - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): paths = dict(paths) # To target elif method == "unweighted": @@ -129,7 +130,7 @@ def shortest_path_length( # To target elif method == "unweighted": lengths = nxcg.single_target_shortest_path_length(G, target) - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): lengths = dict(lengths) elif method == "dijkstra": lengths = nxcg.single_source_dijkstra_path_length( diff --git a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py index 0e98c366e4a..e9c515632ca 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py @@ -17,6 +17,7 @@ import numpy as np import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _groupby, index_dtype, networkx_algorithm @@ -43,7 +44,7 @@ def single_source_shortest_path_length(G, source, cutoff=None): def single_target_shortest_path_length(G, target, cutoff=None): G = _to_graph(G) rv = _bfs(G, target, cutoff, "Target", return_type="length") - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): return iter(rv.items()) return rv @@ -61,7 +62,7 @@ def bidirectional_shortest_path(G, source, target): # TODO PERF: do bidirectional traversal in core G = _to_graph(G) if source not in G or target not in G: - if nx.__version__[:3] <= "3.3": + if _nxver <= (3, 3): raise nx.NodeNotFound( f"Either source {source} or target {target} is not in G" ) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py index 5e4466d7d33..72d0079cf0c 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py @@ -18,6 +18,7 @@ import pylibcugraph as plc import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _groupby, index_dtype, networkx_algorithm @@ -57,7 +58,7 @@ def _bfs(G, source, *, depth_limit=None, reverse=False): return distances[mask], predecessors[mask], node_ids[mask] -if nx.__version__[:3] <= "3.3": +if _nxver <= (3, 3): @networkx_algorithm(is_incomplete=True, version_added="24.02", _plc="bfs") def generic_bfs_edges( @@ -132,13 +133,15 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): raise NotImplementedError( "sort_neighbors argument in bfs_tree is not currently supported" ) + is_compat_graph = isinstance(G, nxcg.Graph) G = _check_G_and_source(G, source) if depth_limit is not None and depth_limit < 1: - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( 1, cp.array([], dtype=index_dtype), cp.array([], dtype=index_dtype), id_to_key=[source], + use_compat_graph=is_compat_graph, ) distances, predecessors, node_ids = _bfs( @@ -148,11 +151,12 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): reverse=reverse, ) if predecessors.size == 0: - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( 1, cp.array([], dtype=index_dtype), cp.array([], dtype=index_dtype), id_to_key=[source], + use_compat_graph=is_compat_graph, ) # TODO: create renumbering helper function(s) unique_node_ids = cp.unique(cp.hstack((predecessors, node_ids))) @@ -170,11 +174,12 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): old_index: new_index for new_index, old_index in enumerate(unique_node_ids.tolist()) } - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( unique_node_ids.size, src_indices, dst_indices, key_to_id=key_to_id, + use_compat_graph=is_compat_graph, ) diff --git a/python/nx-cugraph/nx_cugraph/classes/__init__.py b/python/nx-cugraph/nx_cugraph/classes/__init__.py index 19a5357da55..71168e5364f 100644 --- a/python/nx-cugraph/nx_cugraph/classes/__init__.py +++ b/python/nx-cugraph/nx_cugraph/classes/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,9 +10,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .graph import Graph -from .digraph import DiGraph -from .multigraph import MultiGraph -from .multidigraph import MultiDiGraph +from .graph import CudaGraph, Graph +from .digraph import CudaDiGraph, DiGraph +from .multigraph import CudaMultiGraph, MultiGraph +from .multidigraph import CudaMultiDiGraph, MultiDiGraph from .function import * diff --git a/python/nx-cugraph/nx_cugraph/classes/digraph.py b/python/nx-cugraph/nx_cugraph/classes/digraph.py index e5cfb8f6815..178bf44f16e 100644 --- a/python/nx-cugraph/nx_cugraph/classes/digraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/digraph.py @@ -18,34 +18,108 @@ import cupy as cp import networkx as nx import numpy as np +from networkx.classes.digraph import ( + _CachedPropertyResetterAdjAndSucc, + _CachedPropertyResetterPred, +) import nx_cugraph as nxcg from ..utils import index_dtype -from .graph import Graph +from .graph import CudaGraph, Graph if TYPE_CHECKING: # pragma: no cover from nx_cugraph.typing import AttrKey -__all__ = ["DiGraph"] +__all__ = ["CudaDiGraph", "DiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.DiGraph) -class DiGraph(Graph): - ################# - # Class methods # - ################# +class DiGraph(nx.DiGraph, Graph): + _nx_attrs = ("_node", "_adj", "_succ", "_pred") + + name = Graph.name + _node = Graph._node + + @property + @networkx_api + def _adj(self): + if (adj := self.__dict__["_adj"]) is None: + self._reify_networkx() + adj = self.__dict__["_adj"] + return adj + + @_adj.setter + def _adj(self, val): + self._prepare_setter() + _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + @networkx_api + def _succ(self): + if (succ := self.__dict__["_succ"]) is None: + self._reify_networkx() + succ = self.__dict__["_succ"] + return succ + + @_succ.setter + def _succ(self, val): + self._prepare_setter() + _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + @networkx_api + def _pred(self): + if (pred := self.__dict__["_pred"]) is None: + self._reify_networkx() + pred = self.__dict__["_pred"] + return pred + + @_pred.setter + def _pred(self, val): + self._prepare_setter() + _CachedPropertyResetterPred.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() @classmethod @networkx_api def is_directed(cls) -> bool: return True + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return False + + @classmethod + def to_cudagraph_class(cls) -> type[CudaDiGraph]: + return CudaDiGraph + @classmethod def to_networkx_class(cls) -> type[nx.DiGraph]: return nx.DiGraph + +class CudaDiGraph(CudaGraph): + ################# + # Class methods # + ################# + + is_directed = classmethod(DiGraph.is_directed.__func__) + is_multigraph = classmethod(DiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(DiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(DiGraph.to_networkx_class.__func__) + + @classmethod + def _to_compat_graph_class(cls) -> type[DiGraph]: + return DiGraph + @networkx_api def size(self, weight: AttrKey | None = None) -> int: if weight is not None: @@ -57,7 +131,7 @@ def size(self, weight: AttrKey | None = None) -> int: ########################## @networkx_api - def reverse(self, copy: bool = True) -> DiGraph: + def reverse(self, copy: bool = True) -> CudaDiGraph: return self._copy(not copy, self.__class__, reverse=True) @networkx_api @@ -162,6 +236,7 @@ def to_undirected(self, reciprocal=False, as_view=False): node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=False, ) if as_view: rv.graph = self.graph diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 7c01365c0ac..cfe1e1c87e9 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -20,8 +20,13 @@ import networkx as nx import numpy as np import pylibcugraph as plc +from networkx.classes.graph import ( + _CachedPropertyResetterAdj, + _CachedPropertyResetterNode, +) import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype @@ -40,57 +45,246 @@ any_ndarray, ) -__all__ = ["Graph"] +__all__ = ["CudaGraph", "Graph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.Graph) +# The "everything" cache key is an internal implementation detail of NetworkX +# that may change between releases. +if _nxver < (3, 4): + _CACHE_KEY = ( + True, # Include all edge values + True, # Include all node values + True, # Include `.graph` attributes + ) +else: + _CACHE_KEY = ( + True, # Include all edge values + True, # Include all node values + # `.graph` attributes are always included now + ) + +# Use to indicate when a full conversion to GPU failed so we don't try again. +_CANT_CONVERT_TO_GPU = "_CANT_CONVERT_TO_GPU" + + +# `collections.UserDict` was the preferred way to subclass dict, but now +# subclassing dict directly is much better supported and should work here. +# This class should only be necessary if the user clears the cache manually. +class _GraphCache(dict): + """Cache that ensures Graph will reify into a NetworkX graph when cleared.""" + + _graph: Graph -class Graph: + def __init__(self, graph: Graph): + self._graph = graph + + def clear(self) -> None: + self._graph._reify_networkx() + super().clear() + + +class Graph(nx.Graph): # Tell networkx to dispatch calls with this object to nx-cugraph __networkx_backend__: ClassVar[str] = "cugraph" # nx >=3.2 __networkx_plugin__: ClassVar[str] = "cugraph" # nx <3.2 + # Core attributes of NetowkrX graphs that will be copied and cleared as appropriate. + # These attributes comprise the edge and node data model for NetworkX graphs. + _nx_attrs = ("_node", "_adj") + # Allow networkx dispatch machinery to cache conversions. # This means we should clear the cache if we ever mutate the object! - __networkx_cache__: dict | None + __networkx_cache__: _GraphCache | None # networkx properties graph: dict - graph_attr_dict_factory: ClassVar[type] = dict + # Should we declare type annotations for the rest? + + # Properties that trigger copying to the CPU + def _prepare_setter(self): + """Be careful when setting private attributes which may be used during init.""" + if ( + # If not present, then this must be in init + any(attr not in self.__dict__ for attr in self._nx_attrs) + # Already on the CPU + or not any(self.__dict__[attr] is None for attr in self._nx_attrs) + ): + return + if self._is_on_gpu: + # Copy from GPU to CPU + self._reify_networkx() + return + # Default values + for attr in self._nx_attrs: + if self.__dict__[attr] is None: + if attr == "_succ": + self.__dict__[attr] = self.__dict__["_adj"] + else: + self.__dict__[attr] = {} - # Not networkx properties - # We store edge data in COO format with {src,dst}_indices and edge_values. - src_indices: cp.ndarray[IndexValue] - dst_indices: cp.ndarray[IndexValue] - edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] - edge_masks: dict[AttrKey, cp.ndarray[bool]] - node_values: dict[AttrKey, any_ndarray[NodeValue]] - node_masks: dict[AttrKey, any_ndarray[bool]] - key_to_id: dict[NodeKey, IndexValue] | None - _id_to_key: list[NodeKey] | None - _N: int - _node_ids: cp.ndarray[IndexValue] | None # holds plc.SGGraph.vertices_array data + @property + @networkx_api + def _node(self): + if (node := self.__dict__["_node"]) is None: + self._reify_networkx() + node = self.__dict__["_node"] + return node + + @_node.setter + def _node(self, val): + self._prepare_setter() + _CachedPropertyResetterNode.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() - # Used by graph._get_plc_graph - _plc_type_map: ClassVar[dict[np.dtype, np.dtype]] = { - # signed int - np.dtype(np.int8): np.dtype(np.float32), - np.dtype(np.int16): np.dtype(np.float32), - np.dtype(np.int32): np.dtype(np.float64), - np.dtype(np.int64): np.dtype(np.float64), # raise if abs(x) > 2**53 - # unsigned int - np.dtype(np.uint8): np.dtype(np.float32), - np.dtype(np.uint16): np.dtype(np.float32), - np.dtype(np.uint32): np.dtype(np.float64), - np.dtype(np.uint64): np.dtype(np.float64), # raise if x > 2**53 - # other - np.dtype(np.bool_): np.dtype(np.float32), - np.dtype(np.float16): np.dtype(np.float32), - } - _plc_allowed_edge_types: ClassVar[set[np.dtype]] = { - np.dtype(np.float32), - np.dtype(np.float64), - } + @property + @networkx_api + def _adj(self): + if (adj := self.__dict__["_adj"]) is None: + self._reify_networkx() + adj = self.__dict__["_adj"] + return adj + + @_adj.setter + def _adj(self, val): + self._prepare_setter() + _CachedPropertyResetterAdj.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + def _is_on_gpu(self) -> bool: + """Whether the full graph is on device (in the cache). + + This returns False when only a subset of the graph (such as only + edge indices and edge attribute) is on device. + + The graph may be on host (CPU) and device (GPU) at the same time. + """ + cache = getattr(self, "__networkx_cache__", None) + if not cache: + return False + return _CACHE_KEY in cache.get("backends", {}).get("cugraph", {}) + + @property + def _is_on_cpu(self) -> bool: + """Whether the graph is on host as a NetworkX graph. + + This means the core data structures that comprise a NetworkX graph + (such as ``G._node`` and ``G._adj``) are present. + + The graph may be on host (CPU) and device (GPU) at the same time. + """ + return self.__dict__["_node"] is not None + + @property + def _cudagraph(self): + """Return the full ``CudaGraph`` on device, computing if necessary, or None.""" + nx_cache = getattr(self, "__networkx_cache__", None) + if nx_cache is None: + nx_cache = {} + elif _CANT_CONVERT_TO_GPU in nx_cache: + return None + cache = nx_cache.setdefault("backends", {}).setdefault("cugraph", {}) + if (Gcg := cache.get(_CACHE_KEY)) is not None: + if isinstance(Gcg, Graph): + # This shouldn't happen during normal use, but be extra-careful anyway + return Gcg._cudagraph + return Gcg + if self.__dict__["_node"] is None: + raise RuntimeError( + f"{type(self).__name__} cannot be converted to the GPU, because it is " + "not on the CPU! This is not supposed to be possible. If you believe " + "you have found a bug, please report a minimum reproducible example to " + "https://github.com/rapidsai/cugraph/issues/new/choose" + ) + try: + Gcg = nxcg.from_networkx( + self, preserve_edge_attrs=True, preserve_node_attrs=True + ) + except Exception: + # Should we warn that the full graph can't be on GPU? + nx_cache[_CANT_CONVERT_TO_GPU] = True + return None + Gcg.graph = self.graph + cache[_CACHE_KEY] = Gcg + return Gcg + + @_cudagraph.setter + def _cudagraph(self, val, *, clear_cpu=True): + """Set the full ``CudaGraph`` for this graph, or remove from device if None.""" + if (cache := getattr(self, "__networkx_cache__", None)) is None: + # Should we warn? + return + # TODO: pay close attention to when we should clear the cache, since + # this may or may not be a mutation. + cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + if val is None: + cache.pop(_CACHE_KEY, None) + else: + self.graph = val.graph + cache[_CACHE_KEY] = val + if clear_cpu: + for key in self._nx_attrs: + self.__dict__[key] = None + + @nx.Graph.name.setter + def name(self, s): + # Don't clear the cache when setting the name, since `.graph` is shared. + # There is a very small risk here for the cache to become (slightly) + # insconsistent if graphs from other backends are cached. + self.graph["name"] = s + + @classmethod + @networkx_api + def is_directed(cls) -> bool: + return False + + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return False + + @classmethod + def to_cudagraph_class(cls) -> type[CudaGraph]: + return CudaGraph + + @classmethod + @networkx_api + def to_directed_class(cls) -> type[nxcg.DiGraph]: + return nxcg.DiGraph + + @classmethod + def to_networkx_class(cls) -> type[nx.Graph]: + return nx.Graph + + @classmethod + @networkx_api + def to_undirected_class(cls) -> type[Graph]: + return Graph + + def __init__(self, incoming_graph_data=None, **attr): + super().__init__(incoming_graph_data, **attr) + self.__networkx_cache__ = _GraphCache(self) + + def _reify_networkx(self) -> None: + """Copy graph to host (CPU) if necessary.""" + if self.__dict__["_node"] is None: + # After we make this into an nx graph, we rely on the cache being correct + Gcg = self._cudagraph + G = nxcg.to_networkx(Gcg) + for key in self._nx_attrs: + self.__dict__[key] = G.__dict__[key] + + def _become(self, other: Graph): + if self.__class__ is not other.__class__: + raise TypeError( + "Attempting to update graph inplace with graph of different type!" + ) + # Begin with the simplest implementation; do we need to do more? + self.__dict__.update(other.__dict__) + return self #################### # Creation methods # @@ -109,9 +303,10 @@ def from_coo( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: - new_graph = object.__new__(cls) + ) -> Graph | CudaGraph: + new_graph = object.__new__(cls.to_cudagraph_class()) new_graph.__networkx_cache__ = {} new_graph.src_indices = src_indices new_graph.dst_indices = dst_indices @@ -173,7 +368,8 @@ def from_coo( isolates = nxcg.algorithms.isolate._isolates(new_graph) if len(isolates) > 0: new_graph._node_ids = cp.arange(new_graph._N, dtype=index_dtype) - + if use_compat_graph or use_compat_graph is None and issubclass(cls, Graph): + new_graph = new_graph._to_compat_graph() return new_graph @classmethod @@ -188,8 +384,9 @@ def from_csr( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: N = indptr.size - 1 src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -205,6 +402,7 @@ def from_csr( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=use_compat_graph, **attr, ) @@ -220,8 +418,9 @@ def from_csc( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: N = indptr.size - 1 dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -237,6 +436,7 @@ def from_csc( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=use_compat_graph, **attr, ) @@ -254,8 +454,9 @@ def from_dcsr( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_srcs.get(), cp.diff(indptr).get()) @@ -270,6 +471,7 @@ def from_dcsr( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=use_compat_graph, **attr, ) @@ -287,8 +489,9 @@ def from_dcsc( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_dsts.get(), cp.diff(indptr).get()) @@ -303,13 +506,75 @@ def from_dcsc( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=use_compat_graph, **attr, ) - def __new__(cls, incoming_graph_data=None, **attr) -> Graph: + +class CudaGraph: + # Tell networkx to dispatch calls with this object to nx-cugraph + __networkx_backend__: ClassVar[str] = "cugraph" # nx >=3.2 + __networkx_plugin__: ClassVar[str] = "cugraph" # nx <3.2 + + # Allow networkx dispatch machinery to cache conversions. + # This means we should clear the cache if we ever mutate the object! + __networkx_cache__: dict | None + + # networkx properties + graph: dict + graph_attr_dict_factory: ClassVar[type] = dict + + # Not networkx properties + # We store edge data in COO format with {src,dst}_indices and edge_values. + src_indices: cp.ndarray[IndexValue] + dst_indices: cp.ndarray[IndexValue] + edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] + edge_masks: dict[AttrKey, cp.ndarray[bool]] + node_values: dict[AttrKey, any_ndarray[NodeValue]] + node_masks: dict[AttrKey, any_ndarray[bool]] + key_to_id: dict[NodeKey, IndexValue] | None + _id_to_key: list[NodeKey] | None + _N: int + _node_ids: cp.ndarray[IndexValue] | None # holds plc.SGGraph.vertices_array data + + # Used by graph._get_plc_graph + _plc_type_map: ClassVar[dict[np.dtype, np.dtype]] = { + # signed int + np.dtype(np.int8): np.dtype(np.float32), + np.dtype(np.int16): np.dtype(np.float32), + np.dtype(np.int32): np.dtype(np.float64), + np.dtype(np.int64): np.dtype(np.float64), # raise if abs(x) > 2**53 + # unsigned int + np.dtype(np.uint8): np.dtype(np.float32), + np.dtype(np.uint16): np.dtype(np.float32), + np.dtype(np.uint32): np.dtype(np.float64), + np.dtype(np.uint64): np.dtype(np.float64), # raise if x > 2**53 + # other + np.dtype(np.bool_): np.dtype(np.float32), + np.dtype(np.float16): np.dtype(np.float32), + } + _plc_allowed_edge_types: ClassVar[set[np.dtype]] = { + np.dtype(np.float32), + np.dtype(np.float64), + } + + #################### + # Creation methods # + #################### + + from_coo = classmethod(Graph.from_coo.__func__) + from_csr = classmethod(Graph.from_csr.__func__) + from_csc = classmethod(Graph.from_csc.__func__) + from_dcsr = classmethod(Graph.from_dcsr.__func__) + from_dcsc = classmethod(Graph.from_dcsc.__func__) + + def __new__(cls, incoming_graph_data=None, **attr) -> CudaGraph: if incoming_graph_data is None: new_graph = cls.from_coo( - 0, cp.empty(0, index_dtype), cp.empty(0, index_dtype) + 0, + cp.empty(0, index_dtype), + cp.empty(0, index_dtype), + use_compat_graph=False, ) elif incoming_graph_data.__class__ is cls: new_graph = incoming_graph_data.copy() @@ -318,34 +583,30 @@ def __new__(cls, incoming_graph_data=None, **attr) -> Graph: else: raise NotImplementedError new_graph.graph.update(attr) + # We could return Graph here (if configured), but let's not for now return new_graph ################# # Class methods # ################# - @classmethod - @networkx_api - def is_directed(cls) -> bool: - return False + is_directed = classmethod(Graph.is_directed.__func__) + is_multigraph = classmethod(Graph.is_multigraph.__func__) + to_cudagraph_class = classmethod(Graph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(Graph.to_networkx_class.__func__) @classmethod @networkx_api - def is_multigraph(cls) -> bool: - return False + def to_directed_class(cls) -> type[nxcg.CudaDiGraph]: + return nxcg.CudaDiGraph @classmethod @networkx_api - def to_directed_class(cls) -> type[nxcg.DiGraph]: - return nxcg.DiGraph - - @classmethod - def to_networkx_class(cls) -> type[nx.Graph]: - return nx.Graph + def to_undirected_class(cls) -> type[CudaGraph]: + return CudaGraph @classmethod - @networkx_api - def to_undirected_class(cls) -> type[Graph]: + def _to_compat_graph_class(cls) -> type[Graph]: return Graph ############## @@ -438,7 +699,7 @@ def clear_edges(self) -> None: cache.clear() @networkx_api - def copy(self, as_view: bool = False) -> Graph: + def copy(self, as_view: bool = False) -> CudaGraph: # Does shallow copy in networkx return self._copy(as_view, self.__class__) @@ -534,14 +795,19 @@ def size(self, weight: AttrKey | None = None) -> int: return int(cp.count_nonzero(self.src_indices <= self.dst_indices)) @networkx_api - def to_directed(self, as_view: bool = False) -> nxcg.DiGraph: + def to_directed(self, as_view: bool = False) -> nxcg.CudaDiGraph: return self._copy(as_view, self.to_directed_class()) @networkx_api - def to_undirected(self, as_view: bool = False) -> Graph: + def to_undirected(self, as_view: bool = False) -> CudaGraph: # Does deep copy in networkx return self._copy(as_view, self.to_undirected_class()) + def _to_compat_graph(self) -> Graph: + rv = self._to_compat_graph_class()() + rv._cudagraph = self + return rv + # Not implemented... # adj, adjacency, add_edge, add_edges_from, add_node, # add_nodes_from, add_weighted_edges_from, degree, @@ -552,8 +818,8 @@ def to_undirected(self, as_view: bool = False) -> Graph: # Private methods # ################### - def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): - # DRY warning: see also MultiGraph._copy + def _copy(self, as_view: bool, cls: type[CudaGraph], reverse: bool = False): + # DRY warning: see also CudaMultiGraph._copy src_indices = self.src_indices dst_indices = self.dst_indices edge_values = self.edge_values @@ -593,6 +859,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=False, ) if as_view: rv.graph = self.graph @@ -714,7 +981,7 @@ def _get_plc_graph( ) def _sort_edge_indices(self, primary="src"): - # DRY warning: see also MultiGraph._sort_edge_indices + # DRY warning: see also CudaMultiGraph._sort_edge_indices if primary == "src": stacked = cp.vstack((self.dst_indices, self.src_indices)) elif primary == "dst": @@ -736,7 +1003,7 @@ def _sort_edge_indices(self, primary="src"): {key: val[indices] for key, val in self.edge_masks.items()} ) - def _become(self, other: Graph): + def _become(self, other: CudaGraph): if self.__class__ is not other.__class__: raise TypeError( "Attempting to update graph inplace with graph of different type!" diff --git a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py index 2e7a55a9eb1..5a6595567d2 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py @@ -16,24 +16,51 @@ import nx_cugraph as nxcg -from .digraph import DiGraph -from .multigraph import MultiGraph +from .digraph import CudaDiGraph, DiGraph +from .graph import Graph +from .multigraph import CudaMultiGraph, MultiGraph -__all__ = ["MultiDiGraph"] +__all__ = ["CudaMultiDiGraph", "MultiDiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.MultiDiGraph) -class MultiDiGraph(MultiGraph, DiGraph): +class MultiDiGraph(nx.MultiDiGraph, MultiGraph, DiGraph): + name = Graph.name + _node = Graph._node + _adj = DiGraph._adj + _succ = DiGraph._succ + _pred = DiGraph._pred + @classmethod @networkx_api def is_directed(cls) -> bool: return True + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return True + + @classmethod + def to_cudagraph_class(cls) -> type[CudaMultiDiGraph]: + return CudaMultiDiGraph + @classmethod def to_networkx_class(cls) -> type[nx.MultiDiGraph]: return nx.MultiDiGraph + +class CudaMultiDiGraph(CudaMultiGraph, CudaDiGraph): + is_directed = classmethod(MultiDiGraph.is_directed.__func__) + is_multigraph = classmethod(MultiDiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(MultiDiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(MultiDiGraph.to_networkx_class.__func__) + + @classmethod + def _to_compat_graph_class(cls) -> type[MultiDiGraph]: + return MultiDiGraph + ########################## # NetworkX graph methods # ########################## diff --git a/python/nx-cugraph/nx_cugraph/classes/multigraph.py b/python/nx-cugraph/nx_cugraph/classes/multigraph.py index 23d9faa8734..c8c8f1dfb00 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multigraph.py @@ -22,7 +22,7 @@ import nx_cugraph as nxcg from ..utils import index_dtype -from .graph import Graph +from .graph import CudaGraph, Graph, _GraphCache if TYPE_CHECKING: from nx_cugraph.typing import ( @@ -34,32 +34,47 @@ NodeValue, any_ndarray, ) -__all__ = ["MultiGraph"] +__all__ = ["MultiGraph", "CudaMultiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.MultiGraph) -class MultiGraph(Graph): - # networkx properties - edge_key_dict_factory: ClassVar[type] = dict +class MultiGraph(nx.MultiGraph, Graph): + name = Graph.name + _node = Graph._node + _adj = Graph._adj - # Not networkx properties + @classmethod + @networkx_api + def is_directed(cls) -> bool: + return False - # In a MultiGraph, each edge has a unique `(src, dst, key)` key. - # By default, `key` is 0 if possible, else 1, else 2, etc. - # This key can be any hashable Python object in NetworkX. - # We don't use a dict for our data structure here, because - # that would require a `(src, dst, key)` key. - # Instead, we keep `edge_keys` and/or `edge_indices`. - # `edge_keys` is the list of Python objects for each edge. - # `edge_indices` is for the common case of default multiedge keys, - # in which case we can store it as a cupy array. - # `edge_indices` is generally preferred. It is possible to provide - # both where edge_indices is the default and edge_keys is anything. - # It is also possible for them both to be None, which means the - # default edge indices has not yet been calculated. - edge_indices: cp.ndarray[IndexValue] | None - edge_keys: list[EdgeKey] | None + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return True + + @classmethod + def to_cudagraph_class(cls) -> type[CudaMultiGraph]: + return CudaMultiGraph + + @classmethod + @networkx_api + def to_directed_class(cls) -> type[nxcg.MultiDiGraph]: + return nxcg.MultiDiGraph + + @classmethod + def to_networkx_class(cls) -> type[nx.MultiGraph]: + return nx.MultiGraph + + @classmethod + @networkx_api + def to_undirected_class(cls) -> type[MultiGraph]: + return MultiGraph + + def __init__(self, incoming_graph_data=None, multigraph_input=None, **attr): + super().__init__(incoming_graph_data, multigraph_input, **attr) + self.__networkx_cache__ = _GraphCache(self) #################### # Creation methods # @@ -80,9 +95,10 @@ def from_coo( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: - new_graph = super().from_coo( + ) -> MultiGraph | CudaMultiGraph: + new_graph = super(cls.to_undirected_class(), cls).from_coo( N, src_indices, dst_indices, @@ -92,6 +108,7 @@ def from_coo( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=False, **attr, ) new_graph.edge_indices = edge_indices @@ -102,6 +119,8 @@ def from_coo( and len(new_graph.edge_keys) != src_indices.size ): raise ValueError + if use_compat_graph or use_compat_graph is None and issubclass(cls, Graph): + new_graph = new_graph._to_compat_graph() return new_graph @classmethod @@ -118,8 +137,9 @@ def from_csr( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: N = indptr.size - 1 src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -137,6 +157,7 @@ def from_csr( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=use_compat_graph, **attr, ) @@ -154,8 +175,9 @@ def from_csc( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: N = indptr.size - 1 dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -173,6 +195,7 @@ def from_csc( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=use_compat_graph, **attr, ) @@ -192,8 +215,9 @@ def from_dcsr( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_srcs.get(), cp.diff(indptr).get()) @@ -210,6 +234,7 @@ def from_dcsr( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=use_compat_graph, **attr, ) @@ -229,8 +254,9 @@ def from_dcsc( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> MultiGraph | CudaGraph: dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_dsts.get(), cp.diff(indptr).get()) @@ -247,12 +273,46 @@ def from_dcsc( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=use_compat_graph, **attr, ) + +class CudaMultiGraph(CudaGraph): + # networkx properties + edge_key_dict_factory: ClassVar[type] = dict + + # Not networkx properties + + # In a MultiGraph, each edge has a unique `(src, dst, key)` key. + # By default, `key` is 0 if possible, else 1, else 2, etc. + # This key can be any hashable Python object in NetworkX. + # We don't use a dict for our data structure here, because + # that would require a `(src, dst, key)` key. + # Instead, we keep `edge_keys` and/or `edge_indices`. + # `edge_keys` is the list of Python objects for each edge. + # `edge_indices` is for the common case of default multiedge keys, + # in which case we can store it as a cupy array. + # `edge_indices` is generally preferred. It is possible to provide + # both where edge_indices is the default and edge_keys is anything. + # It is also possible for them both to be None, which means the + # default edge indices has not yet been calculated. + edge_indices: cp.ndarray[IndexValue] | None + edge_keys: list[EdgeKey] | None + + #################### + # Creation methods # + #################### + + from_coo = classmethod(MultiGraph.from_coo.__func__) + from_csr = classmethod(MultiGraph.from_csr.__func__) + from_csc = classmethod(MultiGraph.from_csc.__func__) + from_dcsr = classmethod(MultiGraph.from_dcsr.__func__) + from_dcsc = classmethod(MultiGraph.from_dcsc.__func__) + def __new__( cls, incoming_graph_data=None, multigraph_input=None, **attr - ) -> MultiGraph: + ) -> CudaMultiGraph: if isinstance(incoming_graph_data, dict) and multigraph_input is not False: new_graph = nxcg.from_networkx( nx.MultiGraph(incoming_graph_data, multigraph_input=multigraph_input), @@ -267,28 +327,23 @@ def __new__( # Class methods # ################# - @classmethod - @networkx_api - def is_directed(cls) -> bool: - return False + is_directed = classmethod(MultiGraph.is_directed.__func__) + is_multigraph = classmethod(MultiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(MultiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(MultiGraph.to_networkx_class.__func__) @classmethod @networkx_api - def is_multigraph(cls) -> bool: - return True + def to_directed_class(cls) -> type[nxcg.CudaMultiDiGraph]: + return nxcg.CudaMultiDiGraph @classmethod @networkx_api - def to_directed_class(cls) -> type[nxcg.MultiDiGraph]: - return nxcg.MultiDiGraph - - @classmethod - def to_networkx_class(cls) -> type[nx.MultiGraph]: - return nx.MultiGraph + def to_undirected_class(cls) -> type[CudaMultiGraph]: + return CudaMultiGraph @classmethod - @networkx_api - def to_undirected_class(cls) -> type[MultiGraph]: + def _to_compat_graph_class(cls) -> type[MultiGraph]: return MultiGraph ########################## @@ -308,7 +363,7 @@ def clear_edges(self) -> None: self.edge_keys = None @networkx_api - def copy(self, as_view: bool = False) -> MultiGraph: + def copy(self, as_view: bool = False) -> CudaMultiGraph: # Does shallow copy in networkx return self._copy(as_view, self.__class__) @@ -391,11 +446,11 @@ def has_edge(self, u: NodeKey, v: NodeKey, key: EdgeKey | None = None) -> bool: return any(edge_keys[i] == key for i in indices.tolist()) @networkx_api - def to_directed(self, as_view: bool = False) -> nxcg.MultiDiGraph: + def to_directed(self, as_view: bool = False) -> nxcg.CudaMultiDiGraph: return self._copy(as_view, self.to_directed_class()) @networkx_api - def to_undirected(self, as_view: bool = False) -> MultiGraph: + def to_undirected(self, as_view: bool = False) -> CudaMultiGraph: # Does deep copy in networkx return self._copy(as_view, self.to_undirected_class()) @@ -403,8 +458,8 @@ def to_undirected(self, as_view: bool = False) -> MultiGraph: # Private methods # ################### - def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): - # DRY warning: see also Graph._copy + def _copy(self, as_view: bool, cls: type[CudaGraph], reverse: bool = False): + # DRY warning: see also CudaGraph._copy src_indices = self.src_indices dst_indices = self.dst_indices edge_indices = self.edge_indices @@ -451,6 +506,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=False, ) if as_view: rv.graph = self.graph @@ -460,7 +516,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): return rv def _sort_edge_indices(self, primary="src"): - # DRY warning: see also Graph._sort_edge_indices + # DRY warning: see also CudaGraph._sort_edge_indices if self.edge_indices is None and self.edge_keys is None: return super()._sort_edge_indices(primary=primary) if primary == "src": diff --git a/python/nx-cugraph/nx_cugraph/convert.py b/python/nx-cugraph/nx_cugraph/convert.py index 56d16d837d7..a872f13ac70 100644 --- a/python/nx-cugraph/nx_cugraph/convert.py +++ b/python/nx-cugraph/nx_cugraph/convert.py @@ -12,6 +12,7 @@ # limitations under the License. from __future__ import annotations +import functools import itertools import operator as op from collections import Counter, defaultdict @@ -23,9 +24,13 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from .utils import index_dtype, networkx_algorithm -from .utils.misc import pairwise +from .utils.misc import _And_NotImplementedError, pairwise + +if _nxver >= (3, 4): + from networkx.utils.backends import _get_cache_key, _get_from_cache, _set_to_cache if TYPE_CHECKING: # pragma: no cover from nx_cugraph.typing import AttrKey, Dtype, EdgeValue, NodeValue, any_ndarray @@ -60,6 +65,27 @@ def _iterate_values(graph, adj, is_dicts, func): return func(it), False +# Consider adding this to `utils` if it is useful elsewhere +def _fallback_decorator(func): + """Catch and convert exceptions to ``NotImplementedError``; use as a decorator. + + ``nx.NetworkXError`` are raised without being converted. This allows + falling back to other backends if, for example, conversion to GPU failed. + """ + + @functools.wraps(func) + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except nx.NetworkXError: + raise + except Exception as exc: + raise _And_NotImplementedError(exc) from exc + + return inner + + +@_fallback_decorator def from_networkx( graph: nx.Graph, edge_attrs: AttrKey | dict[AttrKey, EdgeValue | None] | None = None, @@ -74,7 +100,8 @@ def from_networkx( as_directed: bool = False, name: str | None = None, graph_name: str | None = None, -) -> nxcg.Graph: + use_compat_graph: bool | None = False, +) -> nxcg.Graph | nxcg.CudaGraph: """Convert a networkx graph to nx_cugraph graph; can convert all attributes. Parameters @@ -114,10 +141,16 @@ def from_networkx( The name of the algorithm when dispatched from networkx. graph_name : str, optional The name of the graph argument geing converted when dispatched from networkx. + use_compat_graph : bool or None, default False + Indicate whether to return a graph that is compatible with NetworkX graph. + For example, ``nx_cugraph.Graph`` can be used as a NetworkX graph and can + reside in host (CPU) or device (GPU) memory. The default is False, which + will return e.g. ``nx_cugraph.CudaGraph`` that only resides on device (GPU) + and is not fully compatible as a NetworkX graph. Returns ------- - nx_cugraph.Graph + nx_cugraph.Graph or nx_cugraph.CudaGraph Notes ----- @@ -145,6 +178,41 @@ def from_networkx( graph = G else: raise TypeError(f"Expected networkx.Graph; got {type(graph)}") + elif isinstance(graph, nxcg.Graph): + if ( + use_compat_graph + # Use compat graphs by default + or use_compat_graph is None + and (_nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs) + ): + return graph + if graph._is_on_gpu: + return graph._cudagraph + if not graph._is_on_cpu: + raise RuntimeError( + f"{type(graph).__name__} cannot be converted to the GPU, because it is " + "not on the CPU! This is not supposed to be possible. If you believe " + "you have found a bug, please report a minimum reproducible example to " + "https://github.com/rapidsai/cugraph/issues/new/choose" + ) + if _nxver >= (3, 4): + cache_key = _get_cache_key( + edge_attrs=edge_attrs, + node_attrs=node_attrs, + preserve_edge_attrs=preserve_edge_attrs, + preserve_node_attrs=preserve_node_attrs, + preserve_graph_attrs=preserve_graph_attrs, + ) + cache = getattr(graph, "__networkx_cache__", None) + if cache is not None: + cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + compat_key, rv = _get_from_cache(cache, cache_key) + if rv is not None: + if isinstance(rv, nxcg.Graph): + # This shouldn't happen during normal use, but be extra-careful + rv = rv._cudagraph + if rv is not None: + return rv if preserve_all_attrs: preserve_edge_attrs = True @@ -165,7 +233,12 @@ def from_networkx( else: node_attrs = {node_attrs: None} - if graph.__class__ in {nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph}: + if graph.__class__ in { + nx.Graph, + nx.DiGraph, + nx.MultiGraph, + nx.MultiDiGraph, + } or isinstance(graph, nxcg.Graph): # This is a NetworkX private attribute, but is much faster to use adj = graph._adj else: @@ -455,9 +528,9 @@ def func(it, edge_attr=edge_attr, dtype=dtype): # if vals.ndim > 1: ... if graph.is_multigraph(): if graph.is_directed() or as_directed: - klass = nxcg.MultiDiGraph + klass = nxcg.CudaMultiDiGraph else: - klass = nxcg.MultiGraph + klass = nxcg.CudaMultiGraph rv = klass.from_coo( N, src_indices, @@ -469,12 +542,13 @@ def func(it, edge_attr=edge_attr, dtype=dtype): node_masks, key_to_id=key_to_id, edge_keys=edge_keys, + use_compat_graph=False, ) else: if graph.is_directed() or as_directed: - klass = nxcg.DiGraph + klass = nxcg.CudaDiGraph else: - klass = nxcg.Graph + klass = nxcg.CudaGraph rv = klass.from_coo( N, src_indices, @@ -484,9 +558,22 @@ def func(it, edge_attr=edge_attr, dtype=dtype): node_values, node_masks, key_to_id=key_to_id, + use_compat_graph=False, ) if preserve_graph_attrs: rv.graph.update(graph.graph) # deepcopy? + if _nxver >= (3, 4) and isinstance(graph, nxcg.Graph) and cache is not None: + # Make sure this conversion is added to the cache, and make all of + # our graphs share the same `.graph` attribute for consistency. + rv.graph = graph.graph + _set_to_cache(cache, cache_key, rv) + if ( + use_compat_graph + # Use compat graphs by default + or use_compat_graph is None + and (_nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs) + ): + return rv._to_compat_graph() return rv @@ -535,14 +622,16 @@ def _iter_attr_dicts( return full_dicts -def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: +def to_networkx( + G: nxcg.Graph | nxcg.CudaGraph, *, sort_edges: bool = False +) -> nx.Graph: """Convert a nx_cugraph graph to networkx graph. All edge and node attributes and ``G.graph`` properties are converted. Parameters ---------- - G : nx_cugraph.Graph + G : nx_cugraph.Graph or nx_cugraph.CudaGraph sort_edges : bool, default False Whether to sort the edge data of the input graph by (src, dst) indices before converting. This can be useful to convert to networkx graphs @@ -557,6 +646,9 @@ def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: -------- from_networkx : The opposite; convert networkx graph to nx_cugraph graph """ + if isinstance(G, nxcg.Graph): + # These graphs are already NetworkX graphs :) + return G rv = G.to_networkx_class()() id_to_key = G.id_to_key if sort_edges: @@ -623,13 +715,13 @@ def _to_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.Graph | nxcg.DiGraph: +) -> nxcg.CudaGraph | nxcg.CudaDiGraph: """Ensure that input type is a nx_cugraph graph, and convert if necessary. Directed and undirected graphs are both allowed. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): return G if isinstance(G, nx.Graph): return from_networkx( @@ -644,15 +736,15 @@ def _to_directed_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.DiGraph: - """Ensure that input type is a nx_cugraph DiGraph, and convert if necessary. +) -> nxcg.CudaDiGraph: + """Ensure that input type is a nx_cugraph CudaDiGraph, and convert if necessary. Undirected graphs will be converted to directed. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.DiGraph): + if isinstance(G, nxcg.CudaDiGraph): return G - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): return G.to_directed() if isinstance(G, nx.Graph): return from_networkx( @@ -670,13 +762,13 @@ def _to_undirected_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.Graph: - """Ensure that input type is a nx_cugraph Graph, and convert if necessary. +) -> nxcg.CudaGraph: + """Ensure that input type is a nx_cugraph CudaGraph, and convert if necessary. Only undirected graphs are allowed. Directed graphs will raise ValueError. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): if G.is_directed(): raise ValueError("Only undirected graphs supported; got a directed graph") return G @@ -688,7 +780,7 @@ def _to_undirected_graph( raise TypeError -@networkx_algorithm(version_added="24.08") +@networkx_algorithm(version_added="24.08", fallback=True) def from_dict_of_lists(d, create_using=None): from .generators._utils import _create_using_class diff --git a/python/nx-cugraph/nx_cugraph/convert_matrix.py b/python/nx-cugraph/nx_cugraph/convert_matrix.py index 38139b913cf..54975902861 100644 --- a/python/nx-cugraph/nx_cugraph/convert_matrix.py +++ b/python/nx-cugraph/nx_cugraph/convert_matrix.py @@ -14,6 +14,8 @@ import networkx as nx import numpy as np +from nx_cugraph import _nxver + from .generators._utils import _create_using_class from .utils import _cp_iscopied_asarray, index_dtype, networkx_algorithm @@ -24,7 +26,7 @@ # Value columns with string dtype is not supported -@networkx_algorithm(is_incomplete=True, version_added="23.12") +@networkx_algorithm(is_incomplete=True, version_added="23.12", fallback=True) def from_pandas_edgelist( df, source="source", @@ -138,7 +140,7 @@ def from_pandas_edgelist( and ( # In nx <= 3.3, `edge_key` was ignored if `edge_attr` is None edge_attr is not None - or nx.__version__[:3] > "3.3" + or _nxver > (3, 3) ) ): try: @@ -161,7 +163,7 @@ def from_pandas_edgelist( return G -@networkx_algorithm(version_added="23.12") +@networkx_algorithm(version_added="23.12", fallback=True) def from_scipy_sparse_array( A, parallel_edges=False, create_using=None, edge_attribute="weight" ): diff --git a/python/nx-cugraph/nx_cugraph/generators/_utils.py b/python/nx-cugraph/nx_cugraph/generators/_utils.py index e38ace5b28d..bc9ab84bdad 100644 --- a/python/nx-cugraph/nx_cugraph/generators/_utils.py +++ b/python/nx-cugraph/nx_cugraph/generators/_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -16,6 +16,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype @@ -74,7 +75,7 @@ def _common_small_graph(n, nodes, create_using, *, allow_directed=True): return G -def _create_using_class(create_using, *, default=nxcg.Graph): +def _create_using_class(create_using, *, default=nx.Graph): """Handle ``create_using`` argument and return a Graph type from nx_cugraph.""" inplace = False if create_using is None: @@ -85,16 +86,17 @@ def _create_using_class(create_using, *, default=nxcg.Graph): create_using, "is_multigraph" ): raise TypeError("create_using is not a valid graph type or instance") - elif not isinstance(create_using, nxcg.Graph): + elif not isinstance(create_using, (nxcg.Graph, nxcg.CudaGraph)): raise NotImplementedError( f"create_using with object of type {type(create_using)} is not supported " - "by the cugraph backend; only nx_cugraph.Graph objects are allowed." + "by the cugraph backend; only nx_cugraph.Graph or nx_cugraph.CudaGraph " + "objects are allowed." ) else: inplace = True G = create_using G.clear() - if not isinstance(G, nxcg.Graph): + if not isinstance(G, (nxcg.Graph, nxcg.CudaGraph)): if G.is_multigraph(): if G.is_directed(): graph_class = nxcg.MultiDiGraph @@ -104,10 +106,12 @@ def _create_using_class(create_using, *, default=nxcg.Graph): graph_class = nxcg.DiGraph else: graph_class = nxcg.Graph + if _nxver >= (3, 3) and not nx.config.backends.cugraph.use_compat_graphs: + graph_class = graph_class.to_cudagraph_class() if G.__class__ not in {nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph}: raise NotImplementedError( f"create_using with type {type(G)} is not supported by the cugraph " - "backend; only standard networkx or nx_cugraph Graph objects are " + "backend; only standard networkx or nx_cugraph graph objects are " "allowed (but not customized subclasses derived from them)." ) else: diff --git a/python/nx-cugraph/nx_cugraph/generators/classic.py b/python/nx-cugraph/nx_cugraph/generators/classic.py index a548beea34f..cfcb2a3afec 100644 --- a/python/nx-cugraph/nx_cugraph/generators/classic.py +++ b/python/nx-cugraph/nx_cugraph/generators/classic.py @@ -18,6 +18,7 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import _get_int_dtype, index_dtype, networkx_algorithm from ._utils import ( @@ -102,7 +103,9 @@ def complete_graph(n, create_using=None): @networkx_algorithm(version_added="23.12") def complete_multipartite_graph(*subset_sizes): if not subset_sizes: - return nxcg.Graph() + if _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs: + return nxcg.Graph() + return nxcg.CudaGraph() try: subset_sizes = [_ensure_int(size) for size in subset_sizes] except TypeError: @@ -139,6 +142,8 @@ def complete_multipartite_graph(*subset_sizes): dst_indices, node_values={"subset": subsets_array}, id_to_key=nodes, + use_compat_graph=_nxver < (3, 3) + or nx.config.backends.cugraph.use_compat_graphs, ) diff --git a/python/nx-cugraph/nx_cugraph/generators/community.py b/python/nx-cugraph/nx_cugraph/generators/community.py index 9b0e0848de9..4e5063cc345 100644 --- a/python/nx-cugraph/nx_cugraph/generators/community.py +++ b/python/nx-cugraph/nx_cugraph/generators/community.py @@ -11,8 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import cupy as cp +import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import networkx_algorithm from ._utils import ( @@ -42,4 +44,7 @@ def caveman_graph(l, k): # noqa: E741 dst_cliques.extend(dst_clique + i * k for i in range(1, l)) src_indices = cp.hstack(src_cliques) dst_indices = cp.hstack(dst_cliques) - return nxcg.Graph.from_coo(l * k, src_indices, dst_indices) + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + l * k, src_indices, dst_indices, use_compat_graph=use_compat_graph + ) diff --git a/python/nx-cugraph/nx_cugraph/generators/ego.py b/python/nx-cugraph/nx_cugraph/generators/ego.py index 66c9c8b95ee..9a91fa0b6c3 100644 --- a/python/nx-cugraph/nx_cugraph/generators/ego.py +++ b/python/nx-cugraph/nx_cugraph/generators/ego.py @@ -32,7 +32,10 @@ def ego_graph( ): """Weighted ego_graph with negative cycles is not yet supported. `NotImplementedError` will be raised if there are negative `distance` edge weights.""" # noqa: E501 if isinstance(G, nx.Graph): + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + is_compat_graph = False if n not in G: if distance is None: raise nx.NodeNotFound(f"Source {n} is not in G") @@ -100,7 +103,10 @@ def ego_graph( node_mask &= node_ids != src_index node_ids = node_ids[node_mask] if node_ids.size == G._N: - return G.copy() + rv = G.copy() + if is_compat_graph: + return rv._to_compat_graph() + return rv # TODO: create renumbering helper function(s) node_ids.sort() # TODO: is this ever necessary? Keep for safety node_values = {key: val[node_ids] for key, val in G.node_values.items()} @@ -137,6 +143,7 @@ def ego_graph( "node_values": node_values, "node_masks": node_masks, "key_to_id": key_to_id, + "use_compat_graph": False, } if G.is_multigraph(): if G.edge_keys is not None: @@ -147,6 +154,8 @@ def ego_graph( kwargs["edge_indices"] = G.edge_indices[edge_mask] rv = G.__class__.from_coo(**kwargs) rv.graph.update(G.graph) + if is_compat_graph: + return rv._to_compat_graph() return rv diff --git a/python/nx-cugraph/nx_cugraph/generators/small.py b/python/nx-cugraph/nx_cugraph/generators/small.py index 45487571cda..d0c03cb7dd4 100644 --- a/python/nx-cugraph/nx_cugraph/generators/small.py +++ b/python/nx-cugraph/nx_cugraph/generators/small.py @@ -14,6 +14,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype, networkx_algorithm from ._utils import _IS_NX32_OR_LESS, _create_using_class @@ -449,7 +450,14 @@ def pappus_graph(): index_dtype, ) # fmt: on - return nxcg.Graph.from_coo(18, src_indices, dst_indices, name="Pappus Graph") + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 18, + src_indices, + dst_indices, + name="Pappus Graph", + use_compat_graph=use_compat_graph, + ) @networkx_algorithm(version_added="23.12") diff --git a/python/nx-cugraph/nx_cugraph/generators/social.py b/python/nx-cugraph/nx_cugraph/generators/social.py index 07e82c63fbf..09d405e7561 100644 --- a/python/nx-cugraph/nx_cugraph/generators/social.py +++ b/python/nx-cugraph/nx_cugraph/generators/social.py @@ -11,9 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import cupy as cp +import networkx as nx import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype, networkx_algorithm @@ -77,7 +79,8 @@ def davis_southern_women_graph(): "E13", "E14", ] # fmt: on - return nxcg.Graph.from_coo( + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( 32, src_indices, dst_indices, @@ -85,6 +88,7 @@ def davis_southern_women_graph(): id_to_key=women + events, top=women, bottom=events, + use_compat_graph=use_compat_graph, ) @@ -111,7 +115,14 @@ def florentine_families_graph(): "Salviati", "Strozzi", "Tornabuoni" ] # fmt: on - return nxcg.Graph.from_coo(15, src_indices, dst_indices, id_to_key=nodes) + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 15, + src_indices, + dst_indices, + id_to_key=nodes, + use_compat_graph=use_compat_graph, + ) @networkx_algorithm(version_added="23.12") @@ -165,13 +176,15 @@ def karate_club_graph(): "Officer", "Officer", "Officer", "Officer", "Officer", "Officer", ]) # fmt: on - return nxcg.Graph.from_coo( + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( 34, src_indices, dst_indices, edge_values={"weight": weights}, node_values={"club": clubs}, name="Zachary's Karate Club", + use_compat_graph=use_compat_graph, ) @@ -289,6 +302,12 @@ def les_miserables_graph(): "Zephine", ] # fmt: on - return nxcg.Graph.from_coo( - 77, src_indices, dst_indices, edge_values={"weight": weights}, id_to_key=nodes + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 77, + src_indices, + dst_indices, + edge_values={"weight": weights}, + id_to_key=nodes, + use_compat_graph=use_compat_graph, ) diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index 4007230efa9..1a3d08409a2 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -18,6 +18,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver class BackendInterface: @@ -32,11 +33,19 @@ def convert_from_nx(graph, *args, edge_attrs=None, weight=None, **kwargs): "edge_attrs and weight arguments should not both be given" ) edge_attrs = {weight: 1} - return nxcg.from_networkx(graph, *args, edge_attrs=edge_attrs, **kwargs) + return nxcg.from_networkx( + graph, + *args, + edge_attrs=edge_attrs, + use_compat_graph=_nxver < (3, 3) + or nx.config.backends.cugraph.use_compat_graphs, + **kwargs, + ) @staticmethod def convert_to_nx(obj, *, name: str | None = None): - if isinstance(obj, nxcg.Graph): + if isinstance(obj, nxcg.CudaGraph): + # Observe that this does not try to convert Graph! return nxcg.to_networkx(obj) return obj @@ -62,19 +71,32 @@ def key(testpath): return (testname, frozenset({classname, filename})) return (testname, frozenset({filename})) + use_compat_graph = ( + _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + ) + fallback = use_compat_graph or nx.utils.backends._dispatchable._fallback_to_nx + # Reasons for xfailing + # For nx version <= 3.1 no_weights = "weighted implementation not currently supported" no_multigraph = "multigraphs not currently supported" + # For nx version <= 3.2 + nx_cugraph_in_test_setup = ( + "nx-cugraph Graph is incompatible in test setup in nx versions < 3.3" + ) + # For all versions louvain_different = "Louvain may be different due to RNG" - no_string_dtype = "string edge values not currently supported" sssp_path_different = "sssp may choose a different valid path" + tuple_elements_preferred = "elements are tuples instead of lists" + no_mixed_dtypes_for_nodes = ( + # This one is tricky b/c we don't raise; all dtypes are treated as str + "mixed dtypes (str, int, float) for single node property not supported" + ) + # These shouldn't fail if using Graph or falling back to networkx + no_string_dtype = "string edge values not currently supported" no_object_dtype_for_edges = ( "Edges don't support object dtype (lists, strings, etc.)" ) - tuple_elements_preferred = "elements are tuples instead of lists" - nx_cugraph_in_test_setup = ( - "nx-cugraph Graph is incompatible in test setup in nx versions < 3.3" - ) xfail = { # This is removed while strongly_connected_components() is not @@ -98,38 +120,6 @@ def key(testpath): "test_cycles.py:TestMinimumCycleBasis." "test_gh6787_and_edge_attribute_names" ): sssp_path_different, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr_and_node_attr" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr_subgraph_hash" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:" - "test_isomorphic_edge_attr_and_node_attr_subgraph_hash" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPNoEdgeTypes.test_summary_graph" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPUndirected.test_summary_graph" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPDirected.test_summary_graph" - ): no_object_dtype_for_edges, - key("test_gexf.py:TestGEXF.test_relabel"): no_object_dtype_for_edges, - key( - "test_gml.py:TestGraph.test_parse_gml_cytoscape_bug" - ): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_parse_gml"): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_read_gml"): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_data_types"): no_object_dtype_for_edges, - key( - "test_gml.py:TestPropertyLists.test_reading_graph_with_list_property" - ): no_object_dtype_for_edges, key( "test_relabel.py:" "test_relabel_preserve_node_order_partial_mapping_with_copy_false" @@ -138,48 +128,107 @@ def key(testpath): "test_gml.py:" "TestPropertyLists.test_reading_graph_with_single_element_list_property" ): tuple_elements_preferred, - key( - "test_relabel.py:" - "TestRelabel.test_relabel_multidigraph_inout_merge_nodes" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_inplace" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_merge_inplace" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_inout_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_merge_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_nonnumeric_key" - ): no_string_dtype, - key("test_contraction.py:test_multigraph_path"): no_object_dtype_for_edges, - key( - "test_contraction.py:test_directed_multigraph_path" - ): no_object_dtype_for_edges, - key( - "test_contraction.py:test_multigraph_blockmodel" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPUndirectedMulti.test_summary_graph" - ): no_string_dtype, - key( - "test_summarization.py:TestSNAPDirectedMulti.test_summary_graph" - ): no_string_dtype, } + if not fallback: + xfail.update( + { + key( + "test_graph_hashing.py:test_isomorphic_edge_attr" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:test_isomorphic_edge_attr_and_node_attr" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:test_isomorphic_edge_attr_subgraph_hash" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:" + "test_isomorphic_edge_attr_and_node_attr_subgraph_hash" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPNoEdgeTypes.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPUndirected.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPDirected.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_gexf.py:TestGEXF.test_relabel" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_parse_gml_cytoscape_bug" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_parse_gml" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_read_gml" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_data_types" + ): no_object_dtype_for_edges, + key( + "test_gml.py:" + "TestPropertyLists.test_reading_graph_with_list_property" + ): no_object_dtype_for_edges, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_inout_merge_nodes" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multigraph_merge_inplace" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_merge_inplace" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_inout_copy" + ): no_string_dtype, + key( + "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_copy" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_merge_copy" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multigraph_nonnumeric_key" + ): no_string_dtype, + key( + "test_contraction.py:test_multigraph_path" + ): no_object_dtype_for_edges, + key( + "test_contraction.py:test_directed_multigraph_path" + ): no_object_dtype_for_edges, + key( + "test_contraction.py:test_multigraph_blockmodel" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:" + "TestSNAPUndirectedMulti.test_summary_graph" + ): no_string_dtype, + key( + "test_summarization.py:TestSNAPDirectedMulti.test_summary_graph" + ): no_string_dtype, + } + ) + else: + xfail.update( + { + key( + "test_gml.py:" + "TestPropertyLists.test_reading_graph_with_list_property" + ): no_mixed_dtypes_for_nodes, + } + ) - from packaging.version import parse - - nxver = parse(nx.__version__) - - if nxver.major == 3 and nxver.minor <= 2: + if _nxver <= (3, 2): xfail.update( { # NetworkX versions prior to 3.2.1 have tests written to @@ -216,7 +265,7 @@ def key(testpath): } ) - if nxver.major == 3 and nxver.minor <= 1: + if _nxver <= (3, 1): # MAINT: networkx 3.0, 3.1 # NetworkX 3.2 added the ability to "fallback to nx" if backend algorithms # raise NotImplementedError or `can_run` returns False. The tests below @@ -332,24 +381,25 @@ def key(testpath): xfail[key("test_louvain.py:test_threshold")] = ( "Louvain does not support seed parameter" ) - if nxver.major == 3 and nxver.minor >= 2: - xfail.update( - { - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_multi_attr_incl_target" - ): no_string_dtype, - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_multidigraph_and_edge_attr" - ): no_string_dtype, - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_int_attr_name" - ): no_string_dtype, - } - ) - if nxver.minor == 2: + if _nxver >= (3, 2): + if not fallback: + xfail.update( + { + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_multi_attr_incl_target" + ): no_string_dtype, + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_multidigraph_and_edge_attr" + ): no_string_dtype, + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_int_attr_name" + ): no_string_dtype, + } + ) + if _nxver[1] == 2: different_iteration_order = "Different graph data iteration order" xfail.update( { @@ -366,7 +416,7 @@ def key(testpath): ): different_iteration_order, } ) - elif nxver.minor >= 3: + elif _nxver[1] >= 3: xfail.update( { key("test_louvain.py:test_max_level"): louvain_different, diff --git a/python/nx-cugraph/nx_cugraph/relabel.py b/python/nx-cugraph/nx_cugraph/relabel.py index 20d1337a99c..e38e18c779e 100644 --- a/python/nx-cugraph/nx_cugraph/relabel.py +++ b/python/nx-cugraph/nx_cugraph/relabel.py @@ -29,13 +29,18 @@ @networkx_algorithm(version_added="24.08") def relabel_nodes(G, mapping, copy=True): + G_orig = G if isinstance(G, nx.Graph): - if not copy: + is_compat_graph = isinstance(G, nxcg.Graph) + if not copy and not is_compat_graph: raise RuntimeError( "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.relabel_nodes`" ) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + is_compat_graph = False + it = range(G._N) if G.key_to_id is None else G.id_to_key if callable(mapping): previd_to_key = [mapping(node) for node in it] @@ -225,12 +230,13 @@ def relabel_nodes(G, mapping, copy=True): node_masks=node_masks, id_to_key=newid_to_key, key_to_id=key_to_newid, + use_compat_graph=is_compat_graph, **extra_kwargs, ) rv.graph.update(G.graph) if not copy: - G._become(rv) - return G + G_orig._become(rv) + return G_orig return rv @@ -241,7 +247,10 @@ def convert_node_labels_to_integers( if ordering not in {"default", "sorted", "increasing degree", "decreasing degree"}: raise nx.NetworkXError(f"Unknown node ordering: {ordering}") if isinstance(G, nx.Graph): + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + is_compat_graph = False G = G.copy() if label_attribute is not None: prev_vals = G.id_to_key @@ -279,4 +288,6 @@ def convert_node_labels_to_integers( key_to_id = G.key_to_id G.key_to_id = {i: key_to_id[n] for i, (d, n) in enumerate(pairs, first_label)} G._id_to_key = id_to_key + if is_compat_graph: + return G._to_compat_graph() return G diff --git a/python/nx-cugraph/nx_cugraph/tests/test_bfs.py b/python/nx-cugraph/nx_cugraph/tests/test_bfs.py index c2b22e98949..ad2c62c1fb9 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_bfs.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_bfs.py @@ -12,11 +12,10 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse -nxver = parse(nx.__version__) +from nx_cugraph import _nxver -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test clustering", allow_module_level=True) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_classes.py b/python/nx-cugraph/nx_cugraph/tests/test_classes.py new file mode 100644 index 00000000000..0ac238b3558 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_classes.py @@ -0,0 +1,77 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import nx_cugraph as nxcg + + +def test_class_to_class(): + """Basic sanity checks to ensure metadata relating graph classes are accurate.""" + for prefix in ["", "Cuda"]: + for suffix in ["Graph", "DiGraph", "MultiGraph", "MultiDiGraph"]: + cls_name = f"{prefix}{suffix}" + cls = getattr(nxcg, cls_name) + assert cls.__name__ == cls_name + G = cls() + assert cls is G.__class__ + # cudagraph + val = cls.to_cudagraph_class() + val2 = G.to_cudagraph_class() + assert val is val2 + assert val.__name__ == f"Cuda{suffix}" + assert val.__module__.startswith("nx_cugraph") + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + # networkx + val = cls.to_networkx_class() + val2 = G.to_networkx_class() + assert val is val2 + assert val.__name__ == suffix + assert val.__module__.startswith("networkx") + val = val() + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + # directed + val = cls.to_directed_class() + val2 = G.to_directed_class() + assert val is val2 + assert val.__module__.startswith("nx_cugraph") + assert val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + if "Di" in suffix: + assert val is cls + else: + assert "Di" in val.__name__ + assert prefix in val.__name__ + assert cls.to_undirected_class() is cls + # undirected + val = cls.to_undirected_class() + val2 = G.to_undirected_class() + assert val is val2 + assert val.__module__.startswith("nx_cugraph") + assert not val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + if "Di" not in suffix: + assert val is cls + else: + assert "Di" not in val.__name__ + assert prefix in val.__name__ + assert cls.to_directed_class() is cls + # "zero" + if prefix == "Cuda": + val = cls._to_compat_graph_class() + val2 = G._to_compat_graph_class() + assert val is val2 + assert val.__name__ == suffix + assert val.__module__.startswith("nx_cugraph") + assert val.to_cudagraph_class() is cls + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() diff --git a/python/nx-cugraph/nx_cugraph/tests/test_cluster.py b/python/nx-cugraph/nx_cugraph/tests/test_cluster.py index ad4770f1ab8..fd8e1b3cf13 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_cluster.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_cluster.py @@ -12,11 +12,10 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse -nxver = parse(nx.__version__) +from nx_cugraph import _nxver -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test clustering", allow_module_level=True) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_convert.py b/python/nx-cugraph/nx_cugraph/tests/test_convert.py index 634b28e961c..3d109af8a74 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_convert.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_convert.py @@ -13,13 +13,10 @@ import cupy as cp import networkx as nx import pytest -from packaging.version import parse import nx_cugraph as nxcg from nx_cugraph import interface -nxver = parse(nx.__version__) - @pytest.mark.parametrize( "graph_class", [nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph] diff --git a/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py b/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py index 5474f9d79e3..0697a744e85 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py @@ -12,16 +12,13 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse import nx_cugraph as nxcg +from nx_cugraph import _nxver from .testing_utils import assert_graphs_equal -nxver = parse(nx.__version__) - - -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test ego_graph", allow_module_level=True) @@ -49,7 +46,12 @@ def test_ego_graph_cycle_graph( kwargs = {"radius": radius, "center": center, "undirected": undirected} Hnx = nx.ego_graph(Gnx, n, **kwargs) Hcg = nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") + use_compat_graphs = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + assert_graphs_equal(Hnx, Hcg._cudagraph if use_compat_graphs else Hcg) + Hcg = nx.ego_graph(Gcg, n, **kwargs) assert_graphs_equal(Hnx, Hcg) + Hcg = nx.ego_graph(Gcg._to_compat_graph(), n, **kwargs) + assert_graphs_equal(Hnx, Hcg._cudagraph) with pytest.raises(nx.NodeNotFound, match="not in G"): nx.ego_graph(Gnx, -1, **kwargs) with pytest.raises(nx.NodeNotFound, match="not in G"): @@ -61,20 +63,36 @@ def test_ego_graph_cycle_graph( kwargs["distance"] = "weight" H2nx = nx.ego_graph(Gnx, n, **kwargs) - is_nx32 = nxver.major == 3 and nxver.minor == 2 + is_nx32 = _nxver[:2] == (3, 2) if undirected and Gnx.is_directed() and Gnx.is_multigraph(): if is_nx32: # `should_run` was added in nx 3.3 match = "Weighted ego_graph with undirected=True not implemented" + elif _nxver >= (3, 4): + match = "not implemented by 'cugraph'" else: match = "not implemented by cugraph" - with pytest.raises(RuntimeError, match=match): + with pytest.raises( + RuntimeError if _nxver < (3, 4) else NotImplementedError, match=match + ): nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") with pytest.raises(NotImplementedError, match="ego_graph"): - nx.ego_graph(Gcg, n, **kwargs) + nx.ego_graph(Gcg, n, **kwargs, backend="cugraph") + if _nxver < (3, 4): + with pytest.raises(NotImplementedError, match="ego_graph"): + nx.ego_graph(Gcg, n, **kwargs) + else: + # This is an interesting case. `nxcg.ego_graph` is not implemented for + # these arguments, so it falls back to networkx. Hence, as it is currently + # implemented, the input graph is `nxcg.CudaGraph`, but the output graph + # is `nx.Graph`. Should networkx convert back to "cugraph" backend? + # TODO: make fallback to networkx configurable. + H2cg = nx.ego_graph(Gcg, n, **kwargs) + assert type(H2nx) is type(H2cg) + assert_graphs_equal(H2nx, nxcg.from_networkx(H2cg, preserve_all_attrs=True)) else: H2cg = nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") - assert_graphs_equal(H2nx, H2cg) + assert_graphs_equal(H2nx, H2cg._cudagraph if use_compat_graphs else H2cg) with pytest.raises(nx.NodeNotFound, match="not found in graph"): nx.ego_graph(Gnx, -1, **kwargs) with pytest.raises(nx.NodeNotFound, match="not found in graph"): diff --git a/python/nx-cugraph/nx_cugraph/tests/test_generators.py b/python/nx-cugraph/nx_cugraph/tests/test_generators.py index c751b0fe2b3..5c405f1c93b 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_generators.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_generators.py @@ -13,25 +13,24 @@ import networkx as nx import numpy as np import pytest -from packaging.version import parse import nx_cugraph as nxcg +from nx_cugraph import _nxver from .testing_utils import assert_graphs_equal -nxver = parse(nx.__version__) - - -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test generators", allow_module_level=True) def compare(name, create_using, *args, is_vanilla=False): exc1 = exc2 = None func = getattr(nx, name) - if isinstance(create_using, nxcg.Graph): + if isinstance(create_using, nxcg.CudaGraph): nx_create_using = nxcg.to_networkx(create_using) - elif isinstance(create_using, type) and issubclass(create_using, nxcg.Graph): + elif isinstance(create_using, type) and issubclass( + create_using, (nxcg.Graph, nxcg.CudaGraph) + ): nx_create_using = create_using.to_networkx_class() elif isinstance(create_using, nx.Graph): nx_create_using = create_using.copy() @@ -61,8 +60,27 @@ def compare(name, create_using, *args, is_vanilla=False): exc2 = exc if exc1 is not None or exc2 is not None: assert type(exc1) is type(exc2) + return + if isinstance(Gcg, nxcg.Graph): + # If the graph is empty, it may be on host, otherwise it should be on device + if len(G): + assert Gcg._is_on_gpu + assert not Gcg._is_on_cpu + assert_graphs_equal(G, Gcg._cudagraph) else: assert_graphs_equal(G, Gcg) + # Ensure the output type is correct + if is_vanilla: + if _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs: + assert isinstance(Gcg, nxcg.Graph) + else: + assert isinstance(Gcg, nxcg.CudaGraph) + elif isinstance(create_using, type) and issubclass( + create_using, (nxcg.Graph, nxcg.CudaGraph) + ): + assert type(Gcg) is create_using + elif isinstance(create_using, (nxcg.Graph, nxcg.CudaGraph)): + assert type(Gcg) is type(create_using) N = list(range(-1, 5)) @@ -76,6 +94,10 @@ def compare(name, create_using, *args, is_vanilla=False): nxcg.DiGraph, nxcg.MultiGraph, nxcg.MultiDiGraph, + nxcg.CudaGraph, + nxcg.CudaDiGraph, + nxcg.CudaMultiGraph, + nxcg.CudaMultiDiGraph, # These raise NotImplementedError # nx.Graph(), # nx.DiGraph(), @@ -85,6 +107,10 @@ def compare(name, create_using, *args, is_vanilla=False): nxcg.DiGraph(), nxcg.MultiGraph(), nxcg.MultiDiGraph(), + nxcg.CudaGraph(), + nxcg.CudaDiGraph(), + nxcg.CudaMultiGraph(), + nxcg.CudaMultiDiGraph(), None, object, # Bad input 7, # Bad input @@ -158,7 +184,7 @@ def compare(name, create_using, *args, is_vanilla=False): @pytest.mark.parametrize("create_using", COMPLETE_CREATE_USING) def test_generator_noarg(name, create_using): print(name, create_using, type(create_using)) - if isinstance(create_using, nxcg.Graph) and name in { + if isinstance(create_using, nxcg.CudaGraph) and name in { # fmt: off "bull_graph", "chvatal_graph", "cubical_graph", "diamond_graph", "house_graph", "house_x_graph", "icosahedral_graph", "krackhardt_kite_graph", diff --git a/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py b/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py index 3120995a2b2..40a361b1084 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py @@ -47,7 +47,7 @@ def _create_Gs(): @pytest.mark.parametrize("Gnx", _create_Gs()) @pytest.mark.parametrize("reciprocal", [False, True]) def test_to_undirected_directed(Gnx, reciprocal): - Gcg = nxcg.DiGraph(Gnx) + Gcg = nxcg.CudaDiGraph(Gnx) assert_graphs_equal(Gnx, Gcg) Hnx1 = Gnx.to_undirected(reciprocal=reciprocal) Hcg1 = Gcg.to_undirected(reciprocal=reciprocal) @@ -62,6 +62,6 @@ def test_multidigraph_to_undirected(): Gnx.add_edge(0, 1) Gnx.add_edge(0, 1) Gnx.add_edge(1, 0) - Gcg = nxcg.MultiDiGraph(Gnx) + Gcg = nxcg.CudaMultiDiGraph(Gnx) with pytest.raises(NotImplementedError): Gcg.to_undirected() diff --git a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py index 176b531a6e7..1a61c69b3e7 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py @@ -14,13 +14,10 @@ import inspect import networkx as nx -from packaging.version import parse import nx_cugraph as nxcg from nx_cugraph.utils import networkx_algorithm -nxver = parse(nx.__version__) - def test_match_signature_and_names(): """Simple test to ensure our signatures and basic module layout match networkx.""" diff --git a/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py b/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py index a8f189a4745..9208eea09f2 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -26,7 +26,7 @@ def test_get_edge_data(test_nxcugraph): G.add_edge(0, 3) G.add_edge(0, 3) if test_nxcugraph: - G = nxcg.MultiGraph(G) + G = nxcg.CudaMultiGraph(G) default = object() assert G.get_edge_data(0, 0, default=default) is default assert G.get_edge_data("a", "b", default=default) is default @@ -60,7 +60,7 @@ def test_get_edge_data(test_nxcugraph): G = nx.MultiGraph() G.add_edge(0, 1) if test_nxcugraph: - G = nxcg.MultiGraph(G) + G = nxcg.CudaMultiGraph(G) assert G.get_edge_data(0, 1, default=default) == {0: {}} assert G.get_edge_data(0, 1, 0, default=default) == {} assert G.get_edge_data(0, 1, 1, default=default) is default diff --git a/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py index 0b437df2d2f..252f9e6bbb8 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py @@ -12,19 +12,23 @@ # limitations under the License. import networkx as nx import pandas as pd -from pytest import approx +import pytest def test_pagerank_multigraph(): """ - Ensures correct differences between pagerank results for Graphs - vs. MultiGraphs generated using from_pandas_edgelist() + Ensures correct pagerank for Graphs and MultiGraphs when using from_pandas_edgelist. + + PageRank for MultiGraph should give different result compared to Graph; when using + a Graph, the duplicate edges should be dropped. """ - df = pd.DataFrame({"source": [0, 1, 1, 1, 1, 1, 1, 2], - "target": [1, 2, 2, 2, 2, 2, 2, 3]}) + df = pd.DataFrame( + {"source": [0, 1, 1, 1, 1, 1, 1, 2], "target": [1, 2, 2, 2, 2, 2, 2, 3]} + ) expected_pr_for_G = nx.pagerank(nx.from_pandas_edgelist(df)) expected_pr_for_MultiG = nx.pagerank( - nx.from_pandas_edgelist(df, create_using=nx.MultiGraph)) + nx.from_pandas_edgelist(df, create_using=nx.MultiGraph) + ) G = nx.from_pandas_edgelist(df, backend="cugraph") actual_pr_for_G = nx.pagerank(G, backend="cugraph") @@ -32,5 +36,5 @@ def test_pagerank_multigraph(): MultiG = nx.from_pandas_edgelist(df, create_using=nx.MultiGraph, backend="cugraph") actual_pr_for_MultiG = nx.pagerank(MultiG, backend="cugraph") - assert actual_pr_for_G == approx(expected_pr_for_G) - assert actual_pr_for_MultiG == approx(expected_pr_for_MultiG) + assert actual_pr_for_G == pytest.approx(expected_pr_for_G) + assert actual_pr_for_MultiG == pytest.approx(expected_pr_for_MultiG) diff --git a/python/nx-cugraph/nx_cugraph/tests/testing_utils.py b/python/nx-cugraph/nx_cugraph/tests/testing_utils.py index 529a96efd81..50836acf55f 100644 --- a/python/nx-cugraph/nx_cugraph/tests/testing_utils.py +++ b/python/nx-cugraph/nx_cugraph/tests/testing_utils.py @@ -17,7 +17,7 @@ def assert_graphs_equal(Gnx, Gcg): assert isinstance(Gnx, nx.Graph) - assert isinstance(Gcg, nxcg.Graph) + assert isinstance(Gcg, nxcg.CudaGraph) assert (a := Gnx.number_of_nodes()) == (b := Gcg.number_of_nodes()), (a, b) assert (a := Gnx.number_of_edges()) == (b := Gcg.number_of_edges()), (a, b) assert (a := Gnx.is_directed()) == (b := Gcg.is_directed()), (a, b) diff --git a/python/nx-cugraph/nx_cugraph/utils/decorators.py b/python/nx-cugraph/nx_cugraph/utils/decorators.py index 3c5de4f2936..16486996ba0 100644 --- a/python/nx-cugraph/nx_cugraph/utils/decorators.py +++ b/python/nx-cugraph/nx_cugraph/utils/decorators.py @@ -16,10 +16,14 @@ from textwrap import dedent import networkx as nx +from networkx import NetworkXError from networkx.utils.decorators import nodes_or_number, not_implemented_for +from nx_cugraph import _nxver from nx_cugraph.interface import BackendInterface +from .misc import _And_NotImplementedError + try: from networkx.utils.backends import _registered_algorithms except ModuleNotFoundError: @@ -44,6 +48,7 @@ class networkx_algorithm: version_added: str is_incomplete: bool is_different: bool + _fallback: bool _plc_names: set[str] | None def __new__( @@ -59,6 +64,7 @@ def __new__( version_added: str, # Required is_incomplete: bool = False, # See self.extra_doc for details if True is_different: bool = False, # See self.extra_doc for details if True + fallback: bool = False, # Change non-nx exceptions to NotImplementedError _plc: str | set[str] | None = None, # Hidden from user, may be removed someday ): if func is None: @@ -70,10 +76,11 @@ def __new__( version_added=version_added, is_incomplete=is_incomplete, is_different=is_different, + fallback=fallback, _plc=_plc, ) instance = object.__new__(cls) - if nodes_or_number is not None and nx.__version__[:3] > "3.2": + if nodes_or_number is not None and _nxver > (3, 2): func = nx.utils.decorators.nodes_or_number(nodes_or_number)(func) # update_wrapper sets __wrapped__, which will be used for the signature update_wrapper(instance, func) @@ -100,6 +107,7 @@ def __new__( instance.version_added = version_added instance.is_incomplete = is_incomplete instance.is_different = is_different + instance.fallback = fallback # The docstring on our function is added to the NetworkX docstring. instance.extra_doc = ( dedent(func.__doc__.lstrip("\n").rstrip()) if func.__doc__ else None @@ -113,7 +121,7 @@ def __new__( # Set methods so they are in __dict__ instance._can_run = instance._can_run instance._should_run = instance._should_run - if nodes_or_number is not None and nx.__version__[:3] <= "3.2": + if nodes_or_number is not None and _nxver <= (3, 2): instance = nx.utils.decorators.nodes_or_number(nodes_or_number)(instance) return instance @@ -136,7 +144,14 @@ def _should_run(self, func): self.should_run = func def __call__(self, /, *args, **kwargs): - return self.__wrapped__(*args, **kwargs) + if not self.fallback: + return self.__wrapped__(*args, **kwargs) + try: + return self.__wrapped__(*args, **kwargs) + except NetworkXError: + raise + except Exception as exc: + raise _And_NotImplementedError(exc) from exc def __reduce__(self): return _restore_networkx_dispatched, (self.name,) diff --git a/python/nx-cugraph/nx_cugraph/utils/misc.py b/python/nx-cugraph/nx_cugraph/utils/misc.py index 8526524f1de..01c25dd5983 100644 --- a/python/nx-cugraph/nx_cugraph/utils/misc.py +++ b/python/nx-cugraph/nx_cugraph/utils/misc.py @@ -194,7 +194,7 @@ def _get_int_dtype( def _get_float_dtype( - dtype: Dtype, *, graph: nxcg.Graph | None = None, weight: EdgeKey | None = None + dtype: Dtype, *, graph: nxcg.CudaGraph | None = None, weight: EdgeKey | None = None ): """Promote dtype to float32 or float64 as appropriate.""" if dtype is None: @@ -238,3 +238,37 @@ def _cp_iscopied_asarray(a, *args, orig_object=None, **kwargs): ): return False, arr return True, arr + + +class _And_NotImplementedError(NotImplementedError): + """Additionally make an exception a ``NotImplementedError``. + + For example: + + >>> try: + ... raise _And_NotImplementedError(KeyError("missing")) + ... except KeyError: + ... pass + + or + + >>> try: + ... raise _And_NotImplementedError(KeyError("missing")) + ... except NotImplementedError: + ... pass + + """ + + def __new__(cls, exc): + exc_type = type(exc) + if issubclass(exc_type, NotImplementedError): + new_type = exc_type + else: + new_type = type( + f"{exc_type.__name__}{cls.__name__}", + (exc_type, NotImplementedError), + {}, + ) + instance = NotImplementedError.__new__(new_type) + instance.__init__(*exc.args) + return instance diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index 3d4aa25ab95..cf4b98afe77 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ [project.optional-dependencies] test = [ - "packaging>=21", "pandas", "pytest", "pytest-benchmark", @@ -170,6 +169,7 @@ external = [ ] ignore = [ # Would be nice to fix these + "B905", # `zip()` without an explicit `strict=` parameter (Note: possible since py39 was dropped; we should do this!) "D100", # Missing docstring in public module "D101", # Missing docstring in public class "D102", # Missing docstring in public method @@ -215,6 +215,7 @@ ignore = [ "SIM105", # Use contextlib.suppress(...) instead of try-except-pass (Note: try-except-pass is much faster) "SIM108", # Use ternary operator ... instead of if-else-block (Note: if-else better for coverage and sometimes clearer) "TRY003", # Avoid specifying long messages outside the exception class (Note: why?) + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` (Note: tuple is faster for now) # Ignored categories "C90", # mccabe (Too strict, but maybe we should make things less complex) @@ -241,6 +242,7 @@ ignore = [ # Allow assert, print, RNG, and no docstring "nx_cugraph/**/tests/*py" = ["S101", "S311", "T201", "D103", "D100"] "_nx_cugraph/__init__.py" = ["E501"] +"nx_cugraph/__init__.py" = ["E402"] # Allow module level import not at top of file "nx_cugraph/algorithms/**/*py" = ["D205", "D401"] # Allow flexible docstrings for algorithms "nx_cugraph/generators/**/*py" = ["D205", "D401"] # Allow flexible docstrings for generators "nx_cugraph/interface.py" = ["D401"] # Flexible docstrings diff --git a/python/nx-cugraph/run_nx_tests.sh b/python/nx-cugraph/run_nx_tests.sh index bceec53b7d5..5fb173cf939 100755 --- a/python/nx-cugraph/run_nx_tests.sh +++ b/python/nx-cugraph/run_nx_tests.sh @@ -18,6 +18,10 @@ # testing takes longer. Without it, tests will xfail when encountering a # function that we don't implement. # +# NX_CUGRAPH_USE_COMPAT_GRAPHS, {"True", "False"}, default is "True" +# Whether to use `nxcg.Graph` as the nx_cugraph backend graph. +# A Graph should be a compatible NetworkX graph, so fewer tests should fail. +# # Coverage of `nx_cugraph.algorithms` is reported and is a good sanity check # that algorithms run.