diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index f3892fbd3c4..adf3273e311 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -62,6 +62,7 @@ sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugr sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/server/cugraph_service_server/__init__.py sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/pylibcugraph/pylibcugraph/__init__.py sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/nx_cugraph/__init__.py +sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/_nx_cugraph/__init__.py # Python pyproject.toml updates sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph/pyproject.toml diff --git a/python/nx-cugraph/.flake8 b/python/nx-cugraph/.flake8 index 3a2e3fb8617..c5874e54f7e 100644 --- a/python/nx-cugraph/.flake8 +++ b/python/nx-cugraph/.flake8 @@ -11,3 +11,4 @@ extend-ignore = per-file-ignores = nx_cugraph/tests/*.py:T201, __init__.py:F401,F403, + _nx_cugraph/__init__.py:E501, diff --git a/python/nx-cugraph/Makefile b/python/nx-cugraph/Makefile index c9caf147d53..6e1b98ee6e9 100644 --- a/python/nx-cugraph/Makefile +++ b/python/nx-cugraph/Makefile @@ -1,7 +1,17 @@ # Copyright (c) 2023, NVIDIA CORPORATION. SHELL= /bin/bash +.PHONY: all +all: plugin-info lint + +.PHONY: lint lint: git ls-files | xargs pre-commit run --config lint.yaml --files + +.PHONY: lint-update lint-update: pre-commit autoupdate --config lint.yaml + +.PHONY: plugin-info +plugin-info: + python _nx_cugraph/__init__.py diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py new file mode 100644 index 00000000000..9b3332106ec --- /dev/null +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -0,0 +1,88 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tell NetworkX about the cugraph backend. This file can update itself: + +$ make plugin-info # Recommended method for development + +or + +$ python _nx_cugraph/__init__.py +""" + +# Entries between BEGIN and END are automatically generated +_info = { + "backend_name": "cugraph", + "project": "nx-cugraph", + "package": "nx_cugraph", + "url": "https://github.com/rapidsai/cugraph/tree/branch-23.10/python/nx-cugraph", + "short_summary": "GPU-accelerated backend.", + # "description": "TODO", + "functions": { + # BEGIN: functions + "betweenness_centrality", + "edge_betweenness_centrality", + "louvain_communities", + # END: functions + }, + "extra_docstrings": { + # BEGIN: extra_docstrings + "betweenness_centrality": "`weight` parameter is not yet supported.", + "edge_betweenness_centrality": "`weight` parameter is not yet supported.", + "louvain_communities": "`threshold` and `seed` parameters are currently ignored.", + # END: extra_docstrings + }, + "extra_parameters": { + # BEGIN: extra_parameters + "louvain_communities": { + "max_level : int, optional": "Upper limit of the number of macro-iterations.", + }, + # END: extra_parameters + }, +} + + +def get_info(): + """Target of ``networkx.plugin_info`` entry point. + + This tells NetworkX about the cugraph backend without importing nx_cugraph. + """ + # Convert to e.g. `{"functions": {"myfunc": {"extra_docstring": ...}}}` + d = _info.copy() + info_keys = { + "extra_docstrings": "extra_docstring", + "extra_parameters": "extra_parameters", + } + d["functions"] = { + func: { + new_key: vals[func] + for old_key, new_key in info_keys.items() + if func in (vals := d[old_key]) + } + for func in d["functions"] + } + for key in info_keys: + del d[key] + return d + + +__version__ = "23.10.00" + +if __name__ == "__main__": + from pathlib import Path + + from _nx_cugraph.core import main + + filepath = Path(__file__) + text = main(filepath) + with filepath.open("w") as f: + f.write(text) diff --git a/python/nx-cugraph/_nx_cugraph/core.py b/python/nx-cugraph/_nx_cugraph/core.py new file mode 100644 index 00000000000..72f9203897e --- /dev/null +++ b/python/nx-cugraph/_nx_cugraph/core.py @@ -0,0 +1,90 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities to help keep _nx_cugraph up to date.""" + + +def get_functions(): + from nx_cugraph.interface import BackendInterface + from nx_cugraph.utils import networkx_algorithm + + return { + key: val + for key, val in vars(BackendInterface).items() + if isinstance(val, networkx_algorithm) + } + + +def get_extra_docstrings(functions=None): + if functions is None: + functions = get_functions() + return {key: val.extra_doc for key, val in functions.items() if val.extra_doc} + + +def get_extra_parameters(functions=None): + if functions is None: + functions = get_functions() + return {key: val.extra_params for key, val in functions.items() if val.extra_params} + + +def update_text(text, lines_to_add, target, indent=" " * 8): + begin = f"# BEGIN: {target}\n" + end = f"# END: {target}\n" + start = text.index(begin) + stop = text.index(end) + to_add = "\n".join([f"{indent}{line}" for line in lines_to_add]) + return f"{text[:start]}{begin}{to_add}\n{indent}{text[stop:]}" + + +def dict_to_lines(d, *, indent=""): + for key in sorted(d): + val = d[key] + if "\n" not in val: + yield f"{indent}{key!r}: {val!r}," + else: + yield f"{indent}{key!r}: (" + *lines, last_line = val.split("\n") + for line in lines: + line += "\n" + yield f" {indent}{line!r}" + yield f" {indent}{last_line!r}" + yield f"{indent})," + + +def main(filepath): + from pathlib import Path + + filepath = Path(filepath) + with filepath.open() as f: + orig_text = f.read() + text = orig_text + + # Update functions + functions = get_functions() + to_add = [f'"{name}",' for name in sorted(functions)] + text = update_text(text, to_add, "functions") + + # Update extra_docstrings + extra_docstrings = get_extra_docstrings(functions) + to_add = list(dict_to_lines(extra_docstrings)) + text = update_text(text, to_add, "extra_docstrings") + + # Update extra_parameters + extra_parameters = get_extra_parameters(functions) + to_add = [] + for name in sorted(extra_parameters): + params = extra_parameters[name] + to_add.append(f"{name!r}: {{") + to_add.extend(dict_to_lines(params, indent=" " * 4)) + to_add.append("},") + text = update_text(text, to_add, "extra_parameters") + return text diff --git a/python/nx-cugraph/lint.yaml b/python/nx-cugraph/lint.yaml index dba061bd6b5..6a462a6af79 100644 --- a/python/nx-cugraph/lint.yaml +++ b/python/nx-cugraph/lint.yaml @@ -31,7 +31,7 @@ repos: - id: validate-pyproject name: Validate pyproject.toml - repo: https://github.com/PyCQA/autoflake - rev: v2.2.0 + rev: v2.2.1 hooks: - id: autoflake args: [--in-place] @@ -40,17 +40,17 @@ repos: hooks: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.13.0 hooks: - id: pyupgrade args: [--py39-plus] - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.9.1 hooks: - id: black # - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.286 + rev: v0.0.291 hooks: - id: ruff args: [--fix-only, --show-fixes] @@ -58,11 +58,12 @@ repos: rev: 6.1.0 hooks: - id: flake8 + args: ['--per-file-ignores=_nx_cugraph/__init__.py:E501'] # Why is this necessary? additional_dependencies: &flake8_dependencies - # These versions need updated manually - - flake8==6.1.0 - - flake8-bugbear==23.7.10 - - flake8-simplify==0.20.0 + # These versions need updated manually + - flake8==6.1.0 + - flake8-bugbear==23.9.16 + - flake8-simplify==0.20.0 - repo: https://github.com/asottile/yesqa rev: v1.5.0 hooks: @@ -76,7 +77,7 @@ repos: additional_dependencies: [tomli] files: ^(nx_cugraph|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.286 + rev: v0.0.291 hooks: - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/python/nx-cugraph/nx_cugraph/__init__.py b/python/nx-cugraph/nx_cugraph/__init__.py index 28066fe2b02..4a0e95a109f 100644 --- a/python/nx-cugraph/nx_cugraph/__init__.py +++ b/python/nx-cugraph/nx_cugraph/__init__.py @@ -12,9 +12,21 @@ # limitations under the License. from networkx.exception import * -from . import algorithms, classes, convert, utils -from .algorithms import * +from . import utils + +from . import classes from .classes import * + +from . import convert from .convert import * +# from . import convert_matrix +# from .convert_matrix import * + +# from . import generators +# from .generators import * + +from . import algorithms +from .algorithms import * + __version__ = "23.10.00" diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/betweenness.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/betweenness.py index b777919f86f..104ac87414c 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/centrality/betweenness.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/betweenness.py @@ -13,7 +13,7 @@ import pylibcugraph as plc from nx_cugraph.convert import _to_graph -from nx_cugraph.utils import _handle_seed, networkx_algorithm +from nx_cugraph.utils import _seed_to_int, networkx_algorithm __all__ = ["betweenness_centrality", "edge_betweenness_centrality"] @@ -22,11 +22,12 @@ def betweenness_centrality( G, k=None, normalized=True, weight=None, endpoints=False, seed=None ): + """`weight` parameter is not yet supported.""" if weight is not None: raise NotImplementedError( "Weighted implementation of betweenness centrality not currently supported" ) - seed = _handle_seed(seed) + seed = _seed_to_int(seed) G = _to_graph(G, weight) node_ids, values = plc.betweenness_centrality( resource_handle=plc.ResourceHandle(), @@ -47,6 +48,7 @@ def _(G, k=None, normalized=True, weight=None, endpoints=False, seed=None): @networkx_algorithm def edge_betweenness_centrality(G, k=None, normalized=True, weight=None, seed=None): + """`weight` parameter is not yet supported.""" if weight is not None: raise NotImplementedError( "Weighted implementation of betweenness centrality not currently supported" diff --git a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py index ca5f05c2014..a183b59fe1d 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py @@ -17,7 +17,7 @@ from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( _groupby, - _handle_seed, + _seed_to_int, networkx_algorithm, not_implemented_for, ) @@ -26,16 +26,17 @@ @not_implemented_for("directed") -@networkx_algorithm(extra_params="max_level") +@networkx_algorithm( + extra_params={ + "max_level : int, optional": "Upper limit of the number of macro-iterations." + } +) def louvain_communities( G, weight="weight", resolution=1, threshold=0.0000001, seed=None, *, max_level=None ): - """`threshold` and `seed` parameters are currently ignored. - - Extra parameter: `max_level` controls the maximum number of levels of the algorithm. - """ + """`threshold` and `seed` parameters are currently ignored.""" # NetworkX allows both directed and undirected, but cugraph only allows undirected. - seed = _handle_seed(seed) # Unused, but ensure it's valid for future compatibility + seed = _seed_to_int(seed) # Unused, but ensure it's valid for future compatibility G = _to_undirected_graph(G, weight) if G.row_indices.size == 0: # TODO: PLC doesn't handle empty graphs gracefully! @@ -46,8 +47,8 @@ def louvain_communities( resource_handle=plc.ResourceHandle(), graph=G._get_plc_graph(), max_level=max_level, # TODO: add this parameter to NetworkX + threshold=threshold, resolution=resolution, - # threshold=threshold, # TODO: add this parameter to PLC do_expensive_check=False, ) groups = _groupby(clusters, vertices) diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index cc750cd2d5b..2ad23acd940 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -62,9 +62,7 @@ def key(testpath): # Reasons for xfailing no_weights = "weighted implementation not currently supported" no_multigraph = "multigraphs not currently supported" - louvain_different = ( - "Louvain may be different due to RNG or unsupported threshold parameter" - ) + louvain_different = "Louvain may be different due to RNG" xfail = {} @@ -176,7 +174,6 @@ def key(testpath): ): louvain_different, key("test_louvain.py:test_none_weight_param"): louvain_different, key("test_louvain.py:test_multigraph"): louvain_different, - key("test_louvain.py:test_threshold"): louvain_different, } ) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py index 64d3704dd65..ecfda1397db 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py @@ -45,11 +45,14 @@ def test_match_signature_and_names(): assert orig_sig == func_sig else: # Ignore extra parameters added to nx-cugraph algorithm + # The key of func.extra_params may be like "max_level : int, optional", + # but we only want "max_level" here. + extra_params = {name.split(" ")[0] for name in func.extra_params} assert orig_sig == func_sig.replace( parameters=[ p for name, p in func_sig.parameters.items() - if name not in func.extra_params + if name not in extra_params ] ) if func.can_run is not nxcg.utils.decorators._default_can_run: diff --git a/python/nx-cugraph/nx_cugraph/utils/decorators.py b/python/nx-cugraph/nx_cugraph/utils/decorators.py index 3dbdb07e87f..0f15d236ecd 100644 --- a/python/nx-cugraph/nx_cugraph/utils/decorators.py +++ b/python/nx-cugraph/nx_cugraph/utils/decorators.py @@ -10,13 +10,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from functools import partial, update_wrapper -from networkx.utils.decorators import not_implemented_for +from networkx.utils.decorators import nodes_or_number, not_implemented_for from nx_cugraph.interface import BackendInterface -__all__ = ["not_implemented_for", "networkx_algorithm"] +try: + from networkx.utils.backends import _registered_algorithms +except ModuleNotFoundError: + from networkx.classes.backends import _registered_algorithms + + +__all__ = ["not_implemented_for", "nodes_or_number", "networkx_algorithm"] def networkx_class(api): @@ -28,7 +36,17 @@ def inner(func): class networkx_algorithm: - def __new__(cls, func=None, *, name=None, extra_params=None): + name: str + extra_doc: str | None + extra_params: dict[str, str] | None + + def __new__( + cls, + func=None, + *, + name: str | None = None, + extra_params: dict[str, str] | str | None = None, + ): if func is None: return partial(networkx_algorithm, name=name, extra_params=extra_params) instance = object.__new__(cls) @@ -37,13 +55,20 @@ def __new__(cls, func=None, *, name=None, extra_params=None): instance.__defaults__ = func.__defaults__ instance.__kwdefaults__ = func.__kwdefaults__ instance.name = func.__name__ if name is None else name - # TODO: should extra_params be a dict[str, str] that describes the parameters? if extra_params is None: - instance.extra_params = None + pass elif isinstance(extra_params, str): - instance.extra_params = {extra_params} - else: - instance.extra_params = set(extra_params) + extra_params = {extra_params: ""} + elif not isinstance(extra_params, dict): + raise TypeError( + f"extra_params must be dict, str, or None; got {type(extra_params)}" + ) + instance.extra_params = extra_params + # The docstring on our function is added to the NetworkX docstring. + instance.extra_doc = func.__doc__ + # Copy __doc__ from NetworkX + if instance.name in _registered_algorithms: + instance.__doc__ = _registered_algorithms[instance.name].__doc__ instance.can_run = _default_can_run setattr(BackendInterface, instance.name, instance) # Set methods so they are in __dict__ diff --git a/python/nx-cugraph/nx_cugraph/utils/misc.py b/python/nx-cugraph/nx_cugraph/utils/misc.py index 64c0be066f2..72e4094b8b7 100644 --- a/python/nx-cugraph/nx_cugraph/utils/misc.py +++ b/python/nx-cugraph/nx_cugraph/utils/misc.py @@ -18,7 +18,7 @@ import cupy as cp -__all__ = ["_groupby", "_handle_seed"] +__all__ = ["_groupby", "_seed_to_int"] def _groupby(groups: cp.ndarray, values: cp.ndarray) -> dict[int, cp.ndarray]: @@ -51,8 +51,8 @@ def _groupby(groups: cp.ndarray, values: cp.ndarray) -> dict[int, cp.ndarray]: return rv -def _handle_seed(seed: int | Random | None) -> int: - """Handle seed argument and ensure it is what pylibcugraph needs: an int.""" +def _seed_to_int(seed: int | Random | None) -> int: + """Handle any valid seed argument and convert it to an int if necessary.""" if seed is None: return if isinstance(seed, Random): diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index 95e9c256e5d..db3b3a22545 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -54,6 +54,9 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [project.entry-points."networkx.plugins"] cugraph = "nx_cugraph.interface:BackendInterface" +[project.entry-points."networkx.plugin_info"] +cugraph = "_nx_cugraph:get_info" + [tool.setuptools] license-files = ["LICENSE"] @@ -61,6 +64,8 @@ license-files = ["LICENSE"] include = [ "nx_cugraph*", "nx_cugraph.*", + "_nx_cugraph*", + "_nx_cugraph.*", ] [tool.black] @@ -75,6 +80,7 @@ float_to_top = true default_section = "THIRDPARTY" known_first_party = "nx_cugraph" line_length = 88 +extend_skip_glob = ["nx_cugraph/__init__.py"] [tool.pytest.ini_options] minversion = "6.0" @@ -128,6 +134,9 @@ exclude_lines = [ # https://github.com/charliermarsh/ruff/ line-length = 88 target-version = "py39" +unfixable = [ + "F841", # unused-variable (Note: can leave useless expression) +] select = [ "ALL", ] @@ -203,6 +212,7 @@ ignore = [ "__init__.py" = ["F401"] # Allow unused imports (w/o defining `__all__`) # Allow assert, print, RNG, and no docstring "nx_cugraph/**/tests/*py" = ["S101", "S311", "T201", "D103", "D100"] +"_nx_cugraph/__init__.py" = ["E501"] [tool.ruff.flake8-annotations] mypy-init-return = true