From 8b1a437275b2a0c4477afec4cc41eb3281b12517 Mon Sep 17 00:00:00 2001 From: Mike McCarty Date: Thu, 19 Sep 2024 01:43:32 -0400 Subject: [PATCH 1/9] Recommending `miniforge` for conda install (#4650) Recommending `miniforge` for conda install in installation docs. Authors: - Mike McCarty (https://github.com/mmccarty) - Bradley Dice (https://github.com/bdice) - Alex Barghi (https://github.com/alexbarghi-nv) Approvers: - Rick Ratzel (https://github.com/rlratzel) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4650 --- .../notebooks/get_node_storage.ipynb | 2 +- .../heterogeneous_dataloader_benchmark.ipynb | 2 +- .../homogenous_dataloader_benchmark.ipynb | 6 +- .../cugraph/notebooks/feature_storage.ipynb | 2 +- .../source/installation/getting_cugraph.md | 2 +- .../installation/getting_wholegraph.md | 2 +- notebooks/demo/mg_pagerank.ipynb | 280 +++++++++--------- 7 files changed, 148 insertions(+), 148 deletions(-) diff --git a/benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb b/benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb index 95b456c7812..4681c8ec825 100644 --- a/benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb +++ b/benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb @@ -18,7 +18,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } diff --git a/benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb b/benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb index d3b054bb0ee..2c4a934827a 100644 --- a/benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb +++ b/benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb @@ -176,7 +176,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/dgl/dataloading/dataloader.py:859: DGLWarning: Dataloader CPU affinity opt is not enabled, consider switching it on (see enable_cpu_affinity() or CPU best practices for DGL [https://docs.dgl.ai/tutorials/cpu/cpu_best_practises.html])\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/dgl/dataloading/dataloader.py:859: DGLWarning: Dataloader CPU affinity opt is not enabled, consider switching it on (see enable_cpu_affinity() or CPU best practices for DGL [https://docs.dgl.ai/tutorials/cpu/cpu_best_practises.html])\n", " dgl_warning(f'Dataloader CPU affinity opt is not enabled, consider switching it on '\n" ] }, diff --git a/benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb b/benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb index ea1e9b34965..ecd111dabdf 100644 --- a/benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb +++ b/benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb @@ -26,7 +26,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } @@ -190,7 +190,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/dgl/dataloading/dataloader.py:859: DGLWarning: Dataloader CPU affinity opt is not enabled, consider switching it on (see enable_cpu_affinity() or CPU best practices for DGL [https://docs.dgl.ai/tutorials/cpu/cpu_best_practises.html])\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/dgl/dataloading/dataloader.py:859: DGLWarning: Dataloader CPU affinity opt is not enabled, consider switching it on (see enable_cpu_affinity() or CPU best practices for DGL [https://docs.dgl.ai/tutorials/cpu/cpu_best_practises.html])\n", " dgl_warning(f'Dataloader CPU affinity opt is not enabled, consider switching it on '\n" ] }, @@ -278,7 +278,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/distributed/worker.py:2988: UserWarning: Large object of size 1.42 MiB detected in task graph: \n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/distributed/worker.py:2988: UserWarning: Large object of size 1.42 MiB detected in task graph: \n", " [b'\\xad\\xd1\\xe3\\x9c\\x96\\x83O\\xb3\\xba1\\x86\\x94\\xb6\\ ... =int32), False]\n", "Consider scattering large objects ahead of time\n", "with client.scatter to reduce scheduler burden and \n", diff --git a/benchmarks/cugraph/notebooks/feature_storage.ipynb b/benchmarks/cugraph/notebooks/feature_storage.ipynb index 7413ac00cde..440d76fbdb4 100644 --- a/benchmarks/cugraph/notebooks/feature_storage.ipynb +++ b/benchmarks/cugraph/notebooks/feature_storage.ipynb @@ -18,7 +18,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } diff --git a/docs/cugraph/source/installation/getting_cugraph.md b/docs/cugraph/source/installation/getting_cugraph.md index 41ec9a67e1f..01bc9e379c9 100644 --- a/docs/cugraph/source/installation/getting_cugraph.md +++ b/docs/cugraph/source/installation/getting_cugraph.md @@ -21,7 +21,7 @@ The RAPIDS Docker containers contain all RAPIDS packages, including all from cuG ## Conda -It is easy to install cuGraph using conda. You can get a minimal conda installation with [Miniconda](https://conda.io/miniconda.html) or get the full installation with [Anaconda](https://www.anaconda.com/download). +It is easy to install cuGraph using conda. You can get a minimal conda installation with [miniforge](https://github.com/conda-forge/miniforge). cuGraph Conda packages * cugraph - this will also import: diff --git a/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md b/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md index 57314dcd426..80c666d6593 100644 --- a/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md +++ b/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md @@ -21,7 +21,7 @@ The RAPIDS Docker containers (as of Release 23.10) contain all RAPIDS packages, ## Conda -It is easy to install WholeGraph using conda. You can get a minimal conda installation with [Miniconda](https://conda.io/miniconda.html) or get the full installation with [Anaconda](https://www.anaconda.com/download). +It is easy to install WholeGraph using conda. You can get a minimal conda installation with [miniforge](https://github.com/conda-forge/miniforge). WholeGraph conda packages * libwholegraph diff --git a/notebooks/demo/mg_pagerank.ipynb b/notebooks/demo/mg_pagerank.ipynb index bb333048450..e3314f80b3c 100644 --- a/notebooks/demo/mg_pagerank.ipynb +++ b/notebooks/demo/mg_pagerank.ipynb @@ -219,250 +219,250 @@ "text": [ "2023-05-12 09:25:01,974 - distributed.sizeof - WARNING - Sizeof calculation failed. Defaulting to 0.95 MiB\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", " return sizeof(obj)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", " return meth(arg, *args, **kwargs)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", " + df._index.memory_usage()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", " if self.levels:\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", " self._compute_levels_and_codes()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", " code, cats = cudf.Series._from_data({None: col}).factorize()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", " return cudf.core.algorithms.factorize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", " labels = values._column._label_encoding(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", " order = order.take(left_gather_map, check_bounds=False).argsort()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", " return self.as_frame()._get_sorted_inds(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", " return libcudf.sort.order_by(to_sort, ascending, na_position)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", " File \"sort.pyx\", line 141, in cudf._lib.sort.order_by\n", - "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniconda3/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", + "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniforge/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", "2023-05-12 09:25:01,976 - distributed.sizeof - WARNING - Sizeof calculation failed. Defaulting to 0.95 MiB\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", " return sizeof(obj)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", " return meth(arg, *args, **kwargs)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", " + df._index.memory_usage()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", " if self.levels:\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", " self._compute_levels_and_codes()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", " code, cats = cudf.Series._from_data({None: col}).factorize()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", " return cudf.core.algorithms.factorize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", " labels = values._column._label_encoding(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", " order = order.take(left_gather_map, check_bounds=False).argsort()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", " return self.as_frame()._get_sorted_inds(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", " return libcudf.sort.order_by(to_sort, ascending, na_position)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", " File \"sort.pyx\", line 141, in cudf._lib.sort.order_by\n", - "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniconda3/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", + "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniforge/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", "2023-05-12 09:25:03,767 - distributed.sizeof - WARNING - Sizeof calculation failed. Defaulting to 0.95 MiB\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", " return sizeof(obj)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", " return meth(arg, *args, **kwargs)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", " + df._index.memory_usage()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", " if self.levels:\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", " self._compute_levels_and_codes()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", " code, cats = cudf.Series._from_data({None: col}).factorize()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", " return cudf.core.algorithms.factorize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", " labels = values._column._label_encoding(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", " order = order.take(left_gather_map, check_bounds=False).argsort()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", " return self.as_frame()._get_sorted_inds(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", " return libcudf.sort.order_by(to_sort, ascending, na_position)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", " File \"sort.pyx\", line 141, in cudf._lib.sort.order_by\n", - "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniconda3/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", + "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniforge/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", "2023-05-12 09:25:03,768 - distributed.sizeof - WARNING - Sizeof calculation failed. Defaulting to 0.95 MiB\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", " return sizeof(obj)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", " return meth(arg, *args, **kwargs)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", " + df._index.memory_usage()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", " if self.levels:\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", " self._compute_levels_and_codes()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", " code, cats = cudf.Series._from_data({None: col}).factorize()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", " return cudf.core.algorithms.factorize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", " labels = values._column._label_encoding(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", " order = order.take(left_gather_map, check_bounds=False).argsort()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", " return self.as_frame()._get_sorted_inds(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", " return libcudf.sort.order_by(to_sort, ascending, na_position)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", " File \"sort.pyx\", line 141, in cudf._lib.sort.order_by\n", - "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniconda3/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", + "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniforge/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", "2023-05-12 09:25:03,820 - distributed.worker - ERROR - Could not deserialize task ('len-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d', 1)\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2923, in loads_function\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2923, in loads_function\n", " result = cache_loads[bytes_object]\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/collections.py\", line 24, in __getitem__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/collections.py\", line 24, in __getitem__\n", " value = super().__getitem__(key)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/collections/__init__.py\", line 1106, in __getitem__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/collections/__init__.py\", line 1106, in __getitem__\n", " raise KeyError(key)\n", "KeyError: b'\\x80\\x05\\x95>\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x11dask.optimization\\x94\\x8c\\x10SubgraphCallable\\x94\\x93\\x94(}\\x94(\\x8cKlen-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d\\x94\\x8cZassign-getitem-len-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d\\x94\\x8c*rename-01db283bd79fee66f232920c8dc6b55e_.0\\x94\\x8c;getitem-to_frame-rename-01db283bd79fee66f232920c8dc6b55e_.0\\x94\\x8c+getitem-3499fd71ac25ebbc1a06991edea6067c_.0\\x94\\x8c\\t_operator\\x94\\x8c\\x07getitem\\x94\\x93\\x94\\x8c/reset_index-f4c18304ca92859ccd09f44cf89b4b43_.0\\x94\\x8c\\x13__dask_blockwise__1\\x94\\x87\\x94h\\x0c(\\x8c\\ndask.utils\\x94\\x8c\\x05apply\\x94\\x93\\x94h\\x0f\\x8c\\x0cmethodcaller\\x94\\x93\\x94\\x8c\\x0breset_index\\x94\\x85\\x94R\\x94]\\x94\\x8c\\x13__dask_blockwise__5\\x94a\\x8c\\x08builtins\\x94\\x8c\\x04dict\\x94\\x93\\x94]\\x94]\\x94(\\x8c\\x04drop\\x94\\x89ea\\x86\\x94t\\x94h\\x07(h\\x11\\x8c\\x13dask.dataframe.core\\x94\\x8c\\x11apply_and_enforce\\x94\\x93\\x94]\\x94((h\\x11h#]\\x94h\\x0bh\\x0c\\x8c\\x13__dask_blockwise__0\\x94\\x87\\x94ah\\x1b]\\x94(]\\x94(\\x8c\\x05_func\\x94h\\x13\\x8c\\x08to_frame\\x94\\x85\\x94R\\x94e]\\x94(\\x8c\\x05_meta\\x94\\x8c\\x08builtins\\x94\\x8c\\x07getattr\\x94\\x93\\x94\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94\\x8c\\x10host_deserialize\\x94\\x86\\x94R\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94C0\\x80\\x04\\x95%\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94.\\x94\\x8c\\x0ccolumn_names\\x94C\\x14\\x80\\x04\\x95\\t\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x03src\\x94\\x85\\x94.\\x94\\x8c\\x07columns\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94C=\\x80\\x04\\x952\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x1acudf.core.column.numerical\\x94\\x8c\\x0fNumericalColumn\\x94\\x93\\x94.\\x94\\x8c\\x05dtype\\x94CB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i4\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94\\x8c\\x18dtype-is-cudf-serialized\\x94\\x89\\x8c\\x04data\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94CI\\x80\\x04\\x95>\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x14SpillableBufferSlice\\x94\\x93\\x94.\\x94\\x8c\\x0bframe_count\\x94K\\x01u\\x8c\\x04mask\\x94}\\x94(hGCD\\x80\\x04\\x959\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x0fSpillableBuffer\\x94\\x93\\x94.\\x94hIK\\x01u\\x8c\\x04size\\x94K\\x00hIK\\x02u\\x85\\x94\\x8c\\x05index\\x94}\\x94(\\x8c\\x0cindex_column\\x94}\\x94(\\x8c\\x05start\\x94K\\x00\\x8c\\x04stop\\x94K\\x00\\x8c\\x04step\\x94K\\x01u\\x8c\\x04name\\x94C\\x04\\x80\\x04N.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i8\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94\\x8c\\x0ftype-serialized\\x94C-\\x80\\x04\\x95\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x0fcudf.core.index\\x94\\x8c\\nRangeIndex\\x94\\x93\\x94.\\x94hIK\\x00u\\x8c\\x11index_frame_count\\x94K\\x00\\x8c\\x07is-cuda\\x94]\\x94(\\x88\\x88e\\x8c\\x07lengths\\x94]\\x94(K\\x00K\\x00e\\x8c\\twriteable\\x94NN\\x86\\x94u]\\x94(\\x8c\\x12numpy.core.numeric\\x94\\x8c\\x0b_frombuffer\\x94\\x93\\x94(C\\x00\\x94\\x8c\\x05numpy\\x94hB\\x93\\x94\\x8c\\x02u1\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01|\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94bK\\x00\\x85\\x94\\x8c\\x01C\\x94t\\x94R\\x94he(C\\x00\\x94hkK\\x00\\x85\\x94hot\\x94R\\x94e\\x86\\x94R\\x94ee\\x86\\x94t\\x94\\x8c\\x13__dask_blockwise__2\\x94eh\\x1b]\\x94(]\\x94(h*h\\x13\\x8c\\x06rename\\x94\\x85\\x94R\\x94e]\\x94(h/h2h5h6\\x86\\x94R\\x94}\\x94(h:C0\\x80\\x04\\x95%\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94.\\x94h}\\x94(h@C=\\x80\\x04\\x952\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x1acudf.core.column.numerical\\x94\\x8c\\x0fNumericalColumn\\x94\\x93\\x94.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i4\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94hD\\x89hE}\\x94(hGCI\\x80\\x04\\x95>\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x14SpillableBufferSlice\\x94\\x93\\x94.\\x94hIK\\x01uhMK\\x00hIK\\x01u\\x85\\x94hO}\\x94(hQ}\\x94(hSK\\x00hTK\\x00hUK\\x01uhVC\\x04\\x80\\x04N.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i8\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94hYC-\\x80\\x04\\x95\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x0fcudf.core.index\\x94\\x8c\\nRangeIndex\\x94\\x93\\x94.\\x94hIK\\x00uh[K\\x00h\\\\]\\x94\\x88ah^]\\x94K\\x00ah`N\\x85\\x94u]\\x94he(C\\x00\\x94hkK\\x00\\x85\\x94hot\\x94R\\x94a\\x86\\x94R\\x94e]\\x94(h>h\\x1b]\\x94]\\x94(\\x8c\\x03src\\x94h\\x9eea\\x86\\x94ee\\x86\\x94t\\x94h\\x05(h\\x11h!\\x8c\\x10_reduction_chunk\\x94\\x93\\x94]\\x94h\\x0b(\\x8c\\x16dask.dataframe.methods\\x94\\x8c\\x06assign\\x94\\x93\\x94h\\x06h\\rh\\x08t\\x94h&\\x87\\x94ah\\x1b]\\x94]\\x94(\\x8c\\taca_chunk\\x94h0\\x8c\\x03len\\x94\\x93\\x94ea\\x86\\x94t\\x94\\x8c\\x13__dask_blockwise__0\\x94h\\x9e\\x8c\\x13__dask_blockwise__1\\x94\\x8c\\x03dst\\x94\\x8c\\x13__dask_blockwise__2\\x94N\\x8c\\x13__dask_blockwise__3\\x94\\x8c)to_frame-804980ae30b71d28f0a6bd3d5b7610f9\\x94\\x8c\\x13__dask_blockwise__4\\x94\\x8c(getitem-15414b72be12e28054238b44933937ab\\x94\\x8c\\x13__dask_blockwise__6\\x94\\x8c3cudf-aggregate-agg-c50c2d97de169ca4f41e43a92a042630\\x94uh\\x04\\x8c\\x13__dask_blockwise__5\\x94\\x85\\x94\\x8c6subgraph_callable-b4ca530e-8895-432e-b553-40a7b5892ab2\\x94t\\x94R\\x94.'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2244, in execute\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2244, in execute\n", " function, args, kwargs = await self._maybe_deserialize_task(ts)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2216, in _maybe_deserialize_task\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2216, in _maybe_deserialize_task\n", " function, args, kwargs = _deserialize(*ts.run_spec)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2937, in _deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2937, in _deserialize\n", " function = loads_function(function)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2925, in loads_function\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2925, in loads_function\n", " result = pickle.loads(bytes_object)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py\", line 96, in loads\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py\", line 96, in loads\n", " return pickle.loads(x)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 176, in host_deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 176, in host_deserialize\n", " obj = cls.device_deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 130, in device_deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 130, in device_deserialize\n", " return typ.deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py\", line 1019, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py\", line 1019, in deserialize\n", " obj = super().deserialize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 106, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 106, in deserialize\n", " columns = deserialize_columns(header[\"columns\"], frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 2450, in deserialize_columns\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 2450, in deserialize_columns\n", " colobj = col_typ.deserialize(meta, frames[:col_frame_count])\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1216, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1216, in deserialize\n", " data, frames = unpack(header[\"data\"], frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1204, in unpack\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1204, in unpack\n", " obj = klass.deserialize(header, frames[:count])\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 574, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 574, in deserialize\n", " return SpillableBuffer.deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py\", line 335, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py\", line 335, in deserialize\n", " return cls._from_device_memory(frame)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 235, in _from_device_memory\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 235, in _from_device_memory\n", " ret._finalize_init(ptr_desc={\"type\": \"gpu\"}, exposed=exposed)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 206, in _finalize_init\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 206, in _finalize_init\n", " raise ValueError(\n", "ValueError: cannot create without a global spill manager\n", "2023-05-12 09:25:03,817 - distributed.worker - ERROR - Could not deserialize task ('len-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d', 0)\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2923, in loads_function\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2923, in loads_function\n", " result = cache_loads[bytes_object]\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/collections.py\", line 24, in __getitem__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/collections.py\", line 24, in __getitem__\n", " value = super().__getitem__(key)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/collections/__init__.py\", line 1106, in __getitem__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/collections/__init__.py\", line 1106, in __getitem__\n", " raise KeyError(key)\n", "KeyError: b'\\x80\\x05\\x95>\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x11dask.optimization\\x94\\x8c\\x10SubgraphCallable\\x94\\x93\\x94(}\\x94(\\x8cKlen-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d\\x94\\x8cZassign-getitem-len-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d\\x94\\x8c*rename-01db283bd79fee66f232920c8dc6b55e_.0\\x94\\x8c;getitem-to_frame-rename-01db283bd79fee66f232920c8dc6b55e_.0\\x94\\x8c+getitem-3499fd71ac25ebbc1a06991edea6067c_.0\\x94\\x8c\\t_operator\\x94\\x8c\\x07getitem\\x94\\x93\\x94\\x8c/reset_index-f4c18304ca92859ccd09f44cf89b4b43_.0\\x94\\x8c\\x13__dask_blockwise__1\\x94\\x87\\x94h\\x0c(\\x8c\\ndask.utils\\x94\\x8c\\x05apply\\x94\\x93\\x94h\\x0f\\x8c\\x0cmethodcaller\\x94\\x93\\x94\\x8c\\x0breset_index\\x94\\x85\\x94R\\x94]\\x94\\x8c\\x13__dask_blockwise__5\\x94a\\x8c\\x08builtins\\x94\\x8c\\x04dict\\x94\\x93\\x94]\\x94]\\x94(\\x8c\\x04drop\\x94\\x89ea\\x86\\x94t\\x94h\\x07(h\\x11\\x8c\\x13dask.dataframe.core\\x94\\x8c\\x11apply_and_enforce\\x94\\x93\\x94]\\x94((h\\x11h#]\\x94h\\x0bh\\x0c\\x8c\\x13__dask_blockwise__0\\x94\\x87\\x94ah\\x1b]\\x94(]\\x94(\\x8c\\x05_func\\x94h\\x13\\x8c\\x08to_frame\\x94\\x85\\x94R\\x94e]\\x94(\\x8c\\x05_meta\\x94\\x8c\\x08builtins\\x94\\x8c\\x07getattr\\x94\\x93\\x94\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94\\x8c\\x10host_deserialize\\x94\\x86\\x94R\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94C0\\x80\\x04\\x95%\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94.\\x94\\x8c\\x0ccolumn_names\\x94C\\x14\\x80\\x04\\x95\\t\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x03src\\x94\\x85\\x94.\\x94\\x8c\\x07columns\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94C=\\x80\\x04\\x952\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x1acudf.core.column.numerical\\x94\\x8c\\x0fNumericalColumn\\x94\\x93\\x94.\\x94\\x8c\\x05dtype\\x94CB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i4\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94\\x8c\\x18dtype-is-cudf-serialized\\x94\\x89\\x8c\\x04data\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94CI\\x80\\x04\\x95>\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x14SpillableBufferSlice\\x94\\x93\\x94.\\x94\\x8c\\x0bframe_count\\x94K\\x01u\\x8c\\x04mask\\x94}\\x94(hGCD\\x80\\x04\\x959\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x0fSpillableBuffer\\x94\\x93\\x94.\\x94hIK\\x01u\\x8c\\x04size\\x94K\\x00hIK\\x02u\\x85\\x94\\x8c\\x05index\\x94}\\x94(\\x8c\\x0cindex_column\\x94}\\x94(\\x8c\\x05start\\x94K\\x00\\x8c\\x04stop\\x94K\\x00\\x8c\\x04step\\x94K\\x01u\\x8c\\x04name\\x94C\\x04\\x80\\x04N.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i8\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94\\x8c\\x0ftype-serialized\\x94C-\\x80\\x04\\x95\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x0fcudf.core.index\\x94\\x8c\\nRangeIndex\\x94\\x93\\x94.\\x94hIK\\x00u\\x8c\\x11index_frame_count\\x94K\\x00\\x8c\\x07is-cuda\\x94]\\x94(\\x88\\x88e\\x8c\\x07lengths\\x94]\\x94(K\\x00K\\x00e\\x8c\\twriteable\\x94NN\\x86\\x94u]\\x94(\\x8c\\x12numpy.core.numeric\\x94\\x8c\\x0b_frombuffer\\x94\\x93\\x94(C\\x00\\x94\\x8c\\x05numpy\\x94hB\\x93\\x94\\x8c\\x02u1\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01|\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94bK\\x00\\x85\\x94\\x8c\\x01C\\x94t\\x94R\\x94he(C\\x00\\x94hkK\\x00\\x85\\x94hot\\x94R\\x94e\\x86\\x94R\\x94ee\\x86\\x94t\\x94\\x8c\\x13__dask_blockwise__2\\x94eh\\x1b]\\x94(]\\x94(h*h\\x13\\x8c\\x06rename\\x94\\x85\\x94R\\x94e]\\x94(h/h2h5h6\\x86\\x94R\\x94}\\x94(h:C0\\x80\\x04\\x95%\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94.\\x94h}\\x94(h@C=\\x80\\x04\\x952\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x1acudf.core.column.numerical\\x94\\x8c\\x0fNumericalColumn\\x94\\x93\\x94.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i4\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94hD\\x89hE}\\x94(hGCI\\x80\\x04\\x95>\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x14SpillableBufferSlice\\x94\\x93\\x94.\\x94hIK\\x01uhMK\\x00hIK\\x01u\\x85\\x94hO}\\x94(hQ}\\x94(hSK\\x00hTK\\x00hUK\\x01uhVC\\x04\\x80\\x04N.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i8\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94hYC-\\x80\\x04\\x95\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x0fcudf.core.index\\x94\\x8c\\nRangeIndex\\x94\\x93\\x94.\\x94hIK\\x00uh[K\\x00h\\\\]\\x94\\x88ah^]\\x94K\\x00ah`N\\x85\\x94u]\\x94he(C\\x00\\x94hkK\\x00\\x85\\x94hot\\x94R\\x94a\\x86\\x94R\\x94e]\\x94(h>h\\x1b]\\x94]\\x94(\\x8c\\x03src\\x94h\\x9eea\\x86\\x94ee\\x86\\x94t\\x94h\\x05(h\\x11h!\\x8c\\x10_reduction_chunk\\x94\\x93\\x94]\\x94h\\x0b(\\x8c\\x16dask.dataframe.methods\\x94\\x8c\\x06assign\\x94\\x93\\x94h\\x06h\\rh\\x08t\\x94h&\\x87\\x94ah\\x1b]\\x94]\\x94(\\x8c\\taca_chunk\\x94h0\\x8c\\x03len\\x94\\x93\\x94ea\\x86\\x94t\\x94\\x8c\\x13__dask_blockwise__0\\x94h\\x9e\\x8c\\x13__dask_blockwise__1\\x94\\x8c\\x03dst\\x94\\x8c\\x13__dask_blockwise__2\\x94N\\x8c\\x13__dask_blockwise__3\\x94\\x8c)to_frame-804980ae30b71d28f0a6bd3d5b7610f9\\x94\\x8c\\x13__dask_blockwise__4\\x94\\x8c(getitem-15414b72be12e28054238b44933937ab\\x94\\x8c\\x13__dask_blockwise__6\\x94\\x8c3cudf-aggregate-agg-c50c2d97de169ca4f41e43a92a042630\\x94uh\\x04\\x8c\\x13__dask_blockwise__5\\x94\\x85\\x94\\x8c6subgraph_callable-b4ca530e-8895-432e-b553-40a7b5892ab2\\x94t\\x94R\\x94.'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2244, in execute\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2244, in execute\n", " function, args, kwargs = await self._maybe_deserialize_task(ts)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2216, in _maybe_deserialize_task\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2216, in _maybe_deserialize_task\n", " function, args, kwargs = _deserialize(*ts.run_spec)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2937, in _deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2937, in _deserialize\n", " function = loads_function(function)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2925, in loads_function\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2925, in loads_function\n", " result = pickle.loads(bytes_object)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py\", line 96, in loads\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py\", line 96, in loads\n", " return pickle.loads(x)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 176, in host_deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 176, in host_deserialize\n", " obj = cls.device_deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 130, in device_deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 130, in device_deserialize\n", " return typ.deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py\", line 1019, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py\", line 1019, in deserialize\n", " obj = super().deserialize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 106, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 106, in deserialize\n", " columns = deserialize_columns(header[\"columns\"], frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 2450, in deserialize_columns\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 2450, in deserialize_columns\n", " colobj = col_typ.deserialize(meta, frames[:col_frame_count])\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1216, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1216, in deserialize\n", " data, frames = unpack(header[\"data\"], frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1204, in unpack\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1204, in unpack\n", " obj = klass.deserialize(header, frames[:count])\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 574, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 574, in deserialize\n", " return SpillableBuffer.deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py\", line 335, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py\", line 335, in deserialize\n", " return cls._from_device_memory(frame)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 235, in _from_device_memory\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 235, in _from_device_memory\n", " ret._finalize_init(ptr_desc={\"type\": \"gpu\"}, exposed=exposed)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 206, in _finalize_init\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 206, in _finalize_init\n", " raise ValueError(\n", "ValueError: cannot create without a global spill manager\n" ] @@ -475,34 +475,34 @@ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[6], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39m# Create a directed graph using the source (src) and destination (dst) vertex pairs from the Dataframe \u001b[39;00m\n\u001b[1;32m 2\u001b[0m G \u001b[39m=\u001b[39m cugraph\u001b[39m.\u001b[39mGraph(directed\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[0;32m----> 3\u001b[0m G\u001b[39m.\u001b[39;49mfrom_dask_cudf_edgelist(e_list, source\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39msrc\u001b[39;49m\u001b[39m'\u001b[39;49m, destination\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39mdst\u001b[39;49m\u001b[39m'\u001b[39;49m)\n\u001b[1;32m 5\u001b[0m \u001b[39m# Print time\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mRead, load and renumber: \u001b[39m\u001b[39m\"\u001b[39m, time\u001b[39m.\u001b[39mtime()\u001b[39m-\u001b[39mt_start, \u001b[39m\"\u001b[39m\u001b[39ms\u001b[39m\u001b[39m\"\u001b[39m)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cugraph/structure/graph_classes.py:309\u001b[0m, in \u001b[0;36mGraph.from_dask_cudf_edgelist\u001b[0;34m(self, input_ddf, source, destination, edge_attr, renumber, store_transposed, legacy_renum_only)\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_Impl\u001b[39m.\u001b[39medgelist \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 308\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mGraph already has values\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 309\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_Impl\u001b[39m.\u001b[39;49m_simpleDistributedGraphImpl__from_edgelist(\n\u001b[1;32m 310\u001b[0m input_ddf,\n\u001b[1;32m 311\u001b[0m source,\n\u001b[1;32m 312\u001b[0m destination,\n\u001b[1;32m 313\u001b[0m edge_attr,\n\u001b[1;32m 314\u001b[0m renumber,\n\u001b[1;32m 315\u001b[0m store_transposed,\n\u001b[1;32m 316\u001b[0m legacy_renum_only,\n\u001b[1;32m 317\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cugraph/structure/graph_implementation/simpleDistributedGraph.py:272\u001b[0m, in \u001b[0;36msimpleDistributedGraphImpl.__from_edgelist\u001b[0;34m(self, input_ddf, source, destination, edge_attr, renumber, store_transposed, legacy_renum_only)\u001b[0m\n\u001b[1;32m 268\u001b[0m dst_col_name \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrenumber_map\u001b[39m.\u001b[39mrenumbered_dst_col_name\n\u001b[1;32m 270\u001b[0m ddf \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39medgelist\u001b[39m.\u001b[39medgelist_df\n\u001b[0;32m--> 272\u001b[0m num_edges \u001b[39m=\u001b[39m \u001b[39mlen\u001b[39;49m(ddf)\n\u001b[1;32m 273\u001b[0m edge_data \u001b[39m=\u001b[39m get_distributed_data(ddf)\n\u001b[1;32m 275\u001b[0m graph_props \u001b[39m=\u001b[39m GraphProperties(\n\u001b[1;32m 276\u001b[0m is_multigraph\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mproperties\u001b[39m.\u001b[39mmulti_edge,\n\u001b[1;32m 277\u001b[0m is_symmetric\u001b[39m=\u001b[39m\u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mproperties\u001b[39m.\u001b[39mdirected,\n\u001b[1;32m 278\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/dataframe/core.py:4775\u001b[0m, in \u001b[0;36mDataFrame.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 4773\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m\u001b[39m__len__\u001b[39m()\n\u001b[1;32m 4774\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 4775\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlen\u001b[39;49m(s)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/dataframe/core.py:843\u001b[0m, in \u001b[0;36m_Frame.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 840\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__len__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 841\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mreduction(\n\u001b[1;32m 842\u001b[0m \u001b[39mlen\u001b[39;49m, np\u001b[39m.\u001b[39;49msum, token\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mlen\u001b[39;49m\u001b[39m\"\u001b[39;49m, meta\u001b[39m=\u001b[39;49m\u001b[39mint\u001b[39;49m, split_every\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m\n\u001b[0;32m--> 843\u001b[0m )\u001b[39m.\u001b[39;49mcompute()\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/base.py:314\u001b[0m, in \u001b[0;36mDaskMethodsMixin.compute\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcompute\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 291\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Compute this dask collection\u001b[39;00m\n\u001b[1;32m 292\u001b[0m \n\u001b[1;32m 293\u001b[0m \u001b[39m This turns a lazy Dask collection into its in-memory equivalent.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[39m dask.base.compute\u001b[39;00m\n\u001b[1;32m 313\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 314\u001b[0m (result,) \u001b[39m=\u001b[39m compute(\u001b[39mself\u001b[39;49m, traverse\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 315\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/base.py:599\u001b[0m, in \u001b[0;36mcompute\u001b[0;34m(traverse, optimize_graph, scheduler, get, *args, **kwargs)\u001b[0m\n\u001b[1;32m 596\u001b[0m keys\u001b[39m.\u001b[39mappend(x\u001b[39m.\u001b[39m__dask_keys__())\n\u001b[1;32m 597\u001b[0m postcomputes\u001b[39m.\u001b[39mappend(x\u001b[39m.\u001b[39m__dask_postcompute__())\n\u001b[0;32m--> 599\u001b[0m results \u001b[39m=\u001b[39m schedule(dsk, keys, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 600\u001b[0m \u001b[39mreturn\u001b[39;00m repack([f(r, \u001b[39m*\u001b[39ma) \u001b[39mfor\u001b[39;00m r, (f, a) \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(results, postcomputes)])\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:3186\u001b[0m, in \u001b[0;36mClient.get\u001b[0;34m(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)\u001b[0m\n\u001b[1;32m 3184\u001b[0m should_rejoin \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 3185\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 3186\u001b[0m results \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgather(packed, asynchronous\u001b[39m=\u001b[39;49masynchronous, direct\u001b[39m=\u001b[39;49mdirect)\n\u001b[1;32m 3187\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 3188\u001b[0m \u001b[39mfor\u001b[39;00m f \u001b[39min\u001b[39;00m futures\u001b[39m.\u001b[39mvalues():\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:2345\u001b[0m, in \u001b[0;36mClient.gather\u001b[0;34m(self, futures, errors, direct, asynchronous)\u001b[0m\n\u001b[1;32m 2343\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 2344\u001b[0m local_worker \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m-> 2345\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msync(\n\u001b[1;32m 2346\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_gather,\n\u001b[1;32m 2347\u001b[0m futures,\n\u001b[1;32m 2348\u001b[0m errors\u001b[39m=\u001b[39;49merrors,\n\u001b[1;32m 2349\u001b[0m direct\u001b[39m=\u001b[39;49mdirect,\n\u001b[1;32m 2350\u001b[0m local_worker\u001b[39m=\u001b[39;49mlocal_worker,\n\u001b[1;32m 2351\u001b[0m asynchronous\u001b[39m=\u001b[39;49masynchronous,\n\u001b[1;32m 2352\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:349\u001b[0m, in \u001b[0;36mSyncMethodMixin.sync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[39mreturn\u001b[39;00m future\n\u001b[1;32m 348\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 349\u001b[0m \u001b[39mreturn\u001b[39;00m sync(\n\u001b[1;32m 350\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mloop, func, \u001b[39m*\u001b[39;49margs, callback_timeout\u001b[39m=\u001b[39;49mcallback_timeout, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs\n\u001b[1;32m 351\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:416\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 414\u001b[0m \u001b[39mif\u001b[39;00m error:\n\u001b[1;32m 415\u001b[0m typ, exc, tb \u001b[39m=\u001b[39m error\n\u001b[0;32m--> 416\u001b[0m \u001b[39mraise\u001b[39;00m exc\u001b[39m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m 417\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 418\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:389\u001b[0m, in \u001b[0;36msync..f\u001b[0;34m()\u001b[0m\n\u001b[1;32m 387\u001b[0m future \u001b[39m=\u001b[39m wait_for(future, callback_timeout)\n\u001b[1;32m 388\u001b[0m future \u001b[39m=\u001b[39m asyncio\u001b[39m.\u001b[39mensure_future(future)\n\u001b[0;32m--> 389\u001b[0m result \u001b[39m=\u001b[39m \u001b[39myield\u001b[39;00m future\n\u001b[1;32m 390\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 391\u001b[0m error \u001b[39m=\u001b[39m sys\u001b[39m.\u001b[39mexc_info()\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/tornado/gen.py:769\u001b[0m, in \u001b[0;36mRunner.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 766\u001b[0m exc_info \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 768\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 769\u001b[0m value \u001b[39m=\u001b[39m future\u001b[39m.\u001b[39;49mresult()\n\u001b[1;32m 770\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 771\u001b[0m exc_info \u001b[39m=\u001b[39m sys\u001b[39m.\u001b[39mexc_info()\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:2208\u001b[0m, in \u001b[0;36mClient._gather\u001b[0;34m(self, futures, errors, direct, local_worker)\u001b[0m\n\u001b[1;32m 2206\u001b[0m exc \u001b[39m=\u001b[39m CancelledError(key)\n\u001b[1;32m 2207\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2208\u001b[0m \u001b[39mraise\u001b[39;00m exception\u001b[39m.\u001b[39mwith_traceback(traceback)\n\u001b[1;32m 2209\u001b[0m \u001b[39mraise\u001b[39;00m exc\n\u001b[1;32m 2210\u001b[0m \u001b[39mif\u001b[39;00m errors \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mskip\u001b[39m\u001b[39m\"\u001b[39m:\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py:79\u001b[0m, in \u001b[0;36minner\u001b[0;34m()\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@wraps\u001b[39m(func)\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39minner\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds):\n\u001b[1;32m 78\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_recreate_cm():\n\u001b[0;32m---> 79\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py:2937\u001b[0m, in \u001b[0;36m_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2934\u001b[0m \u001b[39m# Some objects require threadlocal state during deserialization, e.g. to\u001b[39;00m\n\u001b[1;32m 2935\u001b[0m \u001b[39m# detect the current worker\u001b[39;00m\n\u001b[1;32m 2936\u001b[0m \u001b[39mif\u001b[39;00m function \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 2937\u001b[0m function \u001b[39m=\u001b[39m loads_function(function)\n\u001b[1;32m 2938\u001b[0m \u001b[39mif\u001b[39;00m args \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(args, \u001b[39mbytes\u001b[39m):\n\u001b[1;32m 2939\u001b[0m args \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(args)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py:2925\u001b[0m, in \u001b[0;36mloads_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2923\u001b[0m result \u001b[39m=\u001b[39m cache_loads[bytes_object]\n\u001b[1;32m 2924\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m:\n\u001b[0;32m-> 2925\u001b[0m result \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(bytes_object)\n\u001b[1;32m 2926\u001b[0m cache_loads[bytes_object] \u001b[39m=\u001b[39m result\n\u001b[1;32m 2927\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py:96\u001b[0m, in \u001b[0;36mloads\u001b[0;34m()\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[39mreturn\u001b[39;00m pickle\u001b[39m.\u001b[39mloads(x, buffers\u001b[39m=\u001b[39mbuffers)\n\u001b[1;32m 95\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 96\u001b[0m \u001b[39mreturn\u001b[39;00m pickle\u001b[39m.\u001b[39mloads(x)\n\u001b[1;32m 97\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 98\u001b[0m logger\u001b[39m.\u001b[39minfo(\u001b[39m\"\u001b[39m\u001b[39mFailed to deserialize \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m, x[:\u001b[39m10000\u001b[39m], exc_info\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py:176\u001b[0m, in \u001b[0;36mhost_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Perform device-side deserialization tasks.\u001b[39;00m\n\u001b[1;32m 155\u001b[0m \n\u001b[1;32m 156\u001b[0m \u001b[39mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[39m:meta private:\u001b[39;00m\n\u001b[1;32m 171\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 172\u001b[0m frames \u001b[39m=\u001b[39m [\n\u001b[1;32m 173\u001b[0m cudf\u001b[39m.\u001b[39mcore\u001b[39m.\u001b[39mbuffer\u001b[39m.\u001b[39mas_buffer(f) \u001b[39mif\u001b[39;00m c \u001b[39melse\u001b[39;00m f\n\u001b[1;32m 174\u001b[0m \u001b[39mfor\u001b[39;00m c, f \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(header[\u001b[39m\"\u001b[39m\u001b[39mis-cuda\u001b[39m\u001b[39m\"\u001b[39m], \u001b[39mmap\u001b[39m(\u001b[39mmemoryview\u001b[39m, frames))\n\u001b[1;32m 175\u001b[0m ]\n\u001b[0;32m--> 176\u001b[0m obj \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39mdevice_deserialize(header, frames)\n\u001b[1;32m 177\u001b[0m \u001b[39mreturn\u001b[39;00m obj\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py:130\u001b[0m, in \u001b[0;36mdevice_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 125\u001b[0m typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 126\u001b[0m frames \u001b[39m=\u001b[39m [\n\u001b[1;32m 127\u001b[0m cudf\u001b[39m.\u001b[39mcore\u001b[39m.\u001b[39mbuffer\u001b[39m.\u001b[39mas_buffer(f) \u001b[39mif\u001b[39;00m c \u001b[39melse\u001b[39;00m \u001b[39mmemoryview\u001b[39m(f)\n\u001b[1;32m 128\u001b[0m \u001b[39mfor\u001b[39;00m c, f \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(header[\u001b[39m\"\u001b[39m\u001b[39mis-cuda\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 129\u001b[0m ]\n\u001b[0;32m--> 130\u001b[0m \u001b[39mreturn\u001b[39;00m typ\u001b[39m.\u001b[39mdeserialize(header, frames)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py:1019\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 1017\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdeserialize\u001b[39m(\u001b[39mcls\u001b[39m, header, frames):\n\u001b[1;32m 1018\u001b[0m index_nframes \u001b[39m=\u001b[39m header[\u001b[39m\"\u001b[39m\u001b[39mindex_frame_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[0;32m-> 1019\u001b[0m obj \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39mdeserialize(\n\u001b[1;32m 1020\u001b[0m header, frames[header[\u001b[39m\"\u001b[39m\u001b[39mindex_frame_count\u001b[39m\u001b[39m\"\u001b[39m] :]\n\u001b[1;32m 1021\u001b[0m )\n\u001b[1;32m 1023\u001b[0m idx_typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 1024\u001b[0m index \u001b[39m=\u001b[39m idx_typ\u001b[39m.\u001b[39mdeserialize(header[\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m], frames[:index_nframes])\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py:106\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 104\u001b[0m cls_deserialize \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 105\u001b[0m column_names \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mcolumn_names\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m--> 106\u001b[0m columns \u001b[39m=\u001b[39m deserialize_columns(header[\u001b[39m\"\u001b[39m\u001b[39mcolumns\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 107\u001b[0m \u001b[39mreturn\u001b[39;00m cls_deserialize\u001b[39m.\u001b[39m_from_data(\u001b[39mdict\u001b[39m(\u001b[39mzip\u001b[39m(column_names, columns)))\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:2450\u001b[0m, in \u001b[0;36mdeserialize_columns\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2448\u001b[0m col_frame_count \u001b[39m=\u001b[39m meta[\u001b[39m\"\u001b[39m\u001b[39mframe_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 2449\u001b[0m col_typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(meta[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m-> 2450\u001b[0m colobj \u001b[39m=\u001b[39m col_typ\u001b[39m.\u001b[39mdeserialize(meta, frames[:col_frame_count])\n\u001b[1;32m 2451\u001b[0m columns\u001b[39m.\u001b[39mappend(colobj)\n\u001b[1;32m 2452\u001b[0m \u001b[39m# Advance frames\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:1216\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1214\u001b[0m dtype \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mdtype\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 1215\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m header:\n\u001b[0;32m-> 1216\u001b[0m data, frames \u001b[39m=\u001b[39m unpack(header[\u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 1217\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 1218\u001b[0m data \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:1204\u001b[0m, in \u001b[0;36munpack\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1202\u001b[0m count \u001b[39m=\u001b[39m header[\u001b[39m\"\u001b[39m\u001b[39mframe_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 1203\u001b[0m klass \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m-> 1204\u001b[0m obj \u001b[39m=\u001b[39m klass\u001b[39m.\u001b[39mdeserialize(header, frames[:count])\n\u001b[1;32m 1205\u001b[0m \u001b[39mreturn\u001b[39;00m obj, frames[count:]\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:574\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 568\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdeserialize\u001b[39m(\u001b[39mcls\u001b[39m, header: \u001b[39mdict\u001b[39m, frames: \u001b[39mlist\u001b[39m):\n\u001b[1;32m 569\u001b[0m \u001b[39m# TODO: because of the hack in `SpillableBuffer.serialize()` where\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 572\u001b[0m \u001b[39m# deserialize into `SpillableBufferSlice` when the frames hasn't been\u001b[39;00m\n\u001b[1;32m 573\u001b[0m \u001b[39m# copied.\u001b[39;00m\n\u001b[0;32m--> 574\u001b[0m \u001b[39mreturn\u001b[39;00m SpillableBuffer\u001b[39m.\u001b[39mdeserialize(header, frames)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py:335\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[39mreturn\u001b[39;00m frame \u001b[39m# The frame is already deserialized\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(frame, \u001b[39m\"\u001b[39m\u001b[39m__cuda_array_interface__\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m--> 335\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_from_device_memory(frame)\n\u001b[1;32m 336\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_from_host_memory(frame)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:235\u001b[0m, in \u001b[0;36m_from_device_memory\u001b[0;34m()\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Create a spillabe buffer from device memory.\u001b[39;00m\n\u001b[1;32m 219\u001b[0m \n\u001b[1;32m 220\u001b[0m \u001b[39mNo data is being copied.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[39m Buffer representing the same device memory as `data`\u001b[39;00m\n\u001b[1;32m 233\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 234\u001b[0m ret \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m_from_device_memory(data)\n\u001b[0;32m--> 235\u001b[0m ret\u001b[39m.\u001b[39m_finalize_init(ptr_desc\u001b[39m=\u001b[39m{\u001b[39m\"\u001b[39m\u001b[39mtype\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mgpu\u001b[39m\u001b[39m\"\u001b[39m}, exposed\u001b[39m=\u001b[39mexposed)\n\u001b[1;32m 236\u001b[0m \u001b[39mreturn\u001b[39;00m ret\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:206\u001b[0m, in \u001b[0;36m_finalize_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m 204\u001b[0m manager \u001b[39m=\u001b[39m get_global_manager()\n\u001b[1;32m 205\u001b[0m \u001b[39mif\u001b[39;00m manager \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 206\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m 207\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mcannot create \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m without \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 208\u001b[0m \u001b[39m\"\u001b[39m\u001b[39ma global spill manager\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 209\u001b[0m )\n\u001b[1;32m 211\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_manager \u001b[39m=\u001b[39m manager\n\u001b[1;32m 212\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_manager\u001b[39m.\u001b[39madd(\u001b[39mself\u001b[39m)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cugraph/structure/graph_classes.py:309\u001b[0m, in \u001b[0;36mGraph.from_dask_cudf_edgelist\u001b[0;34m(self, input_ddf, source, destination, edge_attr, renumber, store_transposed, legacy_renum_only)\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_Impl\u001b[39m.\u001b[39medgelist \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 308\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mGraph already has values\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 309\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_Impl\u001b[39m.\u001b[39;49m_simpleDistributedGraphImpl__from_edgelist(\n\u001b[1;32m 310\u001b[0m input_ddf,\n\u001b[1;32m 311\u001b[0m source,\n\u001b[1;32m 312\u001b[0m destination,\n\u001b[1;32m 313\u001b[0m edge_attr,\n\u001b[1;32m 314\u001b[0m renumber,\n\u001b[1;32m 315\u001b[0m store_transposed,\n\u001b[1;32m 316\u001b[0m legacy_renum_only,\n\u001b[1;32m 317\u001b[0m )\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cugraph/structure/graph_implementation/simpleDistributedGraph.py:272\u001b[0m, in \u001b[0;36msimpleDistributedGraphImpl.__from_edgelist\u001b[0;34m(self, input_ddf, source, destination, edge_attr, renumber, store_transposed, legacy_renum_only)\u001b[0m\n\u001b[1;32m 268\u001b[0m dst_col_name \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrenumber_map\u001b[39m.\u001b[39mrenumbered_dst_col_name\n\u001b[1;32m 270\u001b[0m ddf \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39medgelist\u001b[39m.\u001b[39medgelist_df\n\u001b[0;32m--> 272\u001b[0m num_edges \u001b[39m=\u001b[39m \u001b[39mlen\u001b[39;49m(ddf)\n\u001b[1;32m 273\u001b[0m edge_data \u001b[39m=\u001b[39m get_distributed_data(ddf)\n\u001b[1;32m 275\u001b[0m graph_props \u001b[39m=\u001b[39m GraphProperties(\n\u001b[1;32m 276\u001b[0m is_multigraph\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mproperties\u001b[39m.\u001b[39mmulti_edge,\n\u001b[1;32m 277\u001b[0m is_symmetric\u001b[39m=\u001b[39m\u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mproperties\u001b[39m.\u001b[39mdirected,\n\u001b[1;32m 278\u001b[0m )\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/dataframe/core.py:4775\u001b[0m, in \u001b[0;36mDataFrame.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 4773\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m\u001b[39m__len__\u001b[39m()\n\u001b[1;32m 4774\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 4775\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlen\u001b[39;49m(s)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/dataframe/core.py:843\u001b[0m, in \u001b[0;36m_Frame.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 840\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__len__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 841\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mreduction(\n\u001b[1;32m 842\u001b[0m \u001b[39mlen\u001b[39;49m, np\u001b[39m.\u001b[39;49msum, token\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mlen\u001b[39;49m\u001b[39m\"\u001b[39;49m, meta\u001b[39m=\u001b[39;49m\u001b[39mint\u001b[39;49m, split_every\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m\n\u001b[0;32m--> 843\u001b[0m )\u001b[39m.\u001b[39;49mcompute()\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/base.py:314\u001b[0m, in \u001b[0;36mDaskMethodsMixin.compute\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcompute\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 291\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Compute this dask collection\u001b[39;00m\n\u001b[1;32m 292\u001b[0m \n\u001b[1;32m 293\u001b[0m \u001b[39m This turns a lazy Dask collection into its in-memory equivalent.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[39m dask.base.compute\u001b[39;00m\n\u001b[1;32m 313\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 314\u001b[0m (result,) \u001b[39m=\u001b[39m compute(\u001b[39mself\u001b[39;49m, traverse\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 315\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/base.py:599\u001b[0m, in \u001b[0;36mcompute\u001b[0;34m(traverse, optimize_graph, scheduler, get, *args, **kwargs)\u001b[0m\n\u001b[1;32m 596\u001b[0m keys\u001b[39m.\u001b[39mappend(x\u001b[39m.\u001b[39m__dask_keys__())\n\u001b[1;32m 597\u001b[0m postcomputes\u001b[39m.\u001b[39mappend(x\u001b[39m.\u001b[39m__dask_postcompute__())\n\u001b[0;32m--> 599\u001b[0m results \u001b[39m=\u001b[39m schedule(dsk, keys, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 600\u001b[0m \u001b[39mreturn\u001b[39;00m repack([f(r, \u001b[39m*\u001b[39ma) \u001b[39mfor\u001b[39;00m r, (f, a) \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(results, postcomputes)])\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:3186\u001b[0m, in \u001b[0;36mClient.get\u001b[0;34m(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)\u001b[0m\n\u001b[1;32m 3184\u001b[0m should_rejoin \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 3185\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 3186\u001b[0m results \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgather(packed, asynchronous\u001b[39m=\u001b[39;49masynchronous, direct\u001b[39m=\u001b[39;49mdirect)\n\u001b[1;32m 3187\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 3188\u001b[0m \u001b[39mfor\u001b[39;00m f \u001b[39min\u001b[39;00m futures\u001b[39m.\u001b[39mvalues():\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:2345\u001b[0m, in \u001b[0;36mClient.gather\u001b[0;34m(self, futures, errors, direct, asynchronous)\u001b[0m\n\u001b[1;32m 2343\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 2344\u001b[0m local_worker \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m-> 2345\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msync(\n\u001b[1;32m 2346\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_gather,\n\u001b[1;32m 2347\u001b[0m futures,\n\u001b[1;32m 2348\u001b[0m errors\u001b[39m=\u001b[39;49merrors,\n\u001b[1;32m 2349\u001b[0m direct\u001b[39m=\u001b[39;49mdirect,\n\u001b[1;32m 2350\u001b[0m local_worker\u001b[39m=\u001b[39;49mlocal_worker,\n\u001b[1;32m 2351\u001b[0m asynchronous\u001b[39m=\u001b[39;49masynchronous,\n\u001b[1;32m 2352\u001b[0m )\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:349\u001b[0m, in \u001b[0;36mSyncMethodMixin.sync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[39mreturn\u001b[39;00m future\n\u001b[1;32m 348\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 349\u001b[0m \u001b[39mreturn\u001b[39;00m sync(\n\u001b[1;32m 350\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mloop, func, \u001b[39m*\u001b[39;49margs, callback_timeout\u001b[39m=\u001b[39;49mcallback_timeout, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs\n\u001b[1;32m 351\u001b[0m )\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:416\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 414\u001b[0m \u001b[39mif\u001b[39;00m error:\n\u001b[1;32m 415\u001b[0m typ, exc, tb \u001b[39m=\u001b[39m error\n\u001b[0;32m--> 416\u001b[0m \u001b[39mraise\u001b[39;00m exc\u001b[39m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m 417\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 418\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:389\u001b[0m, in \u001b[0;36msync..f\u001b[0;34m()\u001b[0m\n\u001b[1;32m 387\u001b[0m future \u001b[39m=\u001b[39m wait_for(future, callback_timeout)\n\u001b[1;32m 388\u001b[0m future \u001b[39m=\u001b[39m asyncio\u001b[39m.\u001b[39mensure_future(future)\n\u001b[0;32m--> 389\u001b[0m result \u001b[39m=\u001b[39m \u001b[39myield\u001b[39;00m future\n\u001b[1;32m 390\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 391\u001b[0m error \u001b[39m=\u001b[39m sys\u001b[39m.\u001b[39mexc_info()\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/tornado/gen.py:769\u001b[0m, in \u001b[0;36mRunner.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 766\u001b[0m exc_info \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 768\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 769\u001b[0m value \u001b[39m=\u001b[39m future\u001b[39m.\u001b[39;49mresult()\n\u001b[1;32m 770\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 771\u001b[0m exc_info \u001b[39m=\u001b[39m sys\u001b[39m.\u001b[39mexc_info()\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:2208\u001b[0m, in \u001b[0;36mClient._gather\u001b[0;34m(self, futures, errors, direct, local_worker)\u001b[0m\n\u001b[1;32m 2206\u001b[0m exc \u001b[39m=\u001b[39m CancelledError(key)\n\u001b[1;32m 2207\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2208\u001b[0m \u001b[39mraise\u001b[39;00m exception\u001b[39m.\u001b[39mwith_traceback(traceback)\n\u001b[1;32m 2209\u001b[0m \u001b[39mraise\u001b[39;00m exc\n\u001b[1;32m 2210\u001b[0m \u001b[39mif\u001b[39;00m errors \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mskip\u001b[39m\u001b[39m\"\u001b[39m:\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py:79\u001b[0m, in \u001b[0;36minner\u001b[0;34m()\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@wraps\u001b[39m(func)\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39minner\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds):\n\u001b[1;32m 78\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_recreate_cm():\n\u001b[0;32m---> 79\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py:2937\u001b[0m, in \u001b[0;36m_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2934\u001b[0m \u001b[39m# Some objects require threadlocal state during deserialization, e.g. to\u001b[39;00m\n\u001b[1;32m 2935\u001b[0m \u001b[39m# detect the current worker\u001b[39;00m\n\u001b[1;32m 2936\u001b[0m \u001b[39mif\u001b[39;00m function \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 2937\u001b[0m function \u001b[39m=\u001b[39m loads_function(function)\n\u001b[1;32m 2938\u001b[0m \u001b[39mif\u001b[39;00m args \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(args, \u001b[39mbytes\u001b[39m):\n\u001b[1;32m 2939\u001b[0m args \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(args)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py:2925\u001b[0m, in \u001b[0;36mloads_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2923\u001b[0m result \u001b[39m=\u001b[39m cache_loads[bytes_object]\n\u001b[1;32m 2924\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m:\n\u001b[0;32m-> 2925\u001b[0m result \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(bytes_object)\n\u001b[1;32m 2926\u001b[0m cache_loads[bytes_object] \u001b[39m=\u001b[39m result\n\u001b[1;32m 2927\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py:96\u001b[0m, in \u001b[0;36mloads\u001b[0;34m()\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[39mreturn\u001b[39;00m pickle\u001b[39m.\u001b[39mloads(x, buffers\u001b[39m=\u001b[39mbuffers)\n\u001b[1;32m 95\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 96\u001b[0m \u001b[39mreturn\u001b[39;00m pickle\u001b[39m.\u001b[39mloads(x)\n\u001b[1;32m 97\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 98\u001b[0m logger\u001b[39m.\u001b[39minfo(\u001b[39m\"\u001b[39m\u001b[39mFailed to deserialize \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m, x[:\u001b[39m10000\u001b[39m], exc_info\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py:176\u001b[0m, in \u001b[0;36mhost_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Perform device-side deserialization tasks.\u001b[39;00m\n\u001b[1;32m 155\u001b[0m \n\u001b[1;32m 156\u001b[0m \u001b[39mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[39m:meta private:\u001b[39;00m\n\u001b[1;32m 171\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 172\u001b[0m frames \u001b[39m=\u001b[39m [\n\u001b[1;32m 173\u001b[0m cudf\u001b[39m.\u001b[39mcore\u001b[39m.\u001b[39mbuffer\u001b[39m.\u001b[39mas_buffer(f) \u001b[39mif\u001b[39;00m c \u001b[39melse\u001b[39;00m f\n\u001b[1;32m 174\u001b[0m \u001b[39mfor\u001b[39;00m c, f \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(header[\u001b[39m\"\u001b[39m\u001b[39mis-cuda\u001b[39m\u001b[39m\"\u001b[39m], \u001b[39mmap\u001b[39m(\u001b[39mmemoryview\u001b[39m, frames))\n\u001b[1;32m 175\u001b[0m ]\n\u001b[0;32m--> 176\u001b[0m obj \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39mdevice_deserialize(header, frames)\n\u001b[1;32m 177\u001b[0m \u001b[39mreturn\u001b[39;00m obj\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py:130\u001b[0m, in \u001b[0;36mdevice_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 125\u001b[0m typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 126\u001b[0m frames \u001b[39m=\u001b[39m [\n\u001b[1;32m 127\u001b[0m cudf\u001b[39m.\u001b[39mcore\u001b[39m.\u001b[39mbuffer\u001b[39m.\u001b[39mas_buffer(f) \u001b[39mif\u001b[39;00m c \u001b[39melse\u001b[39;00m \u001b[39mmemoryview\u001b[39m(f)\n\u001b[1;32m 128\u001b[0m \u001b[39mfor\u001b[39;00m c, f \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(header[\u001b[39m\"\u001b[39m\u001b[39mis-cuda\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 129\u001b[0m ]\n\u001b[0;32m--> 130\u001b[0m \u001b[39mreturn\u001b[39;00m typ\u001b[39m.\u001b[39mdeserialize(header, frames)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py:1019\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 1017\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdeserialize\u001b[39m(\u001b[39mcls\u001b[39m, header, frames):\n\u001b[1;32m 1018\u001b[0m index_nframes \u001b[39m=\u001b[39m header[\u001b[39m\"\u001b[39m\u001b[39mindex_frame_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[0;32m-> 1019\u001b[0m obj \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39mdeserialize(\n\u001b[1;32m 1020\u001b[0m header, frames[header[\u001b[39m\"\u001b[39m\u001b[39mindex_frame_count\u001b[39m\u001b[39m\"\u001b[39m] :]\n\u001b[1;32m 1021\u001b[0m )\n\u001b[1;32m 1023\u001b[0m idx_typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 1024\u001b[0m index \u001b[39m=\u001b[39m idx_typ\u001b[39m.\u001b[39mdeserialize(header[\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m], frames[:index_nframes])\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py:106\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 104\u001b[0m cls_deserialize \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 105\u001b[0m column_names \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mcolumn_names\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m--> 106\u001b[0m columns \u001b[39m=\u001b[39m deserialize_columns(header[\u001b[39m\"\u001b[39m\u001b[39mcolumns\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 107\u001b[0m \u001b[39mreturn\u001b[39;00m cls_deserialize\u001b[39m.\u001b[39m_from_data(\u001b[39mdict\u001b[39m(\u001b[39mzip\u001b[39m(column_names, columns)))\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:2450\u001b[0m, in \u001b[0;36mdeserialize_columns\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2448\u001b[0m col_frame_count \u001b[39m=\u001b[39m meta[\u001b[39m\"\u001b[39m\u001b[39mframe_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 2449\u001b[0m col_typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(meta[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m-> 2450\u001b[0m colobj \u001b[39m=\u001b[39m col_typ\u001b[39m.\u001b[39mdeserialize(meta, frames[:col_frame_count])\n\u001b[1;32m 2451\u001b[0m columns\u001b[39m.\u001b[39mappend(colobj)\n\u001b[1;32m 2452\u001b[0m \u001b[39m# Advance frames\u001b[39;00m\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:1216\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1214\u001b[0m dtype \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mdtype\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 1215\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m header:\n\u001b[0;32m-> 1216\u001b[0m data, frames \u001b[39m=\u001b[39m unpack(header[\u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 1217\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 1218\u001b[0m data \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:1204\u001b[0m, in \u001b[0;36munpack\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1202\u001b[0m count \u001b[39m=\u001b[39m header[\u001b[39m\"\u001b[39m\u001b[39mframe_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 1203\u001b[0m klass \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m-> 1204\u001b[0m obj \u001b[39m=\u001b[39m klass\u001b[39m.\u001b[39mdeserialize(header, frames[:count])\n\u001b[1;32m 1205\u001b[0m \u001b[39mreturn\u001b[39;00m obj, frames[count:]\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:574\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 568\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdeserialize\u001b[39m(\u001b[39mcls\u001b[39m, header: \u001b[39mdict\u001b[39m, frames: \u001b[39mlist\u001b[39m):\n\u001b[1;32m 569\u001b[0m \u001b[39m# TODO: because of the hack in `SpillableBuffer.serialize()` where\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 572\u001b[0m \u001b[39m# deserialize into `SpillableBufferSlice` when the frames hasn't been\u001b[39;00m\n\u001b[1;32m 573\u001b[0m \u001b[39m# copied.\u001b[39;00m\n\u001b[0;32m--> 574\u001b[0m \u001b[39mreturn\u001b[39;00m SpillableBuffer\u001b[39m.\u001b[39mdeserialize(header, frames)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py:335\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[39mreturn\u001b[39;00m frame \u001b[39m# The frame is already deserialized\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(frame, \u001b[39m\"\u001b[39m\u001b[39m__cuda_array_interface__\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m--> 335\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_from_device_memory(frame)\n\u001b[1;32m 336\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_from_host_memory(frame)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:235\u001b[0m, in \u001b[0;36m_from_device_memory\u001b[0;34m()\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Create a spillabe buffer from device memory.\u001b[39;00m\n\u001b[1;32m 219\u001b[0m \n\u001b[1;32m 220\u001b[0m \u001b[39mNo data is being copied.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[39m Buffer representing the same device memory as `data`\u001b[39;00m\n\u001b[1;32m 233\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 234\u001b[0m ret \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m_from_device_memory(data)\n\u001b[0;32m--> 235\u001b[0m ret\u001b[39m.\u001b[39m_finalize_init(ptr_desc\u001b[39m=\u001b[39m{\u001b[39m\"\u001b[39m\u001b[39mtype\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mgpu\u001b[39m\u001b[39m\"\u001b[39m}, exposed\u001b[39m=\u001b[39mexposed)\n\u001b[1;32m 236\u001b[0m \u001b[39mreturn\u001b[39;00m ret\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:206\u001b[0m, in \u001b[0;36m_finalize_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m 204\u001b[0m manager \u001b[39m=\u001b[39m get_global_manager()\n\u001b[1;32m 205\u001b[0m \u001b[39mif\u001b[39;00m manager \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 206\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m 207\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mcannot create \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m without \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 208\u001b[0m \u001b[39m\"\u001b[39m\u001b[39ma global spill manager\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 209\u001b[0m )\n\u001b[1;32m 211\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_manager \u001b[39m=\u001b[39m manager\n\u001b[1;32m 212\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_manager\u001b[39m.\u001b[39madd(\u001b[39mself\u001b[39m)\n", "\u001b[0;31mValueError\u001b[0m: cannot create without a global spill manager" ] } From fd714aab5b9d192d8b85214cefed1ab76ea5f5f1 Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Thu, 19 Sep 2024 12:06:39 -0400 Subject: [PATCH 2/9] DOC v24.12 Updates [skip ci] --- .../cuda11.8-conda/devcontainer.json | 6 +- .devcontainer/cuda11.8-pip/devcontainer.json | 8 +- .../cuda12.5-conda/devcontainer.json | 6 +- .devcontainer/cuda12.5-pip/devcontainer.json | 8 +- .github/workflows/build.yaml | 36 ++++----- .github/workflows/pr.yaml | 48 +++++------ .github/workflows/test.yaml | 18 ++--- VERSION | 2 +- .../all_cuda-118_arch-x86_64.yaml | 30 +++---- .../all_cuda-125_arch-x86_64.yaml | 30 +++---- .../cugraph-service/conda_build_config.yaml | 2 +- conda/recipes/cugraph/conda_build_config.yaml | 2 +- .../pylibcugraph/conda_build_config.yaml | 2 +- dependencies.yaml | 80 +++++++++---------- notebooks/README.md | 8 +- .../conda/cugraph_dgl_dev_cuda-118.yaml | 4 +- python/cugraph-dgl/pyproject.toml | 6 +- python/cugraph-equivariant/pyproject.toml | 2 +- .../conda/cugraph_pyg_dev_cuda-118.yaml | 4 +- python/cugraph-pyg/pyproject.toml | 6 +- python/cugraph-service/server/pyproject.toml | 16 ++-- python/cugraph/pyproject.toml | 24 +++--- python/nx-cugraph/pyproject.toml | 2 +- python/pylibcugraph/pyproject.toml | 10 +-- 24 files changed, 180 insertions(+), 180 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 9c83535b771..f5886540252 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda11.8-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index a559be18077..270bfa239ad 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,24 +5,24 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-ucx1.15.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda11.8-ucx1.15.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda11.8-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { "version": "11.8", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/cuda", diff --git a/.devcontainer/cuda12.5-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json index ca10c04edee..e31428e4b0c 100644 --- a/.devcontainer/cuda12.5-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 6e2bf45700a..835274999ba 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -5,24 +5,24 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { "version": "12.5", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/cuda", diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bc489ffd3f0..b272fb43e35 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -47,7 +47,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -77,13 +77,13 @@ jobs: date: ${{ inputs.date }} script: ci/build_wheel_pylibcugraph.sh extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.10 + extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -93,7 +93,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -101,12 +101,12 @@ jobs: date: ${{ inputs.date }} script: ci/build_wheel_cugraph.sh extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.10 + extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -116,7 +116,7 @@ jobs: wheel-build-nx-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -126,7 +126,7 @@ jobs: wheel-publish-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -136,7 +136,7 @@ jobs: wheel-build-cugraph-dgl: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -146,7 +146,7 @@ jobs: wheel-publish-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -156,7 +156,7 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -166,7 +166,7 @@ jobs: wheel-publish-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -175,7 +175,7 @@ jobs: package-name: cugraph-pyg wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -185,7 +185,7 @@ jobs: wheel-publish-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index dacd9a93399..2220aa8aad6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,29 +34,29 @@ jobs: - wheel-tests-cugraph-equivariant - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: pull-request node_type: cpu32 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 with: build_type: pull-request enable_check_symbols: true @@ -64,19 +64,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 with: build_type: pull-request conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -86,7 +86,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -96,63 +96,63 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.10 + extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph.sh extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.10 + extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: pull-request script: ci/test_wheel_cugraph.sh wheel-build-nx-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_nx-cugraph.sh wheel-tests-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh wheel-build-cugraph-dgl: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-dgl.sh wheel-tests-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: pull-request script: ci/test_wheel_cugraph-dgl.sh @@ -160,35 +160,35 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-pyg.sh wheel-tests-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: pull-request script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-equivariant.sh wheel-tests-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: pull-request script: ci/test_wheel_cugraph-equivariant.sh matrix_filter: map(select(.ARCH == "amd64")) devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 957d29ce72b..5fbdd276bd6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibcugraph.sh wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: script: ci/test_wheel_cugraph.sh wheel-tests-nx-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: script: ci/test_wheel_nx-cugraph.sh wheel-tests-cugraph-dgl: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/VERSION b/VERSION index 7c7ba04436f..af28c42b528 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.10.00 +24.12.00 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 18cca40c320..b95343be301 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -16,22 +16,22 @@ dependencies: - cuda-nvtx - cuda-version=11.8 - cudatoolkit -- cudf==24.10.*,>=0.0.0a0 +- cudf==24.12.*,>=0.0.0a0 - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- dask-cudf==24.10.*,>=0.0.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- dask-cudf==24.12.*,>=0.0.0a0 - doxygen - fsspec>=0.6.0 - gcc_linux-64=11.* - graphviz - ipython -- libcudf==24.10.*,>=0.0.0a0 -- libcugraphops==24.10.*,>=0.0.0a0 -- libraft-headers==24.10.*,>=0.0.0a0 -- libraft==24.10.*,>=0.0.0a0 -- librmm==24.10.*,>=0.0.0a0 +- libcudf==24.12.*,>=0.0.0a0 +- libcugraphops==24.12.*,>=0.0.0a0 +- libraft-headers==24.12.*,>=0.0.0a0 +- libraft==24.12.*,>=0.0.0a0 +- librmm==24.12.*,>=0.0.0a0 - nbsphinx - nccl>=2.9.9 - networkx>=2.5.1 @@ -49,9 +49,9 @@ dependencies: - pre-commit - pydantic - pydata-sphinx-theme -- pylibcugraphops==24.10.*,>=0.0.0a0 -- pylibraft==24.10.*,>=0.0.0a0 -- pylibwholegraph==24.10.*,>=0.0.0a0 +- pylibcugraphops==24.12.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 +- pylibwholegraph==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov @@ -59,12 +59,12 @@ dependencies: - pytest-xdist - python-louvain - pytorch>=2.0,<2.2.0a0 -- raft-dask==24.10.*,>=0.0.0a0 +- raft-dask==24.12.*,>=0.0.0a0 - rapids-build-backend>=0.3.1,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark - requests -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn>=0.23.1 - scipy @@ -77,7 +77,7 @@ dependencies: - torchdata - torchmetrics - ucx-proc=*=gpu -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 - wget - wheel name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index ef20371e0f5..ae152cebacf 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -18,26 +18,26 @@ dependencies: - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.5 -- cudf==24.10.*,>=0.0.0a0 +- cudf==24.12.*,>=0.0.0a0 - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- dask-cudf==24.10.*,>=0.0.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- dask-cudf==24.12.*,>=0.0.0a0 - doxygen - fsspec>=0.6.0 - gcc_linux-64=11.* - graphviz - ipython - libcublas-dev -- libcudf==24.10.*,>=0.0.0a0 -- libcugraphops==24.10.*,>=0.0.0a0 +- libcudf==24.12.*,>=0.0.0a0 +- libcugraphops==24.12.*,>=0.0.0a0 - libcurand-dev - libcusolver-dev - libcusparse-dev -- libraft-headers==24.10.*,>=0.0.0a0 -- libraft==24.10.*,>=0.0.0a0 -- librmm==24.10.*,>=0.0.0a0 +- libraft-headers==24.12.*,>=0.0.0a0 +- libraft==24.12.*,>=0.0.0a0 +- librmm==24.12.*,>=0.0.0a0 - nbsphinx - nccl>=2.9.9 - networkx>=2.5.1 @@ -54,9 +54,9 @@ dependencies: - pre-commit - pydantic - pydata-sphinx-theme -- pylibcugraphops==24.10.*,>=0.0.0a0 -- pylibraft==24.10.*,>=0.0.0a0 -- pylibwholegraph==24.10.*,>=0.0.0a0 +- pylibcugraphops==24.12.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 +- pylibwholegraph==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov @@ -64,12 +64,12 @@ dependencies: - pytest-xdist - python-louvain - pytorch>=2.0,<2.2.0a0 -- raft-dask==24.10.*,>=0.0.0a0 +- raft-dask==24.12.*,>=0.0.0a0 - rapids-build-backend>=0.3.1,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark - requests -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn>=0.23.1 - scipy @@ -82,7 +82,7 @@ dependencies: - torchdata - torchmetrics - ucx-proc=*=gpu -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 - wget - wheel name: all_cuda-125_arch-x86_64 diff --git a/conda/recipes/cugraph-service/conda_build_config.yaml b/conda/recipes/cugraph-service/conda_build_config.yaml index 2ac251ab10a..67ed3e26b0e 100644 --- a/conda/recipes/cugraph-service/conda_build_config.yaml +++ b/conda/recipes/cugraph-service/conda_build_config.yaml @@ -1,2 +1,2 @@ ucx_py_version: - - "0.40.*" + - "0.41.*" diff --git a/conda/recipes/cugraph/conda_build_config.yaml b/conda/recipes/cugraph/conda_build_config.yaml index 2525441f92d..10f2e15c550 100644 --- a/conda/recipes/cugraph/conda_build_config.yaml +++ b/conda/recipes/cugraph/conda_build_config.yaml @@ -20,4 +20,4 @@ c_stdlib_version: - "2.17" ucx_py_version: - - "0.40.*" + - "0.41.*" diff --git a/conda/recipes/pylibcugraph/conda_build_config.yaml b/conda/recipes/pylibcugraph/conda_build_config.yaml index 2525441f92d..10f2e15c550 100644 --- a/conda/recipes/pylibcugraph/conda_build_config.yaml +++ b/conda/recipes/pylibcugraph/conda_build_config.yaml @@ -20,4 +20,4 @@ c_stdlib_version: - "2.17" ucx_py_version: - - "0.40.*" + - "0.41.*" diff --git a/dependencies.yaml b/dependencies.yaml index 2c8335868ba..091f2db4548 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -426,12 +426,12 @@ dependencies: packages: - c-compiler - cxx-compiler - - libcudf==24.10.*,>=0.0.0a0 + - libcudf==24.12.*,>=0.0.0a0 # Deprecate libcugraphops - - libcugraphops==24.10.*,>=0.0.0a0 - - libraft-headers==24.10.*,>=0.0.0a0 - - libraft==24.10.*,>=0.0.0a0 - - librmm==24.10.*,>=0.0.0a0 + - libcugraphops==24.12.*,>=0.0.0a0 + - libraft-headers==24.12.*,>=0.0.0a0 + - libraft==24.12.*,>=0.0.0a0 + - librmm==24.12.*,>=0.0.0a0 - openmpi # Required for building cpp-mgtests (multi-GPU tests) specific: - output_types: [conda] @@ -521,8 +521,8 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - &dask rapids-dask-dependency==24.10.*,>=0.0.0a0 - - &dask_cuda dask-cuda==24.10.*,>=0.0.0a0 + - &dask rapids-dask-dependency==24.12.*,>=0.0.0a0 + - &dask_cuda dask-cuda==24.12.*,>=0.0.0a0 - &numba numba>=0.57 - &numpy numpy>=1.23,<2.0a0 - output_types: conda @@ -532,7 +532,7 @@ dependencies: - requests - nccl>=2.9.9 - ucx-proc=*=gpu - - &ucx_py_unsuffixed ucx-py==0.40.*,>=0.0.0a0 + - &ucx_py_unsuffixed ucx-py==0.41.*,>=0.0.0a0 - output_types: pyproject packages: # cudf uses fsspec but is protocol independent. cugraph @@ -545,12 +545,12 @@ dependencies: cuda: "11.*" cuda_suffixed: "true" packages: - - &ucx_py_cu11 ucx-py-cu11==0.40.*,>=0.0.0a0 + - &ucx_py_cu11 ucx-py-cu11==0.41.*,>=0.0.0a0 - matrix: cuda: "12.*" cuda_suffixed: "true" packages: - - &ucx_py_cu12 ucx-py-cu12==0.40.*,>=0.0.0a0 + - &ucx_py_cu12 ucx-py-cu12==0.41.*,>=0.0.0a0 - matrix: packages: - *ucx_py_unsuffixed @@ -573,15 +573,15 @@ dependencies: cuda: "11.*" cuda_suffixed: "true" packages: - - &cugraph_cu11 cugraph-cu11==24.10.*,>=0.0.0a0 + - &cugraph_cu11 cugraph-cu11==24.12.*,>=0.0.0a0 - matrix: cuda: "12.*" cuda_suffixed: "true" packages: - - &cugraph_cu12 cugraph-cu12==24.10.*,>=0.0.0a0 + - &cugraph_cu12 cugraph-cu12==24.12.*,>=0.0.0a0 - matrix: packages: - - &cugraph_unsuffixed cugraph==24.10.*,>=0.0.0a0 + - &cugraph_unsuffixed cugraph==24.12.*,>=0.0.0a0 python_run_cugraph_pyg: common: - output_types: [conda, pyproject] @@ -629,19 +629,19 @@ dependencies: cuda_suffixed: "true" packages: - *cugraph_cu11 - - cugraph-service-client-cu11==24.10.*,>=0.0.0a0 + - cugraph-service-client-cu11==24.12.*,>=0.0.0a0 - *ucx_py_cu11 - matrix: cuda: "12.*" cuda_suffixed: "true" packages: - *cugraph_cu12 - - cugraph-service-client-cu12==24.10.*,>=0.0.0a0 + - cugraph-service-client-cu12==24.12.*,>=0.0.0a0 - *ucx_py_cu12 - matrix: packages: - *cugraph_unsuffixed - - cugraph-service-client==24.10.*,>=0.0.0a0 + - cugraph-service-client==24.12.*,>=0.0.0a0 - *ucx_py_unsuffixed test_cpp: common: @@ -677,7 +677,7 @@ dependencies: - scikit-learn>=0.23.1 - output_types: [conda] packages: - - &pylibwholegraph_unsuffixed pylibwholegraph==24.10.*,>=0.0.0a0 + - &pylibwholegraph_unsuffixed pylibwholegraph==24.12.*,>=0.0.0a0 - *thrift test_python_pylibcugraph: common: @@ -759,19 +759,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibwholegraph-cu12==24.10.*,>=0.0.0a0 + - pylibwholegraph-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibwholegraph-cu11==24.10.*,>=0.0.0a0 + - pylibwholegraph-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} depends_on_rmm: common: - output_types: conda packages: - - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 + - &rmm_unsuffixed rmm==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -784,19 +784,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - rmm-cu12==24.10.*,>=0.0.0a0 + - rmm-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - rmm-cu11==24.10.*,>=0.0.0a0 + - rmm-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_unsuffixed]} depends_on_cudf: common: - output_types: conda packages: - - &cudf_unsuffixed cudf==24.10.*,>=0.0.0a0 + - &cudf_unsuffixed cudf==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -809,19 +809,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - cudf-cu12==24.10.*,>=0.0.0a0 + - cudf-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - cudf-cu11==24.10.*,>=0.0.0a0 + - cudf-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*cudf_unsuffixed]} depends_on_dask_cudf: common: - output_types: conda packages: - - &dask_cudf_unsuffixed dask-cudf==24.10.*,>=0.0.0a0 + - &dask_cudf_unsuffixed dask-cudf==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -834,19 +834,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - dask-cudf-cu12==24.10.*,>=0.0.0a0 + - dask-cudf-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - dask-cudf-cu11==24.10.*,>=0.0.0a0 + - dask-cudf-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*dask_cudf_unsuffixed]} depends_on_pylibraft: common: - output_types: conda packages: - - &pylibraft_unsuffixed pylibraft==24.10.*,>=0.0.0a0 + - &pylibraft_unsuffixed pylibraft==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -859,19 +859,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibraft-cu12==24.10.*,>=0.0.0a0 + - pylibraft-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibraft-cu11==24.10.*,>=0.0.0a0 + - pylibraft-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*pylibraft_unsuffixed]} depends_on_raft_dask: common: - output_types: conda packages: - - &raft_dask_unsuffixed raft-dask==24.10.*,>=0.0.0a0 + - &raft_dask_unsuffixed raft-dask==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -884,19 +884,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - raft-dask-cu12==24.10.*,>=0.0.0a0 + - raft-dask-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - raft-dask-cu11==24.10.*,>=0.0.0a0 + - raft-dask-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*raft_dask_unsuffixed]} depends_on_pylibcugraph: common: - output_types: conda packages: - - &pylibcugraph_unsuffixed pylibcugraph==24.10.*,>=0.0.0a0 + - &pylibcugraph_unsuffixed pylibcugraph==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -909,12 +909,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibcugraph-cu12==24.10.*,>=0.0.0a0 + - pylibcugraph-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibcugraph-cu11==24.10.*,>=0.0.0a0 + - pylibcugraph-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*pylibcugraph_unsuffixed]} # deprecate pylibcugraphops @@ -922,7 +922,7 @@ dependencies: common: - output_types: conda packages: - - &pylibcugraphops_unsuffixed pylibcugraphops==24.10.*,>=0.0.0a0 + - &pylibcugraphops_unsuffixed pylibcugraphops==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -935,12 +935,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibcugraphops-cu12==24.10.*,>=0.0.0a0 + - pylibcugraphops-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibcugraphops-cu11==24.10.*,>=0.0.0a0 + - pylibcugraphops-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*pylibcugraphops_unsuffixed]} depends_on_cupy: diff --git a/notebooks/README.md b/notebooks/README.md index a8f094c340b..f0d0a25b9dd 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -66,13 +66,13 @@ The easiest way to run the notebooks is to get the latest [rapidsai/notebooks](h For example, get the latest (as of writing the document) nightly image (`a` after the version number indicates that an image is nightly) with cuda 12.0 using ```sh -docker pull rapidsai/notebooks:24.10a-cuda12.0-py3.10 +docker pull rapidsai/notebooks:24.12a-cuda12.0-py3.10 ``` And, then run a container based on the image using ```sh -docker run --rm -it --pull always --gpus all --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p 8888:8888 rapidsai/notebooks:24.10a-cuda12.0-py3.10 +docker run --rm -it --pull always --gpus all --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p 8888:8888 rapidsai/notebooks:24.12a-cuda12.0-py3.10 ``` You are all set. Run and edit cugraph notebooks from a browser at url http://127.0.0.1:8888/lab/tree/cugraph/cugraph_benchmarks @@ -88,8 +88,8 @@ ssh -L 127.0.0.1:8888:127.0.0.1:8888 [USER_NAME@][REMOTE_HOST_NAME or REMOTE_HO and then run the container in your remote machine. ```sh -docker pull rapidsai/notebooks:24.10a-cuda12.0-py3.10 -docker run --rm -it --pull always --gpus all --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p 8888:8888 rapidsai/notebooks:24.10a-cuda12.0-py3.10 +docker pull rapidsai/notebooks:24.12a-cuda12.0-py3.10 +docker run --rm -it --pull always --gpus all --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p 8888:8888 rapidsai/notebooks:24.12a-cuda12.0-py3.10 ``` You can run and edit cugraph notebooks at url http://127.0.0.1:8888/lab/tree/cugraph/cugraph_benchmarks as if they are running locally. diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index ea30b652286..4e6bdc20232 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -9,11 +9,11 @@ channels: - conda-forge - nvidia dependencies: -- cugraph==24.10.*,>=0.0.0a0 +- cugraph==24.12.*,>=0.0.0a0 - dgl>=1.1.0.cu* - pandas - pre-commit -- pylibcugraphops==24.10.*,>=0.0.0a0 +- pylibcugraphops==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 0cfeb10822a..c1044efd7e7 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -24,16 +24,16 @@ classifiers = [ "Programming Language :: Python", ] dependencies = [ - "cugraph==24.10.*,>=0.0.0a0", + "cugraph==24.12.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<2.0a0", - "pylibcugraphops==24.10.*,>=0.0.0a0", + "pylibcugraphops==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.optional-dependencies] test = [ "pandas", - "pylibwholegraph==24.10.*,>=0.0.0a0", + "pylibwholegraph==24.12.*,>=0.0.0a0", "pytest", "pytest-benchmark", "pytest-cov", diff --git a/python/cugraph-equivariant/pyproject.toml b/python/cugraph-equivariant/pyproject.toml index 2cc25885b84..7713e89ac20 100644 --- a/python/cugraph-equivariant/pyproject.toml +++ b/python/cugraph-equivariant/pyproject.toml @@ -37,7 +37,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "pylibcugraphops==24.10.*,>=0.0.0a0", + "pylibcugraphops==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index bd1ca33af70..6a0523f1c2d 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -9,11 +9,11 @@ channels: - conda-forge - nvidia dependencies: -- cugraph==24.10.*,>=0.0.0a0 +- cugraph==24.12.*,>=0.0.0a0 - pandas - pre-commit - pyg>=2.5,<2.6 -- pylibcugraphops==24.10.*,>=0.0.0a0 +- pylibcugraphops==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index d206d6001cc..e3fb0eee98f 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -29,10 +29,10 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "cugraph==24.10.*,>=0.0.0a0", + "cugraph==24.12.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<2.0a0", - "pylibcugraphops==24.10.*,>=0.0.0a0", + "pylibcugraphops==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] @@ -42,7 +42,7 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [project.optional-dependencies] test = [ "pandas", - "pylibwholegraph==24.10.*,>=0.0.0a0", + "pylibwholegraph==24.12.*,>=0.0.0a0", "pytest", "pytest-benchmark", "pytest-cov", diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index b9789c201d2..c850397b6fc 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -20,18 +20,18 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "cudf==24.10.*,>=0.0.0a0", - "cugraph-service-client==24.10.*,>=0.0.0a0", - "cugraph==24.10.*,>=0.0.0a0", + "cudf==24.12.*,>=0.0.0a0", + "cugraph-service-client==24.12.*,>=0.0.0a0", + "cugraph==24.12.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", - "dask-cuda==24.10.*,>=0.0.0a0", - "dask-cudf==24.10.*,>=0.0.0a0", + "dask-cuda==24.12.*,>=0.0.0a0", + "dask-cudf==24.12.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<2.0a0", - "rapids-dask-dependency==24.10.*,>=0.0.0a0", - "rmm==24.10.*,>=0.0.0a0", + "rapids-dask-dependency==24.12.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", "thriftpy2!=0.5.0,!=0.5.1", - "ucx-py==0.40.*,>=0.0.0a0", + "ucx-py==0.41.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index 31721c8a568..142d0bcd5fa 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -23,18 +23,18 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "cudf==24.10.*,>=0.0.0a0", + "cudf==24.12.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", - "dask-cuda==24.10.*,>=0.0.0a0", - "dask-cudf==24.10.*,>=0.0.0a0", + "dask-cuda==24.12.*,>=0.0.0a0", + "dask-cudf==24.12.*,>=0.0.0a0", "fsspec[http]>=0.6.0", "numba>=0.57", "numpy>=1.23,<2.0a0", - "pylibcugraph==24.10.*,>=0.0.0a0", - "raft-dask==24.10.*,>=0.0.0a0", - "rapids-dask-dependency==24.10.*,>=0.0.0a0", - "rmm==24.10.*,>=0.0.0a0", - "ucx-py==0.40.*,>=0.0.0a0", + "pylibcugraph==24.12.*,>=0.0.0a0", + "raft-dask==24.12.*,>=0.0.0a0", + "rapids-dask-dependency==24.12.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", + "ucx-py==0.41.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -49,7 +49,7 @@ test = [ "networkx>=2.5.1", "numpy>=1.23,<2.0a0", "pandas", - "pylibwholegraph==24.10.*,>=0.0.0a0", + "pylibwholegraph==24.12.*,>=0.0.0a0", "pytest", "pytest-benchmark", "pytest-cov", @@ -82,9 +82,9 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", "ninja", - "pylibcugraph==24.10.*,>=0.0.0a0", - "pylibraft==24.10.*,>=0.0.0a0", - "rmm==24.10.*,>=0.0.0a0", + "pylibcugraph==24.12.*,>=0.0.0a0", + "pylibraft==24.12.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index e7b4ea44dd8..239d894f5b7 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "networkx>=3.0", "numpy>=1.23,<2.0a0", - "pylibcugraph==24.10.*,>=0.0.0a0", + "pylibcugraph==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.optional-dependencies] diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 92c417f0372..98bbe255e3e 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -27,8 +27,8 @@ dependencies = [ "nvidia-curand", "nvidia-cusolver", "nvidia-cusparse", - "pylibraft==24.10.*,>=0.0.0a0", - "rmm==24.10.*,>=0.0.0a0", + "pylibraft==24.12.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -40,7 +40,7 @@ classifiers = [ [project.optional-dependencies] test = [ - "cudf==24.10.*,>=0.0.0a0", + "cudf==24.12.*,>=0.0.0a0", "numpy>=1.23,<2.0a0", "pandas", "pytest", @@ -74,7 +74,7 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4,!=3.30.0", "ninja", - "pylibraft==24.10.*,>=0.0.0a0", - "rmm==24.10.*,>=0.0.0a0", + "pylibraft==24.12.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" From 7e058e2961b5b8be05877282d26051f36dd73c53 Mon Sep 17 00:00:00 2001 From: Rick Ratzel <3039903+rlratzel@users.noreply.github.com> Date: Fri, 20 Sep 2024 09:12:30 -0500 Subject: [PATCH 3/9] Drops duplicate edges in non-MultiGraph PLC `SGGraph` instances (#4658) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Graph input with duplicate edges intended for `Graph`/`DiGraph` instances resulted in internal PLC `SGGraph` instances with duplicate edges, which were effectively treated as MultiGraphs and caused incorrect results from algorithms like `pagerank`. This PR sets the `drop_multi_edges` PLC `SGGraph` ctor option to have PLC remove duplicate edges on `SGGraph` creation. The overhead to drop duplicate edges for non-MultiGraphs is negligible, and in the case of a large test graph (wikipedia data, 37.5M nodes, 464.5M edges) resulted in an overall _speedup_ for pagerank going from 12.2 seconds to 10.7 seconds on my workstation, likely due to fewer edges to process a minor slowdown from 10.5s to 10.7s. _edit: after several re-runs, the pagerank runtime before the change settled to 10.5, and the runtime after the change was typically 10.7._ A test was added that uses pagerank to ensure Graphs vs. MultiGraphs are handled correctly and duplicate edges are dropped as needed. The results when run without `drop_multi_edges` set: ``` > assert actual_pr_for_G == approx(expected_pr_for_G) E assert {0: 0.0875795...7955580949783} == approx({0: 0....32 ± 1.8e-07}) E E comparison failed. Mismatched elements: 4 / 4: E Max absolute difference: 0.08785887916592061 E Max relative difference: 0.5007959662968462 E Index | Obtained | Expected E 0 | 0.08757955580949783 | 0.17543839772251532 ± 1.8e-07 E 1 | 0.41242048144340515 | 0.32456160227748454 ± 3.2e-07 E 2 | 0.41242048144340515 | 0.32456160227748454 ± 3.2e-07 E 3 | 0.08757955580949783 | 0.17543839772251532 ± 1.8e-07 ``` The same test passes when run with the changes in this PR to set `drop_multi_edges`. Authors: - Rick Ratzel (https://github.com/rlratzel) Approvers: - Erik Welch (https://github.com/eriknw) URL: https://github.com/rapidsai/cugraph/pull/4658 --- python/nx-cugraph/nx_cugraph/classes/graph.py | 9 +++++ .../nx_cugraph/tests/test_pagerank.py | 36 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 python/nx-cugraph/nx_cugraph/tests/test_pagerank.py diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 7425eacb2b4..7c01365c0ac 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -689,6 +689,14 @@ def _get_plc_graph( src_indices = src_indices.astype(index_dtype) dst_indices = dst_indices.astype(index_dtype) + # This sets drop_multi_edges=True for non-multigraph input, which means + # the data in self.src_indices and self.dst_indices may not be + # identical to that contained in the returned pcl.SGGraph (the returned + # SGGraph may have fewer edges since duplicates are dropped). Ideally + # self.src_indices and self.dst_indices would be updated to have + # duplicate edges removed for non-multigraph instances, but that + # requires additional code which would be redundant and likely not as + # performant as the code in PLC. return plc.SGGraph( resource_handle=plc.ResourceHandle(), graph_properties=plc.GraphProperties( @@ -702,6 +710,7 @@ def _get_plc_graph( renumber=False, do_expensive_check=False, vertices_array=self._node_ids, + drop_multi_edges=not self.is_multigraph(), ) def _sort_edge_indices(self, primary="src"): diff --git a/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py new file mode 100644 index 00000000000..0b437df2d2f --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import pandas as pd +from pytest import approx + + +def test_pagerank_multigraph(): + """ + Ensures correct differences between pagerank results for Graphs + vs. MultiGraphs generated using from_pandas_edgelist() + """ + df = pd.DataFrame({"source": [0, 1, 1, 1, 1, 1, 1, 2], + "target": [1, 2, 2, 2, 2, 2, 2, 3]}) + expected_pr_for_G = nx.pagerank(nx.from_pandas_edgelist(df)) + expected_pr_for_MultiG = nx.pagerank( + nx.from_pandas_edgelist(df, create_using=nx.MultiGraph)) + + G = nx.from_pandas_edgelist(df, backend="cugraph") + actual_pr_for_G = nx.pagerank(G, backend="cugraph") + + MultiG = nx.from_pandas_edgelist(df, create_using=nx.MultiGraph, backend="cugraph") + actual_pr_for_MultiG = nx.pagerank(MultiG, backend="cugraph") + + assert actual_pr_for_G == approx(expected_pr_for_G) + assert actual_pr_for_MultiG == approx(expected_pr_for_MultiG) From 607ee5bcdc5fa75f097d5739169e9d2911106378 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:11:08 -0500 Subject: [PATCH 4/9] Update update-version.sh to use packaging lib (#4664) This PR updates the update-version.sh script to use the packaging library, given that setuptools is no longer included by default in Python 3.12. --- ci/release/update-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 36ab7252117..5859ebde953 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -45,8 +45,8 @@ function sed_runner() { echo "${NEXT_FULL_TAG}" > VERSION # Need to distutils-normalize the original version -NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") -NEXT_UCXX_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_UCXX_SHORT_TAG}'))") +NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))") +NEXT_UCXX_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_UCXX_SHORT_TAG}'))") DEPENDENCIES=( cudf From 59217ad009ad3febd13629b9eebb5681e47e3ab0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 24 Sep 2024 17:20:07 -0500 Subject: [PATCH 5/9] nx-cugraph: Updates nxcg.Graph classes for API-compatibility with NetworkX Graph classes, needed for zero code change graph generators (#4629) This is an alternative approach to #4558 for enabling GPU-accelerated NetworkX to "just work". It has similarities to #4558. I opted to make separate classes such as `ZeroGraph`, which I think makes for cleaner separation and gives us and users more control. There are a few lingering TODOs and code comments to tidy up, but I don't think there are any show-stoppers. I have not updated methods (such as `number_of_nodes`) to optimistically try to use GPU if possible, b/c this is not strictly necessary, but we should update these soon. I have run NetworkX tests with these classes using https://github.com/networkx/networkx/pull/7585 and https://github.com/networkx/networkx/pull/7600. We need the behavior in 7585, and 7600 is only useful for testing. There are only 3 new failing tests, and 3 tests that hang (I'll run them overnight to see if they finish). Here's a test summary: ``` 5548 passed, 24 skipped, 16 xfailed, 25 xpassed ``` Note that 25 tests that were failing now pass. I have not investigated test failures, xfails, or xpasses yet. I would like to add tests too. We rely heavily on the networkx cache. I think this is preferred. It is late for me. I will describe and show how and why this works later. I opted for `zero=` and `ZeroGraph`, because I find them delightful! Renaming is trivial if other terms are preferred. CC @quasiben Authors: - Erik Welch (https://github.com/eriknw) Approvers: - Bradley Dice (https://github.com/bdice) - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4629 --- ci/run_nx_cugraph_pytests.sh | 3 +- ci/test_python.sh | 2 +- ci/test_wheel.sh | 1 + .../all_cuda-118_arch-x86_64.yaml | 1 - .../all_cuda-125_arch-x86_64.yaml | 1 - dependencies.yaml | 1 - python/nx-cugraph/_nx_cugraph/__init__.py | 17 +- python/nx-cugraph/lint.yaml | 16 +- python/nx-cugraph/nx_cugraph/__init__.py | 14 +- .../algorithms/bipartite/generators.py | 3 +- .../algorithms/community/louvain.py | 6 +- .../nx-cugraph/nx_cugraph/algorithms/core.py | 7 +- .../algorithms/link_analysis/hits_alg.py | 3 +- .../nx_cugraph/algorithms/operators/unary.py | 12 +- .../algorithms/shortest_paths/generic.py | 5 +- .../algorithms/shortest_paths/unweighted.py | 5 +- .../traversal/breadth_first_search.py | 13 +- .../nx-cugraph/nx_cugraph/classes/__init__.py | 10 +- .../nx-cugraph/nx_cugraph/classes/digraph.py | 89 +++- python/nx-cugraph/nx_cugraph/classes/graph.py | 399 +++++++++++++++--- .../nx_cugraph/classes/multidigraph.py | 35 +- .../nx_cugraph/classes/multigraph.py | 152 ++++--- python/nx-cugraph/nx_cugraph/convert.py | 132 +++++- .../nx-cugraph/nx_cugraph/convert_matrix.py | 8 +- .../nx_cugraph/generators/_utils.py | 16 +- .../nx_cugraph/generators/classic.py | 7 +- .../nx_cugraph/generators/community.py | 7 +- .../nx-cugraph/nx_cugraph/generators/ego.py | 11 +- .../nx-cugraph/nx_cugraph/generators/small.py | 10 +- .../nx_cugraph/generators/social.py | 29 +- python/nx-cugraph/nx_cugraph/interface.py | 248 ++++++----- python/nx-cugraph/nx_cugraph/relabel.py | 17 +- .../nx-cugraph/nx_cugraph/tests/test_bfs.py | 5 +- .../nx_cugraph/tests/test_classes.py | 77 ++++ .../nx_cugraph/tests/test_cluster.py | 5 +- .../nx_cugraph/tests/test_convert.py | 3 - .../nx_cugraph/tests/test_ego_graph.py | 36 +- .../nx_cugraph/tests/test_generators.py | 42 +- .../nx_cugraph/tests/test_graph_methods.py | 4 +- .../nx_cugraph/tests/test_match_api.py | 3 - .../nx_cugraph/tests/test_multigraph.py | 6 +- .../nx_cugraph/tests/test_pagerank.py | 20 +- .../nx_cugraph/tests/testing_utils.py | 2 +- .../nx-cugraph/nx_cugraph/utils/decorators.py | 21 +- python/nx-cugraph/nx_cugraph/utils/misc.py | 36 +- python/nx-cugraph/pyproject.toml | 4 +- python/nx-cugraph/run_nx_tests.sh | 4 + 47 files changed, 1196 insertions(+), 352 deletions(-) create mode 100644 python/nx-cugraph/nx_cugraph/tests/test_classes.py diff --git a/ci/run_nx_cugraph_pytests.sh b/ci/run_nx_cugraph_pytests.sh index b0caffd0a0f..0e309d1e2d4 100755 --- a/ci/run_nx_cugraph_pytests.sh +++ b/ci/run_nx_cugraph_pytests.sh @@ -6,4 +6,5 @@ set -euo pipefail # Support invoking run_nx_cugraph_pytests.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/nx-cugraph/nx_cugraph -pytest --capture=no --cache-clear --benchmark-disable "$@" tests +NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest --capture=no --cache-clear --benchmark-disable "$@" tests +NX_CUGRAPH_USE_COMPAT_GRAPHS=True pytest --capture=no --cache-clear --benchmark-disable "$@" tests diff --git a/ci/test_python.sh b/ci/test_python.sh index e8c8272e8d6..810284b8c97 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -108,7 +108,7 @@ echo "nx-cugraph coverage from networkx tests: $_coverage" echo $_coverage | awk '{ if ($NF == "0.0%") exit 1 }' # Ensure all algorithms were called by comparing covered lines to function lines. # Run our tests again (they're fast enough) to add their coverage, then create coverage.json -pytest \ +NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest \ --pyargs nx_cugraph \ --config-file=../pyproject.toml \ --cov-config=../pyproject.toml \ diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 158704e08d1..e3690dfde6e 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -37,6 +37,7 @@ else DASK_DISTRIBUTED__SCHEDULER__WORKER_TTL="1000s" \ DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT="1000s" \ DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT="1000s" \ + NX_CUGRAPH_USE_COMPAT_GRAPHS=False \ python -m pytest \ -v \ --import-mode=append \ diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 18cca40c320..533f23cd7ac 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -44,7 +44,6 @@ dependencies: - nvcc_linux-64=11.8 - ogb - openmpi -- packaging>=21 - pandas - pre-commit - pydantic diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index ef20371e0f5..084a6adfd31 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -49,7 +49,6 @@ dependencies: - numpydoc - ogb - openmpi -- packaging>=21 - pandas - pre-commit - pydantic diff --git a/dependencies.yaml b/dependencies.yaml index 2c8335868ba..76048be2010 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -688,7 +688,6 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - packaging>=21 # not needed by nx-cugraph tests, but is required for running networkx tests - pytest-mpl cugraph_dgl_dev: diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index 41c18c27ecf..428d266dd2e 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -22,6 +22,7 @@ $ python _nx_cugraph/__init__.py """ +import os from _nx_cugraph._version import __version__ @@ -293,12 +294,20 @@ def get_info(): for key in info_keys: del d[key] + + d["default_config"] = { + "use_compat_graphs": os.environ.get("NX_CUGRAPH_USE_COMPAT_GRAPHS", "true") + .strip() + .lower() + == "true", + } return d -def _check_networkx_version(): - import warnings +def _check_networkx_version() -> tuple[int, int]: + """Check the version of networkx and return ``(major, minor)`` version tuple.""" import re + import warnings import networkx as nx @@ -321,6 +330,10 @@ def _check_networkx_version(): f"{nx.__version__}. Please upgrade (or fix) your Python environment." ) + nxver_major = int(version_major) + nxver_minor = int(re.match(r"^\d+", version_minor).group()) + return (nxver_major, nxver_minor) + if __name__ == "__main__": from pathlib import Path diff --git a/python/nx-cugraph/lint.yaml b/python/nx-cugraph/lint.yaml index b2184a185c4..dab2ea70ef1 100644 --- a/python/nx-cugraph/lint.yaml +++ b/python/nx-cugraph/lint.yaml @@ -26,7 +26,7 @@ repos: - id: mixed-line-ending - id: trailing-whitespace - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.18 + rev: v0.19 hooks: - id: validate-pyproject name: Validate pyproject.toml @@ -40,29 +40,29 @@ repos: hooks: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.17.0 hooks: - id: pyupgrade args: [--py310-plus] - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black # - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.4 + rev: v0.6.7 hooks: - id: ruff args: [--fix-only, --show-fixes] # --unsafe-fixes] - repo: https://github.com/PyCQA/flake8 - rev: 7.1.0 + rev: 7.1.1 hooks: - id: flake8 args: ['--per-file-ignores=_nx_cugraph/__init__.py:E501', '--extend-ignore=B020,SIM105'] # Why is this necessary? additional_dependencies: &flake8_dependencies # These versions need updated manually - - flake8==7.1.0 - - flake8-bugbear==24.4.26 + - flake8==7.1.1 + - flake8-bugbear==24.8.19 - flake8-simplify==0.21.0 - repo: https://github.com/asottile/yesqa rev: v1.5.0 @@ -77,7 +77,7 @@ repos: additional_dependencies: [tomli] files: ^(nx_cugraph|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.4 + rev: v0.6.7 hooks: - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/python/nx-cugraph/nx_cugraph/__init__.py b/python/nx-cugraph/nx_cugraph/__init__.py index 542256fa781..4404e57f645 100644 --- a/python/nx-cugraph/nx_cugraph/__init__.py +++ b/python/nx-cugraph/nx_cugraph/__init__.py @@ -12,6 +12,11 @@ # limitations under the License. from networkx.exception import * +from _nx_cugraph._version import __git_commit__, __version__ +from _nx_cugraph import _check_networkx_version + +_nxver: tuple[int, int] = _check_networkx_version() + from . import utils from . import classes @@ -32,7 +37,10 @@ from . import algorithms from .algorithms import * -from _nx_cugraph._version import __git_commit__, __version__ -from _nx_cugraph import _check_networkx_version +from .interface import BackendInterface -_check_networkx_version() +BackendInterface.Graph = classes.Graph +BackendInterface.DiGraph = classes.DiGraph +BackendInterface.MultiGraph = classes.MultiGraph +BackendInterface.MultiDiGraph = classes.MultiDiGraph +del BackendInterface diff --git a/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py index 60276b7d41b..214970235c6 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py @@ -16,6 +16,7 @@ import networkx as nx import numpy as np +from nx_cugraph import _nxver from nx_cugraph.generators._utils import _create_using_class, _number_and_nodes from nx_cugraph.utils import index_dtype, networkx_algorithm @@ -48,7 +49,7 @@ def complete_bipartite_graph(n1, n2, create_using=None): nodes.extend(range(n2) if nodes2 is None else nodes2) if len(set(nodes)) != len(nodes): raise nx.NetworkXError("Inputs n1 and n2 must contain distinct nodes") - if nx.__version__[:3] <= "3.3": + if _nxver <= (3, 3): name = f"complete_bipartite_graph({orig_n1}, {orig_n2})" else: name = f"complete_bipartite_graph({n1}, {n2})" diff --git a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py index ea1318060e0..52c512c454d 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py @@ -12,9 +12,9 @@ # limitations under the License. import warnings -import networkx as nx import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( _dtype_param, @@ -27,7 +27,7 @@ __all__ = ["louvain_communities"] # max_level argument was added to NetworkX 3.3 -if nx.__version__[:3] <= "3.2": +if _nxver <= (3, 2): _max_level_param = { "max_level : int, optional": ( "Upper limit of the number of macro-iterations (max: 500)." @@ -81,7 +81,7 @@ def _louvain_communities( node_ids, clusters, modularity = plc.louvain( resource_handle=plc.ResourceHandle(), graph=G._get_plc_graph(weight, 1, dtype), - max_level=max_level, # TODO: add this parameter to NetworkX + max_level=max_level, threshold=threshold, resolution=resolution, do_expensive_check=False, diff --git a/python/nx-cugraph/nx_cugraph/algorithms/core.py b/python/nx-cugraph/nx_cugraph/algorithms/core.py index 8eb9a9946e7..e69ee88a17c 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/core.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/core.py @@ -15,6 +15,7 @@ import pylibcugraph as plc import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( _get_int_dtype, @@ -58,9 +59,12 @@ def _(G): @networkx_algorithm(is_incomplete=True, version_added="23.12", _plc="k_truss_subgraph") def k_truss(G, k): if is_nx := isinstance(G, nx.Graph): + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + is_compat_graph = False if nxcg.number_of_selfloops(G) > 0: - if nx.__version__[:3] <= "3.2": + if _nxver <= (3, 2): exc_class = nx.NetworkXError else: exc_class = nx.NetworkXNotImplemented @@ -128,6 +132,7 @@ def k_truss(G, k): node_values, node_masks, key_to_id=key_to_id, + use_compat_graph=is_compat_graph, ) new_graph.graph.update(G.graph) return new_graph diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py index e529b83ab1a..cc59fd5eb64 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py @@ -15,6 +15,7 @@ import numpy as np import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import ( _dtype_param, @@ -53,7 +54,7 @@ def hits( if nstart is not None: nstart = G._dict_to_nodearray(nstart, 0, dtype) if max_iter <= 0: - if nx.__version__[:3] <= "3.2": + if _nxver <= (3, 2): raise ValueError("`maxiter` must be a positive integer.") raise nx.PowerIterationFailedConvergence(max_iter) try: diff --git a/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py b/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py index f53b3458949..75dc5fbc706 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py @@ -23,6 +23,7 @@ @networkx_algorithm(version_added="24.02") def complement(G): + is_compat_graph = isinstance(G, nxcg.Graph) G = _to_graph(G) N = G._N # Upcast to int64 so indices don't overflow. @@ -43,6 +44,7 @@ def complement(G): src_indices.astype(index_dtype), dst_indices.astype(index_dtype), key_to_id=G.key_to_id, + use_compat_graph=is_compat_graph, ) @@ -51,10 +53,16 @@ def reverse(G, copy=True): if not G.is_directed(): raise nx.NetworkXError("Cannot reverse an undirected graph.") if isinstance(G, nx.Graph): - if not copy: + is_compat_graph = isinstance(G, nxcg.Graph) + if not copy and not is_compat_graph: raise RuntimeError( "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.reverse`" ) G = nxcg.from_networkx(G, preserve_all_attrs=True) - return G.reverse(copy=copy) + else: + is_compat_graph = False + rv = G.reverse(copy=copy) + if is_compat_graph: + return rv._to_compat_graph() + return rv diff --git a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py index 7d6d77f34a4..ab3c7214303 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py @@ -14,6 +14,7 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _dtype_param, _get_float_dtype, networkx_algorithm @@ -57,7 +58,7 @@ def shortest_path( paths = nxcg.all_pairs_dijkstra_path(G, weight=weight, dtype=dtype) else: # method == 'bellman-ford': paths = nxcg.all_pairs_bellman_ford_path(G, weight=weight, dtype=dtype) - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): paths = dict(paths) # To target elif method == "unweighted": @@ -129,7 +130,7 @@ def shortest_path_length( # To target elif method == "unweighted": lengths = nxcg.single_target_shortest_path_length(G, target) - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): lengths = dict(lengths) elif method == "dijkstra": lengths = nxcg.single_source_dijkstra_path_length( diff --git a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py index 0e98c366e4a..e9c515632ca 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py @@ -17,6 +17,7 @@ import numpy as np import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _groupby, index_dtype, networkx_algorithm @@ -43,7 +44,7 @@ def single_source_shortest_path_length(G, source, cutoff=None): def single_target_shortest_path_length(G, target, cutoff=None): G = _to_graph(G) rv = _bfs(G, target, cutoff, "Target", return_type="length") - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): return iter(rv.items()) return rv @@ -61,7 +62,7 @@ def bidirectional_shortest_path(G, source, target): # TODO PERF: do bidirectional traversal in core G = _to_graph(G) if source not in G or target not in G: - if nx.__version__[:3] <= "3.3": + if _nxver <= (3, 3): raise nx.NodeNotFound( f"Either source {source} or target {target} is not in G" ) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py index 5e4466d7d33..72d0079cf0c 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py @@ -18,6 +18,7 @@ import pylibcugraph as plc import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _groupby, index_dtype, networkx_algorithm @@ -57,7 +58,7 @@ def _bfs(G, source, *, depth_limit=None, reverse=False): return distances[mask], predecessors[mask], node_ids[mask] -if nx.__version__[:3] <= "3.3": +if _nxver <= (3, 3): @networkx_algorithm(is_incomplete=True, version_added="24.02", _plc="bfs") def generic_bfs_edges( @@ -132,13 +133,15 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): raise NotImplementedError( "sort_neighbors argument in bfs_tree is not currently supported" ) + is_compat_graph = isinstance(G, nxcg.Graph) G = _check_G_and_source(G, source) if depth_limit is not None and depth_limit < 1: - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( 1, cp.array([], dtype=index_dtype), cp.array([], dtype=index_dtype), id_to_key=[source], + use_compat_graph=is_compat_graph, ) distances, predecessors, node_ids = _bfs( @@ -148,11 +151,12 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): reverse=reverse, ) if predecessors.size == 0: - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( 1, cp.array([], dtype=index_dtype), cp.array([], dtype=index_dtype), id_to_key=[source], + use_compat_graph=is_compat_graph, ) # TODO: create renumbering helper function(s) unique_node_ids = cp.unique(cp.hstack((predecessors, node_ids))) @@ -170,11 +174,12 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): old_index: new_index for new_index, old_index in enumerate(unique_node_ids.tolist()) } - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( unique_node_ids.size, src_indices, dst_indices, key_to_id=key_to_id, + use_compat_graph=is_compat_graph, ) diff --git a/python/nx-cugraph/nx_cugraph/classes/__init__.py b/python/nx-cugraph/nx_cugraph/classes/__init__.py index 19a5357da55..71168e5364f 100644 --- a/python/nx-cugraph/nx_cugraph/classes/__init__.py +++ b/python/nx-cugraph/nx_cugraph/classes/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,9 +10,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .graph import Graph -from .digraph import DiGraph -from .multigraph import MultiGraph -from .multidigraph import MultiDiGraph +from .graph import CudaGraph, Graph +from .digraph import CudaDiGraph, DiGraph +from .multigraph import CudaMultiGraph, MultiGraph +from .multidigraph import CudaMultiDiGraph, MultiDiGraph from .function import * diff --git a/python/nx-cugraph/nx_cugraph/classes/digraph.py b/python/nx-cugraph/nx_cugraph/classes/digraph.py index e5cfb8f6815..178bf44f16e 100644 --- a/python/nx-cugraph/nx_cugraph/classes/digraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/digraph.py @@ -18,34 +18,108 @@ import cupy as cp import networkx as nx import numpy as np +from networkx.classes.digraph import ( + _CachedPropertyResetterAdjAndSucc, + _CachedPropertyResetterPred, +) import nx_cugraph as nxcg from ..utils import index_dtype -from .graph import Graph +from .graph import CudaGraph, Graph if TYPE_CHECKING: # pragma: no cover from nx_cugraph.typing import AttrKey -__all__ = ["DiGraph"] +__all__ = ["CudaDiGraph", "DiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.DiGraph) -class DiGraph(Graph): - ################# - # Class methods # - ################# +class DiGraph(nx.DiGraph, Graph): + _nx_attrs = ("_node", "_adj", "_succ", "_pred") + + name = Graph.name + _node = Graph._node + + @property + @networkx_api + def _adj(self): + if (adj := self.__dict__["_adj"]) is None: + self._reify_networkx() + adj = self.__dict__["_adj"] + return adj + + @_adj.setter + def _adj(self, val): + self._prepare_setter() + _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + @networkx_api + def _succ(self): + if (succ := self.__dict__["_succ"]) is None: + self._reify_networkx() + succ = self.__dict__["_succ"] + return succ + + @_succ.setter + def _succ(self, val): + self._prepare_setter() + _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + @networkx_api + def _pred(self): + if (pred := self.__dict__["_pred"]) is None: + self._reify_networkx() + pred = self.__dict__["_pred"] + return pred + + @_pred.setter + def _pred(self, val): + self._prepare_setter() + _CachedPropertyResetterPred.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() @classmethod @networkx_api def is_directed(cls) -> bool: return True + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return False + + @classmethod + def to_cudagraph_class(cls) -> type[CudaDiGraph]: + return CudaDiGraph + @classmethod def to_networkx_class(cls) -> type[nx.DiGraph]: return nx.DiGraph + +class CudaDiGraph(CudaGraph): + ################# + # Class methods # + ################# + + is_directed = classmethod(DiGraph.is_directed.__func__) + is_multigraph = classmethod(DiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(DiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(DiGraph.to_networkx_class.__func__) + + @classmethod + def _to_compat_graph_class(cls) -> type[DiGraph]: + return DiGraph + @networkx_api def size(self, weight: AttrKey | None = None) -> int: if weight is not None: @@ -57,7 +131,7 @@ def size(self, weight: AttrKey | None = None) -> int: ########################## @networkx_api - def reverse(self, copy: bool = True) -> DiGraph: + def reverse(self, copy: bool = True) -> CudaDiGraph: return self._copy(not copy, self.__class__, reverse=True) @networkx_api @@ -162,6 +236,7 @@ def to_undirected(self, reciprocal=False, as_view=False): node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=False, ) if as_view: rv.graph = self.graph diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 7c01365c0ac..cfe1e1c87e9 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -20,8 +20,13 @@ import networkx as nx import numpy as np import pylibcugraph as plc +from networkx.classes.graph import ( + _CachedPropertyResetterAdj, + _CachedPropertyResetterNode, +) import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype @@ -40,57 +45,246 @@ any_ndarray, ) -__all__ = ["Graph"] +__all__ = ["CudaGraph", "Graph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.Graph) +# The "everything" cache key is an internal implementation detail of NetworkX +# that may change between releases. +if _nxver < (3, 4): + _CACHE_KEY = ( + True, # Include all edge values + True, # Include all node values + True, # Include `.graph` attributes + ) +else: + _CACHE_KEY = ( + True, # Include all edge values + True, # Include all node values + # `.graph` attributes are always included now + ) + +# Use to indicate when a full conversion to GPU failed so we don't try again. +_CANT_CONVERT_TO_GPU = "_CANT_CONVERT_TO_GPU" + + +# `collections.UserDict` was the preferred way to subclass dict, but now +# subclassing dict directly is much better supported and should work here. +# This class should only be necessary if the user clears the cache manually. +class _GraphCache(dict): + """Cache that ensures Graph will reify into a NetworkX graph when cleared.""" + + _graph: Graph -class Graph: + def __init__(self, graph: Graph): + self._graph = graph + + def clear(self) -> None: + self._graph._reify_networkx() + super().clear() + + +class Graph(nx.Graph): # Tell networkx to dispatch calls with this object to nx-cugraph __networkx_backend__: ClassVar[str] = "cugraph" # nx >=3.2 __networkx_plugin__: ClassVar[str] = "cugraph" # nx <3.2 + # Core attributes of NetowkrX graphs that will be copied and cleared as appropriate. + # These attributes comprise the edge and node data model for NetworkX graphs. + _nx_attrs = ("_node", "_adj") + # Allow networkx dispatch machinery to cache conversions. # This means we should clear the cache if we ever mutate the object! - __networkx_cache__: dict | None + __networkx_cache__: _GraphCache | None # networkx properties graph: dict - graph_attr_dict_factory: ClassVar[type] = dict + # Should we declare type annotations for the rest? + + # Properties that trigger copying to the CPU + def _prepare_setter(self): + """Be careful when setting private attributes which may be used during init.""" + if ( + # If not present, then this must be in init + any(attr not in self.__dict__ for attr in self._nx_attrs) + # Already on the CPU + or not any(self.__dict__[attr] is None for attr in self._nx_attrs) + ): + return + if self._is_on_gpu: + # Copy from GPU to CPU + self._reify_networkx() + return + # Default values + for attr in self._nx_attrs: + if self.__dict__[attr] is None: + if attr == "_succ": + self.__dict__[attr] = self.__dict__["_adj"] + else: + self.__dict__[attr] = {} - # Not networkx properties - # We store edge data in COO format with {src,dst}_indices and edge_values. - src_indices: cp.ndarray[IndexValue] - dst_indices: cp.ndarray[IndexValue] - edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] - edge_masks: dict[AttrKey, cp.ndarray[bool]] - node_values: dict[AttrKey, any_ndarray[NodeValue]] - node_masks: dict[AttrKey, any_ndarray[bool]] - key_to_id: dict[NodeKey, IndexValue] | None - _id_to_key: list[NodeKey] | None - _N: int - _node_ids: cp.ndarray[IndexValue] | None # holds plc.SGGraph.vertices_array data + @property + @networkx_api + def _node(self): + if (node := self.__dict__["_node"]) is None: + self._reify_networkx() + node = self.__dict__["_node"] + return node + + @_node.setter + def _node(self, val): + self._prepare_setter() + _CachedPropertyResetterNode.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() - # Used by graph._get_plc_graph - _plc_type_map: ClassVar[dict[np.dtype, np.dtype]] = { - # signed int - np.dtype(np.int8): np.dtype(np.float32), - np.dtype(np.int16): np.dtype(np.float32), - np.dtype(np.int32): np.dtype(np.float64), - np.dtype(np.int64): np.dtype(np.float64), # raise if abs(x) > 2**53 - # unsigned int - np.dtype(np.uint8): np.dtype(np.float32), - np.dtype(np.uint16): np.dtype(np.float32), - np.dtype(np.uint32): np.dtype(np.float64), - np.dtype(np.uint64): np.dtype(np.float64), # raise if x > 2**53 - # other - np.dtype(np.bool_): np.dtype(np.float32), - np.dtype(np.float16): np.dtype(np.float32), - } - _plc_allowed_edge_types: ClassVar[set[np.dtype]] = { - np.dtype(np.float32), - np.dtype(np.float64), - } + @property + @networkx_api + def _adj(self): + if (adj := self.__dict__["_adj"]) is None: + self._reify_networkx() + adj = self.__dict__["_adj"] + return adj + + @_adj.setter + def _adj(self, val): + self._prepare_setter() + _CachedPropertyResetterAdj.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + def _is_on_gpu(self) -> bool: + """Whether the full graph is on device (in the cache). + + This returns False when only a subset of the graph (such as only + edge indices and edge attribute) is on device. + + The graph may be on host (CPU) and device (GPU) at the same time. + """ + cache = getattr(self, "__networkx_cache__", None) + if not cache: + return False + return _CACHE_KEY in cache.get("backends", {}).get("cugraph", {}) + + @property + def _is_on_cpu(self) -> bool: + """Whether the graph is on host as a NetworkX graph. + + This means the core data structures that comprise a NetworkX graph + (such as ``G._node`` and ``G._adj``) are present. + + The graph may be on host (CPU) and device (GPU) at the same time. + """ + return self.__dict__["_node"] is not None + + @property + def _cudagraph(self): + """Return the full ``CudaGraph`` on device, computing if necessary, or None.""" + nx_cache = getattr(self, "__networkx_cache__", None) + if nx_cache is None: + nx_cache = {} + elif _CANT_CONVERT_TO_GPU in nx_cache: + return None + cache = nx_cache.setdefault("backends", {}).setdefault("cugraph", {}) + if (Gcg := cache.get(_CACHE_KEY)) is not None: + if isinstance(Gcg, Graph): + # This shouldn't happen during normal use, but be extra-careful anyway + return Gcg._cudagraph + return Gcg + if self.__dict__["_node"] is None: + raise RuntimeError( + f"{type(self).__name__} cannot be converted to the GPU, because it is " + "not on the CPU! This is not supposed to be possible. If you believe " + "you have found a bug, please report a minimum reproducible example to " + "https://github.com/rapidsai/cugraph/issues/new/choose" + ) + try: + Gcg = nxcg.from_networkx( + self, preserve_edge_attrs=True, preserve_node_attrs=True + ) + except Exception: + # Should we warn that the full graph can't be on GPU? + nx_cache[_CANT_CONVERT_TO_GPU] = True + return None + Gcg.graph = self.graph + cache[_CACHE_KEY] = Gcg + return Gcg + + @_cudagraph.setter + def _cudagraph(self, val, *, clear_cpu=True): + """Set the full ``CudaGraph`` for this graph, or remove from device if None.""" + if (cache := getattr(self, "__networkx_cache__", None)) is None: + # Should we warn? + return + # TODO: pay close attention to when we should clear the cache, since + # this may or may not be a mutation. + cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + if val is None: + cache.pop(_CACHE_KEY, None) + else: + self.graph = val.graph + cache[_CACHE_KEY] = val + if clear_cpu: + for key in self._nx_attrs: + self.__dict__[key] = None + + @nx.Graph.name.setter + def name(self, s): + # Don't clear the cache when setting the name, since `.graph` is shared. + # There is a very small risk here for the cache to become (slightly) + # insconsistent if graphs from other backends are cached. + self.graph["name"] = s + + @classmethod + @networkx_api + def is_directed(cls) -> bool: + return False + + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return False + + @classmethod + def to_cudagraph_class(cls) -> type[CudaGraph]: + return CudaGraph + + @classmethod + @networkx_api + def to_directed_class(cls) -> type[nxcg.DiGraph]: + return nxcg.DiGraph + + @classmethod + def to_networkx_class(cls) -> type[nx.Graph]: + return nx.Graph + + @classmethod + @networkx_api + def to_undirected_class(cls) -> type[Graph]: + return Graph + + def __init__(self, incoming_graph_data=None, **attr): + super().__init__(incoming_graph_data, **attr) + self.__networkx_cache__ = _GraphCache(self) + + def _reify_networkx(self) -> None: + """Copy graph to host (CPU) if necessary.""" + if self.__dict__["_node"] is None: + # After we make this into an nx graph, we rely on the cache being correct + Gcg = self._cudagraph + G = nxcg.to_networkx(Gcg) + for key in self._nx_attrs: + self.__dict__[key] = G.__dict__[key] + + def _become(self, other: Graph): + if self.__class__ is not other.__class__: + raise TypeError( + "Attempting to update graph inplace with graph of different type!" + ) + # Begin with the simplest implementation; do we need to do more? + self.__dict__.update(other.__dict__) + return self #################### # Creation methods # @@ -109,9 +303,10 @@ def from_coo( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: - new_graph = object.__new__(cls) + ) -> Graph | CudaGraph: + new_graph = object.__new__(cls.to_cudagraph_class()) new_graph.__networkx_cache__ = {} new_graph.src_indices = src_indices new_graph.dst_indices = dst_indices @@ -173,7 +368,8 @@ def from_coo( isolates = nxcg.algorithms.isolate._isolates(new_graph) if len(isolates) > 0: new_graph._node_ids = cp.arange(new_graph._N, dtype=index_dtype) - + if use_compat_graph or use_compat_graph is None and issubclass(cls, Graph): + new_graph = new_graph._to_compat_graph() return new_graph @classmethod @@ -188,8 +384,9 @@ def from_csr( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: N = indptr.size - 1 src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -205,6 +402,7 @@ def from_csr( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=use_compat_graph, **attr, ) @@ -220,8 +418,9 @@ def from_csc( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: N = indptr.size - 1 dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -237,6 +436,7 @@ def from_csc( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=use_compat_graph, **attr, ) @@ -254,8 +454,9 @@ def from_dcsr( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_srcs.get(), cp.diff(indptr).get()) @@ -270,6 +471,7 @@ def from_dcsr( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=use_compat_graph, **attr, ) @@ -287,8 +489,9 @@ def from_dcsc( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_dsts.get(), cp.diff(indptr).get()) @@ -303,13 +506,75 @@ def from_dcsc( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=use_compat_graph, **attr, ) - def __new__(cls, incoming_graph_data=None, **attr) -> Graph: + +class CudaGraph: + # Tell networkx to dispatch calls with this object to nx-cugraph + __networkx_backend__: ClassVar[str] = "cugraph" # nx >=3.2 + __networkx_plugin__: ClassVar[str] = "cugraph" # nx <3.2 + + # Allow networkx dispatch machinery to cache conversions. + # This means we should clear the cache if we ever mutate the object! + __networkx_cache__: dict | None + + # networkx properties + graph: dict + graph_attr_dict_factory: ClassVar[type] = dict + + # Not networkx properties + # We store edge data in COO format with {src,dst}_indices and edge_values. + src_indices: cp.ndarray[IndexValue] + dst_indices: cp.ndarray[IndexValue] + edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] + edge_masks: dict[AttrKey, cp.ndarray[bool]] + node_values: dict[AttrKey, any_ndarray[NodeValue]] + node_masks: dict[AttrKey, any_ndarray[bool]] + key_to_id: dict[NodeKey, IndexValue] | None + _id_to_key: list[NodeKey] | None + _N: int + _node_ids: cp.ndarray[IndexValue] | None # holds plc.SGGraph.vertices_array data + + # Used by graph._get_plc_graph + _plc_type_map: ClassVar[dict[np.dtype, np.dtype]] = { + # signed int + np.dtype(np.int8): np.dtype(np.float32), + np.dtype(np.int16): np.dtype(np.float32), + np.dtype(np.int32): np.dtype(np.float64), + np.dtype(np.int64): np.dtype(np.float64), # raise if abs(x) > 2**53 + # unsigned int + np.dtype(np.uint8): np.dtype(np.float32), + np.dtype(np.uint16): np.dtype(np.float32), + np.dtype(np.uint32): np.dtype(np.float64), + np.dtype(np.uint64): np.dtype(np.float64), # raise if x > 2**53 + # other + np.dtype(np.bool_): np.dtype(np.float32), + np.dtype(np.float16): np.dtype(np.float32), + } + _plc_allowed_edge_types: ClassVar[set[np.dtype]] = { + np.dtype(np.float32), + np.dtype(np.float64), + } + + #################### + # Creation methods # + #################### + + from_coo = classmethod(Graph.from_coo.__func__) + from_csr = classmethod(Graph.from_csr.__func__) + from_csc = classmethod(Graph.from_csc.__func__) + from_dcsr = classmethod(Graph.from_dcsr.__func__) + from_dcsc = classmethod(Graph.from_dcsc.__func__) + + def __new__(cls, incoming_graph_data=None, **attr) -> CudaGraph: if incoming_graph_data is None: new_graph = cls.from_coo( - 0, cp.empty(0, index_dtype), cp.empty(0, index_dtype) + 0, + cp.empty(0, index_dtype), + cp.empty(0, index_dtype), + use_compat_graph=False, ) elif incoming_graph_data.__class__ is cls: new_graph = incoming_graph_data.copy() @@ -318,34 +583,30 @@ def __new__(cls, incoming_graph_data=None, **attr) -> Graph: else: raise NotImplementedError new_graph.graph.update(attr) + # We could return Graph here (if configured), but let's not for now return new_graph ################# # Class methods # ################# - @classmethod - @networkx_api - def is_directed(cls) -> bool: - return False + is_directed = classmethod(Graph.is_directed.__func__) + is_multigraph = classmethod(Graph.is_multigraph.__func__) + to_cudagraph_class = classmethod(Graph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(Graph.to_networkx_class.__func__) @classmethod @networkx_api - def is_multigraph(cls) -> bool: - return False + def to_directed_class(cls) -> type[nxcg.CudaDiGraph]: + return nxcg.CudaDiGraph @classmethod @networkx_api - def to_directed_class(cls) -> type[nxcg.DiGraph]: - return nxcg.DiGraph - - @classmethod - def to_networkx_class(cls) -> type[nx.Graph]: - return nx.Graph + def to_undirected_class(cls) -> type[CudaGraph]: + return CudaGraph @classmethod - @networkx_api - def to_undirected_class(cls) -> type[Graph]: + def _to_compat_graph_class(cls) -> type[Graph]: return Graph ############## @@ -438,7 +699,7 @@ def clear_edges(self) -> None: cache.clear() @networkx_api - def copy(self, as_view: bool = False) -> Graph: + def copy(self, as_view: bool = False) -> CudaGraph: # Does shallow copy in networkx return self._copy(as_view, self.__class__) @@ -534,14 +795,19 @@ def size(self, weight: AttrKey | None = None) -> int: return int(cp.count_nonzero(self.src_indices <= self.dst_indices)) @networkx_api - def to_directed(self, as_view: bool = False) -> nxcg.DiGraph: + def to_directed(self, as_view: bool = False) -> nxcg.CudaDiGraph: return self._copy(as_view, self.to_directed_class()) @networkx_api - def to_undirected(self, as_view: bool = False) -> Graph: + def to_undirected(self, as_view: bool = False) -> CudaGraph: # Does deep copy in networkx return self._copy(as_view, self.to_undirected_class()) + def _to_compat_graph(self) -> Graph: + rv = self._to_compat_graph_class()() + rv._cudagraph = self + return rv + # Not implemented... # adj, adjacency, add_edge, add_edges_from, add_node, # add_nodes_from, add_weighted_edges_from, degree, @@ -552,8 +818,8 @@ def to_undirected(self, as_view: bool = False) -> Graph: # Private methods # ################### - def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): - # DRY warning: see also MultiGraph._copy + def _copy(self, as_view: bool, cls: type[CudaGraph], reverse: bool = False): + # DRY warning: see also CudaMultiGraph._copy src_indices = self.src_indices dst_indices = self.dst_indices edge_values = self.edge_values @@ -593,6 +859,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=False, ) if as_view: rv.graph = self.graph @@ -714,7 +981,7 @@ def _get_plc_graph( ) def _sort_edge_indices(self, primary="src"): - # DRY warning: see also MultiGraph._sort_edge_indices + # DRY warning: see also CudaMultiGraph._sort_edge_indices if primary == "src": stacked = cp.vstack((self.dst_indices, self.src_indices)) elif primary == "dst": @@ -736,7 +1003,7 @@ def _sort_edge_indices(self, primary="src"): {key: val[indices] for key, val in self.edge_masks.items()} ) - def _become(self, other: Graph): + def _become(self, other: CudaGraph): if self.__class__ is not other.__class__: raise TypeError( "Attempting to update graph inplace with graph of different type!" diff --git a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py index 2e7a55a9eb1..5a6595567d2 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py @@ -16,24 +16,51 @@ import nx_cugraph as nxcg -from .digraph import DiGraph -from .multigraph import MultiGraph +from .digraph import CudaDiGraph, DiGraph +from .graph import Graph +from .multigraph import CudaMultiGraph, MultiGraph -__all__ = ["MultiDiGraph"] +__all__ = ["CudaMultiDiGraph", "MultiDiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.MultiDiGraph) -class MultiDiGraph(MultiGraph, DiGraph): +class MultiDiGraph(nx.MultiDiGraph, MultiGraph, DiGraph): + name = Graph.name + _node = Graph._node + _adj = DiGraph._adj + _succ = DiGraph._succ + _pred = DiGraph._pred + @classmethod @networkx_api def is_directed(cls) -> bool: return True + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return True + + @classmethod + def to_cudagraph_class(cls) -> type[CudaMultiDiGraph]: + return CudaMultiDiGraph + @classmethod def to_networkx_class(cls) -> type[nx.MultiDiGraph]: return nx.MultiDiGraph + +class CudaMultiDiGraph(CudaMultiGraph, CudaDiGraph): + is_directed = classmethod(MultiDiGraph.is_directed.__func__) + is_multigraph = classmethod(MultiDiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(MultiDiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(MultiDiGraph.to_networkx_class.__func__) + + @classmethod + def _to_compat_graph_class(cls) -> type[MultiDiGraph]: + return MultiDiGraph + ########################## # NetworkX graph methods # ########################## diff --git a/python/nx-cugraph/nx_cugraph/classes/multigraph.py b/python/nx-cugraph/nx_cugraph/classes/multigraph.py index 23d9faa8734..c8c8f1dfb00 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multigraph.py @@ -22,7 +22,7 @@ import nx_cugraph as nxcg from ..utils import index_dtype -from .graph import Graph +from .graph import CudaGraph, Graph, _GraphCache if TYPE_CHECKING: from nx_cugraph.typing import ( @@ -34,32 +34,47 @@ NodeValue, any_ndarray, ) -__all__ = ["MultiGraph"] +__all__ = ["MultiGraph", "CudaMultiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.MultiGraph) -class MultiGraph(Graph): - # networkx properties - edge_key_dict_factory: ClassVar[type] = dict +class MultiGraph(nx.MultiGraph, Graph): + name = Graph.name + _node = Graph._node + _adj = Graph._adj - # Not networkx properties + @classmethod + @networkx_api + def is_directed(cls) -> bool: + return False - # In a MultiGraph, each edge has a unique `(src, dst, key)` key. - # By default, `key` is 0 if possible, else 1, else 2, etc. - # This key can be any hashable Python object in NetworkX. - # We don't use a dict for our data structure here, because - # that would require a `(src, dst, key)` key. - # Instead, we keep `edge_keys` and/or `edge_indices`. - # `edge_keys` is the list of Python objects for each edge. - # `edge_indices` is for the common case of default multiedge keys, - # in which case we can store it as a cupy array. - # `edge_indices` is generally preferred. It is possible to provide - # both where edge_indices is the default and edge_keys is anything. - # It is also possible for them both to be None, which means the - # default edge indices has not yet been calculated. - edge_indices: cp.ndarray[IndexValue] | None - edge_keys: list[EdgeKey] | None + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return True + + @classmethod + def to_cudagraph_class(cls) -> type[CudaMultiGraph]: + return CudaMultiGraph + + @classmethod + @networkx_api + def to_directed_class(cls) -> type[nxcg.MultiDiGraph]: + return nxcg.MultiDiGraph + + @classmethod + def to_networkx_class(cls) -> type[nx.MultiGraph]: + return nx.MultiGraph + + @classmethod + @networkx_api + def to_undirected_class(cls) -> type[MultiGraph]: + return MultiGraph + + def __init__(self, incoming_graph_data=None, multigraph_input=None, **attr): + super().__init__(incoming_graph_data, multigraph_input, **attr) + self.__networkx_cache__ = _GraphCache(self) #################### # Creation methods # @@ -80,9 +95,10 @@ def from_coo( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: - new_graph = super().from_coo( + ) -> MultiGraph | CudaMultiGraph: + new_graph = super(cls.to_undirected_class(), cls).from_coo( N, src_indices, dst_indices, @@ -92,6 +108,7 @@ def from_coo( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + use_compat_graph=False, **attr, ) new_graph.edge_indices = edge_indices @@ -102,6 +119,8 @@ def from_coo( and len(new_graph.edge_keys) != src_indices.size ): raise ValueError + if use_compat_graph or use_compat_graph is None and issubclass(cls, Graph): + new_graph = new_graph._to_compat_graph() return new_graph @classmethod @@ -118,8 +137,9 @@ def from_csr( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: N = indptr.size - 1 src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -137,6 +157,7 @@ def from_csr( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=use_compat_graph, **attr, ) @@ -154,8 +175,9 @@ def from_csc( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: N = indptr.size - 1 dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -173,6 +195,7 @@ def from_csc( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=use_compat_graph, **attr, ) @@ -192,8 +215,9 @@ def from_dcsr( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_srcs.get(), cp.diff(indptr).get()) @@ -210,6 +234,7 @@ def from_dcsr( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=use_compat_graph, **attr, ) @@ -229,8 +254,9 @@ def from_dcsc( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> MultiGraph | CudaGraph: dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_dsts.get(), cp.diff(indptr).get()) @@ -247,12 +273,46 @@ def from_dcsc( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=use_compat_graph, **attr, ) + +class CudaMultiGraph(CudaGraph): + # networkx properties + edge_key_dict_factory: ClassVar[type] = dict + + # Not networkx properties + + # In a MultiGraph, each edge has a unique `(src, dst, key)` key. + # By default, `key` is 0 if possible, else 1, else 2, etc. + # This key can be any hashable Python object in NetworkX. + # We don't use a dict for our data structure here, because + # that would require a `(src, dst, key)` key. + # Instead, we keep `edge_keys` and/or `edge_indices`. + # `edge_keys` is the list of Python objects for each edge. + # `edge_indices` is for the common case of default multiedge keys, + # in which case we can store it as a cupy array. + # `edge_indices` is generally preferred. It is possible to provide + # both where edge_indices is the default and edge_keys is anything. + # It is also possible for them both to be None, which means the + # default edge indices has not yet been calculated. + edge_indices: cp.ndarray[IndexValue] | None + edge_keys: list[EdgeKey] | None + + #################### + # Creation methods # + #################### + + from_coo = classmethod(MultiGraph.from_coo.__func__) + from_csr = classmethod(MultiGraph.from_csr.__func__) + from_csc = classmethod(MultiGraph.from_csc.__func__) + from_dcsr = classmethod(MultiGraph.from_dcsr.__func__) + from_dcsc = classmethod(MultiGraph.from_dcsc.__func__) + def __new__( cls, incoming_graph_data=None, multigraph_input=None, **attr - ) -> MultiGraph: + ) -> CudaMultiGraph: if isinstance(incoming_graph_data, dict) and multigraph_input is not False: new_graph = nxcg.from_networkx( nx.MultiGraph(incoming_graph_data, multigraph_input=multigraph_input), @@ -267,28 +327,23 @@ def __new__( # Class methods # ################# - @classmethod - @networkx_api - def is_directed(cls) -> bool: - return False + is_directed = classmethod(MultiGraph.is_directed.__func__) + is_multigraph = classmethod(MultiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(MultiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(MultiGraph.to_networkx_class.__func__) @classmethod @networkx_api - def is_multigraph(cls) -> bool: - return True + def to_directed_class(cls) -> type[nxcg.CudaMultiDiGraph]: + return nxcg.CudaMultiDiGraph @classmethod @networkx_api - def to_directed_class(cls) -> type[nxcg.MultiDiGraph]: - return nxcg.MultiDiGraph - - @classmethod - def to_networkx_class(cls) -> type[nx.MultiGraph]: - return nx.MultiGraph + def to_undirected_class(cls) -> type[CudaMultiGraph]: + return CudaMultiGraph @classmethod - @networkx_api - def to_undirected_class(cls) -> type[MultiGraph]: + def _to_compat_graph_class(cls) -> type[MultiGraph]: return MultiGraph ########################## @@ -308,7 +363,7 @@ def clear_edges(self) -> None: self.edge_keys = None @networkx_api - def copy(self, as_view: bool = False) -> MultiGraph: + def copy(self, as_view: bool = False) -> CudaMultiGraph: # Does shallow copy in networkx return self._copy(as_view, self.__class__) @@ -391,11 +446,11 @@ def has_edge(self, u: NodeKey, v: NodeKey, key: EdgeKey | None = None) -> bool: return any(edge_keys[i] == key for i in indices.tolist()) @networkx_api - def to_directed(self, as_view: bool = False) -> nxcg.MultiDiGraph: + def to_directed(self, as_view: bool = False) -> nxcg.CudaMultiDiGraph: return self._copy(as_view, self.to_directed_class()) @networkx_api - def to_undirected(self, as_view: bool = False) -> MultiGraph: + def to_undirected(self, as_view: bool = False) -> CudaMultiGraph: # Does deep copy in networkx return self._copy(as_view, self.to_undirected_class()) @@ -403,8 +458,8 @@ def to_undirected(self, as_view: bool = False) -> MultiGraph: # Private methods # ################### - def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): - # DRY warning: see also Graph._copy + def _copy(self, as_view: bool, cls: type[CudaGraph], reverse: bool = False): + # DRY warning: see also CudaGraph._copy src_indices = self.src_indices dst_indices = self.dst_indices edge_indices = self.edge_indices @@ -451,6 +506,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + use_compat_graph=False, ) if as_view: rv.graph = self.graph @@ -460,7 +516,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): return rv def _sort_edge_indices(self, primary="src"): - # DRY warning: see also Graph._sort_edge_indices + # DRY warning: see also CudaGraph._sort_edge_indices if self.edge_indices is None and self.edge_keys is None: return super()._sort_edge_indices(primary=primary) if primary == "src": diff --git a/python/nx-cugraph/nx_cugraph/convert.py b/python/nx-cugraph/nx_cugraph/convert.py index 56d16d837d7..a872f13ac70 100644 --- a/python/nx-cugraph/nx_cugraph/convert.py +++ b/python/nx-cugraph/nx_cugraph/convert.py @@ -12,6 +12,7 @@ # limitations under the License. from __future__ import annotations +import functools import itertools import operator as op from collections import Counter, defaultdict @@ -23,9 +24,13 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from .utils import index_dtype, networkx_algorithm -from .utils.misc import pairwise +from .utils.misc import _And_NotImplementedError, pairwise + +if _nxver >= (3, 4): + from networkx.utils.backends import _get_cache_key, _get_from_cache, _set_to_cache if TYPE_CHECKING: # pragma: no cover from nx_cugraph.typing import AttrKey, Dtype, EdgeValue, NodeValue, any_ndarray @@ -60,6 +65,27 @@ def _iterate_values(graph, adj, is_dicts, func): return func(it), False +# Consider adding this to `utils` if it is useful elsewhere +def _fallback_decorator(func): + """Catch and convert exceptions to ``NotImplementedError``; use as a decorator. + + ``nx.NetworkXError`` are raised without being converted. This allows + falling back to other backends if, for example, conversion to GPU failed. + """ + + @functools.wraps(func) + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except nx.NetworkXError: + raise + except Exception as exc: + raise _And_NotImplementedError(exc) from exc + + return inner + + +@_fallback_decorator def from_networkx( graph: nx.Graph, edge_attrs: AttrKey | dict[AttrKey, EdgeValue | None] | None = None, @@ -74,7 +100,8 @@ def from_networkx( as_directed: bool = False, name: str | None = None, graph_name: str | None = None, -) -> nxcg.Graph: + use_compat_graph: bool | None = False, +) -> nxcg.Graph | nxcg.CudaGraph: """Convert a networkx graph to nx_cugraph graph; can convert all attributes. Parameters @@ -114,10 +141,16 @@ def from_networkx( The name of the algorithm when dispatched from networkx. graph_name : str, optional The name of the graph argument geing converted when dispatched from networkx. + use_compat_graph : bool or None, default False + Indicate whether to return a graph that is compatible with NetworkX graph. + For example, ``nx_cugraph.Graph`` can be used as a NetworkX graph and can + reside in host (CPU) or device (GPU) memory. The default is False, which + will return e.g. ``nx_cugraph.CudaGraph`` that only resides on device (GPU) + and is not fully compatible as a NetworkX graph. Returns ------- - nx_cugraph.Graph + nx_cugraph.Graph or nx_cugraph.CudaGraph Notes ----- @@ -145,6 +178,41 @@ def from_networkx( graph = G else: raise TypeError(f"Expected networkx.Graph; got {type(graph)}") + elif isinstance(graph, nxcg.Graph): + if ( + use_compat_graph + # Use compat graphs by default + or use_compat_graph is None + and (_nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs) + ): + return graph + if graph._is_on_gpu: + return graph._cudagraph + if not graph._is_on_cpu: + raise RuntimeError( + f"{type(graph).__name__} cannot be converted to the GPU, because it is " + "not on the CPU! This is not supposed to be possible. If you believe " + "you have found a bug, please report a minimum reproducible example to " + "https://github.com/rapidsai/cugraph/issues/new/choose" + ) + if _nxver >= (3, 4): + cache_key = _get_cache_key( + edge_attrs=edge_attrs, + node_attrs=node_attrs, + preserve_edge_attrs=preserve_edge_attrs, + preserve_node_attrs=preserve_node_attrs, + preserve_graph_attrs=preserve_graph_attrs, + ) + cache = getattr(graph, "__networkx_cache__", None) + if cache is not None: + cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + compat_key, rv = _get_from_cache(cache, cache_key) + if rv is not None: + if isinstance(rv, nxcg.Graph): + # This shouldn't happen during normal use, but be extra-careful + rv = rv._cudagraph + if rv is not None: + return rv if preserve_all_attrs: preserve_edge_attrs = True @@ -165,7 +233,12 @@ def from_networkx( else: node_attrs = {node_attrs: None} - if graph.__class__ in {nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph}: + if graph.__class__ in { + nx.Graph, + nx.DiGraph, + nx.MultiGraph, + nx.MultiDiGraph, + } or isinstance(graph, nxcg.Graph): # This is a NetworkX private attribute, but is much faster to use adj = graph._adj else: @@ -455,9 +528,9 @@ def func(it, edge_attr=edge_attr, dtype=dtype): # if vals.ndim > 1: ... if graph.is_multigraph(): if graph.is_directed() or as_directed: - klass = nxcg.MultiDiGraph + klass = nxcg.CudaMultiDiGraph else: - klass = nxcg.MultiGraph + klass = nxcg.CudaMultiGraph rv = klass.from_coo( N, src_indices, @@ -469,12 +542,13 @@ def func(it, edge_attr=edge_attr, dtype=dtype): node_masks, key_to_id=key_to_id, edge_keys=edge_keys, + use_compat_graph=False, ) else: if graph.is_directed() or as_directed: - klass = nxcg.DiGraph + klass = nxcg.CudaDiGraph else: - klass = nxcg.Graph + klass = nxcg.CudaGraph rv = klass.from_coo( N, src_indices, @@ -484,9 +558,22 @@ def func(it, edge_attr=edge_attr, dtype=dtype): node_values, node_masks, key_to_id=key_to_id, + use_compat_graph=False, ) if preserve_graph_attrs: rv.graph.update(graph.graph) # deepcopy? + if _nxver >= (3, 4) and isinstance(graph, nxcg.Graph) and cache is not None: + # Make sure this conversion is added to the cache, and make all of + # our graphs share the same `.graph` attribute for consistency. + rv.graph = graph.graph + _set_to_cache(cache, cache_key, rv) + if ( + use_compat_graph + # Use compat graphs by default + or use_compat_graph is None + and (_nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs) + ): + return rv._to_compat_graph() return rv @@ -535,14 +622,16 @@ def _iter_attr_dicts( return full_dicts -def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: +def to_networkx( + G: nxcg.Graph | nxcg.CudaGraph, *, sort_edges: bool = False +) -> nx.Graph: """Convert a nx_cugraph graph to networkx graph. All edge and node attributes and ``G.graph`` properties are converted. Parameters ---------- - G : nx_cugraph.Graph + G : nx_cugraph.Graph or nx_cugraph.CudaGraph sort_edges : bool, default False Whether to sort the edge data of the input graph by (src, dst) indices before converting. This can be useful to convert to networkx graphs @@ -557,6 +646,9 @@ def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: -------- from_networkx : The opposite; convert networkx graph to nx_cugraph graph """ + if isinstance(G, nxcg.Graph): + # These graphs are already NetworkX graphs :) + return G rv = G.to_networkx_class()() id_to_key = G.id_to_key if sort_edges: @@ -623,13 +715,13 @@ def _to_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.Graph | nxcg.DiGraph: +) -> nxcg.CudaGraph | nxcg.CudaDiGraph: """Ensure that input type is a nx_cugraph graph, and convert if necessary. Directed and undirected graphs are both allowed. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): return G if isinstance(G, nx.Graph): return from_networkx( @@ -644,15 +736,15 @@ def _to_directed_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.DiGraph: - """Ensure that input type is a nx_cugraph DiGraph, and convert if necessary. +) -> nxcg.CudaDiGraph: + """Ensure that input type is a nx_cugraph CudaDiGraph, and convert if necessary. Undirected graphs will be converted to directed. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.DiGraph): + if isinstance(G, nxcg.CudaDiGraph): return G - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): return G.to_directed() if isinstance(G, nx.Graph): return from_networkx( @@ -670,13 +762,13 @@ def _to_undirected_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.Graph: - """Ensure that input type is a nx_cugraph Graph, and convert if necessary. +) -> nxcg.CudaGraph: + """Ensure that input type is a nx_cugraph CudaGraph, and convert if necessary. Only undirected graphs are allowed. Directed graphs will raise ValueError. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): if G.is_directed(): raise ValueError("Only undirected graphs supported; got a directed graph") return G @@ -688,7 +780,7 @@ def _to_undirected_graph( raise TypeError -@networkx_algorithm(version_added="24.08") +@networkx_algorithm(version_added="24.08", fallback=True) def from_dict_of_lists(d, create_using=None): from .generators._utils import _create_using_class diff --git a/python/nx-cugraph/nx_cugraph/convert_matrix.py b/python/nx-cugraph/nx_cugraph/convert_matrix.py index 38139b913cf..54975902861 100644 --- a/python/nx-cugraph/nx_cugraph/convert_matrix.py +++ b/python/nx-cugraph/nx_cugraph/convert_matrix.py @@ -14,6 +14,8 @@ import networkx as nx import numpy as np +from nx_cugraph import _nxver + from .generators._utils import _create_using_class from .utils import _cp_iscopied_asarray, index_dtype, networkx_algorithm @@ -24,7 +26,7 @@ # Value columns with string dtype is not supported -@networkx_algorithm(is_incomplete=True, version_added="23.12") +@networkx_algorithm(is_incomplete=True, version_added="23.12", fallback=True) def from_pandas_edgelist( df, source="source", @@ -138,7 +140,7 @@ def from_pandas_edgelist( and ( # In nx <= 3.3, `edge_key` was ignored if `edge_attr` is None edge_attr is not None - or nx.__version__[:3] > "3.3" + or _nxver > (3, 3) ) ): try: @@ -161,7 +163,7 @@ def from_pandas_edgelist( return G -@networkx_algorithm(version_added="23.12") +@networkx_algorithm(version_added="23.12", fallback=True) def from_scipy_sparse_array( A, parallel_edges=False, create_using=None, edge_attribute="weight" ): diff --git a/python/nx-cugraph/nx_cugraph/generators/_utils.py b/python/nx-cugraph/nx_cugraph/generators/_utils.py index e38ace5b28d..bc9ab84bdad 100644 --- a/python/nx-cugraph/nx_cugraph/generators/_utils.py +++ b/python/nx-cugraph/nx_cugraph/generators/_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -16,6 +16,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype @@ -74,7 +75,7 @@ def _common_small_graph(n, nodes, create_using, *, allow_directed=True): return G -def _create_using_class(create_using, *, default=nxcg.Graph): +def _create_using_class(create_using, *, default=nx.Graph): """Handle ``create_using`` argument and return a Graph type from nx_cugraph.""" inplace = False if create_using is None: @@ -85,16 +86,17 @@ def _create_using_class(create_using, *, default=nxcg.Graph): create_using, "is_multigraph" ): raise TypeError("create_using is not a valid graph type or instance") - elif not isinstance(create_using, nxcg.Graph): + elif not isinstance(create_using, (nxcg.Graph, nxcg.CudaGraph)): raise NotImplementedError( f"create_using with object of type {type(create_using)} is not supported " - "by the cugraph backend; only nx_cugraph.Graph objects are allowed." + "by the cugraph backend; only nx_cugraph.Graph or nx_cugraph.CudaGraph " + "objects are allowed." ) else: inplace = True G = create_using G.clear() - if not isinstance(G, nxcg.Graph): + if not isinstance(G, (nxcg.Graph, nxcg.CudaGraph)): if G.is_multigraph(): if G.is_directed(): graph_class = nxcg.MultiDiGraph @@ -104,10 +106,12 @@ def _create_using_class(create_using, *, default=nxcg.Graph): graph_class = nxcg.DiGraph else: graph_class = nxcg.Graph + if _nxver >= (3, 3) and not nx.config.backends.cugraph.use_compat_graphs: + graph_class = graph_class.to_cudagraph_class() if G.__class__ not in {nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph}: raise NotImplementedError( f"create_using with type {type(G)} is not supported by the cugraph " - "backend; only standard networkx or nx_cugraph Graph objects are " + "backend; only standard networkx or nx_cugraph graph objects are " "allowed (but not customized subclasses derived from them)." ) else: diff --git a/python/nx-cugraph/nx_cugraph/generators/classic.py b/python/nx-cugraph/nx_cugraph/generators/classic.py index a548beea34f..cfcb2a3afec 100644 --- a/python/nx-cugraph/nx_cugraph/generators/classic.py +++ b/python/nx-cugraph/nx_cugraph/generators/classic.py @@ -18,6 +18,7 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import _get_int_dtype, index_dtype, networkx_algorithm from ._utils import ( @@ -102,7 +103,9 @@ def complete_graph(n, create_using=None): @networkx_algorithm(version_added="23.12") def complete_multipartite_graph(*subset_sizes): if not subset_sizes: - return nxcg.Graph() + if _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs: + return nxcg.Graph() + return nxcg.CudaGraph() try: subset_sizes = [_ensure_int(size) for size in subset_sizes] except TypeError: @@ -139,6 +142,8 @@ def complete_multipartite_graph(*subset_sizes): dst_indices, node_values={"subset": subsets_array}, id_to_key=nodes, + use_compat_graph=_nxver < (3, 3) + or nx.config.backends.cugraph.use_compat_graphs, ) diff --git a/python/nx-cugraph/nx_cugraph/generators/community.py b/python/nx-cugraph/nx_cugraph/generators/community.py index 9b0e0848de9..4e5063cc345 100644 --- a/python/nx-cugraph/nx_cugraph/generators/community.py +++ b/python/nx-cugraph/nx_cugraph/generators/community.py @@ -11,8 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import cupy as cp +import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import networkx_algorithm from ._utils import ( @@ -42,4 +44,7 @@ def caveman_graph(l, k): # noqa: E741 dst_cliques.extend(dst_clique + i * k for i in range(1, l)) src_indices = cp.hstack(src_cliques) dst_indices = cp.hstack(dst_cliques) - return nxcg.Graph.from_coo(l * k, src_indices, dst_indices) + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + l * k, src_indices, dst_indices, use_compat_graph=use_compat_graph + ) diff --git a/python/nx-cugraph/nx_cugraph/generators/ego.py b/python/nx-cugraph/nx_cugraph/generators/ego.py index 66c9c8b95ee..9a91fa0b6c3 100644 --- a/python/nx-cugraph/nx_cugraph/generators/ego.py +++ b/python/nx-cugraph/nx_cugraph/generators/ego.py @@ -32,7 +32,10 @@ def ego_graph( ): """Weighted ego_graph with negative cycles is not yet supported. `NotImplementedError` will be raised if there are negative `distance` edge weights.""" # noqa: E501 if isinstance(G, nx.Graph): + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + is_compat_graph = False if n not in G: if distance is None: raise nx.NodeNotFound(f"Source {n} is not in G") @@ -100,7 +103,10 @@ def ego_graph( node_mask &= node_ids != src_index node_ids = node_ids[node_mask] if node_ids.size == G._N: - return G.copy() + rv = G.copy() + if is_compat_graph: + return rv._to_compat_graph() + return rv # TODO: create renumbering helper function(s) node_ids.sort() # TODO: is this ever necessary? Keep for safety node_values = {key: val[node_ids] for key, val in G.node_values.items()} @@ -137,6 +143,7 @@ def ego_graph( "node_values": node_values, "node_masks": node_masks, "key_to_id": key_to_id, + "use_compat_graph": False, } if G.is_multigraph(): if G.edge_keys is not None: @@ -147,6 +154,8 @@ def ego_graph( kwargs["edge_indices"] = G.edge_indices[edge_mask] rv = G.__class__.from_coo(**kwargs) rv.graph.update(G.graph) + if is_compat_graph: + return rv._to_compat_graph() return rv diff --git a/python/nx-cugraph/nx_cugraph/generators/small.py b/python/nx-cugraph/nx_cugraph/generators/small.py index 45487571cda..d0c03cb7dd4 100644 --- a/python/nx-cugraph/nx_cugraph/generators/small.py +++ b/python/nx-cugraph/nx_cugraph/generators/small.py @@ -14,6 +14,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype, networkx_algorithm from ._utils import _IS_NX32_OR_LESS, _create_using_class @@ -449,7 +450,14 @@ def pappus_graph(): index_dtype, ) # fmt: on - return nxcg.Graph.from_coo(18, src_indices, dst_indices, name="Pappus Graph") + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 18, + src_indices, + dst_indices, + name="Pappus Graph", + use_compat_graph=use_compat_graph, + ) @networkx_algorithm(version_added="23.12") diff --git a/python/nx-cugraph/nx_cugraph/generators/social.py b/python/nx-cugraph/nx_cugraph/generators/social.py index 07e82c63fbf..09d405e7561 100644 --- a/python/nx-cugraph/nx_cugraph/generators/social.py +++ b/python/nx-cugraph/nx_cugraph/generators/social.py @@ -11,9 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import cupy as cp +import networkx as nx import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype, networkx_algorithm @@ -77,7 +79,8 @@ def davis_southern_women_graph(): "E13", "E14", ] # fmt: on - return nxcg.Graph.from_coo( + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( 32, src_indices, dst_indices, @@ -85,6 +88,7 @@ def davis_southern_women_graph(): id_to_key=women + events, top=women, bottom=events, + use_compat_graph=use_compat_graph, ) @@ -111,7 +115,14 @@ def florentine_families_graph(): "Salviati", "Strozzi", "Tornabuoni" ] # fmt: on - return nxcg.Graph.from_coo(15, src_indices, dst_indices, id_to_key=nodes) + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 15, + src_indices, + dst_indices, + id_to_key=nodes, + use_compat_graph=use_compat_graph, + ) @networkx_algorithm(version_added="23.12") @@ -165,13 +176,15 @@ def karate_club_graph(): "Officer", "Officer", "Officer", "Officer", "Officer", "Officer", ]) # fmt: on - return nxcg.Graph.from_coo( + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( 34, src_indices, dst_indices, edge_values={"weight": weights}, node_values={"club": clubs}, name="Zachary's Karate Club", + use_compat_graph=use_compat_graph, ) @@ -289,6 +302,12 @@ def les_miserables_graph(): "Zephine", ] # fmt: on - return nxcg.Graph.from_coo( - 77, src_indices, dst_indices, edge_values={"weight": weights}, id_to_key=nodes + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 77, + src_indices, + dst_indices, + edge_values={"weight": weights}, + id_to_key=nodes, + use_compat_graph=use_compat_graph, ) diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index 4007230efa9..1a3d08409a2 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -18,6 +18,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver class BackendInterface: @@ -32,11 +33,19 @@ def convert_from_nx(graph, *args, edge_attrs=None, weight=None, **kwargs): "edge_attrs and weight arguments should not both be given" ) edge_attrs = {weight: 1} - return nxcg.from_networkx(graph, *args, edge_attrs=edge_attrs, **kwargs) + return nxcg.from_networkx( + graph, + *args, + edge_attrs=edge_attrs, + use_compat_graph=_nxver < (3, 3) + or nx.config.backends.cugraph.use_compat_graphs, + **kwargs, + ) @staticmethod def convert_to_nx(obj, *, name: str | None = None): - if isinstance(obj, nxcg.Graph): + if isinstance(obj, nxcg.CudaGraph): + # Observe that this does not try to convert Graph! return nxcg.to_networkx(obj) return obj @@ -62,19 +71,32 @@ def key(testpath): return (testname, frozenset({classname, filename})) return (testname, frozenset({filename})) + use_compat_graph = ( + _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + ) + fallback = use_compat_graph or nx.utils.backends._dispatchable._fallback_to_nx + # Reasons for xfailing + # For nx version <= 3.1 no_weights = "weighted implementation not currently supported" no_multigraph = "multigraphs not currently supported" + # For nx version <= 3.2 + nx_cugraph_in_test_setup = ( + "nx-cugraph Graph is incompatible in test setup in nx versions < 3.3" + ) + # For all versions louvain_different = "Louvain may be different due to RNG" - no_string_dtype = "string edge values not currently supported" sssp_path_different = "sssp may choose a different valid path" + tuple_elements_preferred = "elements are tuples instead of lists" + no_mixed_dtypes_for_nodes = ( + # This one is tricky b/c we don't raise; all dtypes are treated as str + "mixed dtypes (str, int, float) for single node property not supported" + ) + # These shouldn't fail if using Graph or falling back to networkx + no_string_dtype = "string edge values not currently supported" no_object_dtype_for_edges = ( "Edges don't support object dtype (lists, strings, etc.)" ) - tuple_elements_preferred = "elements are tuples instead of lists" - nx_cugraph_in_test_setup = ( - "nx-cugraph Graph is incompatible in test setup in nx versions < 3.3" - ) xfail = { # This is removed while strongly_connected_components() is not @@ -98,38 +120,6 @@ def key(testpath): "test_cycles.py:TestMinimumCycleBasis." "test_gh6787_and_edge_attribute_names" ): sssp_path_different, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr_and_node_attr" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr_subgraph_hash" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:" - "test_isomorphic_edge_attr_and_node_attr_subgraph_hash" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPNoEdgeTypes.test_summary_graph" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPUndirected.test_summary_graph" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPDirected.test_summary_graph" - ): no_object_dtype_for_edges, - key("test_gexf.py:TestGEXF.test_relabel"): no_object_dtype_for_edges, - key( - "test_gml.py:TestGraph.test_parse_gml_cytoscape_bug" - ): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_parse_gml"): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_read_gml"): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_data_types"): no_object_dtype_for_edges, - key( - "test_gml.py:TestPropertyLists.test_reading_graph_with_list_property" - ): no_object_dtype_for_edges, key( "test_relabel.py:" "test_relabel_preserve_node_order_partial_mapping_with_copy_false" @@ -138,48 +128,107 @@ def key(testpath): "test_gml.py:" "TestPropertyLists.test_reading_graph_with_single_element_list_property" ): tuple_elements_preferred, - key( - "test_relabel.py:" - "TestRelabel.test_relabel_multidigraph_inout_merge_nodes" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_inplace" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_merge_inplace" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_inout_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_merge_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_nonnumeric_key" - ): no_string_dtype, - key("test_contraction.py:test_multigraph_path"): no_object_dtype_for_edges, - key( - "test_contraction.py:test_directed_multigraph_path" - ): no_object_dtype_for_edges, - key( - "test_contraction.py:test_multigraph_blockmodel" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPUndirectedMulti.test_summary_graph" - ): no_string_dtype, - key( - "test_summarization.py:TestSNAPDirectedMulti.test_summary_graph" - ): no_string_dtype, } + if not fallback: + xfail.update( + { + key( + "test_graph_hashing.py:test_isomorphic_edge_attr" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:test_isomorphic_edge_attr_and_node_attr" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:test_isomorphic_edge_attr_subgraph_hash" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:" + "test_isomorphic_edge_attr_and_node_attr_subgraph_hash" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPNoEdgeTypes.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPUndirected.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPDirected.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_gexf.py:TestGEXF.test_relabel" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_parse_gml_cytoscape_bug" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_parse_gml" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_read_gml" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_data_types" + ): no_object_dtype_for_edges, + key( + "test_gml.py:" + "TestPropertyLists.test_reading_graph_with_list_property" + ): no_object_dtype_for_edges, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_inout_merge_nodes" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multigraph_merge_inplace" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_merge_inplace" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_inout_copy" + ): no_string_dtype, + key( + "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_copy" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_merge_copy" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multigraph_nonnumeric_key" + ): no_string_dtype, + key( + "test_contraction.py:test_multigraph_path" + ): no_object_dtype_for_edges, + key( + "test_contraction.py:test_directed_multigraph_path" + ): no_object_dtype_for_edges, + key( + "test_contraction.py:test_multigraph_blockmodel" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:" + "TestSNAPUndirectedMulti.test_summary_graph" + ): no_string_dtype, + key( + "test_summarization.py:TestSNAPDirectedMulti.test_summary_graph" + ): no_string_dtype, + } + ) + else: + xfail.update( + { + key( + "test_gml.py:" + "TestPropertyLists.test_reading_graph_with_list_property" + ): no_mixed_dtypes_for_nodes, + } + ) - from packaging.version import parse - - nxver = parse(nx.__version__) - - if nxver.major == 3 and nxver.minor <= 2: + if _nxver <= (3, 2): xfail.update( { # NetworkX versions prior to 3.2.1 have tests written to @@ -216,7 +265,7 @@ def key(testpath): } ) - if nxver.major == 3 and nxver.minor <= 1: + if _nxver <= (3, 1): # MAINT: networkx 3.0, 3.1 # NetworkX 3.2 added the ability to "fallback to nx" if backend algorithms # raise NotImplementedError or `can_run` returns False. The tests below @@ -332,24 +381,25 @@ def key(testpath): xfail[key("test_louvain.py:test_threshold")] = ( "Louvain does not support seed parameter" ) - if nxver.major == 3 and nxver.minor >= 2: - xfail.update( - { - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_multi_attr_incl_target" - ): no_string_dtype, - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_multidigraph_and_edge_attr" - ): no_string_dtype, - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_int_attr_name" - ): no_string_dtype, - } - ) - if nxver.minor == 2: + if _nxver >= (3, 2): + if not fallback: + xfail.update( + { + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_multi_attr_incl_target" + ): no_string_dtype, + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_multidigraph_and_edge_attr" + ): no_string_dtype, + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_int_attr_name" + ): no_string_dtype, + } + ) + if _nxver[1] == 2: different_iteration_order = "Different graph data iteration order" xfail.update( { @@ -366,7 +416,7 @@ def key(testpath): ): different_iteration_order, } ) - elif nxver.minor >= 3: + elif _nxver[1] >= 3: xfail.update( { key("test_louvain.py:test_max_level"): louvain_different, diff --git a/python/nx-cugraph/nx_cugraph/relabel.py b/python/nx-cugraph/nx_cugraph/relabel.py index 20d1337a99c..e38e18c779e 100644 --- a/python/nx-cugraph/nx_cugraph/relabel.py +++ b/python/nx-cugraph/nx_cugraph/relabel.py @@ -29,13 +29,18 @@ @networkx_algorithm(version_added="24.08") def relabel_nodes(G, mapping, copy=True): + G_orig = G if isinstance(G, nx.Graph): - if not copy: + is_compat_graph = isinstance(G, nxcg.Graph) + if not copy and not is_compat_graph: raise RuntimeError( "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.relabel_nodes`" ) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + is_compat_graph = False + it = range(G._N) if G.key_to_id is None else G.id_to_key if callable(mapping): previd_to_key = [mapping(node) for node in it] @@ -225,12 +230,13 @@ def relabel_nodes(G, mapping, copy=True): node_masks=node_masks, id_to_key=newid_to_key, key_to_id=key_to_newid, + use_compat_graph=is_compat_graph, **extra_kwargs, ) rv.graph.update(G.graph) if not copy: - G._become(rv) - return G + G_orig._become(rv) + return G_orig return rv @@ -241,7 +247,10 @@ def convert_node_labels_to_integers( if ordering not in {"default", "sorted", "increasing degree", "decreasing degree"}: raise nx.NetworkXError(f"Unknown node ordering: {ordering}") if isinstance(G, nx.Graph): + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + is_compat_graph = False G = G.copy() if label_attribute is not None: prev_vals = G.id_to_key @@ -279,4 +288,6 @@ def convert_node_labels_to_integers( key_to_id = G.key_to_id G.key_to_id = {i: key_to_id[n] for i, (d, n) in enumerate(pairs, first_label)} G._id_to_key = id_to_key + if is_compat_graph: + return G._to_compat_graph() return G diff --git a/python/nx-cugraph/nx_cugraph/tests/test_bfs.py b/python/nx-cugraph/nx_cugraph/tests/test_bfs.py index c2b22e98949..ad2c62c1fb9 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_bfs.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_bfs.py @@ -12,11 +12,10 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse -nxver = parse(nx.__version__) +from nx_cugraph import _nxver -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test clustering", allow_module_level=True) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_classes.py b/python/nx-cugraph/nx_cugraph/tests/test_classes.py new file mode 100644 index 00000000000..0ac238b3558 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_classes.py @@ -0,0 +1,77 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import nx_cugraph as nxcg + + +def test_class_to_class(): + """Basic sanity checks to ensure metadata relating graph classes are accurate.""" + for prefix in ["", "Cuda"]: + for suffix in ["Graph", "DiGraph", "MultiGraph", "MultiDiGraph"]: + cls_name = f"{prefix}{suffix}" + cls = getattr(nxcg, cls_name) + assert cls.__name__ == cls_name + G = cls() + assert cls is G.__class__ + # cudagraph + val = cls.to_cudagraph_class() + val2 = G.to_cudagraph_class() + assert val is val2 + assert val.__name__ == f"Cuda{suffix}" + assert val.__module__.startswith("nx_cugraph") + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + # networkx + val = cls.to_networkx_class() + val2 = G.to_networkx_class() + assert val is val2 + assert val.__name__ == suffix + assert val.__module__.startswith("networkx") + val = val() + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + # directed + val = cls.to_directed_class() + val2 = G.to_directed_class() + assert val is val2 + assert val.__module__.startswith("nx_cugraph") + assert val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + if "Di" in suffix: + assert val is cls + else: + assert "Di" in val.__name__ + assert prefix in val.__name__ + assert cls.to_undirected_class() is cls + # undirected + val = cls.to_undirected_class() + val2 = G.to_undirected_class() + assert val is val2 + assert val.__module__.startswith("nx_cugraph") + assert not val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + if "Di" not in suffix: + assert val is cls + else: + assert "Di" not in val.__name__ + assert prefix in val.__name__ + assert cls.to_directed_class() is cls + # "zero" + if prefix == "Cuda": + val = cls._to_compat_graph_class() + val2 = G._to_compat_graph_class() + assert val is val2 + assert val.__name__ == suffix + assert val.__module__.startswith("nx_cugraph") + assert val.to_cudagraph_class() is cls + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() diff --git a/python/nx-cugraph/nx_cugraph/tests/test_cluster.py b/python/nx-cugraph/nx_cugraph/tests/test_cluster.py index ad4770f1ab8..fd8e1b3cf13 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_cluster.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_cluster.py @@ -12,11 +12,10 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse -nxver = parse(nx.__version__) +from nx_cugraph import _nxver -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test clustering", allow_module_level=True) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_convert.py b/python/nx-cugraph/nx_cugraph/tests/test_convert.py index 634b28e961c..3d109af8a74 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_convert.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_convert.py @@ -13,13 +13,10 @@ import cupy as cp import networkx as nx import pytest -from packaging.version import parse import nx_cugraph as nxcg from nx_cugraph import interface -nxver = parse(nx.__version__) - @pytest.mark.parametrize( "graph_class", [nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph] diff --git a/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py b/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py index 5474f9d79e3..0697a744e85 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py @@ -12,16 +12,13 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse import nx_cugraph as nxcg +from nx_cugraph import _nxver from .testing_utils import assert_graphs_equal -nxver = parse(nx.__version__) - - -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test ego_graph", allow_module_level=True) @@ -49,7 +46,12 @@ def test_ego_graph_cycle_graph( kwargs = {"radius": radius, "center": center, "undirected": undirected} Hnx = nx.ego_graph(Gnx, n, **kwargs) Hcg = nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") + use_compat_graphs = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + assert_graphs_equal(Hnx, Hcg._cudagraph if use_compat_graphs else Hcg) + Hcg = nx.ego_graph(Gcg, n, **kwargs) assert_graphs_equal(Hnx, Hcg) + Hcg = nx.ego_graph(Gcg._to_compat_graph(), n, **kwargs) + assert_graphs_equal(Hnx, Hcg._cudagraph) with pytest.raises(nx.NodeNotFound, match="not in G"): nx.ego_graph(Gnx, -1, **kwargs) with pytest.raises(nx.NodeNotFound, match="not in G"): @@ -61,20 +63,36 @@ def test_ego_graph_cycle_graph( kwargs["distance"] = "weight" H2nx = nx.ego_graph(Gnx, n, **kwargs) - is_nx32 = nxver.major == 3 and nxver.minor == 2 + is_nx32 = _nxver[:2] == (3, 2) if undirected and Gnx.is_directed() and Gnx.is_multigraph(): if is_nx32: # `should_run` was added in nx 3.3 match = "Weighted ego_graph with undirected=True not implemented" + elif _nxver >= (3, 4): + match = "not implemented by 'cugraph'" else: match = "not implemented by cugraph" - with pytest.raises(RuntimeError, match=match): + with pytest.raises( + RuntimeError if _nxver < (3, 4) else NotImplementedError, match=match + ): nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") with pytest.raises(NotImplementedError, match="ego_graph"): - nx.ego_graph(Gcg, n, **kwargs) + nx.ego_graph(Gcg, n, **kwargs, backend="cugraph") + if _nxver < (3, 4): + with pytest.raises(NotImplementedError, match="ego_graph"): + nx.ego_graph(Gcg, n, **kwargs) + else: + # This is an interesting case. `nxcg.ego_graph` is not implemented for + # these arguments, so it falls back to networkx. Hence, as it is currently + # implemented, the input graph is `nxcg.CudaGraph`, but the output graph + # is `nx.Graph`. Should networkx convert back to "cugraph" backend? + # TODO: make fallback to networkx configurable. + H2cg = nx.ego_graph(Gcg, n, **kwargs) + assert type(H2nx) is type(H2cg) + assert_graphs_equal(H2nx, nxcg.from_networkx(H2cg, preserve_all_attrs=True)) else: H2cg = nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") - assert_graphs_equal(H2nx, H2cg) + assert_graphs_equal(H2nx, H2cg._cudagraph if use_compat_graphs else H2cg) with pytest.raises(nx.NodeNotFound, match="not found in graph"): nx.ego_graph(Gnx, -1, **kwargs) with pytest.raises(nx.NodeNotFound, match="not found in graph"): diff --git a/python/nx-cugraph/nx_cugraph/tests/test_generators.py b/python/nx-cugraph/nx_cugraph/tests/test_generators.py index c751b0fe2b3..5c405f1c93b 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_generators.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_generators.py @@ -13,25 +13,24 @@ import networkx as nx import numpy as np import pytest -from packaging.version import parse import nx_cugraph as nxcg +from nx_cugraph import _nxver from .testing_utils import assert_graphs_equal -nxver = parse(nx.__version__) - - -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test generators", allow_module_level=True) def compare(name, create_using, *args, is_vanilla=False): exc1 = exc2 = None func = getattr(nx, name) - if isinstance(create_using, nxcg.Graph): + if isinstance(create_using, nxcg.CudaGraph): nx_create_using = nxcg.to_networkx(create_using) - elif isinstance(create_using, type) and issubclass(create_using, nxcg.Graph): + elif isinstance(create_using, type) and issubclass( + create_using, (nxcg.Graph, nxcg.CudaGraph) + ): nx_create_using = create_using.to_networkx_class() elif isinstance(create_using, nx.Graph): nx_create_using = create_using.copy() @@ -61,8 +60,27 @@ def compare(name, create_using, *args, is_vanilla=False): exc2 = exc if exc1 is not None or exc2 is not None: assert type(exc1) is type(exc2) + return + if isinstance(Gcg, nxcg.Graph): + # If the graph is empty, it may be on host, otherwise it should be on device + if len(G): + assert Gcg._is_on_gpu + assert not Gcg._is_on_cpu + assert_graphs_equal(G, Gcg._cudagraph) else: assert_graphs_equal(G, Gcg) + # Ensure the output type is correct + if is_vanilla: + if _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs: + assert isinstance(Gcg, nxcg.Graph) + else: + assert isinstance(Gcg, nxcg.CudaGraph) + elif isinstance(create_using, type) and issubclass( + create_using, (nxcg.Graph, nxcg.CudaGraph) + ): + assert type(Gcg) is create_using + elif isinstance(create_using, (nxcg.Graph, nxcg.CudaGraph)): + assert type(Gcg) is type(create_using) N = list(range(-1, 5)) @@ -76,6 +94,10 @@ def compare(name, create_using, *args, is_vanilla=False): nxcg.DiGraph, nxcg.MultiGraph, nxcg.MultiDiGraph, + nxcg.CudaGraph, + nxcg.CudaDiGraph, + nxcg.CudaMultiGraph, + nxcg.CudaMultiDiGraph, # These raise NotImplementedError # nx.Graph(), # nx.DiGraph(), @@ -85,6 +107,10 @@ def compare(name, create_using, *args, is_vanilla=False): nxcg.DiGraph(), nxcg.MultiGraph(), nxcg.MultiDiGraph(), + nxcg.CudaGraph(), + nxcg.CudaDiGraph(), + nxcg.CudaMultiGraph(), + nxcg.CudaMultiDiGraph(), None, object, # Bad input 7, # Bad input @@ -158,7 +184,7 @@ def compare(name, create_using, *args, is_vanilla=False): @pytest.mark.parametrize("create_using", COMPLETE_CREATE_USING) def test_generator_noarg(name, create_using): print(name, create_using, type(create_using)) - if isinstance(create_using, nxcg.Graph) and name in { + if isinstance(create_using, nxcg.CudaGraph) and name in { # fmt: off "bull_graph", "chvatal_graph", "cubical_graph", "diamond_graph", "house_graph", "house_x_graph", "icosahedral_graph", "krackhardt_kite_graph", diff --git a/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py b/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py index 3120995a2b2..40a361b1084 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py @@ -47,7 +47,7 @@ def _create_Gs(): @pytest.mark.parametrize("Gnx", _create_Gs()) @pytest.mark.parametrize("reciprocal", [False, True]) def test_to_undirected_directed(Gnx, reciprocal): - Gcg = nxcg.DiGraph(Gnx) + Gcg = nxcg.CudaDiGraph(Gnx) assert_graphs_equal(Gnx, Gcg) Hnx1 = Gnx.to_undirected(reciprocal=reciprocal) Hcg1 = Gcg.to_undirected(reciprocal=reciprocal) @@ -62,6 +62,6 @@ def test_multidigraph_to_undirected(): Gnx.add_edge(0, 1) Gnx.add_edge(0, 1) Gnx.add_edge(1, 0) - Gcg = nxcg.MultiDiGraph(Gnx) + Gcg = nxcg.CudaMultiDiGraph(Gnx) with pytest.raises(NotImplementedError): Gcg.to_undirected() diff --git a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py index 176b531a6e7..1a61c69b3e7 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py @@ -14,13 +14,10 @@ import inspect import networkx as nx -from packaging.version import parse import nx_cugraph as nxcg from nx_cugraph.utils import networkx_algorithm -nxver = parse(nx.__version__) - def test_match_signature_and_names(): """Simple test to ensure our signatures and basic module layout match networkx.""" diff --git a/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py b/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py index a8f189a4745..9208eea09f2 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -26,7 +26,7 @@ def test_get_edge_data(test_nxcugraph): G.add_edge(0, 3) G.add_edge(0, 3) if test_nxcugraph: - G = nxcg.MultiGraph(G) + G = nxcg.CudaMultiGraph(G) default = object() assert G.get_edge_data(0, 0, default=default) is default assert G.get_edge_data("a", "b", default=default) is default @@ -60,7 +60,7 @@ def test_get_edge_data(test_nxcugraph): G = nx.MultiGraph() G.add_edge(0, 1) if test_nxcugraph: - G = nxcg.MultiGraph(G) + G = nxcg.CudaMultiGraph(G) assert G.get_edge_data(0, 1, default=default) == {0: {}} assert G.get_edge_data(0, 1, 0, default=default) == {} assert G.get_edge_data(0, 1, 1, default=default) is default diff --git a/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py index 0b437df2d2f..252f9e6bbb8 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py @@ -12,19 +12,23 @@ # limitations under the License. import networkx as nx import pandas as pd -from pytest import approx +import pytest def test_pagerank_multigraph(): """ - Ensures correct differences between pagerank results for Graphs - vs. MultiGraphs generated using from_pandas_edgelist() + Ensures correct pagerank for Graphs and MultiGraphs when using from_pandas_edgelist. + + PageRank for MultiGraph should give different result compared to Graph; when using + a Graph, the duplicate edges should be dropped. """ - df = pd.DataFrame({"source": [0, 1, 1, 1, 1, 1, 1, 2], - "target": [1, 2, 2, 2, 2, 2, 2, 3]}) + df = pd.DataFrame( + {"source": [0, 1, 1, 1, 1, 1, 1, 2], "target": [1, 2, 2, 2, 2, 2, 2, 3]} + ) expected_pr_for_G = nx.pagerank(nx.from_pandas_edgelist(df)) expected_pr_for_MultiG = nx.pagerank( - nx.from_pandas_edgelist(df, create_using=nx.MultiGraph)) + nx.from_pandas_edgelist(df, create_using=nx.MultiGraph) + ) G = nx.from_pandas_edgelist(df, backend="cugraph") actual_pr_for_G = nx.pagerank(G, backend="cugraph") @@ -32,5 +36,5 @@ def test_pagerank_multigraph(): MultiG = nx.from_pandas_edgelist(df, create_using=nx.MultiGraph, backend="cugraph") actual_pr_for_MultiG = nx.pagerank(MultiG, backend="cugraph") - assert actual_pr_for_G == approx(expected_pr_for_G) - assert actual_pr_for_MultiG == approx(expected_pr_for_MultiG) + assert actual_pr_for_G == pytest.approx(expected_pr_for_G) + assert actual_pr_for_MultiG == pytest.approx(expected_pr_for_MultiG) diff --git a/python/nx-cugraph/nx_cugraph/tests/testing_utils.py b/python/nx-cugraph/nx_cugraph/tests/testing_utils.py index 529a96efd81..50836acf55f 100644 --- a/python/nx-cugraph/nx_cugraph/tests/testing_utils.py +++ b/python/nx-cugraph/nx_cugraph/tests/testing_utils.py @@ -17,7 +17,7 @@ def assert_graphs_equal(Gnx, Gcg): assert isinstance(Gnx, nx.Graph) - assert isinstance(Gcg, nxcg.Graph) + assert isinstance(Gcg, nxcg.CudaGraph) assert (a := Gnx.number_of_nodes()) == (b := Gcg.number_of_nodes()), (a, b) assert (a := Gnx.number_of_edges()) == (b := Gcg.number_of_edges()), (a, b) assert (a := Gnx.is_directed()) == (b := Gcg.is_directed()), (a, b) diff --git a/python/nx-cugraph/nx_cugraph/utils/decorators.py b/python/nx-cugraph/nx_cugraph/utils/decorators.py index 3c5de4f2936..16486996ba0 100644 --- a/python/nx-cugraph/nx_cugraph/utils/decorators.py +++ b/python/nx-cugraph/nx_cugraph/utils/decorators.py @@ -16,10 +16,14 @@ from textwrap import dedent import networkx as nx +from networkx import NetworkXError from networkx.utils.decorators import nodes_or_number, not_implemented_for +from nx_cugraph import _nxver from nx_cugraph.interface import BackendInterface +from .misc import _And_NotImplementedError + try: from networkx.utils.backends import _registered_algorithms except ModuleNotFoundError: @@ -44,6 +48,7 @@ class networkx_algorithm: version_added: str is_incomplete: bool is_different: bool + _fallback: bool _plc_names: set[str] | None def __new__( @@ -59,6 +64,7 @@ def __new__( version_added: str, # Required is_incomplete: bool = False, # See self.extra_doc for details if True is_different: bool = False, # See self.extra_doc for details if True + fallback: bool = False, # Change non-nx exceptions to NotImplementedError _plc: str | set[str] | None = None, # Hidden from user, may be removed someday ): if func is None: @@ -70,10 +76,11 @@ def __new__( version_added=version_added, is_incomplete=is_incomplete, is_different=is_different, + fallback=fallback, _plc=_plc, ) instance = object.__new__(cls) - if nodes_or_number is not None and nx.__version__[:3] > "3.2": + if nodes_or_number is not None and _nxver > (3, 2): func = nx.utils.decorators.nodes_or_number(nodes_or_number)(func) # update_wrapper sets __wrapped__, which will be used for the signature update_wrapper(instance, func) @@ -100,6 +107,7 @@ def __new__( instance.version_added = version_added instance.is_incomplete = is_incomplete instance.is_different = is_different + instance.fallback = fallback # The docstring on our function is added to the NetworkX docstring. instance.extra_doc = ( dedent(func.__doc__.lstrip("\n").rstrip()) if func.__doc__ else None @@ -113,7 +121,7 @@ def __new__( # Set methods so they are in __dict__ instance._can_run = instance._can_run instance._should_run = instance._should_run - if nodes_or_number is not None and nx.__version__[:3] <= "3.2": + if nodes_or_number is not None and _nxver <= (3, 2): instance = nx.utils.decorators.nodes_or_number(nodes_or_number)(instance) return instance @@ -136,7 +144,14 @@ def _should_run(self, func): self.should_run = func def __call__(self, /, *args, **kwargs): - return self.__wrapped__(*args, **kwargs) + if not self.fallback: + return self.__wrapped__(*args, **kwargs) + try: + return self.__wrapped__(*args, **kwargs) + except NetworkXError: + raise + except Exception as exc: + raise _And_NotImplementedError(exc) from exc def __reduce__(self): return _restore_networkx_dispatched, (self.name,) diff --git a/python/nx-cugraph/nx_cugraph/utils/misc.py b/python/nx-cugraph/nx_cugraph/utils/misc.py index 8526524f1de..01c25dd5983 100644 --- a/python/nx-cugraph/nx_cugraph/utils/misc.py +++ b/python/nx-cugraph/nx_cugraph/utils/misc.py @@ -194,7 +194,7 @@ def _get_int_dtype( def _get_float_dtype( - dtype: Dtype, *, graph: nxcg.Graph | None = None, weight: EdgeKey | None = None + dtype: Dtype, *, graph: nxcg.CudaGraph | None = None, weight: EdgeKey | None = None ): """Promote dtype to float32 or float64 as appropriate.""" if dtype is None: @@ -238,3 +238,37 @@ def _cp_iscopied_asarray(a, *args, orig_object=None, **kwargs): ): return False, arr return True, arr + + +class _And_NotImplementedError(NotImplementedError): + """Additionally make an exception a ``NotImplementedError``. + + For example: + + >>> try: + ... raise _And_NotImplementedError(KeyError("missing")) + ... except KeyError: + ... pass + + or + + >>> try: + ... raise _And_NotImplementedError(KeyError("missing")) + ... except NotImplementedError: + ... pass + + """ + + def __new__(cls, exc): + exc_type = type(exc) + if issubclass(exc_type, NotImplementedError): + new_type = exc_type + else: + new_type = type( + f"{exc_type.__name__}{cls.__name__}", + (exc_type, NotImplementedError), + {}, + ) + instance = NotImplementedError.__new__(new_type) + instance.__init__(*exc.args) + return instance diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index e7b4ea44dd8..98de089a92c 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ [project.optional-dependencies] test = [ - "packaging>=21", "pandas", "pytest", "pytest-benchmark", @@ -170,6 +169,7 @@ external = [ ] ignore = [ # Would be nice to fix these + "B905", # `zip()` without an explicit `strict=` parameter (Note: possible since py39 was dropped; we should do this!) "D100", # Missing docstring in public module "D101", # Missing docstring in public class "D102", # Missing docstring in public method @@ -215,6 +215,7 @@ ignore = [ "SIM105", # Use contextlib.suppress(...) instead of try-except-pass (Note: try-except-pass is much faster) "SIM108", # Use ternary operator ... instead of if-else-block (Note: if-else better for coverage and sometimes clearer) "TRY003", # Avoid specifying long messages outside the exception class (Note: why?) + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` (Note: tuple is faster for now) # Ignored categories "C90", # mccabe (Too strict, but maybe we should make things less complex) @@ -241,6 +242,7 @@ ignore = [ # Allow assert, print, RNG, and no docstring "nx_cugraph/**/tests/*py" = ["S101", "S311", "T201", "D103", "D100"] "_nx_cugraph/__init__.py" = ["E501"] +"nx_cugraph/__init__.py" = ["E402"] # Allow module level import not at top of file "nx_cugraph/algorithms/**/*py" = ["D205", "D401"] # Allow flexible docstrings for algorithms "nx_cugraph/generators/**/*py" = ["D205", "D401"] # Allow flexible docstrings for generators "nx_cugraph/interface.py" = ["D401"] # Flexible docstrings diff --git a/python/nx-cugraph/run_nx_tests.sh b/python/nx-cugraph/run_nx_tests.sh index bceec53b7d5..5fb173cf939 100755 --- a/python/nx-cugraph/run_nx_tests.sh +++ b/python/nx-cugraph/run_nx_tests.sh @@ -18,6 +18,10 @@ # testing takes longer. Without it, tests will xfail when encountering a # function that we don't implement. # +# NX_CUGRAPH_USE_COMPAT_GRAPHS, {"True", "False"}, default is "True" +# Whether to use `nxcg.Graph` as the nx_cugraph backend graph. +# A Graph should be a compatible NetworkX graph, so fewer tests should fail. +# # Coverage of `nx_cugraph.algorithms` is reported and is a good sanity check # that algorithms run. From 27f2256eede2949c04a94136500ec6c1b7fbbc29 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 24 Sep 2024 23:25:18 -0500 Subject: [PATCH 6/9] bump NCCL floor to 2.18.1.1, include nccl.h where it's needed (#4661) Contributes to https://github.com/rapidsai/build-planning/issues/102 Some RAPIDS libraries are using `ncclCommSplit()`, which was introduced in `nccl==2.18.1.1`. This is part of a series of PRs across RAPIDS updating libraries' pins to `nccl>=2.18.1.1` to ensure they get a new-enough version that supports that. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Chuck Hastings (https://github.com/ChuckHastings) - Vyas Ramasubramani (https://github.com/vyasr) - https://github.com/jakirkham URL: https://github.com/rapidsai/cugraph/pull/4661 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/libcugraph/conda_build_config.yaml | 2 +- cpp/include/cugraph/mtmg/instance_manager.hpp | 2 ++ cpp/include/cugraph/mtmg/resource_manager.hpp | 2 ++ cpp/tests/mtmg/multi_node_threaded_test.cu | 1 + dependencies.yaml | 2 +- 7 files changed, 9 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 533f23cd7ac..fd91edd8adc 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -33,7 +33,7 @@ dependencies: - libraft==24.10.*,>=0.0.0a0 - librmm==24.10.*,>=0.0.0a0 - nbsphinx -- nccl>=2.9.9 +- nccl>=2.18.1.1 - networkx>=2.5.1 - networkx>=3.0 - ninja diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 084a6adfd31..19da750601b 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -39,7 +39,7 @@ dependencies: - libraft==24.10.*,>=0.0.0a0 - librmm==24.10.*,>=0.0.0a0 - nbsphinx -- nccl>=2.9.9 +- nccl>=2.18.1.1 - networkx>=2.5.1 - networkx>=3.0 - ninja diff --git a/conda/recipes/libcugraph/conda_build_config.yaml b/conda/recipes/libcugraph/conda_build_config.yaml index 26aa428d7f5..6b50d0aad23 100644 --- a/conda/recipes/libcugraph/conda_build_config.yaml +++ b/conda/recipes/libcugraph/conda_build_config.yaml @@ -17,7 +17,7 @@ doxygen_version: - ">=1.8.11" nccl_version: - - ">=2.9.9" + - ">=2.18.1.1" c_stdlib: - sysroot diff --git a/cpp/include/cugraph/mtmg/instance_manager.hpp b/cpp/include/cugraph/mtmg/instance_manager.hpp index a2111804997..759635b4a34 100644 --- a/cpp/include/cugraph/mtmg/instance_manager.hpp +++ b/cpp/include/cugraph/mtmg/instance_manager.hpp @@ -20,6 +20,8 @@ #include +#include + #include namespace cugraph { diff --git a/cpp/include/cugraph/mtmg/resource_manager.hpp b/cpp/include/cugraph/mtmg/resource_manager.hpp index a9e4b81f894..e9d25c4576b 100644 --- a/cpp/include/cugraph/mtmg/resource_manager.hpp +++ b/cpp/include/cugraph/mtmg/resource_manager.hpp @@ -27,6 +27,8 @@ #include #include +#include + #include namespace cugraph { diff --git a/cpp/tests/mtmg/multi_node_threaded_test.cu b/cpp/tests/mtmg/multi_node_threaded_test.cu index 06ccd4a7fa1..374c432aac5 100644 --- a/cpp/tests/mtmg/multi_node_threaded_test.cu +++ b/cpp/tests/mtmg/multi_node_threaded_test.cu @@ -39,6 +39,7 @@ #include #include +#include #include #include diff --git a/dependencies.yaml b/dependencies.yaml index 76048be2010..cd34fcf2f70 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -530,7 +530,7 @@ dependencies: - aiohttp - fsspec>=0.6.0 - requests - - nccl>=2.9.9 + - nccl>=2.18.1.1 - ucx-proc=*=gpu - &ucx_py_unsuffixed ucx-py==0.40.*,>=0.0.0a0 - output_types: pyproject From 87987af74d65e9dedeaea96a9ea4ac7c3c068615 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 25 Sep 2024 11:09:46 -0400 Subject: [PATCH 7/9] Install mg test executables (#4656) `rapids_test_add()` does not install the executables we need, because it's checking the value of `${MPIEXEC_EXECUTABLE}` and seeing that it's not a target. Install the target separately. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Bradley Dice (https://github.com/bdice) - Chuck Hastings (https://github.com/ChuckHastings) URL: https://github.com/rapidsai/cugraph/pull/4656 --- cpp/tests/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 09b1431e33b..3752e823659 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -220,6 +220,7 @@ function(ConfigureTestMG CMAKE_TEST_NAME) GPUS ${GPU_COUNT} PERCENT 100 INSTALL_COMPONENT_SET testing_mg + INSTALL_TARGET ${CMAKE_TEST_NAME} ) set_tests_properties(${CMAKE_TEST_NAME} PROPERTIES LABELS "CUGRAPH_MG") @@ -302,6 +303,7 @@ function(ConfigureCTestMG CMAKE_TEST_NAME) GPUS ${GPU_COUNT} PERCENT 100 INSTALL_COMPONENT_SET testing_mg + INSTALL_TARGET ${CMAKE_TEST_NAME} ) set_tests_properties(${CMAKE_TEST_NAME} PROPERTIES LABELS "CUGRAPH_C_MG") From 1e5b3287e628a5cf8e60da14c25174baf0e5cc1a Mon Sep 17 00:00:00 2001 From: Ralph Liu <137829296+nv-rliu@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:21:35 -0400 Subject: [PATCH 8/9] Fix `cit-patents` Dataset for `nx-cugraph` Benchmark (#4666) Replace `_` with `-` Authors: - Ralph Liu (https://github.com/nv-rliu) Approvers: - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4666 --- benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh b/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh index a1d32474e5f..3059e3d4bdf 100755 --- a/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh +++ b/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh @@ -30,7 +30,7 @@ algos=" datasets=" netscience email_Eu_core - cit_patents + cit-patents hollywood soc-livejournal " From 36c190ac8378527ac1e7445eb3773068f9179dea Mon Sep 17 00:00:00 2001 From: Alex Barghi <105237337+alexbarghi-nv@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:42:32 -0400 Subject: [PATCH 9/9] [FEA] Biased Sampling in cuGraph-DGL (#4595) Adds support for biased sampling to cuGraph-DGL. Resolves rapidsai/cugraph-gnn#25 Merge after #4583, #4586, #4607 Authors: - Alex Barghi (https://github.com/alexbarghi-nv) - Ralph Liu (https://github.com/nv-rliu) - Seunghwa Kang (https://github.com/seunghwak) Approvers: - Ray Douglass (https://github.com/raydouglass) - Tingyu Wang (https://github.com/tingyu66) - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4595 --- .../dataloading/neighbor_sampler.py | 19 ++- python/cugraph-dgl/cugraph_dgl/graph.py | 148 ++++++++++-------- .../tests/dataloading/test_dataloader.py | 55 ++++++- .../tests/dataloading/test_dataloader_mg.py | 81 +++++++++- .../cugraph/gnn/data_loading/dist_sampler.py | 2 +- .../pylibcugraph/biased_neighbor_sample.pyx | 2 +- .../pylibcugraph/uniform_neighbor_sample.pyx | 2 +- 7 files changed, 224 insertions(+), 85 deletions(-) diff --git a/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py b/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py index 87d111adcba..4ec513cbf9b 100644 --- a/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py +++ b/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py @@ -18,7 +18,7 @@ from typing import Sequence, Optional, Union, List, Tuple, Iterator -from cugraph.gnn import UniformNeighborSampler, DistSampleWriter +from cugraph.gnn import UniformNeighborSampler, BiasedNeighborSampler, DistSampleWriter from cugraph.utilities.utils import import_optional import cugraph_dgl @@ -93,7 +93,6 @@ def __init__( If provided, the probability of each neighbor being sampled is proportional to the edge feature with the given name. Mutually exclusive with mask. - Currently unsupported. mask: str Optional. If proivided, only neighbors where the edge mask @@ -133,10 +132,6 @@ def __init__( raise NotImplementedError( "Edge masking is currently unsupported by cuGraph-DGL" ) - if prob: - raise NotImplementedError( - "Edge masking is currently unsupported by cuGraph-DGL" - ) if prefetch_edge_feats: warnings.warn("'prefetch_edge_feats' is ignored by cuGraph-DGL") if prefetch_node_feats: @@ -146,6 +141,8 @@ def __init__( if fused: warnings.warn("'fused' is ignored by cuGraph-DGL") + self.__prob_attr = prob + self.fanouts = fanouts_per_layer reverse_fanouts = fanouts_per_layer.copy() reverse_fanouts.reverse() @@ -180,8 +177,14 @@ def sample( format=kwargs.pop("format", "parquet"), ) - ds = UniformNeighborSampler( - g._graph(self.edge_dir), + sampling_clx = ( + UniformNeighborSampler + if self.__prob_attr is None + else BiasedNeighborSampler + ) + + ds = sampling_clx( + g._graph(self.edge_dir, prob_attr=self.__prob_attr), writer, compression="CSR", fanout=self._reversed_fanout_vals, diff --git a/python/cugraph-dgl/cugraph_dgl/graph.py b/python/cugraph-dgl/cugraph_dgl/graph.py index 011ab736d00..138e645838a 100644 --- a/python/cugraph-dgl/cugraph_dgl/graph.py +++ b/python/cugraph-dgl/cugraph_dgl/graph.py @@ -312,7 +312,7 @@ def add_edges( self.__graph = None self.__vertex_offsets = None - def num_nodes(self, ntype: str = None) -> int: + def num_nodes(self, ntype: Optional[str] = None) -> int: """ Returns the number of nodes of ntype, or if ntype is not provided, the total number of nodes in the graph. @@ -322,7 +322,7 @@ def num_nodes(self, ntype: str = None) -> int: return self.__num_nodes_dict[ntype] - def number_of_nodes(self, ntype: str = None) -> int: + def number_of_nodes(self, ntype: Optional[str] = None) -> int: """ Alias for num_nodes. """ @@ -381,7 +381,7 @@ def _vertex_offsets(self) -> Dict[str, int]: return dict(self.__vertex_offsets) - def __get_edgelist(self) -> Dict[str, "torch.Tensor"]: + def __get_edgelist(self, prob_attr=None) -> Dict[str, "torch.Tensor"]: """ This function always returns src/dst labels with respect to the out direction. @@ -431,63 +431,71 @@ def __get_edgelist(self) -> Dict[str, "torch.Tensor"]: ) ) + num_edges_t = torch.tensor( + [self.__edge_indices[et].shape[1] for et in sorted_keys], device="cuda" + ) + if self.is_multi_gpu: rank = torch.distributed.get_rank() world_size = torch.distributed.get_world_size() - num_edges_t = torch.tensor( - [self.__edge_indices[et].shape[1] for et in sorted_keys], device="cuda" - ) num_edges_all_t = torch.empty( world_size, num_edges_t.numel(), dtype=torch.int64, device="cuda" ) torch.distributed.all_gather_into_tensor(num_edges_all_t, num_edges_t) - if rank > 0: - start_offsets = num_edges_all_t[:rank].T.sum(axis=1) - edge_id_array = torch.concat( + start_offsets = num_edges_all_t[:rank].T.sum(axis=1) + + else: + rank = 0 + start_offsets = torch.zeros( + (len(sorted_keys),), dtype=torch.int64, device="cuda" + ) + num_edges_all_t = num_edges_t.reshape((1, num_edges_t.numel())) + + # Use pinned memory here for fast access to CPU/WG storage + edge_id_array_per_type = [ + torch.arange( + start_offsets[i], + start_offsets[i] + num_edges_all_t[rank][i], + dtype=torch.int64, + device="cpu", + ).pin_memory() + for i in range(len(sorted_keys)) + ] + + # Retrieve the weights from the appropriate feature(s) + # DGL implicitly requires all edge types use the same + # feature name. + if prob_attr is None: + weights = None + else: + if len(sorted_keys) > 1: + weights = torch.concat( [ - torch.arange( - start_offsets[i], - start_offsets[i] + num_edges_all_t[rank][i], - dtype=torch.int64, - device="cuda", - ) - for i in range(len(sorted_keys)) + self.edata[prob_attr][sorted_keys[i]][ix] + for i, ix in enumerate(edge_id_array_per_type) ] ) else: - edge_id_array = torch.concat( - [ - torch.arange( - self.__edge_indices[et].shape[1], - dtype=torch.int64, - device="cuda", - ) - for et in sorted_keys - ] - ) + weights = self.edata[prob_attr][edge_id_array_per_type[0]] - else: - # single GPU - edge_id_array = torch.concat( - [ - torch.arange( - self.__edge_indices[et].shape[1], - dtype=torch.int64, - device="cuda", - ) - for et in sorted_keys - ] - ) + # Safe to move this to cuda because the consumer will always + # move it to cuda if it isn't already there. + edge_id_array = torch.concat(edge_id_array_per_type).cuda() - return { + edgelist_dict = { "src": edge_index[0], "dst": edge_index[1], "etp": edge_type_array, "eid": edge_id_array, } + if weights is not None: + edgelist_dict["wgt"] = weights + + return edgelist_dict + @property def is_homogeneous(self): return len(self.__num_edges_dict) <= 1 and len(self.__num_nodes_dict) <= 1 @@ -508,7 +516,9 @@ def _resource_handle(self): return self.__handle def _graph( - self, direction: str + self, + direction: str, + prob_attr: Optional[str] = None, ) -> Union[pylibcugraph.SGGraph, pylibcugraph.MGGraph]: """ Gets the pylibcugraph Graph object with edges pointing in the given direction @@ -522,12 +532,16 @@ def _graph( is_multigraph=True, is_symmetric=False ) - if self.__graph is not None and self.__graph[1] != direction: - self.__graph = None + if self.__graph is not None: + if ( + self.__graph["direction"] != direction + or self.__graph["prob_attr"] != prob_attr + ): + self.__graph = None if self.__graph is None: src_col, dst_col = ("src", "dst") if direction == "out" else ("dst", "src") - edgelist_dict = self.__get_edgelist() + edgelist_dict = self.__get_edgelist(prob_attr=prob_attr) if self.is_multi_gpu: rank = torch.distributed.get_rank() @@ -536,33 +550,35 @@ def _graph( vertices_array = cupy.arange(self.num_nodes(), dtype="int64") vertices_array = cupy.array_split(vertices_array, world_size)[rank] - self.__graph = ( - pylibcugraph.MGGraph( - self._resource_handle, - graph_properties, - [cupy.asarray(edgelist_dict[src_col]).astype("int64")], - [cupy.asarray(edgelist_dict[dst_col]).astype("int64")], - vertices_array=[vertices_array], - edge_id_array=[cupy.asarray(edgelist_dict["eid"])], - edge_type_array=[cupy.asarray(edgelist_dict["etp"])], - ), - direction, + graph = pylibcugraph.MGGraph( + self._resource_handle, + graph_properties, + [cupy.asarray(edgelist_dict[src_col]).astype("int64")], + [cupy.asarray(edgelist_dict[dst_col]).astype("int64")], + vertices_array=[vertices_array], + edge_id_array=[cupy.asarray(edgelist_dict["eid"])], + edge_type_array=[cupy.asarray(edgelist_dict["etp"])], + weight_array=[cupy.asarray(edgelist_dict["wgt"])] + if "wgt" in edgelist_dict + else None, ) else: - self.__graph = ( - pylibcugraph.SGGraph( - self._resource_handle, - graph_properties, - cupy.asarray(edgelist_dict[src_col]).astype("int64"), - cupy.asarray(edgelist_dict[dst_col]).astype("int64"), - vertices_array=cupy.arange(self.num_nodes(), dtype="int64"), - edge_id_array=cupy.asarray(edgelist_dict["eid"]), - edge_type_array=cupy.asarray(edgelist_dict["etp"]), - ), - direction, + graph = pylibcugraph.SGGraph( + self._resource_handle, + graph_properties, + cupy.asarray(edgelist_dict[src_col]).astype("int64"), + cupy.asarray(edgelist_dict[dst_col]).astype("int64"), + vertices_array=cupy.arange(self.num_nodes(), dtype="int64"), + edge_id_array=cupy.asarray(edgelist_dict["eid"]), + edge_type_array=cupy.asarray(edgelist_dict["etp"]), + weight_array=cupy.asarray(edgelist_dict["wgt"]) + if "wgt" in edgelist_dict + else None, ) - return self.__graph[0] + self.__graph = {"graph": graph, "direction": direction, "prob_attr": prob_attr} + + return self.__graph["graph"] def _has_n_emb(self, ntype: str, emb_name: str) -> bool: return (ntype, emb_name) in self.__ndata_storage diff --git a/python/cugraph-dgl/cugraph_dgl/tests/dataloading/test_dataloader.py b/python/cugraph-dgl/cugraph_dgl/tests/dataloading/test_dataloader.py index ef47875463d..419ec7790a9 100644 --- a/python/cugraph-dgl/cugraph_dgl/tests/dataloading/test_dataloader.py +++ b/python/cugraph-dgl/cugraph_dgl/tests/dataloading/test_dataloader.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + import cugraph_dgl.dataloading import pytest @@ -48,9 +49,12 @@ def test_dataloader_basic_homogeneous(): assert len(out_t) <= 2 -def sample_dgl_graphs(g, train_nid, fanouts, batch_size=1): +def sample_dgl_graphs(g, train_nid, fanouts, batch_size=1, prob_attr=None): # Single fanout to match cugraph - sampler = dgl.dataloading.NeighborSampler(fanouts) + sampler = dgl.dataloading.NeighborSampler( + fanouts, + prob=prob_attr, + ) dataloader = dgl.dataloading.DataLoader( g, train_nid, @@ -71,8 +75,13 @@ def sample_dgl_graphs(g, train_nid, fanouts, batch_size=1): return dgl_output -def sample_cugraph_dgl_graphs(cugraph_g, train_nid, fanouts, batch_size=1): - sampler = cugraph_dgl.dataloading.NeighborSampler(fanouts) +def sample_cugraph_dgl_graphs( + cugraph_g, train_nid, fanouts, batch_size=1, prob_attr=None +): + sampler = cugraph_dgl.dataloading.NeighborSampler( + fanouts, + prob=prob_attr, + ) dataloader = cugraph_dgl.dataloading.FutureDataLoader( cugraph_g, @@ -126,3 +135,41 @@ def test_same_homogeneousgraph_results(ix, batch_size): dgl_output[0]["blocks"][0].num_edges() == cugraph_output[0]["blocks"][0].num_edges() ) + + +@pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +@pytest.mark.skipif(isinstance(dgl, MissingModule), reason="dgl not available") +def test_dataloader_biased_homogeneous(): + src = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8]) + dst = torch.tensor([0, 0, 0, 0, 1, 1, 1, 1]) + wgt = torch.tensor([1, 1, 2, 0, 0, 0, 2, 1], dtype=torch.float32) + + train_nid = torch.tensor([0, 1]) + # Create a heterograph with 3 node types and 3 edges types. + dgl_g = dgl.graph((src, dst)) + dgl_g.edata["wgt"] = wgt + + cugraph_g = cugraph_dgl.Graph(is_multi_gpu=False) + cugraph_g.add_nodes(9) + cugraph_g.add_edges(u=src, v=dst, data={"wgt": wgt}) + + dgl_output = sample_dgl_graphs(dgl_g, train_nid, [4], batch_size=2, prob_attr="wgt") + cugraph_output = sample_cugraph_dgl_graphs( + cugraph_g, train_nid, [4], batch_size=2, prob_attr="wgt" + ) + + cugraph_output_nodes = cugraph_output[0]["output_nodes"].cpu().numpy() + dgl_output_nodes = dgl_output[0]["output_nodes"].cpu().numpy() + + np.testing.assert_array_equal( + np.sort(cugraph_output_nodes), np.sort(dgl_output_nodes) + ) + assert ( + dgl_output[0]["blocks"][0].num_dst_nodes() + == cugraph_output[0]["blocks"][0].num_dst_nodes() + ) + assert ( + dgl_output[0]["blocks"][0].num_edges() + == cugraph_output[0]["blocks"][0].num_edges() + ) + assert 5 == cugraph_output[0]["blocks"][0].num_edges() diff --git a/python/cugraph-dgl/cugraph_dgl/tests/dataloading/test_dataloader_mg.py b/python/cugraph-dgl/cugraph_dgl/tests/dataloading/test_dataloader_mg.py index b32233f16a6..061f4fa2077 100644 --- a/python/cugraph-dgl/cugraph_dgl/tests/dataloading/test_dataloader_mg.py +++ b/python/cugraph-dgl/cugraph_dgl/tests/dataloading/test_dataloader_mg.py @@ -82,9 +82,18 @@ def test_dataloader_basic_homogeneous(): ) -def sample_dgl_graphs(g, train_nid, fanouts, batch_size=1): +def sample_dgl_graphs( + g, + train_nid, + fanouts, + batch_size=1, + prob_attr=None, +): # Single fanout to match cugraph - sampler = dgl.dataloading.NeighborSampler(fanouts) + sampler = dgl.dataloading.NeighborSampler( + fanouts, + prob=prob_attr, + ) dataloader = dgl.dataloading.DataLoader( g, train_nid, @@ -105,8 +114,17 @@ def sample_dgl_graphs(g, train_nid, fanouts, batch_size=1): return dgl_output -def sample_cugraph_dgl_graphs(cugraph_g, train_nid, fanouts, batch_size=1): - sampler = cugraph_dgl.dataloading.NeighborSampler(fanouts) +def sample_cugraph_dgl_graphs( + cugraph_g, + train_nid, + fanouts, + batch_size=1, + prob_attr=None, +): + sampler = cugraph_dgl.dataloading.NeighborSampler( + fanouts, + prob=prob_attr, + ) dataloader = cugraph_dgl.dataloading.FutureDataLoader( cugraph_g, @@ -179,3 +197,58 @@ def test_same_homogeneousgraph_results_mg(ix, batch_size): args=(world_size, uid, ix, batch_size), nprocs=world_size, ) + + +def run_test_dataloader_biased_homogeneous(rank, world_size, uid): + init_pytorch_worker(rank, world_size, uid, True) + + src = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8]) + (rank * 9) + dst = torch.tensor([0, 0, 0, 0, 1, 1, 1, 1]) + (rank * 9) + wgt = torch.tensor( + [0.1, 0.1, 0.2, 0, 0, 0, 0.2, 0.1] * world_size, dtype=torch.float32 + ) + + train_nid = torch.tensor([0, 1]) + (rank * 9) + # Create a heterograph with 3 node types and 3 edge types. + dgl_g = dgl.graph((src, dst)) + dgl_g.edata["wgt"] = wgt[:8] + + cugraph_g = cugraph_dgl.Graph(is_multi_gpu=True) + cugraph_g.add_nodes(9 * world_size) + cugraph_g.add_edges(u=src, v=dst, data={"wgt": wgt}) + + dgl_output = sample_dgl_graphs(dgl_g, train_nid, [4], batch_size=2, prob_attr="wgt") + cugraph_output = sample_cugraph_dgl_graphs( + cugraph_g, train_nid, [4], batch_size=2, prob_attr="wgt" + ) + + cugraph_output_nodes = cugraph_output[0]["output_nodes"].cpu().numpy() + dgl_output_nodes = dgl_output[0]["output_nodes"].cpu().numpy() + + np.testing.assert_array_equal( + np.sort(cugraph_output_nodes), np.sort(dgl_output_nodes) + ) + assert ( + dgl_output[0]["blocks"][0].num_dst_nodes() + == cugraph_output[0]["blocks"][0].num_dst_nodes() + ) + assert ( + dgl_output[0]["blocks"][0].num_edges() + == cugraph_output[0]["blocks"][0].num_edges() + ) + + assert 5 == cugraph_output[0]["blocks"][0].num_edges() + + +@pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +@pytest.mark.skipif(isinstance(dgl, MissingModule), reason="dgl not available") +def test_dataloader_biased_homogeneous_mg(): + uid = cugraph_comms_create_unique_id() + # Limit the number of GPUs this test is run with + world_size = torch.cuda.device_count() + + torch.multiprocessing.spawn( + run_test_dataloader_biased_homogeneous, + args=(world_size, uid), + nprocs=world_size, + ) diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index a49139961fd..52ffd8fadfd 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -776,7 +776,7 @@ def sample_batches( label_to_output_comm_rank=cupy.asarray(label_to_output_comm_rank), h_fan_out=np.array(self.__fanout, dtype="int32"), with_replacement=self.__with_replacement, - do_expensive_check=True, + do_expensive_check=False, with_edge_properties=True, random_state=random_state + rank, prior_sources_behavior=self.__prior_sources_behavior, diff --git a/python/pylibcugraph/pylibcugraph/biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/biased_neighbor_sample.pyx index 77f1f04c394..2dd138d5d06 100644 --- a/python/pylibcugraph/pylibcugraph/biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/biased_neighbor_sample.pyx @@ -118,7 +118,7 @@ def biased_neighbor_sample(ResourceHandle resource_handle, Device array containing the list of starting vertices for sampling. h_fan_out: numpy array type - Host array containing the brancing out (fan-out) degrees per + Host array containing the branching out (fan-out) degrees per starting vertex for each hop level. with_replacement: bool diff --git a/python/pylibcugraph/pylibcugraph/uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/uniform_neighbor_sample.pyx index c25c9119985..f3e2336d8f6 100644 --- a/python/pylibcugraph/pylibcugraph/uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/uniform_neighbor_sample.pyx @@ -117,7 +117,7 @@ def uniform_neighbor_sample(ResourceHandle resource_handle, Device array containing the list of starting vertices for sampling. h_fan_out: numpy array type - Host array containing the brancing out (fan-out) degrees per + Host array containing the branching out (fan-out) degrees per starting vertex for each hop level. with_replacement: bool