From 800930903c2e6d7078504448c3c8414ad58de3fb Mon Sep 17 00:00:00 2001 From: Joseph Nke <76006812+jnke2016@users.noreply.github.com> Date: Wed, 11 Sep 2024 02:20:50 +0100 Subject: [PATCH 1/8] Update edge triangle count to call a non detail primitive (#4630) Create a non detail primitive that iterates over each input vertex pair and returns the common destination neighbor list pair in a CSR-like format. closes #3475 Authors: - Joseph Nke (https://github.com/jnke2016) Approvers: - Chuck Hastings (https://github.com/ChuckHastings) - Seunghwa Kang (https://github.com/seunghwak) URL: https://github.com/rapidsai/cugraph/pull/4630 --- cpp/include/cugraph/algorithms.hpp | 6 +- .../community/edge_triangle_count_impl.cuh | 24 +++---- .../edge_triangle_count_mg_v32_e32.cu | 3 +- .../edge_triangle_count_mg_v32_e64.cu | 3 +- .../edge_triangle_count_mg_v64_e64.cu | 3 +- .../edge_triangle_count_sg_v32_e32.cu | 3 +- .../edge_triangle_count_sg_v32_e64.cu | 3 +- .../edge_triangle_count_sg_v64_e64.cu | 3 +- .../prims/per_v_pair_dst_nbr_intersection.cuh | 64 +++++++++++++++++++ 9 files changed, 93 insertions(+), 19 deletions(-) create mode 100644 cpp/src/prims/per_v_pair_dst_nbr_intersection.cuh diff --git a/cpp/include/cugraph/algorithms.hpp b/cpp/include/cugraph/algorithms.hpp index faeb7ad8f83..ed42460ed8e 100644 --- a/cpp/include/cugraph/algorithms.hpp +++ b/cpp/include/cugraph/algorithms.hpp @@ -1873,12 +1873,16 @@ void triangle_count(raft::handle_t const& handle, * @param handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, and * handles to various CUDA libraries) to run graph algorithms. * @param graph_view Graph view object. + * * @param do_expensive_check A flag to run expensive checks for input arguments (if set to + * `true`). * * @return edge_property_t containing the edge triangle count */ template edge_property_t, edge_t> edge_triangle_count( - raft::handle_t const& handle, graph_view_t const& graph_view); + raft::handle_t const& handle, + graph_view_t const& graph_view, + bool do_expensive_check = false); /* * @brief Compute K-Truss. diff --git a/cpp/src/community/edge_triangle_count_impl.cuh b/cpp/src/community/edge_triangle_count_impl.cuh index 225687c4cf0..e3501065008 100644 --- a/cpp/src/community/edge_triangle_count_impl.cuh +++ b/cpp/src/community/edge_triangle_count_impl.cuh @@ -18,8 +18,8 @@ #include "detail/graph_partition_utils.cuh" #include "prims/edge_bucket.cuh" +#include "prims/per_v_pair_dst_nbr_intersection.cuh" #include "prims/transform_e.cuh" -#include "prims/transform_reduce_dst_nbr_intersection_of_e_endpoints_by_v.cuh" #include #include @@ -124,7 +124,8 @@ struct extract_q_r { template edge_property_t, edge_t> edge_triangle_count_impl( raft::handle_t const& handle, - graph_view_t const& graph_view) + graph_view_t const& graph_view, + bool do_expensive_check) { using weight_t = float; rmm::device_uvector edgelist_srcs(0, handle.get_stream()); @@ -158,14 +159,11 @@ edge_property_t, edge_t> edge_t num_remaining_edges -= chunk_size; // Perform 'nbr_intersection' in chunks to reduce peak memory. auto [intersection_offsets, intersection_indices] = - detail::nbr_intersection(handle, - graph_view, - cugraph::edge_dummy_property_t{}.view(), - edge_first + prev_chunk_size, - edge_first + prev_chunk_size + chunk_size, - std::array{true, true}, - false /*FIXME: pass 'do_expensive_check' as argument*/); - + per_v_pair_dst_nbr_intersection(handle, + graph_view, + edge_first + prev_chunk_size, + edge_first + prev_chunk_size + chunk_size, + do_expensive_check); // Update the number of triangles of each (p, q) edges by looking at their intersection // size thrust::for_each( @@ -365,9 +363,11 @@ edge_property_t, edge_t> edge_t template edge_property_t, edge_t> edge_triangle_count( - raft::handle_t const& handle, graph_view_t const& graph_view) + raft::handle_t const& handle, + graph_view_t const& graph_view, + bool do_expensive_check) { - return detail::edge_triangle_count_impl(handle, graph_view); + return detail::edge_triangle_count_impl(handle, graph_view, do_expensive_check); } } // namespace cugraph diff --git a/cpp/src/community/edge_triangle_count_mg_v32_e32.cu b/cpp/src/community/edge_triangle_count_mg_v32_e32.cu index 1212a13323b..5e333139ddf 100644 --- a/cpp/src/community/edge_triangle_count_mg_v32_e32.cu +++ b/cpp/src/community/edge_triangle_count_mg_v32_e32.cu @@ -20,6 +20,7 @@ namespace cugraph { // SG instantiation template edge_property_t, int32_t> edge_triangle_count( raft::handle_t const& handle, - cugraph::graph_view_t const& graph_view); + cugraph::graph_view_t const& graph_view, + bool do_expensive_check); } // namespace cugraph diff --git a/cpp/src/community/edge_triangle_count_mg_v32_e64.cu b/cpp/src/community/edge_triangle_count_mg_v32_e64.cu index 64ee195c7ee..adab2d1fede 100644 --- a/cpp/src/community/edge_triangle_count_mg_v32_e64.cu +++ b/cpp/src/community/edge_triangle_count_mg_v32_e64.cu @@ -20,6 +20,7 @@ namespace cugraph { // SG instantiation template edge_property_t, int64_t> edge_triangle_count( raft::handle_t const& handle, - cugraph::graph_view_t const& graph_view); + cugraph::graph_view_t const& graph_view, + bool do_expensive_check); } // namespace cugraph diff --git a/cpp/src/community/edge_triangle_count_mg_v64_e64.cu b/cpp/src/community/edge_triangle_count_mg_v64_e64.cu index 67c19e5ac52..1f321b2149f 100644 --- a/cpp/src/community/edge_triangle_count_mg_v64_e64.cu +++ b/cpp/src/community/edge_triangle_count_mg_v64_e64.cu @@ -20,6 +20,7 @@ namespace cugraph { // SG instantiation template edge_property_t, int64_t> edge_triangle_count( raft::handle_t const& handle, - cugraph::graph_view_t const& graph_view); + cugraph::graph_view_t const& graph_view, + bool do_expensive_check); } // namespace cugraph diff --git a/cpp/src/community/edge_triangle_count_sg_v32_e32.cu b/cpp/src/community/edge_triangle_count_sg_v32_e32.cu index d6a215aa456..3e16a2cf7ef 100644 --- a/cpp/src/community/edge_triangle_count_sg_v32_e32.cu +++ b/cpp/src/community/edge_triangle_count_sg_v32_e32.cu @@ -20,6 +20,7 @@ namespace cugraph { // SG instantiation template edge_property_t, int32_t> edge_triangle_count( raft::handle_t const& handle, - cugraph::graph_view_t const& graph_view); + cugraph::graph_view_t const& graph_view, + bool do_expensive_check); } // namespace cugraph diff --git a/cpp/src/community/edge_triangle_count_sg_v32_e64.cu b/cpp/src/community/edge_triangle_count_sg_v32_e64.cu index e70fa45c257..24a8de868e0 100644 --- a/cpp/src/community/edge_triangle_count_sg_v32_e64.cu +++ b/cpp/src/community/edge_triangle_count_sg_v32_e64.cu @@ -20,6 +20,7 @@ namespace cugraph { // SG instantiation template edge_property_t, int64_t> edge_triangle_count( raft::handle_t const& handle, - cugraph::graph_view_t const& graph_view); + cugraph::graph_view_t const& graph_view, + bool do_expensive_check); } // namespace cugraph diff --git a/cpp/src/community/edge_triangle_count_sg_v64_e64.cu b/cpp/src/community/edge_triangle_count_sg_v64_e64.cu index 849603f781b..81f814df713 100644 --- a/cpp/src/community/edge_triangle_count_sg_v64_e64.cu +++ b/cpp/src/community/edge_triangle_count_sg_v64_e64.cu @@ -20,6 +20,7 @@ namespace cugraph { // SG instantiation template edge_property_t, int64_t> edge_triangle_count( raft::handle_t const& handle, - cugraph::graph_view_t const& graph_view); + cugraph::graph_view_t const& graph_view, + bool do_expensive_check); } // namespace cugraph diff --git a/cpp/src/prims/per_v_pair_dst_nbr_intersection.cuh b/cpp/src/prims/per_v_pair_dst_nbr_intersection.cuh new file mode 100644 index 00000000000..01c76e5085a --- /dev/null +++ b/cpp/src/prims/per_v_pair_dst_nbr_intersection.cuh @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "prims/detail/nbr_intersection.cuh" + +#include + +#include + +#include + +namespace cugraph { + +/** + * @brief Iterate over each input vertex pair and returns the common destination neighbor list + * pair in a CSR-like format + * + * Iterate over every vertex pair; intersect destination neighbor lists of the two vertices in the + * pair and store the result in a CSR-like format + * + * @tparam GraphViewType Type of the passed non-owning graph object. + * @tparam VertexPairIterator Type of the iterator for input vertex pairs. + * @param handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, and + * handles to various CUDA libraries) to run graph algorithms. + * @param graph_view Non-owning graph object. + * @param vertex_pair_first Iterator pointing to the first (inclusive) input vertex pair. + * @param vertex_pair_last Iterator pointing to the last (exclusive) input vertex pair. + * @param do_expensive_check A flag to run expensive checks for input arguments (if set to `true`). + * @return std::tuple Tuple of intersection offsets and indices. + */ +template +std::tuple, rmm::device_uvector> +per_v_pair_dst_nbr_intersection(raft::handle_t const& handle, + GraphViewType const& graph_view, + VertexPairIterator vertex_pair_first, + VertexPairIterator vertex_pair_last, + bool do_expensive_check = false) +{ + static_assert(!GraphViewType::is_storage_transposed); + + return detail::nbr_intersection(handle, + graph_view, + cugraph::edge_dummy_property_t{}.view(), + vertex_pair_first, + vertex_pair_last, + std::array{true, true}, + do_expensive_check); +} + +} // namespace cugraph From 7c86b98dc5cea7b30a5d3af6af942236dbf3d949 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 11 Sep 2024 11:28:24 -0500 Subject: [PATCH 2/8] Add support for Python 3.12 (#4647) Contributes to https://github.com/rapidsai/build-planning/issues/40 This PR adds support for Python 3.12. ## Notes for Reviewers This is part of ongoing work to add Python 3.12 support across RAPIDS. It temporarily introduces a build/test matrix including Python 3.12, from https://github.com/rapidsai/shared-workflows/pull/213. A follow-up PR will revert back to pointing at the `branch-24.10` branch of `shared-workflows` once all RAPIDS repos have added Python 3.12 support. ### This will fail until all dependencies have been updates to Python 3.12 CI here is expected to fail until all of this project's upstream dependencies support Python 3.12. This can be merged whenever all CI jobs are passing. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4647 --- .github/workflows/build.yaml | 32 +++++++------- .github/workflows/pr.yaml | 44 +++++++++---------- .github/workflows/test.yaml | 18 ++++---- dependencies.yaml | 6 ++- .../source/installation/getting_cugraph.md | 2 +- .../source/installation/source_build.md | 2 +- .../cugraph/source/tutorials/basic_cugraph.md | 2 +- .../wholegraph/installation/source_build.md | 2 +- notebooks/demo/nx_cugraph_demo.ipynb | 2 +- python/cugraph-equivariant/pyproject.toml | 1 + python/cugraph-pyg/pyproject.toml | 1 + python/cugraph-service/client/pyproject.toml | 1 + python/cugraph-service/server/pyproject.toml | 1 + python/cugraph/cugraph/testing/resultset.py | 9 +++- python/cugraph/pyproject.toml | 1 + python/nx-cugraph/README.md | 2 +- python/nx-cugraph/pyproject.toml | 3 +- python/pylibcugraph/pyproject.toml | 1 + 18 files changed, 73 insertions(+), 57 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bc489ffd3f0..3ad8aef5820 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -47,7 +47,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -83,7 +83,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -93,7 +93,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -106,7 +106,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -116,7 +116,7 @@ jobs: wheel-build-nx-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -126,7 +126,7 @@ jobs: wheel-publish-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -136,7 +136,7 @@ jobs: wheel-build-cugraph-dgl: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -146,7 +146,7 @@ jobs: wheel-publish-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -156,7 +156,7 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -166,7 +166,7 @@ jobs: wheel-publish-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -175,7 +175,7 @@ jobs: package-name: cugraph-pyg wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -185,7 +185,7 @@ jobs: wheel-publish-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index dacd9a93399..d25a0b81b13 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,29 +34,29 @@ jobs: - wheel-tests-cugraph-equivariant - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: pull-request node_type: cpu32 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 with: build_type: pull-request enable_check_symbols: true @@ -64,19 +64,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: pull-request conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -86,7 +86,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -96,7 +96,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh @@ -107,14 +107,14 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_cugraph.sh @@ -124,35 +124,35 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request script: ci/test_wheel_cugraph.sh wheel-build-nx-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_nx-cugraph.sh wheel-tests-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh wheel-build-cugraph-dgl: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_cugraph-dgl.sh wheel-tests-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request script: ci/test_wheel_cugraph-dgl.sh @@ -160,35 +160,35 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_cugraph-pyg.sh wheel-tests-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_cugraph-equivariant.sh wheel-tests-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request script: ci/test_wheel_cugraph-equivariant.sh matrix_filter: map(select(.ARCH == "amd64")) devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@python-3.12 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 957d29ce72b..32447711811 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibcugraph.sh wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: script: ci/test_wheel_cugraph.sh wheel-tests-nx-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: script: ci/test_wheel_nx-cugraph.sh wheel-tests-cugraph-dgl: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/dependencies.yaml b/dependencies.yaml index 8619b32e929..2c8335868ba 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -489,8 +489,12 @@ dependencies: packages: - python=3.11 - matrix: + py: "3.12" packages: - - python>=3.10,<3.12 + - python=3.12 + - matrix: + packages: + - python>=3.10,<3.13 python_build_rapids: common: - output_types: [conda, pyproject, requirements] diff --git a/docs/cugraph/source/installation/getting_cugraph.md b/docs/cugraph/source/installation/getting_cugraph.md index ff89fffea7c..41ec9a67e1f 100644 --- a/docs/cugraph/source/installation/getting_cugraph.md +++ b/docs/cugraph/source/installation/getting_cugraph.md @@ -45,7 +45,7 @@ conda install -c rapidsai -c conda-forge -c nvidia cugraph cuda-version=12.0 Alternatively, use `cuda-version=11.8` for packages supporting CUDA 11. -Note: This conda installation only applies to Linux and Python versions 3.10/3.11. +Note: This conda installation only applies to Linux and Python versions 3.10/3.11/3.12.
diff --git a/docs/cugraph/source/installation/source_build.md b/docs/cugraph/source/installation/source_build.md index 80f2d97d40d..243a62e5c81 100644 --- a/docs/cugraph/source/installation/source_build.md +++ b/docs/cugraph/source/installation/source_build.md @@ -177,7 +177,7 @@ Run either the C++ or the Python tests with datasets make test ``` -Note: This conda installation only applies to Linux and Python versions 3.10 and 3.11. +Note: This conda installation only applies to Linux and Python versions 3.10, 3.11, and 3.12. ### (OPTIONAL) Set environment variable on activation diff --git a/docs/cugraph/source/tutorials/basic_cugraph.md b/docs/cugraph/source/tutorials/basic_cugraph.md index 6a9b4877b7f..a0c9ad576b2 100644 --- a/docs/cugraph/source/tutorials/basic_cugraph.md +++ b/docs/cugraph/source/tutorials/basic_cugraph.md @@ -5,7 +5,7 @@ CuGraph is part of [Rapids](https://docs.rapids.ai/user-guide) and has the following system requirements: * NVIDIA GPU, Volta architecture or later, with [compute capability](https://developer.nvidia.com/cuda-gpus) 7.0+ * CUDA 11.2, 11.4, 11.5, 11.8, 12.0, 12.2, or 12.5 - * Python version 3.10 or 3.11 + * Python version 3.10, 3.11, or 3.12 * NetworkX >= version 3.3 or newer in order to use use [NetworkX Configs](https://networkx.org/documentation/stable/reference/backends.html#module-networkx.utils.configs) **This is required for use of nx-cuGraph, [see below](#cugraph-using-networkx-code).** ## Installation diff --git a/docs/cugraph/source/wholegraph/installation/source_build.md b/docs/cugraph/source/wholegraph/installation/source_build.md index 33d7c98b28e..7213cbfb096 100644 --- a/docs/cugraph/source/wholegraph/installation/source_build.md +++ b/docs/cugraph/source/wholegraph/installation/source_build.md @@ -176,7 +176,7 @@ Run either the C++ or the Python tests with datasets ``` -Note: This conda installation only applies to Linux and Python versions 3.10 and 3.11. +Note: This conda installation only applies to Linux and Python versions 3.10, 3.11, and 3.12. ## Creating documentation diff --git a/notebooks/demo/nx_cugraph_demo.ipynb b/notebooks/demo/nx_cugraph_demo.ipynb index 2e3c3c861f8..f1ce80aa188 100644 --- a/notebooks/demo/nx_cugraph_demo.ipynb +++ b/notebooks/demo/nx_cugraph_demo.ipynb @@ -20,7 +20,7 @@ "Using `nx-cugraph` with this notebook requires the following: \n", "- NVIDIA GPU, Pascal architecture or later\n", "- CUDA 11.2, 11.4, 11.5, 11.8, or 12.0\n", - "- Python versions 3.10 or 3.11\n", + "- Python versions 3.10, 3.11, or 3.12\n", "- NetworkX >= version 3.2\n", " - _NetworkX 3.0 supports dispatching and is compatible with `nx-cugraph`, but this notebook will demonstrate features added in 3.2_\n", " - At the time of this writing, NetworkX 3.2 is only available from source and can be installed by following the [development version install instructions](https://github.com/networkx/networkx/blob/main/INSTALL.rst#install-the-development-version).\n", diff --git a/python/cugraph-equivariant/pyproject.toml b/python/cugraph-equivariant/pyproject.toml index 66e23bc798e..2cc25885b84 100644 --- a/python/cugraph-equivariant/pyproject.toml +++ b/python/cugraph-equivariant/pyproject.toml @@ -34,6 +34,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ "pylibcugraphops==24.10.*,>=0.0.0a0", diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index fdb484ccdb4..d206d6001cc 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ "cugraph==24.10.*,>=0.0.0a0", diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml index 06366821316..ac5e6bad0d5 100644 --- a/python/cugraph-service/client/pyproject.toml +++ b/python/cugraph-service/client/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.urls] diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index 7144c3cebcb..b9789c201d2 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -38,6 +38,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.scripts] diff --git a/python/cugraph/cugraph/testing/resultset.py b/python/cugraph/cugraph/testing/resultset.py index 9570d7f3e04..f557ad13089 100644 --- a/python/cugraph/cugraph/testing/resultset.py +++ b/python/cugraph/cugraph/testing/resultset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import warnings import tarfile import urllib.request @@ -108,7 +109,11 @@ def load_resultset(resultset_name, resultset_download_url): if not compressed_file_path.exists(): urllib.request.urlretrieve(resultset_download_url, compressed_file_path) tar = tarfile.open(str(compressed_file_path), "r:gz") - tar.extractall(str(curr_resultset_download_dir)) + # TODO: pass filter="fully_trusted" when minimum supported Python version >=3.12 + # ref: https://docs.python.org/3/library/tarfile.html#tarfile-extraction-filter + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + tar.extractall(str(curr_resultset_download_dir)) tar.close() # FIXME: This assumes separator is " ", but should this be configurable? diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index ac7048f37a5..31721c8a568 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -41,6 +41,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.optional-dependencies] diff --git a/python/nx-cugraph/README.md b/python/nx-cugraph/README.md index b18b995161a..c3ca0b880a9 100644 --- a/python/nx-cugraph/README.md +++ b/python/nx-cugraph/README.md @@ -9,7 +9,7 @@ to run supported algorithms with GPU acceleration. nx-cugraph requires the following: * NVIDIA GPU, Volta architecture or later, with [compute capability](https://developer.nvidia.com/cuda-gpus) 7.0+ * CUDA 11.2, 11.4, 11.5, 11.8, 12.0, 12.2, or 12.5 - * Python version 3.10 or 3.11 + * Python version 3.10, 3.11, or 3.12 * NetworkX >= version 3.0 (version 3.2 or higher recommended) More details about system requirements can be found in the [RAPIDS System Requirements documentation](https://docs.rapids.ai/install#system-req). diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index 6784e9607f1..e7b4ea44dd8 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", "Intended Audience :: Developers", "Topic :: Software Development :: Libraries :: Python Modules", @@ -89,7 +90,7 @@ matrix-entry = "cuda_suffixed=true" [tool.black] line-length = 88 -target-version = ["py310", "py311"] +target-version = ["py310", "py311", "py312"] [tool.isort] sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"] diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 45a59afa76c..92c417f0372 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -35,6 +35,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.optional-dependencies] From a4baa5b49dd76ca1a9c5eaaf8a61169dc242ff00 Mon Sep 17 00:00:00 2001 From: Ralph Liu <137829296+nv-rliu@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:01:15 -0400 Subject: [PATCH 3/8] Add `--cpu-only` or `--gpu-only` Arguments to `nx-cugraph` Benchmark (#4648) This PR is a follow-up to https://github.com/rapidsai/cugraph/pull/4616 This PR adds the following CLI argument options for `benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh` - `--cpu-only` - `--gpu-only` And adds minor updates to the README Authors: - Ralph Liu (https://github.com/nv-rliu) Approvers: - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4648 --- benchmarks/nx-cugraph/pytest-based/README.md | 17 +++++++++++++---- .../pytest-based/run-main-benchmarks.sh | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/benchmarks/nx-cugraph/pytest-based/README.md b/benchmarks/nx-cugraph/pytest-based/README.md index 4ea0f127a51..781550fa560 100644 --- a/benchmarks/nx-cugraph/pytest-based/README.md +++ b/benchmarks/nx-cugraph/pytest-based/README.md @@ -19,13 +19,22 @@ Our current benchmarks provide the following datasets: ### Scripts #### 1. `run-main-benchmarks.sh` -This script allows users to run selected algorithms across multiple datasets and backends. All results are stored inside a sub-directory (`logs/`) and output files are named based on the combination of parameters for that benchmark. +This script allows users to run a small set of commonly-used algorithms across multiple datasets and backends. All results are stored inside a sub-directory (`logs/`) and output files are named based on the combination of parameters for that benchmark. -NOTE: If running with all algorithms, datasets, and backends, this script may take a few hours to finish running. +NOTE: If running with all algorithms and datasets using NetworkX without an accelerated backend, this script may take a few hours to finish running. **Usage:** + - Run with `--cpu-only`: ```bash - bash run-main-benchmarks.sh # edit this script directly + ./run-main-benchmarks.sh --cpu-only + ``` + - Run with `--gpu-only`: + ```bash + ./run-main-benchmarks.sh --gpu-only + ``` + - Run without any arguments (all backends): + ```bash + ./run-main-benchmarks.sh ``` #### 2. `get_graph_bench_dataset.py` @@ -37,7 +46,7 @@ This script downloads the specified dataset using `cugraph.datasets`. ``` #### 3. `create_results_summary_page.py` -This script is designed to be run after `run-gap-benchmarks.sh` in order to generate an HTML page displaying a results table comparing default NetworkX to nx-cugraph. The script also provides information about the current system. +This script is designed to be run after `run-gap-benchmarks.sh` in order to generate an HTML page displaying a results table comparing default NetworkX to nx-cugraph. The script also provides information about the current system, so it should be run on the machine on which benchmarks were run. **Usage:** ```bash diff --git a/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh b/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh index 1a81fe4b80a..a1d32474e5f 100755 --- a/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh +++ b/benchmarks/nx-cugraph/pytest-based/run-main-benchmarks.sh @@ -40,10 +40,26 @@ backends=" None cugraph-preconverted " +# check for --cpu-only or --gpu-only args +if [[ "$#" -eq 1 ]]; then + case $1 in + --cpu-only) + backends="None" + ;; + --gpu-only) + backends="cugraph-preconverted" + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +fi for algo in $algos; do for dataset in $datasets; do - python get_graph_bench_dataset.py $dataset + # this script can be used to download benchmarking datasets by name via cugraph.datasets + python get_graph_bench_dataset.py $dataset for backend in $backends; do name="${backend}__${algo}__${dataset}" echo "Running: $backend, $dataset, bench_$algo" From 4d2dd27393ddb7223fc9375e0496c0df981c3954 Mon Sep 17 00:00:00 2001 From: Chuck Hastings <45364586+ChuckHastings@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:12:48 -0400 Subject: [PATCH 4/8] Biased Random Walks and Node2Vec implementation (#4645) Replaces #4499 Closes #4499 @G-Cornett implemented biased random walks and node2vec during his internship. This PR includes his changes to implement those algorithms. Authors: - Chuck Hastings (https://github.com/ChuckHastings) - Garrett Cornett (https://github.com/G-Cornett) Approvers: - Joseph Nke (https://github.com/jnke2016) - Naim (https://github.com/naimnv) - Seunghwa Kang (https://github.com/seunghwak) URL: https://github.com/rapidsai/cugraph/pull/4645 --- cpp/include/cugraph/algorithms.hpp | 12 +- cpp/src/c_api/random_walks.cpp | 34 +- .../sample_and_compute_local_nbr_indices.cuh | 11 +- ...r_v_random_select_transform_outgoing_e.cuh | 6 +- cpp/src/sampling/random_walks_impl.cuh | 897 ++++++++++++++---- cpp/src/sampling/random_walks_mg_v32_e32.cu | 24 +- cpp/src/sampling/random_walks_mg_v32_e64.cu | 24 +- cpp/src/sampling/random_walks_mg_v64_e64.cu | 24 +- cpp/src/sampling/random_walks_sg_v32_e32.cu | 24 +- cpp/src/sampling/random_walks_sg_v32_e64.cu | 24 +- cpp/src/sampling/random_walks_sg_v64_e64.cu | 24 +- cpp/tests/c_api/sg_random_walks_test.c | 89 +- cpp/tests/sampling/mg_random_walks_test.cpp | 20 +- cpp/tests/sampling/random_walks_check.cuh | 2 +- cpp/tests/sampling/sg_random_walks_test.cpp | 64 +- 15 files changed, 899 insertions(+), 380 deletions(-) diff --git a/cpp/include/cugraph/algorithms.hpp b/cpp/include/cugraph/algorithms.hpp index ed42460ed8e..7e5af4ac686 100644 --- a/cpp/include/cugraph/algorithms.hpp +++ b/cpp/include/cugraph/algorithms.hpp @@ -1579,11 +1579,11 @@ std:: template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed = std::numeric_limits::max()); + size_t max_length); /** * @brief returns biased random walks from starting sources, where each path is of given @@ -1623,11 +1623,11 @@ uniform_random_walks(raft::handle_t const& handle, template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed = std::numeric_limits::max()); + size_t max_length); /** * @brief returns biased random walks with node2vec biases from starting sources, @@ -1670,13 +1670,13 @@ biased_random_walks(raft::handle_t const& handle, template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, weight_t p, - weight_t q, - uint64_t seed = std::numeric_limits::max()); + weight_t q); #ifndef NO_CUGRAPH_OPS /** diff --git a/cpp/src/c_api/random_walks.cpp b/cpp/src/c_api/random_walks.cpp index b9a2c8e4f60..705d2108437 100644 --- a/cpp/src/c_api/random_walks.cpp +++ b/cpp/src/c_api/random_walks.cpp @@ -16,6 +16,7 @@ #include "c_api/abstract_functor.hpp" #include "c_api/graph.hpp" +#include "c_api/random.hpp" #include "c_api/resource_handle.hpp" #include "c_api/utils.hpp" @@ -153,10 +154,11 @@ namespace { struct uniform_random_walks_functor : public cugraph::c_api::abstract_functor { raft::handle_t const& handle_; + // FIXME: rng_state_ should be passed as a parameter + cugraph::c_api::cugraph_rng_state_t* rng_state_{nullptr}; cugraph::c_api::cugraph_graph_t* graph_{nullptr}; cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertices_{nullptr}; size_t max_length_{0}; - size_t seed_{0}; cugraph::c_api::cugraph_random_walk_result_t* result_{nullptr}; uniform_random_walks_functor(cugraph_resource_handle_t const* handle, @@ -222,13 +224,17 @@ struct uniform_random_walks_functor : public cugraph::c_api::abstract_functor { graph_view.local_vertex_partition_range_last(), false); + // FIXME: remove once rng_state passed as parameter + rng_state_ = reinterpret_cast( + new cugraph::c_api::cugraph_rng_state_t{raft::random::RngState{0}}); + auto [paths, weights] = cugraph::uniform_random_walks( handle_, + rng_state_->rng_state_, graph_view, (edge_weights != nullptr) ? std::make_optional(edge_weights->view()) : std::nullopt, raft::device_span{start_vertices.data(), start_vertices.size()}, - max_length_, - seed_); + max_length_); // // Need to unrenumber the vertices in the resulting paths @@ -255,11 +261,12 @@ struct uniform_random_walks_functor : public cugraph::c_api::abstract_functor { struct biased_random_walks_functor : public cugraph::c_api::abstract_functor { raft::handle_t const& handle_; + // FIXME: rng_state_ should be passed as a parameter + cugraph::c_api::cugraph_rng_state_t* rng_state_{nullptr}; cugraph::c_api::cugraph_graph_t* graph_{nullptr}; cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertices_{nullptr}; size_t max_length_{0}; cugraph::c_api::cugraph_random_walk_result_t* result_{nullptr}; - uint64_t seed_{0}; biased_random_walks_functor(cugraph_resource_handle_t const* handle, cugraph_graph_t* graph, @@ -326,13 +333,17 @@ struct biased_random_walks_functor : public cugraph::c_api::abstract_functor { graph_view.local_vertex_partition_range_last(), false); + // FIXME: remove once rng_state passed as parameter + rng_state_ = reinterpret_cast( + new cugraph::c_api::cugraph_rng_state_t{raft::random::RngState{0}}); + auto [paths, weights] = cugraph::biased_random_walks( handle_, + rng_state_->rng_state_, graph_view, edge_weights->view(), raft::device_span{start_vertices.data(), start_vertices.size()}, - max_length_, - seed_); + max_length_); // // Need to unrenumber the vertices in the resulting paths @@ -354,12 +365,13 @@ struct biased_random_walks_functor : public cugraph::c_api::abstract_functor { struct node2vec_random_walks_functor : public cugraph::c_api::abstract_functor { raft::handle_t const& handle_; + // FIXME: rng_state_ should be passed as a parameter + cugraph::c_api::cugraph_rng_state_t* rng_state_{nullptr}; cugraph::c_api::cugraph_graph_t* graph_{nullptr}; cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertices_{nullptr}; size_t max_length_{0}; double p_{0}; double q_{0}; - uint64_t seed_{0}; cugraph::c_api::cugraph_random_walk_result_t* result_{nullptr}; node2vec_random_walks_functor(cugraph_resource_handle_t const* handle, @@ -431,15 +443,19 @@ struct node2vec_random_walks_functor : public cugraph::c_api::abstract_functor { graph_view.local_vertex_partition_range_last(), false); + // FIXME: remove once rng_state passed as parameter + rng_state_ = reinterpret_cast( + new cugraph::c_api::cugraph_rng_state_t{raft::random::RngState{0}}); + auto [paths, weights] = cugraph::node2vec_random_walks( handle_, + rng_state_->rng_state_, graph_view, (edge_weights != nullptr) ? std::make_optional(edge_weights->view()) : std::nullopt, raft::device_span{start_vertices.data(), start_vertices.size()}, max_length_, static_cast(p_), - static_cast(q_), - seed_); + static_cast(q_)); // FIXME: Need to fix invalid_vtx issue here. We can't unrenumber max_vertex_id+1 // properly... diff --git a/cpp/src/prims/detail/sample_and_compute_local_nbr_indices.cuh b/cpp/src/prims/detail/sample_and_compute_local_nbr_indices.cuh index 7d4750c0554..a6a164d36c1 100644 --- a/cpp/src/prims/detail/sample_and_compute_local_nbr_indices.cuh +++ b/cpp/src/prims/detail/sample_and_compute_local_nbr_indices.cuh @@ -392,11 +392,11 @@ compute_unique_keys(raft::handle_t const& handle, cuda::proclaim_return_type( [unique_key_first = get_dataframe_buffer_begin(aggregate_local_frontier_unique_keys) + local_frontier_unique_key_displacements[i], - num_unique_keys = local_frontier_unique_key_sizes[i]] __device__(key_t key) { + unique_key_last = get_dataframe_buffer_begin(aggregate_local_frontier_unique_keys) + + local_frontier_unique_key_displacements[i] + + local_frontier_unique_key_sizes[i]] __device__(key_t key) { return static_cast(thrust::distance( - unique_key_first, - thrust::lower_bound( - thrust::seq, unique_key_first, unique_key_first + num_unique_keys, key))); + unique_key_first, thrust::find(thrust::seq, unique_key_first, unique_key_last, key))); })); } @@ -1759,8 +1759,7 @@ biased_sample_and_compute_local_nbr_indices( std::optional> key_indices{std::nullopt}; std::vector local_frontier_sample_offsets{}; if (with_replacement) { - // computet segmented inclusive sums (one segment per seed) - + // compute segmented inclusive sums (one segment per seed) auto unique_key_first = thrust::make_transform_iterator( thrust::make_counting_iterator(size_t{0}), cuda::proclaim_return_type( diff --git a/cpp/src/prims/per_v_random_select_transform_outgoing_e.cuh b/cpp/src/prims/per_v_random_select_transform_outgoing_e.cuh index 03514e52e6e..9d0f711d106 100644 --- a/cpp/src/prims/per_v_random_select_transform_outgoing_e.cuh +++ b/cpp/src/prims/per_v_random_select_transform_outgoing_e.cuh @@ -351,7 +351,7 @@ per_v_random_select_transform_e(raft::handle_t const& handle, uniform_sample_and_compute_local_nbr_indices( handle, graph_view, - (minor_comm_size > 1) ? get_dataframe_buffer_begin(*aggregate_local_frontier) + (minor_comm_size > 1) ? get_dataframe_buffer_cbegin(*aggregate_local_frontier) : frontier.begin(), local_frontier_displacements, local_frontier_sizes, @@ -363,7 +363,7 @@ per_v_random_select_transform_e(raft::handle_t const& handle, biased_sample_and_compute_local_nbr_indices( handle, graph_view, - (minor_comm_size > 1) ? get_dataframe_buffer_begin(*aggregate_local_frontier) + (minor_comm_size > 1) ? get_dataframe_buffer_cbegin(*aggregate_local_frontier) : frontier.begin(), edge_bias_src_value_input, edge_bias_dst_value_input, @@ -392,7 +392,7 @@ per_v_random_select_transform_e(raft::handle_t const& handle, graph_view.local_edge_partition_view(i)); auto edge_partition_frontier_key_first = - ((minor_comm_size > 1) ? get_dataframe_buffer_begin(*aggregate_local_frontier) + ((minor_comm_size > 1) ? get_dataframe_buffer_cbegin(*aggregate_local_frontier) : frontier.begin()) + local_frontier_displacements[i]; auto edge_partition_sample_local_nbr_index_first = diff --git a/cpp/src/sampling/random_walks_impl.cuh b/cpp/src/sampling/random_walks_impl.cuh index d582893d756..6c10fc473f3 100644 --- a/cpp/src/sampling/random_walks_impl.cuh +++ b/cpp/src/sampling/random_walks_impl.cuh @@ -17,7 +17,10 @@ #pragma once #include "detail/graph_partition_utils.cuh" +#include "prims/detail/nbr_intersection.cuh" #include "prims/per_v_random_select_transform_outgoing_e.cuh" +#include "prims/property_op_utils.cuh" +#include "prims/update_edge_src_dst_property.cuh" #include "prims/vertex_frontier.cuh" #include @@ -25,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -46,13 +50,6 @@ namespace cugraph { namespace detail { -inline uint64_t get_current_time_nanoseconds() -{ - auto cur = std::chrono::steady_clock::now(); - return static_cast( - std::chrono::duration_cast(cur.time_since_epoch()).count()); -} - template struct sample_edges_op_t { template @@ -70,21 +67,129 @@ struct sample_edges_op_t { } }; +template +struct biased_random_walk_e_bias_op_t { + __device__ bias_t + operator()(vertex_t, vertex_t, bias_t src_out_weight_sum, thrust::nullopt_t, bias_t weight) const + { + return weight / src_out_weight_sum; + } +}; + +template +struct biased_sample_edges_op_t { + __device__ thrust::tuple operator()( + vertex_t, vertex_t dst, weight_t, thrust::nullopt_t, weight_t weight) const + { + return thrust::make_tuple(dst, weight); + } +}; + +template +struct node2vec_random_walk_e_bias_op_t { + bias_t p_{}; + bias_t q_{}; + raft::device_span intersection_offsets_{}; + raft::device_span intersection_indices_{}; + raft::device_span current_vertices_{}; + raft::device_span prev_vertices_{}; + + // Unweighted Bias Operator + template + __device__ std::enable_if_t, bias_t> operator()( + thrust::tuple tagged_src, + vertex_t dst, + thrust::nullopt_t, + thrust::nullopt_t, + thrust::nullopt_t) const + { + // Check tag (prev vert) for destination + if (dst == thrust::get<1>(tagged_src)) { return 1.0 / p_; } + // Search zipped vertices for tagged src + auto lower_itr = thrust::lower_bound( + thrust::seq, + thrust::make_zip_iterator(current_vertices_.begin(), prev_vertices_.begin()), + thrust::make_zip_iterator(current_vertices_.end(), prev_vertices_.end()), + tagged_src); + auto low_idx = thrust::distance( + thrust::make_zip_iterator(current_vertices_.begin(), prev_vertices_.begin()), lower_itr); + auto intersection_index_first = intersection_indices_.begin() + intersection_offsets_[low_idx]; + auto intersection_index_last = + intersection_indices_.begin() + intersection_offsets_[low_idx + 1]; + auto itr = + thrust::lower_bound(thrust::seq, intersection_index_first, intersection_index_last, dst); + return (itr != intersection_index_last && *itr == dst) ? 1.0 : 1.0 / q_; + } + + // Weighted Bias Operator + template + __device__ std::enable_if_t, bias_t> operator()( + thrust::tuple tagged_src, + vertex_t dst, + thrust::nullopt_t, + thrust::nullopt_t, + W) const + { + // Check tag (prev vert) for destination + if (dst == thrust::get<1>(tagged_src)) { return 1.0 / p_; } + // Search zipped vertices for tagged src + auto lower_itr = thrust::lower_bound( + thrust::seq, + thrust::make_zip_iterator(current_vertices_.begin(), prev_vertices_.begin()), + thrust::make_zip_iterator(current_vertices_.end(), prev_vertices_.end()), + tagged_src); + auto low_idx = thrust::distance( + thrust::make_zip_iterator(current_vertices_.begin(), prev_vertices_.begin()), lower_itr); + auto intersection_index_first = intersection_indices_.begin() + intersection_offsets_[low_idx]; + auto intersection_index_last = + intersection_indices_.begin() + intersection_offsets_[low_idx + 1]; + auto itr = + thrust::lower_bound(thrust::seq, intersection_index_first, intersection_index_last, dst); + return (itr != intersection_index_last && *itr == dst) ? 1.0 : 1.0 / q_; + } +}; + +template +struct node2vec_sample_edges_op_t { + template + __device__ std::enable_if_t, vertex_t> operator()( + thrust::tuple tagged_src, + vertex_t dst, + thrust::nullopt_t, + thrust::nullopt_t, + thrust::nullopt_t) const + { + return dst; + } + + template + __device__ std::enable_if_t, thrust::tuple> operator()( + thrust::tuple tagged_src, + vertex_t dst, + thrust::nullopt_t, + thrust::nullopt_t, + W w) const + { + return thrust::make_tuple(dst, w); + } +}; + template struct uniform_selector { - raft::random::RngState rng_state_; - - uniform_selector(uint64_t seed) : rng_state_(seed) {} + raft::random::RngState& rng_state_; + static constexpr bool is_second_order_ = false; template std::tuple, + std::optional>, std::optional>> follow_random_edge( raft::handle_t const& handle, GraphViewType const& graph_view, std::optional> edge_weight_view, - rmm::device_uvector const& current_vertices) + rmm::device_uvector&& current_vertices, + std::optional>&& previous_vertices) { using vertex_t = typename GraphViewType::vertex_type; @@ -133,30 +238,67 @@ struct uniform_selector { minors = std::move(sample_e_op_results); } - return std::make_tuple(std::move(minors), std::move(weights)); + return std::make_tuple(std::move(minors), std::move(previous_vertices), std::move(weights)); } }; template struct biased_selector { - uint64_t seed_{0}; + raft::random::RngState& rng_state_; + static constexpr bool is_second_order_ = false; template std::tuple, + std::optional>, std::optional>> follow_random_edge( raft::handle_t const& handle, GraphViewType const& graph_view, std::optional> edge_weight_view, - rmm::device_uvector const& current_vertices) + rmm::device_uvector&& current_vertices, + std::optional>&& previous_vertices) { - // To do biased sampling, I need out_weights instead of out_degrees. - // Then I generate a random float between [0, out_weights[v]). Then - // instead of making a decision based on the index I need to find - // upper_bound (or is it lower_bound) of the random number and - // the cumulative weight. - CUGRAPH_FAIL("biased sampling not implemented"); + // Create vertex frontier + using vertex_t = typename GraphViewType::vertex_type; + + using tag_t = void; + + cugraph::vertex_frontier_t vertex_frontier( + handle, 1); + + vertex_frontier.bucket(0).insert(current_vertices.begin(), current_vertices.end()); + + auto vertex_weight_sum = compute_out_weight_sums(handle, graph_view, *edge_weight_view); + edge_src_property_t edge_src_out_weight_sums(handle, graph_view); + update_edge_src_property(handle, + graph_view, + vertex_frontier.bucket(0).begin(), + vertex_frontier.bucket(0).end(), + vertex_weight_sum.data(), + edge_src_out_weight_sums.mutable_view()); + auto [sample_offsets, sample_e_op_results] = cugraph::per_v_random_select_transform_outgoing_e( + handle, + graph_view, + vertex_frontier.bucket(0), + edge_src_out_weight_sums.view(), + cugraph::edge_dst_dummy_property_t{}.view(), + *edge_weight_view, + biased_random_walk_e_bias_op_t{}, + edge_src_out_weight_sums.view(), + cugraph::edge_dst_dummy_property_t{}.view(), + *edge_weight_view, + biased_sample_edges_op_t{}, + rng_state_, + size_t{1}, + true, + std::make_optional( + thrust::make_tuple(vertex_t{cugraph::invalid_vertex_id::value}, weight_t{0.0}))); + + // Return results + return std::make_tuple(std::move(std::get<0>(sample_e_op_results)), + std::move(previous_vertices), + std::move(std::get<1>(sample_e_op_results))); } }; @@ -164,26 +306,232 @@ template struct node2vec_selector { weight_t p_; weight_t q_; - uint64_t seed_{0}; + raft::random::RngState& rng_state_; + static constexpr bool is_second_order_ = true; template std::tuple, + std::optional>, std::optional>> follow_random_edge( raft::handle_t const& handle, GraphViewType const& graph_view, std::optional> edge_weight_view, - rmm::device_uvector const& current_vertices) + rmm::device_uvector&& current_vertices, + std::optional>&& previous_vertices) { - // To do node2vec, I need the following: - // 1) transform_reduce_dst_nbr_intersection_of_e_endpoints_by_v to compute the sum of the - // node2vec style weights - // 2) Generate a random number between [0, output_from_trdnioeebv[v]) - // 3) a sampling value that lets me pick the correct edge based on the same computation - // (essentially weighted sampling, but with a function that computes the weight rather - // than just using the edge weights) - CUGRAPH_FAIL("node2vec not implemented"); + // Create vertex frontier + using vertex_t = typename GraphViewType::vertex_type; + + using tag_t = vertex_t; + + // Zip previous and current vertices for nbr_intersection() + auto intersection_pairs = + thrust::make_zip_iterator(current_vertices.begin(), (*previous_vertices).begin()); + + auto [intersection_offsets, intersection_indices] = + detail::nbr_intersection(handle, + graph_view, + cugraph::edge_dummy_property_t{}.view(), + intersection_pairs, + intersection_pairs + current_vertices.size(), + std::array{true, true}, + false); + + rmm::device_uvector intersection_counts(size_t{0}, handle.get_stream()); + rmm::device_uvector aggregate_offsets(size_t{0}, handle.get_stream()); + rmm::device_uvector aggregate_currents(size_t{0}, handle.get_stream()); + rmm::device_uvector aggregate_previous(size_t{0}, handle.get_stream()); + rmm::device_uvector aggregate_indices(size_t{0}, handle.get_stream()); + + // Aggregate intersection data across minor comm + if constexpr (GraphViewType::is_multi_gpu) { + intersection_counts.resize(intersection_offsets.size(), handle.get_stream()); + thrust::adjacent_difference(handle.get_thrust_policy(), + intersection_offsets.begin(), + intersection_offsets.end(), + intersection_counts.begin()); + + auto recv_counts = cugraph::host_scalar_allgather( + handle.get_subcomm(cugraph::partition_manager::minor_comm_name()), + current_vertices.size(), + handle.get_stream()); + + std::vector displacements(recv_counts.size()); + std::exclusive_scan(recv_counts.begin(), recv_counts.end(), displacements.begin(), size_t{0}); + + aggregate_offsets.resize(displacements.back() + recv_counts.back() + 1, handle.get_stream()); + aggregate_offsets.set_element_to_zero_async(aggregate_offsets.size() - 1, + handle.get_stream()); + + cugraph::device_allgatherv(handle.get_subcomm(cugraph::partition_manager::minor_comm_name()), + intersection_counts.begin() + 1, + aggregate_offsets.begin(), + recv_counts, + displacements, + handle.get_stream()); + + thrust::exclusive_scan(handle.get_thrust_policy(), + aggregate_offsets.begin(), + aggregate_offsets.end(), + aggregate_offsets.begin()); + + aggregate_currents.resize(displacements.back() + recv_counts.back(), handle.get_stream()); + + cugraph::device_allgatherv(handle.get_subcomm(cugraph::partition_manager::minor_comm_name()), + current_vertices.begin(), + aggregate_currents.begin(), + recv_counts, + displacements, + handle.get_stream()); + + aggregate_previous.resize(displacements.back() + recv_counts.back(), handle.get_stream()); + + cugraph::device_allgatherv(handle.get_subcomm(cugraph::partition_manager::minor_comm_name()), + (*previous_vertices).begin(), + aggregate_previous.begin(), + recv_counts, + displacements, + handle.get_stream()); + + recv_counts = cugraph::host_scalar_allgather( + handle.get_subcomm(cugraph::partition_manager::minor_comm_name()), + intersection_offsets.back_element(handle.get_stream()), + handle.get_stream()); + + displacements.resize(recv_counts.size()); + std::exclusive_scan(recv_counts.begin(), recv_counts.end(), displacements.begin(), size_t{0}); + + aggregate_indices.resize(displacements.back() + recv_counts.back(), handle.get_stream()); + + cugraph::device_allgatherv(handle.get_subcomm(cugraph::partition_manager::minor_comm_name()), + intersection_indices.begin(), + aggregate_indices.begin(), + recv_counts, + displacements, + handle.get_stream()); + } + + cugraph::vertex_frontier_t vertex_frontier( + handle, 1); + vertex_frontier.bucket(0).insert( + thrust::make_zip_iterator(current_vertices.begin(), (*previous_vertices).begin()), + thrust::make_zip_iterator(current_vertices.end(), (*previous_vertices).end())); + + // Create data structs for results + rmm::device_uvector minors(0, handle.get_stream()); + std::optional> weights{std::nullopt}; + + if (edge_weight_view) { + auto [sample_offsets, sample_e_op_results] = + cugraph::per_v_random_select_transform_outgoing_e( + handle, + graph_view, + vertex_frontier.bucket(0), + cugraph::edge_src_dummy_property_t{}.view(), + cugraph::edge_dst_dummy_property_t{}.view(), + *edge_weight_view, + GraphViewType::is_multi_gpu + ? node2vec_random_walk_e_bias_op_t{p_, + q_, + raft::device_span( + aggregate_offsets.data(), + aggregate_offsets.size()), + raft::device_span( + aggregate_indices.data(), + aggregate_indices.size()), + raft::device_span( + aggregate_currents.data(), + aggregate_currents.size()), + raft::device_span( + aggregate_previous.data(), + aggregate_previous.size())} + : node2vec_random_walk_e_bias_op_t{p_, + q_, + raft::device_span( + intersection_offsets.data(), + intersection_offsets.size()), + raft::device_span( + intersection_indices.data(), + intersection_indices.size()), + raft::device_span< + vertex_t const>(current_vertices.data(), + current_vertices.size()), + raft::device_span( + (*previous_vertices).data(), + (*previous_vertices).size())}, + cugraph::edge_src_dummy_property_t{}.view(), + cugraph::edge_dst_dummy_property_t{}.view(), + *edge_weight_view, + node2vec_sample_edges_op_t{}, + rng_state_, + size_t{1}, + true, + std::make_optional(thrust::make_tuple( + vertex_t{cugraph::invalid_vertex_id::value}, weight_t{0.0}))); + minors = std::move(std::get<0>(sample_e_op_results)); + weights = std::move(std::get<1>(sample_e_op_results)); + } else { + auto [sample_offsets, sample_e_op_results] = + cugraph::per_v_random_select_transform_outgoing_e( + handle, + graph_view, + vertex_frontier.bucket(0), + cugraph::edge_src_dummy_property_t{}.view(), + cugraph::edge_dst_dummy_property_t{}.view(), + cugraph::edge_dummy_property_t{}.view(), + GraphViewType::is_multi_gpu + ? node2vec_random_walk_e_bias_op_t{p_, + q_, + raft::device_span( + aggregate_offsets.data(), + aggregate_offsets.size()), + raft::device_span( + aggregate_indices.data(), + aggregate_indices.size()), + raft::device_span( + aggregate_currents.data(), + aggregate_currents.size()), + raft::device_span( + aggregate_previous.data(), + aggregate_previous.size())} + : node2vec_random_walk_e_bias_op_t{p_, + q_, + raft::device_span( + intersection_offsets.data(), + intersection_offsets.size()), + raft::device_span( + intersection_indices.data(), + intersection_indices.size()), + raft::device_span< + vertex_t const>(current_vertices.data(), + current_vertices.size()), + raft::device_span( + (*previous_vertices).data(), + (*previous_vertices).size())}, + cugraph::edge_src_dummy_property_t{}.view(), + cugraph::edge_dst_dummy_property_t{}.view(), + cugraph::edge_dummy_property_t{}.view(), + node2vec_sample_edges_op_t{}, + rng_state_, + size_t{1}, + true, + std::make_optional(vertex_t{cugraph::invalid_vertex_id::value})); + minors = std::move(sample_e_op_results); + } + + *previous_vertices = std::move(current_vertices); + + return std::make_tuple(std::move(minors), std::move(previous_vertices), std::move(weights)); } }; @@ -221,6 +569,16 @@ random_walk_impl(raft::handle_t const& handle, ? std::make_optional>(0, handle.get_stream()) : std::nullopt; + auto previous_vertices = (random_selector.is_second_order_) + ? std::make_optional>( + current_vertices.size(), handle.get_stream()) + : std::nullopt; + if (previous_vertices) { + raft::copy((*previous_vertices).data(), + start_vertices.data(), + start_vertices.size(), + handle.get_stream()); + } raft::copy( current_vertices.data(), start_vertices.data(), start_vertices.size(), handle.get_stream()); detail::sequence_fill( @@ -255,25 +613,73 @@ random_walk_impl(raft::handle_t const& handle, auto& minor_comm = handle.get_subcomm(cugraph::partition_manager::minor_comm_name()); auto const minor_comm_size = minor_comm.get_size(); - // Shuffle vertices to correct GPU to compute random indices - std::forward_as_tuple(std::tie(current_vertices, current_gpu, current_position), - std::ignore) = - cugraph::groupby_gpu_id_and_shuffle_values( - handle.get_comms(), + if (previous_vertices) { + std::forward_as_tuple( + std::tie(current_vertices, current_gpu, current_position, previous_vertices), + std::ignore) = + cugraph::groupby_gpu_id_and_shuffle_values( + handle.get_comms(), + thrust::make_zip_iterator(current_vertices.begin(), + current_gpu.begin(), + current_position.begin(), + previous_vertices->begin()), + thrust::make_zip_iterator(current_vertices.end(), + current_gpu.end(), + current_position.end(), + previous_vertices->end()), + [key_func = + cugraph::detail::compute_gpu_id_from_int_vertex_t{ + {vertex_partition_range_lasts.begin(), vertex_partition_range_lasts.size()}, + major_comm_size, + minor_comm_size}] __device__(auto val) { return key_func(thrust::get<0>(val)); }, + handle.get_stream()); + } else { + // Shuffle vertices to correct GPU to compute random indices + std::forward_as_tuple(std::tie(current_vertices, current_gpu, current_position), + std::ignore) = + cugraph::groupby_gpu_id_and_shuffle_values( + handle.get_comms(), + thrust::make_zip_iterator( + current_vertices.begin(), current_gpu.begin(), current_position.begin()), + thrust::make_zip_iterator( + current_vertices.end(), current_gpu.end(), current_position.end()), + [key_func = + cugraph::detail::compute_gpu_id_from_int_vertex_t{ + {vertex_partition_range_lasts.begin(), vertex_partition_range_lasts.size()}, + major_comm_size, + minor_comm_size}] __device__(auto val) { return key_func(thrust::get<0>(val)); }, + handle.get_stream()); + } + } + + // Sort for nbr_intersection, must sort all together + if (previous_vertices) { + if constexpr (multi_gpu) { + thrust::sort(handle.get_thrust_policy(), + thrust::make_zip_iterator(current_vertices.begin(), + (*previous_vertices).begin(), + current_position.begin(), + current_gpu.begin()), + thrust::make_zip_iterator(current_vertices.end(), + (*previous_vertices).end(), + current_position.end(), + current_gpu.end())); + } else { + thrust::sort( + handle.get_thrust_policy(), thrust::make_zip_iterator( - current_vertices.begin(), current_gpu.begin(), current_position.begin()), + current_vertices.begin(), (*previous_vertices).begin(), current_position.begin()), thrust::make_zip_iterator( - current_vertices.end(), current_gpu.end(), current_position.end()), - [key_func = - cugraph::detail::compute_gpu_id_from_int_vertex_t{ - {vertex_partition_range_lasts.begin(), vertex_partition_range_lasts.size()}, - major_comm_size, - minor_comm_size}] __device__(auto val) { return key_func(thrust::get<0>(val)); }, - handle.get_stream()); + current_vertices.end(), (*previous_vertices).end(), current_position.end())); + } } - std::tie(current_vertices, new_weights) = - random_selector.follow_random_edge(handle, graph_view, edge_weight_view, current_vertices); + std::tie(current_vertices, previous_vertices, new_weights) = + random_selector.follow_random_edge(handle, + graph_view, + edge_weight_view, + std::move(current_vertices), + std::move(previous_vertices)); // FIXME: remove_if has a 32-bit overflow issue // (https://github.com/NVIDIA/thrust/issues/1302) Seems unlikely here (the goal of @@ -281,164 +687,244 @@ random_walk_impl(raft::handle_t const& handle, CUGRAPH_EXPECTS( current_vertices.size() < static_cast(std::numeric_limits::max()), "remove_if will fail, current_vertices.size() is too large"); - + size_t compacted_length{0}; if constexpr (multi_gpu) { if (result_weights) { - auto input_iter = thrust::make_zip_iterator(current_vertices.begin(), - new_weights->begin(), - current_gpu.begin(), - current_position.begin()); - - auto compacted_length = thrust::distance( - input_iter, - thrust::remove_if(handle.get_thrust_policy(), - input_iter, - input_iter + current_vertices.size(), - current_vertices.begin(), - [] __device__(auto dst) { - return (dst == cugraph::invalid_vertex_id::value); - })); - - current_vertices.resize(compacted_length, handle.get_stream()); - new_weights->resize(compacted_length, handle.get_stream()); - current_gpu.resize(compacted_length, handle.get_stream()); - current_position.resize(compacted_length, handle.get_stream()); - - // Shuffle back to original GPU - auto current_iter = thrust::make_zip_iterator(current_vertices.begin(), + if (previous_vertices) { + auto input_iter = thrust::make_zip_iterator(current_vertices.begin(), + new_weights->begin(), + current_gpu.begin(), + current_position.begin(), + previous_vertices->begin()); + + compacted_length = thrust::distance( + input_iter, + thrust::remove_if(handle.get_thrust_policy(), + input_iter, + input_iter + current_vertices.size(), + current_vertices.begin(), + [] __device__(auto dst) { + return (dst == cugraph::invalid_vertex_id::value); + })); + } else { + auto input_iter = thrust::make_zip_iterator(current_vertices.begin(), new_weights->begin(), current_gpu.begin(), current_position.begin()); - std::forward_as_tuple( - std::tie(current_vertices, *new_weights, current_gpu, current_position), std::ignore) = - cugraph::groupby_gpu_id_and_shuffle_values( - handle.get_comms(), - current_iter, - current_iter + current_vertices.size(), - [] __device__(auto val) { return thrust::get<2>(val); }, - handle.get_stream()); - - thrust::for_each( - handle.get_thrust_policy(), - thrust::make_zip_iterator( - current_vertices.begin(), new_weights->begin(), current_position.begin()), - thrust::make_zip_iterator( - current_vertices.end(), new_weights->end(), current_position.end()), - [result_verts = result_vertices.data(), - result_wgts = result_weights->data(), - level, - max_length] __device__(auto tuple) { - vertex_t v = thrust::get<0>(tuple); - weight_t w = thrust::get<1>(tuple); - size_t pos = thrust::get<2>(tuple); - result_verts[pos * (max_length + 1) + level + 1] = v; - result_wgts[pos * max_length + level] = w; - }); + compacted_length = thrust::distance( + input_iter, + thrust::remove_if(handle.get_thrust_policy(), + input_iter, + input_iter + current_vertices.size(), + current_vertices.begin(), + [] __device__(auto dst) { + return (dst == cugraph::invalid_vertex_id::value); + })); + } } else { - auto input_iter = thrust::make_zip_iterator( - current_vertices.begin(), current_gpu.begin(), current_position.begin()); - - auto compacted_length = thrust::distance( - input_iter, - thrust::remove_if(handle.get_thrust_policy(), - input_iter, - input_iter + current_vertices.size(), - current_vertices.begin(), - [] __device__(auto dst) { - return (dst == cugraph::invalid_vertex_id::value); - })); - - current_vertices.resize(compacted_length, handle.get_stream()); - current_gpu.resize(compacted_length, handle.get_stream()); - current_position.resize(compacted_length, handle.get_stream()); - - // Shuffle back to original GPU - auto current_iter = thrust::make_zip_iterator( - current_vertices.begin(), current_gpu.begin(), current_position.begin()); - - std::forward_as_tuple(std::tie(current_vertices, current_gpu, current_position), - std::ignore) = - cugraph::groupby_gpu_id_and_shuffle_values( - handle.get_comms(), - current_iter, - current_iter + current_vertices.size(), - [] __device__(auto val) { return thrust::get<1>(val); }, - handle.get_stream()); - - thrust::for_each( - handle.get_thrust_policy(), - thrust::make_zip_iterator(current_vertices.begin(), current_position.begin()), - thrust::make_zip_iterator(current_vertices.end(), current_position.end()), - [result_verts = result_vertices.data(), level, max_length] __device__(auto tuple) { - vertex_t v = thrust::get<0>(tuple); - size_t pos = thrust::get<1>(tuple); - result_verts[pos * (max_length + 1) + level + 1] = v; - }); + if (previous_vertices) { + auto input_iter = thrust::make_zip_iterator(current_vertices.begin(), + current_gpu.begin(), + current_position.begin(), + previous_vertices->begin()); + + compacted_length = thrust::distance( + input_iter, + thrust::remove_if(handle.get_thrust_policy(), + input_iter, + input_iter + current_vertices.size(), + current_vertices.begin(), + [] __device__(auto dst) { + return (dst == cugraph::invalid_vertex_id::value); + })); + } else { + auto input_iter = thrust::make_zip_iterator( + current_vertices.begin(), current_gpu.begin(), current_position.begin()); + + compacted_length = thrust::distance( + input_iter, + thrust::remove_if(handle.get_thrust_policy(), + input_iter, + input_iter + current_vertices.size(), + current_vertices.begin(), + [] __device__(auto dst) { + return (dst == cugraph::invalid_vertex_id::value); + })); + } } } else { if (result_weights) { - auto input_iter = thrust::make_zip_iterator( - current_vertices.begin(), new_weights->begin(), current_position.begin()); - - auto compacted_length = thrust::distance( - input_iter, - thrust::remove_if(handle.get_thrust_policy(), - input_iter, - input_iter + current_vertices.size(), - current_vertices.begin(), - [] __device__(auto dst) { - return (dst == cugraph::invalid_vertex_id::value); - })); - - current_vertices.resize(compacted_length, handle.get_stream()); - new_weights->resize(compacted_length, handle.get_stream()); - current_position.resize(compacted_length, handle.get_stream()); - - thrust::for_each( - handle.get_thrust_policy(), - thrust::make_zip_iterator( - current_vertices.begin(), new_weights->begin(), current_position.begin()), - thrust::make_zip_iterator( - current_vertices.end(), new_weights->end(), current_position.end()), - [result_verts = result_vertices.data(), - result_wgts = result_weights->data(), - level, - max_length] __device__(auto tuple) { - vertex_t v = thrust::get<0>(tuple); - weight_t w = thrust::get<1>(tuple); - size_t pos = thrust::get<2>(tuple); - result_verts[pos * (max_length + 1) + level + 1] = v; - result_wgts[pos * max_length + level] = w; - }); + if (previous_vertices) { + auto input_iter = thrust::make_zip_iterator(current_vertices.begin(), + new_weights->begin(), + current_position.begin(), + previous_vertices->begin()); + + compacted_length = thrust::distance( + input_iter, + thrust::remove_if(handle.get_thrust_policy(), + input_iter, + input_iter + current_vertices.size(), + current_vertices.begin(), + [] __device__(auto dst) { + return (dst == cugraph::invalid_vertex_id::value); + })); + } else { + auto input_iter = thrust::make_zip_iterator( + current_vertices.begin(), new_weights->begin(), current_position.begin()); + + compacted_length = thrust::distance( + input_iter, + thrust::remove_if(handle.get_thrust_policy(), + input_iter, + input_iter + current_vertices.size(), + current_vertices.begin(), + [] __device__(auto dst) { + return (dst == cugraph::invalid_vertex_id::value); + })); + } } else { - auto input_iter = - thrust::make_zip_iterator(current_vertices.begin(), current_position.begin()); - - auto compacted_length = thrust::distance( - input_iter, - thrust::remove_if(handle.get_thrust_policy(), - input_iter, - input_iter + current_vertices.size(), - current_vertices.begin(), - [] __device__(auto dst) { - return (dst == cugraph::invalid_vertex_id::value); - })); - - current_vertices.resize(compacted_length, handle.get_stream()); - current_position.resize(compacted_length, handle.get_stream()); - - thrust::for_each( - handle.get_thrust_policy(), - thrust::make_zip_iterator(current_vertices.begin(), current_position.begin()), - thrust::make_zip_iterator(current_vertices.end(), current_position.end()), - [result_verts = result_vertices.data(), level, max_length] __device__(auto tuple) { - vertex_t v = thrust::get<0>(tuple); - size_t pos = thrust::get<1>(tuple); - result_verts[pos * (max_length + 1) + level + 1] = v; - }); + if (previous_vertices) { + auto input_iter = thrust::make_zip_iterator( + current_vertices.begin(), current_position.begin(), previous_vertices->begin()); + + compacted_length = thrust::distance( + input_iter, + thrust::remove_if(handle.get_thrust_policy(), + input_iter, + input_iter + current_vertices.size(), + current_vertices.begin(), + [] __device__(auto dst) { + return (dst == cugraph::invalid_vertex_id::value); + })); + } else { + auto input_iter = + thrust::make_zip_iterator(current_vertices.begin(), current_position.begin()); + + compacted_length = thrust::distance( + input_iter, + thrust::remove_if(handle.get_thrust_policy(), + input_iter, + input_iter + current_vertices.size(), + current_vertices.begin(), + [] __device__(auto dst) { + return (dst == cugraph::invalid_vertex_id::value); + })); + } + } + } + + // Moved out of if statements to cut down on code duplication + current_vertices.resize(compacted_length, handle.get_stream()); + current_vertices.shrink_to_fit(handle.get_stream()); + current_position.resize(compacted_length, handle.get_stream()); + current_position.shrink_to_fit(handle.get_stream()); + if (result_weights) { + new_weights->resize(compacted_length, handle.get_stream()); + new_weights->shrink_to_fit(handle.get_stream()); + } + if (previous_vertices) { + previous_vertices->resize(compacted_length, handle.get_stream()); + previous_vertices->shrink_to_fit(handle.get_stream()); + } + if constexpr (multi_gpu) { + current_gpu.resize(compacted_length, handle.get_stream()); + current_gpu.shrink_to_fit(handle.get_stream()); + + // Shuffle back to original GPU + if (previous_vertices) { + if (result_weights) { + auto current_iter = thrust::make_zip_iterator(current_vertices.begin(), + new_weights->begin(), + current_gpu.begin(), + current_position.begin(), + previous_vertices->begin()); + + std::forward_as_tuple( + std::tie( + current_vertices, *new_weights, current_gpu, current_position, *previous_vertices), + std::ignore) = + cugraph::groupby_gpu_id_and_shuffle_values( + handle.get_comms(), + current_iter, + current_iter + current_vertices.size(), + [] __device__(auto val) { return thrust::get<2>(val); }, + handle.get_stream()); + } else { + auto current_iter = thrust::make_zip_iterator(current_vertices.begin(), + current_gpu.begin(), + current_position.begin(), + previous_vertices->begin()); + + std::forward_as_tuple( + std::tie(current_vertices, current_gpu, current_position, *previous_vertices), + std::ignore) = + cugraph::groupby_gpu_id_and_shuffle_values( + handle.get_comms(), + current_iter, + current_iter + current_vertices.size(), + [] __device__(auto val) { return thrust::get<1>(val); }, + handle.get_stream()); + } + } else { + if (result_weights) { + auto current_iter = thrust::make_zip_iterator(current_vertices.begin(), + new_weights->begin(), + current_gpu.begin(), + current_position.begin()); + + std::forward_as_tuple( + std::tie(current_vertices, *new_weights, current_gpu, current_position), std::ignore) = + cugraph::groupby_gpu_id_and_shuffle_values( + handle.get_comms(), + current_iter, + current_iter + current_vertices.size(), + [] __device__(auto val) { return thrust::get<2>(val); }, + handle.get_stream()); + } else { + auto current_iter = thrust::make_zip_iterator( + current_vertices.begin(), current_gpu.begin(), current_position.begin()); + + std::forward_as_tuple(std::tie(current_vertices, current_gpu, current_position), + std::ignore) = + cugraph::groupby_gpu_id_and_shuffle_values( + handle.get_comms(), + current_iter, + current_iter + current_vertices.size(), + [] __device__(auto val) { return thrust::get<1>(val); }, + handle.get_stream()); + } } } + + if (result_weights) { + thrust::for_each(handle.get_thrust_policy(), + thrust::make_zip_iterator( + current_vertices.begin(), new_weights->begin(), current_position.begin()), + thrust::make_zip_iterator( + current_vertices.end(), new_weights->end(), current_position.end()), + [result_verts = result_vertices.data(), + result_wgts = result_weights->data(), + level, + max_length] __device__(auto tuple) { + vertex_t v = thrust::get<0>(tuple); + weight_t w = thrust::get<1>(tuple); + size_t pos = thrust::get<2>(tuple); + result_verts[pos * (max_length + 1) + level + 1] = v; + result_wgts[pos * max_length + level] = w; + }); + } else { + thrust::for_each( + handle.get_thrust_policy(), + thrust::make_zip_iterator(current_vertices.begin(), current_position.begin()), + thrust::make_zip_iterator(current_vertices.end(), current_position.end()), + [result_verts = result_vertices.data(), level, max_length] __device__(auto tuple) { + vertex_t v = thrust::get<0>(tuple); + size_t pos = thrust::get<1>(tuple); + result_verts[pos * (max_length + 1) + level + 1] = v; + }); + } } return std::make_tuple(std::move(result_vertices), std::move(result_weights)); @@ -449,11 +935,11 @@ random_walk_impl(raft::handle_t const& handle, template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed) + size_t max_length) { CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); @@ -462,18 +948,17 @@ uniform_random_walks(raft::handle_t const& handle, edge_weight_view, start_vertices, max_length, - detail::uniform_selector( - (seed == 0 ? detail::get_current_time_nanoseconds() : seed))); + detail::uniform_selector{rng_state}); } template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed) + size_t max_length) { CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); @@ -483,30 +968,28 @@ biased_random_walks(raft::handle_t const& handle, std::optional>{edge_weight_view}, start_vertices, max_length, - detail::biased_selector{(seed == 0 ? detail::get_current_time_nanoseconds() : seed)}); + detail::biased_selector{rng_state}); } template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, weight_t p, - weight_t q, - uint64_t seed) + weight_t q) { CUGRAPH_EXPECTS(!graph_view.has_edge_mask(), "unimplemented."); - return detail::random_walk_impl( - handle, - graph_view, - edge_weight_view, - start_vertices, - max_length, - detail::node2vec_selector{ - p, q, (seed == 0 ? detail::get_current_time_nanoseconds() : seed)}); + return detail::random_walk_impl(handle, + graph_view, + edge_weight_view, + start_vertices, + max_length, + detail::node2vec_selector{p, q, rng_state}); } } // namespace cugraph diff --git a/cpp/src/sampling/random_walks_mg_v32_e32.cu b/cpp/src/sampling/random_walks_mg_v32_e32.cu index 421d3e9c818..abe5386da1c 100644 --- a/cpp/src/sampling/random_walks_mg_v32_e32.cu +++ b/cpp/src/sampling/random_walks_mg_v32_e32.cu @@ -22,54 +22,54 @@ namespace cugraph { template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, float p, - float q, - uint64_t seed); + float q); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, double p, - double q, - uint64_t seed); + double q); } // namespace cugraph diff --git a/cpp/src/sampling/random_walks_mg_v32_e64.cu b/cpp/src/sampling/random_walks_mg_v32_e64.cu index d38af65a505..b1bf1a19b77 100644 --- a/cpp/src/sampling/random_walks_mg_v32_e64.cu +++ b/cpp/src/sampling/random_walks_mg_v32_e64.cu @@ -22,54 +22,54 @@ namespace cugraph { template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, float p, - float q, - uint64_t seed); + float q); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, double p, - double q, - uint64_t seed); + double q); } // namespace cugraph diff --git a/cpp/src/sampling/random_walks_mg_v64_e64.cu b/cpp/src/sampling/random_walks_mg_v64_e64.cu index 9dedc893242..13cc899e50d 100644 --- a/cpp/src/sampling/random_walks_mg_v64_e64.cu +++ b/cpp/src/sampling/random_walks_mg_v64_e64.cu @@ -22,54 +22,54 @@ namespace cugraph { template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, float p, - float q, - uint64_t seed); + float q); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, double p, - double q, - uint64_t seed); + double q); } // namespace cugraph diff --git a/cpp/src/sampling/random_walks_sg_v32_e32.cu b/cpp/src/sampling/random_walks_sg_v32_e32.cu index 7b64d107250..383917c0248 100644 --- a/cpp/src/sampling/random_walks_sg_v32_e32.cu +++ b/cpp/src/sampling/random_walks_sg_v32_e32.cu @@ -22,54 +22,54 @@ namespace cugraph { template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, float p, - float q, - uint64_t seed); + float q); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, double p, - double q, - uint64_t seed); + double q); } // namespace cugraph diff --git a/cpp/src/sampling/random_walks_sg_v32_e64.cu b/cpp/src/sampling/random_walks_sg_v32_e64.cu index d9ea09f36ef..98d2bb02d88 100644 --- a/cpp/src/sampling/random_walks_sg_v32_e64.cu +++ b/cpp/src/sampling/random_walks_sg_v32_e64.cu @@ -22,54 +22,54 @@ namespace cugraph { template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, float p, - float q, - uint64_t seed); + float q); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, double p, - double q, - uint64_t seed); + double q); } // namespace cugraph diff --git a/cpp/src/sampling/random_walks_sg_v64_e64.cu b/cpp/src/sampling/random_walks_sg_v64_e64.cu index 0b9be107276..c139acec4b7 100644 --- a/cpp/src/sampling/random_walks_sg_v64_e64.cu +++ b/cpp/src/sampling/random_walks_sg_v64_e64.cu @@ -22,54 +22,54 @@ namespace cugraph { template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> uniform_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> biased_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, edge_property_view_t edge_weight_view, raft::device_span start_vertices, - size_t max_length, - uint64_t seed); + size_t max_length); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, float p, - float q, - uint64_t seed); + float q); template std::tuple, std::optional>> node2vec_random_walks(raft::handle_t const& handle, + raft::random::RngState& rng_state, graph_view_t const& graph_view, std::optional> edge_weight_view, raft::device_span start_vertices, size_t max_length, double p, - double q, - uint64_t seed); + double q); } // namespace cugraph diff --git a/cpp/tests/c_api/sg_random_walks_test.c b/cpp/tests/c_api/sg_random_walks_test.c index 14108d91c04..a4a77b5775a 100644 --- a/cpp/tests/c_api/sg_random_walks_test.c +++ b/cpp/tests/c_api/sg_random_walks_test.c @@ -192,9 +192,6 @@ int generic_biased_random_walks_test(vertex_t* h_src, ret_code = cugraph_biased_random_walks(handle, graph, d_start_view, max_depth, &result, &ret_error); -#if 1 - TEST_ASSERT(test_ret_value, ret_code != CUGRAPH_SUCCESS, "biased_random_walks should have failed") -#else TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "biased_random_walks failed."); @@ -208,10 +205,10 @@ int generic_biased_random_walks_test(vertex_t* h_src, size_t wgts_size = cugraph_type_erased_device_array_view_size(wgts); vertex_t h_result_verts[verts_size]; - vertex_t h_result_wgts[wgts_size]; + weight_t h_result_wgts[wgts_size]; - ret_code = - cugraph_type_erased_device_array_view_copy_to_host(handle, (byte_t*)h_verts, verts, &ret_error); + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_verts, verts, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); ret_code = cugraph_type_erased_device_array_view_copy_to_host( @@ -231,23 +228,35 @@ int generic_biased_random_walks_test(vertex_t* h_src, M[h_src[i]][h_dst[i]] = h_wgt[i]; TEST_ASSERT(test_ret_value, - cugraph_random_walk_result_get_max_path_length() == max_depth, + cugraph_random_walk_result_get_max_path_length(result) == max_depth, "path length does not match"); for (int i = 0; (i < num_starts) && (test_ret_value == 0); ++i) { - TEST_ASSERT(test_ret_value, - M[h_start[i]][h_result_verts[i * (max_depth + 1)]] == h_result_wgts[i * max_depth], - "biased_random_walks got edge that doesn't exist"); - for (size_t j = 1; j < cugraph_random_walk_result_get_max_path_length(); ++j) - TEST_ASSERT( - test_ret_value, - M[h_start[i * (max_depth + 1) + j - 1]][h_result_verts[i * (max_depth + 1) + j]] == - h_result_wgts[i * max_depth + j - 1], - "biased_random_walks got edge that doesn't exist"); + TEST_ASSERT( + test_ret_value, h_start[i] == h_result_verts[i * (max_depth + 1)], "start of path not found"); + for (size_t j = 0; j < max_depth; ++j) { + int src_index = i * (max_depth + 1) + j; + int dst_index = src_index + 1; + if (h_result_verts[dst_index] < 0) { + if (h_result_verts[src_index] >= 0) { + int departing_count = 0; + for (int k = 0; k < num_vertices; ++k) { + if (M[h_result_verts[src_index]][k] >= 0) departing_count++; + } + TEST_ASSERT(test_ret_value, + departing_count == 0, + "biased_random_walks found no edge when an edge exists"); + } + } else { + TEST_ASSERT(test_ret_value, + M[h_result_verts[src_index]][h_result_verts[dst_index]] == + h_result_wgts[i * max_depth + j], + "biased_random_walks got edge that doesn't exist"); + } + } } cugraph_random_walk_result_free(result); -#endif cugraph_sg_graph_free(graph); cugraph_free_resource_handle(handle); @@ -302,10 +311,6 @@ int generic_node2vec_random_walks_test(vertex_t* h_src, ret_code = cugraph_node2vec_random_walks( handle, graph, d_start_view, max_depth, p, q, &result, &ret_error); -#if 1 - TEST_ASSERT( - test_ret_value, ret_code != CUGRAPH_SUCCESS, "node2vec_random_walks should have failed") -#else TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "node2vec_random_walks failed."); @@ -319,10 +324,10 @@ int generic_node2vec_random_walks_test(vertex_t* h_src, size_t wgts_size = cugraph_type_erased_device_array_view_size(wgts); vertex_t h_result_verts[verts_size]; - vertex_t h_result_wgts[wgts_size]; + weight_t h_result_wgts[wgts_size]; - ret_code = - cugraph_type_erased_device_array_view_copy_to_host(handle, (byte_t*)h_verts, verts, &ret_error); + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_verts, verts, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); ret_code = cugraph_type_erased_device_array_view_copy_to_host( @@ -342,23 +347,35 @@ int generic_node2vec_random_walks_test(vertex_t* h_src, M[h_src[i]][h_dst[i]] = h_wgt[i]; TEST_ASSERT(test_ret_value, - cugraph_random_walk_result_get_max_path_length() == max_depth, + cugraph_random_walk_result_get_max_path_length(result) == max_depth, "path length does not match"); for (int i = 0; (i < num_starts) && (test_ret_value == 0); ++i) { - TEST_ASSERT(test_ret_value, - M[h_start[i]][h_result_verts[i * (max_depth + 1)]] == h_result_wgts[i * max_depth], - "node2vec_random_walks got edge that doesn't exist"); - for (size_t j = 1; j < max_depth; ++j) - TEST_ASSERT( - test_ret_value, - M[h_start[i * (max_depth + 1) + j - 1]][h_result_verts[i * (max_depth + 1) + j]] == - h_result_wgts[i * max_depth + j - 1], - "node2vec_random_walks got edge that doesn't exist"); + TEST_ASSERT( + test_ret_value, h_start[i] == h_result_verts[i * (max_depth + 1)], "start of path not found"); + for (size_t j = 0; j < max_depth; ++j) { + int src_index = i * (max_depth + 1) + j; + int dst_index = src_index + 1; + if (h_result_verts[dst_index] < 0) { + if (h_result_verts[src_index] >= 0) { + int departing_count = 0; + for (int k = 0; k < num_vertices; ++k) { + if (M[h_result_verts[src_index]][k] >= 0) departing_count++; + } + TEST_ASSERT(test_ret_value, + departing_count == 0, + "node2vec_random_walks found no edge when an edge exists"); + } + } else { + TEST_ASSERT(test_ret_value, + M[h_result_verts[src_index]][h_result_verts[dst_index]] == + h_result_wgts[i * max_depth + j], + "node2vec_random_walks got edge that doesn't exist"); + } + } } cugraph_random_walk_result_free(result); -#endif cugraph_sg_graph_free(graph); cugraph_free_resource_handle(handle); @@ -390,7 +407,7 @@ int test_biased_random_walks() vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; - weight_t wgt[] = {0, 1, 2, 3, 4, 5, 6, 7}; + weight_t wgt[] = {1, 2, 3, 4, 5, 6, 7, 8}; vertex_t start[] = {2, 2}; return generic_biased_random_walks_test( diff --git a/cpp/tests/sampling/mg_random_walks_test.cpp b/cpp/tests/sampling/mg_random_walks_test.cpp index c2ad5c37e9e..e2415c08e60 100644 --- a/cpp/tests/sampling/mg_random_walks_test.cpp +++ b/cpp/tests/sampling/mg_random_walks_test.cpp @@ -44,8 +44,10 @@ struct UniformRandomWalks_Usecase { raft::device_span start_vertices, size_t max_depth) { + raft::random::RngState rng_state(static_cast(handle.get_comms().get_rank())); + return cugraph::uniform_random_walks( - handle, graph_view, edge_weight_view, start_vertices, max_depth, seed); + handle, rng_state, graph_view, edge_weight_view, start_vertices, max_depth); } bool expect_throw() { return false; } @@ -66,12 +68,13 @@ struct BiasedRandomWalks_Usecase { { CUGRAPH_EXPECTS(edge_weight_view.has_value(), "Biased random walk requires edge weights."); + raft::random::RngState rng_state(static_cast(handle.get_comms().get_rank())); + return cugraph::biased_random_walks( - handle, graph_view, *edge_weight_view, start_vertices, max_depth, seed); + handle, rng_state, graph_view, *edge_weight_view, start_vertices, max_depth); } - // FIXME: Not currently implemented - bool expect_throw() { return true; } + bool expect_throw() { return !test_weighted; } }; struct Node2VecRandomWalks_Usecase { @@ -89,18 +92,19 @@ struct Node2VecRandomWalks_Usecase { raft::device_span start_vertices, size_t max_depth) { + raft::random::RngState rng_state(static_cast(handle.get_comms().get_rank())); + return cugraph::node2vec_random_walks(handle, + rng_state, graph_view, edge_weight_view, start_vertices, max_depth, static_cast(p), - static_cast(q), - seed); + static_cast(q)); } - // FIXME: Not currently implemented - bool expect_throw() { return true; } + bool expect_throw() { return false; } }; template diff --git a/cpp/tests/sampling/random_walks_check.cuh b/cpp/tests/sampling/random_walks_check.cuh index 0fd73b5bba7..380b97a5b84 100644 --- a/cpp/tests/sampling/random_walks_check.cuh +++ b/cpp/tests/sampling/random_walks_check.cuh @@ -108,7 +108,7 @@ void random_walks_validate( (int)d, (float)w); } else { - printf("edge (%d,%d) NOT FOUND\n", (int)s, (int)d); + printf("edge (%d,%d), weight %g NOT FOUND\n", (int)s, (int)d, (float)w); } return 1; diff --git a/cpp/tests/sampling/sg_random_walks_test.cpp b/cpp/tests/sampling/sg_random_walks_test.cpp index 7409c2ab758..4bcfebc6d51 100644 --- a/cpp/tests/sampling/sg_random_walks_test.cpp +++ b/cpp/tests/sampling/sg_random_walks_test.cpp @@ -40,8 +40,10 @@ struct UniformRandomWalks_Usecase { raft::device_span start_vertices, size_t num_paths) { + raft::random::RngState rng_state(0); + return cugraph::uniform_random_walks( - handle, graph_view, edge_weight_view, start_vertices, num_paths, seed); + handle, rng_state, graph_view, edge_weight_view, start_vertices, num_paths); } bool expect_throw() { return false; } @@ -62,12 +64,13 @@ struct BiasedRandomWalks_Usecase { { CUGRAPH_EXPECTS(edge_weight_view.has_value(), "Biased random walk requires edge weights."); + raft::random::RngState rng_state(0); + return cugraph::biased_random_walks( - handle, graph_view, *edge_weight_view, start_vertices, num_paths, seed); + handle, rng_state, graph_view, *edge_weight_view, start_vertices, num_paths); } - // FIXME: Not currently implemented - bool expect_throw() { return true; } + bool expect_throw() { return !test_weighted; } }; struct Node2VecRandomWalks_Usecase { @@ -85,18 +88,19 @@ struct Node2VecRandomWalks_Usecase { raft::device_span start_vertices, size_t num_paths) { + raft::random::RngState rng_state(0); + return cugraph::node2vec_random_walks(handle, + rng_state, graph_view, edge_weight_view, start_vertices, num_paths, static_cast(p), - static_cast(q), - seed); + static_cast(q)); } - // FIXME: Not currently implemented - bool expect_throw() { return true; } + bool expect_throw() { return false; } }; template @@ -197,9 +201,6 @@ using Tests_Node2VecRandomWalks_File = using Tests_Node2VecRandomWalks_Rmat = Tests_RandomWalks>; -#if 0 -// FIXME: We should use these tests, gtest-1.11.0 makes it a runtime error -// to define and not instantiate these. TEST_P(Tests_UniformRandomWalks_File, Initialize_i32_i32_f) { run_current_test( @@ -211,7 +212,6 @@ TEST_P(Tests_UniformRandomWalks_Rmat, Initialize_i32_i32_f) run_current_test( override_Rmat_Usecase_with_cmd_line_arguments(GetParam())); } -#endif TEST_P(Tests_BiasedRandomWalks_File, Initialize_i32_i32_f) { @@ -237,19 +237,12 @@ TEST_P(Tests_Node2VecRandomWalks_Rmat, Initialize_i32_i32_f) override_Rmat_Usecase_with_cmd_line_arguments(GetParam())); } -#if 0 -// FIXME: Not sure why these are failing, but we're refactoring anyway. INSTANTIATE_TEST_SUITE_P( simple_test, Tests_UniformRandomWalks_File, - ::testing::Combine( - ::testing::Values(UniformRandomWalks_Usecase{false, 0, true}, - UniformRandomWalks_Usecase{true, 0, true}), - ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx"), - cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), - cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), - cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); -#endif + ::testing::Combine(::testing::Values(UniformRandomWalks_Usecase{false, 0, true}, + UniformRandomWalks_Usecase{true, 0, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); INSTANTIATE_TEST_SUITE_P( file_test, @@ -265,6 +258,16 @@ INSTANTIATE_TEST_SUITE_P( Node2VecRandomWalks_Usecase{4, 8, true, 0, true}), ::testing::Values(cugraph::test::File_Usecase("test/datasets/karate.mtx")))); +INSTANTIATE_TEST_SUITE_P( + file_large_test, + Tests_UniformRandomWalks_File, + ::testing::Combine( + ::testing::Values(UniformRandomWalks_Usecase{false, 0, true}, + UniformRandomWalks_Usecase{true, 0, true}), + ::testing::Values(cugraph::test::File_Usecase("test/datasets/web-Google.mtx"), + cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), + cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); + INSTANTIATE_TEST_SUITE_P( file_large_test, Tests_BiasedRandomWalks_File, @@ -285,23 +288,20 @@ INSTANTIATE_TEST_SUITE_P( cugraph::test::File_Usecase("test/datasets/ljournal-2008.mtx"), cugraph::test::File_Usecase("test/datasets/webbase-1M.mtx")))); -#if 0 -// FIXME: Not sure why these are failing, but we're refactoring anyway. INSTANTIATE_TEST_SUITE_P( rmat_small_test, Tests_UniformRandomWalks_Rmat, - ::testing::Combine(::testing::Values(UniformRandomWalks_Usecase{false, 0, true}, - UniformRandomWalks_Usecase{true, 0, true}), - ::testing::Values(cugraph::test::Rmat_Usecase( - 10, 16, 0.57, 0.19, 0.19, 0, false, false)))); + ::testing::Combine( + ::testing::Values(UniformRandomWalks_Usecase{false, 0, true}, + UniformRandomWalks_Usecase{true, 0, true}), + ::testing::Values(cugraph::test::Rmat_Usecase(10, 16, 0.57, 0.19, 0.19, 0, false, false)))); INSTANTIATE_TEST_SUITE_P( rmat_benchmark_test, Tests_UniformRandomWalks_Rmat, - ::testing::Combine(::testing::Values(UniformRandomWalks_Usecase{true, 0, false}), - ::testing::Values(cugraph::test::Rmat_Usecase( - 20, 32, 0.57, 0.19, 0.19, 0, false, false)))); -#endif + ::testing::Combine( + ::testing::Values(UniformRandomWalks_Usecase{true, 0, false}), + ::testing::Values(cugraph::test::Rmat_Usecase(20, 32, 0.57, 0.19, 0.19, 0, false, false)))); INSTANTIATE_TEST_SUITE_P( rmat_small_test, From 467d818a67405adcfda01c57d43c303fbcd1789e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 18 Sep 2024 05:12:05 -0500 Subject: [PATCH 5/8] Update flake8 to 7.1.1. (#4652) We need to update flake8 to fix a false-positive that appears with older flake8 versions on Python 3.12. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) - Alex Barghi (https://github.com/alexbarghi-nv) - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/cugraph/pull/4652 --- .pre-commit-config.yaml | 4 ++-- .../cugraph-dgl/python-script/ogbn_mag_benchmark.py | 4 ++-- python/cugraph-dgl/cugraph_dgl/nn/conv/base.py | 4 ++-- .../examples/graphsage/node-classification-dask.py | 4 +++- .../examples/graphsage/node-classification.py | 4 +++- .../examples/multi_trainer_MG_example/model.py | 2 +- .../examples/multi_trainer_MG_example/workflow_mnmg.py | 2 +- .../examples/multi_trainer_MG_example/workflow_snmg.py | 2 +- .../cugraph-pyg/cugraph_pyg/data/dask_graph_store.py | 2 +- python/cugraph-pyg/cugraph_pyg/examples/gcn_dist_sg.py | 2 +- python/cugraph-service/tests/test_e2e.py | 6 +++--- python/cugraph-service/tests/test_mg_e2e.py | 6 +++--- python/cugraph/cugraph/tests/generators/test_rmat.py | 4 +++- .../cugraph/cugraph/tests/generators/test_rmat_mg.py | 6 ++++-- .../cugraph/tests/sampling/test_bulk_sampler.py | 2 +- .../cugraph/cugraph/tests/structure/test_graph_mg.py | 4 ++-- python/cugraph/cugraph/tests/traversal/test_sssp.py | 4 ++-- python/cugraph/cugraph/traversal/sssp.py | 10 ++++++---- 18 files changed, 41 insertions(+), 31 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b351478fa9..8ff284210b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: files: ^(python/.*|benchmarks/.*)$ exclude: ^python/nx-cugraph/ - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 7.1.1 hooks: - id: flake8 args: ["--config=.flake8"] @@ -34,7 +34,7 @@ repos: hooks: - id: yesqa additional_dependencies: - - flake8==6.0.0 + - flake8==7.1.1 - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.6 hooks: diff --git a/benchmarks/cugraph-dgl/python-script/ogbn_mag_benchmark.py b/benchmarks/cugraph-dgl/python-script/ogbn_mag_benchmark.py index 539fe333b1e..55ff0043e30 100644 --- a/benchmarks/cugraph-dgl/python-script/ogbn_mag_benchmark.py +++ b/benchmarks/cugraph-dgl/python-script/ogbn_mag_benchmark.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -126,4 +126,4 @@ def sampling_func(g, seed_nodes, labels, train_loader): st = time.time() sampling_func(g, subset_split_idx["train"], labels, train_loader) et = time.time() - print(f"Sampling time taken = {et-st} s") + print(f"Sampling time taken = {et - st} s") diff --git a/python/cugraph-dgl/cugraph_dgl/nn/conv/base.py b/python/cugraph-dgl/cugraph_dgl/nn/conv/base.py index d2460f814c9..fcd5a26aee6 100644 --- a/python/cugraph-dgl/cugraph_dgl/nn/conv/base.py +++ b/python/cugraph-dgl/cugraph_dgl/nn/conv/base.py @@ -129,7 +129,7 @@ def __init__( if csrc_ids is not None: if csrc_ids.numel() != self._num_src_nodes + 1: raise RuntimeError( - f"Size mismatch for 'csrc_ids': expected ({size[0]+1},), " + f"Size mismatch for 'csrc_ids': expected ({size[0] + 1},), " f"but got {tuple(csrc_ids.size())}" ) csrc_ids = csrc_ids.contiguous() @@ -137,7 +137,7 @@ def __init__( if cdst_ids is not None: if cdst_ids.numel() != self._num_dst_nodes + 1: raise RuntimeError( - f"Size mismatch for 'cdst_ids': expected ({size[1]+1},), " + f"Size mismatch for 'cdst_ids': expected ({size[1] + 1},), " f"but got {tuple(cdst_ids.size())}" ) cdst_ids = cdst_ids.contiguous() diff --git a/python/cugraph-dgl/examples/graphsage/node-classification-dask.py b/python/cugraph-dgl/examples/graphsage/node-classification-dask.py index 992669e4284..0481f9566bc 100644 --- a/python/cugraph-dgl/examples/graphsage/node-classification-dask.py +++ b/python/cugraph-dgl/examples/graphsage/node-classification-dask.py @@ -205,7 +205,9 @@ def train(args, device, g, dataset, model): et = time.time() - print(f"Time taken for epoch {epoch} with batch_size {batch_size} = {et-st} s") + print( + f"Time taken for epoch {epoch} with batch_size {batch_size} = {et - st} s" + ) acc = evaluate(model, g, val_dataloader) print( "Epoch {:05d} | Loss {:.4f} | Accuracy {:.4f} ".format( diff --git a/python/cugraph-dgl/examples/graphsage/node-classification.py b/python/cugraph-dgl/examples/graphsage/node-classification.py index 2b8b687efab..56ac41c09b4 100644 --- a/python/cugraph-dgl/examples/graphsage/node-classification.py +++ b/python/cugraph-dgl/examples/graphsage/node-classification.py @@ -215,7 +215,9 @@ def train(args, device, g, dataset, model, directory): et = time.time() - print(f"Time taken for epoch {epoch} with batch_size {batch_size} = {et-st} s") + print( + f"Time taken for epoch {epoch} with batch_size {batch_size} = {et - st} s" + ) acc = evaluate(model, g, val_dataloader) print( "Epoch {:05d} | Loss {:.4f} | Accuracy {:.4f} ".format( diff --git a/python/cugraph-dgl/examples/multi_trainer_MG_example/model.py b/python/cugraph-dgl/examples/multi_trainer_MG_example/model.py index d3aad2ab309..3e0c0454905 100644 --- a/python/cugraph-dgl/examples/multi_trainer_MG_example/model.py +++ b/python/cugraph-dgl/examples/multi_trainer_MG_example/model.py @@ -134,7 +134,7 @@ def train_model(model, g, opt, train_dataloader, num_epochs, rank, val_nid): et = time.time() print( f"Total time taken for num_epochs {num_epochs} " - f"with batch_size {train_dataloader._batch_size} = {et-st} s on rank ={rank}" + f"with batch_size {train_dataloader._batch_size} = {et - st} s on rank ={rank}" ) if rank == 0: val_acc = layerwise_infer(g, val_nid, model, 1024 * 5, "cuda") diff --git a/python/cugraph-dgl/examples/multi_trainer_MG_example/workflow_mnmg.py b/python/cugraph-dgl/examples/multi_trainer_MG_example/workflow_mnmg.py index b1878b37d4e..11afe466014 100644 --- a/python/cugraph-dgl/examples/multi_trainer_MG_example/workflow_mnmg.py +++ b/python/cugraph-dgl/examples/multi_trainer_MG_example/workflow_mnmg.py @@ -234,7 +234,7 @@ def run_workflow( torch.distributed.barrier() total_et = time.time() print( - f"Total time taken on n_epochs {n_epochs} = {total_et-total_st} s", + f"Total time taken on n_epochs {n_epochs} = {total_et - total_st} s", f"measured by worker = {global_rank}", ) diff --git a/python/cugraph-dgl/examples/multi_trainer_MG_example/workflow_snmg.py b/python/cugraph-dgl/examples/multi_trainer_MG_example/workflow_snmg.py index da5c2b4d64e..001d7fb82dc 100644 --- a/python/cugraph-dgl/examples/multi_trainer_MG_example/workflow_snmg.py +++ b/python/cugraph-dgl/examples/multi_trainer_MG_example/workflow_snmg.py @@ -207,7 +207,7 @@ def run_workflow(rank, world_size, cugraph_id, dataset, temp_dir): torch.distributed.barrier() total_et = time.time() print( - f"Total time taken on n_epochs {n_epochs} = {total_et-total_st} s", + f"Total time taken on n_epochs {n_epochs} = {total_et - total_st} s", f"measured by worker = {rank}", ) diff --git a/python/cugraph-pyg/cugraph_pyg/data/dask_graph_store.py b/python/cugraph-pyg/cugraph_pyg/data/dask_graph_store.py index c805cd496c8..6195f3118a4 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/dask_graph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/dask_graph_store.py @@ -150,7 +150,7 @@ def is_set(self, key): if key not in self.__dataclass_fields__: raise KeyError(key) attr = getattr(self, key) - return type(attr) != _field_status or attr != _field_status.UNSET + return type(attr) is not _field_status or attr != _field_status.UNSET def is_fully_specified(self): """ diff --git a/python/cugraph-pyg/cugraph_pyg/examples/gcn_dist_sg.py b/python/cugraph-pyg/cugraph_pyg/examples/gcn_dist_sg.py index b299fc2a1a1..09d874bd87d 100644 --- a/python/cugraph-pyg/cugraph_pyg/examples/gcn_dist_sg.py +++ b/python/cugraph-pyg/cugraph_pyg/examples/gcn_dist_sg.py @@ -66,7 +66,7 @@ def train(epoch: int): torch.cuda.synchronize() print( f"Average Training Iteration Time (s/iter): \ - {(time.perf_counter() - start_avg_time)/(i-warmup_steps):.6f}" + {(time.perf_counter() - start_avg_time) / (i - warmup_steps):.6f}" ) diff --git a/python/cugraph-service/tests/test_e2e.py b/python/cugraph-service/tests/test_e2e.py index c9b3d24f20e..3079a2423c7 100644 --- a/python/cugraph-service/tests/test_e2e.py +++ b/python/cugraph-service/tests/test_e2e.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -307,8 +307,8 @@ def test_load_call_unload_extension(client, extension1): assert len(results) == 2 assert len(results[0]) == 33 assert len(results[1]) == 21 - assert type(results[0][0]) == int - assert type(results[1][0]) == float + assert type(results[0][0]) is int + assert type(results[1][0]) is float assert results[0][0] == 9 assert results[1][0] == 9.0 diff --git a/python/cugraph-service/tests/test_mg_e2e.py b/python/cugraph-service/tests/test_mg_e2e.py index 39c1195151d..5526593aee0 100644 --- a/python/cugraph-service/tests/test_mg_e2e.py +++ b/python/cugraph-service/tests/test_mg_e2e.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -413,8 +413,8 @@ def test_call_extension_result_on_device( assert len(results) == 2 assert len(results[0]) == array1_len assert len(results[1]) == array2_len - assert type(results[0][0]) == int - assert type(results[1][0]) == float + assert type(results[0][0]) is int + assert type(results[1][0]) is float assert results[0][0] == 9 assert results[1][0] == 9.0 else: diff --git a/python/cugraph/cugraph/tests/generators/test_rmat.py b/python/cugraph/cugraph/tests/generators/test_rmat.py index 1cee0461686..87cbe636fdc 100644 --- a/python/cugraph/cugraph/tests/generators/test_rmat.py +++ b/python/cugraph/cugraph/tests/generators/test_rmat.py @@ -27,7 +27,9 @@ _scale_values = [2, 4, 16] _scale_test_ids = [f"scale={x}" for x in _scale_values] _graph_types = [cugraph.Graph, None, int] -_graph_test_ids = [f"create_using={getattr(x,'__name__',str(x))}" for x in _graph_types] +_graph_test_ids = [ + f"create_using={getattr(x, '__name__', str(x))}" for x in _graph_types +] _clip_and_flip = [False, True] _clip_and_flip_test_ids = [f"clip_and_flip={x}" for x in _clip_and_flip] _scramble_vertex_ids = [False, True] diff --git a/python/cugraph/cugraph/tests/generators/test_rmat_mg.py b/python/cugraph/cugraph/tests/generators/test_rmat_mg.py index 0e1808d2f80..44a6b3a2fc1 100644 --- a/python/cugraph/cugraph/tests/generators/test_rmat_mg.py +++ b/python/cugraph/cugraph/tests/generators/test_rmat_mg.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -34,7 +34,9 @@ _scale_values = [2, 4, 16] _scale_test_ids = [f"scale={x}" for x in _scale_values] _graph_types = [cugraph.Graph, None, int] -_graph_test_ids = [f"create_using={getattr(x,'__name__',str(x))}" for x in _graph_types] +_graph_test_ids = [ + f"create_using={getattr(x, '__name__', str(x))}" for x in _graph_types +] def _call_rmat(scale, num_edges, create_using, mg=True): diff --git a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py index 65bcce78771..3c5d6428001 100644 --- a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py +++ b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py @@ -119,7 +119,7 @@ def test_bulk_sampler_remainder(scratch_dir): assert b in recovered_samples["batch_id"].values_host.tolist() for x in range(0, 6, 2): - subdir = f"{x}-{x+1}" + subdir = f"{x}-{x + 1}" df = cudf.read_parquet(os.path.join(samples_path, f"batch={subdir}.parquet")) assert ((df.batch_id == x) | (df.batch_id == (x + 1))).all() diff --git a/python/cugraph/cugraph/tests/structure/test_graph_mg.py b/python/cugraph/cugraph/tests/structure/test_graph_mg.py index cba61731e9a..f2cc1583f93 100644 --- a/python/cugraph/cugraph/tests/structure/test_graph_mg.py +++ b/python/cugraph/cugraph/tests/structure/test_graph_mg.py @@ -303,7 +303,7 @@ def test_mg_graph_serializable(dask_client, input_combo): G = input_combo["MGGraph"] dask_client.publish_dataset(shared_g=G) shared_g = dask_client.get_dataset("shared_g") - assert type(shared_g) == type(G) + assert type(shared_g) is type(G) assert G.number_of_vertices() == shared_g.number_of_vertices() assert G.number_of_edges() == shared_g.number_of_edges() # cleanup @@ -314,7 +314,7 @@ def test_mg_graph_serializable(dask_client, input_combo): def test_mg_graph_copy(): G = cugraph.MultiGraph(directed=True) G_c = copy.deepcopy(G) - assert type(G) == type(G_c) + assert type(G) is type(G_c) @pytest.mark.mg diff --git a/python/cugraph/cugraph/tests/traversal/test_sssp.py b/python/cugraph/cugraph/tests/traversal/test_sssp.py index 58288e022e8..ceb6040275d 100644 --- a/python/cugraph/cugraph/tests/traversal/test_sssp.py +++ b/python/cugraph/cugraph/tests/traversal/test_sssp.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -486,7 +486,7 @@ def test_scipy_api_compat(): distances = cugraph.shortest_path( input_coo_matrix, source=0, return_predecessors=False ) - assert type(distances) != tuple + assert type(distances) is not tuple with pytest.raises(ValueError): cugraph.shortest_path(input_coo_matrix, source=0, unweighted=False) diff --git a/python/cugraph/cugraph/traversal/sssp.py b/python/cugraph/cugraph/traversal/sssp.py index 5ab97e60390..bb98b5a9a29 100644 --- a/python/cugraph/cugraph/traversal/sssp.py +++ b/python/cugraph/cugraph/traversal/sssp.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -36,7 +36,7 @@ def _ensure_args( # checks common to all input types if (method is not None) and (method != "auto"): raise ValueError("only 'auto' is currently accepted for method") - if (indices is not None) and (type(indices) == list): + if (indices is not None) and (type(indices) is list): raise ValueError("indices currently cannot be a list-like type") if (indices is not None) and (source is not None): raise TypeError("cannot specify both 'source' and 'indices'") @@ -70,9 +70,11 @@ def _ensure_args( # Check for non-Graph-type inputs else: - if (directed is not None) and (type(directed) != bool): + if (directed is not None) and (type(directed) is not bool): raise ValueError("'directed' must be a bool") - if (return_predecessors is not None) and (type(return_predecessors) != bool): + if (return_predecessors is not None) and ( + type(return_predecessors) is not bool + ): raise ValueError("'return_predecessors' must be a bool") if (unweighted is not None) and (unweighted is not True): raise ValueError("'unweighted' currently must be True if " "specified") From eb8963ab40f6ec7f8260e5e74dae5c7421b455d9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 18 Sep 2024 14:42:15 -0500 Subject: [PATCH 6/8] Use CI workflow branch 'branch-24.10' again (#4654) All RAPIDS libraries have been updated with Python 3.12 support, so Python 3.12 changes have been merged into `branch-24.10` of `shared-workflows`: https://github.com/rapidsai/shared-workflows/pull/213 This updates GitHub Actions configs here to that branch. --- .github/workflows/build.yaml | 32 +++++++++++++------------- .github/workflows/pr.yaml | 44 ++++++++++++++++++------------------ .github/workflows/test.yaml | 18 +++++++-------- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3ad8aef5820..bc489ffd3f0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -47,7 +47,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -83,7 +83,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -93,7 +93,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -106,7 +106,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -116,7 +116,7 @@ jobs: wheel-build-nx-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -126,7 +126,7 @@ jobs: wheel-publish-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -136,7 +136,7 @@ jobs: wheel-build-cugraph-dgl: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -146,7 +146,7 @@ jobs: wheel-publish-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -156,7 +156,7 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-publish-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -166,7 +166,7 @@ jobs: wheel-publish-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -175,7 +175,7 @@ jobs: package-name: cugraph-pyg wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -185,7 +185,7 @@ jobs: wheel-publish-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d25a0b81b13..dacd9a93399 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,29 +34,29 @@ jobs: - wheel-tests-cugraph-equivariant - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request node_type: cpu32 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: pull-request enable_check_symbols: true @@ -64,19 +64,19 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: pull-request conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -86,7 +86,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -96,7 +96,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh @@ -107,14 +107,14 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_cugraph.sh @@ -124,35 +124,35 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_cugraph.sh wheel-build-nx-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_nx-cugraph.sh wheel-tests-nx-cugraph: needs: wheel-build-nx-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh wheel-build-cugraph-dgl: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_cugraph-dgl.sh wheel-tests-cugraph-dgl: needs: wheel-build-cugraph-dgl secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_cugraph-dgl.sh @@ -160,35 +160,35 @@ jobs: wheel-build-cugraph-pyg: needs: wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_cugraph-pyg.sh wheel-tests-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_cugraph-equivariant.sh wheel-tests-cugraph-equivariant: needs: wheel-build-cugraph-equivariant secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_cugraph-equivariant.sh matrix_filter: map(select(.ARCH == "amd64")) devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 32447711811..957d29ce72b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +42,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: script: ci/test_wheel_pylibcugraph.sh wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: script: ci/test_wheel_cugraph.sh wheel-tests-nx-cugraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: script: ci/test_wheel_nx-cugraph.sh wheel-tests-cugraph-dgl: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-equivariant: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} From 8b1a437275b2a0c4477afec4cc41eb3281b12517 Mon Sep 17 00:00:00 2001 From: Mike McCarty Date: Thu, 19 Sep 2024 01:43:32 -0400 Subject: [PATCH 7/8] Recommending `miniforge` for conda install (#4650) Recommending `miniforge` for conda install in installation docs. Authors: - Mike McCarty (https://github.com/mmccarty) - Bradley Dice (https://github.com/bdice) - Alex Barghi (https://github.com/alexbarghi-nv) Approvers: - Rick Ratzel (https://github.com/rlratzel) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4650 --- .../notebooks/get_node_storage.ipynb | 2 +- .../heterogeneous_dataloader_benchmark.ipynb | 2 +- .../homogenous_dataloader_benchmark.ipynb | 6 +- .../cugraph/notebooks/feature_storage.ipynb | 2 +- .../source/installation/getting_cugraph.md | 2 +- .../installation/getting_wholegraph.md | 2 +- notebooks/demo/mg_pagerank.ipynb | 280 +++++++++--------- 7 files changed, 148 insertions(+), 148 deletions(-) diff --git a/benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb b/benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb index 95b456c7812..4681c8ec825 100644 --- a/benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb +++ b/benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb @@ -18,7 +18,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } diff --git a/benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb b/benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb index d3b054bb0ee..2c4a934827a 100644 --- a/benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb +++ b/benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb @@ -176,7 +176,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/dgl/dataloading/dataloader.py:859: DGLWarning: Dataloader CPU affinity opt is not enabled, consider switching it on (see enable_cpu_affinity() or CPU best practices for DGL [https://docs.dgl.ai/tutorials/cpu/cpu_best_practises.html])\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/dgl/dataloading/dataloader.py:859: DGLWarning: Dataloader CPU affinity opt is not enabled, consider switching it on (see enable_cpu_affinity() or CPU best practices for DGL [https://docs.dgl.ai/tutorials/cpu/cpu_best_practises.html])\n", " dgl_warning(f'Dataloader CPU affinity opt is not enabled, consider switching it on '\n" ] }, diff --git a/benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb b/benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb index ea1e9b34965..ecd111dabdf 100644 --- a/benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb +++ b/benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb @@ -26,7 +26,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } @@ -190,7 +190,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/dgl/dataloading/dataloader.py:859: DGLWarning: Dataloader CPU affinity opt is not enabled, consider switching it on (see enable_cpu_affinity() or CPU best practices for DGL [https://docs.dgl.ai/tutorials/cpu/cpu_best_practises.html])\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/dgl/dataloading/dataloader.py:859: DGLWarning: Dataloader CPU affinity opt is not enabled, consider switching it on (see enable_cpu_affinity() or CPU best practices for DGL [https://docs.dgl.ai/tutorials/cpu/cpu_best_practises.html])\n", " dgl_warning(f'Dataloader CPU affinity opt is not enabled, consider switching it on '\n" ] }, @@ -278,7 +278,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/distributed/worker.py:2988: UserWarning: Large object of size 1.42 MiB detected in task graph: \n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/distributed/worker.py:2988: UserWarning: Large object of size 1.42 MiB detected in task graph: \n", " [b'\\xad\\xd1\\xe3\\x9c\\x96\\x83O\\xb3\\xba1\\x86\\x94\\xb6\\ ... =int32), False]\n", "Consider scattering large objects ahead of time\n", "with client.scatter to reduce scheduler burden and \n", diff --git a/benchmarks/cugraph/notebooks/feature_storage.ipynb b/benchmarks/cugraph/notebooks/feature_storage.ipynb index 7413ac00cde..440d76fbdb4 100644 --- a/benchmarks/cugraph/notebooks/feature_storage.ipynb +++ b/benchmarks/cugraph/notebooks/feature_storage.ipynb @@ -18,7 +18,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/datasets/vjawa/miniconda3/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + "/datasets/vjawa/miniforge/envs/all_cuda-115_arch-x86_64/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } diff --git a/docs/cugraph/source/installation/getting_cugraph.md b/docs/cugraph/source/installation/getting_cugraph.md index 41ec9a67e1f..01bc9e379c9 100644 --- a/docs/cugraph/source/installation/getting_cugraph.md +++ b/docs/cugraph/source/installation/getting_cugraph.md @@ -21,7 +21,7 @@ The RAPIDS Docker containers contain all RAPIDS packages, including all from cuG ## Conda -It is easy to install cuGraph using conda. You can get a minimal conda installation with [Miniconda](https://conda.io/miniconda.html) or get the full installation with [Anaconda](https://www.anaconda.com/download). +It is easy to install cuGraph using conda. You can get a minimal conda installation with [miniforge](https://github.com/conda-forge/miniforge). cuGraph Conda packages * cugraph - this will also import: diff --git a/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md b/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md index 57314dcd426..80c666d6593 100644 --- a/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md +++ b/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md @@ -21,7 +21,7 @@ The RAPIDS Docker containers (as of Release 23.10) contain all RAPIDS packages, ## Conda -It is easy to install WholeGraph using conda. You can get a minimal conda installation with [Miniconda](https://conda.io/miniconda.html) or get the full installation with [Anaconda](https://www.anaconda.com/download). +It is easy to install WholeGraph using conda. You can get a minimal conda installation with [miniforge](https://github.com/conda-forge/miniforge). WholeGraph conda packages * libwholegraph diff --git a/notebooks/demo/mg_pagerank.ipynb b/notebooks/demo/mg_pagerank.ipynb index bb333048450..e3314f80b3c 100644 --- a/notebooks/demo/mg_pagerank.ipynb +++ b/notebooks/demo/mg_pagerank.ipynb @@ -219,250 +219,250 @@ "text": [ "2023-05-12 09:25:01,974 - distributed.sizeof - WARNING - Sizeof calculation failed. Defaulting to 0.95 MiB\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", " return sizeof(obj)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", " return meth(arg, *args, **kwargs)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", " + df._index.memory_usage()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", " if self.levels:\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", " self._compute_levels_and_codes()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", " code, cats = cudf.Series._from_data({None: col}).factorize()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", " return cudf.core.algorithms.factorize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", " labels = values._column._label_encoding(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", " order = order.take(left_gather_map, check_bounds=False).argsort()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", " return self.as_frame()._get_sorted_inds(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", " return libcudf.sort.order_by(to_sort, ascending, na_position)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", " File \"sort.pyx\", line 141, in cudf._lib.sort.order_by\n", - "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniconda3/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", + "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniforge/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", "2023-05-12 09:25:01,976 - distributed.sizeof - WARNING - Sizeof calculation failed. Defaulting to 0.95 MiB\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", " return sizeof(obj)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", " return meth(arg, *args, **kwargs)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", " + df._index.memory_usage()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", " if self.levels:\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", " self._compute_levels_and_codes()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", " code, cats = cudf.Series._from_data({None: col}).factorize()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", " return cudf.core.algorithms.factorize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", " labels = values._column._label_encoding(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", " order = order.take(left_gather_map, check_bounds=False).argsort()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", " return self.as_frame()._get_sorted_inds(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", " return libcudf.sort.order_by(to_sort, ascending, na_position)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", " File \"sort.pyx\", line 141, in cudf._lib.sort.order_by\n", - "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniconda3/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", + "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniforge/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", "2023-05-12 09:25:03,767 - distributed.sizeof - WARNING - Sizeof calculation failed. Defaulting to 0.95 MiB\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", " return sizeof(obj)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", " return meth(arg, *args, **kwargs)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", " + df._index.memory_usage()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", " if self.levels:\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", " self._compute_levels_and_codes()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", " code, cats = cudf.Series._from_data({None: col}).factorize()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", " return cudf.core.algorithms.factorize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", " labels = values._column._label_encoding(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", " order = order.take(left_gather_map, check_bounds=False).argsort()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", " return self.as_frame()._get_sorted_inds(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", " return libcudf.sort.order_by(to_sort, ascending, na_position)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", " File \"sort.pyx\", line 141, in cudf._lib.sort.order_by\n", - "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniconda3/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", + "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniforge/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", "2023-05-12 09:25:03,768 - distributed.sizeof - WARNING - Sizeof calculation failed. Defaulting to 0.95 MiB\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/sizeof.py\", line 17, in safe_sizeof\n", " return sizeof(obj)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/utils.py\", line 642, in __call__\n", " return meth(arg, *args, **kwargs)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask_cudf/backends.py\", line 430, in sizeof_cudf_dataframe\n", " + df._index.memory_usage()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 1594, in memory_usage\n", " if self.levels:\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 605, in levels\n", " self._compute_levels_and_codes()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/multiindex.py\", line 748, in _compute_levels_and_codes\n", " code, cats = cudf.Series._from_data({None: col}).factorize()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/single_column_frame.py\", line 311, in factorize\n", " return cudf.core.algorithms.factorize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/algorithms.py\", line 138, in factorize\n", " labels = values._column._label_encoding(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1385, in _label_encoding\n", " order = order.take(left_gather_map, check_bounds=False).argsort()\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1101, in argsort\n", " return self.as_frame()._get_sorted_inds(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 1572, in _get_sorted_inds\n", " return libcudf.sort.order_by(to_sort, ascending, na_position)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", " File \"sort.pyx\", line 141, in cudf._lib.sort.order_by\n", - "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniconda3/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", + "MemoryError: std::bad_alloc: out_of_memory: CUDA error at: /home/dacosta/miniforge/envs/cugraph_0411/include/rmm/mr/device/cuda_memory_resource.hpp\n", "2023-05-12 09:25:03,820 - distributed.worker - ERROR - Could not deserialize task ('len-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d', 1)\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2923, in loads_function\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2923, in loads_function\n", " result = cache_loads[bytes_object]\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/collections.py\", line 24, in __getitem__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/collections.py\", line 24, in __getitem__\n", " value = super().__getitem__(key)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/collections/__init__.py\", line 1106, in __getitem__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/collections/__init__.py\", line 1106, in __getitem__\n", " raise KeyError(key)\n", "KeyError: b'\\x80\\x05\\x95>\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x11dask.optimization\\x94\\x8c\\x10SubgraphCallable\\x94\\x93\\x94(}\\x94(\\x8cKlen-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d\\x94\\x8cZassign-getitem-len-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d\\x94\\x8c*rename-01db283bd79fee66f232920c8dc6b55e_.0\\x94\\x8c;getitem-to_frame-rename-01db283bd79fee66f232920c8dc6b55e_.0\\x94\\x8c+getitem-3499fd71ac25ebbc1a06991edea6067c_.0\\x94\\x8c\\t_operator\\x94\\x8c\\x07getitem\\x94\\x93\\x94\\x8c/reset_index-f4c18304ca92859ccd09f44cf89b4b43_.0\\x94\\x8c\\x13__dask_blockwise__1\\x94\\x87\\x94h\\x0c(\\x8c\\ndask.utils\\x94\\x8c\\x05apply\\x94\\x93\\x94h\\x0f\\x8c\\x0cmethodcaller\\x94\\x93\\x94\\x8c\\x0breset_index\\x94\\x85\\x94R\\x94]\\x94\\x8c\\x13__dask_blockwise__5\\x94a\\x8c\\x08builtins\\x94\\x8c\\x04dict\\x94\\x93\\x94]\\x94]\\x94(\\x8c\\x04drop\\x94\\x89ea\\x86\\x94t\\x94h\\x07(h\\x11\\x8c\\x13dask.dataframe.core\\x94\\x8c\\x11apply_and_enforce\\x94\\x93\\x94]\\x94((h\\x11h#]\\x94h\\x0bh\\x0c\\x8c\\x13__dask_blockwise__0\\x94\\x87\\x94ah\\x1b]\\x94(]\\x94(\\x8c\\x05_func\\x94h\\x13\\x8c\\x08to_frame\\x94\\x85\\x94R\\x94e]\\x94(\\x8c\\x05_meta\\x94\\x8c\\x08builtins\\x94\\x8c\\x07getattr\\x94\\x93\\x94\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94\\x8c\\x10host_deserialize\\x94\\x86\\x94R\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94C0\\x80\\x04\\x95%\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94.\\x94\\x8c\\x0ccolumn_names\\x94C\\x14\\x80\\x04\\x95\\t\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x03src\\x94\\x85\\x94.\\x94\\x8c\\x07columns\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94C=\\x80\\x04\\x952\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x1acudf.core.column.numerical\\x94\\x8c\\x0fNumericalColumn\\x94\\x93\\x94.\\x94\\x8c\\x05dtype\\x94CB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i4\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94\\x8c\\x18dtype-is-cudf-serialized\\x94\\x89\\x8c\\x04data\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94CI\\x80\\x04\\x95>\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x14SpillableBufferSlice\\x94\\x93\\x94.\\x94\\x8c\\x0bframe_count\\x94K\\x01u\\x8c\\x04mask\\x94}\\x94(hGCD\\x80\\x04\\x959\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x0fSpillableBuffer\\x94\\x93\\x94.\\x94hIK\\x01u\\x8c\\x04size\\x94K\\x00hIK\\x02u\\x85\\x94\\x8c\\x05index\\x94}\\x94(\\x8c\\x0cindex_column\\x94}\\x94(\\x8c\\x05start\\x94K\\x00\\x8c\\x04stop\\x94K\\x00\\x8c\\x04step\\x94K\\x01u\\x8c\\x04name\\x94C\\x04\\x80\\x04N.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i8\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94\\x8c\\x0ftype-serialized\\x94C-\\x80\\x04\\x95\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x0fcudf.core.index\\x94\\x8c\\nRangeIndex\\x94\\x93\\x94.\\x94hIK\\x00u\\x8c\\x11index_frame_count\\x94K\\x00\\x8c\\x07is-cuda\\x94]\\x94(\\x88\\x88e\\x8c\\x07lengths\\x94]\\x94(K\\x00K\\x00e\\x8c\\twriteable\\x94NN\\x86\\x94u]\\x94(\\x8c\\x12numpy.core.numeric\\x94\\x8c\\x0b_frombuffer\\x94\\x93\\x94(C\\x00\\x94\\x8c\\x05numpy\\x94hB\\x93\\x94\\x8c\\x02u1\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01|\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94bK\\x00\\x85\\x94\\x8c\\x01C\\x94t\\x94R\\x94he(C\\x00\\x94hkK\\x00\\x85\\x94hot\\x94R\\x94e\\x86\\x94R\\x94ee\\x86\\x94t\\x94\\x8c\\x13__dask_blockwise__2\\x94eh\\x1b]\\x94(]\\x94(h*h\\x13\\x8c\\x06rename\\x94\\x85\\x94R\\x94e]\\x94(h/h2h5h6\\x86\\x94R\\x94}\\x94(h:C0\\x80\\x04\\x95%\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94.\\x94h}\\x94(h@C=\\x80\\x04\\x952\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x1acudf.core.column.numerical\\x94\\x8c\\x0fNumericalColumn\\x94\\x93\\x94.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i4\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94hD\\x89hE}\\x94(hGCI\\x80\\x04\\x95>\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x14SpillableBufferSlice\\x94\\x93\\x94.\\x94hIK\\x01uhMK\\x00hIK\\x01u\\x85\\x94hO}\\x94(hQ}\\x94(hSK\\x00hTK\\x00hUK\\x01uhVC\\x04\\x80\\x04N.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i8\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94hYC-\\x80\\x04\\x95\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x0fcudf.core.index\\x94\\x8c\\nRangeIndex\\x94\\x93\\x94.\\x94hIK\\x00uh[K\\x00h\\\\]\\x94\\x88ah^]\\x94K\\x00ah`N\\x85\\x94u]\\x94he(C\\x00\\x94hkK\\x00\\x85\\x94hot\\x94R\\x94a\\x86\\x94R\\x94e]\\x94(h>h\\x1b]\\x94]\\x94(\\x8c\\x03src\\x94h\\x9eea\\x86\\x94ee\\x86\\x94t\\x94h\\x05(h\\x11h!\\x8c\\x10_reduction_chunk\\x94\\x93\\x94]\\x94h\\x0b(\\x8c\\x16dask.dataframe.methods\\x94\\x8c\\x06assign\\x94\\x93\\x94h\\x06h\\rh\\x08t\\x94h&\\x87\\x94ah\\x1b]\\x94]\\x94(\\x8c\\taca_chunk\\x94h0\\x8c\\x03len\\x94\\x93\\x94ea\\x86\\x94t\\x94\\x8c\\x13__dask_blockwise__0\\x94h\\x9e\\x8c\\x13__dask_blockwise__1\\x94\\x8c\\x03dst\\x94\\x8c\\x13__dask_blockwise__2\\x94N\\x8c\\x13__dask_blockwise__3\\x94\\x8c)to_frame-804980ae30b71d28f0a6bd3d5b7610f9\\x94\\x8c\\x13__dask_blockwise__4\\x94\\x8c(getitem-15414b72be12e28054238b44933937ab\\x94\\x8c\\x13__dask_blockwise__6\\x94\\x8c3cudf-aggregate-agg-c50c2d97de169ca4f41e43a92a042630\\x94uh\\x04\\x8c\\x13__dask_blockwise__5\\x94\\x85\\x94\\x8c6subgraph_callable-b4ca530e-8895-432e-b553-40a7b5892ab2\\x94t\\x94R\\x94.'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2244, in execute\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2244, in execute\n", " function, args, kwargs = await self._maybe_deserialize_task(ts)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2216, in _maybe_deserialize_task\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2216, in _maybe_deserialize_task\n", " function, args, kwargs = _deserialize(*ts.run_spec)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2937, in _deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2937, in _deserialize\n", " function = loads_function(function)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2925, in loads_function\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2925, in loads_function\n", " result = pickle.loads(bytes_object)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py\", line 96, in loads\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py\", line 96, in loads\n", " return pickle.loads(x)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 176, in host_deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 176, in host_deserialize\n", " obj = cls.device_deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 130, in device_deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 130, in device_deserialize\n", " return typ.deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py\", line 1019, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py\", line 1019, in deserialize\n", " obj = super().deserialize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 106, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 106, in deserialize\n", " columns = deserialize_columns(header[\"columns\"], frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 2450, in deserialize_columns\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 2450, in deserialize_columns\n", " colobj = col_typ.deserialize(meta, frames[:col_frame_count])\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1216, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1216, in deserialize\n", " data, frames = unpack(header[\"data\"], frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1204, in unpack\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1204, in unpack\n", " obj = klass.deserialize(header, frames[:count])\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 574, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 574, in deserialize\n", " return SpillableBuffer.deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py\", line 335, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py\", line 335, in deserialize\n", " return cls._from_device_memory(frame)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 235, in _from_device_memory\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 235, in _from_device_memory\n", " ret._finalize_init(ptr_desc={\"type\": \"gpu\"}, exposed=exposed)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 206, in _finalize_init\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 206, in _finalize_init\n", " raise ValueError(\n", "ValueError: cannot create without a global spill manager\n", "2023-05-12 09:25:03,817 - distributed.worker - ERROR - Could not deserialize task ('len-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d', 0)\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2923, in loads_function\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2923, in loads_function\n", " result = cache_loads[bytes_object]\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/collections.py\", line 24, in __getitem__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/collections.py\", line 24, in __getitem__\n", " value = super().__getitem__(key)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/collections/__init__.py\", line 1106, in __getitem__\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/collections/__init__.py\", line 1106, in __getitem__\n", " raise KeyError(key)\n", "KeyError: b'\\x80\\x05\\x95>\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x11dask.optimization\\x94\\x8c\\x10SubgraphCallable\\x94\\x93\\x94(}\\x94(\\x8cKlen-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d\\x94\\x8cZassign-getitem-len-chunk-319fe46af5510615b2fae86c6e732896-841a12bf4568ebb80eb2030cc4d9651d\\x94\\x8c*rename-01db283bd79fee66f232920c8dc6b55e_.0\\x94\\x8c;getitem-to_frame-rename-01db283bd79fee66f232920c8dc6b55e_.0\\x94\\x8c+getitem-3499fd71ac25ebbc1a06991edea6067c_.0\\x94\\x8c\\t_operator\\x94\\x8c\\x07getitem\\x94\\x93\\x94\\x8c/reset_index-f4c18304ca92859ccd09f44cf89b4b43_.0\\x94\\x8c\\x13__dask_blockwise__1\\x94\\x87\\x94h\\x0c(\\x8c\\ndask.utils\\x94\\x8c\\x05apply\\x94\\x93\\x94h\\x0f\\x8c\\x0cmethodcaller\\x94\\x93\\x94\\x8c\\x0breset_index\\x94\\x85\\x94R\\x94]\\x94\\x8c\\x13__dask_blockwise__5\\x94a\\x8c\\x08builtins\\x94\\x8c\\x04dict\\x94\\x93\\x94]\\x94]\\x94(\\x8c\\x04drop\\x94\\x89ea\\x86\\x94t\\x94h\\x07(h\\x11\\x8c\\x13dask.dataframe.core\\x94\\x8c\\x11apply_and_enforce\\x94\\x93\\x94]\\x94((h\\x11h#]\\x94h\\x0bh\\x0c\\x8c\\x13__dask_blockwise__0\\x94\\x87\\x94ah\\x1b]\\x94(]\\x94(\\x8c\\x05_func\\x94h\\x13\\x8c\\x08to_frame\\x94\\x85\\x94R\\x94e]\\x94(\\x8c\\x05_meta\\x94\\x8c\\x08builtins\\x94\\x8c\\x07getattr\\x94\\x93\\x94\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94\\x8c\\x10host_deserialize\\x94\\x86\\x94R\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94C0\\x80\\x04\\x95%\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94.\\x94\\x8c\\x0ccolumn_names\\x94C\\x14\\x80\\x04\\x95\\t\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x03src\\x94\\x85\\x94.\\x94\\x8c\\x07columns\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94C=\\x80\\x04\\x952\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x1acudf.core.column.numerical\\x94\\x8c\\x0fNumericalColumn\\x94\\x93\\x94.\\x94\\x8c\\x05dtype\\x94CB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i4\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94\\x8c\\x18dtype-is-cudf-serialized\\x94\\x89\\x8c\\x04data\\x94}\\x94(\\x8c\\x0ftype-serialized\\x94CI\\x80\\x04\\x95>\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x14SpillableBufferSlice\\x94\\x93\\x94.\\x94\\x8c\\x0bframe_count\\x94K\\x01u\\x8c\\x04mask\\x94}\\x94(hGCD\\x80\\x04\\x959\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x0fSpillableBuffer\\x94\\x93\\x94.\\x94hIK\\x01u\\x8c\\x04size\\x94K\\x00hIK\\x02u\\x85\\x94\\x8c\\x05index\\x94}\\x94(\\x8c\\x0cindex_column\\x94}\\x94(\\x8c\\x05start\\x94K\\x00\\x8c\\x04stop\\x94K\\x00\\x8c\\x04step\\x94K\\x01u\\x8c\\x04name\\x94C\\x04\\x80\\x04N.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i8\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94\\x8c\\x0ftype-serialized\\x94C-\\x80\\x04\\x95\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x0fcudf.core.index\\x94\\x8c\\nRangeIndex\\x94\\x93\\x94.\\x94hIK\\x00u\\x8c\\x11index_frame_count\\x94K\\x00\\x8c\\x07is-cuda\\x94]\\x94(\\x88\\x88e\\x8c\\x07lengths\\x94]\\x94(K\\x00K\\x00e\\x8c\\twriteable\\x94NN\\x86\\x94u]\\x94(\\x8c\\x12numpy.core.numeric\\x94\\x8c\\x0b_frombuffer\\x94\\x93\\x94(C\\x00\\x94\\x8c\\x05numpy\\x94hB\\x93\\x94\\x8c\\x02u1\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01|\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94bK\\x00\\x85\\x94\\x8c\\x01C\\x94t\\x94R\\x94he(C\\x00\\x94hkK\\x00\\x85\\x94hot\\x94R\\x94e\\x86\\x94R\\x94ee\\x86\\x94t\\x94\\x8c\\x13__dask_blockwise__2\\x94eh\\x1b]\\x94(]\\x94(h*h\\x13\\x8c\\x06rename\\x94\\x85\\x94R\\x94e]\\x94(h/h2h5h6\\x86\\x94R\\x94}\\x94(h:C0\\x80\\x04\\x95%\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x13cudf.core.dataframe\\x94\\x8c\\tDataFrame\\x94\\x93\\x94.\\x94h}\\x94(h@C=\\x80\\x04\\x952\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x1acudf.core.column.numerical\\x94\\x8c\\x0fNumericalColumn\\x94\\x93\\x94.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i4\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94hD\\x89hE}\\x94(hGCI\\x80\\x04\\x95>\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c!cudf.core.buffer.spillable_buffer\\x94\\x8c\\x14SpillableBufferSlice\\x94\\x93\\x94.\\x94hIK\\x01uhMK\\x00hIK\\x01u\\x85\\x94hO}\\x94(hQ}\\x94(hSK\\x00hTK\\x00hUK\\x01uhVC\\x04\\x80\\x04N.\\x94hBCB\\x80\\x04\\x957\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x05numpy\\x94\\x8c\\x05dtype\\x94\\x93\\x94\\x8c\\x02i8\\x94\\x89\\x88\\x87\\x94R\\x94(K\\x03\\x8c\\x01<\\x94NNNJ\\xff\\xff\\xff\\xffJ\\xff\\xff\\xff\\xffK\\x00t\\x94b.\\x94hYC-\\x80\\x04\\x95\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\x0fcudf.core.index\\x94\\x8c\\nRangeIndex\\x94\\x93\\x94.\\x94hIK\\x00uh[K\\x00h\\\\]\\x94\\x88ah^]\\x94K\\x00ah`N\\x85\\x94u]\\x94he(C\\x00\\x94hkK\\x00\\x85\\x94hot\\x94R\\x94a\\x86\\x94R\\x94e]\\x94(h>h\\x1b]\\x94]\\x94(\\x8c\\x03src\\x94h\\x9eea\\x86\\x94ee\\x86\\x94t\\x94h\\x05(h\\x11h!\\x8c\\x10_reduction_chunk\\x94\\x93\\x94]\\x94h\\x0b(\\x8c\\x16dask.dataframe.methods\\x94\\x8c\\x06assign\\x94\\x93\\x94h\\x06h\\rh\\x08t\\x94h&\\x87\\x94ah\\x1b]\\x94]\\x94(\\x8c\\taca_chunk\\x94h0\\x8c\\x03len\\x94\\x93\\x94ea\\x86\\x94t\\x94\\x8c\\x13__dask_blockwise__0\\x94h\\x9e\\x8c\\x13__dask_blockwise__1\\x94\\x8c\\x03dst\\x94\\x8c\\x13__dask_blockwise__2\\x94N\\x8c\\x13__dask_blockwise__3\\x94\\x8c)to_frame-804980ae30b71d28f0a6bd3d5b7610f9\\x94\\x8c\\x13__dask_blockwise__4\\x94\\x8c(getitem-15414b72be12e28054238b44933937ab\\x94\\x8c\\x13__dask_blockwise__6\\x94\\x8c3cudf-aggregate-agg-c50c2d97de169ca4f41e43a92a042630\\x94uh\\x04\\x8c\\x13__dask_blockwise__5\\x94\\x85\\x94\\x8c6subgraph_callable-b4ca530e-8895-432e-b553-40a7b5892ab2\\x94t\\x94R\\x94.'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2244, in execute\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2244, in execute\n", " function, args, kwargs = await self._maybe_deserialize_task(ts)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2216, in _maybe_deserialize_task\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2216, in _maybe_deserialize_task\n", " function, args, kwargs = _deserialize(*ts.run_spec)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py\", line 79, in inner\n", " return func(*args, **kwds)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2937, in _deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2937, in _deserialize\n", " function = loads_function(function)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2925, in loads_function\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py\", line 2925, in loads_function\n", " result = pickle.loads(bytes_object)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py\", line 96, in loads\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py\", line 96, in loads\n", " return pickle.loads(x)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 176, in host_deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 176, in host_deserialize\n", " obj = cls.device_deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 130, in device_deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py\", line 130, in device_deserialize\n", " return typ.deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py\", line 1019, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py\", line 1019, in deserialize\n", " obj = super().deserialize(\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 106, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py\", line 106, in deserialize\n", " columns = deserialize_columns(header[\"columns\"], frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 2450, in deserialize_columns\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 2450, in deserialize_columns\n", " colobj = col_typ.deserialize(meta, frames[:col_frame_count])\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1216, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1216, in deserialize\n", " data, frames = unpack(header[\"data\"], frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1204, in unpack\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py\", line 1204, in unpack\n", " obj = klass.deserialize(header, frames[:count])\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 574, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 574, in deserialize\n", " return SpillableBuffer.deserialize(header, frames)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py\", line 335, in deserialize\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py\", line 335, in deserialize\n", " return cls._from_device_memory(frame)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 235, in _from_device_memory\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 235, in _from_device_memory\n", " ret._finalize_init(ptr_desc={\"type\": \"gpu\"}, exposed=exposed)\n", - " File \"/home/dacosta/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 206, in _finalize_init\n", + " File \"/home/dacosta/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py\", line 206, in _finalize_init\n", " raise ValueError(\n", "ValueError: cannot create without a global spill manager\n" ] @@ -475,34 +475,34 @@ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[6], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39m# Create a directed graph using the source (src) and destination (dst) vertex pairs from the Dataframe \u001b[39;00m\n\u001b[1;32m 2\u001b[0m G \u001b[39m=\u001b[39m cugraph\u001b[39m.\u001b[39mGraph(directed\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[0;32m----> 3\u001b[0m G\u001b[39m.\u001b[39;49mfrom_dask_cudf_edgelist(e_list, source\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39msrc\u001b[39;49m\u001b[39m'\u001b[39;49m, destination\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39mdst\u001b[39;49m\u001b[39m'\u001b[39;49m)\n\u001b[1;32m 5\u001b[0m \u001b[39m# Print time\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mRead, load and renumber: \u001b[39m\u001b[39m\"\u001b[39m, time\u001b[39m.\u001b[39mtime()\u001b[39m-\u001b[39mt_start, \u001b[39m\"\u001b[39m\u001b[39ms\u001b[39m\u001b[39m\"\u001b[39m)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cugraph/structure/graph_classes.py:309\u001b[0m, in \u001b[0;36mGraph.from_dask_cudf_edgelist\u001b[0;34m(self, input_ddf, source, destination, edge_attr, renumber, store_transposed, legacy_renum_only)\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_Impl\u001b[39m.\u001b[39medgelist \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 308\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mGraph already has values\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 309\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_Impl\u001b[39m.\u001b[39;49m_simpleDistributedGraphImpl__from_edgelist(\n\u001b[1;32m 310\u001b[0m input_ddf,\n\u001b[1;32m 311\u001b[0m source,\n\u001b[1;32m 312\u001b[0m destination,\n\u001b[1;32m 313\u001b[0m edge_attr,\n\u001b[1;32m 314\u001b[0m renumber,\n\u001b[1;32m 315\u001b[0m store_transposed,\n\u001b[1;32m 316\u001b[0m legacy_renum_only,\n\u001b[1;32m 317\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cugraph/structure/graph_implementation/simpleDistributedGraph.py:272\u001b[0m, in \u001b[0;36msimpleDistributedGraphImpl.__from_edgelist\u001b[0;34m(self, input_ddf, source, destination, edge_attr, renumber, store_transposed, legacy_renum_only)\u001b[0m\n\u001b[1;32m 268\u001b[0m dst_col_name \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrenumber_map\u001b[39m.\u001b[39mrenumbered_dst_col_name\n\u001b[1;32m 270\u001b[0m ddf \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39medgelist\u001b[39m.\u001b[39medgelist_df\n\u001b[0;32m--> 272\u001b[0m num_edges \u001b[39m=\u001b[39m \u001b[39mlen\u001b[39;49m(ddf)\n\u001b[1;32m 273\u001b[0m edge_data \u001b[39m=\u001b[39m get_distributed_data(ddf)\n\u001b[1;32m 275\u001b[0m graph_props \u001b[39m=\u001b[39m GraphProperties(\n\u001b[1;32m 276\u001b[0m is_multigraph\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mproperties\u001b[39m.\u001b[39mmulti_edge,\n\u001b[1;32m 277\u001b[0m is_symmetric\u001b[39m=\u001b[39m\u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mproperties\u001b[39m.\u001b[39mdirected,\n\u001b[1;32m 278\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/dataframe/core.py:4775\u001b[0m, in \u001b[0;36mDataFrame.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 4773\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m\u001b[39m__len__\u001b[39m()\n\u001b[1;32m 4774\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 4775\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlen\u001b[39;49m(s)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/dataframe/core.py:843\u001b[0m, in \u001b[0;36m_Frame.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 840\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__len__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 841\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mreduction(\n\u001b[1;32m 842\u001b[0m \u001b[39mlen\u001b[39;49m, np\u001b[39m.\u001b[39;49msum, token\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mlen\u001b[39;49m\u001b[39m\"\u001b[39;49m, meta\u001b[39m=\u001b[39;49m\u001b[39mint\u001b[39;49m, split_every\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m\n\u001b[0;32m--> 843\u001b[0m )\u001b[39m.\u001b[39;49mcompute()\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/base.py:314\u001b[0m, in \u001b[0;36mDaskMethodsMixin.compute\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcompute\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 291\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Compute this dask collection\u001b[39;00m\n\u001b[1;32m 292\u001b[0m \n\u001b[1;32m 293\u001b[0m \u001b[39m This turns a lazy Dask collection into its in-memory equivalent.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[39m dask.base.compute\u001b[39;00m\n\u001b[1;32m 313\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 314\u001b[0m (result,) \u001b[39m=\u001b[39m compute(\u001b[39mself\u001b[39;49m, traverse\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 315\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/dask/base.py:599\u001b[0m, in \u001b[0;36mcompute\u001b[0;34m(traverse, optimize_graph, scheduler, get, *args, **kwargs)\u001b[0m\n\u001b[1;32m 596\u001b[0m keys\u001b[39m.\u001b[39mappend(x\u001b[39m.\u001b[39m__dask_keys__())\n\u001b[1;32m 597\u001b[0m postcomputes\u001b[39m.\u001b[39mappend(x\u001b[39m.\u001b[39m__dask_postcompute__())\n\u001b[0;32m--> 599\u001b[0m results \u001b[39m=\u001b[39m schedule(dsk, keys, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 600\u001b[0m \u001b[39mreturn\u001b[39;00m repack([f(r, \u001b[39m*\u001b[39ma) \u001b[39mfor\u001b[39;00m r, (f, a) \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(results, postcomputes)])\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:3186\u001b[0m, in \u001b[0;36mClient.get\u001b[0;34m(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)\u001b[0m\n\u001b[1;32m 3184\u001b[0m should_rejoin \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 3185\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 3186\u001b[0m results \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgather(packed, asynchronous\u001b[39m=\u001b[39;49masynchronous, direct\u001b[39m=\u001b[39;49mdirect)\n\u001b[1;32m 3187\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 3188\u001b[0m \u001b[39mfor\u001b[39;00m f \u001b[39min\u001b[39;00m futures\u001b[39m.\u001b[39mvalues():\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:2345\u001b[0m, in \u001b[0;36mClient.gather\u001b[0;34m(self, futures, errors, direct, asynchronous)\u001b[0m\n\u001b[1;32m 2343\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 2344\u001b[0m local_worker \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m-> 2345\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msync(\n\u001b[1;32m 2346\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_gather,\n\u001b[1;32m 2347\u001b[0m futures,\n\u001b[1;32m 2348\u001b[0m errors\u001b[39m=\u001b[39;49merrors,\n\u001b[1;32m 2349\u001b[0m direct\u001b[39m=\u001b[39;49mdirect,\n\u001b[1;32m 2350\u001b[0m local_worker\u001b[39m=\u001b[39;49mlocal_worker,\n\u001b[1;32m 2351\u001b[0m asynchronous\u001b[39m=\u001b[39;49masynchronous,\n\u001b[1;32m 2352\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:349\u001b[0m, in \u001b[0;36mSyncMethodMixin.sync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[39mreturn\u001b[39;00m future\n\u001b[1;32m 348\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 349\u001b[0m \u001b[39mreturn\u001b[39;00m sync(\n\u001b[1;32m 350\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mloop, func, \u001b[39m*\u001b[39;49margs, callback_timeout\u001b[39m=\u001b[39;49mcallback_timeout, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs\n\u001b[1;32m 351\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:416\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 414\u001b[0m \u001b[39mif\u001b[39;00m error:\n\u001b[1;32m 415\u001b[0m typ, exc, tb \u001b[39m=\u001b[39m error\n\u001b[0;32m--> 416\u001b[0m \u001b[39mraise\u001b[39;00m exc\u001b[39m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m 417\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 418\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:389\u001b[0m, in \u001b[0;36msync..f\u001b[0;34m()\u001b[0m\n\u001b[1;32m 387\u001b[0m future \u001b[39m=\u001b[39m wait_for(future, callback_timeout)\n\u001b[1;32m 388\u001b[0m future \u001b[39m=\u001b[39m asyncio\u001b[39m.\u001b[39mensure_future(future)\n\u001b[0;32m--> 389\u001b[0m result \u001b[39m=\u001b[39m \u001b[39myield\u001b[39;00m future\n\u001b[1;32m 390\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 391\u001b[0m error \u001b[39m=\u001b[39m sys\u001b[39m.\u001b[39mexc_info()\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/tornado/gen.py:769\u001b[0m, in \u001b[0;36mRunner.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 766\u001b[0m exc_info \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 768\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 769\u001b[0m value \u001b[39m=\u001b[39m future\u001b[39m.\u001b[39;49mresult()\n\u001b[1;32m 770\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 771\u001b[0m exc_info \u001b[39m=\u001b[39m sys\u001b[39m.\u001b[39mexc_info()\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:2208\u001b[0m, in \u001b[0;36mClient._gather\u001b[0;34m(self, futures, errors, direct, local_worker)\u001b[0m\n\u001b[1;32m 2206\u001b[0m exc \u001b[39m=\u001b[39m CancelledError(key)\n\u001b[1;32m 2207\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2208\u001b[0m \u001b[39mraise\u001b[39;00m exception\u001b[39m.\u001b[39mwith_traceback(traceback)\n\u001b[1;32m 2209\u001b[0m \u001b[39mraise\u001b[39;00m exc\n\u001b[1;32m 2210\u001b[0m \u001b[39mif\u001b[39;00m errors \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mskip\u001b[39m\u001b[39m\"\u001b[39m:\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/contextlib.py:79\u001b[0m, in \u001b[0;36minner\u001b[0;34m()\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@wraps\u001b[39m(func)\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39minner\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds):\n\u001b[1;32m 78\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_recreate_cm():\n\u001b[0;32m---> 79\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py:2937\u001b[0m, in \u001b[0;36m_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2934\u001b[0m \u001b[39m# Some objects require threadlocal state during deserialization, e.g. to\u001b[39;00m\n\u001b[1;32m 2935\u001b[0m \u001b[39m# detect the current worker\u001b[39;00m\n\u001b[1;32m 2936\u001b[0m \u001b[39mif\u001b[39;00m function \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 2937\u001b[0m function \u001b[39m=\u001b[39m loads_function(function)\n\u001b[1;32m 2938\u001b[0m \u001b[39mif\u001b[39;00m args \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(args, \u001b[39mbytes\u001b[39m):\n\u001b[1;32m 2939\u001b[0m args \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(args)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py:2925\u001b[0m, in \u001b[0;36mloads_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2923\u001b[0m result \u001b[39m=\u001b[39m cache_loads[bytes_object]\n\u001b[1;32m 2924\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m:\n\u001b[0;32m-> 2925\u001b[0m result \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(bytes_object)\n\u001b[1;32m 2926\u001b[0m cache_loads[bytes_object] \u001b[39m=\u001b[39m result\n\u001b[1;32m 2927\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py:96\u001b[0m, in \u001b[0;36mloads\u001b[0;34m()\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[39mreturn\u001b[39;00m pickle\u001b[39m.\u001b[39mloads(x, buffers\u001b[39m=\u001b[39mbuffers)\n\u001b[1;32m 95\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 96\u001b[0m \u001b[39mreturn\u001b[39;00m pickle\u001b[39m.\u001b[39mloads(x)\n\u001b[1;32m 97\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 98\u001b[0m logger\u001b[39m.\u001b[39minfo(\u001b[39m\"\u001b[39m\u001b[39mFailed to deserialize \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m, x[:\u001b[39m10000\u001b[39m], exc_info\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py:176\u001b[0m, in \u001b[0;36mhost_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Perform device-side deserialization tasks.\u001b[39;00m\n\u001b[1;32m 155\u001b[0m \n\u001b[1;32m 156\u001b[0m \u001b[39mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[39m:meta private:\u001b[39;00m\n\u001b[1;32m 171\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 172\u001b[0m frames \u001b[39m=\u001b[39m [\n\u001b[1;32m 173\u001b[0m cudf\u001b[39m.\u001b[39mcore\u001b[39m.\u001b[39mbuffer\u001b[39m.\u001b[39mas_buffer(f) \u001b[39mif\u001b[39;00m c \u001b[39melse\u001b[39;00m f\n\u001b[1;32m 174\u001b[0m \u001b[39mfor\u001b[39;00m c, f \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(header[\u001b[39m\"\u001b[39m\u001b[39mis-cuda\u001b[39m\u001b[39m\"\u001b[39m], \u001b[39mmap\u001b[39m(\u001b[39mmemoryview\u001b[39m, frames))\n\u001b[1;32m 175\u001b[0m ]\n\u001b[0;32m--> 176\u001b[0m obj \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39mdevice_deserialize(header, frames)\n\u001b[1;32m 177\u001b[0m \u001b[39mreturn\u001b[39;00m obj\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py:130\u001b[0m, in \u001b[0;36mdevice_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 125\u001b[0m typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 126\u001b[0m frames \u001b[39m=\u001b[39m [\n\u001b[1;32m 127\u001b[0m cudf\u001b[39m.\u001b[39mcore\u001b[39m.\u001b[39mbuffer\u001b[39m.\u001b[39mas_buffer(f) \u001b[39mif\u001b[39;00m c \u001b[39melse\u001b[39;00m \u001b[39mmemoryview\u001b[39m(f)\n\u001b[1;32m 128\u001b[0m \u001b[39mfor\u001b[39;00m c, f \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(header[\u001b[39m\"\u001b[39m\u001b[39mis-cuda\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 129\u001b[0m ]\n\u001b[0;32m--> 130\u001b[0m \u001b[39mreturn\u001b[39;00m typ\u001b[39m.\u001b[39mdeserialize(header, frames)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py:1019\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 1017\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdeserialize\u001b[39m(\u001b[39mcls\u001b[39m, header, frames):\n\u001b[1;32m 1018\u001b[0m index_nframes \u001b[39m=\u001b[39m header[\u001b[39m\"\u001b[39m\u001b[39mindex_frame_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[0;32m-> 1019\u001b[0m obj \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39mdeserialize(\n\u001b[1;32m 1020\u001b[0m header, frames[header[\u001b[39m\"\u001b[39m\u001b[39mindex_frame_count\u001b[39m\u001b[39m\"\u001b[39m] :]\n\u001b[1;32m 1021\u001b[0m )\n\u001b[1;32m 1023\u001b[0m idx_typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 1024\u001b[0m index \u001b[39m=\u001b[39m idx_typ\u001b[39m.\u001b[39mdeserialize(header[\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m], frames[:index_nframes])\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py:106\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 104\u001b[0m cls_deserialize \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 105\u001b[0m column_names \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mcolumn_names\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m--> 106\u001b[0m columns \u001b[39m=\u001b[39m deserialize_columns(header[\u001b[39m\"\u001b[39m\u001b[39mcolumns\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 107\u001b[0m \u001b[39mreturn\u001b[39;00m cls_deserialize\u001b[39m.\u001b[39m_from_data(\u001b[39mdict\u001b[39m(\u001b[39mzip\u001b[39m(column_names, columns)))\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:2450\u001b[0m, in \u001b[0;36mdeserialize_columns\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2448\u001b[0m col_frame_count \u001b[39m=\u001b[39m meta[\u001b[39m\"\u001b[39m\u001b[39mframe_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 2449\u001b[0m col_typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(meta[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m-> 2450\u001b[0m colobj \u001b[39m=\u001b[39m col_typ\u001b[39m.\u001b[39mdeserialize(meta, frames[:col_frame_count])\n\u001b[1;32m 2451\u001b[0m columns\u001b[39m.\u001b[39mappend(colobj)\n\u001b[1;32m 2452\u001b[0m \u001b[39m# Advance frames\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:1216\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1214\u001b[0m dtype \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mdtype\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 1215\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m header:\n\u001b[0;32m-> 1216\u001b[0m data, frames \u001b[39m=\u001b[39m unpack(header[\u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 1217\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 1218\u001b[0m data \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:1204\u001b[0m, in \u001b[0;36munpack\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1202\u001b[0m count \u001b[39m=\u001b[39m header[\u001b[39m\"\u001b[39m\u001b[39mframe_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 1203\u001b[0m klass \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m-> 1204\u001b[0m obj \u001b[39m=\u001b[39m klass\u001b[39m.\u001b[39mdeserialize(header, frames[:count])\n\u001b[1;32m 1205\u001b[0m \u001b[39mreturn\u001b[39;00m obj, frames[count:]\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:574\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 568\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdeserialize\u001b[39m(\u001b[39mcls\u001b[39m, header: \u001b[39mdict\u001b[39m, frames: \u001b[39mlist\u001b[39m):\n\u001b[1;32m 569\u001b[0m \u001b[39m# TODO: because of the hack in `SpillableBuffer.serialize()` where\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 572\u001b[0m \u001b[39m# deserialize into `SpillableBufferSlice` when the frames hasn't been\u001b[39;00m\n\u001b[1;32m 573\u001b[0m \u001b[39m# copied.\u001b[39;00m\n\u001b[0;32m--> 574\u001b[0m \u001b[39mreturn\u001b[39;00m SpillableBuffer\u001b[39m.\u001b[39mdeserialize(header, frames)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py:335\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[39mreturn\u001b[39;00m frame \u001b[39m# The frame is already deserialized\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(frame, \u001b[39m\"\u001b[39m\u001b[39m__cuda_array_interface__\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m--> 335\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_from_device_memory(frame)\n\u001b[1;32m 336\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_from_host_memory(frame)\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:235\u001b[0m, in \u001b[0;36m_from_device_memory\u001b[0;34m()\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Create a spillabe buffer from device memory.\u001b[39;00m\n\u001b[1;32m 219\u001b[0m \n\u001b[1;32m 220\u001b[0m \u001b[39mNo data is being copied.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[39m Buffer representing the same device memory as `data`\u001b[39;00m\n\u001b[1;32m 233\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 234\u001b[0m ret \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m_from_device_memory(data)\n\u001b[0;32m--> 235\u001b[0m ret\u001b[39m.\u001b[39m_finalize_init(ptr_desc\u001b[39m=\u001b[39m{\u001b[39m\"\u001b[39m\u001b[39mtype\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mgpu\u001b[39m\u001b[39m\"\u001b[39m}, exposed\u001b[39m=\u001b[39mexposed)\n\u001b[1;32m 236\u001b[0m \u001b[39mreturn\u001b[39;00m ret\n", - "File \u001b[0;32m~/miniconda3/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:206\u001b[0m, in \u001b[0;36m_finalize_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m 204\u001b[0m manager \u001b[39m=\u001b[39m get_global_manager()\n\u001b[1;32m 205\u001b[0m \u001b[39mif\u001b[39;00m manager \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 206\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m 207\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mcannot create \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m without \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 208\u001b[0m \u001b[39m\"\u001b[39m\u001b[39ma global spill manager\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 209\u001b[0m )\n\u001b[1;32m 211\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_manager \u001b[39m=\u001b[39m manager\n\u001b[1;32m 212\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_manager\u001b[39m.\u001b[39madd(\u001b[39mself\u001b[39m)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cugraph/structure/graph_classes.py:309\u001b[0m, in \u001b[0;36mGraph.from_dask_cudf_edgelist\u001b[0;34m(self, input_ddf, source, destination, edge_attr, renumber, store_transposed, legacy_renum_only)\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_Impl\u001b[39m.\u001b[39medgelist \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 308\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mGraph already has values\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 309\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_Impl\u001b[39m.\u001b[39;49m_simpleDistributedGraphImpl__from_edgelist(\n\u001b[1;32m 310\u001b[0m input_ddf,\n\u001b[1;32m 311\u001b[0m source,\n\u001b[1;32m 312\u001b[0m destination,\n\u001b[1;32m 313\u001b[0m edge_attr,\n\u001b[1;32m 314\u001b[0m renumber,\n\u001b[1;32m 315\u001b[0m store_transposed,\n\u001b[1;32m 316\u001b[0m legacy_renum_only,\n\u001b[1;32m 317\u001b[0m )\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cugraph/structure/graph_implementation/simpleDistributedGraph.py:272\u001b[0m, in \u001b[0;36msimpleDistributedGraphImpl.__from_edgelist\u001b[0;34m(self, input_ddf, source, destination, edge_attr, renumber, store_transposed, legacy_renum_only)\u001b[0m\n\u001b[1;32m 268\u001b[0m dst_col_name \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrenumber_map\u001b[39m.\u001b[39mrenumbered_dst_col_name\n\u001b[1;32m 270\u001b[0m ddf \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39medgelist\u001b[39m.\u001b[39medgelist_df\n\u001b[0;32m--> 272\u001b[0m num_edges \u001b[39m=\u001b[39m \u001b[39mlen\u001b[39;49m(ddf)\n\u001b[1;32m 273\u001b[0m edge_data \u001b[39m=\u001b[39m get_distributed_data(ddf)\n\u001b[1;32m 275\u001b[0m graph_props \u001b[39m=\u001b[39m GraphProperties(\n\u001b[1;32m 276\u001b[0m is_multigraph\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mproperties\u001b[39m.\u001b[39mmulti_edge,\n\u001b[1;32m 277\u001b[0m is_symmetric\u001b[39m=\u001b[39m\u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mproperties\u001b[39m.\u001b[39mdirected,\n\u001b[1;32m 278\u001b[0m )\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/dataframe/core.py:4775\u001b[0m, in \u001b[0;36mDataFrame.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 4773\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m\u001b[39m__len__\u001b[39m()\n\u001b[1;32m 4774\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 4775\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlen\u001b[39;49m(s)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/dataframe/core.py:843\u001b[0m, in \u001b[0;36m_Frame.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 840\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__len__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 841\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mreduction(\n\u001b[1;32m 842\u001b[0m \u001b[39mlen\u001b[39;49m, np\u001b[39m.\u001b[39;49msum, token\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mlen\u001b[39;49m\u001b[39m\"\u001b[39;49m, meta\u001b[39m=\u001b[39;49m\u001b[39mint\u001b[39;49m, split_every\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m\n\u001b[0;32m--> 843\u001b[0m )\u001b[39m.\u001b[39;49mcompute()\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/base.py:314\u001b[0m, in \u001b[0;36mDaskMethodsMixin.compute\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcompute\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 291\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Compute this dask collection\u001b[39;00m\n\u001b[1;32m 292\u001b[0m \n\u001b[1;32m 293\u001b[0m \u001b[39m This turns a lazy Dask collection into its in-memory equivalent.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[39m dask.base.compute\u001b[39;00m\n\u001b[1;32m 313\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 314\u001b[0m (result,) \u001b[39m=\u001b[39m compute(\u001b[39mself\u001b[39;49m, traverse\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 315\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/dask/base.py:599\u001b[0m, in \u001b[0;36mcompute\u001b[0;34m(traverse, optimize_graph, scheduler, get, *args, **kwargs)\u001b[0m\n\u001b[1;32m 596\u001b[0m keys\u001b[39m.\u001b[39mappend(x\u001b[39m.\u001b[39m__dask_keys__())\n\u001b[1;32m 597\u001b[0m postcomputes\u001b[39m.\u001b[39mappend(x\u001b[39m.\u001b[39m__dask_postcompute__())\n\u001b[0;32m--> 599\u001b[0m results \u001b[39m=\u001b[39m schedule(dsk, keys, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 600\u001b[0m \u001b[39mreturn\u001b[39;00m repack([f(r, \u001b[39m*\u001b[39ma) \u001b[39mfor\u001b[39;00m r, (f, a) \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(results, postcomputes)])\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:3186\u001b[0m, in \u001b[0;36mClient.get\u001b[0;34m(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)\u001b[0m\n\u001b[1;32m 3184\u001b[0m should_rejoin \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 3185\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 3186\u001b[0m results \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgather(packed, asynchronous\u001b[39m=\u001b[39;49masynchronous, direct\u001b[39m=\u001b[39;49mdirect)\n\u001b[1;32m 3187\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 3188\u001b[0m \u001b[39mfor\u001b[39;00m f \u001b[39min\u001b[39;00m futures\u001b[39m.\u001b[39mvalues():\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:2345\u001b[0m, in \u001b[0;36mClient.gather\u001b[0;34m(self, futures, errors, direct, asynchronous)\u001b[0m\n\u001b[1;32m 2343\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 2344\u001b[0m local_worker \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m-> 2345\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msync(\n\u001b[1;32m 2346\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_gather,\n\u001b[1;32m 2347\u001b[0m futures,\n\u001b[1;32m 2348\u001b[0m errors\u001b[39m=\u001b[39;49merrors,\n\u001b[1;32m 2349\u001b[0m direct\u001b[39m=\u001b[39;49mdirect,\n\u001b[1;32m 2350\u001b[0m local_worker\u001b[39m=\u001b[39;49mlocal_worker,\n\u001b[1;32m 2351\u001b[0m asynchronous\u001b[39m=\u001b[39;49masynchronous,\n\u001b[1;32m 2352\u001b[0m )\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:349\u001b[0m, in \u001b[0;36mSyncMethodMixin.sync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[39mreturn\u001b[39;00m future\n\u001b[1;32m 348\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 349\u001b[0m \u001b[39mreturn\u001b[39;00m sync(\n\u001b[1;32m 350\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mloop, func, \u001b[39m*\u001b[39;49margs, callback_timeout\u001b[39m=\u001b[39;49mcallback_timeout, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs\n\u001b[1;32m 351\u001b[0m )\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:416\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 414\u001b[0m \u001b[39mif\u001b[39;00m error:\n\u001b[1;32m 415\u001b[0m typ, exc, tb \u001b[39m=\u001b[39m error\n\u001b[0;32m--> 416\u001b[0m \u001b[39mraise\u001b[39;00m exc\u001b[39m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m 417\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 418\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/utils.py:389\u001b[0m, in \u001b[0;36msync..f\u001b[0;34m()\u001b[0m\n\u001b[1;32m 387\u001b[0m future \u001b[39m=\u001b[39m wait_for(future, callback_timeout)\n\u001b[1;32m 388\u001b[0m future \u001b[39m=\u001b[39m asyncio\u001b[39m.\u001b[39mensure_future(future)\n\u001b[0;32m--> 389\u001b[0m result \u001b[39m=\u001b[39m \u001b[39myield\u001b[39;00m future\n\u001b[1;32m 390\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 391\u001b[0m error \u001b[39m=\u001b[39m sys\u001b[39m.\u001b[39mexc_info()\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/tornado/gen.py:769\u001b[0m, in \u001b[0;36mRunner.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 766\u001b[0m exc_info \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 768\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 769\u001b[0m value \u001b[39m=\u001b[39m future\u001b[39m.\u001b[39;49mresult()\n\u001b[1;32m 770\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 771\u001b[0m exc_info \u001b[39m=\u001b[39m sys\u001b[39m.\u001b[39mexc_info()\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/client.py:2208\u001b[0m, in \u001b[0;36mClient._gather\u001b[0;34m(self, futures, errors, direct, local_worker)\u001b[0m\n\u001b[1;32m 2206\u001b[0m exc \u001b[39m=\u001b[39m CancelledError(key)\n\u001b[1;32m 2207\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2208\u001b[0m \u001b[39mraise\u001b[39;00m exception\u001b[39m.\u001b[39mwith_traceback(traceback)\n\u001b[1;32m 2209\u001b[0m \u001b[39mraise\u001b[39;00m exc\n\u001b[1;32m 2210\u001b[0m \u001b[39mif\u001b[39;00m errors \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mskip\u001b[39m\u001b[39m\"\u001b[39m:\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/contextlib.py:79\u001b[0m, in \u001b[0;36minner\u001b[0;34m()\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@wraps\u001b[39m(func)\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39minner\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds):\n\u001b[1;32m 78\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_recreate_cm():\n\u001b[0;32m---> 79\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py:2937\u001b[0m, in \u001b[0;36m_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2934\u001b[0m \u001b[39m# Some objects require threadlocal state during deserialization, e.g. to\u001b[39;00m\n\u001b[1;32m 2935\u001b[0m \u001b[39m# detect the current worker\u001b[39;00m\n\u001b[1;32m 2936\u001b[0m \u001b[39mif\u001b[39;00m function \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 2937\u001b[0m function \u001b[39m=\u001b[39m loads_function(function)\n\u001b[1;32m 2938\u001b[0m \u001b[39mif\u001b[39;00m args \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(args, \u001b[39mbytes\u001b[39m):\n\u001b[1;32m 2939\u001b[0m args \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(args)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/worker.py:2925\u001b[0m, in \u001b[0;36mloads_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2923\u001b[0m result \u001b[39m=\u001b[39m cache_loads[bytes_object]\n\u001b[1;32m 2924\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m:\n\u001b[0;32m-> 2925\u001b[0m result \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(bytes_object)\n\u001b[1;32m 2926\u001b[0m cache_loads[bytes_object] \u001b[39m=\u001b[39m result\n\u001b[1;32m 2927\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/distributed/protocol/pickle.py:96\u001b[0m, in \u001b[0;36mloads\u001b[0;34m()\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[39mreturn\u001b[39;00m pickle\u001b[39m.\u001b[39mloads(x, buffers\u001b[39m=\u001b[39mbuffers)\n\u001b[1;32m 95\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 96\u001b[0m \u001b[39mreturn\u001b[39;00m pickle\u001b[39m.\u001b[39mloads(x)\n\u001b[1;32m 97\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 98\u001b[0m logger\u001b[39m.\u001b[39minfo(\u001b[39m\"\u001b[39m\u001b[39mFailed to deserialize \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m, x[:\u001b[39m10000\u001b[39m], exc_info\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py:176\u001b[0m, in \u001b[0;36mhost_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Perform device-side deserialization tasks.\u001b[39;00m\n\u001b[1;32m 155\u001b[0m \n\u001b[1;32m 156\u001b[0m \u001b[39mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[39m:meta private:\u001b[39;00m\n\u001b[1;32m 171\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 172\u001b[0m frames \u001b[39m=\u001b[39m [\n\u001b[1;32m 173\u001b[0m cudf\u001b[39m.\u001b[39mcore\u001b[39m.\u001b[39mbuffer\u001b[39m.\u001b[39mas_buffer(f) \u001b[39mif\u001b[39;00m c \u001b[39melse\u001b[39;00m f\n\u001b[1;32m 174\u001b[0m \u001b[39mfor\u001b[39;00m c, f \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(header[\u001b[39m\"\u001b[39m\u001b[39mis-cuda\u001b[39m\u001b[39m\"\u001b[39m], \u001b[39mmap\u001b[39m(\u001b[39mmemoryview\u001b[39m, frames))\n\u001b[1;32m 175\u001b[0m ]\n\u001b[0;32m--> 176\u001b[0m obj \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39mdevice_deserialize(header, frames)\n\u001b[1;32m 177\u001b[0m \u001b[39mreturn\u001b[39;00m obj\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/abc.py:130\u001b[0m, in \u001b[0;36mdevice_deserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 125\u001b[0m typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 126\u001b[0m frames \u001b[39m=\u001b[39m [\n\u001b[1;32m 127\u001b[0m cudf\u001b[39m.\u001b[39mcore\u001b[39m.\u001b[39mbuffer\u001b[39m.\u001b[39mas_buffer(f) \u001b[39mif\u001b[39;00m c \u001b[39melse\u001b[39;00m \u001b[39mmemoryview\u001b[39m(f)\n\u001b[1;32m 128\u001b[0m \u001b[39mfor\u001b[39;00m c, f \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(header[\u001b[39m\"\u001b[39m\u001b[39mis-cuda\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 129\u001b[0m ]\n\u001b[0;32m--> 130\u001b[0m \u001b[39mreturn\u001b[39;00m typ\u001b[39m.\u001b[39mdeserialize(header, frames)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/dataframe.py:1019\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 1017\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdeserialize\u001b[39m(\u001b[39mcls\u001b[39m, header, frames):\n\u001b[1;32m 1018\u001b[0m index_nframes \u001b[39m=\u001b[39m header[\u001b[39m\"\u001b[39m\u001b[39mindex_frame_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[0;32m-> 1019\u001b[0m obj \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39mdeserialize(\n\u001b[1;32m 1020\u001b[0m header, frames[header[\u001b[39m\"\u001b[39m\u001b[39mindex_frame_count\u001b[39m\u001b[39m\"\u001b[39m] :]\n\u001b[1;32m 1021\u001b[0m )\n\u001b[1;32m 1023\u001b[0m idx_typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 1024\u001b[0m index \u001b[39m=\u001b[39m idx_typ\u001b[39m.\u001b[39mdeserialize(header[\u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m], frames[:index_nframes])\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/frame.py:106\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 104\u001b[0m cls_deserialize \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 105\u001b[0m column_names \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mcolumn_names\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m--> 106\u001b[0m columns \u001b[39m=\u001b[39m deserialize_columns(header[\u001b[39m\"\u001b[39m\u001b[39mcolumns\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 107\u001b[0m \u001b[39mreturn\u001b[39;00m cls_deserialize\u001b[39m.\u001b[39m_from_data(\u001b[39mdict\u001b[39m(\u001b[39mzip\u001b[39m(column_names, columns)))\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:2450\u001b[0m, in \u001b[0;36mdeserialize_columns\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2448\u001b[0m col_frame_count \u001b[39m=\u001b[39m meta[\u001b[39m\"\u001b[39m\u001b[39mframe_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 2449\u001b[0m col_typ \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(meta[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m-> 2450\u001b[0m colobj \u001b[39m=\u001b[39m col_typ\u001b[39m.\u001b[39mdeserialize(meta, frames[:col_frame_count])\n\u001b[1;32m 2451\u001b[0m columns\u001b[39m.\u001b[39mappend(colobj)\n\u001b[1;32m 2452\u001b[0m \u001b[39m# Advance frames\u001b[39;00m\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:1216\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1214\u001b[0m dtype \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mdtype\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m 1215\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m header:\n\u001b[0;32m-> 1216\u001b[0m data, frames \u001b[39m=\u001b[39m unpack(header[\u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m], frames)\n\u001b[1;32m 1217\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 1218\u001b[0m data \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/column/column.py:1204\u001b[0m, in \u001b[0;36munpack\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1202\u001b[0m count \u001b[39m=\u001b[39m header[\u001b[39m\"\u001b[39m\u001b[39mframe_count\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 1203\u001b[0m klass \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39mloads(header[\u001b[39m\"\u001b[39m\u001b[39mtype-serialized\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m-> 1204\u001b[0m obj \u001b[39m=\u001b[39m klass\u001b[39m.\u001b[39mdeserialize(header, frames[:count])\n\u001b[1;32m 1205\u001b[0m \u001b[39mreturn\u001b[39;00m obj, frames[count:]\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:574\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 568\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdeserialize\u001b[39m(\u001b[39mcls\u001b[39m, header: \u001b[39mdict\u001b[39m, frames: \u001b[39mlist\u001b[39m):\n\u001b[1;32m 569\u001b[0m \u001b[39m# TODO: because of the hack in `SpillableBuffer.serialize()` where\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 572\u001b[0m \u001b[39m# deserialize into `SpillableBufferSlice` when the frames hasn't been\u001b[39;00m\n\u001b[1;32m 573\u001b[0m \u001b[39m# copied.\u001b[39;00m\n\u001b[0;32m--> 574\u001b[0m \u001b[39mreturn\u001b[39;00m SpillableBuffer\u001b[39m.\u001b[39mdeserialize(header, frames)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/buffer.py:335\u001b[0m, in \u001b[0;36mdeserialize\u001b[0;34m()\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[39mreturn\u001b[39;00m frame \u001b[39m# The frame is already deserialized\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(frame, \u001b[39m\"\u001b[39m\u001b[39m__cuda_array_interface__\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m--> 335\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_from_device_memory(frame)\n\u001b[1;32m 336\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_from_host_memory(frame)\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:235\u001b[0m, in \u001b[0;36m_from_device_memory\u001b[0;34m()\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Create a spillabe buffer from device memory.\u001b[39;00m\n\u001b[1;32m 219\u001b[0m \n\u001b[1;32m 220\u001b[0m \u001b[39mNo data is being copied.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[39m Buffer representing the same device memory as `data`\u001b[39;00m\n\u001b[1;32m 233\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 234\u001b[0m ret \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m_from_device_memory(data)\n\u001b[0;32m--> 235\u001b[0m ret\u001b[39m.\u001b[39m_finalize_init(ptr_desc\u001b[39m=\u001b[39m{\u001b[39m\"\u001b[39m\u001b[39mtype\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mgpu\u001b[39m\u001b[39m\"\u001b[39m}, exposed\u001b[39m=\u001b[39mexposed)\n\u001b[1;32m 236\u001b[0m \u001b[39mreturn\u001b[39;00m ret\n", + "File \u001b[0;32m~/miniforge/envs/cugraph_0411/lib/python3.10/site-packages/cudf/core/buffer/spillable_buffer.py:206\u001b[0m, in \u001b[0;36m_finalize_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m 204\u001b[0m manager \u001b[39m=\u001b[39m get_global_manager()\n\u001b[1;32m 205\u001b[0m \u001b[39mif\u001b[39;00m manager \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 206\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m 207\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mcannot create \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m without \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 208\u001b[0m \u001b[39m\"\u001b[39m\u001b[39ma global spill manager\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 209\u001b[0m )\n\u001b[1;32m 211\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_manager \u001b[39m=\u001b[39m manager\n\u001b[1;32m 212\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_manager\u001b[39m.\u001b[39madd(\u001b[39mself\u001b[39m)\n", "\u001b[0;31mValueError\u001b[0m: cannot create without a global spill manager" ] } From 7e058e2961b5b8be05877282d26051f36dd73c53 Mon Sep 17 00:00:00 2001 From: Rick Ratzel <3039903+rlratzel@users.noreply.github.com> Date: Fri, 20 Sep 2024 09:12:30 -0500 Subject: [PATCH 8/8] Drops duplicate edges in non-MultiGraph PLC `SGGraph` instances (#4658) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Graph input with duplicate edges intended for `Graph`/`DiGraph` instances resulted in internal PLC `SGGraph` instances with duplicate edges, which were effectively treated as MultiGraphs and caused incorrect results from algorithms like `pagerank`. This PR sets the `drop_multi_edges` PLC `SGGraph` ctor option to have PLC remove duplicate edges on `SGGraph` creation. The overhead to drop duplicate edges for non-MultiGraphs is negligible, and in the case of a large test graph (wikipedia data, 37.5M nodes, 464.5M edges) resulted in an overall _speedup_ for pagerank going from 12.2 seconds to 10.7 seconds on my workstation, likely due to fewer edges to process a minor slowdown from 10.5s to 10.7s. _edit: after several re-runs, the pagerank runtime before the change settled to 10.5, and the runtime after the change was typically 10.7._ A test was added that uses pagerank to ensure Graphs vs. MultiGraphs are handled correctly and duplicate edges are dropped as needed. The results when run without `drop_multi_edges` set: ``` > assert actual_pr_for_G == approx(expected_pr_for_G) E assert {0: 0.0875795...7955580949783} == approx({0: 0....32 ± 1.8e-07}) E E comparison failed. Mismatched elements: 4 / 4: E Max absolute difference: 0.08785887916592061 E Max relative difference: 0.5007959662968462 E Index | Obtained | Expected E 0 | 0.08757955580949783 | 0.17543839772251532 ± 1.8e-07 E 1 | 0.41242048144340515 | 0.32456160227748454 ± 3.2e-07 E 2 | 0.41242048144340515 | 0.32456160227748454 ± 3.2e-07 E 3 | 0.08757955580949783 | 0.17543839772251532 ± 1.8e-07 ``` The same test passes when run with the changes in this PR to set `drop_multi_edges`. Authors: - Rick Ratzel (https://github.com/rlratzel) Approvers: - Erik Welch (https://github.com/eriknw) URL: https://github.com/rapidsai/cugraph/pull/4658 --- python/nx-cugraph/nx_cugraph/classes/graph.py | 9 +++++ .../nx_cugraph/tests/test_pagerank.py | 36 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 python/nx-cugraph/nx_cugraph/tests/test_pagerank.py diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 7425eacb2b4..7c01365c0ac 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -689,6 +689,14 @@ def _get_plc_graph( src_indices = src_indices.astype(index_dtype) dst_indices = dst_indices.astype(index_dtype) + # This sets drop_multi_edges=True for non-multigraph input, which means + # the data in self.src_indices and self.dst_indices may not be + # identical to that contained in the returned pcl.SGGraph (the returned + # SGGraph may have fewer edges since duplicates are dropped). Ideally + # self.src_indices and self.dst_indices would be updated to have + # duplicate edges removed for non-multigraph instances, but that + # requires additional code which would be redundant and likely not as + # performant as the code in PLC. return plc.SGGraph( resource_handle=plc.ResourceHandle(), graph_properties=plc.GraphProperties( @@ -702,6 +710,7 @@ def _get_plc_graph( renumber=False, do_expensive_check=False, vertices_array=self._node_ids, + drop_multi_edges=not self.is_multigraph(), ) def _sort_edge_indices(self, primary="src"): diff --git a/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py new file mode 100644 index 00000000000..0b437df2d2f --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import pandas as pd +from pytest import approx + + +def test_pagerank_multigraph(): + """ + Ensures correct differences between pagerank results for Graphs + vs. MultiGraphs generated using from_pandas_edgelist() + """ + df = pd.DataFrame({"source": [0, 1, 1, 1, 1, 1, 1, 2], + "target": [1, 2, 2, 2, 2, 2, 2, 3]}) + expected_pr_for_G = nx.pagerank(nx.from_pandas_edgelist(df)) + expected_pr_for_MultiG = nx.pagerank( + nx.from_pandas_edgelist(df, create_using=nx.MultiGraph)) + + G = nx.from_pandas_edgelist(df, backend="cugraph") + actual_pr_for_G = nx.pagerank(G, backend="cugraph") + + MultiG = nx.from_pandas_edgelist(df, create_using=nx.MultiGraph, backend="cugraph") + actual_pr_for_MultiG = nx.pagerank(MultiG, backend="cugraph") + + assert actual_pr_for_G == approx(expected_pr_for_G) + assert actual_pr_for_MultiG == approx(expected_pr_for_MultiG)