From 6b3d3e3f06044c86f26d9773b6daf7ab68857321 Mon Sep 17 00:00:00 2001 From: Joseph Nke <76006812+jnke2016@users.noreply.github.com> Date: Mon, 20 Nov 2023 19:44:48 -0600 Subject: [PATCH 1/6] Enable parallel mode (#3875) This PR enables parallel mode Closes https://github.com/rapidsai/graph_dl/issues/328 Authors: - Joseph Nke (https://github.com/jnke2016) - Rick Ratzel (https://github.com/rlratzel) Approvers: - Chuck Hastings (https://github.com/ChuckHastings) - Naim (https://github.com/naimnv) - Brad Rees (https://github.com/BradReesWork) - Rick Ratzel (https://github.com/rlratzel) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cugraph/pull/3875 --- cpp/CMakeLists.txt | 2 + .../cugraph/detail/collect_comm_wrapper.hpp | 42 +++ cpp/include/cugraph_c/graph_functions.h | 47 ++- cpp/src/c_api/allgather.cpp | 282 ++++++++++++++ cpp/src/c_api/extract_ego.cpp | 2 + cpp/src/c_api/induced_subgraph.cpp | 3 + cpp/src/c_api/induced_subgraph_result.cpp | 24 ++ cpp/src/c_api/induced_subgraph_result.hpp | 4 +- cpp/src/c_api/legacy_k_truss.cpp | 3 + cpp/src/detail/collect_comm_wrapper.cu | 57 +++ dependencies.yaml | 3 +- python/cugraph/cugraph/structure/__init__.py | 5 + .../cugraph/structure/replicate_edgelist.py | 351 ++++++++++++++++++ .../community/test_induced_subgraph_mg.py | 2 +- .../internals/test_replicate_edgelist_mg.py | 128 +++++++ python/cugraph/pyproject.toml | 1 + .../pylibcugraph/pylibcugraph/CMakeLists.txt | 1 + python/pylibcugraph/pylibcugraph/__init__.py | 2 + .../_cugraph_c/graph_functions.pxd | 26 +- .../pylibcugraph/replicate_edgelist.pyx | 202 ++++++++++ 20 files changed, 1182 insertions(+), 5 deletions(-) create mode 100644 cpp/include/cugraph/detail/collect_comm_wrapper.hpp create mode 100644 cpp/src/c_api/allgather.cpp create mode 100644 cpp/src/detail/collect_comm_wrapper.cu create mode 100644 python/cugraph/cugraph/structure/replicate_edgelist.py create mode 100644 python/cugraph/cugraph/tests/internals/test_replicate_edgelist_mg.py create mode 100644 python/pylibcugraph/pylibcugraph/replicate_edgelist.pyx diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3e867643041..626d62cffa5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -192,6 +192,7 @@ set(CUGRAPH_SOURCES src/detail/shuffle_vertex_pairs.cu src/detail/collect_local_vertex_values.cu src/detail/groupby_and_count.cu + src/detail/collect_comm_wrapper.cu src/sampling/random_walks_mg.cu src/community/detail/common_methods_mg.cu src/community/detail/common_methods_sg.cu @@ -443,6 +444,7 @@ add_library(cugraph_c src/c_api/labeling_result.cpp src/c_api/weakly_connected_components.cpp src/c_api/strongly_connected_components.cpp + src/c_api/allgather.cpp src/c_api/legacy_k_truss.cpp ) add_library(cugraph::cugraph_c ALIAS cugraph_c) diff --git a/cpp/include/cugraph/detail/collect_comm_wrapper.hpp b/cpp/include/cugraph/detail/collect_comm_wrapper.hpp new file mode 100644 index 00000000000..b791c593f41 --- /dev/null +++ b/cpp/include/cugraph/detail/collect_comm_wrapper.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include + +namespace cugraph { +namespace detail { + +/** + * @brief Gather the span of data from all ranks and broadcast the combined data to all ranks. + * + * @param[in] handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, + * and handles to various CUDA libraries) to run graph algorithms. + * @param[in] comm Raft comms that manages underlying NCCL comms handles across the ranks. + * @param[in] d_input The span of data to perform the 'allgatherv'. + * + * @return A vector containing the combined data of all ranks. + */ +template +rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +} // namespace detail +} // namespace cugraph diff --git a/cpp/include/cugraph_c/graph_functions.h b/cpp/include/cugraph_c/graph_functions.h index 655324df284..19b69922fa5 100644 --- a/cpp/include/cugraph_c/graph_functions.h +++ b/cpp/include/cugraph_c/graph_functions.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -136,6 +136,24 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_destinatio cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_weights( cugraph_induced_subgraph_result_t* induced_subgraph); +/** + * @brief Get the edge ids + * + * @param [in] induced_subgraph Opaque pointer to induced subgraph + * @return type erased array view of edge ids + */ +cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_ids( + cugraph_induced_subgraph_result_t* induced_subgraph); + +/** + * @brief Get the edge types + * + * @param [in] induced_subgraph Opaque pointer to induced subgraph + * @return type erased array view of edge types + */ +cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_type_ids( + cugraph_induced_subgraph_result_t* induced_subgraph); + /** * @brief Get the subgraph offsets * @@ -184,6 +202,33 @@ cugraph_error_code_t cugraph_extract_induced_subgraph( cugraph_induced_subgraph_result_t** result, cugraph_error_t** error); +// FIXME: Rename the return type +/** + * @brief Gather edgelist + * + * This function collects the edgelist from all ranks and stores the combine edgelist + * in each rank + * + * @param [in] handle Handle for accessing resources. + * @param [in] src Device array containing the source vertex ids. + * @param [in] dst Device array containing the destination vertex ids + * @param [in] weights Optional device array containing the edge weights + * @param [in] edge_ids Optional device array containing the edge ids for each edge. + * @param [in] edge_type_ids Optional device array containing the edge types for each edge + * @param [out] result Opaque pointer to gathered edgelist result + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_allgather(const cugraph_resource_handle_t* handle, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + cugraph_induced_subgraph_result_t** result, + cugraph_error_t** error); + #ifdef __cplusplus } #endif diff --git a/cpp/src/c_api/allgather.cpp b/cpp/src/c_api/allgather.cpp new file mode 100644 index 00000000000..7ef401aa6b7 --- /dev/null +++ b/cpp/src/c_api/allgather.cpp @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { + +struct create_allgather_functor : public cugraph::c_api::abstract_functor { + raft::handle_t const& handle_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* src_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids_; + cugraph::c_api::cugraph_induced_subgraph_result_t* result_{}; + + create_allgather_functor( + raft::handle_t const& handle, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* src, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids) + : abstract_functor(), + handle_(handle), + src_(src), + dst_(dst), + weights_(weights), + edge_ids_(edge_ids), + edge_type_ids_(edge_type_ids) + { + } + + template + void operator()() + { + std::optional> edgelist_srcs{std::nullopt}; + if (src_) { + edgelist_srcs = rmm::device_uvector(src_->size_, handle_.get_stream()); + raft::copy( + edgelist_srcs->data(), src_->as_type(), src_->size_, handle_.get_stream()); + } + + std::optional> edgelist_dsts{std::nullopt}; + if (dst_) { + edgelist_dsts = rmm::device_uvector(dst_->size_, handle_.get_stream()); + raft::copy( + edgelist_dsts->data(), dst_->as_type(), dst_->size_, handle_.get_stream()); + } + + std::optional> edgelist_weights{std::nullopt}; + if (weights_) { + edgelist_weights = rmm::device_uvector(weights_->size_, handle_.get_stream()); + raft::copy(edgelist_weights->data(), + weights_->as_type(), + weights_->size_, + handle_.get_stream()); + } + + std::optional> edgelist_ids{std::nullopt}; + if (edge_ids_) { + edgelist_ids = rmm::device_uvector(edge_ids_->size_, handle_.get_stream()); + raft::copy( + edgelist_ids->data(), edge_ids_->as_type(), edge_ids_->size_, handle_.get_stream()); + } + + std::optional> edgelist_type_ids{std::nullopt}; + if (edge_type_ids_) { + edgelist_type_ids = + rmm::device_uvector(edge_type_ids_->size_, handle_.get_stream()); + raft::copy(edgelist_type_ids->data(), + edge_type_ids_->as_type(), + edge_type_ids_->size_, + handle_.get_stream()); + } + + auto& comm = handle_.get_comms(); + + if (edgelist_srcs) { + edgelist_srcs = cugraph::detail::device_allgatherv( + handle_, + comm, + raft::device_span(edgelist_srcs->data(), edgelist_srcs->size())); + } + + if (edgelist_dsts) { + edgelist_dsts = cugraph::detail::device_allgatherv( + handle_, + comm, + raft::device_span(edgelist_dsts->data(), edgelist_dsts->size())); + } + + rmm::device_uvector edge_offsets(2, handle_.get_stream()); + + std::vector h_edge_offsets{ + {0, edgelist_srcs ? edgelist_srcs->size() : edgelist_weights->size()}}; + raft::update_device( + edge_offsets.data(), h_edge_offsets.data(), h_edge_offsets.size(), handle_.get_stream()); + + cugraph::c_api::cugraph_induced_subgraph_result_t* result = NULL; + + if (edgelist_weights) { + edgelist_weights = cugraph::detail::device_allgatherv( + handle_, + comm, + raft::device_span(edgelist_weights->data(), edgelist_weights->size())); + } + + if (edgelist_ids) { + edgelist_ids = cugraph::detail::device_allgatherv( + handle_, comm, raft::device_span(edgelist_ids->data(), edgelist_ids->size())); + } + + if (edgelist_type_ids) { + edgelist_type_ids = + cugraph::detail::device_allgatherv(handle_, + comm, + raft::device_span( + edgelist_type_ids->data(), edgelist_type_ids->size())); + } + + result = new cugraph::c_api::cugraph_induced_subgraph_result_t{ + edgelist_srcs + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edgelist_srcs, src_->type_) + : NULL, + edgelist_dsts + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edgelist_dsts, dst_->type_) + : NULL, + edgelist_weights + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edgelist_weights, weights_->type_) + : NULL, + edgelist_ids + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*edgelist_ids, edge_ids_->type_) + : NULL, + edgelist_type_ids ? new cugraph::c_api::cugraph_type_erased_device_array_t( + *edgelist_type_ids, edge_type_ids_->type_) + : NULL, + new cugraph::c_api::cugraph_type_erased_device_array_t(edge_offsets, + cugraph_data_type_id_t::SIZE_T)}; + + result_ = reinterpret_cast(result); + } +}; + +} // namespace + +extern "C" cugraph_error_code_t cugraph_allgather( + const cugraph_resource_handle_t* handle, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + cugraph_induced_subgraph_result_t** edgelist, + cugraph_error_t** error) +{ + *edgelist = nullptr; + *error = nullptr; + + auto p_handle = reinterpret_cast(handle); + auto p_src = + reinterpret_cast(src); + auto p_dst = + reinterpret_cast(dst); + auto p_weights = + reinterpret_cast(weights); + + auto p_edge_ids = + reinterpret_cast(edge_ids); + + auto p_edge_type_ids = + reinterpret_cast(edge_type_ids); + + CAPI_EXPECTS((dst == nullptr) || (src == nullptr) || p_src->size_ == p_dst->size_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != dst size.", + *error); + CAPI_EXPECTS((dst == nullptr) || (src == nullptr) || p_src->type_ == p_dst->type_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src type != dst type.", + *error); + + CAPI_EXPECTS((weights == nullptr) || (src == nullptr) || (p_weights->size_ == p_src->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != weights size.", + *error); + + cugraph_data_type_id_t vertex_type; + cugraph_data_type_id_t edge_type; + cugraph_data_type_id_t weight_type; + cugraph_data_type_id_t edge_type_id_type; + + if (src != nullptr) { + vertex_type = p_src->type_; + } else { + vertex_type = cugraph_data_type_id_t::INT32; + } + + if (weights != nullptr) { + weight_type = p_weights->type_; + } else { + weight_type = cugraph_data_type_id_t::FLOAT32; + } + + if (edge_ids != nullptr) { + edge_type = p_edge_ids->type_; + } else { + edge_type = cugraph_data_type_id_t::INT32; + } + + if (edge_type_ids != nullptr) { + edge_type_id_type = p_edge_type_ids->type_; + } else { + edge_type_id_type = cugraph_data_type_id_t::INT32; + } + + if (src != nullptr) { + CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids->size_ == p_src->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge id prop size", + *error); + + CAPI_EXPECTS((edge_type_ids == nullptr) || (p_edge_type_ids->size_ == p_src->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge type prop size", + *error); + } + + constexpr bool multi_gpu = false; + constexpr bool store_transposed = false; + + ::create_allgather_functor functor( + *p_handle->handle_, p_src, p_dst, p_weights, p_edge_ids, p_edge_type_ids); + + try { + cugraph::c_api::vertex_dispatcher( + vertex_type, edge_type, weight_type, edge_type_id_type, store_transposed, multi_gpu, functor); + + if (functor.error_code_ != CUGRAPH_SUCCESS) { + *error = reinterpret_cast(functor.error_.release()); + return functor.error_code_; + } + + *edgelist = reinterpret_cast(functor.result_); + } catch (std::exception const& ex) { + *error = reinterpret_cast(new cugraph::c_api::cugraph_error_t{ex.what()}); + return CUGRAPH_UNKNOWN_ERROR; + } + + return CUGRAPH_SUCCESS; +} diff --git a/cpp/src/c_api/extract_ego.cpp b/cpp/src/c_api/extract_ego.cpp index 8f510b79023..931d58b5185 100644 --- a/cpp/src/c_api/extract_ego.cpp +++ b/cpp/src/c_api/extract_ego.cpp @@ -135,6 +135,8 @@ struct extract_ego_functor : public cugraph::c_api::abstract_functor { new cugraph::c_api::cugraph_type_erased_device_array_t(dst, graph_->vertex_type_), wgt ? new cugraph::c_api::cugraph_type_erased_device_array_t(*wgt, graph_->weight_type_) : NULL, + NULL, + NULL, new cugraph::c_api::cugraph_type_erased_device_array_t(edge_offsets, cugraph_data_type_id_t::SIZE_T)}; } diff --git a/cpp/src/c_api/induced_subgraph.cpp b/cpp/src/c_api/induced_subgraph.cpp index a1bbcb60825..ac56301e231 100644 --- a/cpp/src/c_api/induced_subgraph.cpp +++ b/cpp/src/c_api/induced_subgraph.cpp @@ -147,11 +147,14 @@ struct induced_subgraph_functor : public cugraph::c_api::abstract_functor { graph_view.vertex_partition_range_lasts(), do_expensive_check_); + // FIXME: Add support for edge_id and edge_type_id. result_ = new cugraph::c_api::cugraph_induced_subgraph_result_t{ new cugraph::c_api::cugraph_type_erased_device_array_t(src, graph_->vertex_type_), new cugraph::c_api::cugraph_type_erased_device_array_t(dst, graph_->vertex_type_), wgt ? new cugraph::c_api::cugraph_type_erased_device_array_t(*wgt, graph_->weight_type_) : NULL, + NULL, + NULL, new cugraph::c_api::cugraph_type_erased_device_array_t(graph_offsets, SIZE_T)}; } } diff --git a/cpp/src/c_api/induced_subgraph_result.cpp b/cpp/src/c_api/induced_subgraph_result.cpp index b9ad0e0d66f..5226872d404 100644 --- a/cpp/src/c_api/induced_subgraph_result.cpp +++ b/cpp/src/c_api/induced_subgraph_result.cpp @@ -45,6 +45,28 @@ extern "C" cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get internal_pointer->wgt_->view()); } +extern "C" cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_ids( + cugraph_induced_subgraph_result_t* induced_subgraph) +{ + auto internal_pointer = + reinterpret_cast(induced_subgraph); + return (internal_pointer->edge_ids_ == nullptr) + ? NULL + : reinterpret_cast( + internal_pointer->edge_ids_->view()); +} + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_type_ids( + cugraph_induced_subgraph_result_t* induced_subgraph) +{ + auto internal_pointer = + reinterpret_cast(induced_subgraph); + return (internal_pointer->edge_type_ids_ == nullptr) + ? NULL + : reinterpret_cast( + internal_pointer->edge_type_ids_->view()); +} + extern "C" cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_subgraph_offsets( cugraph_induced_subgraph_result_t* induced_subgraph) { @@ -62,6 +84,8 @@ extern "C" void cugraph_induced_subgraph_result_free( delete internal_pointer->src_; delete internal_pointer->dst_; delete internal_pointer->wgt_; + delete internal_pointer->edge_ids_; + delete internal_pointer->edge_type_ids_; delete internal_pointer->subgraph_offsets_; delete internal_pointer; } diff --git a/cpp/src/c_api/induced_subgraph_result.hpp b/cpp/src/c_api/induced_subgraph_result.hpp index acc99b617f4..6f02a699605 100644 --- a/cpp/src/c_api/induced_subgraph_result.hpp +++ b/cpp/src/c_api/induced_subgraph_result.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,8 @@ struct cugraph_induced_subgraph_result_t { cugraph_type_erased_device_array_t* src_{}; cugraph_type_erased_device_array_t* dst_{}; cugraph_type_erased_device_array_t* wgt_{}; + cugraph_type_erased_device_array_t* edge_ids_{}; + cugraph_type_erased_device_array_t* edge_type_ids_{}; cugraph_type_erased_device_array_t* subgraph_offsets_{}; }; diff --git a/cpp/src/c_api/legacy_k_truss.cpp b/cpp/src/c_api/legacy_k_truss.cpp index 90e0894783a..90db9fc133c 100644 --- a/cpp/src/c_api/legacy_k_truss.cpp +++ b/cpp/src/c_api/legacy_k_truss.cpp @@ -123,12 +123,15 @@ struct k_truss_functor : public cugraph::c_api::abstract_functor { raft::update_device( edge_offsets.data(), h_edge_offsets.data(), h_edge_offsets.size(), handle_.get_stream()); + // FIXME: Add support for edge_id and edge_type_id. result_ = new cugraph::c_api::cugraph_induced_subgraph_result_t{ new cugraph::c_api::cugraph_type_erased_device_array_t(result_src, graph_->vertex_type_), new cugraph::c_api::cugraph_type_erased_device_array_t(result_dst, graph_->vertex_type_), wgt ? new cugraph::c_api::cugraph_type_erased_device_array_t(*result_wgt, graph_->weight_type_) : NULL, + NULL, + NULL, new cugraph::c_api::cugraph_type_erased_device_array_t(edge_offsets, cugraph_data_type_id_t::SIZE_T)}; } diff --git a/cpp/src/detail/collect_comm_wrapper.cu b/cpp/src/detail/collect_comm_wrapper.cu new file mode 100644 index 00000000000..7ce2241c677 --- /dev/null +++ b/cpp/src/detail/collect_comm_wrapper.cu @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +#include +#include + +namespace cugraph { +namespace detail { + +template +rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input) +{ + auto gathered_v = cugraph::device_allgatherv(handle, comm, d_input); + + return gathered_v; +} + +template rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +template rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +template rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +template rmm::device_uvector device_allgatherv(raft::handle_t const& handle, + raft::comms::comms_t const& comm, + raft::device_span d_input); + +} // namespace detail +} // namespace cugraph diff --git a/dependencies.yaml b/dependencies.yaml index a89acd9288b..2c0918ad117 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -380,6 +380,7 @@ dependencies: - &dask rapids-dask-dependency==23.12.* - &dask_cuda dask-cuda==23.12.* - &numba numba>=0.57 + - &numpy numpy>=1.21 - &ucx_py ucx-py==0.35.* - output_types: conda packages: @@ -399,7 +400,7 @@ dependencies: - output_types: [conda, pyproject] packages: - networkx>=3.0 - - &numpy numpy>=1.21 + - *numpy python_run_cugraph_dgl: common: - output_types: [conda, pyproject] diff --git a/python/cugraph/cugraph/structure/__init__.py b/python/cugraph/cugraph/structure/__init__.py index d7e0ff62358..94f34fd23f3 100644 --- a/python/cugraph/cugraph/structure/__init__.py +++ b/python/cugraph/cugraph/structure/__init__.py @@ -25,6 +25,11 @@ ) from cugraph.structure.number_map import NumberMap from cugraph.structure.symmetrize import symmetrize, symmetrize_df, symmetrize_ddf +from cugraph.structure.replicate_edgelist import ( + replicate_edgelist, + replicate_cudf_dataframe, + replicate_cudf_series, +) from cugraph.structure.convert_matrix import ( from_edgelist, from_cudf_edgelist, diff --git a/python/cugraph/cugraph/structure/replicate_edgelist.py b/python/cugraph/cugraph/structure/replicate_edgelist.py new file mode 100644 index 00000000000..d413e50e485 --- /dev/null +++ b/python/cugraph/cugraph/structure/replicate_edgelist.py @@ -0,0 +1,351 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dask_cudf +import cudf +from dask.distributed import wait, default_client +import numpy as np +from pylibcugraph import ( + ResourceHandle, + replicate_edgelist as pylibcugraph_replicate_edgelist, +) + +from cugraph.dask.common.part_utils import ( + get_persisted_df_worker_map, + persist_dask_df_equal_parts_per_worker, +) + +import dask +import cupy as cp +import cugraph.dask.comms.comms as Comms +from typing import Union, Tuple + + +# FIXME: Convert it to a general-purpose util function +def _convert_to_cudf(cp_arrays: Tuple[cp.ndarray], col_names: list) -> cudf.DataFrame: + """ + Creates a cudf Dataframe from cupy arrays + """ + src, dst, wgt, edge_id, edge_type_id, _ = cp_arrays + gathered_edgelist_df = cudf.DataFrame() + gathered_edgelist_df[col_names[0]] = src + gathered_edgelist_df[col_names[1]] = dst + if wgt is not None: + gathered_edgelist_df[col_names[2]] = wgt + if edge_id is not None: + gathered_edgelist_df[col_names[3]] = edge_id + if edge_type_id is not None: + gathered_edgelist_df[col_names[4]] = edge_type_id + + return gathered_edgelist_df + + +def _call_plc_replicate_edgelist( + sID: bytes, edgelist_df: cudf.DataFrame, col_names: list +) -> cudf.DataFrame: + edgelist_df = edgelist_df[0] + cp_arrays = pylibcugraph_replicate_edgelist( + resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()), + src_array=edgelist_df[col_names[0]], + dst_array=edgelist_df[col_names[1]], + weight_array=edgelist_df[col_names[2]] if len(col_names) > 2 else None, + edge_id_array=edgelist_df[col_names[3]] if len(col_names) > 3 else None, + edge_type_id_array=edgelist_df[col_names[4]] if len(col_names) > 4 else None, + ) + return _convert_to_cudf(cp_arrays, col_names) + + +def _call_plc_replicate_dataframe(sID: bytes, df: cudf.DataFrame) -> cudf.DataFrame: + df = df[0] + df_replicated = cudf.DataFrame() + for col_name in df.columns: + cp_array = pylibcugraph_replicate_edgelist( + resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()), + src_array=df[col_name] + if df[col_name].dtype in [np.int32, np.int64] + else None, + dst_array=None, + weight_array=df[col_name] + if df[col_name].dtype in [np.float32, np.float64] + else None, + edge_id_array=None, + edge_type_id_array=None, + ) + src, _, wgt, _, _, _ = cp_array + if src is not None: + df_replicated[col_name] = src + elif wgt is not None: + df_replicated[col_name] = wgt + + return df_replicated + + +def _call_plc_replicate_series(sID: bytes, series: cudf.Series) -> cudf.Series: + series = series[0] + series_replicated = cudf.Series() + cp_array = pylibcugraph_replicate_edgelist( + resource_handle=ResourceHandle(Comms.get_handle(sID).getHandle()), + src_array=series if series.dtype in [np.int32, np.int64] else None, + dst_array=None, + weight_array=series if series.dtype in [np.float32, np.float64] else None, + edge_id_array=None, + edge_type_id_array=None, + ) + src, _, wgt, _, _, _ = cp_array + if src is not None: + series_replicated = cudf.Series(src) + elif wgt is not None: + series_replicated = cudf.Series(wgt) + + return series_replicated + + +def _mg_call_plc_replicate( + client: dask.distributed.client.Client, + sID: bytes, + dask_object: dict, + input_type: str, + col_names: list, +) -> Union[dask_cudf.DataFrame, dask_cudf.Series]: + + if input_type == "dataframe": + result = [ + client.submit( + _call_plc_replicate_dataframe, + sID, + edata, + workers=[w], + allow_other_workers=False, + pure=False, + ) + for w, edata in dask_object.items() + ] + elif input_type == "dataframe": + result = [ + client.submit( + _call_plc_replicate_series, + sID, + edata, + workers=[w], + allow_other_workers=False, + pure=False, + ) + for w, edata in dask_object.items() + ] + elif input_type == "edgelist": + result = [ + client.submit( + _call_plc_replicate_edgelist, + sID, + edata, + col_names, + workers=[w], + allow_other_workers=False, + pure=False, + ) + for w, edata in dask_object.items() + ] + + ddf = dask_cudf.from_delayed(result, verify_meta=False).persist() + wait(ddf) + wait([r.release() for r in result]) + return ddf + + +def replicate_edgelist( + edgelist_ddf: Union[dask_cudf.DataFrame, cudf.DataFrame] = None, + source="src", + destination="dst", + weight=None, + edge_id=None, + edge_type=None, +) -> dask_cudf.DataFrame: + """ + Replicate edges across all GPUs + + Parameters + ---------- + + edgelist_ddf: cudf.DataFrame or dask_cudf.DataFrame + A DataFrame that contains edge information. + + source : str or array-like + source column name or array of column names + + destination : str or array-like + destination column name or array of column names + + weight : str, optional (default=None) + Name of the weight column in the input dataframe. + + edge_id : str, optional (default=None) + Name of the edge id column in the input dataframe. + + edge_type : str, optional (default=None) + Name of the edge type column in the input dataframe. + + Returns + ------- + df : dask_cudf.DataFrame + A distributed dataframe where each partition contains the + combined edgelist from all GPUs. If a cudf.DataFrame was passed + as input, the edgelist will be replicated across all the other + GPUs in the cluster. If as dask_cudf.DataFrame was passed as input, + each partition will be filled with the edges of all partitions + in the dask_cudf.DataFrame. + + """ + + _client = default_client() + + if isinstance(edgelist_ddf, cudf.DataFrame): + edgelist_ddf = dask_cudf.from_cudf( + edgelist_ddf, npartitions=len(Comms.get_workers()) + ) + col_names = [source, destination] + + if weight is not None: + col_names.append(weight) + if edge_id is not None: + col_names.append(edge_id) + if edge_type is not None: + col_names.append(edge_type) + + if not (set(col_names).issubset(set(edgelist_ddf.columns))): + raise ValueError( + "Invalid column names were provided: valid columns names are " + f"{edgelist_ddf.columns}" + ) + + edgelist_ddf = persist_dask_df_equal_parts_per_worker(edgelist_ddf, _client) + edgelist_ddf = get_persisted_df_worker_map(edgelist_ddf, _client) + + ddf = _mg_call_plc_replicate( + _client, + Comms.get_session_id(), + edgelist_ddf, + "edgelist", + col_names, + ) + + return ddf + + +def replicate_cudf_dataframe(cudf_dataframe): + """ + Replicate dataframe across all GPUs + + Parameters + ---------- + + cudf_dataframe: cudf.DataFrame or dask_cudf.DataFrame + + Returns + ------- + df : dask_cudf.DataFrame + A distributed dataframe where each partition contains the + combined dataframe from all GPUs. If a cudf.DataFrame was passed + as input, the dataframe will be replicated across all the other + GPUs in the cluster. If as dask_cudf.DataFrame was passed as input, + each partition will be filled with the datafame of all partitions + in the dask_cudf.DataFrame. + + """ + + supported_types = [np.int32, np.int64, np.float32, np.float64] + if not all(dtype in supported_types for dtype in cudf_dataframe.dtypes): + raise TypeError( + "The supported types are 'int32', 'int64', 'float32', 'float64'" + ) + + _client = default_client() + + if not isinstance(cudf_dataframe, dask_cudf.DataFrame): + if isinstance(cudf_dataframe, cudf.DataFrame): + df = dask_cudf.from_cudf( + cudf_dataframe, npartitions=len(Comms.get_workers()) + ) + elif not isinstance(cudf_dataframe, dask_cudf.DataFrame): + raise TypeError( + "The variable 'cudf_dataframe' must be of type " + f"'cudf/dask_cudf.dataframe', got type {type(cudf_dataframe)}" + ) + else: + df = cudf_dataframe + + df = persist_dask_df_equal_parts_per_worker(df, _client) + df = get_persisted_df_worker_map(df, _client) + + ddf = _mg_call_plc_replicate( + _client, + Comms.get_session_id(), + df, + "dataframe", + ) + + return ddf + + +def replicate_cudf_series(cudf_series): + """ + Replicate series across all GPUs + + Parameters + ---------- + + cudf_series: cudf.Series or dask_cudf.Series + + Returns + ------- + series : dask_cudf.Series + A distributed series where each partition contains the + combined series from all GPUs. If a cudf.Series was passed + as input, the Series will be replicated across all the other + GPUs in the cluster. If as dask_cudf.Series was passed as input, + each partition will be filled with the series of all partitions + in the dask_cudf.Series. + + """ + + supported_types = [np.int32, np.int64, np.float32, np.float64] + if cudf_series.dtype not in supported_types: + raise TypeError( + "The supported types are 'int32', 'int64', 'float32', 'float64'" + ) + + _client = default_client() + + if not isinstance(cudf_series, dask_cudf.Series): + if isinstance(cudf_series, cudf.Series): + series = dask_cudf.from_cudf( + cudf_series, npartitions=len(Comms.get_workers()) + ) + elif not isinstance(cudf_series, dask_cudf.Series): + raise TypeError( + "The variable 'cudf_series' must be of type " + f"'cudf/dask_cudf.series', got type {type(cudf_series)}" + ) + else: + series = cudf_series + + series = persist_dask_df_equal_parts_per_worker(series, _client) + series = get_persisted_df_worker_map(series, _client) + + series = _mg_call_plc_replicate( + _client, + Comms.get_session_id(), + series, + "series", + ) + + return series diff --git a/python/cugraph/cugraph/tests/community/test_induced_subgraph_mg.py b/python/cugraph/cugraph/tests/community/test_induced_subgraph_mg.py index d93fa3b547d..8d80611a54c 100644 --- a/python/cugraph/cugraph/tests/community/test_induced_subgraph_mg.py +++ b/python/cugraph/cugraph/tests/community/test_induced_subgraph_mg.py @@ -93,7 +93,7 @@ def input_expected_output(input_combo): srcs = G.view_edge_list()["0"] dsts = G.view_edge_list()["1"] vertices = cudf.concat([srcs, dsts]).drop_duplicates() - vertices = vertices.sample(num_seeds).astype("int32") + vertices = vertices.sample(num_seeds, replace=True).astype("int32") # print randomly sample n seeds from the graph print("\nvertices: \n", vertices) diff --git a/python/cugraph/cugraph/tests/internals/test_replicate_edgelist_mg.py b/python/cugraph/cugraph/tests/internals/test_replicate_edgelist_mg.py new file mode 100644 index 00000000000..3bdb5c079ef --- /dev/null +++ b/python/cugraph/cugraph/tests/internals/test_replicate_edgelist_mg.py @@ -0,0 +1,128 @@ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc + +import pytest + +import dask_cudf +import numpy as np +from cugraph.testing import UNDIRECTED_DATASETS, karate_disjoint + +from cugraph.structure.replicate_edgelist import replicate_edgelist +from cudf.testing.testing import assert_frame_equal +from pylibcugraph.testing.utils import gen_fixture_params_product + + +# ============================================================================= +# Pytest Setup / Teardown - called for each test function +# ============================================================================= +def setup_function(): + gc.collect() + + +edgeWeightCol = "weights" +edgeIdCol = "edge_id" +edgeTypeCol = "edge_type" +srcCol = "src" +dstCol = "dst" + + +input_data = UNDIRECTED_DATASETS + [karate_disjoint] +datasets = [pytest.param(d) for d in input_data] + +fixture_params = gen_fixture_params_product( + (datasets, "graph_file"), + ([True, False], "distributed"), + ([True, False], "use_weights"), + ([True, False], "use_edge_ids"), + ([True, False], "use_edge_type_ids"), +) + + +@pytest.fixture(scope="module", params=fixture_params) +def input_combo(request): + """ + Simply return the current combination of params as a dictionary for use in + tests or other parameterized fixtures. + """ + return dict( + zip( + ( + "graph_file", + "use_weights", + "use_edge_ids", + "use_edge_type_ids", + "distributed", + ), + request.param, + ) + ) + + +# ============================================================================= +# Tests +# ============================================================================= +# @pytest.mark.skipif( +# is_single_gpu(), reason="skipping MG testing on Single GPU system" +# ) +@pytest.mark.mg +def test_mg_replicate_edgelist(dask_client, input_combo): + df = input_combo["graph_file"].get_edgelist() + distributed = input_combo["distributed"] + + use_weights = input_combo["use_weights"] + use_edge_ids = input_combo["use_edge_ids"] + use_edge_type_ids = input_combo["use_edge_type_ids"] + + columns = [srcCol, dstCol] + weight = None + edge_id = None + edge_type = None + + if use_weights: + df = df.rename(columns={"wgt": edgeWeightCol}) + columns.append(edgeWeightCol) + weight = edgeWeightCol + if use_edge_ids: + df = df.reset_index().rename(columns={"index": edgeIdCol}) + df[edgeIdCol] = df[edgeIdCol].astype(df[srcCol].dtype) + columns.append(edgeIdCol) + edge_id = edgeIdCol + if use_edge_type_ids: + df[edgeTypeCol] = np.random.randint(0, 10, size=len(df)) + df[edgeTypeCol] = df[edgeTypeCol].astype(df[srcCol].dtype) + columns.append(edgeTypeCol) + edge_type = edgeTypeCol + + if distributed: + # Distribute the edges across all ranks + num_workers = len(dask_client.scheduler_info()["workers"]) + df = dask_cudf.from_cudf(df, npartitions=num_workers) + ddf = replicate_edgelist( + df[columns], weight=weight, edge_id=edge_id, edge_type=edge_type + ) + + if distributed: + df = df.compute() + + for i in range(ddf.npartitions): + result_df = ( + ddf.get_partition(i) + .compute() + .sort_values([srcCol, dstCol]) + .reset_index(drop=True) + ) + expected_df = df[columns].sort_values([srcCol, dstCol]).reset_index(drop=True) + + assert_frame_equal(expected_df, result_df, check_dtype=False, check_like=True) diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index 319900b3de3..bd426291c8d 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "dask-cudf==23.12.*", "fsspec[http]>=0.6.0", "numba>=0.57", + "numpy>=1.21", "pylibcugraph==23.12.*", "raft-dask==23.12.*", "rapids-dask-dependency==23.12.*", diff --git a/python/pylibcugraph/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/pylibcugraph/CMakeLists.txt index 6618c50122c..c2e22fc1ff7 100644 --- a/python/pylibcugraph/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/pylibcugraph/CMakeLists.txt @@ -56,6 +56,7 @@ set(cython_sources uniform_random_walks.pyx utils.pyx weakly_connected_components.pyx + replicate_edgelist.pyx ) set(linked_libraries cugraph::cugraph;cugraph::cugraph_c) diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 30f1c2d0fb1..1d02498ea30 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -87,6 +87,8 @@ from pylibcugraph.generate_rmat_edgelists import generate_rmat_edgelists +from pylibcugraph.replicate_edgelist import replicate_edgelist + from pylibcugraph.k_truss_subgraph import k_truss_subgraph from pylibcugraph.jaccard_coefficients import jaccard_coefficients diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd index f18e9848182..8b3a629956c 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_functions.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -138,6 +138,16 @@ cdef extern from "cugraph_c/graph_functions.h": cugraph_induced_subgraph_result_t* induced_subgraph ) + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_induced_subgraph_get_edge_ids( + cugraph_induced_subgraph_result_t* induced_subgraph + ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_induced_subgraph_get_edge_type_ids( + cugraph_induced_subgraph_result_t* induced_subgraph + ) + cdef cugraph_type_erased_device_array_view_t* \ cugraph_induced_subgraph_get_subgraph_offsets( cugraph_induced_subgraph_result_t* induced_subgraph @@ -158,3 +168,17 @@ cdef extern from "cugraph_c/graph_functions.h": cugraph_induced_subgraph_result_t** result, cugraph_error_t** error ) + + ########################################################################### + # allgather + cdef cugraph_error_code_t \ + cugraph_allgather( + const cugraph_resource_handle_t* handle, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + cugraph_induced_subgraph_result_t** result, + cugraph_error_t** error + ) diff --git a/python/pylibcugraph/pylibcugraph/replicate_edgelist.pyx b/python/pylibcugraph/pylibcugraph/replicate_edgelist.pyx new file mode 100644 index 00000000000..3763d4bc69d --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/replicate_edgelist.pyx @@ -0,0 +1,202 @@ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Have cython use python 3 syntax +# cython: language_level = 3 + + +from pylibcugraph._cugraph_c.resource_handle cimport ( + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c.error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c.array cimport ( + cugraph_type_erased_device_array_view_t, + cugraph_type_erased_device_array_view_free, +) +from pylibcugraph._cugraph_c.graph_functions cimport ( + cugraph_allgather, + cugraph_induced_subgraph_result_t, + cugraph_induced_subgraph_get_sources, + cugraph_induced_subgraph_get_destinations, + cugraph_induced_subgraph_get_edge_weights, + cugraph_induced_subgraph_get_edge_ids, + cugraph_induced_subgraph_get_edge_type_ids, + cugraph_induced_subgraph_get_subgraph_offsets, + cugraph_induced_subgraph_result_free, +) +from pylibcugraph.resource_handle cimport ( + ResourceHandle, +) +from pylibcugraph.utils cimport ( + assert_success, + assert_CAI_type, + copy_to_cupy_array, + create_cugraph_type_erased_device_array_view_from_py_obj +) + + +def replicate_edgelist(ResourceHandle resource_handle, + src_array, + dst_array, + weight_array, + edge_id_array, + edge_type_id_array): + """ + Replicate edges across all GPUs + + Parameters + ---------- + resource_handle : ResourceHandle + Handle to the underlying device resources needed for referencing data + and running algorithms. + + src_array : device array type, optional + Device array containing the vertex identifiers of the source of each + directed edge. The order of the array corresponds to the ordering of the + dst_array, where the ith item in src_array and the ith item in dst_array + define the ith edge of the graph. + + dst_array : device array type, optional + Device array containing the vertex identifiers of the destination of + each directed edge. The order of the array corresponds to the ordering + of the src_array, where the ith item in src_array and the ith item in + dst_array define the ith edge of the graph. + + weight_array : device array type, optional + Device array containing the weight values of each directed edge. The + order of the array corresponds to the ordering of the src_array and + dst_array arrays, where the ith item in weight_array is the weight value + of the ith edge of the graph. + + edge_id_array : device array type, optional + Device array containing the edge id values of each directed edge. The + order of the array corresponds to the ordering of the src_array and + dst_array arrays, where the ith item in edge_id_array is the id value + of the ith edge of the graph. + + edge_type_id_array : device array type, optional + Device array containing the edge type id values of each directed edge. The + order of the array corresponds to the ordering of the src_array and + dst_array arrays, where the ith item in edge_type_id_array is the type id + value of the ith edge of the graph. + + Returns + ------- + return cupy arrays of 'src' and/or 'dst' and/or 'weight'and/or 'edge_id' + and/or 'edge_type_id'. + """ + assert_CAI_type(src_array, "src_array", True) + assert_CAI_type(dst_array, "dst_array", True) + assert_CAI_type(weight_array, "weight_array", True) + assert_CAI_type(edge_id_array, "edge_id_array", True) + assert_CAI_type(edge_type_id_array, "edge_type_id_array", True) + cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ + resource_handle.c_resource_handle_ptr + + cdef cugraph_induced_subgraph_result_t* result_ptr + cdef cugraph_error_code_t error_code + cdef cugraph_error_t* error_ptr + + cdef cugraph_type_erased_device_array_view_t* srcs_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(src_array) + + cdef cugraph_type_erased_device_array_view_t* dsts_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(dst_array) + + + cdef cugraph_type_erased_device_array_view_t* weights_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(weight_array) + + cdef cugraph_type_erased_device_array_view_t* edge_ids_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(edge_id_array) + + cdef cugraph_type_erased_device_array_view_t* edge_type_ids_view_ptr = \ + create_cugraph_type_erased_device_array_view_from_py_obj(edge_type_id_array) + + error_code = cugraph_allgather(c_resource_handle_ptr, + srcs_view_ptr, + dsts_view_ptr, + weights_view_ptr, + edge_ids_view_ptr, + edge_type_ids_view_ptr, + &result_ptr, + &error_ptr) + assert_success(error_code, error_ptr, "replicate_edgelist") + # Extract individual device array pointers from result and copy to cupy + # arrays for returning. + cdef cugraph_type_erased_device_array_view_t* sources_ptr + if src_array is not None: + sources_ptr = cugraph_induced_subgraph_get_sources(result_ptr) + cdef cugraph_type_erased_device_array_view_t* destinations_ptr + if dst_array is not None: + destinations_ptr = cugraph_induced_subgraph_get_destinations(result_ptr) + cdef cugraph_type_erased_device_array_view_t* edge_weights_ptr = \ + cugraph_induced_subgraph_get_edge_weights(result_ptr) + cdef cugraph_type_erased_device_array_view_t* edge_ids_ptr = \ + cugraph_induced_subgraph_get_edge_ids(result_ptr) + cdef cugraph_type_erased_device_array_view_t* edge_type_ids_ptr = \ + cugraph_induced_subgraph_get_edge_type_ids(result_ptr) + cdef cugraph_type_erased_device_array_view_t* subgraph_offsets_ptr = \ + cugraph_induced_subgraph_get_subgraph_offsets(result_ptr) + + # FIXME: Get ownership of the result data instead of performing a copy + # for perfomance improvement + + cupy_sources = None + cupy_destinations = None + cupy_edge_weights = None + cupy_edge_ids = None + cupy_edge_type_ids = None + + if src_array is not None: + cupy_sources = copy_to_cupy_array( + c_resource_handle_ptr, sources_ptr) + + if dst_array is not None: + cupy_destinations = copy_to_cupy_array( + c_resource_handle_ptr, destinations_ptr) + + if weight_array is not None: + cupy_edge_weights = copy_to_cupy_array( + c_resource_handle_ptr, edge_weights_ptr) + + if edge_id_array is not None: + cupy_edge_ids = copy_to_cupy_array( + c_resource_handle_ptr, edge_ids_ptr) + + if edge_type_id_array is not None: + cupy_edge_type_ids = copy_to_cupy_array( + c_resource_handle_ptr, edge_type_ids_ptr) + + cupy_subgraph_offsets = copy_to_cupy_array( + c_resource_handle_ptr, subgraph_offsets_ptr) + + # Free pointer + cugraph_induced_subgraph_result_free(result_ptr) + if src_array is not None: + cugraph_type_erased_device_array_view_free(srcs_view_ptr) + if dst_array is not None: + cugraph_type_erased_device_array_view_free(dsts_view_ptr) + if weight_array is not None: + cugraph_type_erased_device_array_view_free(weights_view_ptr) + if edge_id_array is not None: + cugraph_type_erased_device_array_view_free(edge_ids_view_ptr) + if edge_type_id_array is not None: + cugraph_type_erased_device_array_view_free(edge_type_ids_view_ptr) + + return (cupy_sources, cupy_destinations, + cupy_edge_weights, cupy_edge_ids, + cupy_edge_type_ids, cupy_subgraph_offsets) From 3f1c7b5ef8cd296aa5ad012cf855abdd2dfeb84b Mon Sep 17 00:00:00 2001 From: Chuck Hastings <45364586+ChuckHastings@users.noreply.github.com> Date: Mon, 20 Nov 2023 21:17:24 -0500 Subject: [PATCH 2/6] Update C API graph creation function signatures (#3982) Updating the C API graph creation functions to support the following: * Add support for isolated vertices * Add MG optimization to support multiple device arrays per rank as input and concatenate them internally * Add MG optimization to internally compute the number of edges via allreduce rather than requiring it as an input parameter (this can be expensive to compute in python) This PR implements these features. Some simple tests exist to check for isolate vertices (by running pagerank which generates a different result if the graph has isolated vertices). A simple test for multiple input arrays exists for the MG case. Closes #3947 Closes #3974 Authors: - Chuck Hastings (https://github.com/ChuckHastings) - Naim (https://github.com/naimnv) Approvers: - Naim (https://github.com/naimnv) - Joseph Nke (https://github.com/jnke2016) - Seunghwa Kang (https://github.com/seunghwak) URL: https://github.com/rapidsai/cugraph/pull/3982 --- cpp/CMakeLists.txt | 2 + cpp/include/cugraph/graph_functions.hpp | 67 +++ cpp/include/cugraph_c/graph.h | 193 ++++++- cpp/include/cugraph_c/resource_handle.h | 12 + cpp/src/c_api/graph_mg.cpp | 511 +++++++++++------- cpp/src/c_api/graph_sg.cpp | 170 +++++- cpp/src/c_api/resource_handle.cpp | 9 +- cpp/src/structure/detail/structure_utils.cuh | 61 ++- cpp/src/structure/remove_multi_edges.cu | 92 ++++ cpp/src/structure/remove_multi_edges_impl.cuh | 310 +++++++++++ cpp/src/structure/remove_self_loops.cu | 92 ++++ cpp/src/structure/remove_self_loops_impl.cuh | 94 ++++ cpp/tests/c_api/create_graph_test.c | 498 ++++++++++++++++- cpp/tests/c_api/mg_create_graph_test.c | 400 +++++++++++++- 14 files changed, 2265 insertions(+), 246 deletions(-) create mode 100644 cpp/src/structure/remove_multi_edges.cu create mode 100644 cpp/src/structure/remove_multi_edges_impl.cuh create mode 100644 cpp/src/structure/remove_self_loops.cu create mode 100644 cpp/src/structure/remove_self_loops_impl.cuh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 626d62cffa5..836d5569ef7 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -202,6 +202,8 @@ set(CUGRAPH_SOURCES src/community/detail/mis_mg.cu src/detail/utility_wrappers.cu src/structure/graph_view_mg.cu + src/structure/remove_self_loops.cu + src/structure/remove_multi_edges.cu src/utilities/path_retrieval.cu src/structure/legacy/graph.cu src/linear_assignment/legacy/hungarian.cu diff --git a/cpp/include/cugraph/graph_functions.hpp b/cpp/include/cugraph/graph_functions.hpp index 5c1e9d5311f..6a75a420bf8 100644 --- a/cpp/include/cugraph/graph_functions.hpp +++ b/cpp/include/cugraph/graph_functions.hpp @@ -973,4 +973,71 @@ renumber_sampled_edgelist( label_offsets, bool do_expensive_check = false); +/** + * @brief Remove self loops from an edge list + * + * @tparam vertex_t Type of vertex identifiers. Needs to be an integral type. + * @tparam edge_t Type of edge identifiers. Needs to be an integral type. + * @tparam weight_t Type of edge weight. Currently float and double are supported. + * @tparam edge_type_t Type of edge type. Needs to be an integral type. + * + * @param handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, and + * handles to various CUDA libraries) to run graph algorithms. + * @param edgelist_srcs List of source vertex ids + * @param edgelist_dsts List of destination vertex ids + * @param edgelist_weights Optional list of edge weights + * @param edgelist_edge_ids Optional list of edge ids + * @param edgelist_edge_types Optional list of edge types + * @return Tuple of vectors storing edge sources, destinations, optional weights, + * optional edge ids, optional edge types. + */ +template +std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +/** + * @brief Remove all but one edge when a multi-edge exists. Note that this function does not use + * stable methods. When a multi-edge exists, one of the edges will remain, there is no + * guarantee on which one will remain. + * + * In an MG context it is assumed that edges have been shuffled to the proper GPU, + * in which case any multi-edges will be on the same GPU. + * + * @tparam vertex_t Type of vertex identifiers. Needs to be an integral type. + * @tparam edge_t Type of edge identifiers. Needs to be an integral type. + * @tparam weight_t Type of edge weight. Currently float and double are supported. + * @tparam edge_type_t Type of edge type. Needs to be an integral type. + * + * @param handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator, and + * handles to various CUDA libraries) to run graph algorithms. + * @param edgelist_srcs List of source vertex ids + * @param edgelist_dsts List of destination vertex ids + * @param edgelist_weights Optional list of edge weights + * @param edgelist_edge_ids Optional list of edge ids + * @param edgelist_edge_types Optional list of edge types + * @return Tuple of vectors storing edge sources, destinations, optional weights, + * optional edge ids, optional edge types. + */ +template +std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + } // namespace cugraph diff --git a/cpp/include/cugraph_c/graph.h b/cpp/include/cugraph_c/graph.h index e910d8b1244..88176a9c1b6 100644 --- a/cpp/include/cugraph_c/graph.h +++ b/cpp/include/cugraph_c/graph.h @@ -35,10 +35,11 @@ typedef struct { bool_t is_multigraph; } cugraph_graph_properties_t; -// FIXME: Add support for specifying isolated vertices /** * @brief Construct an SG graph * + * @deprecated This API will be deleted, use cugraph_graph_create_sg instead + * * @param [in] handle Handle for accessing resources * @param [in] properties Properties of the constructed graph * @param [in] src Device array containing the source vertex ids. @@ -51,11 +52,11 @@ typedef struct { argument that can be NULL if edge types are not used. * @param [in] store_transposed If true create the graph initially in transposed format * @param [in] renumber If true, renumber vertices to make an efficient data structure. - * If false, do not renumber. Renumbering is required if the vertices are not sequential - * integer values from 0 to num_vertices. + * If false, do not renumber. Renumbering enables some significant optimizations within + * the graph primitives library, so it is strongly encouraged. Renumbering is required if + * the vertices are not sequential integer values from 0 to num_vertices. * @param [in] do_expensive_check If true, do expensive checks to validate the input data * is consistent with software assumptions. If false bypass these checks. - * @param [in] properties Properties of the graph * @param [out] graph A pointer to the graph object * @param [out] error Pointer to an error object storing details of any error. Will * be populated if error code is not CUGRAPH_SUCCESS @@ -76,9 +77,63 @@ cugraph_error_code_t cugraph_sg_graph_create( cugraph_graph_t** graph, cugraph_error_t** error); +/** + * @brief Construct an SG graph + * + * @param [in] handle Handle for accessing resources + * @param [in] properties Properties of the constructed graph + * @param [in] vertices Optional device array containing a list of vertex ids + * (specify NULL if we should create vertex ids from the + * unique contents of @p src and @p dst) + * @param [in] src Device array containing the source vertex ids. + * @param [in] dst Device array containing the destination vertex ids + * @param [in] weights Device array containing the edge weights. Note that an unweighted + * graph can be created by passing weights == NULL. + * @param [in] edge_ids Device array containing the edge ids for each edge. Optional + argument that can be NULL if edge ids are not used. + * @param [in] edge_type_ids Device array containing the edge types for each edge. Optional + argument that can be NULL if edge types are not used. + * @param [in] store_transposed If true create the graph initially in transposed format + * @param [in] renumber If true, renumber vertices to make an efficient data structure. + * If false, do not renumber. Renumbering enables some significant optimizations within + * the graph primitives library, so it is strongly encouraged. Renumbering is required if + * the vertices are not sequential integer values from 0 to num_vertices. + * @param [in] drop_self_loops If true, drop any self loops that exist in the provided edge list. + * @param [in] drop_multi_edges If true, drop any multi edges that exist in the provided edge list. + * Note that setting this flag will arbitrarily select one instance of a multi edge to be the + * edge that survives. If the edges have properties that should be honored (e.g. sum the + weights, + * or take the maximum weight), the caller should do that on not rely on this flag. + * @param [in] do_expensive_check If true, do expensive checks to validate the input data + * is consistent with software assumptions. If false bypass these checks. + * @param [out] graph A pointer to the graph object + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * + * @return error code + */ +cugraph_error_code_t cugraph_graph_create_sg( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* vertices, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error); + /** * @brief Construct an SG graph from a CSR input * + * @deprecated This API will be deleted, use cugraph_graph_create_sg_from_csr instead + * * @param [in] handle Handle for accessing resources * @param [in] properties Properties of the constructed graph * @param [in] offsets Device array containing the CSR offsets array @@ -91,11 +146,11 @@ cugraph_error_code_t cugraph_sg_graph_create( argument that can be NULL if edge types are not used. * @param [in] store_transposed If true create the graph initially in transposed format * @param [in] renumber If true, renumber vertices to make an efficient data structure. - * If false, do not renumber. Renumbering is required if the vertices are not sequential - * integer values from 0 to num_vertices. + * If false, do not renumber. Renumbering enables some significant optimizations within + * the graph primitives library, so it is strongly encouraged. Renumbering is required if + * the vertices are not sequential integer values from 0 to num_vertices. * @param [in] do_expensive_check If true, do expensive checks to validate the input data * is consistent with software assumptions. If false bypass these checks. - * @param [in] properties Properties of the graph * @param [out] graph A pointer to the graph object * @param [out] error Pointer to an error object storing details of any error. Will * be populated if error code is not CUGRAPH_SUCCESS @@ -117,18 +172,50 @@ cugraph_error_code_t cugraph_sg_graph_create_from_csr( cugraph_error_t** error); /** - * @brief Destroy an SG graph + * @brief Construct an SG graph from a CSR input * - * @param [in] graph A pointer to the graph object to destroy + * @param [in] handle Handle for accessing resources + * @param [in] properties Properties of the constructed graph + * @param [in] offsets Device array containing the CSR offsets array + * @param [in] indices Device array containing the destination vertex ids + * @param [in] weights Device array containing the edge weights. Note that an unweighted + * graph can be created by passing weights == NULL. + * @param [in] edge_ids Device array containing the edge ids for each edge. Optional + argument that can be NULL if edge ids are not used. + * @param [in] edge_type_ids Device array containing the edge types for each edge. Optional + argument that can be NULL if edge types are not used. + * @param [in] store_transposed If true create the graph initially in transposed format + * @param [in] renumber If true, renumber vertices to make an efficient data structure. + * If false, do not renumber. Renumbering enables some significant optimizations within + * the graph primitives library, so it is strongly encouraged. Renumbering is required if + * the vertices are not sequential integer values from 0 to num_vertices. + * @param [in] do_expensive_check If true, do expensive checks to validate the input data + * is consistent with software assumptions. If false bypass these checks. + * @param [out] graph A pointer to the graph object + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * + * @return error code */ -// FIXME: This should probably just be cugraph_graph_free -// but didn't want to confuse with original cugraph_free_graph -void cugraph_sg_graph_free(cugraph_graph_t* graph); +cugraph_error_code_t cugraph_graph_create_sg_from_csr( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* offsets, + const cugraph_type_erased_device_array_view_t* indices, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error); -// FIXME: Add support for specifying isolated vertices /** * @brief Construct an MG graph * + * @deprecated This API will be deleted, use cugraph_graph_create_mg instead + * * @param [in] handle Handle for accessing resources * @param [in] properties Properties of the constructed graph * @param [in] src Device array containing the source vertex ids @@ -165,13 +252,89 @@ cugraph_error_code_t cugraph_mg_graph_create( cugraph_graph_t** graph, cugraph_error_t** error); +/** + * @brief Construct an MG graph + * + * @param [in] handle Handle for accessing resources + * @param [in] properties Properties of the constructed graph + * @param [in] vertices List of device arrays containing the unique vertex ids. + * If NULL we will construct this internally using the unique + * entries specified in src and dst + * All entries in this list will be concatenated on this GPU + * into a single array. + * @param [in] src List of device array containing the source vertex ids + * All entries in this list will be concatenated on this GPU + * into a single array. + * @param [in] dst List of device array containing the destination vertex ids + * All entries in this list will be concatenated on this GPU + * into a single array. + * @param [in] weights List of device array containing the edge weights. Note that an + * unweighted graph can be created by passing weights == NULL. If a weighted graph is to be + * created, the weights device array should be created on each rank, but the pointer can be NULL and + * the size 0 if there are no inputs provided by this rank All entries in this list will be + * concatenated on this GPU into a single array. + * @param [in] edge_ids List of device array containing the edge ids for each edge. Optional + * argument that can be NULL if edge ids are not used. + * All entries in this list will be concatenated on this GPU + * into a single array. + * @param [in] edge_type_ids List of device array containing the edge types for each edge. + * Optional argument that can be NULL if edge types are not used. All entries in this list will be + * concatenated on this GPU into a single array. + * @param [in] store_transposed If true create the graph initially in transposed format + * @param [in] num_arrays The number of arrays specified in @p vertices, @p src, @p dst, @p + * weights, @p edge_ids and @p edge_type_ids + * @param [in] drop_self_loops If true, drop any self loops that exist in the provided edge list. + * @param [in] drop_multi_edges If true, drop any multi edges that exist in the provided edge list. + * Note that setting this flag will arbitrarily select one instance of a multi edge to be the + * edge that survives. If the edges have properties that should be honored (e.g. sum the + * weights, or take the maximum weight), the caller should do that on not rely on this flag. + * @param [in] do_expensive_check If true, do expensive checks to validate the input data + * is consistent with software assumptions. If false bypass these checks. + * @param [out] graph A pointer to the graph object + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_graph_create_mg( + cugraph_resource_handle_t const* handle, + cugraph_graph_properties_t const* properties, + cugraph_type_erased_device_array_view_t const* const* vertices, + cugraph_type_erased_device_array_view_t const* const* src, + cugraph_type_erased_device_array_view_t const* const* dst, + cugraph_type_erased_device_array_view_t const* const* weights, + cugraph_type_erased_device_array_view_t const* const* edge_ids, + cugraph_type_erased_device_array_view_t const* const* edge_type_ids, + bool_t store_transposed, + size_t num_arrays, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error); + +/** + * @brief Destroy an graph + * + * @param [in] graph A pointer to the graph object to destroy + */ +void cugraph_graph_free(cugraph_graph_t* graph); + +/** + * @brief Destroy an SG graph + * + * @deprecated This API will be deleted, use cugraph_graph_free instead + * + * @param [in] graph A pointer to the graph object to destroy + */ +void cugraph_sg_graph_free(cugraph_graph_t* graph); + /** * @brief Destroy an MG graph * + * @deprecated This API will be deleted, use cugraph_graph_free instead + * * @param [in] graph A pointer to the graph object to destroy */ -// FIXME: This should probably just be cugraph_graph_free -// but didn't want to confuse with original cugraph_free_graph void cugraph_mg_graph_free(cugraph_graph_t* graph); /** diff --git a/cpp/include/cugraph_c/resource_handle.h b/cpp/include/cugraph_c/resource_handle.h index a239c24afe9..0e45102aae2 100644 --- a/cpp/include/cugraph_c/resource_handle.h +++ b/cpp/include/cugraph_c/resource_handle.h @@ -57,6 +57,18 @@ typedef struct cugraph_resource_handle_ { */ cugraph_resource_handle_t* cugraph_create_resource_handle(void* raft_handle); +/** + * @brief get comm_size from resource handle + * + * If the resource handle has been configured for multi-gpu, this will return + * the comm_size for this cluster. If the resource handle has not been configured for + * multi-gpu this will always return 1. + * + * @param [in] handle Handle for accessing resources + * @return comm_size + */ +int cugraph_resource_handle_get_comm_size(const cugraph_resource_handle_t* handle); + /** * @brief get rank from resource handle * diff --git a/cpp/src/c_api/graph_mg.cpp b/cpp/src/c_api/graph_mg.cpp index f50c7c08fb6..5413949e3a3 100644 --- a/cpp/src/c_api/graph_mg.cpp +++ b/cpp/src/c_api/graph_mg.cpp @@ -31,40 +31,85 @@ namespace { +template +rmm::device_uvector concatenate( + raft::handle_t const& handle, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* values, + size_t num_arrays) +{ + size_t num_values = std::transform_reduce( + values, values + num_arrays, size_t{0}, std::plus{}, [](auto p) { return p->size_; }); + + rmm::device_uvector results(num_values, handle.get_stream()); + size_t concat_pos{0}; + + for (size_t i = 0; i < num_arrays; ++i) { + raft::copy(results.data() + concat_pos, + values[i]->as_type(), + values[i]->size_, + handle.get_stream()); + concat_pos += values[i]->size_; + } + + return results; +} + struct create_graph_functor : public cugraph::c_api::abstract_functor { raft::handle_t const& handle_; cugraph_graph_properties_t const* properties_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* src_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids_; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids_; - bool_t renumber_; - bool_t check_; + cugraph_data_type_id_t vertex_type_; cugraph_data_type_id_t edge_type_; + cugraph_data_type_id_t weight_type_; + cugraph_data_type_id_t edge_type_id_type_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* vertices_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* src_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* dst_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* weights_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* edge_ids_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* edge_type_ids_; + size_t num_arrays_; + bool_t renumber_; + bool_t drop_self_loops_; + bool_t drop_multi_edges_; + bool_t do_expensive_check_; cugraph::c_api::cugraph_graph_t* result_{}; - create_graph_functor(raft::handle_t const& handle, - cugraph_graph_properties_t const* properties, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* src, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids, - cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids, - bool_t renumber, - bool_t check, - cugraph_data_type_id_t edge_type) + create_graph_functor( + raft::handle_t const& handle, + cugraph_graph_properties_t const* properties, + cugraph_data_type_id_t vertex_type, + cugraph_data_type_id_t edge_type, + cugraph_data_type_id_t weight_type, + cugraph_data_type_id_t edge_type_id_type, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* vertices, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* src, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* dst, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* weights, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* edge_ids, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* const* edge_type_ids, + size_t num_arrays, + bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check) : abstract_functor(), properties_(properties), + vertex_type_(vertex_type), + edge_type_(edge_type), + weight_type_(weight_type), + edge_type_id_type_(edge_type_id_type), handle_(handle), + vertices_(vertices), src_(src), dst_(dst), weights_(weights), edge_ids_(edge_ids), edge_type_ids_(edge_type_ids), + num_arrays_(num_arrays), renumber_(renumber), - check_(check), - edge_type_(edge_type) + drop_self_loops_(drop_self_loops), + drop_multi_edges_(drop_multi_edges), + do_expensive_check_(do_expensive_check) { } @@ -96,49 +141,27 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { edge_type_id_t>> new_edge_types{std::nullopt}; - rmm::device_uvector edgelist_srcs(src_->size_, handle_.get_stream()); - rmm::device_uvector edgelist_dsts(dst_->size_, handle_.get_stream()); + std::optional> vertex_list = + vertices_ ? std::make_optional(concatenate(handle_, vertices_, num_arrays_)) + : std::nullopt; - raft::copy( - edgelist_srcs.data(), src_->as_type(), src_->size_, handle_.get_stream()); - raft::copy( - edgelist_dsts.data(), dst_->as_type(), dst_->size_, handle_.get_stream()); + rmm::device_uvector edgelist_srcs = + concatenate(handle_, src_, num_arrays_); + rmm::device_uvector edgelist_dsts = + concatenate(handle_, dst_, num_arrays_); std::optional> edgelist_weights = - weights_ - ? std::make_optional(rmm::device_uvector(weights_->size_, handle_.get_stream())) - : std::nullopt; - - if (edgelist_weights) { - raft::copy(edgelist_weights->data(), - weights_->as_type(), - weights_->size_, - handle_.get_stream()); - } + weights_ ? std::make_optional(concatenate(handle_, weights_, num_arrays_)) + : std::nullopt; std::optional> edgelist_edge_ids = - edge_ids_ - ? std::make_optional(rmm::device_uvector(edge_ids_->size_, handle_.get_stream())) - : std::nullopt; - - if (edgelist_edge_ids) { - raft::copy(edgelist_edge_ids->data(), - edge_ids_->as_type(), - edge_ids_->size_, - handle_.get_stream()); - } + edge_ids_ ? std::make_optional(concatenate(handle_, edge_ids_, num_arrays_)) + : std::nullopt; std::optional> edgelist_edge_types = - edge_type_ids_ ? std::make_optional(rmm::device_uvector( - edge_type_ids_->size_, handle_.get_stream())) - : std::nullopt; - - if (edgelist_edge_types) { - raft::copy(edgelist_edge_types->data(), - edge_type_ids_->as_type(), - edge_type_ids_->size_, - handle_.get_stream()); - } + edge_type_ids_ + ? std::make_optional(concatenate(handle_, edge_type_ids_, num_arrays_)) + : std::nullopt; std::tie(store_transposed ? edgelist_dsts : edgelist_srcs, store_transposed ? edgelist_srcs : edgelist_dsts, @@ -153,6 +176,11 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { std::move(edgelist_edge_ids), std::move(edgelist_edge_types)); + if (vertex_list) { + vertex_list = cugraph::detail::shuffle_ext_vertices_to_local_gpu_by_vertex_partitioning( + handle_, std::move(*vertex_list)); + } + auto graph = new cugraph::graph_t(handle_); rmm::device_uvector* number_map = @@ -170,6 +198,28 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { cugraph::graph_view_t, edge_type_id_t>(handle_); + if (drop_self_loops_) { + std::tie( + edgelist_srcs, edgelist_dsts, edgelist_weights, edgelist_edge_ids, edgelist_edge_types) = + cugraph::remove_self_loops(handle_, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); + } + + if (drop_multi_edges_) { + std::tie( + edgelist_srcs, edgelist_dsts, edgelist_weights, edgelist_edge_ids, edgelist_edge_types) = + cugraph::remove_multi_edges(handle_, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); + } + std::tie(*graph, new_edge_weights, new_edge_ids, new_edge_types, new_number_map) = cugraph::create_graph_from_edgelist( handle_, - std::nullopt, + std::move(vertex_list), std::move(edgelist_srcs), std::move(edgelist_dsts), std::move(edgelist_weights), @@ -187,7 +237,7 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { std::move(edgelist_edge_types), cugraph::graph_properties_t{properties_->is_symmetric, properties_->is_multigraph}, renumber_, - check_); + do_expensive_check_); if (renumber_) { *number_map = std::move(new_number_map.value()); @@ -204,90 +254,39 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { if (new_edge_types) { *edge_types = std::move(new_edge_types.value()); } // Set up return - auto result = new cugraph::c_api::cugraph_graph_t{ - src_->type_, - edge_type_, - weights_ ? weights_->type_ : cugraph_data_type_id_t::FLOAT32, - edge_type_ids_ ? edge_type_ids_->type_ : cugraph_data_type_id_t::INT32, - store_transposed, - multi_gpu, - graph, - number_map, - new_edge_weights ? edge_weights : nullptr, - new_edge_ids ? edge_ids : nullptr, - new_edge_types ? edge_types : nullptr}; + auto result = new cugraph::c_api::cugraph_graph_t{vertex_type_, + edge_type_, + weight_type_, + edge_type_id_type_, + store_transposed, + multi_gpu, + graph, + number_map, + new_edge_weights ? edge_weights : nullptr, + new_edge_ids ? edge_ids : nullptr, + new_edge_types ? edge_types : nullptr}; result_ = reinterpret_cast(result); } } }; -struct destroy_graph_functor : public cugraph::c_api::abstract_functor { - void* graph_; - void* number_map_; - void* edge_weights_; - void* edge_ids_; - void* edge_types_; - - destroy_graph_functor( - void* graph, void* number_map, void* edge_weights, void* edge_ids, void* edge_types) - : abstract_functor(), - graph_(graph), - number_map_(number_map), - edge_weights_(edge_weights), - edge_ids_(edge_ids), - edge_types_(edge_types) - { - } - - template - void operator()() - { - auto internal_graph_pointer = - reinterpret_cast*>(graph_); - - delete internal_graph_pointer; - - auto internal_number_map_pointer = - reinterpret_cast*>(number_map_); - - delete internal_number_map_pointer; - - auto internal_edge_weight_pointer = reinterpret_cast< - cugraph::edge_property_t, - weight_t>*>(edge_weights_); - if (internal_edge_weight_pointer) { delete internal_edge_weight_pointer; } - - auto internal_edge_id_pointer = reinterpret_cast< - cugraph::edge_property_t, - edge_t>*>(edge_ids_); - if (internal_edge_id_pointer) { delete internal_edge_id_pointer; } - - auto internal_edge_type_pointer = reinterpret_cast< - cugraph::edge_property_t, - edge_type_id_t>*>(edge_types_); - if (internal_edge_type_pointer) { delete internal_edge_type_pointer; } - } -}; - } // namespace -extern "C" cugraph_error_code_t cugraph_mg_graph_create( - const cugraph_resource_handle_t* handle, - const cugraph_graph_properties_t* properties, - const cugraph_type_erased_device_array_view_t* src, - const cugraph_type_erased_device_array_view_t* dst, - const cugraph_type_erased_device_array_view_t* weights, - const cugraph_type_erased_device_array_view_t* edge_ids, - const cugraph_type_erased_device_array_view_t* edge_type_ids, +extern "C" cugraph_error_code_t cugraph_graph_create_mg( + cugraph_resource_handle_t const* handle, + cugraph_graph_properties_t const* properties, + cugraph_type_erased_device_array_view_t const* const* vertices, + cugraph_type_erased_device_array_view_t const* const* src, + cugraph_type_erased_device_array_view_t const* const* dst, + cugraph_type_erased_device_array_view_t const* const* weights, + cugraph_type_erased_device_array_view_t const* const* edge_ids, + cugraph_type_erased_device_array_view_t const* const* edge_type_ids, bool_t store_transposed, - size_t num_edges, - bool_t check, + size_t num_arrays, + bool_t drop_self_loops, + bool_t drop_multi_edges, + bool_t do_expensive_check, cugraph_graph_t** graph, cugraph_error_t** error) { @@ -298,87 +297,198 @@ extern "C" cugraph_error_code_t cugraph_mg_graph_create( *error = nullptr; auto p_handle = reinterpret_cast(handle); + auto p_vertices = + reinterpret_cast( + vertices); auto p_src = - reinterpret_cast(src); + reinterpret_cast(src); auto p_dst = - reinterpret_cast(dst); + reinterpret_cast(dst); auto p_weights = - reinterpret_cast(weights); + reinterpret_cast( + weights); auto p_edge_ids = - reinterpret_cast(edge_ids); + reinterpret_cast( + edge_ids); auto p_edge_type_ids = - reinterpret_cast(edge_type_ids); + reinterpret_cast( + edge_type_ids); + + size_t local_num_edges{0}; + + // + // Determine the type of vertex, weight, edge_type_id across + // multiple input arrays and acros multiple GPUs. Also compute + // the number of edges so we can determine what type to use for + // edge_t + // + cugraph_data_type_id_t vertex_type{cugraph_data_type_id_t::NTYPES}; + cugraph_data_type_id_t weight_type{cugraph_data_type_id_t::NTYPES}; + + for (size_t i = 0; i < num_arrays; ++i) { + CAPI_EXPECTS(p_src[i]->size_ == p_dst[i]->size_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != dst size.", + *error); + + CAPI_EXPECTS(p_src[i]->type_ == p_dst[i]->type_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src type != dst type.", + *error); + + CAPI_EXPECTS((p_vertices == nullptr) || (p_src[i]->type_ == p_vertices[i]->type_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src type != vertices type.", + *error); + + CAPI_EXPECTS((weights == nullptr) || (p_weights[i]->size_ == p_src[i]->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != weights size.", + *error); + + local_num_edges += p_src[i]->size_; + + if (vertex_type == cugraph_data_type_id_t::NTYPES) vertex_type = p_src[i]->type_; + + if (weights != nullptr) { + if (weight_type == cugraph_data_type_id_t::NTYPES) weight_type = p_weights[i]->type_; + } - CAPI_EXPECTS(p_src->size_ == p_dst->size_, - CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src size != dst size.", - *error); - CAPI_EXPECTS(p_src->type_ == p_dst->type_, + CAPI_EXPECTS(p_src[i]->type_ == vertex_type, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: all vertex types must match", + *error); + + CAPI_EXPECTS((weights == nullptr) || (p_weights[i]->type_ == weight_type), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: all weight types must match", + *error); + } + + size_t num_edges = cugraph::host_scalar_allreduce(p_handle->handle_->get_comms(), + local_num_edges, + raft::comms::op_t::SUM, + p_handle->handle_->get_stream()); + + auto vertex_types = cugraph::host_scalar_allgather( + p_handle->handle_->get_comms(), static_cast(vertex_type), p_handle->handle_->get_stream()); + + auto weight_types = cugraph::host_scalar_allgather( + p_handle->handle_->get_comms(), static_cast(weight_type), p_handle->handle_->get_stream()); + + if (vertex_type == cugraph_data_type_id_t::NTYPES) { + // Only true if this GPU had no vertex arrays + vertex_type = static_cast( + *std::min_element(vertex_types.begin(), vertex_types.end())); + } + + if (weight_type == cugraph_data_type_id_t::NTYPES) { + // Only true if this GPU had no weight arrays + weight_type = static_cast( + *std::min_element(weight_types.begin(), weight_types.end())); + } + + CAPI_EXPECTS(std::all_of(vertex_types.begin(), + vertex_types.end(), + [vertex_type](auto t) { return vertex_type == static_cast(t); }), CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src type != dst type.", + "different vertex type used on different GPUs", *error); - CAPI_EXPECTS((weights == nullptr) || (p_weights->size_ == p_src->size_), + CAPI_EXPECTS(std::all_of(weight_types.begin(), + weight_types.end(), + [weight_type](auto t) { return weight_type == static_cast(t); }), CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src size != weights size.", + "different weight type used on different GPUs", *error); cugraph_data_type_id_t edge_type; - cugraph_data_type_id_t weight_type; if (num_edges < int32_threshold) { - edge_type = p_src->type_; + edge_type = static_cast(vertex_types[0]); } else { edge_type = cugraph_data_type_id_t::INT64; } - if (weights != nullptr) { - weight_type = p_weights->type_; - } else { + if (weight_type == cugraph_data_type_id_t::NTYPES) { weight_type = cugraph_data_type_id_t::FLOAT32; } - CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids->type_ == edge_type), - CUGRAPH_INVALID_INPUT, - "Invalid input arguments: Edge id type must match edge type", - *error); + cugraph_data_type_id_t edge_type_id_type{cugraph_data_type_id_t::NTYPES}; - CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids->size_ == p_src->size_), - CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src size != edge id prop size", - *error); + for (size_t i = 0; i < num_arrays; ++i) { + CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids[i]->type_ == edge_type), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: Edge id type must match edge type", + *error); - CAPI_EXPECTS((edge_type_ids == nullptr) || (p_edge_type_ids->size_ == p_src->size_), - CUGRAPH_INVALID_INPUT, - "Invalid input arguments: src size != edge type prop size", - *error); + CAPI_EXPECTS((edge_ids == nullptr) || (p_edge_ids[i]->size_ == p_src[i]->size_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge id prop size", + *error); + + if (edge_type_ids != nullptr) { + CAPI_EXPECTS(p_edge_type_ids[i]->size_ == p_src[i]->size_, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge type prop size", + *error); + + if (edge_type_id_type == cugraph_data_type_id_t::NTYPES) + edge_type_id_type = p_edge_type_ids[i]->type_; + + CAPI_EXPECTS(p_edge_type_ids[i]->type_ == edge_type_id_type, + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src size != edge type prop size", + *error); + } + } + + auto edge_type_id_types = cugraph::host_scalar_allgather(p_handle->handle_->get_comms(), + static_cast(edge_type_id_type), + p_handle->handle_->get_stream()); + + if (edge_type_id_type == cugraph_data_type_id_t::NTYPES) { + // Only true if this GPU had no edge_type_id arrays + edge_type_id_type = static_cast( + *std::min_element(edge_type_id_types.begin(), edge_type_id_types.end())); + } + + CAPI_EXPECTS( + std::all_of(edge_type_id_types.begin(), + edge_type_id_types.end(), + [edge_type_id_type](auto t) { return edge_type_id_type == static_cast(t); }), + CUGRAPH_INVALID_INPUT, + "different edge_type_id type used on different GPUs", + *error); - cugraph_data_type_id_t edge_type_id_type; - if (edge_type_ids == nullptr) { + if (edge_type_id_type == cugraph_data_type_id_t::NTYPES) { edge_type_id_type = cugraph_data_type_id_t::INT32; - } else { - edge_type_id_type = p_edge_type_ids->type_; } + // + // Now we know enough to create the graph + // create_graph_functor functor(*p_handle->handle_, properties, + vertex_type, + edge_type, + weight_type, + edge_type_id_type, + p_vertices, p_src, p_dst, p_weights, p_edge_ids, p_edge_type_ids, + num_arrays, bool_t::TRUE, - check, - edge_type); + drop_self_loops, + drop_multi_edges, + do_expensive_check); try { - cugraph::c_api::vertex_dispatcher(p_src->type_, - edge_type, - weight_type, - edge_type_id_type, - store_transposed, - multi_gpu, - functor); + cugraph::c_api::vertex_dispatcher( + vertex_type, edge_type, weight_type, edge_type_id_type, store_transposed, multi_gpu, functor); if (functor.error_code_ != CUGRAPH_SUCCESS) { *error = reinterpret_cast(functor.error_.release()); @@ -394,25 +504,38 @@ extern "C" cugraph_error_code_t cugraph_mg_graph_create( return CUGRAPH_SUCCESS; } +extern "C" cugraph_error_code_t cugraph_mg_graph_create( + cugraph_resource_handle_t const* handle, + cugraph_graph_properties_t const* properties, + cugraph_type_erased_device_array_view_t const* src, + cugraph_type_erased_device_array_view_t const* dst, + cugraph_type_erased_device_array_view_t const* weights, + cugraph_type_erased_device_array_view_t const* edge_ids, + cugraph_type_erased_device_array_view_t const* edge_type_ids, + bool_t store_transposed, + size_t num_edges, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error) +{ + return cugraph_graph_create_mg(handle, + properties, + NULL, + &src, + &dst, + &weights, + &edge_ids, + &edge_type_ids, + store_transposed, + 1, + FALSE, + FALSE, + do_expensive_check, + graph, + error); +} + extern "C" void cugraph_mg_graph_free(cugraph_graph_t* ptr_graph) { - if (ptr_graph != NULL) { - auto internal_pointer = reinterpret_cast(ptr_graph); - - destroy_graph_functor functor(internal_pointer->graph_, - internal_pointer->number_map_, - internal_pointer->edge_weights_, - internal_pointer->edge_ids_, - internal_pointer->edge_types_); - - cugraph::c_api::vertex_dispatcher(internal_pointer->vertex_type_, - internal_pointer->edge_type_, - internal_pointer->weight_type_, - internal_pointer->edge_type_id_type_, - internal_pointer->store_transposed_, - internal_pointer->multi_gpu_, - functor); - - delete internal_pointer; - } + if (ptr_graph != NULL) { cugraph_graph_free(ptr_graph); } } diff --git a/cpp/src/c_api/graph_sg.cpp b/cpp/src/c_api/graph_sg.cpp index 9536869f123..7793458b53a 100644 --- a/cpp/src/c_api/graph_sg.cpp +++ b/cpp/src/c_api/graph_sg.cpp @@ -33,35 +33,44 @@ namespace { struct create_graph_functor : public cugraph::c_api::abstract_functor { raft::handle_t const& handle_; cugraph_graph_properties_t const* properties_; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* vertices_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* src_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids_; cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids_; bool_t renumber_; + bool_t drop_self_loops_; + bool_t drop_multi_edges_; bool_t do_expensive_check_; cugraph_data_type_id_t edge_type_; cugraph::c_api::cugraph_graph_t* result_{}; create_graph_functor(raft::handle_t const& handle, cugraph_graph_properties_t const* properties, + cugraph::c_api::cugraph_type_erased_device_array_view_t const* vertices, cugraph::c_api::cugraph_type_erased_device_array_view_t const* src, cugraph::c_api::cugraph_type_erased_device_array_view_t const* dst, cugraph::c_api::cugraph_type_erased_device_array_view_t const* weights, cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_ids, cugraph::c_api::cugraph_type_erased_device_array_view_t const* edge_type_ids, bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, bool_t do_expensive_check, cugraph_data_type_id_t edge_type) : abstract_functor(), properties_(properties), handle_(handle), + vertices_(vertices), src_(src), dst_(dst), weights_(weights), edge_ids_(edge_ids), edge_type_ids_(edge_type_ids), renumber_(renumber), + drop_self_loops_(drop_self_loops), + drop_multi_edges_(drop_multi_edges), do_expensive_check_(do_expensive_check), edge_type_(edge_type) { @@ -99,6 +108,18 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { edge_type_id_t>> new_edge_types{std::nullopt}; + std::optional> vertex_list = + vertices_ ? std::make_optional( + rmm::device_uvector(vertices_->size_, handle_.get_stream())) + : std::nullopt; + + if (vertex_list) { + raft::copy(vertex_list->data(), + vertices_->as_type(), + vertices_->size_, + handle_.get_stream()); + } + rmm::device_uvector edgelist_srcs(src_->size_, handle_.get_stream()); rmm::device_uvector edgelist_dsts(dst_->size_, handle_.get_stream()); @@ -160,6 +181,28 @@ struct create_graph_functor : public cugraph::c_api::abstract_functor { cugraph::graph_view_t, edge_type_id_t>(handle_); + if (drop_self_loops_) { + std::tie( + edgelist_srcs, edgelist_dsts, edgelist_weights, edgelist_edge_ids, edgelist_edge_types) = + cugraph::remove_self_loops(handle_, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); + } + + if (drop_multi_edges_) { + std::tie( + edgelist_srcs, edgelist_dsts, edgelist_weights, edgelist_edge_ids, edgelist_edge_types) = + cugraph::remove_multi_edges(handle_, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); + } + std::tie(*graph, new_edge_weights, new_edge_ids, new_edge_types, new_number_map) = cugraph::create_graph_from_edgelist( handle_, - std::nullopt, + std::move(vertex_list), std::move(edgelist_srcs), std::move(edgelist_dsts), std::move(edgelist_weights), @@ -279,6 +322,12 @@ struct create_graph_csr_functor : public cugraph::c_api::abstract_functor { edge_type_id_t>> new_edge_types{std::nullopt}; + std::optional> vertex_list = std::make_optional( + rmm::device_uvector(offsets_->size_ - 1, handle_.get_stream())); + + cugraph::detail::sequence_fill( + handle_.get_stream(), vertex_list->data(), vertex_list->size(), vertex_t{0}); + rmm::device_uvector edgelist_srcs(0, handle_.get_stream()); rmm::device_uvector edgelist_dsts(indices_->size_, handle_.get_stream()); @@ -354,7 +403,7 @@ struct create_graph_csr_functor : public cugraph::c_api::abstract_functor { store_transposed, multi_gpu>( handle_, - std::nullopt, + std::move(vertex_list), std::move(edgelist_srcs), std::move(edgelist_dsts), std::move(edgelist_weights), @@ -452,9 +501,10 @@ struct destroy_graph_functor : public cugraph::c_api::abstract_functor { } // namespace -extern "C" cugraph_error_code_t cugraph_sg_graph_create( +extern "C" cugraph_error_code_t cugraph_graph_create_sg( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* vertices, const cugraph_type_erased_device_array_view_t* src, const cugraph_type_erased_device_array_view_t* dst, const cugraph_type_erased_device_array_view_t* weights, @@ -462,6 +512,8 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( const cugraph_type_erased_device_array_view_t* edge_type_ids, bool_t store_transposed, bool_t renumber, + bool_t drop_self_loops, + bool_t drop_multi_edges, bool_t do_expensive_check, cugraph_graph_t** graph, cugraph_error_t** error) @@ -473,6 +525,8 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( *error = nullptr; auto p_handle = reinterpret_cast(handle); + auto p_vertices = + reinterpret_cast(vertices); auto p_src = reinterpret_cast(src); auto p_dst = @@ -488,6 +542,12 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( CUGRAPH_INVALID_INPUT, "Invalid input arguments: src size != dst size.", *error); + + CAPI_EXPECTS((p_vertices == nullptr) || (p_src->type_ == p_vertices->type_), + CUGRAPH_INVALID_INPUT, + "Invalid input arguments: src type != vertices type.", + *error); + CAPI_EXPECTS(p_src->type_ == p_dst->type_, CUGRAPH_INVALID_INPUT, "Invalid input arguments: src type != dst type.", @@ -533,12 +593,15 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( ::create_graph_functor functor(*p_handle->handle_, properties, + p_vertices, p_src, p_dst, p_weights, p_edge_ids, p_edge_type_ids, renumber, + drop_self_loops, + drop_multi_edges, do_expensive_check, edge_type); @@ -565,7 +628,38 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( return CUGRAPH_SUCCESS; } -cugraph_error_code_t cugraph_sg_graph_create_from_csr( +extern "C" cugraph_error_code_t cugraph_sg_graph_create( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error) +{ + return cugraph_graph_create_sg(handle, + properties, + NULL, + src, + dst, + weights, + edge_ids, + edge_type_ids, + store_transposed, + renumber, + FALSE, + FALSE, + do_expensive_check, + graph, + error); +} + +cugraph_error_code_t cugraph_graph_create_sg_from_csr( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, const cugraph_type_erased_device_array_view_t* offsets, @@ -662,23 +756,55 @@ cugraph_error_code_t cugraph_sg_graph_create_from_csr( return CUGRAPH_SUCCESS; } -extern "C" void cugraph_sg_graph_free(cugraph_graph_t* ptr_graph) +cugraph_error_code_t cugraph_sg_graph_create_from_csr( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_view_t* offsets, + const cugraph_type_erased_device_array_view_t* indices, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_device_array_view_t* edge_ids, + const cugraph_type_erased_device_array_view_t* edge_type_ids, + bool_t store_transposed, + bool_t renumber, + bool_t do_expensive_check, + cugraph_graph_t** graph, + cugraph_error_t** error) { - auto internal_pointer = reinterpret_cast(ptr_graph); - - destroy_graph_functor functor(internal_pointer->graph_, - internal_pointer->number_map_, - internal_pointer->edge_weights_, - internal_pointer->edge_ids_, - internal_pointer->edge_types_); - - cugraph::c_api::vertex_dispatcher(internal_pointer->vertex_type_, - internal_pointer->edge_type_, - internal_pointer->weight_type_, - internal_pointer->edge_type_id_type_, - internal_pointer->store_transposed_, - internal_pointer->multi_gpu_, - functor); - - delete internal_pointer; + return cugraph_graph_create_sg_from_csr(handle, + properties, + offsets, + indices, + weights, + edge_ids, + edge_type_ids, + store_transposed, + renumber, + do_expensive_check, + graph, + error); } + +extern "C" void cugraph_graph_free(cugraph_graph_t* ptr_graph) +{ + if (ptr_graph != NULL) { + auto internal_pointer = reinterpret_cast(ptr_graph); + + destroy_graph_functor functor(internal_pointer->graph_, + internal_pointer->number_map_, + internal_pointer->edge_weights_, + internal_pointer->edge_ids_, + internal_pointer->edge_types_); + + cugraph::c_api::vertex_dispatcher(internal_pointer->vertex_type_, + internal_pointer->edge_type_, + internal_pointer->weight_type_, + internal_pointer->edge_type_id_type_, + internal_pointer->store_transposed_, + internal_pointer->multi_gpu_, + functor); + + delete internal_pointer; + } +} + +extern "C" void cugraph_sg_graph_free(cugraph_graph_t* ptr_graph) { cugraph_graph_free(ptr_graph); } diff --git a/cpp/src/c_api/resource_handle.cpp b/cpp/src/c_api/resource_handle.cpp index 767a6f0add6..75b9537ef49 100644 --- a/cpp/src/c_api/resource_handle.cpp +++ b/cpp/src/c_api/resource_handle.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,3 +41,10 @@ extern "C" int cugraph_resource_handle_get_rank(const cugraph_resource_handle_t* auto& comm = internal->handle_->get_comms(); return static_cast(comm.get_rank()); } + +extern "C" int cugraph_resource_handle_get_comm_size(const cugraph_resource_handle_t* handle) +{ + auto internal = reinterpret_cast(handle); + auto& comm = internal->handle_->get_comms(); + return static_cast(comm.get_size()); +} diff --git a/cpp/src/structure/detail/structure_utils.cuh b/cpp/src/structure/detail/structure_utils.cuh index 01fbccaa53e..c49b62e4543 100644 --- a/cpp/src/structure/detail/structure_utils.cuh +++ b/cpp/src/structure/detail/structure_utils.cuh @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -496,6 +498,63 @@ void sort_adjacency_list(raft::handle_t const& handle, } } -} // namespace detail +template +std::tuple> mark_entries(raft::handle_t const& handle, + size_t num_entries, + comparison_t comparison) +{ + rmm::device_uvector marked_entries(cugraph::packed_bool_size(num_entries), + handle.get_stream()); + + thrust::tabulate(handle.get_thrust_policy(), + marked_entries.begin(), + marked_entries.end(), + [comparison, num_entries] __device__(size_t idx) { + auto word = cugraph::packed_bool_empty_mask(); + size_t start_index = idx * cugraph::packed_bools_per_word(); + size_t bits_in_this_word = + (start_index + cugraph::packed_bools_per_word() < num_entries) + ? cugraph::packed_bools_per_word() + : (num_entries - start_index); + + for (size_t bit = 0; bit < bits_in_this_word; ++bit) { + if (comparison(start_index + bit)) word |= cugraph::packed_bool_mask(bit); + } + + return word; + }); + + size_t bit_count = thrust::transform_reduce( + handle.get_thrust_policy(), + marked_entries.begin(), + marked_entries.end(), + [] __device__(auto word) { return __popc(word); }, + size_t{0}, + thrust::plus()); + + return std::make_tuple(bit_count, std::move(marked_entries)); +} +template +rmm::device_uvector remove_flagged_elements(raft::handle_t const& handle, + rmm::device_uvector&& vector, + raft::device_span remove_flags, + size_t remove_count) +{ + rmm::device_uvector result(vector.size() - remove_count, handle.get_stream()); + + thrust::copy_if( + handle.get_thrust_policy(), + thrust::make_counting_iterator(size_t{0}), + thrust::make_counting_iterator(vector.size()), + thrust::make_transform_output_iterator(result.begin(), + indirection_t{vector.data()}), + [remove_flags] __device__(size_t i) { + return !(remove_flags[cugraph::packed_bool_offset(i)] & cugraph::packed_bool_mask(i)); + }); + + return result; +} + +} // namespace detail } // namespace cugraph diff --git a/cpp/src/structure/remove_multi_edges.cu b/cpp/src/structure/remove_multi_edges.cu new file mode 100644 index 00000000000..ba07d068c0e --- /dev/null +++ b/cpp/src/structure/remove_multi_edges.cu @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +namespace cugraph { + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +} // namespace cugraph diff --git a/cpp/src/structure/remove_multi_edges_impl.cuh b/cpp/src/structure/remove_multi_edges_impl.cuh new file mode 100644 index 00000000000..ab6b1fba8eb --- /dev/null +++ b/cpp/src/structure/remove_multi_edges_impl.cuh @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +// FIXME: mem_frugal_partition should probably not be in shuffle_comm.hpp +// It's used here without any notion of shuffling +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace cugraph { + +namespace detail { + +template +struct hash_src_dst_pair { + int32_t num_groups; + + int32_t __device__ operator()(thrust::tuple t) const + { + vertex_t pair[2]; + pair[0] = thrust::get<0>(t); + pair[1] = thrust::get<1>(t); + cuco::detail::MurmurHash3_32 hash_func{}; + return hash_func.compute_hash(reinterpret_cast(pair), 2 * sizeof(vertex_t)) % + num_groups; + } +}; + +template +std::tuple, rmm::device_uvector> group_multi_edges( + raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + size_t mem_frugal_threshold) +{ + auto pair_first = thrust::make_zip_iterator(edgelist_srcs.begin(), edgelist_dsts.begin()); + + if (edgelist_srcs.size() > mem_frugal_threshold) { + // FIXME: Tuning parameter to address high frequency multi-edges + // Defaulting to 2 which makes the code easier. If + // num_groups > 2 we can evaluate whether to find a good + // midpoint to do 2 sorts, or if we should do more than 2 sorts. + const size_t num_groups{2}; + + auto group_counts = groupby_and_count(pair_first, + pair_first + edgelist_srcs.size(), + hash_src_dst_pair{}, + num_groups, + mem_frugal_threshold, + handle.get_stream()); + + std::vector h_group_counts(group_counts.size()); + raft::update_host( + h_group_counts.data(), group_counts.data(), group_counts.size(), handle.get_stream()); + + thrust::sort(handle.get_thrust_policy(), pair_first, pair_first + h_group_counts[0]); + thrust::sort(handle.get_thrust_policy(), + pair_first + h_group_counts[0], + pair_first + edgelist_srcs.size()); + } else { + thrust::sort(handle.get_thrust_policy(), pair_first, pair_first + edgelist_srcs.size()); + } + + return std::make_tuple(std::move(edgelist_srcs), std::move(edgelist_dsts)); +} + +template +std::tuple, + rmm::device_uvector, + decltype(allocate_dataframe_buffer(size_t{0}, rmm::cuda_stream_view{}))> +group_multi_edges( + raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + decltype(allocate_dataframe_buffer(0, rmm::cuda_stream_view{}))&& edgelist_values, + size_t mem_frugal_threshold) +{ + auto pair_first = thrust::make_zip_iterator(edgelist_srcs.begin(), edgelist_dsts.begin()); + auto value_first = get_dataframe_buffer_begin(edgelist_values); + + if (edgelist_srcs.size() > mem_frugal_threshold) { + // FIXME: Tuning parameter to address high frequency multi-edges + // Defaulting to 2 which makes the code easier. If + // num_groups > 2 we can evaluate whether to find a good + // midpoint to do 2 sorts, or if we should do more than 2 sorts. + const size_t num_groups{2}; + + auto group_counts = groupby_and_count(pair_first, + pair_first + edgelist_srcs.size(), + value_first, + hash_src_dst_pair{}, + num_groups, + mem_frugal_threshold, + handle.get_stream()); + + std::vector h_group_counts(group_counts.size()); + raft::update_host( + h_group_counts.data(), group_counts.data(), group_counts.size(), handle.get_stream()); + + thrust::sort_by_key(handle.get_thrust_policy(), + pair_first, + pair_first + h_group_counts[0], + get_dataframe_buffer_begin(edgelist_values)); + thrust::sort_by_key(handle.get_thrust_policy(), + pair_first + h_group_counts[0], + pair_first + edgelist_srcs.size(), + get_dataframe_buffer_begin(edgelist_values) + h_group_counts[0]); + } else { + thrust::sort_by_key(handle.get_thrust_policy(), + pair_first, + pair_first + edgelist_srcs.size(), + get_dataframe_buffer_begin(edgelist_values)); + } + + return std::make_tuple( + std::move(edgelist_srcs), std::move(edgelist_dsts), std::move(edgelist_values)); +} + +} // namespace detail + +template +std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_multi_edges(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types) +{ + auto total_global_mem = handle.get_device_properties().totalGlobalMem; + size_t element_size = sizeof(vertex_t) * 2; + if (edgelist_weights) { element_size += sizeof(weight_t); } + if (edgelist_edge_ids) { element_size += sizeof(edge_t); } + if (edgelist_edge_types) { element_size += sizeof(edge_type_t); } + + auto constexpr mem_frugal_ratio = + 0.25; // if the expected temporary buffer size exceeds the mem_frugal_ratio of the + // total_global_mem, switch to the memory frugal approach + auto mem_frugal_threshold = + static_cast(static_cast(total_global_mem / element_size) * mem_frugal_ratio); + + if (edgelist_weights) { + if (edgelist_edge_ids) { + if (edgelist_edge_types) { + std::forward_as_tuple(edgelist_srcs, + edgelist_dsts, + std::tie(edgelist_weights, edgelist_edge_ids, edgelist_edge_types)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), + std::move(*edgelist_edge_ids), + std::move(*edgelist_edge_types)), + mem_frugal_threshold); + } else { + std::forward_as_tuple( + edgelist_srcs, edgelist_dsts, std::tie(edgelist_weights, edgelist_edge_ids)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), std::move(*edgelist_edge_ids)), + mem_frugal_threshold); + } + } else { + if (edgelist_edge_types) { + std::forward_as_tuple( + edgelist_srcs, edgelist_dsts, std::tie(edgelist_weights, edgelist_edge_types)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights), std::move(*edgelist_edge_types)), + mem_frugal_threshold); + } else { + std::forward_as_tuple(edgelist_srcs, edgelist_dsts, std::tie(edgelist_weights)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_weights)), + mem_frugal_threshold); + } + } + } else { + if (edgelist_edge_ids) { + if (edgelist_edge_types) { + std::forward_as_tuple( + edgelist_srcs, edgelist_dsts, std::tie(edgelist_edge_ids, edgelist_edge_types)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_edge_ids), std::move(*edgelist_edge_types)), + mem_frugal_threshold); + } else { + std::forward_as_tuple(edgelist_srcs, edgelist_dsts, std::tie(edgelist_edge_ids)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_edge_ids)), + mem_frugal_threshold); + } + } else { + if (edgelist_edge_types) { + std::forward_as_tuple(edgelist_srcs, edgelist_dsts, std::tie(edgelist_edge_types)) = + detail::group_multi_edges>( + handle, + std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::make_tuple(std::move(*edgelist_edge_types)), + mem_frugal_threshold); + } else { + std::tie(edgelist_srcs, edgelist_dsts) = detail::group_multi_edges( + handle, std::move(edgelist_srcs), std::move(edgelist_dsts), mem_frugal_threshold); + } + } + } + + auto [multi_edge_count, multi_edges_to_delete] = + detail::mark_entries(handle, + edgelist_srcs.size(), + [d_edgelist_srcs = edgelist_srcs.data(), + d_edgelist_dsts = edgelist_dsts.data()] __device__(auto idx) { + return (idx > 0) && (d_edgelist_srcs[idx - 1] == d_edgelist_srcs[idx]) && + (d_edgelist_dsts[idx - 1] == d_edgelist_dsts[idx]); + }); + + if (multi_edge_count > 0) { + edgelist_srcs = detail::remove_flagged_elements( + handle, + std::move(edgelist_srcs), + raft::device_span{multi_edges_to_delete.data(), multi_edges_to_delete.size()}, + multi_edge_count); + edgelist_dsts = detail::remove_flagged_elements( + handle, + std::move(edgelist_dsts), + raft::device_span{multi_edges_to_delete.data(), multi_edges_to_delete.size()}, + multi_edge_count); + + if (edgelist_weights) + edgelist_weights = detail::remove_flagged_elements( + handle, + std::move(*edgelist_weights), + raft::device_span{multi_edges_to_delete.data(), + multi_edges_to_delete.size()}, + multi_edge_count); + + if (edgelist_edge_ids) + edgelist_edge_ids = detail::remove_flagged_elements( + handle, + std::move(*edgelist_edge_ids), + raft::device_span{multi_edges_to_delete.data(), + multi_edges_to_delete.size()}, + multi_edge_count); + + if (edgelist_edge_types) + edgelist_edge_types = detail::remove_flagged_elements( + handle, + std::move(*edgelist_edge_types), + raft::device_span{multi_edges_to_delete.data(), + multi_edges_to_delete.size()}, + multi_edge_count); + } + + return std::make_tuple(std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); +} + +} // namespace cugraph diff --git a/cpp/src/structure/remove_self_loops.cu b/cpp/src/structure/remove_self_loops.cu new file mode 100644 index 00000000000..8a66c1e05e3 --- /dev/null +++ b/cpp/src/structure/remove_self_loops.cu @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +namespace cugraph { + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +template std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types); + +} // namespace cugraph diff --git a/cpp/src/structure/remove_self_loops_impl.cuh b/cpp/src/structure/remove_self_loops_impl.cuh new file mode 100644 index 00000000000..161ffeae28e --- /dev/null +++ b/cpp/src/structure/remove_self_loops_impl.cuh @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace cugraph { + +template +std::tuple, + rmm::device_uvector, + std::optional>, + std::optional>, + std::optional>> +remove_self_loops(raft::handle_t const& handle, + rmm::device_uvector&& edgelist_srcs, + rmm::device_uvector&& edgelist_dsts, + std::optional>&& edgelist_weights, + std::optional>&& edgelist_edge_ids, + std::optional>&& edgelist_edge_types) +{ + auto [self_loop_count, self_loops_to_delete] = + detail::mark_entries(handle, + edgelist_srcs.size(), + [d_srcs = edgelist_srcs.data(), d_dsts = edgelist_dsts.data()] __device__( + size_t i) { return d_srcs[i] == d_dsts[i]; }); + + if (self_loop_count > 0) { + edgelist_srcs = detail::remove_flagged_elements( + handle, + std::move(edgelist_srcs), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + edgelist_dsts = detail::remove_flagged_elements( + handle, + std::move(edgelist_dsts), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + + if (edgelist_weights) + edgelist_weights = detail::remove_flagged_elements( + handle, + std::move(*edgelist_weights), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + + if (edgelist_edge_ids) + edgelist_edge_ids = detail::remove_flagged_elements( + handle, + std::move(*edgelist_edge_ids), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + + if (edgelist_edge_types) + edgelist_edge_types = detail::remove_flagged_elements( + handle, + std::move(*edgelist_edge_types), + raft::device_span{self_loops_to_delete.data(), self_loops_to_delete.size()}, + self_loop_count); + } + + return std::make_tuple(std::move(edgelist_srcs), + std::move(edgelist_dsts), + std::move(edgelist_weights), + std::move(edgelist_edge_ids), + std::move(edgelist_edge_types)); +} + +} // namespace cugraph diff --git a/cpp/tests/c_api/create_graph_test.c b/cpp/tests/c_api/create_graph_test.c index 736db761ebd..11da2eb8589 100644 --- a/cpp/tests/c_api/create_graph_test.c +++ b/cpp/tests/c_api/create_graph_test.c @@ -91,8 +91,9 @@ int test_create_sg_graph_simple() handle, wgt_view, (byte_t*)h_wgt, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); - ret_code = cugraph_sg_graph_create(handle, + ret_code = cugraph_graph_create_sg(handle, &properties, + NULL, src_view, dst_view, wgt_view, @@ -101,11 +102,13 @@ int test_create_sg_graph_simple() FALSE, FALSE, FALSE, + FALSE, + FALSE, &graph, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); - cugraph_sg_graph_free(graph); + cugraph_graph_free(graph); cugraph_type_erased_device_array_view_free(wgt_view); cugraph_type_erased_device_array_view_free(dst_view); @@ -300,7 +303,7 @@ int test_create_sg_graph_csr() } cugraph_sample_result_free(result); - cugraph_sg_graph_free(graph); + cugraph_graph_free(graph); cugraph_type_erased_device_array_view_free(wgt_view); cugraph_type_erased_device_array_view_free(indices_view); cugraph_type_erased_device_array_view_free(offsets_view); @@ -382,8 +385,9 @@ int test_create_sg_graph_symmetric_error() handle, wgt_view, (byte_t*)h_wgt, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); - ret_code = cugraph_sg_graph_create(handle, + ret_code = cugraph_graph_create_sg(handle, &properties, + NULL, src_view, dst_view, wgt_view, @@ -391,19 +395,500 @@ int test_create_sg_graph_symmetric_error() NULL, FALSE, FALSE, + FALSE, + FALSE, TRUE, &graph, &ret_error); TEST_ASSERT(test_ret_value, ret_code != CUGRAPH_SUCCESS, "graph creation succeeded but should have failed."); - if (ret_code == CUGRAPH_SUCCESS) cugraph_sg_graph_free(graph); + if (ret_code == CUGRAPH_SUCCESS) cugraph_graph_free(graph); + + cugraph_type_erased_device_array_view_free(wgt_view); + cugraph_type_erased_device_array_view_free(dst_view); + cugraph_type_erased_device_array_view_free(src_view); + cugraph_type_erased_device_array_free(wgt); + cugraph_type_erased_device_array_free(dst); + cugraph_type_erased_device_array_free(src); + + cugraph_free_resource_handle(handle); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_create_sg_graph_with_isolated_vertices() +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 8; + size_t num_vertices = 7; + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + vertex_t h_vertices[] = { 0, 1, 2, 3, 4, 5, 6 }; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_resource_handle_t* handle = NULL; + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + handle = cugraph_create_resource_handle(NULL); + TEST_ASSERT(test_ret_value, handle != NULL, "resource handle creation failed."); + + cugraph_type_erased_device_array_t* vertices; + cugraph_type_erased_device_array_t* src; + cugraph_type_erased_device_array_t* dst; + cugraph_type_erased_device_array_t* wgt; + cugraph_type_erased_device_array_view_t* vertices_view; + cugraph_type_erased_device_array_view_t* src_view; + cugraph_type_erased_device_array_view_t* dst_view; + cugraph_type_erased_device_array_view_t* wgt_view; + + ret_code = + cugraph_type_erased_device_array_create(handle, num_vertices, vertex_tid, &vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + vertices_view = cugraph_type_erased_device_array_view(vertices); + src_view = cugraph_type_erased_device_array_view(src); + dst_view = cugraph_type_erased_device_array_view(dst); + wgt_view = cugraph_type_erased_device_array_view(wgt); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, vertices_view, (byte_t*)h_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, src_view, (byte_t*)h_src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, dst_view, (byte_t*)h_dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view, (byte_t*)h_wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + + ret_code = cugraph_graph_create_sg(handle, + &properties, + vertices_view, + src_view, + dst_view, + wgt_view, + NULL, + NULL, + FALSE, + FALSE, + FALSE, + FALSE, + FALSE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_centrality_result_t* result = NULL; + + // To verify we will call pagerank + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + vertex_t h_result_vertices[num_vertices]; + weight_t h_pageranks[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); + + cugraph_type_erased_device_array_view_free(wgt_view); + cugraph_type_erased_device_array_view_free(dst_view); + cugraph_type_erased_device_array_view_free(src_view); + cugraph_type_erased_device_array_view_free(vertices_view); + cugraph_type_erased_device_array_free(wgt); + cugraph_type_erased_device_array_free(dst); + cugraph_type_erased_device_array_free(src); + cugraph_type_erased_device_array_free(vertices); + + cugraph_free_resource_handle(handle); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_create_sg_graph_csr_with_isolated() +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 8; + size_t num_vertices = 7; + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + /* + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + */ + edge_t h_offsets[] = {0, 1, 3, 6, 7, 8, 8, 8}; + vertex_t h_indices[] = {1, 3, 4, 0, 1, 3, 5, 5}; + vertex_t h_start[] = {0, 1, 2, 3, 4, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_resource_handle_t* handle = NULL; + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + handle = cugraph_create_resource_handle(NULL); + TEST_ASSERT(test_ret_value, handle != NULL, "resource handle creation failed."); + + cugraph_type_erased_device_array_t* offsets; + cugraph_type_erased_device_array_t* indices; + cugraph_type_erased_device_array_t* wgt; + cugraph_type_erased_device_array_view_t* offsets_view; + cugraph_type_erased_device_array_view_t* indices_view; + cugraph_type_erased_device_array_view_t* wgt_view; + + ret_code = cugraph_type_erased_device_array_create( + handle, num_vertices + 1, vertex_tid, &offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "offsets create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &indices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "indices create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + offsets_view = cugraph_type_erased_device_array_view(offsets); + indices_view = cugraph_type_erased_device_array_view(indices); + wgt_view = cugraph_type_erased_device_array_view(wgt); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, offsets_view, (byte_t*)h_offsets, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "offsets copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, indices_view, (byte_t*)h_indices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "indices copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view, (byte_t*)h_wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + + ret_code = cugraph_sg_graph_create_from_csr(handle, + &properties, + offsets_view, + indices_view, + wgt_view, + NULL, + NULL, + FALSE, + FALSE, + FALSE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_centrality_result_t* result = NULL; + + // To verify we will call pagerank + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + vertex_t h_result_vertices[num_vertices]; + weight_t h_pageranks[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); + cugraph_type_erased_device_array_view_free(wgt_view); + cugraph_type_erased_device_array_view_free(indices_view); + cugraph_type_erased_device_array_view_free(offsets_view); + cugraph_type_erased_device_array_free(wgt); + cugraph_type_erased_device_array_free(indices); + cugraph_type_erased_device_array_free(offsets); + + cugraph_free_resource_handle(handle); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_create_sg_graph_with_isolated_vertices_multi_input() +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 66; + size_t num_vertices = 7; + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + vertex_t h_vertices[] = { 0, 1, 2, 3, 4, 5, 6 }; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, + 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5, + 1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f, + 0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.7f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_resource_handle_t* handle = NULL; + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + handle = cugraph_create_resource_handle(NULL); + TEST_ASSERT(test_ret_value, handle != NULL, "resource handle creation failed."); + + cugraph_type_erased_device_array_t* vertices; + cugraph_type_erased_device_array_t* src; + cugraph_type_erased_device_array_t* dst; + cugraph_type_erased_device_array_t* wgt; + cugraph_type_erased_device_array_view_t* vertices_view; + cugraph_type_erased_device_array_view_t* src_view; + cugraph_type_erased_device_array_view_t* dst_view; + cugraph_type_erased_device_array_view_t* wgt_view; + + ret_code = + cugraph_type_erased_device_array_create(handle, num_vertices, vertex_tid, &vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, vertex_tid, &dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, num_edges, weight_tid, &wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + vertices_view = cugraph_type_erased_device_array_view(vertices); + src_view = cugraph_type_erased_device_array_view(src); + dst_view = cugraph_type_erased_device_array_view(dst); + wgt_view = cugraph_type_erased_device_array_view(wgt); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, vertices_view, (byte_t*)h_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, src_view, (byte_t*)h_src, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, dst_view, (byte_t*)h_dst, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view, (byte_t*)h_wgt, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + + ret_code = cugraph_graph_create_sg(handle, + &properties, + vertices_view, + src_view, + dst_view, + wgt_view, + NULL, + NULL, + FALSE, + FALSE, + TRUE, + TRUE, + FALSE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + + cugraph_centrality_result_t* result = NULL; + + // To verify we will call pagerank + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + vertex_t h_result_vertices[num_vertices]; + weight_t h_pageranks[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); cugraph_type_erased_device_array_view_free(wgt_view); cugraph_type_erased_device_array_view_free(dst_view); cugraph_type_erased_device_array_view_free(src_view); + cugraph_type_erased_device_array_view_free(vertices_view); cugraph_type_erased_device_array_free(wgt); cugraph_type_erased_device_array_free(dst); cugraph_type_erased_device_array_free(src); + cugraph_type_erased_device_array_free(vertices); cugraph_free_resource_handle(handle); cugraph_error_free(ret_error); @@ -419,5 +904,8 @@ int main(int argc, char** argv) result |= RUN_TEST(test_create_sg_graph_simple); result |= RUN_TEST(test_create_sg_graph_csr); result |= RUN_TEST(test_create_sg_graph_symmetric_error); + result |= RUN_TEST(test_create_sg_graph_with_isolated_vertices); + result |= RUN_TEST(test_create_sg_graph_csr_with_isolated); + result |= RUN_TEST(test_create_sg_graph_with_isolated_vertices_multi_input); return result; } diff --git a/cpp/tests/c_api/mg_create_graph_test.c b/cpp/tests/c_api/mg_create_graph_test.c index 4c8f2f22982..fec319d1881 100644 --- a/cpp/tests/c_api/mg_create_graph_test.c +++ b/cpp/tests/c_api/mg_create_graph_test.c @@ -17,6 +17,8 @@ #include "c_test_utils.h" /* RUN_TEST */ #include "mg_test_utils.h" /* RUN_TEST */ +#include + #include #include #include @@ -41,7 +43,7 @@ int test_create_mg_graph_simple(const cugraph_resource_handle_t* handle) vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; - cugraph_graph_t* p_graph = NULL; + cugraph_graph_t* graph = NULL; cugraph_graph_properties_t properties; properties.is_symmetric = FALSE; @@ -94,21 +96,25 @@ int test_create_mg_graph_simple(const cugraph_resource_handle_t* handle) handle, wgt_view, (byte_t*)h_wgt, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); - ret_code = cugraph_mg_graph_create(handle, + ret_code = cugraph_graph_create_mg(handle, &properties, - src_view, - dst_view, - wgt_view, + NULL, + (cugraph_type_erased_device_array_view_t const* const*) &src_view, + (cugraph_type_erased_device_array_view_t const* const*) &dst_view, + (cugraph_type_erased_device_array_view_t const* const*) &wgt_view, NULL, NULL, FALSE, - num_edges, + 1, + FALSE, + FALSE, TRUE, - &p_graph, + &graph, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); - cugraph_mg_graph_free(p_graph); + cugraph_graph_free(graph); cugraph_type_erased_device_array_view_free(wgt_view); cugraph_type_erased_device_array_view_free(dst_view); @@ -122,6 +128,382 @@ int test_create_mg_graph_simple(const cugraph_resource_handle_t* handle) return test_ret_value; } +int test_create_mg_graph_multiple_edge_lists(const cugraph_resource_handle_t* handle) +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 8; + size_t num_vertices = 7; + + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + vertex_t h_vertices[] = { 0, 1, 2, 3, 4, 5, 6 }; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + const size_t num_local_arrays = 2; + + cugraph_type_erased_device_array_t* vertices[num_local_arrays]; + cugraph_type_erased_device_array_t* src[num_local_arrays]; + cugraph_type_erased_device_array_t* dst[num_local_arrays]; + cugraph_type_erased_device_array_t* wgt[num_local_arrays]; + cugraph_type_erased_device_array_view_t* vertices_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* src_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* dst_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* wgt_view[num_local_arrays]; + + int my_rank = cugraph_resource_handle_get_rank(handle); + int comm_size = cugraph_resource_handle_get_comm_size(handle); + + size_t local_num_vertices = (num_vertices + comm_size - 1) / comm_size; + size_t local_start_vertex = my_rank * local_num_vertices; + size_t local_num_edges = (num_edges + comm_size - 1) / comm_size; + size_t local_start_edge = my_rank * local_num_edges; + + local_num_edges = (local_num_edges < (num_edges - local_start_edge)) ? local_num_edges : (num_edges - local_start_edge); + local_num_vertices = (local_num_vertices < (num_vertices - local_start_vertex)) ? local_num_vertices : (num_vertices - local_start_vertex); + + for (size_t i = 0 ; i < num_local_arrays ; ++i) { + size_t vertex_count = (local_num_vertices + num_local_arrays - 1) / num_local_arrays; + size_t vertex_start = i * vertex_count; + vertex_count = (vertex_count < (local_num_vertices - vertex_start)) ? vertex_count : (local_num_vertices - vertex_start); + + ret_code = + cugraph_type_erased_device_array_create(handle, vertex_count, vertex_tid, vertices + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + size_t edge_count = (local_num_edges + num_local_arrays - 1) / num_local_arrays; + size_t edge_start = i * edge_count; + edge_count = (edge_count < (local_num_edges - edge_start)) ? edge_count : (local_num_edges - edge_start); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, vertex_tid, src + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, vertex_tid, dst + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, weight_tid, wgt + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + vertices_view[i] = cugraph_type_erased_device_array_view(vertices[i]); + src_view[i] = cugraph_type_erased_device_array_view(src[i]); + dst_view[i] = cugraph_type_erased_device_array_view(dst[i]); + wgt_view[i] = cugraph_type_erased_device_array_view(wgt[i]); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, vertices_view[i], (byte_t*)(h_vertices + local_start_vertex + vertex_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, src_view[i], (byte_t*)(h_src + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, dst_view[i], (byte_t*)(h_dst + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view[i], (byte_t*)(h_wgt + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + } + + ret_code = cugraph_graph_create_mg(handle, + &properties, + (cugraph_type_erased_device_array_view_t const* const*) vertices_view, + (cugraph_type_erased_device_array_view_t const* const*) src_view, + (cugraph_type_erased_device_array_view_t const* const*) dst_view, + (cugraph_type_erased_device_array_view_t const* const*) wgt_view, + NULL, + NULL, + FALSE, + num_local_arrays, + FALSE, + FALSE, + TRUE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + // + // Now call pagerank and check results... + // + cugraph_centrality_result_t* result = NULL; + + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + + // NOTE: Because we get back vertex ids and pageranks, we can simply compare + // the returned values with the expected results for the entire + // graph. Each GPU will have a subset of the total vertices, so + // they will do a subset of the comparisons. + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + size_t num_local_vertices = cugraph_type_erased_device_array_view_size(result_vertices); + + vertex_t h_result_vertices[num_local_vertices]; + weight_t h_pageranks[num_local_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_local_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); + + for (size_t i = 0 ; i < num_local_arrays ; ++i) { + cugraph_type_erased_device_array_view_free(wgt_view[i]); + cugraph_type_erased_device_array_view_free(dst_view[i]); + cugraph_type_erased_device_array_view_free(src_view[i]); + cugraph_type_erased_device_array_view_free(vertices_view[i]); + cugraph_type_erased_device_array_free(wgt[i]); + cugraph_type_erased_device_array_free(dst[i]); + cugraph_type_erased_device_array_free(src[i]); + cugraph_type_erased_device_array_free(vertices[i]); + } + + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_create_mg_graph_multiple_edge_lists_multi_edge(const cugraph_resource_handle_t* handle) +{ + int test_ret_value = 0; + + typedef int32_t vertex_t; + typedef int32_t edge_t; + typedef float weight_t; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + size_t num_edges = 11; + size_t num_vertices = 7; + + double alpha = 0.95; + double epsilon = 0.0001; + size_t max_iterations = 20; + + vertex_t h_vertices[] = { 0, 1, 2, 3, 4, 5, 6 }; + vertex_t h_src[] = {0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5}; + vertex_t h_dst[] = {1, 3, 4, 0, 1, 3, 5, 5, 5, 5, 5}; + weight_t h_wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f, 3.2f, 3.2f, 1.1f}; + weight_t h_result[] = { 0.0859168, 0.158029, 0.0616337, 0.179675, 0.113239, 0.339873, 0.0616337 }; + + cugraph_graph_t* graph = NULL; + cugraph_graph_properties_t properties; + + properties.is_symmetric = FALSE; + properties.is_multigraph = FALSE; + + data_type_id_t vertex_tid = INT32; + data_type_id_t edge_tid = INT32; + data_type_id_t weight_tid = FLOAT32; + + const size_t num_local_arrays = 2; + + cugraph_type_erased_device_array_t* vertices[num_local_arrays]; + cugraph_type_erased_device_array_t* src[num_local_arrays]; + cugraph_type_erased_device_array_t* dst[num_local_arrays]; + cugraph_type_erased_device_array_t* wgt[num_local_arrays]; + cugraph_type_erased_device_array_view_t* vertices_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* src_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* dst_view[num_local_arrays]; + cugraph_type_erased_device_array_view_t* wgt_view[num_local_arrays]; + + int my_rank = cugraph_resource_handle_get_rank(handle); + int comm_size = cugraph_resource_handle_get_comm_size(handle); + + size_t local_num_vertices = (num_vertices + comm_size - 1) / comm_size; + size_t local_start_vertex = my_rank * local_num_vertices; + size_t local_num_edges = (num_edges + comm_size - 1) / comm_size; + size_t local_start_edge = my_rank * local_num_edges; + + local_num_edges = (local_num_edges < (num_edges - local_start_edge)) ? local_num_edges : (num_edges - local_start_edge); + local_num_vertices = (local_num_vertices < (num_vertices - local_start_vertex)) ? local_num_vertices : (num_vertices - local_start_vertex); + + for (size_t i = 0 ; i < num_local_arrays ; ++i) { + size_t vertex_count = (local_num_vertices + num_local_arrays - 1) / num_local_arrays; + size_t vertex_start = i * vertex_count; + vertex_count = (vertex_count < (local_num_vertices - vertex_start)) ? vertex_count : (local_num_vertices - vertex_start); + + ret_code = + cugraph_type_erased_device_array_create(handle, vertex_count, vertex_tid, vertices + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "vertices create failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + size_t edge_count = (local_num_edges + num_local_arrays - 1) / num_local_arrays; + size_t edge_start = i * edge_count; + edge_count = (edge_count < (local_num_edges - edge_start)) ? edge_count : (local_num_edges - edge_start); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, vertex_tid, src + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, vertex_tid, dst + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); + + ret_code = + cugraph_type_erased_device_array_create(handle, edge_count, weight_tid, wgt + i, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + + vertices_view[i] = cugraph_type_erased_device_array_view(vertices[i]); + src_view[i] = cugraph_type_erased_device_array_view(src[i]); + dst_view[i] = cugraph_type_erased_device_array_view(dst[i]); + wgt_view[i] = cugraph_type_erased_device_array_view(wgt[i]); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, vertices_view[i], (byte_t*)(h_vertices + local_start_vertex + vertex_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, src_view[i], (byte_t*)(h_src + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, dst_view[i], (byte_t*)(h_dst + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + handle, wgt_view[i], (byte_t*)(h_wgt + local_start_edge + edge_start), &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); + } + + ret_code = cugraph_graph_create_mg(handle, + &properties, + (cugraph_type_erased_device_array_view_t const* const*) vertices_view, + (cugraph_type_erased_device_array_view_t const* const*) src_view, + (cugraph_type_erased_device_array_view_t const* const*) dst_view, + (cugraph_type_erased_device_array_view_t const* const*) wgt_view, + NULL, + NULL, + FALSE, + num_local_arrays, + TRUE, + TRUE, + TRUE, + &graph, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + TEST_ALWAYS_ASSERT(ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + + // + // Now call pagerank and check results... + // + cugraph_centrality_result_t* result = NULL; + + ret_code = cugraph_pagerank(handle, + graph, + NULL, + NULL, + NULL, + NULL, + alpha, + epsilon, + max_iterations, + FALSE, + &result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); + + // NOTE: Because we get back vertex ids and pageranks, we can simply compare + // the returned values with the expected results for the entire + // graph. Each GPU will have a subset of the total vertices, so + // they will do a subset of the comparisons. + cugraph_type_erased_device_array_view_t* result_vertices; + cugraph_type_erased_device_array_view_t* pageranks; + + result_vertices = cugraph_centrality_result_get_vertices(result); + pageranks = cugraph_centrality_result_get_values(result); + + size_t num_local_vertices = cugraph_type_erased_device_array_view_size(result_vertices); + + vertex_t h_result_vertices[num_local_vertices]; + weight_t h_pageranks[num_local_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_result_vertices, result_vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + handle, (byte_t*)h_pageranks, pageranks, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + for (int i = 0; (i < num_local_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(h_result[h_result_vertices[i]], h_pageranks[i], 0.001), + "pagerank results don't match"); + } + + cugraph_centrality_result_free(result); + cugraph_graph_free(graph); + + for (size_t i = 0 ; i < num_local_arrays ; ++i) { + cugraph_type_erased_device_array_view_free(wgt_view[i]); + cugraph_type_erased_device_array_view_free(dst_view[i]); + cugraph_type_erased_device_array_view_free(src_view[i]); + cugraph_type_erased_device_array_view_free(vertices_view[i]); + cugraph_type_erased_device_array_free(wgt[i]); + cugraph_type_erased_device_array_free(dst[i]); + cugraph_type_erased_device_array_free(src[i]); + cugraph_type_erased_device_array_free(vertices[i]); + } + + cugraph_error_free(ret_error); + + return test_ret_value; +} + /******************************************************************************/ int main(int argc, char** argv) @@ -131,6 +513,8 @@ int main(int argc, char** argv) int result = 0; result |= RUN_MG_TEST(test_create_mg_graph_simple, handle); + result |= RUN_MG_TEST(test_create_mg_graph_multiple_edge_lists, handle); + result |= RUN_MG_TEST(test_create_mg_graph_multiple_edge_lists_multi_edge, handle); cugraph_free_resource_handle(handle); free_mg_raft_handle(raft_handle); From fceb6b7a4b34bc821700ba5729423a0cf76fbbdf Mon Sep 17 00:00:00 2001 From: Brad Rees <34135411+BradReesWork@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:48:41 -0500 Subject: [PATCH 3/6] adding C/C++ API docs (#3938) adding C and C++ API Adding WholeGraph APIs Adding cugraph-ops APIs closes #3406 Authors: - Brad Rees (https://github.com/BradReesWork) - GALI PREM SAGAR (https://github.com/galipremsagar) - Ralph Liu (https://github.com/nv-rliu) - Don Acosta (https://github.com/acostadon) Approvers: - Seunghwa Kang (https://github.com/seunghwak) - Don Acosta (https://github.com/acostadon) - Rick Ratzel (https://github.com/rlratzel) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cugraph/pull/3938 --- .gitignore | 4 +- build.sh | 22 + ci/build_docs.sh | 7 +- ci/release/update-version.sh | 4 + cpp/doxygen/Doxyfile | 765 ++++++++++++------ cpp/include/cugraph_c/centrality_algorithms.h | 26 +- cpp/include/cugraph_c/community_algorithms.h | 1 - cpp/include/cugraph_c/core_algorithms.h | 14 +- cpp/include/cugraph_c/labeling_algorithms.h | 11 +- cpp/include/cugraph_c/sampling_algorithms.h | 3 +- cpp/include/cugraph_c/similarity_algorithms.h | 7 +- docs/cugraph/Makefile | 2 +- .../cugraph-ops/bipartite_operators.rst | 16 - .../api_docs/cugraph-ops/c_cpp/index.rst | 3 + .../api_docs/cugraph-ops/fg_operators.rst | 83 -- .../api_docs/cugraph-ops/graph_types.rst | 33 - .../source/api_docs/cugraph-ops/index.rst | 11 +- .../api_docs/cugraph-ops/mfg_operators.rst | 31 - .../cugraph-ops/{ => python}/dimenet.rst | 4 +- .../cugraph-ops/python/graph_types.rst | 34 + .../api_docs/cugraph-ops/python/index.rst | 13 + .../api_docs/cugraph-ops/python/operators.rst | 93 +++ .../cugraph-ops/{ => python}/pytorch.rst | 22 +- .../api_docs/cugraph-ops/static_operators.rst | 16 - .../api_docs/cugraph-pyg/cugraph_pyg.rst | 4 +- .../source/api_docs/cugraph_c/c_and_cpp.rst | 4 - .../source/api_docs/cugraph_c/centrality.rst | 51 ++ .../source/api_docs/cugraph_c/community.rst | 63 ++ .../source/api_docs/cugraph_c/core.rst | 21 + .../source/api_docs/cugraph_c/index.rst | 16 + .../source/api_docs/cugraph_c/labeling.rst | 20 + .../source/api_docs/cugraph_c/sampling.rst | 37 + .../source/api_docs/cugraph_c/similarity.rst | 25 + .../source/api_docs/cugraph_c/traversal.rst | 30 + docs/cugraph/source/api_docs/index.rst | 33 +- .../service/cugraph_service_client.rst | 2 +- .../service/cugraph_service_server.rst | 2 +- .../source/api_docs/wholegraph/index.rst | 11 + .../wholegraph/libwholegraph/index.rst | 228 ++++++ .../wholegraph/pylibwholegraph/index.rst | 38 + docs/cugraph/source/basics/cugraph_intro.md | 9 +- docs/cugraph/source/conf.py | 5 +- .../source/graph_support/algorithms.md | 12 +- .../graph_support/algorithms/Centrality.md | 12 +- .../graph_support/algorithms/Similarity.md | 6 +- docs/cugraph/source/index.rst | 10 +- .../source/wholegraph/basics/index.rst | 11 + .../wholegraph/basics/wholegraph_intro.md | 135 ++++ .../wholememory_implementation_details.md | 58 ++ .../wholegraph/basics/wholememory_intro.md | 123 +++ .../imgs/device_chunked_wholememory_step1.png | Bin 0 -> 23136 bytes .../imgs/device_chunked_wholememory_step2.png | Bin 0 -> 28201 bytes .../device_continuous_wholememory_step1.png | Bin 0 -> 21107 bytes .../device_continuous_wholememory_step2.png | Bin 0 -> 26434 bytes .../imgs/distributed_wholememory.png | Bin 0 -> 23138 bytes .../wholegraph/imgs/general_wholememory.png | Bin 0 -> 14073 bytes .../imgs/host_mapped_wholememory_step1.png | Bin 0 -> 21733 bytes .../imgs/host_mapped_wholememory_step2.png | Bin 0 -> 32110 bytes .../wholegraph/imgs/wholememory_tensor.png | Bin 0 -> 40367 bytes docs/cugraph/source/wholegraph/index.rst | 14 + .../wholegraph/installation/container.md | 29 + .../installation/getting_wholegraph.md | 48 ++ .../source/wholegraph/installation/index.rst | 9 + .../wholegraph/installation/source_build.md | 187 +++++ 64 files changed, 1951 insertions(+), 527 deletions(-) delete mode 100644 docs/cugraph/source/api_docs/cugraph-ops/bipartite_operators.rst create mode 100644 docs/cugraph/source/api_docs/cugraph-ops/c_cpp/index.rst delete mode 100644 docs/cugraph/source/api_docs/cugraph-ops/fg_operators.rst delete mode 100644 docs/cugraph/source/api_docs/cugraph-ops/graph_types.rst delete mode 100644 docs/cugraph/source/api_docs/cugraph-ops/mfg_operators.rst rename docs/cugraph/source/api_docs/cugraph-ops/{ => python}/dimenet.rst (89%) create mode 100644 docs/cugraph/source/api_docs/cugraph-ops/python/graph_types.rst create mode 100644 docs/cugraph/source/api_docs/cugraph-ops/python/index.rst create mode 100644 docs/cugraph/source/api_docs/cugraph-ops/python/operators.rst rename docs/cugraph/source/api_docs/cugraph-ops/{ => python}/pytorch.rst (59%) delete mode 100644 docs/cugraph/source/api_docs/cugraph-ops/static_operators.rst delete mode 100644 docs/cugraph/source/api_docs/cugraph_c/c_and_cpp.rst create mode 100644 docs/cugraph/source/api_docs/cugraph_c/centrality.rst create mode 100644 docs/cugraph/source/api_docs/cugraph_c/community.rst create mode 100644 docs/cugraph/source/api_docs/cugraph_c/core.rst create mode 100644 docs/cugraph/source/api_docs/cugraph_c/index.rst create mode 100644 docs/cugraph/source/api_docs/cugraph_c/labeling.rst create mode 100644 docs/cugraph/source/api_docs/cugraph_c/sampling.rst create mode 100644 docs/cugraph/source/api_docs/cugraph_c/similarity.rst create mode 100644 docs/cugraph/source/api_docs/cugraph_c/traversal.rst create mode 100644 docs/cugraph/source/api_docs/wholegraph/index.rst create mode 100644 docs/cugraph/source/api_docs/wholegraph/libwholegraph/index.rst create mode 100644 docs/cugraph/source/api_docs/wholegraph/pylibwholegraph/index.rst create mode 100644 docs/cugraph/source/wholegraph/basics/index.rst create mode 100644 docs/cugraph/source/wholegraph/basics/wholegraph_intro.md create mode 100644 docs/cugraph/source/wholegraph/basics/wholememory_implementation_details.md create mode 100644 docs/cugraph/source/wholegraph/basics/wholememory_intro.md create mode 100644 docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step1.png create mode 100644 docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step2.png create mode 100644 docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step1.png create mode 100644 docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step2.png create mode 100644 docs/cugraph/source/wholegraph/imgs/distributed_wholememory.png create mode 100644 docs/cugraph/source/wholegraph/imgs/general_wholememory.png create mode 100644 docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step1.png create mode 100644 docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step2.png create mode 100644 docs/cugraph/source/wholegraph/imgs/wholememory_tensor.png create mode 100644 docs/cugraph/source/wholegraph/index.rst create mode 100644 docs/cugraph/source/wholegraph/installation/container.md create mode 100644 docs/cugraph/source/wholegraph/installation/getting_wholegraph.md create mode 100644 docs/cugraph/source/wholegraph/installation/index.rst create mode 100644 docs/cugraph/source/wholegraph/installation/source_build.md diff --git a/.gitignore b/.gitignore index c6bcf6965d7..358650cfc5a 100644 --- a/.gitignore +++ b/.gitignore @@ -84,8 +84,10 @@ datasets/* # Jupyter Notebooks .ipynb_checkpoints -## Doxygen +## Doxygen and Docs cpp/doxygen/html +docs/cugraph/lib* +docs/cugraph/api/* # created by Dask tests python/dask-worker-space diff --git a/build.sh b/build.sh index 1723e750978..eef19046d85 100755 --- a/build.sh +++ b/build.sh @@ -18,6 +18,8 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) +RAPIDS_VERSION=23.12 + # Valid args to this script (all possible targets and options) - only one per line VALIDARGS=" clean @@ -412,8 +414,28 @@ if hasArg docs || hasArg all; then ${CMAKE_GENERATOR_OPTION} \ ${CMAKE_VERBOSE_OPTION} fi + + for PROJECT in libcugraphops libwholegraph; do + XML_DIR="${REPODIR}/docs/cugraph/${PROJECT}" + rm -rf "${XML_DIR}" + mkdir -p "${XML_DIR}" + export XML_DIR_${PROJECT^^}="$XML_DIR" + + echo "downloading xml for ${PROJECT} into ${XML_DIR}. Environment variable XML_DIR_${PROJECT^^} is set to ${XML_DIR}" + curl -O "https://d1664dvumjb44w.cloudfront.net/${PROJECT}/xml_tar/${RAPIDS_VERSION}/xml.tar.gz" + tar -xzf xml.tar.gz -C "${XML_DIR}" + rm "./xml.tar.gz" + done + cd ${LIBCUGRAPH_BUILD_DIR} cmake --build "${LIBCUGRAPH_BUILD_DIR}" -j${PARALLEL_LEVEL} --target docs_cugraph ${VERBOSE_FLAG} + + echo "making libcugraph doc dir" + rm -rf ${REPODIR}/docs/cugraph/libcugraph + mkdir -p ${REPODIR}/docs/cugraph/libcugraph + + export XML_DIR_LIBCUGRAPH="${REPODIR}/cpp/doxygen/xml" + cd ${REPODIR}/docs/cugraph make html fi diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 3f97f652d41..3f765704bdb 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -29,7 +29,9 @@ rapids-mamba-retry install \ cugraph-pyg \ cugraph-service-server \ cugraph-service-client \ - libcugraph_etl + libcugraph_etl \ + pylibcugraphops \ + pylibwholegraph # This command installs `cugraph-dgl` without its dependencies # since this package can currently only run in `11.6` CTK environments @@ -50,8 +52,7 @@ done rapids-logger "Build CPP docs" pushd cpp/doxygen doxygen Doxyfile -mkdir -p "${RAPIDS_DOCS_DIR}/libcugraph/html" -mv html/* "${RAPIDS_DOCS_DIR}/libcugraph/html" +export XML_DIR_LIBCUGRAPH="$(pwd)/xml" popd rapids-logger "Build Python docs" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 69eb085e7ed..c091bd1ed33 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -54,6 +54,10 @@ sed_runner "s/set(cugraph_version .*)/set(cugraph_version ${NEXT_FULL_TAG})/g" p sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/cugraph/source/conf.py sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/cugraph/source/conf.py + +# build.sh script +sed_runner 's/RAPIDS_VERSION=.*/RAPIDS_VERSION='${NEXT_SHORT_TAG}'/g' build.sh + # Centralized version file update # NOTE: Any script that runs in CI will need to use gha-tool `rapids-generate-version` # and echo it to `VERSION` file to get an alpha spec of the current version diff --git a/cpp/doxygen/Doxyfile b/cpp/doxygen/Doxyfile index 482ff988098..6946bd38bfe 100644 --- a/cpp/doxygen/Doxyfile +++ b/cpp/doxygen/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.8.20 +# Doxyfile 1.9.8 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -12,6 +12,16 @@ # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). +# +# Note: +# +# Use doxygen to compare the used configuration file with the template +# configuration file: +# doxygen -x [configFile] +# Use doxygen to compare the used configuration file with the template +# configuration file without replacing the environment variables or CMake type +# replacement variables: +# doxygen -x_noenv [configFile] #--------------------------------------------------------------------------- # Project related configuration options @@ -32,19 +42,19 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = "libcugraph" +PROJECT_NAME = libcugraph # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER=23.12 +PROJECT_NUMBER = 23.12 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. -PROJECT_BRIEF = GPU accelerated graph analytics +PROJECT_BRIEF = "GPU accelerated graph analytics" # With the PROJECT_LOGO tag one can specify a logo or an icon that is included # in the documentation. The maximum height of the logo should not exceed 55 @@ -60,16 +70,28 @@ PROJECT_LOGO = OUTPUT_DIRECTORY = -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 +# sub-directories (in 2 levels) under the output directory of each output format +# and will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes -# performance problems for the file system. +# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to +# control the number of sub-directories. # The default value is: NO. CREATE_SUBDIRS = NO +# Controls the number of sub-directories that will be created when +# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every +# level increment doubles the number of directories, resulting in 4096 +# directories at level 8 which is the default and also the maximum value. The +# sub-directories are organized in 2 levels, the first level always has a fixed +# number of 16 directories. +# Minimum value: 0, maximum value: 8, default value: 8. +# This tag requires that the tag CREATE_SUBDIRS is set to YES. + +CREATE_SUBDIRS_LEVEL = 8 + # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode @@ -81,26 +103,18 @@ ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, +# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English +# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, +# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with +# English messages), Korean, Korean-en (Korean with English messages), Latvian, +# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, +# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, +# Swedish, Turkish, Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -248,16 +262,16 @@ TAB_SIZE = 4 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = @@ -302,8 +316,8 @@ OPTIMIZE_OUTPUT_SLICE = NO # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make doxygen treat .inc files @@ -313,7 +327,10 @@ OPTIMIZE_OUTPUT_SLICE = NO # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. EXTENSION_MAPPING = cu=C++ \ cuh=C++ @@ -337,6 +354,17 @@ MARKDOWN_SUPPORT = YES TOC_INCLUDE_HEADINGS = 5 +# The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to +# generate identifiers for the Markdown headings. Note: Every identifier is +# unique. +# Possible values are: DOXYGEN use a fixed 'autotoc_md' string followed by a +# sequence number starting at 0 and GITHUB use the lower case version of title +# with any whitespace replaced by '-' and punctuation characters removed. +# The default value is: DOXYGEN. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +MARKDOWN_ID_STYLE = DOXYGEN + # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or @@ -448,19 +476,27 @@ TYPEDEF_HIDES_STRUCT = NO LOOKUP_CACHE_SIZE = 0 -# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use +# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use # during processing. When set to 0 doxygen will based this on the number of # cores available in the system. You can set it explicitly to a value larger # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, -# which efficively disables parallel processing. Please report any issues you +# which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. NUM_PROC_THREADS = 1 +# If the TIMESTAMP tag is set different from NO then each generated page will +# contain the date or date and time when the page was generated. Setting this to +# NO can help when comparing the output of multiple runs. +# Possible values are: YES, NO, DATETIME and DATE. +# The default value is: NO. + +TIMESTAMP = NO + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- @@ -524,6 +560,13 @@ EXTRACT_LOCAL_METHODS = NO EXTRACT_ANON_NSPACES = NO +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation @@ -535,7 +578,8 @@ HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. +# will also hide undocumented C++ concepts if enabled. This option has no effect +# if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO @@ -561,12 +605,20 @@ HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# (including Cygwin) and Mac users are advised to set this option to NO. -# The default value is: system dependent. +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. +# Possible values are: SYSTEM, NO and YES. +# The default value is: SYSTEM. CASE_SENSE_NAMES = YES @@ -584,6 +636,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -741,7 +799,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -787,24 +846,50 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = YES +# If WARN_IF_UNDOC_ENUM_VAL option is set to YES, doxygen will warn about +# undocumented enumeration values. If set to NO, doxygen will accept +# undocumented enumeration values. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: NO. + +WARN_IF_UNDOC_ENUM_VAL = NO + # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then doxygen behaves +# like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined doxygen will not +# write the warning messages in between other messages but write them at the end +# of a run, in case a WARN_LOGFILE is defined the warning messages will be +# besides being in the defined file also be shown at the end of a run, unless +# the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case +# the behavior will remain as with the setting FAIL_ON_WARNINGS. +# Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT. # The default value is: NO. WARN_AS_ERROR = NO @@ -815,13 +900,27 @@ WARN_AS_ERROR = NO # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) +# See also: WARN_LINE_FORMAT # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" +# In the $text part of the WARN_FORMAT command it is possible that a reference +# to a more specific place is given. To make it easier to jump to this place +# (outside of doxygen) the user can define a custom "cut" / "paste" string. +# Example: +# WARN_LINE_FORMAT = "'vi $file +$line'" +# See also: WARN_FORMAT +# The default value is: at line $line of file $file. + +WARN_LINE_FORMAT = "at line $line of file $file" + # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard -# error (stderr). +# error (stderr). In case the file specified cannot be opened for writing the +# warning and error messages are written to standard error. When as file - is +# specified the warning and error messages are written to standard output +# (stdout). WARN_LOGFILE = @@ -842,12 +941,23 @@ INPUT = main_page.md \ # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: https://www.gnu.org/software/libiconv/) for the list of -# possible encodings. +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# See also: INPUT_FILE_ENCODING # The default value is: UTF-8. INPUT_ENCODING = UTF-8 +# This tag can be used to specify the character encoding of the source files +# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify +# character encoding on a per file pattern basis. Doxygen will compare the file +# name with each pattern and apply the encoding instead of the default +# INPUT_ENCODING) if there is a match. The character encodings are a list of the +# form: pattern=encoding (like *.php=ISO-8859-1). See cfg_input_encoding +# "INPUT_ENCODING" for further information on supported encodings. + +INPUT_FILE_ENCODING = + # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. @@ -856,13 +966,15 @@ INPUT_ENCODING = UTF-8 # need to set EXTENSION_MAPPING for the extension otherwise the files are not # read by doxygen. # -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen -# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, -# *.vhdl, *.ucf, *.qsf and *.ice. +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cxxm, +# *.cpp, *.cppm, *.c++, *.c++m, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, +# *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, *.h++, *.ixx, *.l, *.cs, *.d, *.php, +# *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be +# provided as doxygen C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f18, *.f, *.for, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.cpp \ *.hpp \ @@ -907,10 +1019,7 @@ EXCLUDE_PATTERNS = */nvtx/* \ # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* +# ANamespace::AClass, ANamespace::*Test EXCLUDE_SYMBOLS = org::apache @@ -955,6 +1064,11 @@ IMAGE_PATH = # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. # +# Note that doxygen will use the data processed and written to standard output +# for further processing, therefore nothing else, like debug statements or used +# commands (so in case of a Windows batch file always use @echo OFF), should be +# written to standard output. +# # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by doxygen. @@ -996,6 +1110,15 @@ FILTER_SOURCE_PATTERNS = USE_MDFILE_AS_MAINPAGE = main_page.md +# The Fortran standard specifies that for fixed formatted Fortran code all +# characters from position 72 are to be considered as comment. A common +# extension is to allow longer lines before the automatic comment starts. The +# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can +# be processed before the automatic comment starts. +# Minimum value: 7, maximum value: 10000, default value: 72. + +FORTRAN_COMMENT_AFTER = 72 + #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- @@ -1093,17 +1216,11 @@ VERBATIM_HEADERS = YES ALPHABETICAL_INDEX = YES -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. +# The IGNORE_PREFIX tag can be used to specify a prefix (or a list of prefixes) +# that should be ignored while generating the index headers. The IGNORE_PREFIX +# tag works for classes, function and member names. The entity will be placed in +# the alphabetical list under the first letter of the entity name that remains +# after removing the prefix. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = @@ -1115,7 +1232,7 @@ IGNORE_PREFIX = # If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output # The default value is: YES. -GENERATE_HTML = YES +GENERATE_HTML = NO # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -1182,7 +1299,12 @@ HTML_STYLESHEET = # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. +# list). +# Note: Since the styling of scrollbars can currently not be overruled in +# Webkit/Chromium, the styling will be left out of the default doxygen.css if +# one or more extra stylesheets have been specified. So if scrollbar +# customization is desired it has to be added explicitly. For an example see the +# documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = @@ -1197,9 +1319,22 @@ HTML_EXTRA_STYLESHEET = HTML_EXTRA_FILES = +# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output +# should be rendered with a dark or light theme. +# Possible values are: LIGHT always generate light mode output, DARK always +# generate dark mode output, AUTO_LIGHT automatically set the mode according to +# the user preference, use light mode if no preference is set (the default), +# AUTO_DARK automatically set the mode according to the user preference, use +# dark mode if no preference is set and TOGGLE allow to user to switch between +# light and dark mode via a button. +# The default value is: AUTO_LIGHT. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE = AUTO_LIGHT + # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1209,7 +1344,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 270 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1227,15 +1362,6 @@ HTML_COLORSTYLE_SAT = 255 HTML_COLORSTYLE_GAMMA = 80 -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that # are dynamically created via JavaScript. If disabled, the navigation index will @@ -1255,6 +1381,13 @@ HTML_DYNAMIC_MENUS = YES HTML_DYNAMIC_SECTIONS = NO +# If the HTML_CODE_FOLDING tag is set to YES then classes and functions can be +# dynamically folded and expanded in the generated HTML source code. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_CODE_FOLDING = YES + # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to @@ -1270,10 +1403,11 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: https://developer.apple.com/xcode/), introduced with OSX -# 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy # genXcode/_index.html for more information. @@ -1290,6 +1424,13 @@ GENERATE_DOCSET = NO DOCSET_FEEDNAME = "Doxygen generated docs" +# This tag determines the URL of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDURL = + # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. @@ -1315,8 +1456,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1373,6 +1518,16 @@ BINARY_TOC = NO TOC_EXPAND = NO +# The SITEMAP_URL tag is used to specify the full URL of the place where the +# generated documentation will be placed on the server by the user during the +# deployment of the documentation. The generated sitemap is called sitemap.xml +# and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL +# is specified no sitemap is generated. For information about the sitemap +# protocol see https://www.sitemaps.org +# This tag requires that the tag GENERATE_HTML is set to YES. + +SITEMAP_URL = + # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help @@ -1391,7 +1546,8 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1399,8 +1555,8 @@ QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- -# folders). +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1408,16 +1564,16 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = @@ -1429,9 +1585,9 @@ QHP_CUST_FILTER_ATTRS = QHP_SECT_FILTER_ATTRS = -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = @@ -1474,16 +1630,28 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATE_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1508,6 +1676,13 @@ TREEVIEW_WIDTH = 250 EXT_LINKS_IN_WINDOW = NO +# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email +# addresses. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +OBFUSCATE_EMAILS = YES + # If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg # tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see # https://inkscape.org) to generate formulas as SVG images instead of PNGs for @@ -1528,17 +1703,6 @@ HTML_FORMULA_FORMAT = png FORMULA_FONTSIZE = 10 -# Use the FORMULA_TRANSPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - # The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands # to create new LaTeX commands to be used in formulas as building blocks. See # the section "Including formulas" for details. @@ -1556,11 +1720,29 @@ FORMULA_MACROFILE = USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1573,22 +1755,29 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see +# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1635,7 +1824,8 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). +# Xapian (see: +# https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1648,8 +1838,9 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). See the section "External Indexing and -# Searching" for details. +# Xapian (see: +# https://xapian.org/). See the section "External Indexing and Searching" for +# details. # This tag requires that the tag SEARCHENGINE is set to YES. SEARCHENGINE_URL = @@ -1758,29 +1949,31 @@ PAPER_TYPE = a4 EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1823,10 +2016,16 @@ PDF_HYPERLINKS = YES USE_PDFLATEX = YES -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode -# command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# The LATEX_BATCHMODE tag signals the behavior of LaTeX in case of an error. +# Possible values are: NO same as ERROR_STOP, YES same as BATCH, BATCH In batch +# mode nothing is printed on the terminal, errors are scrolled as if is +# hit at every error; missing files that TeX tries to input or request from +# keyboard input (\read on a not open input stream) cause the job to abort, +# NON_STOP In nonstop mode the diagnostic message will appear on the terminal, +# but there is no possibility of user interaction just like in batch mode, +# SCROLL In scroll mode, TeX will stop only for missing files to input or if +# keyboard input is necessary and ERROR_STOP In errorstop mode, TeX will stop at +# each error, asking for user intervention. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1839,16 +2038,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1857,14 +2046,6 @@ LATEX_SOURCE_CODE = NO LATEX_BIB_STYLE = plain -# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated -# page will contain the date and time when the page was generated. Setting this -# to NO can help when comparing the output of multiple runs. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_TIMESTAMP = NO - # The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) # path from which the emoji images will be read. If a relative path is entered, # it will be relative to the LATEX_OUTPUT directory. If left blank the @@ -1929,16 +2110,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -1991,7 +2162,7 @@ MAN_LINKS = NO # captures the structure of the code including all documentation. # The default value is: NO. -GENERATE_XML = NO +GENERATE_XML = YES # The XML_OUTPUT tag is used to specify where the XML pages will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -2035,27 +2206,44 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures +# AutoGen Definitions (see https://autogen.sourceforge.net/) file that captures # the structure of the code including all documentation. Note that this feature # is still experimental and incomplete at the moment. # The default value is: NO. GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# Configuration options related to Sqlite3 output +#--------------------------------------------------------------------------- + +# If the GENERATE_SQLITE3 tag is set to YES doxygen will generate a Sqlite3 +# database with symbols found by doxygen stored in tables. +# The default value is: NO. + +GENERATE_SQLITE3 = NO + +# The SQLITE3_OUTPUT tag is used to specify where the Sqlite3 database will be +# put. If a relative path is entered the value of OUTPUT_DIRECTORY will be put +# in front of it. +# The default directory is: sqlite3. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_OUTPUT = sqlite3 + +# The SQLITE3_OVERWRITE_DB tag is set to YES, the existing doxygen_sqlite3.db +# database file will be recreated with each doxygen run. If set to NO, doxygen +# will warn if an a database file is already found and not modify it. +# The default value is: YES. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_RECREATE_DB = YES + #--------------------------------------------------------------------------- # Configuration options related to the Perl module output #--------------------------------------------------------------------------- @@ -2130,7 +2318,8 @@ SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the -# preprocessor. +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @@ -2197,15 +2386,15 @@ TAGFILES = rmm.tag=https://docs.rapids.ai/api/librmm/22.08 GENERATE_TAGFILE = -# If the ALLEXTERNALS tag is set to YES, all external class will be listed in -# the class index. If set to NO, only the inherited external classes will be -# listed. +# If the ALLEXTERNALS tag is set to YES, all external classes and namespaces +# will be listed in the class and namespace index. If set to NO, only the +# inherited external classes will be listed. # The default value is: NO. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will be +# in the topic index. If set to NO, only the current project's groups will be # listed. # The default value is: YES. @@ -2219,25 +2408,9 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- -# Configuration options related to the dot tool +# Configuration options related to diagram generator tools #--------------------------------------------------------------------------- -# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram -# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to -# NO turns the diagrams off. Note that this option also works with HAVE_DOT -# disabled, but it is recommended to install and use dot, since it yields more -# powerful graphs. -# The default value is: YES. - -CLASS_DIAGRAMS = YES - -# You can include diagrams made with dia in doxygen documentation. Doxygen will -# then run dia to produce the diagram and insert it in the documentation. The -# DIA_PATH tag allows you to specify the directory where the dia binary resides. -# If left empty dia is assumed to be found in the default search path. - -DIA_PATH = - # If set to YES the inheritance and collaboration graphs will hide inheritance # and usage relations if the target is undocumented or is not a class. # The default value is: YES. @@ -2246,7 +2419,7 @@ HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: -# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO # The default value is: NO. @@ -2263,49 +2436,73 @@ HAVE_DOT = YES DOT_NUM_THREADS = 0 -# When you want a differently looking font in the dot files that doxygen -# generates you can specify the font name using DOT_FONTNAME. You need to make -# sure dot is able to find the font, which can be done by putting it in a -# standard location or by setting the DOTFONTPATH environment variable or by -# setting DOT_FONTPATH to the directory containing the font. -# The default value is: Helvetica. +# DOT_COMMON_ATTR is common attributes for nodes, edges and labels of +# subgraphs. When you want a differently looking font in the dot files that +# doxygen generates you can specify fontname, fontcolor and fontsize attributes. +# For details please see Node, +# Edge and Graph Attributes specification You need to make sure dot is able +# to find the font, which can be done by putting it in a standard location or by +# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. Default graphviz fontsize is 14. +# The default value is: fontname=Helvetica,fontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTNAME = Helvetica +DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" -# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of -# dot graphs. -# Minimum value: 4, maximum value: 24, default value: 10. +# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can +# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. Complete documentation about +# arrows shapes. +# The default value is: labelfontname=Helvetica,labelfontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTSIZE = 10 +DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" -# By default doxygen will tell dot to use the default font as specified with -# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set -# the path where dot can find it using this tag. +# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes +# around nodes set 'shape=plain' or 'shape=plaintext' Shapes specification +# The default value is: shape=box,height=0.2,width=0.4. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" + +# You can set the path where dot can find font specified with fontname in +# DOT_COMMON_ATTR and others dot attributes. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for -# each documented class showing the direct and indirect inheritance relations. -# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# If the CLASS_GRAPH tag is set to YES or GRAPH or BUILTIN then doxygen will +# generate a graph for each documented class showing the direct and indirect +# inheritance relations. In case the CLASS_GRAPH tag is set to YES or GRAPH and +# HAVE_DOT is enabled as well, then dot will be used to draw the graph. In case +# the CLASS_GRAPH tag is set to YES and HAVE_DOT is disabled or if the +# CLASS_GRAPH tag is set to BUILTIN, then the built-in generator will be used. +# If the CLASS_GRAPH tag is set to TEXT the direct and indirect inheritance +# relations will be shown as texts / links. +# Possible values are: NO, YES, TEXT, GRAPH and BUILTIN. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a # graph for each documented class showing the direct and indirect implementation # dependencies (inheritance, containment, and class references variables) of the -# class with other documented classes. +# class with other documented classes. Explicit enabling a collaboration graph, +# when COLLABORATION_GRAPH is set to NO, can be accomplished by means of the +# command \collaborationgraph. Disabling a collaboration graph can be +# accomplished by means of the command \hidecollaborationgraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. +# groups, showing the direct groups dependencies. Explicit enabling a group +# dependency graph, when GROUP_GRAPHS is set to NO, can be accomplished by means +# of the command \groupgraph. Disabling a directory graph can be accomplished by +# means of the command \hidegroupgraph. See also the chapter Grouping in the +# manual. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2328,10 +2525,32 @@ UML_LOOK = NO # but if the number exceeds 15, the total amount of fields shown is limited to # 10. # Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag HAVE_DOT is set to YES. +# This tag requires that the tag UML_LOOK is set to YES. UML_LIMIT_NUM_FIELDS = 10 +# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and +# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS +# tag is set to YES, doxygen will add type and arguments for attributes and +# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen +# will not generate fields with class member information in the UML graphs. The +# class diagrams will look similar to the default class diagrams but using UML +# notation for the relationships. +# Possible values are: NO, YES and NONE. +# The default value is: NO. +# This tag requires that the tag UML_LOOK is set to YES. + +DOT_UML_DETAILS = NO + +# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters +# to display on a single line. If the actual line length exceeds this threshold +# significantly it will wrapped across multiple lines. Some heuristics are apply +# to avoid ugly line breaks. +# Minimum value: 0, maximum value: 1000, default value: 17. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_WRAP_THRESHOLD = 17 + # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and # collaboration graphs will show the relations between templates and their # instances. @@ -2343,7 +2562,9 @@ TEMPLATE_RELATIONS = NO # If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to # YES then doxygen will generate a graph for each documented file showing the # direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an include graph, when INCLUDE_GRAPH is is set to NO, +# can be accomplished by means of the command \includegraph. Disabling an +# include graph can be accomplished by means of the command \hideincludegraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2352,7 +2573,10 @@ INCLUDE_GRAPH = YES # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are # set to YES then doxygen will generate a graph for each documented file showing # the direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an included by graph, when INCLUDED_BY_GRAPH is set +# to NO, can be accomplished by means of the command \includedbygraph. Disabling +# an included by graph can be accomplished by means of the command +# \hideincludedbygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2392,16 +2616,26 @@ GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the # dependencies a directory has on other directories in a graphical way. The # dependency relations are determined by the #include relations between the -# files in the directories. +# files in the directories. Explicit enabling a directory graph, when +# DIRECTORY_GRAPH is set to NO, can be accomplished by means of the command +# \directorygraph. Disabling a directory graph can be accomplished by means of +# the command \hidedirectorygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. DIRECTORY_GRAPH = YES +# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels +# of child directories generated in directory dependency graphs by dot. +# Minimum value: 1, maximum value: 25, default value: 1. +# This tag requires that the tag DIRECTORY_GRAPH is set to YES. + +DIR_GRAPH_MAX_DEPTH = 1 + # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). +# https://www.graphviz.org/)). # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order # to make the SVG files visible in IE 9+ (other browsers do not have this # requirement). @@ -2438,11 +2672,12 @@ DOT_PATH = DOTFILE_DIRS = -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. -MSCFILE_DIRS = +DIA_PATH = # The DIAFILE_DIRS tag can be used to specify one or more directories that # contain dia files that are included in the documentation (see the \diafile @@ -2451,10 +2686,10 @@ MSCFILE_DIRS = DIAFILE_DIRS = # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file. If left blank, it is assumed -# PlantUML is not used or called during a preprocessing step. Doxygen will -# generate a warning when it encounters a \startuml command in this case and -# will not generate output for the diagram. +# path where java can find the plantuml.jar file or to the filename of jar file +# to be used. If left blank, it is assumed PlantUML is not used or called during +# a preprocessing step. Doxygen will generate a warning when it encounters a +# \startuml command in this case and will not generate output for the diagram. PLANTUML_JAR_PATH = @@ -2492,18 +2727,6 @@ DOT_GRAPH_MAX_NODES = 50 MAX_DOT_GRAPH_DEPTH = 0 -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not seem -# to support this out of the box. -# -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_TRANSPARENT = NO - # Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) support @@ -2516,14 +2739,34 @@ DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page # explaining the meaning of the various boxes and arrows in the dot generated # graphs. +# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal +# graphical representation for inheritance and collaboration diagrams is used. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GENERATE_LEGEND = YES -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. +# +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. DOT_CLEANUP = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. If the MSCGEN_TOOL tag is left empty (the default), then doxygen will +# use a built-in version of mscgen tool to produce the charts. Alternatively, +# the MSCGEN_TOOL tag can also specify the name an external tool. For instance, +# specifying prog as the value, doxygen will call the tool as prog -T +# -o . The external tool should support +# output file formats "png", "eps", "svg", and "ismap". + +MSCGEN_TOOL = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = diff --git a/cpp/include/cugraph_c/centrality_algorithms.h b/cpp/include/cugraph_c/centrality_algorithms.h index 0ac0e58540f..fb5d4b63b9c 100644 --- a/cpp/include/cugraph_c/centrality_algorithms.h +++ b/cpp/include/cugraph_c/centrality_algorithms.h @@ -23,8 +23,6 @@ #include /** @defgroup centrality Centrality algorithms - * @ingroup c_api - * @{ */ #ifdef __cplusplus @@ -39,7 +37,8 @@ typedef struct { } cugraph_centrality_result_t; /** - * @brief Get the vertex ids from the centrality result + * @ingroup centrality + * @brief Get the vertex ids from the centrality result * * @param [in] result The result from a centrality algorithm * @return type erased array of vertex ids @@ -48,7 +47,8 @@ cugraph_type_erased_device_array_view_t* cugraph_centrality_result_get_vertices( cugraph_centrality_result_t* result); /** - * @brief Get the centrality values from a centrality algorithm result + * @ingroup centrality + * @brief Get the centrality values from a centrality algorithm result * * @param [in] result The result from a centrality algorithm * @return type erased array view of centrality values @@ -57,6 +57,7 @@ cugraph_type_erased_device_array_view_t* cugraph_centrality_result_get_values( cugraph_centrality_result_t* result); /** + * @ingroup centrality * @brief Get the number of iterations executed from the algorithm metadata * * @param [in] result The result from a centrality algorithm @@ -65,6 +66,7 @@ cugraph_type_erased_device_array_view_t* cugraph_centrality_result_get_values( size_t cugraph_centrality_result_get_num_iterations(cugraph_centrality_result_t* result); /** + * @ingroup centrality * @brief Returns true if the centrality algorithm converged * * @param [in] result The result from a centrality algorithm @@ -73,6 +75,7 @@ size_t cugraph_centrality_result_get_num_iterations(cugraph_centrality_result_t* bool_t cugraph_centrality_result_converged(cugraph_centrality_result_t* result); /** + * @ingroup centrality * @brief Free centrality result * * @param [in] result The result from a centrality algorithm @@ -409,6 +412,7 @@ typedef struct { } cugraph_edge_centrality_result_t; /** + * @ingroup centrality * @brief Get the src vertex ids from an edge centrality result * * @param [in] result The result from an edge centrality algorithm @@ -418,6 +422,7 @@ cugraph_type_erased_device_array_view_t* cugraph_edge_centrality_result_get_src_ cugraph_edge_centrality_result_t* result); /** + * @ingroup centrality * @brief Get the dst vertex ids from an edge centrality result * * @param [in] result The result from an edge centrality algorithm @@ -427,6 +432,7 @@ cugraph_type_erased_device_array_view_t* cugraph_edge_centrality_result_get_dst_ cugraph_edge_centrality_result_t* result); /** + * @ingroup centrality * @brief Get the edge ids from an edge centrality result * * @param [in] result The result from an edge centrality algorithm @@ -436,6 +442,7 @@ cugraph_type_erased_device_array_view_t* cugraph_edge_centrality_result_get_edge cugraph_edge_centrality_result_t* result); /** + * @ingroup centrality * @brief Get the centrality values from an edge centrality algorithm result * * @param [in] result The result from an edge centrality algorithm @@ -445,6 +452,7 @@ cugraph_type_erased_device_array_view_t* cugraph_edge_centrality_result_get_valu cugraph_edge_centrality_result_t* result); /** + * @ingroup centrality * @brief Free centrality result * * @param [in] result The result from a centrality algorithm @@ -491,6 +499,7 @@ typedef struct { } cugraph_hits_result_t; /** + * @ingroup centrality * @brief Get the vertex ids from the hits result * * @param [in] result The result from hits @@ -500,6 +509,7 @@ cugraph_type_erased_device_array_view_t* cugraph_hits_result_get_vertices( cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Get the hubs values from the hits result * * @param [in] result The result from hits @@ -509,6 +519,7 @@ cugraph_type_erased_device_array_view_t* cugraph_hits_result_get_hubs( cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Get the authorities values from the hits result * * @param [in] result The result from hits @@ -518,6 +529,7 @@ cugraph_type_erased_device_array_view_t* cugraph_hits_result_get_authorities( cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Get the score differences between the last two iterations * * @param [in] result The result from hits @@ -526,6 +538,7 @@ cugraph_type_erased_device_array_view_t* cugraph_hits_result_get_authorities( double cugraph_hits_result_get_hub_score_differences(cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Get the actual number of iterations * * @param [in] result The result from hits @@ -534,6 +547,7 @@ double cugraph_hits_result_get_hub_score_differences(cugraph_hits_result_t* resu size_t cugraph_hits_result_get_number_of_iterations(cugraph_hits_result_t* result); /** + * @ingroup centrality * @brief Free hits result * * @param [in] result The result from hits @@ -585,7 +599,3 @@ cugraph_error_code_t cugraph_hits( #ifdef __cplusplus } #endif - -/** - * @} - */ diff --git a/cpp/include/cugraph_c/community_algorithms.h b/cpp/include/cugraph_c/community_algorithms.h index 8f1015f8632..feab15c7eeb 100644 --- a/cpp/include/cugraph_c/community_algorithms.h +++ b/cpp/include/cugraph_c/community_algorithms.h @@ -23,7 +23,6 @@ #include /** @defgroup community Community algorithms - * @ingroup c_api * @{ */ diff --git a/cpp/include/cugraph_c/core_algorithms.h b/cpp/include/cugraph_c/core_algorithms.h index c0e348c3cf4..6db3269f61e 100644 --- a/cpp/include/cugraph_c/core_algorithms.h +++ b/cpp/include/cugraph_c/core_algorithms.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,9 @@ #include #include +/** @defgroup core Core algorithms + */ + #ifdef __cplusplus extern "C" { #endif @@ -40,6 +43,7 @@ typedef struct { } cugraph_k_core_result_t; /** + * @ingroup core * @brief Create a core_number result (in case it was previously extracted) * * @param [in] handle Handle for accessing resources @@ -58,6 +62,7 @@ cugraph_error_code_t cugraph_core_result_create( cugraph_error_t** error); /** + * @ingroup core * @brief Get the vertex ids from the core result * * @param [in] result The result from core number @@ -67,6 +72,7 @@ cugraph_type_erased_device_array_view_t* cugraph_core_result_get_vertices( cugraph_core_result_t* result); /** + * @ingroup core * @brief Get the core numbers from the core result * * @param [in] result The result from core number @@ -76,6 +82,7 @@ cugraph_type_erased_device_array_view_t* cugraph_core_result_get_core_numbers( cugraph_core_result_t* result); /** + * @ingroup core * @brief Free core result * * @param [in] result The result from core number @@ -83,6 +90,7 @@ cugraph_type_erased_device_array_view_t* cugraph_core_result_get_core_numbers( void cugraph_core_result_free(cugraph_core_result_t* result); /** + * @ingroup core * @brief Get the src vertex ids from the k-core result * * @param [in] result The result from k-core @@ -92,6 +100,7 @@ cugraph_type_erased_device_array_view_t* cugraph_k_core_result_get_src_vertices( cugraph_k_core_result_t* result); /** + * @ingroup core * @brief Get the dst vertex ids from the k-core result * * @param [in] result The result from k-core @@ -101,6 +110,7 @@ cugraph_type_erased_device_array_view_t* cugraph_k_core_result_get_dst_vertices( cugraph_k_core_result_t* result); /** + * @ingroup core * @brief Get the weights from the k-core result * * Returns NULL if the graph is unweighted @@ -112,6 +122,7 @@ cugraph_type_erased_device_array_view_t* cugraph_k_core_result_get_weights( cugraph_k_core_result_t* result); /** + * @ingroup core * @brief Free k-core result * * @param [in] result The result from k-core @@ -119,6 +130,7 @@ cugraph_type_erased_device_array_view_t* cugraph_k_core_result_get_weights( void cugraph_k_core_result_free(cugraph_k_core_result_t* result); /** + * @ingroup core * @brief Enumeration for computing core number */ typedef enum { diff --git a/cpp/include/cugraph_c/labeling_algorithms.h b/cpp/include/cugraph_c/labeling_algorithms.h index f3e634dafe6..53dcc0d9419 100644 --- a/cpp/include/cugraph_c/labeling_algorithms.h +++ b/cpp/include/cugraph_c/labeling_algorithms.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,6 @@ extern "C" { #endif /** @defgroup labeling Labeling algorithms - * @ingroup c_api - * @{ */ /** @@ -37,6 +35,7 @@ typedef struct { } cugraph_labeling_result_t; /** + * @ingroup labeling * @brief Get the vertex ids from the labeling result * * @param [in] result The result from a labeling algorithm @@ -46,6 +45,7 @@ cugraph_type_erased_device_array_view_t* cugraph_labeling_result_get_vertices( cugraph_labeling_result_t* result); /** + * @ingroup labeling * @brief Get the label values from the labeling result * * @param [in] result The result from a labeling algorithm @@ -55,6 +55,7 @@ cugraph_type_erased_device_array_view_t* cugraph_labeling_result_get_labels( cugraph_labeling_result_t* result); /** + * @ingroup labeling * @brief Free labeling result * * @param [in] result The result from a labeling algorithm @@ -104,7 +105,3 @@ cugraph_error_code_t cugraph_strongly_connected_components(const cugraph_resourc #ifdef __cplusplus } #endif - -/** - * @} - */ diff --git a/cpp/include/cugraph_c/sampling_algorithms.h b/cpp/include/cugraph_c/sampling_algorithms.h index 92fe50ef622..782bb5a3790 100644 --- a/cpp/include/cugraph_c/sampling_algorithms.h +++ b/cpp/include/cugraph_c/sampling_algorithms.h @@ -21,8 +21,7 @@ #include #include -/** @defgroup sampling Sampling algorithms - * @ingroup c_api +/** @defgroup samplingC Sampling algorithms * @{ */ diff --git a/cpp/include/cugraph_c/similarity_algorithms.h b/cpp/include/cugraph_c/similarity_algorithms.h index 1417d8ac566..b8f61b46545 100644 --- a/cpp/include/cugraph_c/similarity_algorithms.h +++ b/cpp/include/cugraph_c/similarity_algorithms.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,9 @@ #include #include +/** @defgroup similarity Similarity algorithms + */ + #ifdef __cplusplus extern "C" { #endif @@ -34,6 +37,7 @@ typedef struct { } cugraph_similarity_result_t; /** + * @ingroup similarity * @brief Get the similarity coefficient array * * @param [in] result The result from a similarity algorithm @@ -43,6 +47,7 @@ cugraph_type_erased_device_array_view_t* cugraph_similarity_result_get_similarit cugraph_similarity_result_t* result); /** + * @ingroup similarity * @brief Free similarity result * * @param [in] result The result from a similarity algorithm diff --git a/docs/cugraph/Makefile b/docs/cugraph/Makefile index 32237aa2cc0..f92d0be6910 100644 --- a/docs/cugraph/Makefile +++ b/docs/cugraph/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = "-v" SPHINXBUILD = sphinx-build SPHINXPROJ = cugraph SOURCEDIR = source diff --git a/docs/cugraph/source/api_docs/cugraph-ops/bipartite_operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/bipartite_operators.rst deleted file mode 100644 index e172309fae2..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/bipartite_operators.rst +++ /dev/null @@ -1,16 +0,0 @@ -============================= -Operators on Bipartite Graphs -============================= - -.. currentmodule:: pylibcugraphops - -Update Edges: Concatenation or Sum of Edge and Node Features ------------------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.update_efeat_bipartite_e2e_concat_fwd - operators.update_efeat_bipartite_e2e_concat_bwd - - operators.update_efeat_bipartite_e2e_sum_fwd - operators.update_efeat_bipartite_e2e_sum_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/c_cpp/index.rst b/docs/cugraph/source/api_docs/cugraph-ops/c_cpp/index.rst new file mode 100644 index 00000000000..5545bebe975 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph-ops/c_cpp/index.rst @@ -0,0 +1,3 @@ +cugraph-ops C++ API Reference +============================= + diff --git a/docs/cugraph/source/api_docs/cugraph-ops/fg_operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/fg_operators.rst deleted file mode 100644 index 387844f684a..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/fg_operators.rst +++ /dev/null @@ -1,83 +0,0 @@ -======================== -Operators on Full Graphs -======================== - -.. currentmodule:: pylibcugraphops - -Simple Neighborhood Aggregator (SAGEConv) ------------------------------------------ -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_simple_fg_n2n_fwd - operators.agg_simple_fg_n2n_bwd - operators.agg_simple_fg_e2n_fwd - operators.agg_simple_fg_e2n_bwd - operators.agg_simple_fg_n2n_e2n_fwd - operators.agg_simple_fg_n2n_e2n_bwd - - operators.agg_concat_fg_n2n_fwd - operators.agg_concat_fg_n2n_bwd - operators.agg_concat_fg_e2n_fwd - operators.agg_concat_fg_e2n_bwd - operators.agg_concat_fg_n2n_e2n_fwd - operators.agg_concat_fg_n2n_e2n_bwd - -Weighted Neighborhood Aggregation ---------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_weighted_fg_n2n_fwd - operators.agg_weighted_fg_n2n_bwd - operators.agg_concat_weighted_fg_n2n_fwd - operators.agg_concat_weighted_fg_n2n_bwd - -Heterogenous Aggregator using Basis Decomposition (RGCNConv) ------------------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_hg_basis_fg_n2n_post_fwd - operators.agg_hg_basis_fg_n2n_post_bwd - -Graph Attention (GATConv/GATv2Conv) ------------------------------------ -.. autosummary:: - :toctree: ../api/ops/ - - operators.mha_gat_fg_n2n_fwd - operators.mha_gat_fg_n2n_bwd - operators.mha_gat_fg_n2n_efeat_fwd - operators.mha_gat_fg_n2n_efeat_bwd - - operators.mha_gat_v2_fg_n2n_fwd - operators.mha_gat_v2_fg_n2n_bwd - operators.mha_gat_v2_fg_n2n_efeat_fwd - operators.mha_gat_v2_fg_n2n_efeat_bwd - -Transformer-like Graph Attention (TransformerConv) --------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.mha_gat_v2_fg_n2n_fwd - operators.mha_gat_v2_fg_n2n_bwd - operators.mha_gat_v2_fg_n2n_efeat_fwd - operators.mha_gat_v2_fg_n2n_efeat_bwd - -Directional Message-Passing (DMPNN) ------------------------------------ -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_dmpnn_fg_e2e_fwd - operators.agg_dmpnn_fg_e2e_bwd - -Graph Pooling -------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.pool_fg_n2s_fwd - operators.pool_fg_n2s_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/graph_types.rst b/docs/cugraph/source/api_docs/cugraph-ops/graph_types.rst deleted file mode 100644 index 9289ce53e39..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/graph_types.rst +++ /dev/null @@ -1,33 +0,0 @@ -=========== -Graph types -=========== - -.. currentmodule:: pylibcugraphops - -Message-Flow Graph (MFG) -------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - make_mfg_csr - -Heterogenous MFG ----------------- -.. autosummary:: - :toctree: ../api/ops/ - - make_mfg_csr_hg - -"Full" Graph (FG) ------------------ -.. autosummary:: - :toctree: ../api/ops/ - - make_fg_csr - -Heterogenous FG ---------------- -.. autosummary:: - :toctree: ../api/ops/ - - make_fg_csr_hg diff --git a/docs/cugraph/source/api_docs/cugraph-ops/index.rst b/docs/cugraph/source/api_docs/cugraph-ops/index.rst index e2338dc1833..fdfd5baab96 100644 --- a/docs/cugraph/source/api_docs/cugraph-ops/index.rst +++ b/docs/cugraph/source/api_docs/cugraph-ops/index.rst @@ -1,4 +1,3 @@ -========================= cugraph-ops API reference ========================= @@ -8,11 +7,5 @@ This page provides a list of all publicly accessible modules, methods and classe :maxdepth: 2 :caption: API Documentation - graph_types - pytorch - mfg_operators - bipartite_operators - static_operators - fg_operators - dimenet - pytorch + python/index + c_cpp/index \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/cugraph-ops/mfg_operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/mfg_operators.rst deleted file mode 100644 index f3dd1faa245..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/mfg_operators.rst +++ /dev/null @@ -1,31 +0,0 @@ -================================ -Operators on Message-Flow Graphs -================================ - -.. currentmodule:: pylibcugraphops - -Simple Neighborhood Aggregator (SAGEConv) ------------------------------------------ -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_simple_mfg_n2n_fwd - operators.agg_simple_mfg_n2n_bwd - operators.agg_concat_mfg_n2n_fwd - operators.agg_concat_mfg_n2n_bwd - -Graph Attention (GATConv) -------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.mha_gat_mfg_n2n_fwd - operators.mha_gat_mfg_n2n_bwd - -Heterogenous Aggregator using Basis Decomposition (RGCNConv) ------------------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.agg_hg_basis_mfg_n2n_post_fwd - operators.agg_hg_basis_mfg_n2n_post_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/dimenet.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/dimenet.rst similarity index 89% rename from docs/cugraph/source/api_docs/cugraph-ops/dimenet.rst rename to docs/cugraph/source/api_docs/cugraph-ops/python/dimenet.rst index b709464c7e6..6fadcc57b22 100644 --- a/docs/cugraph/source/api_docs/cugraph-ops/dimenet.rst +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/dimenet.rst @@ -7,7 +7,7 @@ Dimenet operators Radial Basis Functions ---------------------- .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops dimenet.radial_basis_fwd dimenet.radial_basis_bwd @@ -16,7 +16,7 @@ Radial Basis Functions Edge-to-Edge Aggregation ------------------------- .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops dimenet.agg_edge_to_edge_fwd dimenet.agg_edge_to_edge_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/python/graph_types.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/graph_types.rst new file mode 100644 index 00000000000..141d40393a5 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/graph_types.rst @@ -0,0 +1,34 @@ +=========== +Graph types +=========== + +.. currentmodule:: pylibcugraphops + + +CSC Graph +----------------- +.. autosummary:: + :toctree: ../../api/ops + + make_csc + +Heterogenous CSC Graph +---------------------- +.. autosummary:: + :toctree: ../../api/ops + + make_csc_hg + +Bipartite Graph +----------------- +.. autosummary:: + :toctree: ../../api/ops + + make_bipartite_csc + +Heterogenous Bipartite Graph +---------------------------- +.. autosummary:: + :toctree: ../../api/ops + + make_bipartite_csc_hg diff --git a/docs/cugraph/source/api_docs/cugraph-ops/python/index.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/index.rst new file mode 100644 index 00000000000..082c7741f23 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/index.rst @@ -0,0 +1,13 @@ +cugraph-ops Python API reference +================================ + +This page provides a list of all publicly accessible modules, methods and classes through `pylibcugraphops.*` namespace. + +.. toctree:: + :maxdepth: 2 + :caption: API Documentation + + graph_types + operators + dimenet + pytorch diff --git a/docs/cugraph/source/api_docs/cugraph-ops/python/operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/operators.rst new file mode 100644 index 00000000000..3e6664b2db5 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/operators.rst @@ -0,0 +1,93 @@ +============================= +Operators for Message-Passing +============================= + +.. currentmodule:: pylibcugraphops + +Simple Neighborhood Aggregator (SAGEConv) +----------------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.agg_simple_n2n_fwd + operators.agg_simple_n2n_bwd + operators.agg_simple_e2n_fwd + operators.agg_simple_e2n_bwd + operators.agg_simple_n2n_e2n_fwd + operators.agg_simple_n2n_e2n_bwd + + operators.agg_concat_n2n_fwd + operators.agg_concat_n2n_bwd + operators.agg_concat_e2n_fwd + operators.agg_concat_e2n_bwd + operators.agg_concat_n2n_e2n_fwd + operators.agg_concat_n2n_e2n_bwd + + +Weighted Neighborhood Aggregation +--------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.agg_weighted_n2n_fwd + operators.agg_weighted_n2n_bwd + operators.agg_concat_weighted_n2n_fwd + operators.agg_concat_weighted_n2n_bwd + +Heterogenous Aggregator using Basis Decomposition (RGCNConv) +------------------------------------------------------------ +.. autosummary:: + :toctree: ../../api/ops + + operators.agg_hg_basis_n2n_post_fwd + operators.agg_hg_basis_n2n_post_bwd + +Graph Attention (GATConv/GATv2Conv) +----------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.mha_gat_n2n_fwd + operators.mha_gat_n2n_bwd + operators.mha_gat_n2n_efeat_fwd + operators.mha_gat_n2n_efeat_bwd + + operators.mha_gat_v2_n2n_fwd + operators.mha_gat_v2_n2n_bwd + operators.mha_gat_v2_n2n_efeat_fwd + operators.mha_gat_v2_n2n_efeat_bwd + +Transformer-like Graph Attention (TransformerConv) +-------------------------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.mha_gat_v2_n2n_fwd + operators.mha_gat_v2_n2n_bwd + operators.mha_gat_v2_n2n_efeat_fwd + operators.mha_gat_v2_n2n_efeat_bwd + +Directional Message-Passing (DMPNN) +----------------------------------- +.. autosummary:: + :toctree: ../../api/ops + + operators.agg_dmpnn_e2e_fwd + operators.agg_dmpnn_e2e_bwd + +Update Edges: Concatenation or Sum of Edge and Node Features +------------------------------------------------------------ +.. autosummary:: + :toctree: ../../api/ops + + operators.update_efeat_e2e_concat_fwd + operators.update_efeat_e2e_concat_bwd + + operators.update_efeat_e2e_sum_fwd + operators.update_efeat_e2e_sum_bwd + + operators.update_efeat_e2e_concat_fwd + operators.update_efeat_e2e_concat_bwd + + operators.update_efeat_e2e_sum_fwd + operators.update_efeat_e2e_sum_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-ops/pytorch.rst b/docs/cugraph/source/api_docs/cugraph-ops/python/pytorch.rst similarity index 59% rename from docs/cugraph/source/api_docs/cugraph-ops/pytorch.rst rename to docs/cugraph/source/api_docs/cugraph-ops/python/pytorch.rst index 83800fbc546..d2074df15b0 100644 --- a/docs/cugraph/source/api_docs/cugraph-ops/pytorch.rst +++ b/docs/cugraph/source/api_docs/cugraph-ops/python/pytorch.rst @@ -2,35 +2,35 @@ PyTorch Autograd Wrappers ========================== -.. currentmodule:: pylibcugraphops +.. currentmodule:: pylibcugraphops.pytorch Simple Neighborhood Aggregator (SAGEConv) ----------------------------------------- .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops - pytorch.operators.agg_concat_n2n + operators.agg_concat_n2n Graph Attention (GATConv/GATv2Conv) ----------------------------------- .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops - pytorch.operators.mha_gat_n2n - pytorch.operators.mha_gat_v2_n2n + operators.mha_gat_n2n + operators.mha_gat_v2_n2n Heterogenous Aggregator using Basis Decomposition (RGCNConv) ------------------------------------------------------------ .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops - pytorch.operators.agg_hg_basis_n2n_post + operators.agg_hg_basis_n2n_post Update Edges: Concatenation or Sum of Edge and Node Features ------------------------------------------------------------ .. autosummary:: - :toctree: ../api/ops/ + :toctree: ../../api/ops - pytorch.operators.update_efeat_bipartite_e2e - pytorch.operators.update_efeat_static_e2e + operators.update_efeat_e2e + operators.update_efeat_e2e diff --git a/docs/cugraph/source/api_docs/cugraph-ops/static_operators.rst b/docs/cugraph/source/api_docs/cugraph-ops/static_operators.rst deleted file mode 100644 index f3ecc068f22..00000000000 --- a/docs/cugraph/source/api_docs/cugraph-ops/static_operators.rst +++ /dev/null @@ -1,16 +0,0 @@ -========================== -Operators on Static Graphs -========================== - -.. currentmodule:: pylibcugraphops - -Update Edges: Concatenation or Sum of Edge and Node Features ------------------------------------------------------------- -.. autosummary:: - :toctree: ../api/ops/ - - operators.update_efeat_static_e2e_concat_fwd - operators.update_efeat_static_e2e_concat_bwd - - operators.update_efeat_static_e2e_sum_fwd - operators.update_efeat_static_e2e_sum_bwd diff --git a/docs/cugraph/source/api_docs/cugraph-pyg/cugraph_pyg.rst b/docs/cugraph/source/api_docs/cugraph-pyg/cugraph_pyg.rst index 2cd8969aa66..f7d7f5f2262 100644 --- a/docs/cugraph/source/api_docs/cugraph-pyg/cugraph_pyg.rst +++ b/docs/cugraph/source/api_docs/cugraph-pyg/cugraph_pyg.rst @@ -9,6 +9,6 @@ cugraph-pyg .. autosummary:: :toctree: ../api/cugraph-pyg/ - cugraph_pyg.data.cugraph_store.EXPERIMENTAL__CuGraphStore - cugraph_pyg.sampler.cugraph_sampler.EXPERIMENTAL__CuGraphSampler +.. cugraph_pyg.data.cugraph_store.EXPERIMENTAL__CuGraphStore +.. cugraph_pyg.sampler.cugraph_sampler.EXPERIMENTAL__CuGraphSampler diff --git a/docs/cugraph/source/api_docs/cugraph_c/c_and_cpp.rst b/docs/cugraph/source/api_docs/cugraph_c/c_and_cpp.rst deleted file mode 100644 index 34b812785d3..00000000000 --- a/docs/cugraph/source/api_docs/cugraph_c/c_and_cpp.rst +++ /dev/null @@ -1,4 +0,0 @@ -CuGraph C and C++ API Links -=========================== - -coming soon - see https://docs.rapids.ai/api/libcugraph/nightly/ \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/cugraph_c/centrality.rst b/docs/cugraph/source/api_docs/cugraph_c/centrality.rst new file mode 100644 index 00000000000..f34e26ad76e --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/centrality.rst @@ -0,0 +1,51 @@ +Centrality +========== + +PageRank +-------- +.. doxygenfunction:: cugraph_pagerank + :project: libcugraph + +.. doxygenfunction:: cugraph_pagerank_allow_nonconvergence + :project: libcugraph + +Personalized PageRank +--------------------- +.. doxygenfunction:: cugraph_personalized_pagerank + :project: libcugraph + +.. doxygenfunction:: cugraph_personalized_pagerank_allow_nonconvergence + :project: libcugraph + +Eigenvector Centrality +---------------------- +.. doxygenfunction:: cugraph_eigenvector_centrality + :project: libcugraph + +Katz Centrality +--------------- +.. doxygenfunction:: cugraph_katz_centrality + :project: libcugraph + +Betweenness Centrality +---------------------- +.. doxygenfunction:: cugraph_betweenness_centrality + :project: libcugraph + +Edge Betweenness Centrality +--------------------------- +.. doxygenfunction:: cugraph_edge_betweenness_centrality + :project: libcugraph + +HITS Centrality +--------------- +.. doxygenfunction:: cugraph_hits + :project: libcugraph + +Centrality Support Functions +---------------------------- + .. doxygengroup:: centrality + :project: libcugraph + :members: + :content-only: + diff --git a/docs/cugraph/source/api_docs/cugraph_c/community.rst b/docs/cugraph/source/api_docs/cugraph_c/community.rst new file mode 100644 index 00000000000..0bbfe365c4d --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/community.rst @@ -0,0 +1,63 @@ +Community +========= + +.. role:: py(code) + :language: c + :class: highlight + +``#include `` + +Triangle Counting +----------------- +.. doxygenfunction:: cugraph_triangle_count + :project: libcugraph + +Louvain +------- +.. doxygenfunction:: cugraph_louvain + :project: libcugraph + +Leiden +------ +.. doxygenfunction:: cugraph_leiden + :project: libcugraph + +ECG +--- +.. doxygenfunction:: cugraph_ecg + :project: libcugraph + +Extract Egonet +-------------- +.. doxygenfunction:: cugraph_extract_ego + :project: libcugraph + +Balanced Cut +------------ +.. doxygenfunction:: cugraph_balanced_cut_clustering + :project: libcugraph + +Spectral Clustering - Modularity Maximization +--------------------------------------------- +.. doxygenfunction:: cugraph_spectral_modularity_maximization + :project: libcugraph + +.. doxygenfunction:: cugraph_analyze_clustering_modularity + :project: libcugraph + +Spectral Clusteriong - Edge Cut +------------------------------- +.. doxygenfunction:: cugraph_analyze_clustering_edge_cut + :project: libcugraph + +.. doxygenfunction:: cugraph_analyze_clustering_ratio_cut + :project: libcugraph + + +Community Support Functions +--------------------------- + .. doxygengroup:: community + :project: libcugraph + :members: + :content-only: + diff --git a/docs/cugraph/source/api_docs/cugraph_c/core.rst b/docs/cugraph/source/api_docs/cugraph_c/core.rst new file mode 100644 index 00000000000..34456c65e43 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/core.rst @@ -0,0 +1,21 @@ +Core +==== + + +Core Number +----------- +.. doxygenfunction:: cugraph_core_number + :project: libcugraph + +K-Core +------ +.. doxygenfunction:: cugraph_k_core + :project: libcugraph + + +Core Support Functions +---------------------- + .. doxygengroup:: core + :project: libcugraph + :members: + :content-only: diff --git a/docs/cugraph/source/api_docs/cugraph_c/index.rst b/docs/cugraph/source/api_docs/cugraph_c/index.rst new file mode 100644 index 00000000000..3dd37dbc374 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/index.rst @@ -0,0 +1,16 @@ +=========================== +cuGraph C API documentation +=========================== + + +.. toctree:: + :maxdepth: 3 + :caption: API Documentation + + centrality.rst + community.rst + core.rst + labeling.rst + sampling.rst + similarity.rst + traversal.rst diff --git a/docs/cugraph/source/api_docs/cugraph_c/labeling.rst b/docs/cugraph/source/api_docs/cugraph_c/labeling.rst new file mode 100644 index 00000000000..af105ee8fc9 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/labeling.rst @@ -0,0 +1,20 @@ +Components +========== + + +Weakly Connected Components +--------------------------- +.. doxygenfunction:: cugraph_weakly_connected_components + :project: libcugraph + +Strongly Connected Components +----------------------------- +.. doxygenfunction:: cugraph_strongly_connected_components + :project: libcugraph + +Support +------- + .. doxygengroup:: labeling + :project: libcugraph + :members: + :content-only: \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/cugraph_c/sampling.rst b/docs/cugraph/source/api_docs/cugraph_c/sampling.rst new file mode 100644 index 00000000000..21b837daf93 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/sampling.rst @@ -0,0 +1,37 @@ +Sampling +======== + +Uniform Random Walks +-------------------- +.. doxygenfunction:: cugraph_uniform_random_walks + :project: libcugraph + +Biased Random Walks +-------------------- +.. doxygenfunction:: cugraph_biased_random_walks + :project: libcugraph + +Random Walks via Node2Vec +------------------------- +.. doxygenfunction:: cugraph_node2vec_random_walks + :project: libcugraph + +Node2Vec +-------- +.. doxygenfunction:: cugraph_node2vec + :project: libcugraph + +Uniform Neighborhood Sampling +----------------------------- +.. doxygenfunction:: cugraph_uniform_neighbor_sample_with_edge_properties + :project: libcugraph + +.. doxygenfunction:: cugraph_uniform_neighbor_sample + :project: libcugraph + +Support +------- +.. doxygengroup:: samplingC + :project: libcugraph + :members: + :content-only: diff --git a/docs/cugraph/source/api_docs/cugraph_c/similarity.rst b/docs/cugraph/source/api_docs/cugraph_c/similarity.rst new file mode 100644 index 00000000000..fba07ad206c --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/similarity.rst @@ -0,0 +1,25 @@ +Similarity +========== + + +Jaccard +------- +.. doxygenfunction:: cugraph_jaccard_coefficients + :project: libcugraph + +Sorensen +-------- +.. doxygenfunction:: cugraph_sorensen_coefficients + :project: libcugraph + +Overlap +------- +.. doxygenfunction:: cugraph_overlap_coefficients + :project: libcugraph + +Support +------- +.. doxygengroup:: similarity + :project: libcugraph + :members: + :content-only: \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/cugraph_c/traversal.rst b/docs/cugraph/source/api_docs/cugraph_c/traversal.rst new file mode 100644 index 00000000000..c90760e9e79 --- /dev/null +++ b/docs/cugraph/source/api_docs/cugraph_c/traversal.rst @@ -0,0 +1,30 @@ +Traversal +========== + + +Breadth First Search (BFS) +-------------------------- +.. doxygenfunction:: cugraph_bfs + :project: libcugraph + +Single-Source Shortest-Path (SSSP) +---------------------------------- +.. doxygenfunction:: cugraph_sssp + :project: libcugraph + +Path Extraction +--------------- +.. doxygenfunction:: cugraph_extract_paths + :project: libcugraph + +Extract Max Path Length +----------------------- +.. doxygenfunction:: cugraph_extract_paths_result_get_max_path_length + :project: libcugraph + +Support +------- +.. doxygengroup:: traversal + :project: libcugraph + :members: + :content-only: \ No newline at end of file diff --git a/docs/cugraph/source/api_docs/index.rst b/docs/cugraph/source/api_docs/index.rst index 45f7210f5a2..74ca98bb98d 100644 --- a/docs/cugraph/source/api_docs/index.rst +++ b/docs/cugraph/source/api_docs/index.rst @@ -1,16 +1,39 @@ -Python API reference -==================== +API Reference +============= This page provides a list of all publicly accessible Python modules with in the Graph collection +Core Graph API Documentation +---------------------------- + .. toctree:: - :maxdepth: 2 - :caption: Python API Documentation + :maxdepth: 3 + :caption: Core Graph API Documentation cugraph/index.rst plc/pylibcugraph.rst + cugraph_c/index.rst + cugraph_cpp/index.rst + +Graph Nerual Networks API Documentation +--------------------------------------- + +.. toctree:: + :maxdepth: 3 + :caption: Graph Nerual Networks API Documentation + cugraph-dgl/cugraph_dgl.rst cugraph-pyg/cugraph_pyg.rst - service/index.rst cugraph-ops/index.rst + wholegraph/index.rst + +Additional Graph Packages API Documentation +---------------------------------- + +.. toctree:: + :maxdepth: 3 + :caption: Additional Graph Packages API Documentation + + service/index.rst + diff --git a/docs/cugraph/source/api_docs/service/cugraph_service_client.rst b/docs/cugraph/source/api_docs/service/cugraph_service_client.rst index 383b31d269a..7e344d326f7 100644 --- a/docs/cugraph/source/api_docs/service/cugraph_service_client.rst +++ b/docs/cugraph/source/api_docs/service/cugraph_service_client.rst @@ -9,7 +9,7 @@ cugraph-service .. autosummary:: :toctree: ../api/service/ - cugraph_service_client.client.RunAsyncioThread +.. cugraph_service_client.client.RunAsyncioThread cugraph_service_client.client.run_async cugraph_service_client.client.DeviceArrayAllocator cugraph_service_client.client.CugraphServiceClient diff --git a/docs/cugraph/source/api_docs/service/cugraph_service_server.rst b/docs/cugraph/source/api_docs/service/cugraph_service_server.rst index a7e8b547573..09ca8360b6c 100644 --- a/docs/cugraph/source/api_docs/service/cugraph_service_server.rst +++ b/docs/cugraph/source/api_docs/service/cugraph_service_server.rst @@ -9,6 +9,6 @@ cugraph-service .. autosummary:: :toctree: ../api/service/ - cugraph_service_server.cugraph_handler.call_algo +.. cugraph_service_server.cugraph_handler.call_algo cugraph_service_server.cugraph_handler.ExtensionServerFacade cugraph_service_server.cugraph_handler.CugraphHandler diff --git a/docs/cugraph/source/api_docs/wholegraph/index.rst b/docs/cugraph/source/api_docs/wholegraph/index.rst new file mode 100644 index 00000000000..80e231d4610 --- /dev/null +++ b/docs/cugraph/source/api_docs/wholegraph/index.rst @@ -0,0 +1,11 @@ +WholeGraph API reference +======================== + +This page provides WholeGraph API reference + +.. toctree:: + :maxdepth: 2 + :caption: WholeGraph API Documentation + + libwholegraph/index.rst + pylibwholegraph/index.rst diff --git a/docs/cugraph/source/api_docs/wholegraph/libwholegraph/index.rst b/docs/cugraph/source/api_docs/wholegraph/libwholegraph/index.rst new file mode 100644 index 00000000000..4ef68abef2d --- /dev/null +++ b/docs/cugraph/source/api_docs/wholegraph/libwholegraph/index.rst @@ -0,0 +1,228 @@ +===================== +libwholegraph API doc +===================== + +Doxygen WholeGraph C API documentation +-------------------------------------- +For doxygen documentation, please refer to `Doxygen Documentation <../../doxygen_docs/libwholegraph/html/index.html>`_ + +WholeGraph C API documentation +------------------------------ + +Library Level APIs +++++++++++++++++++ + +.. doxygenenum:: wholememory_error_code_t + :project: libwholegraph +.. doxygenfunction:: wholememory_init + :project: libwholegraph +.. doxygenfunction:: wholememory_finalize + :project: libwholegraph +.. doxygenfunction:: fork_get_device_count + :project: libwholegraph + +WholeMemory Communicator APIs ++++++++++++++++++++++++++++++ + +.. doxygentypedef:: wholememory_comm_t + :project: libwholegraph +.. doxygenstruct:: wholememory_unique_id_t + :project: libwholegraph +.. doxygenfunction:: wholememory_create_unique_id + :project: libwholegraph +.. doxygenfunction:: wholememory_create_communicator + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_communicator + :project: libwholegraph +.. doxygenfunction:: wholememory_communicator_get_rank + :project: libwholegraph +.. doxygenfunction:: wholememory_communicator_get_size + :project: libwholegraph +.. doxygenfunction:: wholememory_communicator_barrier + :project: libwholegraph + +WholeMemoryHandle APIs +++++++++++++++++++++++ + +.. doxygenenum:: wholememory_memory_type_t + :project: libwholegraph +.. doxygenenum:: wholememory_memory_location_t + :project: libwholegraph +.. doxygentypedef:: wholememory_handle_t + :project: libwholegraph +.. doxygenstruct:: wholememory_gref_t + :project: libwholegraph +.. doxygenfunction:: wholememory_malloc + :project: libwholegraph +.. doxygenfunction:: wholememory_free + :project: libwholegraph +.. doxygenfunction:: wholememory_get_communicator + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_type + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_location + :project: libwholegraph +.. doxygenfunction:: wholememory_get_total_size + :project: libwholegraph +.. doxygenfunction:: wholememory_get_data_granularity + :project: libwholegraph +.. doxygenfunction:: wholememory_get_local_memory + :project: libwholegraph +.. doxygenfunction:: wholememory_get_rank_memory + :project: libwholegraph +.. doxygenfunction:: wholememory_get_global_pointer + :project: libwholegraph +.. doxygenfunction:: wholememory_get_global_reference + :project: libwholegraph +.. doxygenfunction:: wholememory_determine_partition_plan + :project: libwholegraph +.. doxygenfunction:: wholememory_determine_entry_partition_plan + :project: libwholegraph +.. doxygenfunction:: wholememory_get_partition_plan + :project: libwholegraph +.. doxygenfunction:: wholememory_load_from_file + :project: libwholegraph +.. doxygenfunction:: wholememory_store_to_file + :project: libwholegraph + +WholeMemoryTensor APIs +++++++++++++++++++++++ + +.. doxygenenum:: wholememory_dtype_t + :project: libwholegraph +.. doxygenstruct:: wholememory_array_description_t + :project: libwholegraph +.. doxygenstruct:: wholememory_matrix_description_t + :project: libwholegraph +.. doxygenstruct:: wholememory_tensor_description_t + :project: libwholegraph +.. doxygentypedef:: wholememory_tensor_t + :project: libwholegraph +.. doxygenfunction:: wholememory_dtype_get_element_size + :project: libwholegraph +.. doxygenfunction:: wholememory_dtype_is_floating_number + :project: libwholegraph +.. doxygenfunction:: wholememory_dtype_is_integer_number + :project: libwholegraph +.. doxygenfunction:: wholememory_create_array_desc + :project: libwholegraph +.. doxygenfunction:: wholememory_create_matrix_desc + :project: libwholegraph +.. doxygenfunction:: wholememory_initialize_tensor_desc + :project: libwholegraph +.. doxygenfunction:: wholememory_copy_array_desc_to_matrix + :project: libwholegraph +.. doxygenfunction:: wholememory_copy_array_desc_to_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_copy_matrix_desc_to_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_convert_tensor_desc_to_array + :project: libwholegraph +.. doxygenfunction:: wholememory_convert_tensor_desc_to_matrix + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_element_count_from_array + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_size_from_array + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_element_count_from_matrix + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_size_from_matrix + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_element_count_from_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_get_memory_size_from_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_unsqueeze_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_create_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_make_tensor_from_pointer + :project: libwholegraph +.. doxygenfunction:: wholememory_make_tensor_from_handle + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_has_handle + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_memory_handle + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_tensor_description + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_global_reference + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_map_local_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_data_pointer + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_entry_per_partition + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_subtensor + :project: libwholegraph +.. doxygenfunction:: wholememory_tensor_get_root + :project: libwholegraph + +Ops on WholeMemory Tensors +++++++++++++++++++++++++++ + +.. doxygenfunction:: wholememory_gather + :project: libwholegraph +.. doxygenfunction:: wholememory_scatter + :project: libwholegraph + +WholeTensorEmbedding APIs ++++++++++++++++++++++++++ + +.. doxygentypedef:: wholememory_embedding_cache_policy_t + :project: libwholegraph +.. doxygentypedef:: wholememory_embedding_optimizer_t + :project: libwholegraph +.. doxygentypedef:: wholememory_embedding_t + :project: libwholegraph +.. doxygenenum:: wholememory_access_type_t + :project: libwholegraph +.. doxygenenum:: wholememory_optimizer_type_t + :project: libwholegraph +.. doxygenfunction:: wholememory_create_embedding_optimizer + :project: libwholegraph +.. doxygenfunction:: wholememory_optimizer_set_parameter + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_embedding_optimizer + :project: libwholegraph +.. doxygenfunction:: wholememory_create_embedding_cache_policy + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_embedding_cache_policy + :project: libwholegraph +.. doxygenfunction:: wholememory_create_embedding + :project: libwholegraph +.. doxygenfunction:: wholememory_destroy_embedding + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_get_embedding_tensor + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_gather + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_gather_gradient_apply + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_get_optimizer_state_names + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_get_optimizer_state + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_writeback_cache + :project: libwholegraph +.. doxygenfunction:: wholememory_embedding_drop_all_cache + :project: libwholegraph + +Ops on graphs stored in WholeMemory ++++++++++++++++++++++++++++++++++++ + +.. doxygenfunction:: wholegraph_csr_unweighted_sample_without_replacement + :project: libwholegraph +.. doxygenfunction:: wholegraph_csr_weighted_sample_without_replacement + :project: libwholegraph + +Miscellaneous Ops for graph ++++++++++++++++++++++++++++ + +.. doxygenfunction:: graph_append_unique + :project: libwholegraph +.. doxygenfunction:: csr_add_self_loop + :project: libwholegraph diff --git a/docs/cugraph/source/api_docs/wholegraph/pylibwholegraph/index.rst b/docs/cugraph/source/api_docs/wholegraph/pylibwholegraph/index.rst new file mode 100644 index 00000000000..67aab00acef --- /dev/null +++ b/docs/cugraph/source/api_docs/wholegraph/pylibwholegraph/index.rst @@ -0,0 +1,38 @@ +======================= +pylibwholegraph API doc +======================= + +.. currentmodule:: pylibwholegraph + +APIs +---- +.. autosummary:: + :toctree: ../../api/wg + + torch.initialize.init_torch_env + torch.initialize.init_torch_env_and_create_wm_comm + torch.initialize.finalize + torch.comm.WholeMemoryCommunicator + torch.comm.set_world_info + torch.comm.create_group_communicator + torch.comm.destroy_communicator + torch.comm.get_global_communicator + torch.comm.get_local_node_communicator + torch.comm.get_local_device_communicator + torch.tensor.WholeMemoryTensor + torch.tensor.create_wholememory_tensor + torch.tensor.create_wholememory_tensor_from_filelist + torch.tensor.destroy_wholememory_tensor + torch.embedding.WholeMemoryOptimizer + torch.embedding.create_wholememory_optimizer + torch.embedding.destroy_wholememory_optimizer + torch.embedding.WholeMemoryCachePolicy + torch.embedding.create_wholememory_cache_policy + torch.embedding.create_builtin_cache_policy + torch.embedding.destroy_wholememory_cache_policy + torch.embedding.WholeMemoryEmbedding + torch.embedding.create_embedding + torch.embedding.create_embedding_from_filelist + torch.embedding.destroy_embedding + torch.embedding.WholeMemoryEmbeddingModule + torch.graph_structure.GraphStructure diff --git a/docs/cugraph/source/basics/cugraph_intro.md b/docs/cugraph/source/basics/cugraph_intro.md index 0684129503f..10d14f8a0d7 100644 --- a/docs/cugraph/source/basics/cugraph_intro.md +++ b/docs/cugraph/source/basics/cugraph_intro.md @@ -21,7 +21,7 @@ call graph algorithms using data stored in a GPU DataFrame, NetworkX Graphs, or CuPy or SciPy sparse Matrix. -# Vision +## Vision The vision of RAPIDS cuGraph is to ___make graph analysis ubiquitous to the point that users just think in terms of analysis and not technologies or frameworks___. This is a goal that many of us on the cuGraph team have been @@ -49,7 +49,7 @@ RAPIDS and DASK allows cuGraph to scale to multiple GPUs to support multi-billion edge graphs. -# Terminology +## Terminology cuGraph is a collection of GPU accelerated graph algorithms and graph utility functions. The application of graph analysis covers a lot of areas. @@ -67,8 +67,7 @@ documentation we will mostly use the terms __Node__ and __Edge__ to better match NetworkX preferred term use, as well as other Python-based tools. At the CUDA/C layer, we favor the mathematical terms of __Vertex__ and __Edge__. -# Roadmap -GitHub does not provide a robust project management interface, and so a roadmap turns into simply a projection of when work will be completed and not a complete picture of everything that needs to be done. To capture the work that requires multiple steps, issues are labels as “EPIC” and include multiple subtasks that could span multiple releases. The EPIC will be in the release where work in expected to be completed. A better roadmap is being worked an image of the roadmap will be posted when ready. - * GitHub Project Board: https://github.com/rapidsai/cugraph/projects/28 + + \ No newline at end of file diff --git a/docs/cugraph/source/conf.py b/docs/cugraph/source/conf.py index 470086b4faa..3f7ef7deb03 100644 --- a/docs/cugraph/source/conf.py +++ b/docs/cugraph/source/conf.py @@ -181,11 +181,10 @@ # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'cugraph', 'cugraph Documentation', - author, 'cugraph', 'One line description of project.', + author, 'cugraph', 'GPU-accelerated graph analysis.', 'Miscellaneous'), ] - # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} @@ -209,7 +208,9 @@ def setup(app): ) breathe_projects = { + 'libcugraph': os.environ['XML_DIR_LIBCUGRAPH'], 'libcugraphops': os.environ['XML_DIR_LIBCUGRAPHOPS'], 'libwholegraph': os.environ['XML_DIR_LIBWHOLEGRAPH'] } + breathe_default_project = "libcugraph" diff --git a/docs/cugraph/source/graph_support/algorithms.md b/docs/cugraph/source/graph_support/algorithms.md index f6cb7c0d8b1..a1b80e92751 100644 --- a/docs/cugraph/source/graph_support/algorithms.md +++ b/docs/cugraph/source/graph_support/algorithms.md @@ -22,7 +22,7 @@ Note: Multi-GPU, or MG, includes support for Multi-Node Multi-GPU (also called M | Category | Notebooks | Scale | Notes | | ----------------- | ---------------------------------- | ------------------- | --------------------------------------------------------------- | -| [Centrality](./algorithms/Centrality.md) | [Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb) | | | +| [Centrality](./algorithms/Centrality.html ) | [Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb) | | | | | [Katz](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Katz.ipynb) | __Multi-GPU__ | | | | [Betweenness Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Betweenness.ipynb) | __Multi-GPU__ | MG as of 23.06 | | | [Edge Betweenness Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Betweenness.ipynb) | __Multi-GPU__ | MG as of 23.08 | @@ -31,12 +31,12 @@ Note: Multi-GPU, or MG, includes support for Multi-Node Multi-GPU (also called M | Community | | | | | | [Leiden](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Louvain.ipynb) | __Multi-GPU__ | MG as of 23.06 | | | [Louvain](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Louvain.ipynb) | __Multi-GPU__ | | -| | [Ensemble Clustering for Graphs](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/ECG.ipynb) | Single-GPU | MG planned for 23.10 | +| | [Ensemble Clustering for Graphs](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/ECG.ipynb) | Single-GPU | MG planned for 24.02 | | | [Spectral-Clustering - Balanced Cut](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Spectral-Clustering.ipynb) | Single-GPU | | | | [Spectral-Clustering - Modularity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Spectral-Clustering.ipynb) | Single-GPU | | | | [Subgraph Extraction](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Subgraph-Extraction.ipyn) | Single-GPU | | | | [Triangle Counting](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/Triangle-Counting.ipynb) | __Multi-GPU__ | | -| | [K-Truss](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/ktruss.ipynb) | Single-GPU | MG planned for 23.10 | +| | [K-Truss](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/community/ktruss.ipynb) | Single-GPU | MG planned for 2024 | | Components | | | | | | [Weakly Connected Components](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/components/ConnectedComponents.ipynb) | __Multi-GPU__ | | | | [Strongly Connected Components](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/components/ConnectedComponents.ipynb) | Single-GPU | | @@ -55,7 +55,7 @@ Note: Multi-GPU, or MG, includes support for Multi-Node Multi-GPU (also called M | | [Pagerank](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_analysis/Pagerank.ipynb) | __Multi-GPU__ | [C++ README](cpp/src/centrality/README.md#Pagerank) | | | [Personal Pagerank]() | __Multi-GPU__ | [C++ README](cpp/src/centrality/README.md#Personalized-Pagerank) | | | [HITS](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_analysis/HITS.ipynb) | __Multi-GPU__ | | -| [Link Prediction](./algorithms/Similarity.md) | | | | +| [Link Prediction](algorithms/Similarity.html) | | | | | | [Jaccard Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Jaccard-Similarity.ipynb) | __Multi-GPU__ | Directed graph only | | | [Weighted Jaccard Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Jaccard-Similarity.ipynb) | Single-GPU | | | | [Overlap Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Overlap-Similarity.ipynb) | **Multi-GPU** | | @@ -65,8 +65,8 @@ Note: Multi-GPU, or MG, includes support for Multi-Node Multi-GPU (also called M | | [Uniform Random Walks RW](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/sampling/RandomWalk.ipynb) | __Multi-GPU__ | | | | *Biased Random Walks (RW)* | --- | | | | Egonet | __Multi-GPU__ | | -| | Node2Vec | Single-GPU | | -| | Uniform Neighborhood sampling | __Multi-GPU__ | | +| | Node2Vec | __Multi-GPU__ | | +| | Neighborhood sampling | __Multi-GPU__ | | | Traversal | | | | | | Breadth First Search (BFS) | __Multi-GPU__ | with cutoff support [C++ README](cpp/src/traversal/README.md#BFS) | | | Single Source Shortest Path (SSSP) | __Multi-GPU__ | [C++ README](cpp/src/traversal/README.md#SSSP) | diff --git a/docs/cugraph/source/graph_support/algorithms/Centrality.md b/docs/cugraph/source/graph_support/algorithms/Centrality.md index e42bbe238c6..f82a1fe123b 100644 --- a/docs/cugraph/source/graph_support/algorithms/Centrality.md +++ b/docs/cugraph/source/graph_support/algorithms/Centrality.md @@ -1,7 +1,7 @@ # cuGraph Centrality Notebooks - + The RAPIDS cuGraph Centrality folder contains a collection of Jupyter Notebooks that demonstrate algorithms to identify and quantify the importance of vertices to the structure of the graph. In the diagram above, the highlighted vertices are highly important and are likely answers to questions like: @@ -15,13 +15,13 @@ But which vertices are most important? The answer depends on which measure/algor |Algorithm |Notebooks Containing |Description | | --------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -|[Degree Centrality](./degree_centrality.md)| [Centrality](./Centrality.ipynb), [Degree](./Degree.ipynb) |Measure based on counting direct connections for each vertex| -|[Betweenness Centrality](./betweenness_centrality.md)| [Centrality](./Centrality.ipynb), [Betweenness](./Betweenness.ipynb) |Number of shortest paths through the vertex| -|[Eigenvector Centrality](./eigenvector_centrality.md)|[Centrality](./Centrality.ipynb), [Eigenvector](./Eigenvector.ipynb)|Measure of connectivity to other important vertices (which also have high connectivity) often referred to as the influence measure of a vertex| -|[Katz Centrality](./katz_centrality.md)|[Centrality](./Centrality.ipynb), [Katz](./Katz.ipynb) |Similar to Eigenvector but has tweaks to measure more weakly connected graph | +|[Degree Centrality](./degree_centrality.html)| [Centrality](./Centrality.ipynb), [Degree](./Degree.ipynb) |Measure based on counting direct connections for each vertex| +|[Betweenness Centrality](./betweenness_centrality.html)| [Centrality](./Centrality.ipynb), [Betweenness](./Betweenness.ipynb) |Number of shortest paths through the vertex| +|[Eigenvector Centrality](./eigenvector_centrality.html)|[Centrality](./Centrality.ipynb), [Eigenvector](./Eigenvector.ipynb)|Measure of connectivity to other important vertices (which also have high connectivity) often referred to as the influence measure of a vertex| +|[Katz Centrality](./katz_centrality.html)|[Centrality](./Centrality.ipynb), [Katz](./Katz.ipynb) |Similar to Eigenvector but has tweaks to measure more weakly connected graph | |Pagerank|[Centrality](./Centrality.ipynb), [Pagerank](../../link_analysis/Pagerank.ipynb) |Classified as both a link analysis and centrality measure by quantifying incoming links from central vertices. | -[System Requirements](../../README.md#requirements) +[System Requirements](../../README.html#requirements) | Author Credit | Date | Update | cuGraph Version | Test Hardware | | --------------|------------|------------------|-----------------|----------------| diff --git a/docs/cugraph/source/graph_support/algorithms/Similarity.md b/docs/cugraph/source/graph_support/algorithms/Similarity.md index 450beb373a2..18c0a94d519 100644 --- a/docs/cugraph/source/graph_support/algorithms/Similarity.md +++ b/docs/cugraph/source/graph_support/algorithms/Similarity.md @@ -15,9 +15,9 @@ Manipulation of the data before or after the graph analytic is not covered here. |Algorithm |Notebooks Containing |Description | | --------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -|[Jaccard Smiliarity](./jaccard_similarity.md)| [Jaccard Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Jaccard-Similarity.ipynb) || -|[Overlap Similarity](./overlap_similarity.md)| [Overlap Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Overlap-Similarity.ipynb) || -|[Sorensen](./sorensen_coefficient.md)|[Sorensen Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Sorensen_coefficient.ipynb)|| +|[Jaccard Smiliarity](./jaccard_similarity.html)| [Jaccard Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Jaccard-Similarity.ipynb) || +|[Overlap Similarity](./overlap_similarity.html)| [Overlap Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Overlap-Similarity.ipynb) || +|[Sorensen](./sorensen_coefficient.html)|[Sorensen Similarity](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_prediction/Sorensen_coefficient.ipynb)|| |Personal Pagerank|[Pagerank](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_analysis/Pagerank.ipynb) || diff --git a/docs/cugraph/source/index.rst b/docs/cugraph/source/index.rst index c5303c21674..955eb6d54db 100644 --- a/docs/cugraph/source/index.rst +++ b/docs/cugraph/source/index.rst @@ -25,12 +25,12 @@ RAPIDS Graph documentation * - :abbr:`libcugraph_etl (C++ renumbering function for strings)` - :abbr:`wholegraph (Shared memory-based GPU-accelerated GNN training)` - - .. - -| | +~~~~~~~~~~~~ +Introduction +~~~~~~~~~~~~ cuGraph is a library of graph algorithms that seamlessly integrates into the RAPIDS data science ecosystem and allows the data scientist to easily call graph algorithms using data stored in GPU DataFrames, NetworkX Graphs, or @@ -39,6 +39,7 @@ even CuPy or SciPy sparse Matrices. Note: We are redoing all of our documents, please be patient as we update the docs and links +| .. toctree:: :maxdepth: 2 @@ -48,9 +49,8 @@ the docs and links installation/index tutorials/index graph_support/index + wholegraph/index references/index - dev_resources/index - releases/index api_docs/index Indices and tables diff --git a/docs/cugraph/source/wholegraph/basics/index.rst b/docs/cugraph/source/wholegraph/basics/index.rst new file mode 100644 index 00000000000..429fe35d601 --- /dev/null +++ b/docs/cugraph/source/wholegraph/basics/index.rst @@ -0,0 +1,11 @@ +====== +Basics +====== + + +.. toctree:: + :maxdepth: 2 + + wholegraph_intro + wholememory_intro + wholememory_implementation_details diff --git a/docs/cugraph/source/wholegraph/basics/wholegraph_intro.md b/docs/cugraph/source/wholegraph/basics/wholegraph_intro.md new file mode 100644 index 00000000000..360f8e0e36b --- /dev/null +++ b/docs/cugraph/source/wholegraph/basics/wholegraph_intro.md @@ -0,0 +1,135 @@ +# WholeGraph Introduction +WholeGraph helps train large-scale Graph Neural Networks(GNN). +WholeGraph provides underlying storage structure called WholeMemory. +WholeMemory is a Tensor like storage and provides multi-GPU support. +It is optimized for NVLink systems like DGX A100 servers. +By working together with cuGraph, cuGraph-Ops, cuGraph-DGL, cuGraph-PyG, and upstream DGL and PyG, +it will be easy to build GNN applications. + +## WholeMemory +WholeMemory can be regarded as a whole view of GPU memory. +WholeMemory exposes a handle of the memory instance no matter how the underlying data is stored across multiple GPUs. +WholeMemory assumes that separate process is used to control each GPU. + +### WholeMemory Basics +To define WholeMemory, we need to specify the following: + +#### 1. Specify the set of GPU to handle the Memory + +Since WholeMemory is owned by a set of GPUs, you must specify the set of GPUs. +This is done by creating [WholeMemory Communicator](#wholememory-communicator) and specifying the WholeMemory Communicator when creating WholeMemory. + +#### 2. Specify the location of the memory + +Although WholeMemory is owned by a set of GPUs, the memory itself can be located in host memory or in device memory. +The location of the memory need to be specified, two types of locations can be specified. + +- **Host memory**: will use pinned host memory as underlying storage. +- **Device memory**: will use GPU device memory as underlying storage. + +#### 3. Specify the address mapping mode of the memory + +As WholeMemory is owned by multiple GPUs, each GPU will access the whole memory space, so we need address mapping. +There are three types of address mapping modes (also known as WholeMemory types), they are: + +- **Continuous**: All memory from each GPU will be mapped into a single continuous memory address space for each GPU. + In this mode, each GPU can directly access the whole memory using a single pointer and offset, just like using normal + device memory. Software will see no difference. Hardware peer to peer access will handle the underlying communication. + +- **Chunked**: Memory from each GPU will be mapped into different memory chunks, one chunk for each GPU. + In this mode, direct access is also supported, but not using a single pointer. Software will see the chunked memory. + However, an abstract layer may help to hide this. + +- **Distributed**: Memory from other GPUs are not mapped into current GPU, so no direct access is supported. + To access memory of other GPU, explicit communication is needed. + +To learn more details about WholeMemory locations and WholeMemory types, please refer to +[WholeMemory Implementation Details](wholememory_implementation_details.md) + +### WholeMemory Communicator +WholeMemory Communicator has two main purpose: + +- **Defines a set of GPUs which works together on WholeMemory.** WholeMemory Communicator is created by all GPUs that + wants to work together. A WholeMemory Communicator can be reused as long as the GPU set needed is the same. +- **Provides underlying communication channel needed by WholeMemory.** WholeMemory may need commuincator between GPUs + during the WholeMemory creation and some OPs on some types of WholeMemory. + +To Create WholeMemory Communicator, a WholeMemory Unique ID needs to be created first, it is usually created by the first GPU in the set of GPUs, and then broadcasted to all GPUs that want to work together. Then all GPUs in this communicator +will call WholeMemory Communicator creation function using this WholeMemory Unique ID, and the rank of current GPU as +well as all GPU count. + +### WholeMemory Granularity +As underlying storage may be partitioned into multiple GPUs physically, this is usually not wanted inside one single +user data block. To help on this, when creating WholeMemory, the granularity of data can be specified. Then the +WholeMemory is considered as multiple block of the same granularity and will not get split inside the granularity. + +### WholeMemory Mapping +As WholeMemory provides a whole view of memory to GPU, to access WholeMemory, mapping is usually needed. +Different types of WholeMemory have different mapping methods supported as their names. +Some mappings supported include +- All the WholeMemory types support mapping the memory range that local GPU is responsible for. + That is, each rank can directly access "Local" memory in all types of WholeMemory. + Here "Local" memory doesn't have to be on current GPU's memory, it can be on host memory or even maybe on other GPU, + but it is guaranteed to be directly accessed by current GPU. +- Chunked and Continuous WholeMemory also support Chunked mapping. That is, memory of all GPUs can be mapped into + current GPU, one continuous chunk for one GPU. Each chunk can be directly accessed by current GPU. But the memory of + different chunks are not guaranteed to be continuous. +- Continuous WholeMemory can be mapped into continuous memory space. That is, memory of all GPUs are mapped into a + single range of virtual memory, accessing to different position of this memory will physically access to different + GPUs. This mapping will be handled by hardware (CPU pagetable or GPU pagetable). + +### Operations on WholeMemory +There are some operations that can be performed on WholeMemory. They are based on the mapping of WholeMemory. +#### Local Operation +As all WholeMemory supports mapping of local memory, so operation on local memory is supported. The operation can be +either read or write. Just use it as GPU memory of current device is OK. +#### Load and Store +To facilitate file operation, Load / Store WholeMemory from file or to file is supported. WholeMemory uses raw binary +file format for disk operation. For Load, the input file can be a single file or a list of files, if it is a list, they +will be logically concatenated together and then loaded. For store, each GPU stores its local memory to file, producing +a list of files. +#### Gather and Scatter +WholeMemory also supports Gather / Scatter operation, usually they operate on a +[WholeMemory Tensor](#wholememory-tensor). + +### WholeMemory Tensor +Compared to PyTorch, WholeMemory is like PyTorch Storage while a WholeMemory Tensor is like a PyTorch Tensor. +For now, WholeMemory supports only 1D and 2D tensors, or arrays and matrices. Only first dimension is partitioned. + +### WholeMemory Embedding +WholeMemory Embedding is just like a 2D WholeMemory Tensor, with two features added. They support cache and sparse +optimizers. +#### Cache Support +To create WholeMemory Embedding with a cache, WholeMemory CachePolicy needs to be be created first. WholeMemoryCachePolicy can be created with following fields: +- **WholeMemory Communicator**: WholeMemory CachePolicy also needs WholeMemory Communicator. + WholeMemory Communicator defines the set of GPUs that cache all the Embedding. + It can be the same as the WholeMemory Communicator used to create WholeMemory Embedding. +- **WholeMemory type**: WholeMemory CachePolicy uses WholeMemory type to specify the WholeMemory type of cache. +- **WholeMemory location**: WholeMemory CachePolicy use WholeMemory location to specify the location of the cache. +- **Access type**: Access type can be readonly or readwrite. +- **Cache ratio**: Specify how much memory the cache will use. This ratio is computed for each GPU set that caches the + whole embedding. + +The two most commonly used caches are: +- **Device cached host memory**: When the WholeMemory Communicator for Cache Policy is the same as the WholeMemory + Communicator used to create WholeMemory Embedding, it means that the cache has same GPU set as WholeMemory Embedding. + So each GPU just caches its own part of raw Embedding. + Most commonly, when raw WholeMemory Embedding is located on host memory, and the cache is on device + memory, each GPU just caches its own part of host memory. +- **Local cached global memory**: The WholeMemory Communicator of WholeMemory CachePolicy can also be a subset of the + WholeMemory Communicator of WholeMemory Embedding. In this case, the subset of GPUs together cache all the embeddings. + Normally, when raw WholeMemory Embedding is partitioned on different machine nodes, and we + want to cache some embeddings in local machine or local GPU, then the subset of GPU can be all the GPUs in the local + machine. For local cached global memory, only readonly is supported. + +#### WholeMemory Embedding Sparse Optimizer +Another feature of WholeMemory Embedding is that WholeMemory Embedding supports embedding training. +To efficiently train large embedding tables, a sparse optimizer is needed. +WholeMemory Embedding Sparse Optimizer can run on a cached or noncached WholeMemory Embedding. +Currently supported optimizers include SGD, Adam, RMSProp and AdaGrad. + +## Graph Structure +Graph structure in WholeGraph is also based on WholeMemory. +In WholeGraph, graph is stored in [CSR format](https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_(CSR,_CRS_or_Yale_format)). +Both ROW_INDEX (noted as `csr_row_ptr`) and COL_INDEX (notated as `csr_col_ind`) are stored in a +WholeMemory Tensor. So loading Graph Structure can use [WholeMemory Tensor Loading mechanism](#load-and-store). diff --git a/docs/cugraph/source/wholegraph/basics/wholememory_implementation_details.md b/docs/cugraph/source/wholegraph/basics/wholememory_implementation_details.md new file mode 100644 index 00000000000..a5541109c4f --- /dev/null +++ b/docs/cugraph/source/wholegraph/basics/wholememory_implementation_details.md @@ -0,0 +1,58 @@ +# WholeMemory Implementation Details +As described in [WholeMemory Introduction](wholegraph_intro.md), there are two WholeMemory location and three +WholeMemory types. So there will be total six WholeMemory. + +| Type | CONTINUOUS | CONTINUOUS | CHUNKED | CHUNKED | DISTRIBUTED | DISTRIBUTED | +|:-------------:|:-----------:|:----------:|:---------:|:---------:|:-----------:|:-----------:| +| Location | DEVICE | HOST | DEVICE | HOST | DEVICE | HOST | +| Allocated by | EACH | FIRST | EACH | FIRST | EACH | EACH | +| Allocate API | Driver | Host | Runtime | Host | Runtime | Runtime | +| IPC Mapping | Unix fd | mmap | cudaIpc | mmap | No IPC map | No IPC map | + +For "Continuous" and "Chunked" types of WholeMemory, all memory is mapped to each GPU, +so these two types are all "Mapped" WholeMemory, in contrast to "Distributed" WholeMemory where all are not mapped. + +## WholeMemory Layout +Since the underlying memory of a single WholeMemory object may be on multiple GPU devices, the WholeGraph library will +partition data into these GPU devices. +The partition method guarantees that each GPU can access one continuous part of the entire memory. +Here "can access" means can directly access from CUDA kernels, but the memory doesn't have to be physically on that GPU. +For example,it can be on host memory or other GPU's device memory that can be access using P2P. +In that case the stored data has its own granularity that shouldn't be split. Data granularity can be specified while +creating WholeMemory. Then each data granularity can be considered as a block of data. + +The follow figure shows the layout of 15 data block over 4 GPUs. +![WholeMemory Layout](../imgs/general_wholememory.png) + +For WholeMemory Tensors, they can be 1D or 2D tensors. +For 1D tensor, data granularity is one element. For 2D tensor, data granularity is its 1D tensor. +The layout will be like this: +![WholeMemory Tensor Layout](../imgs/wholememory_tensor.png) + +## WholeMemory Allocation +As there are six types of WholeMemory, the allocation process of each type are as follows: + +### Device Continuous WholeMemory +For Device Continuous WholeMemory, first a range of virtual address space is reserved in each GPU, which covers the +entire memory range. Then a part of pyhsical memory is allocated in each GPU, as shown in the following figure. +![Device Continuous WholeMemory Allocation Step 1](../imgs/device_continuous_wholememory_step1.png) +After that, each GPU gathers all the memory handles from all GPUs, and maps them to the reserved address space. +![Device Continuous WholeMemory Allocation Step 2](../imgs/device_continuous_wholememory_step2.png) + +### Device Chunked WholeMemory +For Device Chunked WholeMemory, first each GPU allocates its own part of memory using CUDA runtime API, this will create +both a virtual address space and physical memory for its own memory. +![Device Chunked WholeMemory Allocation Step 1](../imgs/device_chunked_wholememory_step1.png) +Each GPU gathers the Ipc handle of memory from all other GPUs, and maps that into its own virtual address space. +![Device Chunked WholeMemory Allocation Step 2](../imgs/device_chunked_wholememory_step2.png) + +### Host Mapped WholeMemory +For Host, Continuous and Chunked are using the same method. First, rank and allocate the host physical and share that to all +ranks. +![Host Mapped WholeMemory Allocation Step 1](../imgs/host_mapped_wholememory_step1.png) +Then each rank registers that host memory to GPU address space. +![Host Mapped WholeMemory Allocation Step 2](../imgs/host_mapped_wholememory_step2.png) + +### Distributed WholeMemory +For Distributed WholeMemory, each GPU just malloc its own part of memory, no need to share to other GPUs. +![Distributed WholeMemory Allocation](../imgs/distributed_wholememory.png) diff --git a/docs/cugraph/source/wholegraph/basics/wholememory_intro.md b/docs/cugraph/source/wholegraph/basics/wholememory_intro.md new file mode 100644 index 00000000000..7209da9471c --- /dev/null +++ b/docs/cugraph/source/wholegraph/basics/wholememory_intro.md @@ -0,0 +1,123 @@ +## WholeMemory +WholeMemory can be regarded as a whole view of GPU memory. +WholeMemory exposes a handle to the memory instance no matter how the underlying data is stored across multiple GPUs. +WholeMemory assumes that a separate process is used to control each GPU. + +### WholeMemory Basics +To define WholeMemory, we need to specify the following: + +#### 1. Specify the set of GPU to handle the Memory + +As WholeMemory is owned by a set of GPUs, so the set of GPUs need to be specified. +This is done by creating [WholeMemory Communicator](#wholememory-communicator) and specify the WholeMemory Communicator +when creating WholeMemory. + +#### 2. Specify the location of the memory + +Although WholeMemory is owned by a set of GPUs, the memory itself can be located on host memory or on device memory. +So the location of the memory needs to be specified. Two types of location can be specified. + +- **Host memory**: will use pinned host memory as underlying storage. +- **Device memory**: will use GPU device memory as underlying storage. + +#### 3. Specify the address mapping mode of the memory + +As WholeMemory is owned by multiple GPUs, each GPU will access the whole memory space, so we need address mapping. +There are three types of address mapping modes (also known as WholeMemory types), they are: + +- **Continuous**: All memory from each GPU will be mapped into a single continuous memory address space for each GPU. + In this mode, each GPU can directly access the whole memory using a single pointer and offset, just like using normal + device memory. Software will see no difference. Hardware peer-to-peer access will handle the underlying communication. + +- **Chunked**: Memory from each GPU will be mapped into different memory chunks, one chunk for each GPU. + In this mode, direct access is also supported, but not using a single pointer. Software will see the chunked memory. + However, an abstract layer can hide this. + +- **Distributed**: Memory from other GPUs is not mapped into current GPU, so no direct access is supported. + To access memory of another GPU, explicit communication is needed. + +If you would like to know more details about WholeMemory locations and WholeMemory types, please refer to +[WholeMemory Implementation Details](wholememory_implementation_details.md) + +### WholeMemory Communicator +WholeMemory Communicator has two main purpose: + +- **Defines a set of GPUs which works together on WholeMemory.** WholeMemory Communicator is created by all GPUs that + wants to work together. A WholeMemory Communicator can be reused as long as the GPU set needed is the same. +- **Provides underlying communication channel needed by WholeMemory.** WholeMemory may need commuincator between GPUs + during the WholeMemory creation and some OPs on some types of WholeMemory. + +To Create WholeMemory Communicator, a WholeMemory Unique ID need to be created first, it is usually created by the first +GPU in the set of GPUs, and then broadcasted to all GPUs that want to work together. Then all GPUs in this communicator +will call WholeMemory Communicator creation function using this WholeMemory Unique ID, and the rank of current GPU as +well as all GPU count. + +### WholeMemory Granularity +As underlying storage may be physically partitioned into multiple GPUs, it is usually not wanted inside one single +user data block. To help with this, when creating WholeMemory, the granularity of data can be specified. Therefore +WholeMemory is considered as multiple blocks of the same granularity and will not get split inside the granularity. + +### WholeMemory Mapping +Since WholeMemory provides a whole view of memory to GPU, mapping is usually needed to access WholeMemory. +Different types of WholeMemory have different mapping methods supported as their names. +Some mappings supported include: +- All the WholeMemory types support mapping the memory range that local GPU is responsible for. + That is, each rank can directly access "Local" memory in all types of WholeMemory. + Here "Local" memory doesn't have to be on current GPU's memory, it can be on host memory or even maybe on other GPU, + but it is guaranteed to be directly accessed by current GPU. +- Chunked and Continuous WholeMemory also support Chunked mapping. That is, memory of all GPUs can be mapped into + current GPU, one continuous chunk for one GPU. Each chunk can be directly accessed by current GPU. But the memory of + different chunks are not guaranteed to be continuous. +- Continuous WholeMemory can be mapped into continuous memory space. That is, memory of all GPUs are mapped into a + single range of virtual memory, accessing different positions of this memory will physically access different + GPUs. This mapping will be handled by hardware (CPU pagetable or GPU pagetable). + +### Operations on WholeMemory +There are some operations that can be performed on WholeMemory. They are based on the mapping of WholeMemory. +#### Local Operation +As all WholeMemory supports mapping of local memory, so operation on local memory is supported. The operation can be +either read or write. Just use it as GPU memory of current device is OK. +#### Load / Store +To facilitate file operation, Load / Store WholeMemory from file or to file is supported. WholeMemory use raw binary +file format for disk operation. For Load, the input file can be single file or a list of files, if it is a list, they +will be logically concatenated together and then loaded. For store, each GPU stores its local memory to file, producing +a list of files. +#### Gather / Scatter +WholeMemory also supports Gather / Scatter operations, usually they operate on a +[WholeMemory Tensor](#wholememory-tensor). + +### WholeMemory Tensor +Compared to PyTorch, WholeMemory is like PyTorch Storage while WholeMemory Tensor is like PyTorch Tensor. +For now, WholeMemory supports only 1D and 2D tensor, or array and matrix. Only first dimension is partitioned. + +### WholeMemory Embedding +WholeMemory Embedding is just like 2D WholeMemory Tensor, with cache support and sparse optimizer support added. +#### Cache Support +WholeMemory Embedding supports cache. To create WholeMemory Embedding with cache, WholeMemory CachePolicy need first be +created. WholeMemoryCachePolicy can be created with following fields: +- **WholeMemory Communicator**: WholeMemory CachePolicy also need WholeMemory Communicator. + This WholeMemory Communicator defines the set of GPUs that cache the all the Embedding. + It can be the same as the WholeMemory Communicator used to create WholeMemory Embedding. +- **WholeMemory type**: WholeMemory CachePolicy uses WholeMemory type to specify the WholeMemory type of the cache. +- **WholeMemory location**: WholeMemory CachePolicy uses WholeMemory location to specify the location of the cache. +- **Access type**: Access type can be readonly or readwrite. +- **Cache ratio**: Specify how much memory the cache will use. This ratio is computed for each GPU set that caches the + whole embedding. + +There are two most commonly used caches. They are: +- **Device cached host memory**: When the WholeMemory Communicator for Cache Policy is the same as the WholeMemory + Communicator used to create WholeMemory Embedding, it means that cache has the same GPU set as WholeMemory Embedding. + So each GPU just cache its own part of raw Embedding. + Normally, when raw WholeMemory Embedding is located on host memory, and the cache is on device + memory, each GPU just caches its own part of host memory. +- **Local cached global memory**: The WholeMemory Communicator of WholeMemory CachePolicy can also be a subset of the + WholeMemory Communicator of WholeMemory Embedding. In this case, the subset of GPUs together cache all the embeddings. + Typically, raw WholeMemory Embedding is partitioned on different machine nodes, and we + want to cache some embeddings in local machine or local GPU, then the subset of GPUs can be all the GPUs on the local + machine. For local cached global memory supports just readonly. + +#### WholeMemory Embedding Sparse Optimizer +Another feature of WholeMemory Embedding is that WholeMemory Embedding supports embedding training. +To efficiently train large embedding tables, a sparse optimizer is needed. +The WholeMemory Embedding Sparse Optimizer can run on cached or non-cached WholeMemory Embedding. +Currently supported optimizers include SGD, Adam, RMSProp and AdaGrad. diff --git a/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step1.png b/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step1.png new file mode 100644 index 0000000000000000000000000000000000000000..b8a0447e6fb9dde5691c987aa734a6ba4ae24663 GIT binary patch literal 23136 zcmeFZ2UOErw=WuWyZJVv=$7UdL~tuei-3T18#a0g)exjgml7aA01K#~6p`K(AxP*o z5Mscs)X*eC5F{uN0tASJ5<)2NAC&#=citWEoIB2akM6JdGYBcRuKoL-Z`c1N z+x)Lzgr7F!V1Y-=YXoI)1U^F^@yoV9+}VI=@Gfv_6Os%{{*Z|_A6=27pH^7BY-p-V z>Zx^1D9W@&IlM0u-9MQ;Xsj}XyZBe@H}Ruq`NW?-w4*Ng7&2#@B38PVIsYb>ie|M& z{ZXOV(5bGVkX2F>`Lot*fXp8+;mW1X+}$^z*98e2zIR^2T2bb^zc){x$=|d4=Bkmn z!0zv7LwGpy*m*^MG`r5>f9Lfy=aTg9@Ak|5rd+#kUKjqKDgNgL^)`D{snZu^)v3kQ zG7j5V1q)%w;Z$<32HrG`u>_M_z_Rh=Okf@EX22c)KveoSF1!`W=@`ueLe#k{gddH zgyWt*jnEMRR@iC?$){=cWjS#qHVC<B1uYLi#=@a;TWt`qj0;s z_^H~4u^RzO%9nJR1N(U;6{s@$qO|Z<+A>vZ52@H0Y z_)KiQYcg^^nusy5*<(~qN@08%e32SPd>s=jANJw%YVyba-y^$YR2Co}%<@tH2wy6x z$=8t0j33)dzOjGJZG4lRfwa^=IDQcKmBFxB&v#2PQrxI|%bQ98jh<+`(Gd{2^?c^s zDLU7>9+~eEfN1y4b^7*=#cH4J3yVE`5p_7qEpXUyf%8a5A%A%o*(^p6jIY=@075Rc zG%lPWuO#*g($$x8#N@_x){4L>O=_g#9vpRhf`oe$JRhW8jf;~1AMz&#@OL-v{0POB zytOqxib0m)DB|GY*#cKvJ#E^J)EAgNXV2xXRclBh4$V*KEYCa=YkxZ)<8vl_P8pmt z$u)YCz}0FH$M*^hJ?iB~EOfAabQ$ zLosu4-1a#}n=EKcFMi?yBRrNf&k#L)d3Bs_x2@44sk1%6NYUD!s|Cx1Z?Iven%@-S z<$bs!wQ`%5Xe~zDt?XDwpHLV;jH~9NLQ4dGb)bc*$?TxWdES@hwmR<9mnoc*bq~T9@_($#*-_0^o=`<%Fi0Nn3Jlw!r(Tst$2WxtXdjKfq6p?+I zythIY(S)FG7jv}y>Nsp`I5Z}LLBPB)(_QQqf?D0$=EE1Rsp>^pjZoI{SlR8Uh+z0)dBmMufIWFN zvZ$hHB&2;gJb?)+xkUEKi351~{r3809p*@jt#?Byu0RSuF_L4o!HD0Wj5in|ZXx!A zWT#%|Zu%_-g>N+pMSO3|1^b+dA;A#3i!wUvmN84cBWHYG9UxY1SoS!vFa$qP%8542 z`$aX7SJKSrNjle^OxzeRM`*q@QCRvfv`}yMErgy4ZlN((S?x~fC&2z-K+S%5-?6<( zSqK^(na81m7UPpX*;Ne=^+fD1enVi0eF0SFw_ zC70*wvALc$Khaq?U)1=e4AEvV7EkPDS2nGhvwr+XL z)Uo=qQK@kngIv{%F!^$q;MEcfh;-cA27d zJptfB644Z{Zpy`bOQZ%#-)4<<7HMshw%5_)V)%6gg&$Ny)7k8~76U9kUAAx)Hv!u) zYI&8aS(@c=xJo3_;Q0jYB#~?A&vd*f<2dk^Nqd#RW8a*I{GNPm%VUumG^&)2-{_A$ z?Ij9Z*(lUUaoI!3tZ%)B!}}wqVquy3I$PuWV{PNMA_xE4|C|Eyd?4bJtIutJ%xT@F zBNSA3k&H3gz8-lmtXxr_JVfFj6~qkSP>t< zR@>V;txo}htkVS^6&Gl}u-VtU8Mv4;qHSp?kfa1#)uP@2_$q--8crIW@;k$L;^L$R z8u_zrgu}?`bo^0(5wGXWdG&2rwn^OHvxl}BB(mAYUmbkOW}zhe?R)g~IzJwe3iYT_ zn7%Jd6=uGHVQ$ShtzXXF>Yvn4+Y7D_Q#8=BK6pW>eQ}^80EE4ODo@36zuZ5G32-?^i(=bZl($BWn4q+5Z;|Y547r-8-p-{PNiDqHem6c9KdO+vH5yO0)7)GX z5&|PzkDMlE>80%LJLMOK0KhMwI71Vsb4~w_7&{1Ha_{fA*HpJwG=MRKM}u^VcYp%W zSw|fJ1F)HP#or2BQ-Wpab975h?a0)OQGKvt_i+xL9g8g-IxMHo5v~8#c;U8P{GSZ~l)PpL#67>>|lG{t$!K>|lAgq3zsw9zXS}~n`5W#mL zwSzq%mA^;$F<>+r*Cwe=>PNSyV&$`bnS3pOK0@&T2zsc2pT(~xAc2u+8UpP5w4xe0 zOP~8C!6`&zQVRsNplnWT^}jw%WP~dKKKs|NJBEI3WUC@eIwoT2GCPKe@m2MUKe;LV&d|eg-TiQk!kF=C((eY=~rAh0_;h{lfm3^?-pa72h^!(vf0(7hoG}%qA-`_=32=h4{_65gp?=ftL8UQ!On=z$_Gi4=nw+ zc*Ju0Mu>?T+`H^qOM~@RUWYbUq;|pt6>yBp$@)GEUvNyMCTLTfF7!Hq=VhVyMxSd; zQ21IkO5eTN2(sUD4ZX9kul3+r)C@~vl0WlD|#U5=uqdP%fPPCH#0< zdu4m|A(l$C(rqU-Vt^W932yU2dTFpI8Oz2 zlF+ddvPLG}r*((CSUew-6r z81Xh)d%9{vxyG%MJ`R6Tc6qd7cF9OpHwW@rP)#+UD(jY5RakP?9VN$p=k8%{mZQp1 z2eD;qtX%bK_vsc7w2fr6hbLzxCdW519li$2vLUjgP6;!LNNWQoJc%T?d2UKSSC^a= z%WT$NawD@5qng#fBF46HKK?pj2|Lx93BzNPn|C|dtbqJI)9E|*sGGUz2G}Fgt)ge& zhg#DAU^D69vc?50BgJKW%PR+RmoeGV5%OU@z#1FwdeFSHa&`H(Sb)ijdK~Ga=Hl?P zY6szzbG;H&33~L+l%ym|Y{O#fF+yeD`LMOSYJ+h@+=?iz0gudY$IRjiozxrJ6WXb# z+mw@~BS!utIMs6O2({4!0$%;rqlL3C<$}I{eRm_VugI*h=b(ZprZD4p{f97C)oh9o zcI%z`D2%;2FVi+fU|z(E`5lxOOclemgpZDnYH57fGHkh+h18TybWQXS_mgLMzj0DK zV~tN+>sM1e@u`$8JG0t5%#DBSb#3lFmZb9j9Et-p(?z_cvpKpmQ}cl-YgtwiiYmX1 zGi2P494f4gD4t+okzbZ|9A@G<47rh;z7~zi(y#R>Y&ZbHcJ_JS@wxD)$jop)7$%;w z`MsU}YtP0-P~S=iL~8jT!YS>w z(i6Y}Np~~EHe1Hze4L-ol1bPsO7Y|s+oMTkRLxjTm+&zJla3D9bk(BL%$XtRU!%Qb z)pGp83=0;Jfz0fOR0an;z<$dg2y9 z6pri8e9kl%c@H)uGXb5!SBky|-{iO`PF5h*QFn2UaNPD*$YKLd z{g=2lpYr^NzbZ7tf7!uQ`xt#ft73oKj`Em=Uos=Xd}O_O+olXULZ#!r0bYNEzz%*I z#;QR7&C8b%b~a^*9D!FfWTiTo@aNY%;o;ChACrNLked5Fe9e~wTw_Kin6V;a{r1&)Rf3phoPtVKm>LcHXS2k3 zL#%9e3P#*JUVI@*heR|-2olQJPY=Z(c`rf8Iz44XYV0P#_>GlkoZj6%pw(s;*ZUi5 zvAs^iY&lAw9*%mSric_o7`2a;za3LKNgwH8_f0Vu38v?@^cmsmocQCp#a?zVX>oQo zJp!t8Qb8-tnsm=Q5dp2|LVG9o^ClMNxtudwMep++>yT_6Dh19_fS_ zc!v(S2JZ5GVWDhW-~Rhg`C}R{p$NnuE zov3qvJkr0n?EH0izwZ`L5JczJ;PATwy#o@2l24*FwJc3ltv7b0cdDZaQ?>x0#0kGu zh55RPGt{?|&iYk$thRX#xIS4+rv~d&bCnrjF=~g7*oWXPXHViKnz^UrmRMP3*d1aY zC%)-0GM?%8jRBN`_Pl9f3Hjmf(=$ZZMCLDT-(!=Q#Z;sfleq|Y7#q}*fB!=8w|F%= z4T!3z6OF>RiYaMI_R}dF7f$FH57jNty-?}xdh#GeGqz0tkU*2YI^((9^aNOStplbn_8YopQ$TM|6;y{7;jHJNfEAbo#y!VfMgCA zFSZL;O6c0t7zJmJQ!IPVeBaBqdK7@SBC9vW#~-v+_<%PMQ(Z0H<`ymk5_iUl-_w@x zaXYq%w!rSJ%&VWwJPwIda53P!GX6Dh{X`ZY>!M{gmhch6Em!$;i^~>ClrVzfK=tR7mgL-zV;SD zU!RNBJS^L6yETk(8-+1ixD0H$g=X$qgxljvmM=6d*y9W>4-N~*pBwA{a*W5ir*2t| zP%QG4*CvScU}#mvPkt;SHt>E)Qbgs*q|V3#327C#9jSZ9>ib+pVPpmskk4BDrm$+8_ zEh<)*YaoyNoUbvz>*1Dn*F%g&hz9$!2j!Zanx$R)FC1~hA6S$Xw(gN`O{2I`AHZFV zYZhTBVQlB3CV{v@My-`3Zawmt9(GlZcHSXP*5e4`xD!_2$822;`${j{$SVE^C?K8^ zOO(wDhPEe-y*8A~K?JONk#%xa=>4DcLLlwlcM{epsdhJXc^^gCF(u-Oia{3Zu&(w% z-_$!>S&z?qQ{j5OjK}a>x?N)2sDT8qPvV18tMc<}Z+av;bKBCq*T0mRl{n&$L#4jY z-b_47OZ=&)00utyGdCc;PS57`sx=`&7IfJ}_M)>#@&s$Cewr~Tz6#yS%@N-W`^D>6 z$WGXY%HH3v*}HTHvJ)(AA+Oq#xOw(r%zMmOLadu_Hh6Yt`f&2M%f z372cgabaJO^i7JDiROBu&2RCLOm3(Fgxh82o*u8SZP**Tm6vy*{otz9h8~3cLH%O! zIyI#HxP5U)YMO8E+Mb$)ml`RS2cX87INuR0z63h;vOQNb@xjx@Fka|jb%>>Y3lQDS06B^ay_2Q4gk~PrpqHCZ zMVIOEB@9;L5i)G7(iJnvVTkm>7~+GkfVFIt7zAtGiSfEa``Z0Z7)ZV2(gT`GUMC|x zBXTY;<;1S-QRyyDYg)gr(Y=_q5M$N0yq{{dje?!udls7W8oqrIW9XJcrS&T-z7-v5 zU*6E*O<+M z;oPg9)(&1d#C>HkERc2L{Gh@h_@I=TS2hdVbrBx&y)o{AUnKn4%@pwSUV^8idM~!8 zu6xOg9C4otYi`|uY8^RmPD_h$c``rLnpy$6oFvA#y7CqRxEggtqJ^e+YEu))N~rC7 zcRM?w`Uca8El>;5whfRLA~=jIc-_c2ziWlH1PryUP5_ zb6OR&_d*E+7i~|WWN7PEfn>#{OSNTUYkTyS9?JGEAyPI7gwtq5206l-6>;AvRGz`x z+4N~HV^(kcrjrX63?1jQhqEfD>ahDtZUQ9T!o?A(>gTt9ZIwliJ!4*L`_3mxx5PZ7 zPu$~_uwFi5Sj-|>hqA@4bkCAsWCXd~$PUPX3&|F6pA+{%j#;w8V`rAtJRLQbwu94{ z`|gIg5V3rnT!TgN);UeCs@+r+BKIcMI*9phL+9i~gOy8mBg(s}puF*wr)0AVrS1AF zz9_WfTUcG<3~Mob>n(Tl(D~?G12V?($XFZLA^pgg+fp3(@C_iXkBQ}FX8mRsWbq9( z$!!A2gMLDC3^lZ{5h;;+{KJ&%o07{VMqB5LkK^FAx&h%+J!n@LrI%6y7H5zacB!&U zZK}i*$(YL<^4nD^+k3D*r{#Z3=v*qD5$Ym#4NH!^;H=!-EOyVb?r22G!ra`skcd%> zk3iPK`)PVk#PMfsMWNlGr#EnX>e0D|e2E?%ZO*lh9?}*%{rz~>7Og{mD+IHA0r9n9-ed@nCw87 z=3d&YHbj^NY!x_crAurh3%nZ}%}-=U9QS{s6FG3DYyeJsBK!DDPD&!Y4Vn2TQA~m; z_Iod43mY(ZprTD7ObbZb&MXPD)pM#LA6%oY3|FuQkn%@ixs&W;zxn{XeyOd&-dxd_ zep6qaR3(s~zjZzO?stQFlkPut*$?5N(eDga)VC#>e<{R9%wZ>_QD z#ccD-;i1~uvGAL8B_nJXBsH~s49%c)AIrcAPM3>QM-wp5D01t^)A)$wBSdyhIt5{O zPIR`a&=9L*foOuAlf1ljYcd|jyw&10kT8~GNb=HKXCrz|x^l@8$rK8~eQ71pW=iel zcI$Jv<7dMR!kDd)sDvu&2Awn&8n~rB8jx07R&;JZ&9At~nc-x2klGY4?p++cu^JGP z%zKdK;mjGuhO&|f_V{@3TOxdwYh_l^De=_;QYW8jP1)2;rQRWm*Hu3Wh|nLYJIz?} zbg;Wh(z5imRi4BY65%7yx5CN(FR1RL48hM+VVfz*<(5g}$1}e@hpzy6hRB>)rQ1DW zU*!ukVi7%IcgQz229~lPYrR^`ehyfY%;RQr-=j`T>z8%2ew&&zcbT3YVCbgDm4 z5E#-ySWFRyQA6c3f%>!;zg_SjShUMIvh-o|Br$W!DHHIeMl+_r_31l3c3gYZ&hEYg z_|M=tLBysv&0pFx6R<_Xk>`bD#`D*Vk8%IXdUx^h2>L;7y-!#5%-&4%ur)(cgCQ1g zKvKq^EAez>J2asfU!n|K9KCzu#KXu`hnGJ~VY}W&Ihl zs+%O&j9r!^6l-R#;ziLb&)CYF{c>`DXy5&ML9Nc3wK@qiY^}>bYIOsnx@wf1OkC}) zYYEIRSK3DKRA7VY<$e<6?LrD7Cz0hldAcJY{rGsW_9mY*$i<(x=^EL`ur@pyf{*Wo zH8m{PHg@+GRTyZvZe7nibqu6cv4o`o;nWb4R!b2E@e&f23{&==4RGvGPe0h)*P+e^ zMSe@#S+d=-{Qq7>{kOG(rt@=i!o0s;F6SJ};Joa!2FZ!;K|v(@v%(I*d$8qWqDcWJPxWaYYmtzu#k(a(ls$cWJa+?Sz zL4jIOOQmDmaSMB8y%G;;7#)&2X3oDCqH9aD+Cmku;ZSzTbM0$|hS}0(*jaSPntIJ! zD2wJ{7NMHukwmv93?oKYC`0L_4%>3Mwv1xCMta6ZBX#!7NSN6w{`@l*bEd4TMv`D-F}5KZM)-O@tXtdIK^#4-;J;2dVOxI99+5T<2ZA1 zu#@Bhn*?Ci1(l9H{T)C|me)k5JLh1dcm>Tr3GNwllKAM_G?62`QII~3R58DeyJW%h0?fQz)}wy5?M|ty$;B%F^Cul`Dk>Zlfut9}z`mp-C)i z?|u;B4|DN>dwaP%E44m;D|)i=97)6dDMB+LwG_`R@$Dk~fposj6eFDFyorLp&TC{) zEArk8$+|$Z(_0mFK%On?8cNR);h^aXINaqA!Um}C6m)v-6P3kfFZwIwx`5zY94!1s zJ?_{*Pmkfev84Kmfl)~w(5o>B~Cej!knAP$rJZ*5Y{H<6CVWW-P)$&}u*rfAT*{Q!~8V#D)||Dp(?zx#NEo=uI7dO zNjdwBx(z_;Ll1F%E`t--#Etl;J@wW_t}Ys!zx%a$()EsMjajc^byGS#M$t!8$_dv4VZ4OnA*ka}Lr3*)OH9$K+PxXC~SLR@gT~ z+};}K<-SKBDk}qagm7K+Ma==P2;yJ4xX02{#kO`^A$Rg@yzyPaz5DkpdBq|a)H<8^ zfRS+p1~OHL1TUYTT}jm8EAG-`|InBM?1G*`IiL+YYLq&Q1?v`~+(t7@chpGdry70Y zsFC~Jjuuflk9>4cGgdvd|(1hA|5#Ae7R`%F^c!@tsXXyi$qH9i^(u*YtDtM){ zLJIfk4`{-By}E>&^hl?82KuQgW2lWv2lW)u^4u@dOS|XhN=8oXFS^9d>eiruTaa4(BHG!coA zEsSY82Igw*Y&Tq8)T8g1D+a2|+n=4iuBihG&=*tJSV4w^hN9&I0%w;tKotic$mM4Q zA(~F7uGm7Lq}*5SOj3-)$5LiDpy zgm$r@L_?>4=%zO?Sm%DXeuo5Hj9$rRmwgO-y0w5Ba-soK&G7EstlyNan{Mgv$5K2jLIE( z@UyuqsuEbTaD$k;t@1ntdF`jL$}M=peh-GcVJ3eRQ06j{ z3AdSKgeTX5t?RD~y@B7sjvBO#-GqL-4dGWVZuU+_>>fQF^y-T;zZNHQrftJQF{ zvRI=5ft$J}#>$>%cK=c>oiGaz^|bUBP5sleF}O%>5!XUPeiYxx^- zZVJ0)cb#b?pBpm@IUJ5*6UZf2^P-^mgXHisF-^4_ zK4`bw!~ceP&F$||=HhavZ@^hq;k4P~pIl86YTOl!M>o8Lj+MS93{=I5sF}vicf?YL zQ-tK@9g-2&-PoVh>`0RVNYjJ{NOKjS<`@Sh?Ep%!PxBI_>4(f`7|xp+g3I~37ktwj ztpNmnva!r_#>$r3h@=SpsmuKusL^(dGA?0pX;Kl$58*btTi6(?vn{kJ1)6Hjw05?w zs}c84KW0oE)eEyj^?sa#4Jol^Ta>~pN(Uldw~ z(C-x5QRWl40Y<0w^`SSzThX zaE_Ad>JU?narA@hYR>p&bf&(&BYulKFGIfxDRmv)>cAk$+9es8>m?;}=cn@pF@}oWgQt?JRw>Y*qTC}JUW1^(12g{#e4J|9XbSG1T zx?+Pa$yQ5;qiuWZwd#Trkw@b-yu=FFO5kOiN3)1DjJCoDmBIOm+1kU*4yK8|CE?A; zwSSsHjYbpe%G16LWyFV?^3)Vf3NhMe{= zbS)Z~%B}U`vg%6cz3&FFJk^7XGZ)A%>P*z{-T=N_;v(x>?d`5Xi&KWD*!YIG(Lv9i z{^ahcIVlmHrvzF0&t`(R>YXd^vqq1yvOn_dGNmOO6p$i)ZR*b1Cg4zG@lnF(lFZ>A z>$VJaGdb0!iJ(ZA*5@KGrIPVQ}IB?L~7-m zmd614>+I9inv)P+j_kmhmhD=Rg1ju*w7NPkZ^?bx1)Yg%?o+vqK3r*TKSWo2D~2P| z0*lP1U}AETSk-a+x=N>YSugqn?1VvfK}s&L0cxI|L~8vT6OkW@i99bqJz|zXitiUI zotsrRg#@5%yg4g9tTb6|KRjGl?FzMcRaR}T_T1Vb+_h%U?ot)sx~Z}AZNGowA`Ol! zQW_Xv(g^4Ne**Or>j=cIN}@bshC?L&NzEAs~|Sl;SyP;N_aks@{=; z`*{vv{zhTLagJ)}$?0q_J_yxuR&QqfKO`CRWTGuzNR=Gy=*w_2{S~GjIA>J}?6b$w z#bwey_iq1xse1$MqhU+kEw3&=ilLP?meoC${y<-)kLfi0?bmn{YAIJ82I$= z0e390NqS*+RueE*Hnq~{wUU$zt3SCf^$ue|vM%ke?`mqjT>V|`uAyUk%mFF6aHMCU zlu~W2TAK|Av`4S>JV$%3{;fS|8pj@>Sv%@%NG1^z-9N+~*RqM5e{!j){^=NSKIio1 zr)|b+oLtv#f(G(%lEY^t~Xq`*(? zUzO&0ZMr1?x$yCBTT7%kKx;mbCc)CmWD3naE>Hh3k^@J<9|4QNv633E1zu_ar%OUzm1{kzgQ$Q}3_H7&<{K3by-{yBewOG1xn@jvEH;~b15RJQ1vScI(=M;{_+ z1Wfm-VcHHcp_7&4coxI)u`?K3V&2s_yAGkHfF)7ID|>$iCNIST^8YB3+B4h~5doao zYKs%KU1uOo&b|o5s0V+y_hn4Z?bmEWaw4f)@Bs)#j@p^3rYGw`F8J-T=Ms>qpqANk z&br8ykKyL?ZGTGG)A2e-i&Jd8wyF?OU-sU4_rsc@^>csn`Ke1MXl&0};twfXH1O}} zL8(f_B1%S5$4RT%^Jh*4-j>PH4-P5@I@lrVrq9gnhK}f7Bm@R^LZWV(Troe;P;bQV zXAs(T%m-H~653uvX{pNF`$-n^&C2eIvR8Wb2gd;c69Zd}^~i5OxxSz}eO=jsh?VZ1 z{a{{Lb`k;wykMNM@0i9J&Iq! z^dWyByk7nCYv6HRS2TrVpm$eOBXrPNV3xC>>8GZ28hrr8Hf{A&p>Z56!RRVWypWE+tvat z9ob32wD}c6TPuJK?%W|-x{!B^-|QgRyH}^ao0hTLCDpttmH$VBlyJ=bm+FvG1Jt>-aH8L4s}rEym8!sko0Kq#4{k9=$8Fvs-w}Rg|d;+PRDqr?xqPq zgL>iut_Axbh{8fCQyXf*jEjJ93ms4+&G3MBbaYzJ{sNbD(p&e#5KI~3@^A|eHlOdG zVV)a6Y5H!McYqW0qky0v%5S=O*2EYR)X^k`a;GUq7^ti3azXg@bQMekN;@?STLPd$ zX!CV;P3S-#3L1XsCx9Yv4}E?nc*Wy)QN&<R z>zUUv)nzo(Y&U3HSCi_tjO4aTDBmykb&#bp0>^QtZOI^B2uYS6t zg0|3OElzy`JL!-Da}Hq0PFpUV{r8n0d6FVdp5)UIXCwZ^P>+a}E7(b?oL*AR1kw!R z{}FUGH^c>O6^h@SX?jSO-e+DdzqkG|F3r>8n%piu>(|5u{5gA7AO-F2`eu6Sg`jdl zi=mgsP6fpEv!Q8a_e=2hKa&C%9(kgbh{gad@*{z+g?M0_6H{!;)GJXc**OlJvKGkF<0uI$_-^I!^E1N_7`*akp zyffid$l21qa?u$2&RBlx8Q%kiE8jSO1N+6w^5Yf=#(4bHqLe4ji`R`4RhSlc!&J%>LP2uE4C}LAUbe|I zX8zY)i~OA8GFa3FTM&?q-ig(vQ}cGiU~&Lu@=*zQnwM<4w%!Q)wCnns+-e1bAEL4HqUZ10dmlEo0}?Kl@YG+QI;&XQq08*g@Fb1@{?P5VWylRY zB1+H8)BhCmi+RcAc2bfzjpO)l2d?h78oau|RpQ;w?c?Q@sSncuwo_S^VZSvN&A(vL zPX0STn?R$913E29rvYl_j1W(zX5n|jyxXhMs<>CdoN|ic$GWCX%y!^nanpJ+a6+H( z2jiWs?Z{~&31}{m^^t}B1#Enrw?f)OB3VD9smTzwlX|oJB=Rk%6Zh!_5#uYBX}7f= z>{S7&=%UJl5@tXFQC+a!6Ev=&UmM6;)T*p{|A{A+L4y18@eekZYcsCy@Tv!|Q zw?ue%B@+BUFA?C}$nL`ZFYK-+-{C2W6VtKdoKO~xo}S9|k#AdfPZJ)D+KNseJrKu5!K-%-N2xIvUWVP6_m$^0nOvulYM1e{!HBNs>?*vd9KD zFAJWBKML=M8Rlw32`r z-@9cdccEo390IuxpBt(i2SQ%gM`>~5LM>oNX_Pw9m*5LD40Hrm7X{MCWlwPa z18Lx6y9AvTd@G$Fv*%SYP^+9fL{yktSkNwcDIHVPY7X)r|L~+E{Bf>6X;QiJI`!|3 zd-UzIsjIQ~(bJ~MriU(vyWvllKdFg@d{^J~Hp2LK{H9UB$L=9#R9jzncu0Wpo%@S- zX zBCCpKphs4PUA!Ws7n11l!f3d6)l+yz-Izs!fU%`5izeq>`+?aNz~UN z;l*AZkdecL3=GVVSVw8Tx0jD5MOuCyGvi-(NBq=JIxjfijWT!p)>e`iBQ^Er!{+S5 zF#=j9L=D(?s}0VX`kHt?FC!(Myw+Lyc26nCRTL#K)D_ahK$@jPJ}LP$O^q*R&Lvsd@t*@4F6z31HF1|Q=291(7+1sfLRjFBlKq0A%tQ(S73MHOs1V4d&C+$y(>_&2WRqQ?#JSQ zSgP|nWS>NX)b!)Af>|EB{=Z(hr9LRmrzH!%n%rmMR*1gIAN?Vu%;v;%JDal~R19rh z<{qH$x@5R-4+z*mmI@hdV>Q8YUBgUquNIif27*FBN!9ASAwL!ST`A^ zRf%8XdEJlvlz&%ltQ;O?&=A(Jl(TdiIw0+qXdz#x^VY|1Bb8o`h|kp7Ch~Nf8o+|w zK9X<_)dAbRwywL%VGURoffiH1bW?dv?j#B+>b|?4l$T=dB9Q$Mcj7eoO3`GLqMP^) z{8L1BX-YB}b0g9CScYI`sqMzW%BMEyzRap*VHf{Q-r&&!Zfi5ah$uBerd z<=;4WxJAjN?{`|3V13K7O6)_FR^Ylw{yz1YP2!%4fQatd;}ESNgJ>_oWP_wKGa~6a z@vk>Glf2E7mk}9*^U}zk*{!;tJc0xj9s_oK<8kB8lDg8NF*zw3GRtG+ zfL!`3x#p$2U25*>eu?xt`3ON81ZrAYCS#>f#dzsqyNUAj)H08CT+c8J`Was*$NnQd zDE%+p$p{&E1UE59X;+rc%jg8)d`^(5Z1O>0TmF+R71!G`>+*Gt5kmRasVjOd*tWV) z0_M7}`qeeefw3LFd(l*{aAvF*Q=IWXd{YCC4H-IhK_t)1UXAqwc2?VyI`3 z@{Fbete(QI1un@85-rMWmJqFznUNum#2Yu7#lX=i* zj1?7&Gz}WNwT`Q6fMMiDQ>;_Z>Qh=AX~A;&qwmox>LJV1K+&I(wHpqcdzFCR=l0GF zVw@!S`U(lQjVv{=DFbW$<(iz14XV!bPZSqy__A_w67%;<18OUYTdfiFN$7JbMt#r zUvohoc6G!12pcZS@G@%D9_WBg(4_BlGJSD@X7aS%D6CC*SWi@!YMD2YN)jr4xbgd4xb8(?TcA|iXC5?$n=t18USfLZ zMXyG2-)W?`%lt@~s9EeB$qM7%R95wj_Ix2ec)5zv>f<;$MO!>uh6K?|BBq7BCXk1y*^jwlqzIlAQj`tTkfzCK02=v}_^lC%9V7%1ydZyseyny2EQ?p;4;iQbBKbN!(3@Wq!b0!O!KD(1Gx5(xTh3 zaphagTazP+K=Ez4KZTY+-;N=y?=F#->Ggp;>_MPTbpiE6A{l3V>Plr#9-(tUVAC9> zNAvIXgTrbSRJ96H!p-dYh>3f45Jdzx`jr$@@-SF zLB2bHzv%?O+6@K!evLAhd!iQ)4H|`0%uVJ-Fy$h9EfTlAM76`}MRv9IPCTHkoTzPx zlbWUUJNBhun|3oEk#OVdwfWP}tE;P(j`n-2nN0fvfq);6l{vo=aE7XVY&Tx0FySdm z3E=cQ&WQubU9)K}x$_8ylUm)i!@E1~3U8hw3;un_O)kJ^KNXF0PW6>GqtQ*JqoYvW z-Q8dF@R@uYQ+@-;zT*nv0}#vPCN2)eyWsM?{~s=4T5f+W#(E4K(OSSWxBPfP(I>kx z=a|u#fWx6qmwOjq0GB|tx4z)Cd4i(#$wt6cO)sy>W53_$JOSJ=^q)V0S-phL&K7;} z&VIV3k6TDbUF~iomHc0LZtxNsV2tzJKp$KPw$=_zi4Tmo_zT2q^t0s(plzqw=wNfg zk46j-Z#piwr(hEl@@MI<_$AtyR2#_te1F)naqF|d%>v>-8d5-)Rd#xi02X1a*wD^O zUn)|R)1Jq8DyfYEE*Ai91ETz!LWxn7tH)5olCfC`O28W?#x+| zI%OjMzw(0Kzu0NC0Xi&8gMb6|_EsDY+o&Mk&pT~;e9Z|RJ8dPvWVvBC)C9jx>A<*z zfu2C1=PAa&`U}u^!Atm$Qz;^JvN`QxH;V7Qj%^X5V+*tPL~h02(rY{iG$f_@_qrrg zK)h|a@h3PY&>3Vz3G;oR*6R3GT;?XJMd$LXeXBni3%Gx{ok z7IXUg_wPH`4g80v-Cdba{X`;268@@n~=yehStuI`QSPY8I z)mePkv1YW<3kGv8`srBf11`jfz5{e212-q|eiT*$S}XaQ4|T9V@3gO5YHu?DKzOgW z&jaU~mti4-`WZo&f%_kX^C~BShMm1|E9d4V z!JWI3CP00ziLPSn%c-X)68#f{E}sHM$WWl{jBw7e`^N|;>C~MOggAEB23nswI-V63z~lY<8p+Z z8v{flUkN*M8mvH<6;?Stb$ie9Gw??_Kn@`H82WbOAKjmqQ}vMKy%15LCdm5F9!j?O z7c7GDprx>B?f)w0-h-OT&H#YNQcJ6jbZa2XfX1TohFEl=kw;vWMHhjs*o_V$q#aq8 z;u0PruaM|E$}Zt))>se?MHcFUr0Xl7k&EkQLx{>MDl1@a4GG91ggoRXA<6A`W!vd= zrZf9*r~l=jdveb?-}%nD$^CxU0JE^K0}iA!qHPn05J93Qh~jsWJ-Ka=KslmI&?cfV zf6Ls*Y9TWg4}$#YxI2nwfaM{84+M=NVhMRZ%Yg$ytLe&AP*!?HQ)Yl@?8%xaB9ACt z#%%W9nwb+QrMI+*zLfL~vqdBP`;^+OuIdw^?#)u2wTFN}HVmB=VW4NoGpdLnE+d$% zVudFf;f>(O+=UWXHKVc0Z7Wqi6@@ZX(qEQ%^+ZMzkjSZju^Y)?u`m}El+sxx+pLAy z;eS7jGUWzQMqoRs`KF)LPZpeYzPcK|n5!#}CCH6_i-Ne{cQiDj7;P}UK<}xzO<`OD zJ%wcX?B*ti+a+(;TciNy58vqg$KkqfVLMU1AbsA#@e!=>A5&tTz zz1dHjD&=3a(ig##y74Q-h1=R_DR3dh#HYW$kdU?VtWxAEp|#jU+J&|JdG(LdT2wT2 zPqJzu{^?{t2s+SJ||zKCM3ECBQ$2 z8+^IRfSzPR5gF(y;Tr7!Y7-xj0@zEUWLsDeD>b%qt3usaglo;-uo!%ND4t`}vIz7q?K;vx!elrdQF zRyoae!A+5km>cwDPl%)cv~_UDJ%s5{;^77+AtiCx_NoA~M@)B`V>SWgBNP;;N;P6@ z(Pnm)EbkLL+ELriJS)-^4(?392QR@mj^d~CE1O`(6LnS-gj%}a&@j+j-)^YKhMnIV zPN$mq`-!P|(YHLDmUGJ)qNs>-ey*|z$_x@O$R|MurB7D^sz<y&;dSF3VCftNONR!g4`s~ymqD_?+^#S8Vg2+CPBUMhfWK=@XYvy1*NVc% zhN$=zSbR@t`2nW(U-gh}J2pWhADtSXsh=Vp675D;Ko*RQpK#C!N@#P*um5kS)?F_kdcE#4rFUGFy@frz^v-^Q&0 zJuOXFzQ7vs;Pi#ZuKJ8tY9AM7#1>8;uy+)pX%0qI`Q5ihmh9{3URlP9SexH&%D|_v z^93LH--8StnN%YMQjRkOQWUI}$Rv#vRJ^0d%ASbAgV=crQKd*=xu~7SG>o3Gt_S>o zo9qwONU{2+@D8p&3|*g3^zB|!avY_72K9t z!kTu~*XBMxLJ|ikqtt+rP5TYqB&@Y?6W<9tcvZif$E+8_Qh=n(_2N;Y`RLx6Peq*a z+0->m%}+=|+lzxp3_W)NVxLEFWv+n`|D|zVcQjpD`&}UiFFQsv>7>NE@zLk7)n8Xx zX6rH1)g%q~B;kb19$_e)$b7QLXa<;M$ETp<)wL+Mnf>Wt$Eo|x&~9x$O$>cDaIOxgQ^cJ;~S}~ z0~Z~qFOwnIGF8SL2)59~Hmm}Me!q!tCWdBT(^7Mh6hEgGN$@{`mG@d;;yQXMloYuR z&=L}7kjUf4A;jp?w)360py~6*ua#unHo~N!=COb?vcLTT+K=xy&cdD^`bXLj{$h6< zB9&*>?lwe1T>HfdCWix*b4rcBAu0*LvHoXR-*H8~k0#w$FiI86UY2}qoAhMNH&P_t z?7cCs9tsE2mb}@KA~0qv2IWrY!IK_jOrolJ^c-O`8{#9!t(Wk}~$J2A780)nmNno61iYAuD={%V{wJucE?{*_W>{fNw zm%6fH<;I+da$j9+U$<%@2XL*emojiCBZxmgLk%QsII*m0dTEMtLVJ2pFrb-}x0BDX zd4I6^kn9t}qGze{>2;DYP2of=-c~!t=FO#gFgVMlz*FM28Ri!4HPm+3auV1q|Q1A4rX56)VRN4$v1Cnu;cJh@xcw_c1gqDZD_9jy0+`4A%t7r4K@LR ziq<^OQ@~+WrJLMx17!@BT##|aP^t2!xwCUiNu_khkx=d)8XF?C1$15Jg8saL7IA&)~YgTdn7kUUYQy zIxk<2cj!eD;%I2|Ik{8pwnFGjrNP96Mx5IM$mTv>IYuCxnb;WCN52$wAxuwzlvid# z#wUkod$Uedf%%0P+(W-_gkP_QL4iu>kUh8 z>{`S}e|V1PSS&yP=i?~pf2(fq|A-XMk2m6OC(GaRD9nB@mzcR-@yR>7wsAiG2Y6{( A_5c6? literal 0 HcmV?d00001 diff --git a/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step2.png b/docs/cugraph/source/wholegraph/imgs/device_chunked_wholememory_step2.png new file mode 100644 index 0000000000000000000000000000000000000000..8b203ce22467786b4b8f7fca87a6cfe9d5798027 GIT binary patch literal 28201 zcmeFZ2~d++yDo}tx3=wof(TB~4v2t)h=3rFZqY_$Qf6d`4GoA8!b}*F*ljB^NLm?X ziYSAO5rKp;(H4j@ML`G@KD z-sgSZ^?gZ~>@2tbu;&LkIk~M?7tT4z$!+qMlUo<~{RZ$aA=yP2!C&hl94ybu)%5L~ z0Uy2%_}TVnIk{TQ=G9-n1D`jAUT}|)lT(P4{aH8RuZED5gAuLH{p^JBW{+?F=h%q5 z3ZK|=rS|W*AD86J{@Q-+yYJ2ZdVZ^Qw4q&{tAqJ=pU+=&e{&DjWLV>q!kz`?*=AOT zTg4h}+NLrRrv75v;jD-EcV^rjZs@(}b`0z-!zqQwmiJA(jY~5?GhZ;$)U*F5E zw`%?7>j#w!45hE{pZ}zQ`}$sP!+pfR{`ZN$+O@y_;KuKVz1MyHu;noKUl+XkJDy)#4iC=wdb81X`-0D;7Q1sE|)savUs!h%WJFHbMw10)2UN zCnP@ti}R2Rtx^>XlbtXme(vlhObeogUz>-<3YIinbP=7(&QM;DhN~`OD&0`(9p5_` zwK?mjV;fJ~F-}?}&B&3hYyvJa%$|oSaI!5yK*uk>d5Lj?BPrrRj|uUIUXnmEwVnr+ zb4%Gv6z(3Z^j_|n9fXSJdP!8z>QD3ecP8b!59;8ubGAJ_=3`>p9)H7c2WQ#>d{cVd zM<2IXCT+eFwXh0zhKSPW<*KN2IqTlvuf&-+B5B>ER$(Cv3vHXqq&mA67&}UD`5w$# zUv;ywHNnT^xyhXw){zPD5rG~9iE|>Uh0goX``yG|3Z0;NxwW_CXW-;ND_1(;{I^sGD--QM)MBAg}m8bj^u*) z)ui`oe%U*}6Ikaq;Vnu<;liImERx*QNHx*YO<=Rdd$Y%teb0?e@QUa-X?PR7S>=#q zlt3&N_CbV7yhHg%YWH2;-#$61IIwQ$XB(&yR~38|#oj}F_uS6MuveZmQ!q&5h{lEY zEZaVH7lu2`F04R>#lp*SL-Pd5xP}q;P#2xVvQVp3L%!3l5#65^5qTwf%P%BJlX9#h zI=YF#7a=}ElRECP`>p8d(exsdR6@Dv3JZsN+oq7)Qi`Ji~B|V zz1*pInn?A@L?{P2$l?g^g3FID^-nuNmPcKkpi(9KO0{1v`WVZphs74QN6KCF5iWoW zWE`B34#d-;;+riK&X9$pT*(RGt2f;i@(_n{ekZ{-X0-=5gz;R+sl-(fIUdkwuOCUC zm6;MzJLQBcZ^-73;(zi`Nd2J2MP*;x`792?J!;+n_oSvtB}m2o*v5w`ji&NGCURcw zODl?;b)ObqprtdjmZ*87cG009fq(2s|4fJ~Ap#*5Xa3Jy`$2FiBhn(Pk|X73yV%Dw zn6s90SB=!fAJv>7OBA~MxI~}xM6Ky#tUcO?XYVcO9aNL%1`J`Irm$;QCzX9ysKAPz zzj&H^gR5GcvysL_M18XyNqvsF2E$CwB4H^V+lJUG4XXJdFDn4m5^f&dLwpO3h(U7h z!(&GxDkE?!owJh^$5MqGfo;t7Bv)VvXa#ECRSFZoQit%fDK2P99YJ^$S-m zXD!!VHQ51`^xV~bQYd<-;Urj{T0a;75v=qwA&cE=PgGGK&qndjQzpjBu%&Jic}cgc zvy>_D*3DY~#>&%N73|XY&dlPE6YEdh23OYdUKe`ndm>#Vkoxg%6UJR34$ky{GVO7w zRotrOAk~!hns-R6AMXrR19!o2Pljy~_!h02QY;9a6ih+}m2M1jX%GHWqxPx}#0U_Zb14XP-0Hg@7RAaCgD##a3G=sp)KN1#;2H9noo`X^Wb}F-KrR#$H4er`a zTEI&QLID;P8X+CAk6kV8`Dlb>*BJ`dy?1L}8CM(0u$25mHm_@vzSHpZXk(AXR1;4< zbKCa(z@6>^fHkMT)e}c13FP+ufgnL2}EH$deal{4Ojgn zE!fqPS+y}JmD`7#HWMce){&OEgAh?BePj$xO+=yf^{3}j3fr1U zyTG!@KY6O!gJr4QnGvajWo>B}%>ZiVR*qhs61bq5WV#Ez@{5z<(t+$jf!zD$wpHyy z7v%gR#z|Jw@TJBDIe@VWhw}?A0C5h z)$Y6DTR+rjdK)CUp29$xY`v*qy`Ij}&Cf5IF8y<85F0P1Pz4pUdx;A(+2Sl+xgOj0 z+B*SO?M-)LtktvDhxQEZ$|i3IPW1N?o|3KV0RpmBwyFr)Ub;8({bTItF*KKn8~^s5 zZsXK!FH$b&@g3hCl!G<5?nLb1WL-SA@qJG9aUINoQt)N%OMxIg_SNm=U{@`PS0kXU z^t5{hDH0;Wt(A!yTPmf3P3@CBoNO`#_ZH9Y$!(oP3dt1ql?sYjg8`>xJ zK%7qfWTR%;y7B6^gN9-XnxDxtdM@>_y0v!xGBAe4qU$q17Va90ckXng^s?SauyEF3=BuYrcpAuZD)U#yP?lUEah@;s&{@i_eC2HX92+xA>^hkK0EDJi^xl@W@2?7 z^G=|HtlcbUtqcu5vSZ^R)xN>Z+A$k@Y_pQMNmn!9x-EiN_chxb)n@%DNK`t{Zt3*H^=kN&)&rIPS`93IAU1=_wU5- zke+RSY24SdqkCj3$2Ydg)*>~KcDPytkIwtlU5{A_nkEz>pNBBo?nwL}WNnuv*vJtd z>oI&s(Xt1fm4|oRMJVDPE6gUS`Pk5o1e(=~X?JGIvYBG2i-db2LwmY>Kh>Y7 zcZ6lYEe2q7BFf2gtJbwA7>uzylNnRnyUFcIn!Yx~8oc>YXdUIa@M_-zGLV%wu#@aG zdL%1oUKR}n4yhz$O;A7VQ$r*+FGr`8Qvrtxj6)uj8IYF@8D=Fa>k~W6YPhZCwlv#T zEV>o@44Zg|c?3NUGvcC}+C>s45I#>5RZ+#lk(gCGKu^3EFZ$i3>iGXLhG5fCSQGFg?Kz z+82JECyM4lRE9K7X2k{WdArf4C$hh`ccFJJ;T@Az6u3>4DBJrCPDXpA9#T2Xdqj{B zsY6zIS#3~C*ul|kx~=w0E5v-dub?t}6cBm!34ttTizbAHqRK?Iq~8{g2Ca4sE|Qk4 z^8AK0y*_4!G|!aUI=nR#j75Z}h`5Lv&MmuoZR%h>F?3FY5sUUhi+MZ;!ZY~=)X^rD z-8HSlDX6y44}ZTtEO|?Cyc7#}*rm{U3 z-)2jc!jSJ%Xo2U{4ENKH4dgQyRD4XDR0IyGT_p&|&WE0{q`NXtSnSjA`{w7Rz$JLI z4t$nLl!kRDRDq4XX0~ydVOt@o2z35@>tbIzDwWtvN`%k)a#6AEqIEi`6LM}3)etXr z3p3EG+ETIy%hLUEibi{0IBtc_*c28pLtTzHip=3zMYi*pZWPre#ZP??RIo1>&;8C$vqA-bI?0?CmzB0$625+%FBoQBLPG?6Zk1 zrGbtmz5aLwDZT)cMONvq>&%(>rBx%TR!>dJ7&7p`y%XEgxhPwq5+_@cQ#L#86}g`< zske*P_3Vy7`~={|rPv{EC5i9OQwVbEt7<|VY<*B3`f{t$JItvR5_z#yi=z)|vT`;~Lpjxb~uSzeS$u5akcibC4G(l$N};>I3# zdOxM<=?Z!%@Xm~nP-SK~iDNX{XGtkE^U(Ergw77Ap^l0UPNX;9RyH|nobL1)H>~NlUlLMJ$Ii8fE7T@5CdBi(-#ye-A_E}q@teMK@0iySo4R0} zWHf+ky_h89HisvR^DZ=-Eb)rg^4|WN7a58pr|lwIhbEM{h3y8bp$u^=$QlAZXxU3v zB_0aKrCeE+_$O8ObH5v3WxLW5;treHo+dtLic77>Nx$89GC~8dp<~X3XfArJsv=D% z%I-T(m@8w*aa}w+=lxnkjy3UGUizfNjrUSct8h=eW`3;gB5*4UHM4LJ+{`__4}?`b z_`9h{MBfQ#?!u6(F|D@!k!jm;`hKm)3Go#Xe{P?)Kx(%=J!W6&AJ}v)0HlwdOup zU%?qyU9Xe=(XK1{H<2GH-LyKpSkoS`i)Pa-NRzh@tbJj8j-~O#?K=V;pBX%2FN53I z5z!@x%iBru8MmGOo@)}e_gNq85t2CWsEWQ9I#}kgq1%A4mLF$h&B8x*DMuVARYTh( z8g$4|xM&~lEudqtE{p2^UITV!*l9-yGqTVbdV zc=jL9G%p`5%nNC9gL<~|b|@pltaFEyeF~5P|C~xDYTtkkMMdc+S;Hxo#%echKFZRR!G)5xNl=9bde{JjbBr&|@Ve010#_ws-dMfB{UfZxW>v)Gr zcgo}NUoK@2Q^VB+$a@=+AGs$m>Ew&Yrrq|(AG6oH*2PX9&hm*hUf;mB?l_%&N7sSS zAw-x;8HRW>?=tH-k1B#D@s|XOzzp*mnK`XU7?1X7`*z@ghm31YyCr@0%Sx0sErOiMVe9kQDZis2Sa3?AQsjPf} zyZlDXsoUD(nSrm4>FKz<^eaklFKsB}{db9fkzbnHscP5e-*=lhPABONp6f!T$hKL- z>h3Yui^@L7sX^|oG)}&BR<`?>D5ix0wRa}+E+a*;%G$L zq~^9?wv5I&-x?NZGCW7n7TUIknxyaN7qN>!_bZhJhODtGiyZ@^9Rp=bv>Y{GuBr*v z4gWcz6Aw-Z>e>-#8#gTQ#`%-OItyn}dd(DA z%Ab`*%0GqeP^!ydFw{wR(nLtb$NcKmO*i%QqpqqgAWkQ`8~YghR_SF;XDab`-(&=Q z7j0E&7uK3Oxe~4BkRS9B1b7li zbF!5$C8R4M&>Q} z=ngfoZuJo->n`~0CEwwP9morr6yEmC?}+Z`xR_hwM8t%VIPop^wRb0@_gS1Bu0325 zw(qN99TmE1lfIJS2hVEJvBxL|y?a6w!yv?b+pyLIwZs~@+9OVv^zhb|l^r&)&jxm& zcYPmgs>^)q=F^-*l%Evr4@=!o<8~|!?Uc0^8J;f*iX5`>Z4PswHczdt*7iOw%c_-g zWGz=kO%z${)kxNYwYoTZ7xOZ8J}zu7XzcI}Q!wWrybfK^CC022@Etr~m3quYNW9xS z+_Ac5Ok?h?z=88|Rl4J8KMxpo`ig|%G1)1vd}p+`X?%sA58HjIp1!fk=N)a+n;GtXzaw!O6Uv})H#-HDPW0H5uVrH!3qw{lQPIVcl7vl{85d9u0jkZ)>W zwWf^5Zc4nfks($ON){ApPLX}fGJ^|(E)snJ)e8YF_SK@U{%00i$M0rBKee-mCzYK< z!xPWyQ3G4vd0uSp>Np>XdYd1@U9P>u?$&+gqgL8H9KqdCbo*kADp_kS__1bNjC$WHI7ZBbd-L)W|!|vpGqY^)8 zZ1~*(CmhX!SmXXLKeh#j!^Ll9Xx~IA|A4EGiWwjg*Gx4<3 zE!WdWWGi~XICvl_CDGXB_$%GG=8R3&NW0}S$%7ZLuFVaN!?frfdC>3sK5XueyqDcB z{tXJ9kFKt>2|mN$>~{l}wY|H0d8~BxSsmtGDQ7XpSyHxw7d%S+c{2MU`3SN+|6EUx zsNa6uH>s}PL8=vdUdUq6JDGVJjl0Q$U+rB8GwQUY{^P2^A)jd%aDO@jAob~0zd%7# zeq^??7i_R7+)Ljz_als7@IC`7ck9_ssb{8mmtav4nk+shef)^EnVB&4j;PC6#}7=a zZQWNV>YOHW&9ez3F;OoUAiB)9VZS_#ts`lYJnq0(eE(NhQ1OYxb{9RQ_qH7a-7N7! zy!?%Zu&s2vR%~Zr0`QcMBIDOU0Ef^2FhDl#)fh+mkDltiVtRb?ysPJ2l-JHo(N2+< z_l=B&l!@>SJ?8EF44gjShvR1LlC6KUvu|FD-i_Bi-sKxZSLVuzPLbUvcA|ab33q7+ zBrd2X57e*YA5Wnk{pi|BAVM#|2Zl5foeJyPQK~HbH$CSE(CvlB5vA1SxF?5?>$Q;# zI+qp_PI;Hjb0xY6X0fpX zn(^`UI0LoHt`9i>v1{b$0lNtrp3r)f9Z{6cVFvd8eN^{o9&xEh&^E(4pc*vFDVFk&H!)A!LR)(#+(NQjWhbbKUu?e9Uap0`9 zhLEDc=H=>SL#7#4QQ}Xh7NX>vV6-7mQw=kS&_8vxLA3OWmz?_@l;Uq0gDh@Ch8Sd-``X>GpwF-oL`4&h2Ys9YLn6Yw@Q!mCMprf#=FRi zB3sdYX&cK-i+O!OczZ6Q3E84WsH7bC&YM(Vjh2JCy7mrzm`>qCsz8`xsBHgC-a(bm zXzF@DqKhOLNXNP6gSDkQY1t&8z#YcwDWEapR~-@k$CHSf0hcmaCiTzYgkF`|N>125 zWcU;2uB*_gUEPmof8I@LE*>7TOip?`RWGO>>kPjgup4XPcnl-a^!011u}DVs3Vc1( z?c{{k)Of%APYM)F&V-Sjot2I9Kt#pWZVl`~`IChKQ|IL6PYP;L_OFox=((7m`?wM-KkCBQHnZHAIT4-!QZLqc1m_?_^i@* zQMy&!y)6rm0KzLDW`>n+JSE5WdL`vziGWlso4^IaBUBytZ62Tt_sI*9LRvl~kvu0mwJ>=z z$!Pg|!}SldjZ@wKQ+$X!%>N!UCLl5>ry>j|17d>{Ol=bNLdpb71%>zaBK0io#Tw0jNhONJK0=e5mhqCMTM(1E<5v9M!vmI0}6Iq|jK z)0bM|GF|T>xTGVpcCTnyQwsWlV5o5yBpoDX%TWhpN$;6by} zon`nP=-Uk>W~sHbFqPa%k1Kj=A&Tq?S>+K1H?vboW;AJ=Yly_XzscDnHB9_0UQ|>Y ze;hh$7lkaucAb%QHrP`q!3H&IpB&nj1X({y)jOIL;W&*QXizf=QEN#TkMPGWLnWTN zJMr~!QQsj#R?dX~1+@&U8I#(`7pZoeP~odjlubN?A%ez%dT}2;8cr7=uEm2x6d%~+ zgl!$7XayYqBM{B2LmEHa7OHynCz_6-MC`%igfRzt9-&;oN;@EQd3!aGoP{c7Fe-ipN^DTS@SBUim zv%=OHrwJ~>Lu8@ht@{SzxoDRI7Oy0BZzBNGjE z;FTh~=-Eht`(g5VpD`f9JB`Oq7w{g7^mwRDGId8ng6qsFa+Q~@!DD>F_drAUF2_28TdIMehiUV9?O@wr9 zy~t8RAGt<%;A(|Qn&L$zNObr}z?+qbwl_tO+mcysKpYkmzKPQ^Fqhn9{ zgRuGGI6K>c+oh{!GeG177@1LcqQ{-}Y7xf|PY^tuJ1V!o)@>?s#w-+Phd#k7rhY6M z<`>cIoeLo7z&9jTO^YEoBJ$;BQC-7uKK=Sa+qDaWkDW9%M_gVUMg4u^ejMNVvL-IB zJ|p?cO8helQj7i)>DrbXM{;U4*;Z!e&-Urs-Efjz2E$@D|LU=3mJe*_)@BEjc{^cy zPi0(@1%_{{gj3MZ2FX%KD_cEASu+?fIqdNfNoo+f9|!ZZ{)0>KK3ZQ7uS8_3pksna zFw>m6dV5>7^Hkjyf|^N0^zk~svQ~2rW0_4w^&I0E(mJiV0Rn!xeL9|VDIdKJPWqG# zLT@^*=w%k4*4fa1#V0QB0P-)P!QH@4To_wmmQ|3j4PTJD8!~(Atc6J3x%FZ`>Fzc9 zVhjBpr-qKJw-)J9&oYoLgU+o#+zU=W_lFa-+`%DS%0F&K85trw&$ z%DJo^k&w%uJpIlYe|5E5`G>9S#bnn&8$*fR05Qlk=W)F~OlGE+H2DJKmaLIyj$T+xl8K}=5aOQ| zlUKuaSlvBg&I|0HKC^QgE2iNd0Yq4B^8R$Gp0kH~|1%o@$O|$q-A#%PLjMD<&wcED z^fz|4n=Bgf7NSjB-s&}IA>m70`GFi;DY5^qd^xFk41e8qA8x_S#%cRkb6kDm?9%!} z?BJ6(JfTvG7|LZ4IiSLgeS*iGYbIOp+ zVQ21jT2L0*i&$XWPrOwsPGL8FxV?AQl8}OuZ>jRNp&^saCwMu=#~pj0z`B0SHa4$8 zdZ!*J_I=$`GBa9lnFfpp(?>)GA|DI87`*_xx57yHv5ze~%{XGV13;f?FLk8+e zKBk5M6Zgl}!z*NQN^YQ&L1x3KXVj?>@xR%BT0FbFmr5+RFZ*k#a)F`lSPz*QxY8QX z2Pq384JW(qcwYdaZoc(9Ta=a^lr#goz4Y@e73RE}3Vk`MW($&N`Lga#9XvD~yj`K) z;48YcUi6ek5?s00(Jt;>t#gcDbne^WlYYxF{Z(Ckr>&-f&A}Z zr>5g=gg|;1EJ+j?KPo&f*@bKAHtDSYdh+%3*2-#dlBDUd{p>J)>%Fmi8rA0-il6n{ zRcX&$31}i-kh=(TIC-fVOR&e?!$Cf7I%nlxq-FojtYfSuC(jKHexom`J=>GGf70-F zDa!hG2>ntlwv3m}v7q}t4WwI8B?Pu>?6KQ5yj!KPKGDgcdjwlnDmfQl7DUg*ACee; zTs}&@974{)mrY*s9ojB)Xg6>u(L)6L3^Bk&vlnOdktg-LW!M9%c{;HT(fXk2R2WFL z*^-yRK_>X_-Hi-NBj3X}1E9{QH~qwFhO^J=c&+#kZSM~ACe(u(fVsV;&F1h~S28_> zO%q0@)_Z3v>l#%c_uss~Y8D|eCW;>cs9&pX6h5+Yph~@zu)BZbjN7;i$ZQlZ2z^}% z*Dt)Q#ygvMn=dlic;+~*kC%H|%6f~s+EIxB^_>fFauhoVvISY`J^9+WNHg}Rf<42f zrVf)~qmX13XcqRAEcO(7Ts*t!9^$?#1uoi!+>YDNpbt;1@|Y-L{C(6}&h`bt3FlZX z#b|t+w5hcCAp)DrhoG^e8CI@uYXXsz+&7k2;7xsQI+b3TqFt~qfg#M22A{s7oVygG zn(-R11$Fj$HOa0Agrz}w-N-nsZed zRfch3#H~muWcL8SzTHsu#ha1q6}uZy`c1Sqe6KYRM#HgbbYE$HVQIR#CghzgDFFw` zOEc2XK5`w$aBhptf!UJzc$o`B*k=M5d(f;~+qMFh((+bM^ltggQ}N%xF*Q(dPLW$N zl0ltLBL$8c-CgH+I_bQYuHk4o(pr{P)AMWr&;4{px;RjHb^uh`OP{WRuLm6E$NiPE zr7&INMUFU-=Ad3|a~Cy{3lc;HB-=NrM1T$~>^KdxvM*V{cJTt~=wo^RzFqh<1!^9e z;rvJG_D#mf^qMp)ENsFOy){!~<9X=x!ViA79JG$Y-EfQhlla~DBO5BsS*gfw0m(k^ zWKz{?)nPC3sK1TJ7eCxUUO{$LKD%!3o_W+ytnNh{DLZ%>J6&%KunEbFWf#$?uI{x} zn(h^xAbGQ0XKBqXX>CW4X$4-waH=5Qqb2-I;^FeUf$>)!VG>ax_B1E`>%M^6*wctr zh)jP&kPu#?)#b_i>EK-bJ?(07NIcHMX)2cDKc=`%$o-I<9 zhXGco-zv_b6d~v#JSALLamHFLZqBGrPx*Fx?)hgnhd;C*H-9>_3!~=Xeyd#bcMV)z z7D?K);CAQF(%LTSZ2FH;dk)^yLd(-aG-oQDTuf3v5)h~njP1Vx-0?~rLTRDG>;TLf zJA1ao92V^k*P%}kmF_;mq?^O=&L_1pMg4xXAo+2kdDnGU+7NA8e3Uv!_`#8e;<1^Y zSlD=p?xJCfmY`^^&XXDqJh0~x>Gl^7EQ&LX0k{2%Rd5p&b-vhs)E~J)9|u^8wdj;c z-@hTR=@d+aOvsu(r9HjI$txMo2PI2E=<~A5;Gw&7L%lsY_yY4bH|7z4a}7dCR{lZ5 zv8RQ4(j&TpwOHyHD}yb05SlyCpjbUYWS*ApqoY1GBIoS*Wvmv=dq4n=?gaQV3%{Ti zESi&*eiEm4F1fZtR(j&|*cy5nAQU}2Ikajb!?}@-r_GjF zA8bU+>6PBVNWz?v=l=7*?-B*azCnz>`S@`q^GCVSKbwMh<1@hZ#-6vliTZ6 z>Y4D8(;+@r=`J1D|0CW99x|M(tIT&=9d29=(GQOMa>c~TvkMWD`p=p6F@^7Ju zs&&ma|Kjyn^u*uziwFI6*b|cC+*etGt9R|p2aqxc5Y}j6E2soh{ioP8pSLwrYxcjw zCoSvSimKNF9-zcq$#15_C;N<2Nrz~hUro*H>+M%8-}1Vb`4*T=aCku06tO{5WK<<4 zYI~RPk|B!qaF8XZrDJX4ej=o4PXCjPe!Y3?-BBq4m7i*y&JA)C6-__1D5xTt?$$@W z+k7jFZ(WX<%$6?=qzmfu3a5qQ%f6$Srdgj21neLiosH3*P_f@=kDck80Mlf#aYis=2Woo&ey-2w>EUSqTfru|`H{D!s zu>x4&G13u4bWcebdti>Q3F!uhG$KC1+I+^IBte*?*rgzkFO>H`nz-UKberU3n?r(e=rwrBbz5hkel0!KOfL=i!S5(f7aJNA!~- zFsYwSfEhkhj#56*sxq)it=bN(^~;wbnYwbJ6<)@0F4c@Z zbl)Vcccd1;$Ks zrj^qqzB8_ior(+|9lEe@bV#@9LP*znbcgT}vfQRQcN7l`ywBGytb=(l;oSUEQZ_PO z&Cz%C>+N6pyOeN0+;avv5Rh_tAHX0mo)*;+n^E4^ql8GR%$p1@;*%guHd2Nl?o`>Z;7=Pd;2 zY;g)Xai5wOn_O>poP5e3razcETDm3t0h2j8-Jx>UB&)|WSH?bFFJfn^kv>z-KmjNc zTaCd0g5<26dJ2BT>;DyRu%8yVp+<&F+MT{kIQS(?QK_tRivatqx%c3a}F0D z_G;1}4O;wEpxzi<>6D4nZeQ%fLeS6XFi}37{@2Ic zQrJid-2P-%5N+t~JIsOmPaNpetl0{4Xa42;f&i>lBL?OiF^%YJQtxK5y${ZIB1%xX4{LD}>B|?}Cv!&&Qgh9N0 zX?!vr-(a;sA}<(n_pHSAFD^#iLU53tU4#wGxjB z>?A$!BbpP}Yv@HAXqbOToO|4e~4a)Db~wm%Im*W({1Jr>yfT&bzqz} zt^Z@=$71^~`Gz$c{sd&p-ky>q(4>Fq#M@vX?CU&4eGeLCGMmMkS{vD8ZXnqfOp<9A zGKI~#^}69CuZaoI<=gr`-)ZM_uPfzl^!0zqYe?j-ZIi#DbBS*vWy-7x`1(QmU*;Fc z*JbNi9-NT8lgW5MowOK2&4+2wQPXh6dU_GSDKCFHK#Ng(AxrAv{x4KxWker!S4Q-zBE+)oz3kveV9wamIIA!LABjoC zPd7<+PoR2aB77MbbA5R|{|O9zwwy}T3j)pB3aV=#wp6^PQ&}@U93f&G&+h+ z?pX+_gSGBNgSK9HDp0hoV`cgueo76X!{{1ili)xS%u~Divvg7p9mw(j+ysuip1w+Z zog>pBPt`*mKvJ&I?(oe>B3tpiP<*lFi^T2t@3&ineB6-xGN%EvCPxRNPij9J`fn?; z{@?A);D3fHAZKZ7nQ8yOsd@?SF!+ipDi=DmZ~Q)W!e2IaIeyb~YB_{W_#Bq>wXmct zz3w?CYmEQi2J~+}gX=%l-wwdNB70k1^~|tN08l2Fqwu?z*DAr+h$OGI#G;R~F%_tA zd7vFDhsvM?&Kl6`k8}2J9ecu$+H!G0(6chV{I6L2!Hg!TwyXQXe0-zU3iBXNTn<*m zG6F3bkcK&|eRK7}XQ)ApRh*^hcO@JnwgxTJnXaS%i+v5jkHqX9C9aY2{(1J|?(Lr8 z{Th>rqcXwe#6#qOY)|L@V^0%?>s?nzs*%$+o@)f52x;PPWa5wAgWiaTB0wI<`b^ob z7KCG5BU@3a-qjy8YA;N7{Bow@Uri>_d~;&>Gl(tpnh8jK;{RJFXjCxB>u}FD5{1IRF25TVNl%5 z0)eW>o*;6#Z}@;2z5F8UIki5@j_0%ynpdXsDm~zB4SF+L?ew*2#Nt=GTzw6G%ki++ z`LoVYqwz?`sOf^3A;Hzy&mnC{w|~GKsHcB!eG^1_yG7Pf{Ti@3^P4hmQ&fV?74NPa z?e)yXi_Oiy^uD%YI>4;80W8qWd}QOE^`m9JhQ(nom(N^YV~MgMBG}}=Y%w4kA`RmexD!6A`j=Fu9W0p6GKb211Uo>8$p9?4IIwlvTMZqXgXKC57cXf`U? z@0jLHtDRbfAw)#%_>w7=Gf=Ol?+@fnu)~3-h-sPDQ>%q?`3KBoM#g-MLte{zGxe{Q za66)e`;>7~&BE27^2~hWlOl}{K&IO zGq=a-lt`>Q_C-%MP#@oz1F6EPq?^a)F+}mZmgwGP|O27;|NXE(lDT=Fy?8fM7 zk+(8AulVsQ82RIDM+vAqCyC-WzJ9U^A&~6PWX7y+1sn&=$l4f`Z`mGzp7%jw;G}wv zy^d&$EG0}Lzp#ckQGLIy?ioUTGX8W)gQaWVDQj|yO2iIY-Yb(g!Nln^Ye2ut;rrIy zY8ORodB4?CC=d6795_1j^-&);3@&k#UG5>GSk;6fL3B6j&yoi(a||X^uK@q!?U8xG zV5kW;)(Uzz*NfzFD{M}zvu-5dC*^BmQzJhxT9kpE{JdpvJy}B@CL8!w2V0$xjo?2w z?Mx1U$GaONviRjQvLU>HZxPT}MR+SdqKe8fT%vv_me@mk_VE4%mqU*Xr_Fhabt4%Ud$K!N|4t@ zN1IWA1b^xo`K;!d3I}CSt5?@bi@cl7p!_2nI-AprPD|(LC>fodrT}H-5t7=N^<@ZD z4($fk1;-q2g721v)PW#U`~Y`vD^?+25vQD-lgWDM`LvYc^A zo<7H*;;F5o`4(1*CCmd_rLAStV3p+W*FT7l!Y5I$Uu*_a228Z+i*=ueUFIg+(JBlZ zoa3(dJ8>(@yBr-m$`SMU>&{`TTimI5ww;x06HJNNA7?QTeZJpWl3QEI!jUA&RM@Ns z_7>CVGq<*xOMTDuK377Q2BJF< z!2qwma!5&95D5cr7#OXIxwtkw{xZXFHm8VyJ!_DTSi1w&u9F?srPt8?Weo^QzN~Yp z1)A9NGcwp-PZR9dr78^2w#?johoe<2?G7|#& zZ^;wH`LYY9TH2>>U2sT#5kl3nz4!E8hMW?ngT6pbJzNUIDJD62vARnBEO%HzRK?IJ zTBW=!*0$OLzelcBpcXK=;a6D|U0=EL^AF{S^Ke_GqHtP;g}PrBI3jW#<7ZLmE(CxPe*#wU_u zxUMSPU&nR~u$;J=6SUkkcDSqp)`*8x{A4El*Ext_()(oBTKgJkX%Nus>OkL@2Z9kC zx51n5KO{3=_ucDI^25VuV7#>dI05md4oROq^+kCzagk*~DpWj*z0y%xj{=Dk zFYLPU5p75Mq;d6XVAKyZ=75CHYL0eXO0q5YK6wg?*O-w3d2Q7GS&$p7 zyBI@9#26mvhdsq-?=# zyoBN0H1SNq$>UbJ(~MR|lIvtp*<9at2)vPHzcovgBXweIZy}nO`F;+#|I1wO3)P@? z4L<;t29!;Bo;Yx-AJj~6&};4RjR6l$-Qvj&a1*6JXbfCC)o^P+AR5t+AIoS5djzEp zfNkyU&A+BftFoa6nBdI@BMaIOW-5DZy$*G1P5!S=yM6AS9ZJ6Lhi z;Qw2qg(Z8wg;SGVlc90?{P{|~O?EJd_33sV6{;FKsulg!Feji|-g*zf-SsQaic1#E z_Fv&vR6TTdd3162g98ed@!hrX|~s@(tG-}$@IYLoiC`L-)xB%8{x z^+%7-jCH9HpvbkkRumG|TB=HIb?q5+@M)#ImqhMA^ciA)_l<(^4YFcMrAE|7rrR*4 zPvtRP;FB$z)!KDvo1AxW7#lrz->EP-!}YJx!OwIuq=k)-3v9%9xfj`tCjW^WF4M84 z7{~L2+aj$F`$WwwUJ1>WEMX-BJO$BqaV17zE%mlTpKwhZfB9C>^EMm3l1;U?fd%^h z0ZzLvv6q%+P;-x=ZLXpn;tIGRzzWGaJt8kQF+-ZPA^GKkpeqsc_T;~pgiGZpY@zwM zyo|8Ah9-A891s_KuV#23Hpqlfs7`tnRay z65S0;VPo2BOSP_)Lv#7}n4s^m9@3NZ47B53KDP1d%#Z>LJVGPO4xEvfJBD?m+A)+0 zZfE^N%lgUbi1(z!t*Vp1YX^Hk9UnPeu|KMa?GhoW4}xi(u-EQ>3(n7AX215>?+cHU z-OXXzt7)D*DR*rAicbM}%1D1wlu+)*q#|o{rTphBTg97N!GNRWy5nIms&*K~6E9p! zrgoR`KC&*dE^M+tAW==aT2dNyWHPQL1NR`*HNP!A3b#5*uYJl8gIDF6=<9Fd`#iz=`=4s&c|o|J~6v5 z2<|<(Z}Ze|$7}6upoRK$G|}5fOBwY^7`G|sP;Cf8ZA_@^scbjHhkUrQ{VE-_8kUQW zH@{wAF5d|C&1X#(@Wy zdaYF?F1S>pY$>})#SO{578C?TSp2-T>oNi^hP($TVO zNPQ)l31781x5ShXe#aq9T88t`2yd?Oi0&Q-9pQ1w2zW{vkhPC@R~)c9{f`8~O26t( zeBshq5BAM+^DoVbz^Z^id+NvgaR-(x0^>x(g{Kz#f^4o2fp_eBqj!ca^k9$3_MZ<4PjC28B&r&h8wT~W^{-t1a_9Y?tM zK=2{NKBe`dpXSOQF<5(b8>9GIkHaCBoqQm;We*EH2zo9lG24k)scrqDhhl){gNtmq zE)S*;Jg7s|Ms(nZ7{v*H8yNb5O=FCTeQ{n|fzf+D=k_ML#C(oSNf}ey z_OOkS6)^+Vp7yPj5#+{@#D%Vw%0>GeDO(Q6)|#bw&R_EL%*+wmRJJf3NCjm{o~AIMpONbl+t z?yXRQ8zL?!3UGS)^4QQ&fgRY<3U;Un6Yo$hZE88lADKCVxMUHrDo^X!*r>uv~+zpxNAP>ubQ~%t^$id$Mxb0z8A$rm&s9 ze9^7(5@)P<(J)n-zY}W?AD;aG>v3pRv>T7|*OcT<*Xc>iWq~Uh{;C?ZPn*}@B&UYk>9aXH`qWt+kGI*B%L_7U^P+RCCoHcRWUIbG}Y zx-Mn+5**?#6yeHZszrrS8j65Zl4@TQ<$<3{E~EDeDNO%7TcF8giri#I%D`&ZT^sP1Hh*rC8SgZ=P_mL-G>5hD z8l@%mTWohY*lM+w5$xu_JRO<(s}r>Y+v<(B)oqBkY^yb;?5`@eoK?;IhqLag7s4VDBev8BWo;K4 z^!wPzq94D+CHDx8n$rEc9dlnN(+Rj-L|AD}Z)o72=f*8InIBkbd$+wy+7#kP53)OB zfC?!L^;{We+iU7cyX-ZK`~tgOF_9y!HEXgO;b)z#pIEF%CM9cT8tO;Pwm)3+*i!SY zF5KHEn}55mj_@+?36_#WUl)18Q6aPV-jGMB;kryUO@|!&0shWhY%Vx{Qljgmg9DQe zv9zi*>%1cOAg*SmhYG78gBGwgjt%vhS8Uz(QOdW(8ThTutz321`N`?Ppv(_RNbgWr zHLjK;Ig>lt@6!O_B6HV03DR-jyGcBL4QcnQ+JgiPTQcn~47$>wp@l|eAGQ)-xGecP zFq*k?oNk;NoB9l>=w0O*q>goro<-jo&VYr zv=!Vw(9L<|-07bUMB`QC>@?)u4S9?=b&w6OKHX(ZHs07Y7{dJeNNvGf%MLEZ<#pa9IcJUM0rFe z`HFK_vU+dd9Cz#ZdM7=VLi^J@voccP<=6eJOKELu%1qnzy;@SPrH>cnc{DuR)%}yZq$u(f!gqvDZnr=C;kpo%Ll-&Piu0XG86P zn1v~I{q%H}xOO{P;%pM|o4w1goH&1OveSes%Ks8SOF#Uo$S>ervpRu2wIpENLODqO)Mu zj;%&Wm^`Rcn-`kG-rX%M(6`Y`ES#cJBZg|Q?+Pv1+vg|8TG6&7{QZ{E?b4$xDOGhY zw^06#RQF?x5rL@sc)<8``Q9vgMBRL0@4D?MyRbFc!B<9*AG(<{zu25PGO=cNMnaDy z5NumH${(aqfI|4(v8l{!*F>qhC|MrCiuciIHqF$u%uPFjHCWxmI`U?yvF*tASFOrkvLo-3uSSTSiI0c?3RvNA!EC@_FcR%2nl-vb8Eg2WVFUOYPx}wLJ+}+${ep z0-6md0{$c-XB%rFB&7N3JoCIbId`Q!I46PjFjDw>>GMA?S2UsNK->Ob=AVVO8`bvO zt^BW^_xBsUGuypTLZ`IBE4Faosfx!Vf4Vu@g zn}W8{?=_IbRKPkWEkiu3%YO)npTYv`GR+=xJGd!UCv3Dm2Sb?0m%e61eR(g{(nH}I-GE(<3xJ6#AesrFC&Ar*eYM|@}Z)u+l; z_s-#K>ISxGq@zC^U!($O@{Pd?=K-@bhj=FNP@S}la;Uf;n4@JcK~m4FiKU=w%~bWV zU%{3kD0sVyd*!n20ZR*XFm1XI(9@(i(I7<7oXy3`T2YICd_F-)MyggCxXh4*C2-X6 z?UJ3OaZ^XiAVidVT@3$B3*`dH`wh4{niS<_^4HbrbN&=@lyMM>`8`CU09^O)6rz2_Qrz zR$sp=KH2g8`OfiYelM=W(ndL$dzS;UjG`m$m35$&e5~x9RYu`};?2LvJ}4_&qJOIU zJkf)nSj?y3Z4v}@LluA|-cmzoozXZ3tvp1vH=sOCI*?1AXt~R>gtn4Ir)M3+)&A>h zG}sD4-S3!JmHA^ZInf;k=It$Kz$#DN z#I7Z|)%^+JE&%BTv(p|D{RX_Z6HSr;5*px29-I(D(;C<`KP?<}wC7tQarwCU4D-U} zmIp62g*X@$cg;B(0%Dm2WOd7D^%AmK_BdzpeG8c*2`3GJjAU8Kq9H2x+1H;fMVO%j zqcZmLQUd|hYW{@)$g)~p{_O~k0Cbe?+$>1to*l<{0b_NE0tF?bMI@-HYgu+e#%lMg zC2|oQ5paL`W<1%*b7aB{`$e8g86e>x+8rVQG$Li~qa&U`gEwp)Sb_~$ge>~^ATXfXk1U9Wiido6(uVOe*>=46Z`=Kx^suG z??oJT1LQ$SZQgPn4a7|ig1&0vOd|p;;{h(VItXXmbVJWwF;r;j0bt~D@du>xeWi&x z|M2pW)r{pMJ>#b2kT#ndL5IvOgHJhx1p~61Cy=hknJAf!;5((P&%7O0ZbjhvbEt9Q z`$jN~->JYV&M5wWSXf6;&}9tiAcV`t@uY9kp)MnwAlbt@X89=^y6gwBL28pp7K)A> zP_h4p=V}e{pp2hxgfRX?O$$c^P2~F7`*au3mbRq{lzJx4U_wSZ1S!2e+Aau>Y zM4%ol!NU?t#Kq9|@Wv8Q*vbyaN0cFHBC$^d^QLL$T6@}Q4p|R)>;fqqakpDeVH0B^ z7%K0*K0TbpIY2rxq-f~UGeanm)`G@DRPZyIloVMmd4`fu$f%eP!~d~~j8Y&J*H%pA zRO$q`m`s?j4=Mwlhlw(qcy*ON?3L^P&NJUHR(4to4!{iN1`3Zqj{4|KIxBx=Lhuez zLP|6@gha@bOsJ!xLlCfuksvyN`B9Q|9&{F>eEbs_b!B=nVCkaY?}(e~ zf%8-jpCn9^6g~o?A~gcF7HTtsO5htuEhcVYnKX(N9r9=b|0nKSjBJrRj(?_pGjowc zP=H}m*(V0D^PmvTIScq=0vZ=21`_jw)_2P{Y>SmWM-}v(zVJE48>LizI(kIz4(^5X zGtj-zq*OS%@FcH!K_8XRFyb7pt4Yk8)}YSS%@tAn29%S0QpwK&jb_P#_fo23Tsa=9 zpvrvr8WVvE0*LMfIh6S?!s*k0un3CpM&v27!&=R4at20DEDmb?Xvd9C$G3}QQUw~2 z(+?aOMX_$SP(cSk1h&n{{0*>NzRA`ewFh*gcek2O(jy=?n;|V)63L;U67ofim`E

o~MNE8|1jUhL<9#)5{syNXL!crz2kVOo%IqL&V8I7SI)7vy z9EpPD_UsJ21hmXE*%5RMD7^j%MY?5C)QeEe#X&&;qD1jNgv7DfPB&FT&ODBvKx(3@ z!tT!3ay|*UY0Bvs9Ird@QQiJV@pP^;>Ss{>m(_sx1FZ9^*r~e^!G4h;l^+A~YNB?+ z$RgCIxDH^Wb>rwBkkl`4$))QZ6t4vX9%>b=a8wl$6|p59&s8TB9RhoY4e6;g2fnXBU+2_vKA z^y5t4tJUQgikI3m$WNr;la)&D*%(-SCwr&7{26TX+&7G#nin3c^ zzp4g{uYsy404P5av}e6IQK91t1K36DYl;oZPah>iI-L2+$@7z5{QInS9dDKW72z@O zMP&BSq8*DV=h90-e25ms{97)Q(9_o|+HnGm<8CAK!D>AqJVS;h0yMt{I8PM{my+X? z)&c45ZivMXh1^rdRKYPXFUSV3ijpML%JFMpVf<*4BoOMwb66bt6sG9E4L$)%3&7;< z#dHN03@hnd(up9VO7rG)9|;A0Z;A&ZR#@4iAb%SQKtU*{-l5C=8Pe~7rTI@2)#Eao zP{@HFoHL_eL|t$7Eu)bov>@En3$`73N%cOZ@+%1%{#a0S5%nOt7f#_*(mFW5$E3Kk zX<{4ept|qIQP7HSsHIQ-%N|7C$gm`Ulc6mac__5}B~xA>L0ALE@cl0-G9k|u_2~|X z?9s%(CD0;94LX>Vc}4#b$}*sHcY@#$uV=8l=wGA$XcD8*dyMQ4pcGW4ur)2y#QsgF$^s zP$znRw8jWuupn4C{i;dKc|i(@rJ!7UGhYf#K!@gYL1-N64y_@`kp@$*RQ21#*~Wjy z(uJiaMKnGUZ!P~vVUa-^eesB6k(c1PKX@r;hBP)Ifru=dd?fW?%$wdj#7qElW!+#= zgJokY*kd7N z1+j>w*N1#G%j&w}V$!+vI)ZS5698JOVnfK@wAF?Qb&Vk8N^@oG z)VE-+n`YfeQUycXqbv{#X|d8N^PDb!HA3ENd!%C#3Vl?iSFyf1IN`to6!gg9!l+A@ z0*dG&Pp%}UQNXW?r?(sa1f#fgBFx{M$8tMx7evzXnL38=0Ja^5U_N;%xS=clU2T&f?<# z#qQEQ#YM=X?}1M_cJV?R_2Ig|JmmRpWaFu;w9ViMtDmsQqC44G*yvcgx<+UmVuz%G z2l&TQ3UiRz4{fM7k_140|G0PAkd1a8AhtQidv_jQx^ma;{0*@!_&+~DApZv?-x%V5 zB{S65AxxF`Lr%;shf(A-!bfIe?R?+3%*-q~iEC`m$?*q|2f_Si#^`Yo5#W{BH_J_x z;Xgx4Xb$0_F49*tQh9*YNuG9D-iv1Q& zk$!$_7i1+ISKsUDK#9sDpIZ>3gk76j2|P&sJ+lI7hC*1e1rCIxZl8lC46s&e(LUh4+SaE zv;*h%okt9j)LYNBGuMoZxGoot-CYsW7W#X^XjJ9+d3a{lEmfi*dgVc7sI}jC6OZym zjN3lOZ>^rVbM*cXbSE?uJDDICNsjwMg@%s^Xls%Zk5DEC7t!G#bw^jq^a%Oh&kAfb zI={vSpSK67cqk}#wr`grQom3F&hxs7v^~vPRIHf=V`Degl&}pUD@>@3ta4usD&&^3 z^vqPOfcg{__h<%XH~v#=0b?;MSOCUo6^m`i3@0X@T%Dj-5mOs7=Wt>ZE0MH(qJ-mm zypaU*gKM?m&7-6cc1Pg-TTkcOL@B$Xu8apiyeb1xIg2e{nODLVhAfn$B$1ixWn%Ur zp0iy9;ovJxy!Ww;cIcRV=fk6p?CC^kV{QnJoG249pHCDb?;1Nz*cD}vK#$hnb5t=+ zV;@cU<`8K+RQ%&lFOQ5<=_rW^Y8^}_0XySDe%9w$NB@MAsz75JX&wSdJ;>3R42|$G z>hM$p|DX`UsfkqEOhhr2?R)Hv|jYDa|_w7mBUI`PX zTLV2AgtLVmOtNVaSF7z<7<)JFkA(?a!HBjuoo7{r{nu@9M$N0rxspp@FOFaruQoT_ zTFfz<@f0BhVFjD~DQTFUMU*?>7@+0X-LZIc4J_5SNZJU~wvolZ(JR<}E?$2k9IWU; zLNGZzr5L&W*GP;AX1OvZF=8{8{7eW+uNL5it_TsOgjXI%gALup7g9F-sil#d*^?hm zjN8b5qLN~TO}1F2j;7&+J~;{GYwl{n`!?azvB7*TG%-r*l+c;JT`#n5Z&tGA-o!dZ zH0w{C0c+aKwl(Oy;)i%q!(gnN)V3CcrgFp7NH#lf^7F+BbIj+CsIH%HRtHC{Uf^uo z5;Nz(wg#)cfz}spyb^#mXXYA!h2R}j-=sOnd%XhK0XR=9hO|w~+7>aAEZlYTEUQhV zrF`QPw6Qo-DyCAIGCGMCVWdt%BiSi9T;k@uzXT*s08Z^3Gm^<(on47_5kWUME@U&c z2@_w}6Zz@T*A9#Df&5(funCgfvPP+pW%bc1&&Uw{v-mff1-oDWnua|_QisGor4!t) z$v`Yc|K4OqPRYd#jt6=*<%wgzEXYc-KI7RI)KxsR#QrBp4db&I)M={~#&V^uw=e_2 zOOAeIGERi`wt=4`x3ag6y>n%VWWT3XBFT{VR}w0tS;`QCXx|&1@!hYzX%WgCIvs8p zWj!*v+p;uZl%f|iNuCJ*RAjdCI-pw?326<-`ct-ZGvQ+s;d%C^yc2;87air5kFg>) zvbPQe*|3`5$chL}MQf1C{C|VIH%zEZ1uOU@&5fO;>~fFAIA!XI{PY2Rn!>WC3bkw% z;>sH)Ok^cuQ0xi;e)gJ(K1#prx24&Tn2MkUuubhF`^s?v z0H2f9c!zVg@4~@;VC>m`tOQr<-=vNCS=80>CfDr2S8rivzID0jJIYfn5N7TYJt-RDh5Feb5_Pn15PeIgC9BFx6Ar$ z8dexgX7P-^Vgj^^mvabNif1cv5=Fs;DMZdanu(8U4yRrafu`@dUyosIqXhT2Y&{`H zcqp)ESBNscDjQ=_j@e1ldu|GmHfSAzkbx@&m8lrcw zA$HU}9h8$AyCd;fSAeL^fy>BtDg&SheXXEDbA0!Qvo!26aHvTe{J9joU8s~2phcy@ zuw@&S(9Ik$<-p8YBDQwpvzUexhJ6HeqO0Z0l86u@c}6!;IZ}am08A**LH)#;@Opt9 z60_%JENMH?Ed{#k=1KZE`!e<6;I;*TbK+}|K$f43GsL=~J-@y12Fa2npbA*PK@^{n zJO>F(({}>Ge;L`q?BIw8@1BZCFKUlbq@Tj}#hENs?$6O#0qgF<5cKc<5;O^a$!oq^ z5mlN1Nv{4Fxy^-#sd6uwih}S>-eAh(8}JQ0@${%of--#ZE(8JO^mW@NTFplMygW+ z?gXTv^{V6cJNk1oEoy*V*cIvnF=#>N?)%Fzn3&O$Jh-9)P@o2<@e6Dz|S=dvX>?VV9jf0xpUE(2uo_;1Z%WM0pk4t#YmWmr%7GQ)~eWVkWKc} zhul2UPQAdHC`BJ8OAJHdcfXd1m{!e}q8tI6crjAhLTx6Sn{0j5L)qB}{$-UC0QrDO zguiVwnjQ@3Y`PHOBi{X{w! zWEX{fTe)H7$~lO4`o_(g!%h4QMP8n6|B1I;H*5AO560sNHV+pe4q#MoOUP2(FfI+%SNJk;!| zZ-`wm$~-`{vJZkQbDe8xi{=rqv$HEaUIu8G1WO7)h!Or!%u^O_I#7D78sqP{Gqs^mL2qzuKI1RqPy6YiZGo%V(8zGa4AM_LKODn6GcR}{ZQ;boazwF z^^2_ga?5rN2_?9tnlib(L=@TYRuX=?cj^eJVJ@E*>dKkaj)Tg*0e^bLxb~2>TNb^1QxxQOw zd#d}Czqqg0a7)^A*Pc6FHH}W28#dCkA(j46XC^*ruGC^%ef}s9@xukgVL0(pf7qQd z=AE_iPpgOv-@xnI)X0SON%O-^naPb;n#p>zc#p3H2_|R#|=7(1v!GfH}0NE@iOR zK=l`g)RrhqnFI$+La$GeS8R49T7p*AS>N7-O(v5RPZ9(=p19u8M_1_Le~;`Il}Y$Z zRFCf3TV<0UVqh_HEpopM;DjuD3|nz-(A;ML8<;HG=emhW7hVvpv{USR%GsBZ?D%fC zu0jvWOi*IlM_`%GY@bhDk)8}n)}S@QUFiU^Rx#_1bv(mW2~ zeYfe$W4O9OM>|;$(dCx+FfBp#(7;-uDs_ic--(U{W92%vjJx~LMWI6Y1^4ujrFRXX zvq%NvshLxh%^Ru7$>f@*FHt8ti*trrA|sXS1pW5ieor&6^x1@6=P)Aal;$Z^#;TX3 z1xq$rA33-3xJ4?1rH+=Ul)t0&Q6F_}Jh?9#6_250KitEUYL?u=WIdzHB7SX+^WAFo z!EN(glMgddV?Gh!N2)JB=uv1h58@68qc02bofJ~Nw`d;Au+H`zmXE=c7fS4$W;aEa z=cvPVURKH%9U}JiPCe~|r#Xx($4#M|n-L{As-c!xkNh&Sy{5hHpt1mb(5gD~xirck zyh!NS!TEZ`vp}|(&~A}cSslSP^q}IlLLNsBOsQBaSM!q247G=*jIA^09SRywI*P&n zx+2(j32V`USw5@FW_!NYc&IY=tbTmuKDFLB(E)8FoL%c(GepDcu&XS2 zC$VN??F>BM8BvK53$c-I`|!ss|~ zOSvJpbb{YS!=qx2VW&2Sb=Z#6>y!Y#D$#cwHfeYSf8J^4yqvB)uiyL0pt1fzo~0WX ze8eZ-^+Uz-PHUMb2&oQ;7fxx6XpkSb9iCyJa0vxXt*(w z-^gI$@q5ot3nyYivhqaE=r2H~x$b#Tg)-#2>e}D5H)E5?=C>J(y-OZbyER3w%HHwv zT8L|@%V|`RdtETWaheImtm%*&1!f&Bc}KKt7nVgj1Jg7rMY;A6wkA)EDg23ro6)jk zG`7GIcPE*?tyHTK&asK&5gYYHBse2$^@J_#Ls(5@LCxgIgPDobwFey7ZR$es;_}Gj z-L$TkL)o)s@1Pg^_y_wE$p+dnH$+e|p7xCuBp9GrU%`7jHh#(tbQ+b7y>dq`9o+(A6#VkAY{7 zZIXL-w{v*vWMoSsy;XMA<~CX%X2yC{T+uTAV(j4Nv_VIhp12fPjsPEKE}x_(~nbwM!P+IP?N%M^6+9D)=Y(!jb0pgHF8e7mq!8 z$eIsJC|o7}29LZwcG}CEJ8gVhKl-HW<{4V{(-x<&79Z_#IWbDxd1xXE`S)T(jNDe%ap7ZbHajOeS|c5mi1@+YjFtvQrf^jx6*VnBxyv;CBXPi16; zi=SAC++>|5F zzkMS?q0BS-txqVtQi?!y2Qwnn)3oSfEyfCXA}$;+&0q5GKBbVTl2Vb!L5zp!+5oaMPd3YPe)oQ&wX-C}BSh z_Z{|tl?E6o5`^ ztjM8}+NY{~%ol7;r2!toM>lME8oW{&T{*M0Z=;{X@n_c~P4_6M zDE4@q7=*`-m>Q@?_lE{lbz?51L{7P<4-JNkj4bn>*Z08YHw_9@XI9LIrdl1VeZLgr zU0SWij#|j$IXMM;vhg{{egIEfeK z4zSW~6^FCm2@+$px%jY4F^_W`_F2%dY`DnCQw`aH!RZ;Q{8j7POZ9Ur@2b`!@3U^m zEvWt=4JXQ{Y{4rK-rojPV3~)Gn2KD#$S|Cs8Wc1yJ7^UY<17T%$<DavC;slL48_kF1+yBZYen^!rFH;Rao+jHjm@-E_u4k}W zcqs#R{o8q1$OFy7OjNaFgUKr!evjN&@uyohNBuoRzLRAc!^{Q-ZMQ1@B$>BXP9-8p zC>LYKi(XG`2_zNOL$Adh=&M$d^U#r{gb<~+e}7d`<5Vb>bv5er4_IqtGcw5*2ef~w zQ}@(dWS;kzg+|mQ+_n*8e{Sr6t~@*?V_8h$m?FTlg=rVaS$Vr`9*+%}r96o^@l6P9 z1mF|;A}d%u6du1&p?m9OhFAM{*m^yuSBMCAWv;HbBC8f&dV?yDPA*~Un|w@OUi6BV zP;sEj6wNN?Wi7ve*1^y@3-i4&7aWNGKeHrWZIAMX5ub3HIEQD5P& zMx+J48I)iRTxY*JfnIMozTNwcqo?a))Oa{YuGuEf)Q9{eJ9qrk^Bj})lw}fwhIN9A z46MP!1&Qu{-hG4KTib{5u@T4p$EAROUL`k{6}V3%!cZCalq}L4e`Lb`$*G(YnVNv3 z3+W9mAM01`B&*_MALDiK4W5k6J9F=G_q1jTHu=pXgk7+6O+IGcE?6N4@!uD296GV| zT}eJAlWsF^&xNsaCf>$x>(!`P(921>r|45{qmuZQl}iho!w2!Q>_X4$Y>fx53qRQ4 zv|JuMd!{Fcqv|zTa2?D$^!lbVBkK8Fa(kHn=2u|b;R3&`Y|PhXVDGqO@OrKFJza6+ zoL*x=LU=wAVu0XShn1gR-TxQ98_HdF_$3SlLvwmst&_Gsjh^ zMuFhMHZy<*#xeU5kL_GTtg4G;@tUk9d+P%13^;Xb;xwk!Zf_n)!=~#evLZv{CSL@u zU3}-iU!qSeiT-C0=?~0=d>)1;C{2W5X_1*#A~lc`8I{RV@%Ph6nq%GPpY`SRCYaP# z=&9cH@rG%9j2u(4H?v5xlvEk>5PRcUg^_Y*_8QtOwpFV{tAw>4?4aH0o$7sO%&ua? z4;ZKlP|F+ssewzb{|JZbOpuVdo2!hQ2$vkTTKG6xs)>76wz_Lzt(PSz+`dQeo`#!J)N?On3fhzk!rfUH6>po#^E*35_t zrAvm;%NdIDgcbT?0b^8wVm9Y8EUYfic5};&h1WNE%_rDxq8A7eKb4t{>u_fn|0>m| zL{K@ePNB@8!XW$NHMVYp2&g3$z9l`)_(|QH8`wIn@8>o4B|v*K!jlV^?UG8L1^4Cz zTq-}3LNQwm9>3L*pGDd_q3H8?sRQNK(|yY-`a^dMymqz$PYY0<(@fZ^8@X2~^3ovV z1v!XuVnlTrq59T`5&aiq&uEaPU&o*3*)RiWtFT+IyiFm8aKs8L%dTS zGN>MAj)7T`?rCg;qzT935sEC|&6&u=rk_u>I-eSx*FF&={diOunH02*wRkpH*sGV}L`HF!5D3*EUBD60U5H_RwaCu*zk# zx^TD#D%R%)<$?jaTWb&ro5-Q98F(2u?)9$Fm~lYS;~8C>!Y>;9C{N}PscB@mqk>P+ zCo?g1;aJQBF+KStRDiKEm{n?R31(7pYLJ^->gzU9(`aU&S|AM8y&u%^%noRsK5=7U zzbn*3HWHRqr{x=$JPu2vd}1&I`xHIw%slu9$d+a+q>fs_Zdf8ep(It#Xhxvp^V1ef zej6ujg_ffT1AWJ5HL)yUUy&8^WKAuazWsFEEs4HjpPg%9x9E0cj(W5KWLU)Zl0tXWG6>{Kn8T- zTnt$z*wincMy&k`D-B-hzFwhYtD?Cu88dIEP}xa4i8&?H$@QT`^U*@jluL>XrCM50 zxm|=e9&x(3xS(?ClITL{*iW^tebs8(7VRtE>5?PqMd#&(YvvJz26^@`9@)fKvV^G( zl1*BrSJq@>0aM;L&j^xUx)}yH)IUN`)Q6=f6V3pWs#81z(seCrwhbjpv2w8i=elw_ zdF@R_i`gq5j^IDia+8WdO(ssyhLExn4-!hcZ;mQ~brQ@ZS&c>wO@`=ZgNBP5&&$eO z*kBzRkFHC`JH8SNrd?3@LSiHOEHWqC3sW5b0}EfE28Ucp(Bg;z7` z4OxfZKJ|l~>mMLgN)^U9GlME63n4F^HZy}kWE2ay1_U%W4F{Befvip#*K|L&{U<*` z1_Ts$9zL80lD+f!|L6}3L9Ye|22u+=hP5yL0QDs*6q+-zzFyDyQeo%I7W>VEJA3)x zk$&LSCJ2qdWad06FE2M+`ue4f+}qW#xE&LexA5~1xNbJ_ddggJBvr#Eu(Z_a9Eg;I zj$^pl7#4)KdW_J;%cHvrn`tSPYHJ#W9_a-hKK1Q_sYa<@;LlReYK+5qhdu=ZJB#GG zh0~Oa27R>_7vazok8Jf`N3vP|p7@?z9YXQ6Vdf~7os zTvY4B=IM)-5`a@2&KC+LNI^wdey{y%;F+LGU{U6BIdx2l1wS?*G98k|u-Zdv@szz$aWajc-c zp$Cheswl6_pZ6Uc7?BKW@L2FFMvv63DOwgTZ+QK9sJ}7hGrVbAvV6~{Io`KORcScX zE`M3kr}HsFPD z!?#%CKP+JERa|nl(WTNG~<^51^y-N-ny<*Qg{NvU8EYqU8Yhmk) z5{35^n!|6Z95#z9sH-3dpKn2%wA*X^^Ipu=^0VnPv}h1rz{eM81(Ro9JO6opXF*8- zsrLXuWE%ONpwl-zJY3=vaPwAbNRDqm1RN5Z@75uisV>A z;EDOSo3y0_e!?3C+y1{E?jaQCT)MYttz0)m=2=)*pPm_`%wK$3d0{}Q;)662T5-5< z&Zn>bf^X?a>b##pgIiLNG|hTvD{%3Tw$(PWkw>{s0w|F(F-G?&m)ok5^~NQj@h&gg(CfG>EqE`ms8HzHup+flhyC ztTt^j^&po;4=qU1gb(+Yyiu~|4TGWl;8^3aRQ|GSRR(oxK0dK7#(Zix;?Vdi!v%|O ze&!f@x4$>Qnr~RE%CV$lRzFC~)-QX0y~X2}iDMpLVyT@IE7U#57KZJ9(Y^P;3L+Ly z0@z3hoS0Omxgb#s=)rqyo9)E5Q;9oj)%y9>a(7X6Y>bsDoZ3@;jjHPiv9vlk zaMlD^Qx~RUT%F;en4Fhl(CpHu0*CVP7iBJ$sIr^Tr^@|iB)DS>p7;=AYu~BM$|Pjq z8iS@sviPrCBo?bSP#YU%#!|)q)Fb*}jTiyA@G@0LXp@%kPA=MO@kh2YPp&?_Vf(wO zhySNGEv>)w9OJ5gOZN^qf%meUbGL=RGE@1g zD6ee79mBrBpDO|ts%k_0(pRx_Dzup_*E4nh0mP2ILA}n&n9FChjdL7%g_cGdU?1!2 z>kaF49~l9RMD_Do#Y-vPAXh#r;4)4o5&KgfG#|xDsz)aeHy)~_J!ovaeS5S;x2OL< zpgsQ0)~)1Is8_L9%fxe@^b|}Eksqlt#LY{2AF0o+BuZ7JlWEZ@u1>vvsU~H^9|Bu4 zVkT11f4?i4^)%fz@gF7=;zVtRP52%tNYIW|A5|}{I4Bj9P*QIQe>K*4sFX&oa%xhP z{nWTPS>*a;g};(+W5){P<+2(X8Oa^-r=rsoZVdd(lz#%Y&@#DhYtDZFY@&-QPd? zTHM^btud!m^xw_H&zVQySpK66ecCR^(=m@FVrtQ%aSdH~ba}+`R1!ZB^h575P%sOC|^_${f8B8#j6We)1yV2ajCIt_wFl=HT4m zf0@O#27(*7Vi~!QgxD0?ppIan7OArIlH9^^3$JmXv_O>Bx~l_zM<-l8QimSSa zEYx4*j(;bFH0OPsKN*7Coaat)X4IZ~qZK!js?yKGA>R3Tjq=lr->5?s=e#_V@IU=P zh4hVA12Lpr$)KuV^D1t-dwF@8w;q-$$j#3==+Wy^f;W)r<*uoaYJoXW4IHoYbLsY@ z?<7@*4P1Q_ZnttZKU{T6qpEz4g%9LCZo!1=Mo4Rx$qAz>Yr0OA9~^lm;AO#7^k9QH z_odCWl9-1QXQk-=+Y*Bz?5ZAi$L_puMJmeojB zl#Fy+3o}&-RdF*y!EwD)X#&scUa9c()r;9Kx{S$JIG*rBZ5_ph*Cn7`YX9XhsH+o7 zIik*oys$uB-lbmi8^y*>uB9SVGG=M8xg^t5c3nOMwk$kT@Fp{3+20rK)e}(TJ|2`N zO(~qm{5(?-wlyA@JdxpRN1_)HInl#yQgiXhk1vzwVw33G<`9jMv#eIcWmS^(85wlZ8J{)=%LRk@Oua_I_8caco`8Pi5jC(CfoqvAH^lsd_}- zhNbc5ut4YzK8CJw*I8$X_k#EQxc>1nJ_vBW{bs1Ct}l<-aV$+>?F<#+0&6?7)x$a^ z{uwXLc{A`#xp=RNKk0fEFmXATh{x+*B-A%It4j{Wc>MD!XQ@~KGSW;n=epNnt$&vD z{tph0me@BNJ>4O$yRNP-sN|9L1jjV97M597B?RCQD{DHjK}2+gx!S#+rP6vCC!_aP zBYS?Vb6H9JqqEGtrC|%hrAvO3*OcoT#Z8Bs!-D-*XM6lRJO&-ZmS*c4iFpr-#QkG( z3L9Z+q5=zsgZ)rSsB*&CCY!k)^8y;$Utf!X6DIEX?^*5->l@f0XS-B{7Ahn z^S2*xwt50MeGUkzm>Vv+pMPe4#H5U)JSW?eP8SZZudln?IJO6V{rs2zygv1Xak-Q* zszT1miuYRl$a2YbDL!U)<%O|#?r?d#$EV6`holCQVNUvkLUudY9stRYUNwLmd-x0= zY3;4gu`aG?mq^9=80;;}fp`w1O{Ze8;A|WpqJxBtm7wUM?%M06wnk&?IGXs-4TBSF zJ2G>B9ROOM^ScCh!n`?`T{Mq~EK*v>@#i4UrSQCd5SMWi`f822cbLXcmdR@ax;i*d zbavqF@<7!cSgd6p*_Ju*3R)nyO9Phb24xWnMYZ=O2B4MdVe8M$)*cxvjo#H%u5%f^ zZrex7jaVA4E6q%oZkfR3KHoVkULSpA8>nsjIwjGZPs`ZH5e+llWj4xx+6A|Q|JBJN zmVr6w^~@s|)|GSI$Jahz6ag<8|Fpktt|eLWy7%)reEX?B6-!=WZCF^>3LaYZsNo)S6c9H&Nh6-ihnMmZZW)N){(tA}csm z7o1NkNb=2DNw+V!YchA;41UOp*Ig^AFDKW8;|7vHP6FHKpI7Gr4W5ef8RR3a>{tE& zm?Ixrkj))3cPvx%xOVRQbPxy{&^#-c!!w@voYQtVcXN*(b6fp7@9$pId)331ivj2? z5BXy|BgO#Kb;{uzUyBTSwM{G>H@=$y_xfbrjw1E-fv$LCQe`=YpygD8%%K#ssV=*t z|G?xg0T?)^T|zqwxKSz=wS14XdYo0blX6~5^yd|M+EcCSx-nm+L|#dWG?i2g0p`0m zgS=4ohEnB%?<^LoGU}_BpP(Gna+<5h1>g}2*%#iToz)Gv(L2z&4-TyBk^?E%P8uWz z!=`YIG%a2UA!Yx(OjQ}~XYU-iUrTy-?k)MdaZT%snWK1$z;z2}xW+_aR*K(9$n1w# zd*a*qXwEgAjcb33^V_XHOR9w(8Lz2#jBo&Vb0d{u7RvqhYhje;rm*kl454YRLW|gy z10Qs)#-K`84uh|$VSNS>Ftkt7M1}_E9)g}sFYsOY{-r^Ioc1{V$2p^{GWlc#}2QTArIAAmx>Rm_)WfjsQtEtJ82zcK&Y0OONy9CapyQ^`FY@K?@A1Ox{WAC z_mt**^%$&haN~vI6wQfBzGuoh&#yN<(|}3VpCpawHu(GzKV!t!A_8wh{M=uJm-PCsz5TX77&d<60rnn+nd+3iV9>$KoexK-#7R$ zgGnYsR>zF1K@#JK;OcI9$V+xqdJrcO^B+<_0a^4vKKD~WD&Ee6d)j~V2j{7!iI$e3 z@o|m*ov|;yI9D4#KDGMbHtWadGU31VLmB;%Ey%(Hah(%AKSthX&4WnSkD%7SMY{gy zgIYi8M2`40Z=t~zYrPY_#NSg<$=4^vEU>Gebp~?OMteNS0!8=v>bt6BS?I7tk+x2G zbvsKxdTnTP?gT*seTO+%AzL`R*JkD@gjBH|fQk(2e-)`d;pU{-rE>cJ)z1G@T$K&} zgOVLI-;bEzeryZ8seg|r|2fSabpJmi=>I$m{HG&qlXem+ZAFAQ;lE))%}a7S;FY1S zmj9OX*bNvNZoD*-jBg^-$2qCD`Yje;H3YH;XT{t~9HnZCxZ5YicE)5Fy&B-vNfeeT zoX0J3Op4-D`6u}YGB5e%WDdV-oUN_B4@-5%p73q_5{KFucBDX3<5;Ca~sE27O(5P zjX8xCr6n~)h1(eX7H~ph4>9meE{$#{?$K=7c z(x+iNBEdP|WORAGapHCoTuLOQtT!$Fy3QAFN1YvjIlDXzel9LRqFpYfV?=3_6_~sK z7zV(?^y=W)hlUpeHoksw1_;D~C~z@9;hn=NIbS`>hKMq6_YQjZ=yqZ{~%Q1$+$-CsuZkVDNdN`e>OTJlfVxc3E|q z;dY2l96TbEs`M!_JWOh8{cLI>Aze-9C|zv_R*6TpUZ#AH-u|iwKWQjE0_av0az+4Y zt*@1I{m333Vd=tqF&!m6EzYWw@JR-3=`q8-N}V8ixi}j(ijDSUPiX|<&Rg8l9~E29 zD6NN&%G)>SBF_sdg$~r_yqa&c#4A86=Hxv#(~Il#LlGnnWI@?E7BjA+D@w{D{fzSU zvsOzwWnRO69q}VbVA!a+U&9n2Gxdp|${wZfQ1~t<@>l!%wrP(xZ>;%bL=ZhbsGqO@ zTPWA_oe7)<1TWKN{|`a@=>pH_X3(JAG8??Y)nD)c5FYv~z*q!s8iFQ^Ai> zeq3tF-8K>4lF2bq1ww=NYLUq4ICq)umzf9h{CMIMj5C^f9U)GAMsPI zwNwO>hA3VHX_%`40~6&>D=u(jcTfn74BH$`k*6&HE(QeU1VEnTvAP!;AkGP2NI*L$ z*>D1Q;k`Yzw7*lr!*cEdDsV}s%tsi+n3Xu^n3r8ZVCR*mj=l}hCg>=0`?VZwz#~?I z%o8i$YADJ~+Z5<>sRfNBXXE|wYDkg6c|QicOjWecQ7HvIm04NYOTtcd>qrxv5~)L; z>OIcQ48IO1t10sqy}5OZg;Ga1i@z*ghZjVlQ4EUc5121&fk61RD>{zmDFTSRlAoU2 z^TxgPDMmj?ul?#eN~TH9-OJaKh3~Rb>9st|ZRD&fAahw)uKB352)_J6!WD}b7Yu{8 z=DC|n@P)J)r@9aseUWL;pDcFd2@ypMhdi_V6N)mL@`>VP=9$oKOS(&mAj%#aV zPFI$rw~2eZ0a~5e>01dOpMWe^{Y0zLRF<#7@eb+6#gjik`~V#NV;}kIuKjV{zCuQ% zUo1v(K1N@1W!zF%$Nh%R$xiZ!T5*WCtw|?l@@W3Zi-emBm7e#$Lae#BqEn?bxdc7-LCp({$c0{qw~|XGvU2@K<_g<=*Pd4NmG3?9$_zU z$5L%58QeJ!U#9`ZV#E8Fv~6C$xoy$;iqE1#9!0)yb8FqjV$yOpq_uzbB1m&-8C>%- zBnoff7X-{+Hm(QkeOvC`lY8=To;}#uQU=w3P#ei2X*nKQp^VL6|8oF!#h8I z2Ia;W<*&wo9i!J+=heWz`i2G_oV7=+tH~m-PwJtt)Uz_xojQK$fuGud8?>nB@Rud; z)jH+B2h?)hMe9o|jEg1H33S)~ULU>KhysW1a}&#cj~tpp?BnJYyj1UgzTEVSsb6+) zKoVQF-=SpZBID-aN46JYeoCkl;?KPmC>K>knVH*6Iw zf^9HE(`Xmd;0gOsMSP-YCNiww~x<2-yY{JlNIk|mXpub*e#-2Jzd>gI3WZ{3)kgdHF`|L_`Bd${vynKAOefNPhrS-8MhJzREAW875F?Z8rWKriWVsK+ix zY(k?CAHHrs16!DD+27$b#hDa;$?=T@Qhg`Ll5Semf5hX}4O?*qpKE2}YM;P;l2E(^ zcWCe5FSWE~I(61!W`vY}GM7%gnu!q?B_Z2OrBq^EtBl}(nU5h>4khPYc2BxmuvL}J z4et%OFgG+k@*ow6g@=hV3C zLY?hjb>^Ky)&43PlbzEj5py#K7w9N%5!{3=aK4>``J5lxy#;rHTRct{5F(Ep9I;kc zoLI&@E%1F_FjTP`LB;f|%t+h;&c-#6m>=pZ8vkob;{0>pL23NO^-`6JKJ7Bw=#c$` zS+U{J-c#jI9XD(CGpLpSrr^%)ePpX{s`NLp@l>*3&aa|!fBt5&PhkP=AEc#~;z!IV zwbNM}TBZ6)=iC>Q+bTZcBiZ8QUM&;UOOP4JN;D5ub31lOQtp?GwZ?b%WK07G3oZ!+ z<+jF;B!U>Ie}zNP$j51M#~ppo7XLV(6#gg51F0VO6G}Yr2~(>_&(WH9#c$+%Os!;p zz&n+N`pV&iMww+Rt9@qWrzggZeP9kVE zPRJmQk%Kds;l!<>x8b#`f|Z(BhaCp&^Y+mR(lNrh_J(4@cn%}qmdY{(a$e*UxRhT@ zSgyUP`w!8}^saa@da^G%^m5v7=Q_^QDoQsou!q69EmbN@o{?^UmFzpaa9gDeKVURr zZP;^hq~%_avX)c33Lf%%@Dhr-Ra$yrdx4?BJnlJg%%l|0>+u&2=WZL(@B^OvHWB~< zN5l-DoAt?s{i=Xu`I3=GcP_QuufDQ0dYk{LqIrGTze4<56l_;)dt}2pWx~mDM_d&O zjZ60&5Yez-1pa+yfQO=WDtcna{kjh>Cx?yYJBh%OE6UH^zIL($*pHpcH!V#v>tGWa zEjuBmR#OS^qL^W(c%h+c;Ox_@hpiuQf{bRW|K`cYf;-zUIrq2e^&RXkv;6&~amnG~}6>05>-&t=qpWeT6?t`9@-nQSvKA}bgetN-PQFsCAL zR_1%8*B|mZlF9y>=Mq9!C-Xv9b$(HRI=7r^b?w4Lxzes6Xt^)#V)V`h2o>(BSGJ4yI+Z6FV4HjM)( zTk!tBt-KohR`YxQ|2n6X!I1F_4`loKw~Zw&_sKMHGG3kjhXudqV7f0BBkhe1wL&&$ zWOuOPcPEwkRN#IDC*OLN2)6buQ(PFV_#=C^HeFf!%Lb^o+LZySP*?juY<>kn3{ar3tn4OWh1kZC?M*P`-6-io$jz1fCVsd; z)1VS3_a>CUWTL)N&}siG*!Xi$8(JAP5KZ*)L75xwZZ-c_;;F};P}S~(Vkw_+u&u8R zaRs|>_HDnVu!5)=I^#;|EF7Eh2=|v)#Zd!#y!4)ziBZqK%_y>8o|BUbS?&3-G7fTj z-jlS5m|AeVaRPeRPMHgmA-BzMlpc%#h5xg;uSg)MwEKgO=KJs5ps>nG?r+tvhfM0}B2ta|n?l?vCWo^u8_ay}+1tUOl=WNDF^0yh3G`qgfaGJyM=W+wO zHA*+u1;TwmZ7HXZ17~5WuA25HP=W+%%vZaf`m2Gi39}im?tm%@;rrMmPOj{WHu^HU z48RK2peFfqLlmvj-8{P2Kpp5yqo~yHEWc0r`}UrEgH*7hiAI~;ibh>X$AT(h8Q;yX zOSvA9p=(JlNp2@;GX52u(no~+x2i?Rrt>T(@qEuqInp+#U&Z^yvhwoDbce!+D|Pt& z4f|TBZv;sHmxIoh-^DJ^GGh`W*Y(JP5buQc%J!X-ylJR+vwWyxE#hv7qa;&*DIvwGAZfmcYTBr}rvJn5~wi zK-qFm$GQmLkuTA$%q*LvJFPtf6dz{=qlCH6K?1cnlbrRJqQwdUaAcFGx;XXW`UP^) zfGY{4?b~i_d~&A;z+kd)K*YBKbquGhysHJ2m`g1;QT~_&Rp)^UJdg$2-b#cQRW*{z z0t-aY@$0-Caz76+=hTj$!Mlk8@j8`;wFlBAyU7^bG63&NP{cTe)28a6=v@L-{=eB+ z;2Bp;hq7Lc#Ok$7V9eHK`XSs2Kl*Ot5V|(#95` zfWxW25ze-swKKigv#?HOI5YvoN>hD7k-m&?)6!mCmk4{*on&rgyiN za9R{dvz96aCBV#lfJyA>r>&3yaU5;*lQQ-aIMf8%SMLGlM}LmWfSPQLIHk}H8hi>w~7%laTVJueT3EL8ZSeYj#K`mmxPEWJG0;Zk!x! z-}BPKPiU(#?<&cV9J$4WsZM3&D}e1;UsN7H07RvvW3?ZMN?AGvl?V`K8}w;PR=lEd z%MaHR99gzOQ>VwL;x5uC|3YfQH&TD6$31NG5ur9t1%1<>HlRA+QY)bWG&=#=c?bm1 zH>Ia2GY#LH0F^h9*X80WV%ha}Pz%uG0vWmDn)^ylU+cKiog$VxdddT?EN;6x4`!f z5bFhYgtJ=I{Z7F>;*BWQhE57-rJ^XZMViqOYHMJR6>@%!k&+YPJJ@u9~F(3Fy=@tMIX+hZ}<;7@yk3dLo-plpE4> zTG$b^Y^X@)Y4!f5Au1Shk&atqh0EedUfdrPRut%i(iHk!ReCT_I&zWA$wH0dC8BQo z3&fzEx&|6iNGI!4M1ky|2fb(Hb3luhL$_G}Lw}abfAK@iFe|wzi39fR`bh)ICf>gz zVPd;V9mmtWF-I2k0f*Roy@cRcpz0mZBLdpPAk$|lNF$)Fy^c&wGX7d%m2Auwws4xd zeCtlYX)Z9Cvlq*0l%lG!f_oF=wB1MNZQ0rk2IyAqoJ$6z+1dI`nHtQcL^w3yI4*?ho;+j zP0smSjMSjt4l|_wxtAz~{OO#z2zwD3W7R@c7aIwSWg6bm@ni>Uw1BP)d^lyXnuq=1 zo<#pNUg;s&)A2Mc1F%zH_#%oT9kFddNukStPCb{!FjEw$+18{NXw}1M+Hkh8*^^Qe z%jq28NA8maEn3d;Dj_!3C+qMqnF`fSEV%OyLowe@A|3s-sDFcU-ah#*Ye!X#TXFf0^bXaoBFn5q4D> zN5E1g$cNG1wy97~F}f9W>afj$*I)kcJ2fob9S;-E$G@7-ckCC5Woh~o?%P+4 ztV#c0x4-rMVwTbPH$exn)mT9h*|Y;66gEg^{ZSePu{wG`0@N|V3DRQ zp%lq?!MA8mB=CT&yh$&0ChuW?p;o!y5_z)ij6@o&GsiGU*Rl_>rS}8r+_hR=|K;@r VzTd9vQIZ42tEa1<%Q~loCIE!7SJMCh literal 0 HcmV?d00001 diff --git a/docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step2.png b/docs/cugraph/source/wholegraph/imgs/device_continuous_wholememory_step2.png new file mode 100644 index 0000000000000000000000000000000000000000..b773b1ef6e9872e8622c808de28510acb5033cb0 GIT binary patch literal 26434 zcmeFZ2~?9;+ct`|)>bVzpg1tp37G_i3WAcfDxv~XWR@Wc0xEB13>E2!RAh1wxoYBtSwELjIj7{oe1K_xsoS*ZR--&ROSVwaX+VPxij| zJzUpy@BKXZ?V|aQ8+LDylau@L{JAri%i+DZk=;R%E@gFlK#JD*jw$koSZlH{Fz_vZo9LGHhwMBH->z3L9J^Vnu7BBx@Z2tYE?_g|)*qK_l4Fi&+-r%u(`vhGmv@&b2^B8($qB9t z<)cIRvy_6O>4jlQV>oyj$)~XheGrpScj+KykJm5U93&h6d39ft_s!~ybh&2t)#q~C z{_OwfQ&+xGj99b!7t1IQz18PYKm0cH&y|m?dDyu62RZd$%9U1My#7|>|NjxGG5+6S zu3JH~4dE(s-yu0Dz9Xkb)YN-|voaBl;Rwzx&MHK{6fPa&6hqLQf*lg{$grr$8a>q$ zyitzFysCgYxA_usKzkt~uqq~A?A62z670+SqO-Fx7D9T;=!(4Zzk^7K5y`v*h zXZ_AY#$83%OkN*v3&NitUK^LxP@o4~aMK7;r1rv{kFPI=p{GSVB*TJw{zuH3D@CQ^ zH!Y5O!jVIEdg$@qSWH>X;F&iumZ@u%cNjxp`YM0FY7NS4jfiU6$FZ4N_j>r`El8k$ zgxIauIveGFYBf2(IcC)iLo&5G7Yw7fDQ1>_sNrhjZCdGC2N5@wt-qX<;j6Id1BcBV_D$ zQFl5~F-rVUQ`GbG3buyd(`zo7q3C5!NmN2H*NzThvTjSz0}p(6sB|V#>gtH8nv+?~ z*iD$_=uK%%xzV3m+S-DmCZ9|$9OSIQSq%52fj|oxFF>>T#T-m0!m$iBEt>Y#<;@&2 zt;_XTc+eY5T6)m?tgNnLWuu(!{L=)6i`#!w_NdZ>$w zi?EfC!rkwHxjODDLnHN7uBcuUHnc@Vg+1nf3+(Ra7PnhSq&C!sH@RW#VDo?apoX6% z6l_7!Sda?hj;Pb~rO@SxeF`H5-++rN^im-c>!^_z7jyJbv-=cwsyWLaK?E+}Y7L4C zOPZYjiL++hYM7D)CV?L}LQC%UhLWa!>{UUatx=tK-2=px#c8kztHn1yCyB)GqH1r8 zPs|p&2eaIIwK+>U!p!wZO7neGTh?~#S4O&)}bXwtYc32z{Vo&E}!+HL*z9R`&tTq#jSmBhO@?k zUn57XERFW^C4-z+(OO*p#;8Zj?Q~A*N}o2SFr2?0;s8Vb1r-Zfj*P`%*d#uANjBL1OB3FFh zQW6RkjZeYE$(GoyVG6Iczy+*xIZcvO$nu&?z^!TMv1&xwN*Ct18?5ec-(3FK)JHNg zFMP$;a?OU60S7p!$nHaxpz})l8NbQ3;V?Z1yV&glOZmeVnmWR7PmmB^gdF*`K7)! z7D1ey>`hWb+6Rw^ata}WLTkNJ%wid-zE1KxvbW90t+A%V|7S0Og1Ax1ULth4*cvEj z(l$5ucv8;q6wdl?!}=@RxGSI22R25focS9=R#-l@SVU#m!))zgymbl1=momOW6{hy z(hl{b+{%4Jnc9?J!EZ#jUFcqVm8t;I3HQ{6Ol&{>C;J`@I0%cAA z_6&$BS2fBlk>A7eQC>5_aMgE^8UXo)OUA#<10r*2?Hts7-n z4>vQ5)x@8h6?65%#=i3yyL=llT)zB3A`VxvkmK#+ss|q89y+6HrfUZI zkN|vNBQUZa`2Oj{Nzs1Jnw<|Du&Q9as)VU(OVLx0Ld7mBomDgylsg{QMg9)#7dN&v1?=Z} zpobte{aG%0S+`iSE^10zNM-qnXeW!S`UyhDnk(0(wmU!OFs7sg!t$MEJpWM7nw1u4N0a!Y(&Xl8#NgeV8d6*8_YhaAp_%*MLPsBxrEaikU)nzGzJgQ#)deiVBriyx zcg2#qXjf@I{=A1I`D?1rQR0m%EL!O#WWOLzJV+=4D|!12#O&h1L|CFH5~SPoZ;Xen zfWg=4qYS`6H1GQf2@K&I;gbfnZNMFLBKgItFW;_YvP^5gW;3}s>oL1A`zNe^SFm8X zQ}KFWBIz44h!Eh@c4y8GeLiSiq}L`Y2Qc-QVEQztX!iDe#FT}7@RS$KrVI>#8MfVc zN0^}_2#LbP(N17Q+x`jZ??MpgRDpeL>_LLzGgBrm$%7d@&2kEC(UB zv#hnawd4S$vNn+^XZc7Op$>k0cLvb}{P;)LR*_x_`01nIr;i|^i@qdSE%6$FxVoEKqL`^K9R4^sS zvT?>D*E123h>W?Iy0-o?>-Lz*Op`M5GYWR{qq$`E%qSjRxO~EZ)?>dknj`Vqjhodl94y z^{>v;FQL&R+12M;YGz!YBVYNQc&Xmy(_LN%F zt!Rel=?}z+{XKkblxlE1%SSea0;_O}UpT{(lY?RyjpD87=g38zCMKlBBMEQS9pWLO zw~EBE^G>-4*3#!TQR4CSuXV&JE@+QC_E(nE(ZHM zd~5CLLu5^>y=0@d2XJpoVu=^)&jt#_BKxi;Il*h7cA(()#eCGrt|ATrn)%}1@%}h9uvQ16O2tp<4?>w z&6v+n(wU-AqJe!GoTAwE*Uo%az{!NVw@9P2N&!yXld;*c4JYg>vY+U{IwyMyrmKtU z=HBg@u4dmy{^W^2WJ)lOM7GamTNWtp+A0_^dK<>(|Hqj>l`qQ6M8`-mfG~~S?7Uhbcn>$*I2F-Q*G73JH z>$qXWi?OG020mTNdI83zlts3yS$&|36@8kg1r(Tlflb|NGsm?Mm=oFmyx zOut|R>%x4`$#yVWQ6-#^JZ&W6oQXqT1S>ijp_Z04MwqOVbgRRYma`WJ&N4)PChMnX zM<07&=RNz=xM+HAlVCJ$hrGqu<@TD^`qL>sg3#s9Wgn^f=2jeE&n~W2pohz@B#uv- z{cL3eoXT#|Vo1Ca1iHsevE6U3W?96u7AWU4hxx(=g@WbD?$CNLSHwP3^*K%j7~gdem2uKtY!WM+gO>3<=FM5>F%6bWNx;buk(Xh(Sc8pxGo1X&>TxsU`57?NWuF-s`EUj+Zy-^sLz1xnr=iLBadT`x+af z5+=#W!(81!&imHDt_+B-(lj$rOEnL4UrOOA0~2jRikSX z>Pwtod7NP43HiQ%WaZU0?Wo-$NFA#U)!=%DzuR_Akln`ZWg7hI?|UZ5zjqYTPInp+ zItg3p2duDLM$q^S0uzmxHaK;kavJ4R_370~@9&Ird!UDsIs7@*N%5e|X5!)O zl?Bg35gGKz6|+P~?58jF#q2Gn+i-%BWbJUL1UO1H)7?dSZ!6jk%-=rM5! z7veWBS_;;jg;;XY$EDLyFDv4yy`4YSwnryip~U%B6%4HM)lb+=eyd;$5-w-do}G z@OR+@YvNjNroh#$RLF2h7wW`Hz3@&{K{{H7>?2evs{FZdwb8Xb z{%R5qt7@rI+<$h!c~=ic^&NT~cD9s$^Z`7XrE3BS5Er#DeAN0Tzt^MBs;g8Sj|p3E z!Ye6!u?fGILO98nkJc$TK(1%D5ZYd9GQ(p1ESz(vh2K;O=_Tow zSEC{4pNTYLi@-#HqW9eW?oe!(r!A{!#wog7%(@D>+gnPyA9s<8H~A&Z=haL@nYKhf zZ*t>ZbK_MDi1do@cI#k(Fha9Eq?(Uy#smibcafPFm&IWf-F*RUacoy*AEgjxRsriSe(U)x58hD{1vUb@N)3fETfXJ{^R_0lAy=}E> zu~{)T33p_G@KlKP2Xa?TUT zav&FN;JE~Wn1;@?{AQBwl>NsPfSyJ5wO@P{lW7=UiXS*SqaCa|-({~8;1>CrQ8kL! z7NoJKmuJzMCX^cwJiF9lCpPm0dB^;>!MY7mM>o?g$UA>#d%6;e^^&QRHStW>a?@4s zxO$&oTIe0to(IpVJV_p1LZV1Dar9%TE!p~!Is+e*N1rMm+(iH4A%4#iYUMM~zgse_ zapn>z)I8wgJE5{?{dn>uxi#X==CGeQdg->qF}RL|X`cPxIlfv;BcEGEp8&y&+ND*% zcJn%0XvNGac6z7FdCT^|kgEuor} z`9Rpavp7V{<_vdq%qP~K-;KLtIOf9$iY_Su*8G@)IvxZn(3N}bHYY^6??PQ11~|bc z`e9ja+b^1}uN>r6$Xg34RX^N_Z9#UZEcUnblO`6%@yG7m&WXni|H^1qS}#{0qTx)u z04YPpfv-KftUF&a4V@zEJ$NQ|GPjGVN?`h=9*^Hk)cEfB&t8~YMVu0V7>zVHwvC`$ zmc{E{mrgYC_j3rIR7Ojg$BPWVtMWjYcx`Fca^QhHS<%O*^~7bj_I-troH)4T56`R$ z=iGOw$Lc9YnQx0 zMCi0{(jwhf$yzWA*_x4owScm|e^pyiX8iHx#;hlk`i+W|(dBY^A>(y6lT$yp@ej-i?R(dT51dA_~8oYSAlN(gEGh9A#Qdlp>?MO2J3C@a|t!s(0lab zU0kSn9r1aW`F*x9c5RefrblWTX8<){ix_5c4eN88%a;{;ZtL_7u*z?kAO!QXzr}C6 za?VRl)ADJ$1J0TZ)``a_b4=yN&#Iw~!$>NkB?x-^^rkoKg#f`Cx&}l7b<=cSxVAvX5 z-Xs*05Fa1jNRqeTsoo>X^2DWK%}3Fhxdw8`t(?2U&+IYrtP~w=ZlPD_;8=GkYhRg4 zo$B4lY4pHfmN2cB^@|#Y4;994JRr{zPzfuoDbM^*1?@~_BL=q2_SAMH1i+qIwO{n> zQ7RFX8;MQd23xfZ1<{}SMF=Nw86Vo=@8la(&76c25&>U=MB?x6`01F6r)#9k#Zdu0L}k=`}eWP z^;52qI!>X+44W5(pz>uy7POU0^*1N7?Njln&SrQwdvwrmABh8D{Y2t8wTtv5x&BG1 zTp>45lc!0tN{h_O@-G`CS*6gYFAGm9rI()Wsy_`e>?2G?T&b;UJTy)i@J>w&`w)7g zJy%^I=pPfKd$Tt&J#B_DJ)YaC`S}eH(qNE|fKjjd>^hc#PGYCfY2)+z30^(=Z7KOD!zIQ0TCvP4Y~n zm+UHI0vmtd-sqY?BNx?o6qrRW;hj*A9|`Q$)Q!4@Bd)+Ul-?UV@xcoQ4TQ^M&4`61)rZ zO6}Uz&^*@DL6zAP3Y%vnatt>18#b>n6U^e`+sxm+WsDvV49KcF5a8dQ=)uI_bI7IR z6Kzn7DAg9@T~lEjUuoNtx#>esJ1TWG>B4gvJK;Q(9}|+g-b+IDAhRP|-cB+UG-GPzrBN zNSo>#KU|IdrfBJJm7KJSwcAGOK0b}3pUpyr5(7W{+RCpzPb8uE$B6xw_l^^fm(_A( z`}zy;9)zHV!g)oB?g{AWhoh~e1NXb;aFX-c#M}2hb@N$>?4-R|f?U0RwT^XIACIWg zz4DGNL?2C8GQ^$?t8k~6ac{Tl{Wj{v-DJrfeKem`QNdo`bU1!LCRd=~9@W_#foXqm zTY@0Z>+dI{#~^J%8LD|utfat5jy%}}`-#Ip#D-P0@#mQHgk)%F%yRaIn3;*?*dw^E zwO7>RiE}A{`sIk)96$QX-ExvQ=3FzDi}BZW2GZ>Dpo%VIxxYVuU9GPh1pYs|?!$PcjefUfb??OgcH6uh%OVM~il05( z4}P@VDTH`t{1vKV5IofpUA`vpZwr#XKjWJ?*_6oN0iF*~_i+l$K;uJOFsCM&3;$6r z#A1Br-u#Bw#MbPgK=hh^r3?dQmTD9#(YPm2x_1DToOkEeSpE06{5Ei$%^6GW;Uz^~ z;xX;1>Xnh5r{& zP6;*j1i$gnRCVkPQa(>I8y$=BZ}6O>9P}5EI};>UPzGm?iWc(-?wC)EU}SJl!=6gG zSJ*D4gMxYnFKw)5`K9BAwbj;TSyO6_D{n|DW1L(X0lckZF2Ye>qf`RpSs{s&+|VW7 zBMM63QWn&ITJ&t;Y4P43CMSwLZHZ{#BsdW%vJ$e8Z3|w!4>4(^rYh)c3!w^1`aL}V z?sBQ)hCNlVd<8uce~>uQLT1cWQDdattF<-rT_=iV^3s{K(O)2(R2Tn3|Ax7D!tN@# zsK&o>v2O~O{LQLQyc)jj)s!a5H@7AAh~> z&1+9y=Ttn5Fs|&*F!0gM{Hu6bw72t8wS{1`Vj8()U6@T=sCbd;2qeh}6x-iVRMXC) zw!&!ZeX$(pXmt9(~^BC|anS%pA}@`o;&t z?wslMr#tInT0Rhiz01ykSw@>yBeL>sSLCJUN`H#1eM-w3D5m4}QO-)gt;~|dW*NGe z>6AB7?4Id&6q8`OujrmLWQ=p0mWFDz?>N|#HsG!1z7k)I@`bv~u7^j2ORaGOac?&N zo?n>))D+C`%wJaiy#qPiQ&~qoS|2<6*QFUzk;vLU$24N^_1T)y_~x*0zuF_J{79wa zr`y*kVZ|M@$=-ilpn2bXh>eLmmFL!}oI;rm(lCf&Cba1$6h>|eHLa)O6UZIzqpvvp zm%)C>u~%Clqc^dtSzQ^8Tfdlr#xjUFcy=l7nRsp*hc?~y$sm{K&AUnHwE4?eaKcYC zY%H5!Je|-vlhVq#N-Ll~+GWx*F#O_gfBFnIG?66oWgI8RESG?|sG>kOk{Mg7Vfx?= z6(Y(-tsFwWVitbM=O!h49F4j_OvRB2dI!6C7gNW_z1Bt?}n&hJ2 znjRb6%+F}LlhNpq#IUX?6Vcy^(+7N&dE(sac(*uZ!#6(4>>f|#vhvSw3fw1O#qg|T z6W>IKJW|zV)B{%_HfA)^*dBJbF0RP;tSkjg8kkxxjb4Wlv2B!qY9p;0 zx}kyFN_>I18>}^=@lwAh2>v(R&~Mpk9a1ZOqLCLmXDSr*FfnjOy_%lXLx97fZVOJ7 z@+N~l@6FCtgE;W?E<5FXtH%fTa!EjPrN4BKa}?}U_r9?9>lCl(bj-t7 zkl8yw*@A3$Rpj->t{;hB-eqsU$9|#}G&mbMDun&~0bl9568}tmkP;?v1|PK*;elD3 z@J6RuHS|5(2jMOmxmZtWcFYHtkfx5Ar>OP?9A8~5VLVU)adXJSuMAQ;pphQcYK^JR;yNs%bp|W&1Gl$@DmO zGh>8d&lF|Gimqoq)iuRW_P3)RkZ)j8a|lZ=gB%lj|MSCNm+p0P#z3)|V%1pm0k~-M zx`Sa)Eju>YcWEfnFVx|k{q-KkU&^wJO4enudr)I^g|H8oE_J<4QBsvHqfxmyFv4cD zVzXgwB(xp1$%=K@IauP5f}O7ta|0$s%6=xi_rc5L_~J<2d!_7X9IA`r<+mS^h2KRz z#Arz*02&q3JN2g80*Y^2#5-0q6Zw zJSXBjk!1vs^>eozc%%xVo-m+lY!@}L3L5y0$JR|u;(p7+}yyDux*UFYNMVO?+|Ih8@-Nc^Evvp zIC@E=!Y{%gD1p!jc4-(yR8O3!cXZ#Zn7JV}um50bU>556H>n!;F3=w+Ik|S_lk(rB zZEn_`6h;nIA73B9*t_yb+{uCJ7LX22vVMqp483>5u(;yf3-%o4*!F$RqaTBo1*27# z&J)|&nRrKtn(!R`Fk#m;^QAkp+$@V*ZY4@oa~U)bn{98Yo2<>rRb!66I`6WYZ3_)5 zsap&8oj9|`LNR%){hM?ReP@R&-9GpiT*JK^oM_&STFmBO-9$-aEdvEs)DHK(T~<^` zS_>^CyLgtNm1?SviZ@8(_)LTk*2NbO5=v1M^UGTNa!Nq7J*By##gGw0qHoCTPnh*LL7|F1ikD&#gNVahnh%`U3Tgv|X)ep&cu%+CnJvFDPL&;< ztfbT*hZVPPd@|z7;%4+`s!CiHjhGIa?|myTJf+qN{S4v{Lsua1_BYks>L)=tR4>CD ztJ{H(n0fOq=Vj--$VO?}8>zdA27JczAp~b{9fOF)UKY`Dh(v797|7gEB^>4#QKDb@ z(cLJG>Zsfk)7j);H4Z+6i7vgglW?AndVp-0nhs%r^rR8r^F218nsiN;g~Ib+weSSf z>Y&UYV7qwpF2sG_RVr5BAJYUHZ8YYKzq6Nz0AdMrMT46EJ57zBZvs9f1$S9bD?d2V zj8Nar`szNr6Z(I|c3uXkR>m5-UIz2@iacYu!#!)j929){o>c^)67n9MA_YW9;N!4s zmh8n(1uY%!7Rk{^3aqiER9(dl%)E&|ylwv677Z$h;VO8rPZvir)TBY| zr9$hhV0;#tPvCS|1;i&o;52VPt(2DTe74l6kap)r8lKLY(;)PQ;tRc6dilyM;mouN zOCM2L%H7$HOr55cS>yX#mIqK%PQZFD;H z3#tUwRNjaYiEIq{Rq^8=4KLI;Sk^cxY9HGkZuY_Y@;$q*sa}Vl?A(F4t3{~(5nk!> z<6%|8ZDLtna@jMDd$E7sC#8&pZM)iS=t5P8uM#R;M~AyS&>jL^=Bf~f0q?MrOwUUYZ+iNGw#22!*uNqnqY)px zV!ZqhP3c^qI5ym-#D=U4~%do1wOCZ3@6>i)=gy^u!#I+?Ey zz#>3mq=%&bCm3bTrh88&lhzW-U{LP~SN|6PxlQ3qEMa}?l1U;2#s)_QCOvWmKwM|A zDElz{e&SV7LqT=)QVZNJB_Q|#X+FNmpe0Ci`_%+?)-Yj=a}IcYMB!fgbdxn0c=-KO z`3D}NFXouo!|JZwzw_j%_QuA%-)<Gxv%L1^ygX0zIDhjisj;2jIw zS%)<_m~%1@$7|=RZBe$dOpVQJFLkc_m$*L&#;eJD)~02k^3}Av!oADv`6LRetPbPW zo>Aw}`*V*2F#+W>ayi768RqJ9uNa3Ub=A&f4*1dAN1d2Q8PD1Hx5mSZ#nUC>eo>6la>PL&sa(W_@diAfgO4$DVQ)M!~HzV$qoI#vkzRMg1_h zgdg%AV7#di4?br@-iphFbwh;z=R)P&%nfJr9$)jjn0wN>%x!bCGw_cr8Z9~6-Kzaf zp%=YsK&HIsOiZdgPIhpMMg2r^q$C ztlzy#FD2TY-iOz->Yi(c(|E^Lb`Wj|Bf|W-tdIYa1G+lgweO#L($HT>nK@fq@l(NM zjc^cWQH$NXeP8}GnXT))t(|nx%D}0lqNGNuabA4(;?Rc3z#9?89b9{VpI&Ed`{>Qg zfTBUl0XBjj=0{J%Av1#Dm6NqeMfpudEu^lw=EdTZ&^j6dAc1!axxL;l`yXVke_^Or zzvYzZ2IgdV&0zc%h!lO{VPEp=FaMM=RC`u1a`CbGHuwp$6ad4x*=bntQ9xbEQs4&a zPJDf%&0jFN-8w;M4KXPy;cE0MzMmq-9LIb)$LSZ{8`V%1~N9QyXl)x82b+mRfzDop)Q^ zeXy_wXO&&RHDuyXDTbu*>2{WitlDg6EyhI{6YrR_SeSjRBzs7k?b9RBOlX`|Ru9k} zt?j!u1TQU~3^>=0`T>aLlwx%_Tcqd<|Bo7Xp<|{x@8`76!xmnRE z18KW(;<<@WsXUg^Z)HJ8PQ7r~^n1UuBo1u@d1MO&R65G{PtGcUgp>;?U!{MkD))Mx zn^&z2W5r7ob5lG$w6j}N2C=%L%khi@WR||xZI_H;u%C)^i=CjB2KkbBKt%`xxqPmQ z!k~ zc6sUqpiKD8MRGDr%Av@<`b~ z9CPH2&)>Yo=>oV61B(epm5%kOxDEeS=L2lc;>V$3ouB^E`Fvz7?rX;RU(xyQ&7SqV z{v|*B`|%!ADcAyO!b8!1b@#${Nv#~x4^cDr`-4&g^d%{c0udzgmDTAkkix#;O!P@!61HA@kPIFAVcm_$G6b(mH|pT-#kBMgXHJ)T z=w=W*3`L*0p{{HfkxZVPDM$5)QkPZEgEXmk>{aRQ8O;Ez1;1-qUk2_gIf#k!z54IgDl66GB zlkZU+uKf43-)jFp5TWNqsWOS{o-e7b_0z4^XAeS*wT>Uc@aZd0l%;s->Yt!a_ekYL z3}32kJ$lzooHV!vlU5wCxf8%9MC7cjZmZJtvL*rD??<=ks7YHr-aJ|dLUf{z7CIChOaEuG#h`lb zd2p&sV`OStI9*oePZINlTa|M)8+_ot2W-&UQn4O2E%Pdwl-|?&3CCZ*Kk_?&7zs`L zLKiWzsBV$T8rENhyQ>s7nev(&dH{q+?Z@}*&TqQmX`p)%<4Xzxq)^(b`Xb(PL!)@k zQn*o#5JGCoqyoMO=&c+vp7i{SmJNJ_e^_v$0+eO{pYMH>?Zy>q&^MS%Qx1eEX%*HkXYrOa!5H-x>R3#Etr#+ zy#`**W=paA)%EOo*%aP=AS#25Ot~%XB@G$MP{b>C1WP87obYGJxO&VDP{ONeXCDJu)Y6 z!drv(0^c9F211Gm>UYYRdleMzDzv0*mGxh2MEWx71C&-S()9~>mx08ut}R7zlTzyb ztLUJV&ww^2=pxihy9ih5{8`XtmB}t)Qi4+~Qvf0&*-SC$NyS#DG@IMhRL$x*=i6Cs zkG^{WYp+?BdL zea4RLU3Ihj=Ss&OKL;xV-)1=7oB5DM(W2j^bL=Kke6!kg;6^{Q9Tmfw{9Sb8fRVQc z`Sc+F#vShv(lWFA{Ku2OR15kaZKhdEMSNyRoXk+K#1%5P`tIgG2-SA@Wma*_$;|6t zx|u6aSPh_YZ8B4casd+3RgjaAgEA|XSW+nEpj9XC2AB9ET|m6Tdm@sSRZ3Ce&}jqE z$Q!~$WDr-wLk+WI8njCj88Uy)k|sT6HW?BJfI$Q}pXtr>*l#mq*L7xl-Vv?gm(-j` zS`VLRJ}e!nTXI2TZI1VbKy4a&t4mz9y0ucMEg^Jtg+81x7;;~%6HE3uVI@+crXg@{ zl?ki)@K#O7OtZt!osqyi>$Ej}%BSYj3)&g)tHoa241A{d588ZqlIXGzmTGLY$>)Dw zjT0D>FR;Fn#aoQ~>BfSz9V!j8>G3J43~Vp6%@?WCWTpm1l_JeQK>~2y=_|6Thy?mC zA!X7^iYTbE{va?8z!WF^us$+iv@Es+ilt0U|0)PsKw{gZOb=9yX_>D9{?YK&u}bb| z6ploz8Q33t;E@lfT-e@x>AV0LQ+!-@RO4#ut8Uo=q>X)t7qooTVzK@+&l-m6D)Uz@ za|8~IeJJlIO|8#*)5@<-pUWSd5W*wmzOY=y?H%XM60z4yvg zu`jQZn5uH9DGQJt-LqS(fgVm`C-{7OUO^@=T#e!hG(dy=Y#$Lwm*9^}yfm=y{e#px z!BXqUIEGB9jh7*Hsf8$@poArsSAXTNS)iu0^sMBuh%#OZ;I*Z!NLpn`QTVW>+CLz% zT2@T}#$y~i5&D?47HGNrH#@(oj{Ofb&OXe05r`|($b7i31U_M4;GpPaR_>taYe~!eshy=9Pi;w;Y+`ElHDY+4U!qG<%A#s$Fd5AQjTdQGcgq$~Zl9w-4;?U)rfSBu zv0KNb9Wj}DD1u|o$CD|5)cz(KMPCJS$b;n}Tfid>cU#DIgMgzspLKzZEzcm%)k?d- z@)6fbep0e}WmQ7V8t^p`N!#Q9Vp@4xS$9ktBf=lFUSi`4zcj~Vb7H@FS@BA9L%CtK zAMPc{G5s+b-^UugNDtOIJiWsWuJsV#=e%l%bf6iQE4fwPDgh0w3PUENT|^EZ;s-49 z!;bEEQoLW{nS%1DlhT_1O4H98Rs*Q11|=5W0D?Sl1&M5Xgxa$^NNy;U?SgcbZD%0p zLD!NQiB+J93$}?EZ9GSCld^=;udVU!t1;{}S;1XH^N~6Rl{hK-n1BZv{5ewfVs|rI^8rjJDOu z{&~$?k2i8}kAKaJ--;qF9dibXvo&942WZ>p-qYY+=FP{r@f$mL)z)>b?J@cSHNJ^L zImw#RiQsJMg}yd!ZQO=ba5VLSB@EmK;>v|FaS+J;kIwA1(;=U^>fru&G9|gX)VAPa zudkA{t}W80Qad;AXg^i1p&#ox)g6?TbjZ5TSLw{0V;Y3NPPAYo8E3qt%3UE`>+(Nd zCN~q?F>sirB;C@~6fi5>;dEGuRs%Sg!y3eVYkOE~@F!G}sCp`{<AOjAT0d(y$M>u*PoNe}K~w#KT4>7e%$^O=oT7QqXUX zDm7b{zh#Vs6f0`y!&BGU2zYfHTDn(`ZMSbW5hhnI6ssb8Ub>$j^clpl(W64A z5n?KQ6_!MWeKv@X1N)leRgt)M%JFYrA6E_>2B)~?hg>VbvHM#S*K&|@uRkRPHy!y` zg8#qwRzf?Y_aGX|O+se}=T`N5Iaka7^;;LI+sp(91ziK1=#rdP{XE2Lq1m0)l~d`Y zlX`x2r-5Akt)Pou-@JaQj2hJj=T%MkNo6mn4)Z3J>5uwi-H2^sRndCDiO_P^znBg-8XzZn9X;gq(oO|bxHM~#|K6~G^OCLjZb zf~kjBm>!e06%XzH*dbYUs@FL0qm@*Z&_SEBW3eO#IOyCGeG#eCWgNlt0|!07$zRVj ziz-(9HWz{t=3!RNa%CI(UHPwy=w?6IX6~t)3Jw1ZyKDXSe*dg6s-v`jTDq|i90o@= zyk$CFu=&%V;7*y;sq0pR(@N8G2-opDk=~^m-_AVCUgOsV-6Flone_2G1bkWhVR~sJ zk2ZQxwzB}^eD{^uZO_bx_c@x^+qION% z)8$?7Ae0A@hOhqqXT#fToMOaw4}6&`1IzY=2u~hr|E9cR=^xWM_P{XmK8hoQ0 zCsIMLr<8*F120-N?@n-6i9P3{4<7BYb?(?kl`;Eok}oa0Uen!RB|IKHW!D(;^ryp} zr*??8@VD_DXU$#X5Au9m%ycGKXz`}cM-%8jZy$UzUj z7OS^&;b%YWTpip_O`cYAw6j~ZnvI{-fu?wL1WczK4R){z%anfJp8e9K-qBejnKEU+ zgOn`V)~pwRa{o5zIek&{*7|rqOI>)yk&wOMy!l~5BJEH>9BSho z{0-+VzxAw)C3w<}^-@^JsiL85Yh=M?g7u@Ax}B{GIW+W>a{#z*p-#3pSD|=D3_x zO3e{HhQ`|{r$R9_?+h(WZY8_?0_y-Hrr+D~s4dMqEH|qw;Cek=FnH5srQ&+Y4VS&~ zWfb4%=c}{{8Q8KVmlfJ&!)KR2Z;M|wsT~N*;=&{94=qOuSnUJRd7qyZ{&vurk2nvp zwS(-rR5-};T6f(o`tbBaWHQGSlGI%%tWv93!s$RBirHGj?GzjANFBGqa62reogHF2 zj`A+B;GU|uq>Mq+D}E_I33KDSKvv+*w`z%$`i3JV#XspnC2pckUk7mH`M~xFvtPkx z^$%tiig%Nn9c(^!7ku{4@QObAREMS+>-D4}{g7UWCgH=e)BSbrY@X&d+CAdBfcjKb za>aWMy?yO!BnUxax=X8fad7t=6G~!5ji0lPgOkHVx;7=kiqx(#^Lsb&x_Q#*+FS##mTW4eQdWS&@yIsrdJnGEv)`~-KAuO}o0b#ADaClLd z+TDix8YGna^Hgk^SLLnzmQsrTwNEZtkys(bT(|>q5aqpif!k6D?|Zd;)HnQ}Z6a|G zJjyOB7Q77vZrZp#JUUYSsYa@%&u%Rrlbc%c55$LKeYtsMr2|idtsBGei?*TVut}Cd zmbYUpnHhY#EuofOpgfQ2RHJ&ae05p~@M(y79r{V&cBXIsWbk`l`f7rS`!Mscu`A~U zL1UsltwwhP4jkql{}bM}vATWWd8+siPhFmAe174C_QS#u;~uI>-e+}Qabdsa_TiM4 z3WR{j1PyU$r_*exOMsImt$F5Bv94^(%(}q25 z(g<=@AcUJlLPE@aXQH(G+kN-#`*r!04`j}q=R7lK&Sd6)j(!YdjA;(11Lyo{3d*M4 z=ez9*$5=ES zpXK(&r`HmXmFJf3KyT%Yo!o#GKi-m)*Sl>yD`q) zk9Rwr{Jiplsq1#hv$S2YLX6IhO9i>57ai?zM{wb6!75S(>nQi$RMmZ1z|cg*`cp*X zc4C;%=qI6Zt=9Wy&vj4!qy?lz%*_CVbdeGF7M{jsH63SwU7XnImrz7H1Y3du$i!FPc93MpS3qaz6W; z?Tb4Yo`Zh<6Z@Ar`>^s#duusKoe#5$pS`-zm!IO)w*=f@=-(6c@~cscL4n(?)@0qt zD>5I)uzV}!wSS7OZACA0cjxZkP85OFzUQBv&i-nd9rY*bZ?2iQejV~FKEJN`(y(=x zowiz9v(3>;Tw3)C!s7iSik{@7N%kgaGPT1IqjRe?66ej>yKQjbez3qKcrSU&-ope=A4cC9vr&5v-xfR;`t28tUJ>$Rrl6rLYJ=x(m`*gY6HtNuGMP2 zeZ)_rfyQEQtIL6mr8d(tH|gP;aotTeKEMnMppJ6oFPzp~HH!N>$XUy=nT~UJ31{t> zv|wNQVnXJ&0*I<~B7~ci8W7oi-E5Ut)}`?wxApvr&f%HpmbH&`#ktF*W*HAsr^-q8 zBO&uBlbsv>uSbBU4xB;Z^A%O7vqFCV()g2d2*f-0fb=!9fI{Z0zf5IRJtI3T&5CD! z(u_0sAM~!Uo}>F9Klb#C9-XGWw`iqs9En?6-=3)n{wn~E+Q3lBlhDTMcv7{yp=#-* zDTgb1V@_A)gM4mIVeQv$Hd1Zy_`(gN9S+CJxTRIPjo&%X?zvOx!+PnfRm(|t*S_1R zGZ@giUf92EdBX-lc`Vp?PgSX+!nFv!#E;B@UW`qsYS%nM@1bf3_l6joEpW%V@23Ty zZ;r1ABVd=I@Ua`hM~##hzB7Y0LCYJ=v>5z&u=&VZ(I%bVgD}@y+JlXVoqY~zDK+ym zLC*v~0^KdvaahY9oIpYOj@9{3C2$^GxvbAt+&_96d%$?cV^~@Yei+fT&-06zTfZD= zclwqBJr>~ZG+(JU#%nd|r;3_X_zBii{2U%MfOh__nz_}V9TRvE;}|wlUWumlB3}EC zi$mk?UV~fa{;i6$2Bb*8Kcs>Ph^x{tY|A-?f5T^pSOb-oU`KT1)k_Uv=&r(z@Cs1U z#bYWw>-Eisg}kJfz6)JaeaC{|pUo;9pY$M=CxxeE8uP!XeTj(O^m{?TE^GXo1vt?u zvbsO>!>)6L=1J={5nU1J%iX$Bw=KiERL^(Uhj!(?D~jwqya}P{q6$*HEOR&_3BY!m z?fKmBV}LHS*;@$J9#GP_++TLxps~YB{zcM*-cfYi4Z}}fzZ(~)bBFVpNo~5#2)*oo zt2_{yAY7IeXxJTVP4okZ9|aZo_PPBk&s+`?2yA-S(RtrNIY`aiHE1$N1Qb>9VWuhR z1_UZCpR=xas%iX}Dv)P^TA-lTu zpsfY63-DP0XxwR|(r+{Hp})E*Z;PUfB72d<;0jHIi%tX(ayD@{@ifzn!E92;Ho!Qn zkFnX+`yu4##^vw8e$$TUt26K?2yYkA3?KqlHz>+LR#SGcDw#FFY1-VbX1-emX2B9v zE*3%$=uuWadR`Rz)YG2nMKJ==MMja zOV32iII^b>?FIXlms$YBy4MZ}tm`OEcRn(*a1M!L;9*UW(%?Jee#haqBd+>`W!n8MaopO2sl5bAL z*^9zO5$Q3wHe)Nr+7@ZHM7zXJs0J<{58nskq|Fp`0vMCB@!`}5h?Y-u1i9-!`&&Ei zhXb5|B)133wykrebfE^RLb4#8!+U%dV;biXA03~&JlQ=N@39fEpt~%Fp|aU=MiL5@ zO=FHcRw-cVW^h#g;p;xw5ahks=nxPDqFiKM{&@gmO#<^iv(PYb2R~s2=z_hsv`T@1 zcsZQ<6eRG89q?4IaSw1Rpqxr2 za^8laOh$l!?TRHWHc`3Kznn#U#>W$7W4z4g8FR&`Arex%WY;{QWOWp1{Lgs6@;QM z1}%9@s)+2sU`ysaLT*S!;o7B{pS~&e{jDVHf$U zLhFXdbA_Zatr91K{Cb8F*tWyrWlDxR$Y};K^ui|%`Nxf zq?oP)5_0ooJ~OKjt*D&#?Xz`HSN4U;?oa1y&F@em9ylfPGQa?!Zc|UI&9Hj;4n9!= zva38riZsU({?%M|zpT_fFz+nfgDB6gW+#KZErjxo1(3O%D7O7)&KVs4*QOh1g z?Wh?c7qprIL!MSFSk~p{9=-9srSeTMVd`gE#BeIr1tpE27RS>OlkX^Dq;g1spFoRp zl$G>}pOFfUtyG3Q1yzEoZ~%AOkhb}N0eX)i;wn3Z)g5>{11HITb9YvbxWHThiBSo~D*U%Ie;_>rrHkQ!6 z3S1Bk5Totm@58}B@kD@r4{0Wv!YHOse3E4iSyW8)>${|?$)*whg$U9)cxC=GU7m-} za{(MX5{F;W=+Pj8Edr3V)(lywaBAUIa@{=5nzkQ!ny%Pfmgb^UI#A0KoqhpSdV(Vv zXt>{Ih-CAvRGzYCBweHdf8lUnP_evqc;VIQ#C?oH`owp%$BI|%c3J}^^3ARdc)WU} zN4;Xb$dy*RP&N&3slaV7Bh96jz#Bmcicmscw?_T{X|jjFu}1S``MyDDZ)-)qC!3R&$lw`txy|s%@jckuXyW*vHbU|m5&`n9!xDKS~%zYL_ zbMVH1`ZfVT>k$iEc-$l4Tr3w{7>iQ2h%r%3?WGIl;DSZHQ@(K^yQuEU^sA;b&-gy*scvqN?280W zGpqhoy1L`7|BcO`9h|DJ_j7h84!o)>fgr7<-(l&U2NhN&k&;y#YTk<=jK*%FMIR1f zr)iafHzcfPH*JvIl)OaDwr@oXr<$drR-W8dcr6O;Nf59U^^vVZ$g<%Gk-e^xJ_b5# zUOm@9Sc+QB8$nqHbe#t=Sv+fVwlWR$b=6*pXDqH*U$d|N_Jss literal 0 HcmV?d00001 diff --git a/docs/cugraph/source/wholegraph/imgs/distributed_wholememory.png b/docs/cugraph/source/wholegraph/imgs/distributed_wholememory.png new file mode 100644 index 0000000000000000000000000000000000000000..e6bbe9f13e9e0b38f5b3459c513ab0992c177c62 GIT binary patch literal 23138 zcmeFZXIN9|+9(`k97j~J&}L|Mdb5CZM?@4UB7~lR6p@;t1PCoyXhA`xN>!8~0RjOe z5F)_>0YVc9AQ2dW&_a=fPJpulvfsV;d(Qj4=eo}Io$tpN^)gw>TF>2|yFB+Dd&^k= z5YHbxAQ0%#jq6u$gFySOKp?KL-+l$IthYUj0RH3hyRCl(gl!X@1U~%Y`lr#KAP_F$ z0Q=5f;Pd{!u3P(oKu7&K|F{P4N(6yG*2_1p{%IZrof$aPaXK(eVkP|aqe#9JPF&ZH zA9whLi{)6S;B>>o;YOLqbX?Qw13jsHrf)%E()7w-zJ;d;&GV*uEPJ4BpB0;LLfYEC zDmQl1o3WouJ;-|yd54m`JO}L`J^2*pd|ddK&yB$$-bj{z$>zG{vSB;$hZL#}es2<} z)1@__#cBx*UQ#6u(sc3xKzCk$A5oA&?%DnH)YO54!}hD=@_m8b_m{c%TI{|D9pbm* z+Wm0h*8hLQ|0k%_2QXE1U&LVb^5(v{gu%u7VW+?WzaOkwMS+}^Z>e!5p);Gqvde?2 z>#Nj|ImOz5A#VCnkSE1SeB>a7paD4Mpv<$N1oc<>Ty zDd8Jr^Vf)1gxPj`YvS;#i8ORo-m*0u>WjNZa5_ujYS4=}Nj2OjEcC)y48ffJ34mSUg|RJD+@WeTOh>A8j{_r|j^8R>nZ&;@2_o;)F>9;t zIFJ*r_qCQ;P%oRa)jf=&D99tZ>pDWe%PtGfu{&H*f!QIg2^Ormtx=)oA-f6iIFz+D z=m9k+F41zpZLiv$)c~C4=MyIZoVxOj2DuF%5{EPaZS*t>)fmekTpApP1cp~;vPy+C zXHYD8nVgML3Z)wKxRt!Ml5H)1Rb-7Xbn2O8kRoX<10{iIjdMj@-fHxXa7!E+Inl+1 zHbR>`B<@R7K@4eep89QW^ZR(5L z2$8;|5tm^?A^|I(QVb8CA1Og1=F42QaG9SJntVaNzM*N(>L9fj+S*L<2)ogZ4A_>{ zyrMDkWhP4rF_gVW7Njz^xlckSeu#-39xd5~s*TsZR;Ycu@~oG5&q83MLRQ?iZLb4H zdoi*^P-n4nxwRI&T30=NU2MHJm0uEzMEEpSPdltTtAXMg=kGk71j(K3iZ&VKE^UP6=ZGQK#w_r~!*H#4$kBFPKaYbBuB@Q78POg`MjaaH?vJMt2Q-IsoG{CH$Y<{?ULH| zq9po2Jq4Y^&WfE{z7&+B zk&2r??Bx_{@0Dcg3IUiv5=UwPDVs<)oRdKCUG5#-aO5chS<%R|o1Joh4TX)nQq=V3 z33H)K)6TODh3cwIZcyQB`MN|Vy~TEmMdo}fW{K>QbItant0wtbMa%7TFdJP7Xz~ZC z=%JUR0pgpDZiFfv`K`{zcwTyP}VzyVR4`G)m;e;q!Wc@LEAe zo9s8*T!ogke%$g|*PqR9U2n-@X<8U~x(s|IB2NPfS9H4nuzd*d%dJJ(H)X#jG0TRL8x%$9e2~LBbgf8I@E?pc+Ukj(lgXZcL_wu$vSQ2K z6oB1o;+R^k`U~F6IgJag0M!>_iI#wP-#qdhN#6p%iI$&fLJ_G zBBRZoR1}n8FXXhKk&A)a%Ijk-=d**pdZu%8zU-Ci*O+fGYB4Ib5736Nvt%0Q<`;%@QNYdZ=*(q~ zHcYI@&&g=dS1YzRkmhlzZ4X$0CIqP*0)-u}=3nJk6F7-KNxF0#z~;Q#Ai(jL?aslf zqgtRa6*>oc5qgh^Pa+FsDc5#V-kEE^@7p>EAvceruB z9uR2V<6XU{=D36(6pXzGxIVUq!)x04um=&jaCF9)w44ZVhGX8El|d6F7fMc&Pqw2CQn-~a_!2qd zdCz%`2>`2m{@ph1U*`nVLMXW0mFSW%xA{(2Z5%-OO%xIJ4f6(}auFc$t7jvN94Jq9 zrAh)&rWepTP`Yn4!?y0sV>NU%M3CL}LIN;$worQmm&0%23n3(Uiv0I;s7%=qgsP)+iFN zNfYSI5oti9I+_;hIoPTuZ?opqH(zoV-OBXaU!#%rUDzN`fGMAjtO}@E++cZ4bAVN) zi<$t&EXo+Icox8xP%z5DR^dm-X1EXKYqaj%VlM;|&s8)#+LTy7I5@b>Q6*jE4I=ebmC`|GLpcdJX=%BmMuJ}Z0N zo*5xJnTkC`Fz%@NXfoofM>L_&oSwO!V0!TkB_-1@O!}pa(<&arQ;N-sB`ZlcL@DkP zUF*itE(u(PvE-BSsv{S+;Y+_-`MD!WcfO`_T$NQO7v;i|^G;PD{o}@1!y)X{##<*% zEwlCo#Hf}LrniOA+dB2Ge1>uNTcx)cwRD>CSFI7n%Jr){HP~@Iqj7yEZX`_tMh;@1 zeYU&~db&aZw(#(IOJslw8@4qEOZ9tEcHXsZddR=a$n@@Da15>84BtZh-uUcP`AnMR z`;FZ7_Tq9+1xT;vPIo^!8U?aZW<*ul+@)qTmw9HECv1bLBRpYnse#JL0bj876Vic1Fqz)yDPj<6)^}#P9RO8*_NKr^_Q3mx#R|cp_M{ z$RDqUdk0zifP&c}ivvEJ2@OqO@}IcnGx9_=#jih>7Pu<^s4L&3z4BCLetYMPM&KT^ zc)06t9`ZY zObh=zNtzyHfKH4%7#r&C2Q@2T;%}aVpi52;T2_YXs3(mJQ!?Dq?Wg9N)S~UEl4nThdqx+L7kZh3jRozI78=Hojq zoN#m0dekN+Gma=CYi(?hqA02TIR4;oaGJIWVT%=ipYLAH5pwBlb?YS~X|kCcn)J8Z zkW`nX0(2Sk*Z_=3biE~$ldX@YEEvU%-+)RTCSQn$!J*K=$F$9T@}HEc9F0lzt4aU`3d$Hl@y+NfE)A%pCrrW-(Dv4jLX?eN)npW{56EE$Fx~A1wxp89!+OX zdFTyc-0Y%~w^=Q~O_0`H^;m5V&S7B%Y9)Dy+3ovSx|t;HH+yJq zI%8iiBoLyO4`pk6_kuMB-NSsgk_oVxr4L`rT`4#^!BYMx`R|qa5qkH17u%~Q*8OMI zv-OSIzo#K*uVORRJ9=TRxbzFX4QzR9Yh%3>MHm%Ut8kQj&fYaO)jJJ!>Fzvnmcfc{ zO*e?ocUC_!({c-t62rpGc-nOPrp`MNjtssIp8Y&7!Wv8v*AA@`)%bfQzMcPkm^i|u zUX9Hh4MWNY4~`+)R{$)4ZAr;*Zi7z{j~t!#ow8fe8rnZ#PM17DZThBg=sS01#f00J z{h!MS&>{2BtkWAHhcB_VP~lX;R^?DPtlpsB?2{*Q`6np=Z0%>YCs^OV2QMuxDFl@W z=zT%lxiF#<$+5FCtEnEjRqnmx(H!V7Mz1cX+FQNOL8nZz8bf5-z60AD0B>NX}dt8u}|gD_&(WCyWw$OtMna_}qxjsjd@P zAqf`E|E@u*Nj=-jq;?QX(5eq+!UJ8ts8;irJ+U3N8MQsr`X2X`ZtlFw?2M0wgt1iL zO>g$ny$&ktM5%PT$iU~042Kh1XrVPV!3l?lH*V3|@47A0`hsRW30minU3M?@E6DOf}rM-Hn`A1L3MO0#5Hl-Qtazlt+WV7 z)TwmR@+jwd-D>lyRRzm3k5eqm3vM2eJ0)Jz*oh_{0-&!l+O$$FV9NDCmc>uZN=CW# zd_A3Fg0G>B+w)tzn+*LB^hAqnb}s8_PQo-HvHn)$!ltXbp>)^yv*kzZ#eRmD2D3JU z$Oj1*C&}!kj$G*3-iQrtwAcXnwW3b$YimXyYLO3>wVJ+m@Bb_KL685imE~c6d~uX{ zk)m;Df6TFJVHr5%0IscasIlM}EpgVBtv{vS2`_dWpikyp7;0GI2=h}q%Cu zQnq~3Z*!==e_LiGDvbxDu{sGbS;eX6%e#QL0HM-tz-{5=Vd%nUUA=r|Yiust;M@E| zO8t+BhcT0%accoLGCk{?Zn0gRTMDDnQnO>~o6Is22?lZ**z*mnL~Kw|inVLrtTUEE zuAXUHLK`2af|us`EL~&*Hq6bjHAQXNFy~N0hTjy^T6acBhfZ!y~KF z64={0#R~XhU|6bKB09I zN{*2$XTyKb$8EcscR5~es#C49E@@Qqx7Gchv3WyxCz%$*Q*ZnsdCH0ZV>c{ujVQ<;3Qf3V7MF;$|jW zB#))7`FQ#0{CzjG9x{7@yT5O#Do4+8$niaV^^eu;*^u-mFZ!Hhyo3%k{?EXMY`|wT zwr|?u$KCez>4$eRwmC8>i_qC5CU{*>ja&3$vQ{P<1$0Ge3*jSuYsiTCKRh^Fp2L`| zeGLiNNhzdFMzb=yXlB1broTDtDeHvJ?}cW8)-_YkXM3LI9AB8!nRe9Z;saS~{T>4! zB`X-KIBu4$58V~9P+QH!55kQf3=5+ri*a7?NV|^6UB(!vD5lH~sSo_g91;9#;ev4g z5}o384xapAR&Twst(L63z$HynTH;tFUoZQ0mZm{;i_ zBhzoCestqLeCfSn4p~*;sj|3JE61&E38fFNH{)`Hrt=bH$PTRGN5!*`&g#6_AGXd{ zQ(@MH)?vpT%fcF~cOENcKSF>f_krM&lkhd)b7cQFt0`lHR*k*^E{)bux`T5Ey5{uf z+Q4UitMj&%e$H8J?je<}{Sy3^IHPTGakX>~);Rax7WZV*PflESPO`ykTU=Z%N5eFn zTGqA0#3kBmRQBFK*lV?8INcV1xTI~qWKMA_%X)t6wb|tv8y2&fTB)&+lPK3ktZ;h8+LI#M;grg9=4vYGShL>TfHh{!e$)s%Ik9$m*u|(XZ{T{6EcfXz0 z7B1HKJz){7E79%c)I#fyt}4uwTcTTi$=Vg;zIu<|TY!@0V&kpT>A0OHhDmmfuywSq`53OPiN3boLE(-N;`kh& zq4b0+kxY5^B@QLE7MirkEHroB}eQ`P>(Sk+@P%I@v^VP(= z!j90=!VinlhL#@Q1Etf|Y?dVgGm^#-wen%7x@vH7hV43^{&KtQfXe)YKQ&iV-qb#l zildBc7wG~U|6kC}nItcK%uMq++WY1vT|x^Ql|1}L!G4*T_?Z3th)*tmf@_XRypxF2 z`ta5BRH~|rrP^0x6O^h?8E>hZQxnsLiAWAJO#Rx8_hW_HF64}??kmTI+LP2oF%j}eJ+5C6X?mS;{jD& z&OB&7eELod_m}Cqk91vs%E8gt7uYAEDy& ziVlqR{mPdUm7NjU?W|D=_exXwuBD~I<}E+sB(Cwi)K+OZ48xZo`|QOe&cEPt#3Fq> zy=F?GQ~}T^zj;k#3C7=Sk+iY}xGGd4>yyrNqWzG9c0#oBZJ>5!cO2HFout~k+C(LK zn-u5`T`fpdD4)E7UbrebD!&jdqvNfdr5y#CcHNXX8mBF6Oxpi$G7%nO^oL{0c*$2U zR<+k)ey5AdK(W%6Npp~##n-Xg0uAp;mbcvc{?ePPYPW^2a1|O#q(Gd5!)tSpvs_9S z+rEz{E)VoIjL%c7=y7u$`+)R7#DVUx7!eqj7V3wdwT+=Y&#|k&F_4>>6Pn{lVjX*8 z7wnsXeC=|~F@(8pz5E7SQjBK7){54>irKi#A*L;l^l5!#$3E1I`1q3Z4C$v)S)y-ue+aXM&i6}$+o8j{N+2s>s)S18mufRM; z&u6$;9TVQbIuT|!V&y&JEas3Hl}2Cn2=Ih?nhaDnXphqVSg)DjuC}Ud{=i=GybYDh zLiVMR@*9!%1++PB0psyGvYpIf2d!rw@PV+C2TtNM&Xvs)VHwVApMbx^%I!I+ZST2w zFGZI~&Z#PfiH=y2ohsvl+7B}dvhf8|t4}2o*+IG`Gu(A*_4XTIS={DrSryY{S|m8y z57tJx2$6cD*=IW&XK3!G6 zI%VHm#yxV1A<8{Pb!j}OJHQCP4l8eCAQalaRO(K&L4wXFQ!uul*-*=+!p( zT9+lYp;|DVM!g%eewWyg23H5lu}_9G&W{hqq?*mtpBo=EoO@Hus9sjRiOk5GsYLhg zg>}tfZsNiwosJCcxk2JON=n$cUhjo@b(DN|>za8j7D7qz>Qas9cVE|xcPc9!m#;$~ zl&8Fo`!m}$&irf7wd7b_OXafsPy<^e1EC#FkV_azbaeslLye@*>YKS5f%*&EmYJDW zr@bX!45~=hoCb`Mf#uDJfRnnJpL7fdVfvOiWU^C53oFoa&g_mDN8_r8PfMdxhFiTe zl}05Gofrd4`uNbS?$RVR3xdqDz5TNMLulR}p=nlM>o@CtjOY^H4JsD}(!lGsS*4%b z5oFg?N1e`NtV<^#%Q|-p6F2(C?H1WjoXRLa!7tSG2p-_2E;Y>bhXWzrF$qreNW+FA znSAn2_MHcd-mHs|^P8K|Bjp}AiAh!q60@gB5xQQO8PmIk)@(n2PAB2pS-^(rX13>V zeK}Yc+V-6f$T$^LmXx)P&#$snQ?&yFSG5H!-UZI?7oX_`rB%J}9k7cRsNb}ONs&Pg zb`6<?W`hOq# z;87sEsYt%(1cS}7|6-0d`Of};zqiN5`)BO}RgqukexG=WKJn!>9o8TBvA|yp@x^d) zT{oJyHN4XVo*?$4RF(Ms(765h!2v#>P%Z}%=e4{0I0c+WR_j4u(TLjT8v}g45)qku z3irl~X;{RIT?$+CSL+$v&Wv#P+W8r&1svace=>WsfDktj%qT9;sXO(4wH2m7?)!x- z6yfzuErY(Yr}Pl>?s-v5(3-(Dc)Rl6<9IyMSWR2XRE=0{l<**51V+RGC4~%c!sL-i zOV;N~G;;=wA?UF;&vHrP%T-Xb&U0M9OA^o~0sS`vC zZMC5eGQ1VYZCWOS!MbWa(8X_IGw_s%mJjX3%NY+?fO*nNOR4C#!$Z()uf&N`) zDD)b`GCjj{*B7^WN}u<+dboG)Eh&`e@X<$ZuMW+8c1jb+Z|4to`Sv(#PhkpX&)d=# zmpwud@lxNZ?gDE8fA+UuQY&pCo5EP%7uVA;6W!*$#nS5MFTw*2tZTy&puX6NmhSE> zX@RWlDT0=DtteN-Ol+H>(ZGw_UkVDav_v{ho(~juH0wn=;e1}MnHdBxNl}CfM!4N} zpmu)$6ivZ#e>sAA!tO+75GzD8OnVm#XGZvbLRD8PUFz4%a&i1 zlswT}{<0vNmOs0iEJ2y5?5QY*Cs58M%n-wxbHpRsqnTy-GBXEI$t3@ zR&YVUvX!=_H-#$fv9>>>fK2oDAk<#sh1Y|9*3n)~^*(W4?LZpOslOurn>uCJ(6_v< zbHCws-zWyF24^}Z5R45?e-qHHZBxNGrc9hQFP6l-?V|KWP6fKXy20OM&g#0;H7sl% zX)u4jQuX~-8aCYihT<2z*QAq`KqD1yWb+1FB_p0s*nrs{P7<=L_k}YkIecE`jMm=Y zM-ZHH}G>X!mN2Lw1g&3-VC4HmeE47&KX$ z=Ye3X7bUJRJ>_Oxf6AANw$Pfck7)N71hPs>+)y6v=w8oBr;ItXaC>DjM*O%5kYnE! z<}p);cMqige)}d;ELjCeP+Ib1M)gkXcHGTS^&zGt2b44)Na~Dtf+rwl6uZ+c?Wh(6 zFQdp?4EORjVn<=?ImA486Mx&<-)`L6)tXRVY;0_1FAfUG<;!{*MbrdM*Kw5E4FM8P z*YT}`P`{u;YvPclJTM)^HzvuPUN9-wX91b73vc!Lb ze}YpF=(~@Jcx~jR0J3Cz5YQRmGiNJtIggSw4G?!n-#-L6N&n-3TX1OAW!O0LlBu!SJsI&A7-E$SsvuVwt;qbCfnk_-Jrx4eQ(sXI zOcN&VQG0W-QcB+*p&biUB5yO~C~GRamp$@%zM{e763k6t9RkS*mtk_bv(EWl%#pQClBC@COVxHdF0&RQo zzsv@rqU3mzd*qK|Y-<@O-x&GV+fe66$iQxeI0ETqK4@?$Dwkt?HJ8jU0LZ3+1Ekpr zDh{gjw9U#y*J^?mnkv=Ro*)pQw}2TkLg>FT$Wi3$tL%`J=S5NIt$qs=U)l)5#huA+ z1GzZjEopUepo(|*`%|Gly56-_)~Wa|b>=u&Q=w+c@!eEqOHHf#dRgJfPe?(JGXPkr z@P^3|I+vyNLX!snMfomtmCPLR0t^z(D<+fAdqKd2KJcn%@@10H)B2G=_Fj#PV&9Hl zt!xNp&O8n)qV-4}c2`#@keW8ApGPfH1%&I_di6exd;KZu)XjRY^va9jLimF9uyD`L z@osfWnOFv}!6pM6tfb1tFFhzoj+?Qn&`U=~3~ zbsv+TqW0t>DCQJ-87vP4D@*c%d@+QA1S79f2`SG?LcZ>)4z*?Nc^#@TqJk@)+$0C_ z{{~*yF$tH`EmBNr@2zec#vGWMj=&ZeE%mZMTeSPn&rRBFo zY<_X*07bBoKJbC5U_lM&5z7f35)%7lAWA`BDFyddV+iC|&tW%XW(q=}R5u&#IZ+GO zESImDkiZ9jtZ@uFhdzq{efI4-AVZ6|9m9ayQy6cqCl^+`V8#E@Jn|NG$-+^cCxGc~ zOg%SWgLX-gwu+_o2#lCBQ0u7mB2+*+Dn^D?S@ML?W`%2W8BL9Ci3`3`R4#+U(Av@@ z)@xdDkJr7_uT9Rk9H>)y6)Y;~WpWUg04G-kdw=#~I`MZ=)u-aW{1Z{i@e<%8-2AB< ze@vjck6n1wTlCNIRDWF9L0k?Tl0J0-?p4N)q>#$k(#xP-r!_#kYe;Q<{<;w)qd&kc z{RcC|>+@3L7h zx)PF(CbFBDHEdrA2e?y92yo(S_w@&GG_G{7ZcwWY9#u7)k*RJ9FDLR^fCheN;hk&x z@vIkv-5)TyiD0$oFSfs6{O%W@Pv~oxX)olp5XgGlW3_42+ZNvF6R~J?rfYn1P1yWv zPCzMU!ucJW354-Qu*Zm!hyQe*7+Ad0;kOcNhQ*Pbxd7Ah9Cg!TJe%lrpw1VlsD8yR z&4kX%O*w$SG+{)Wy5)d->w+6HM+y{{%9~llhhRO)LV*+tWmy$)FA@gPjxM$379Neu75P!T?!0WhML z$O`;_SV;^nRwSctU0nGuVlvcObOj=b@jYGsky*=ipfr1DAu zCcR__sPQC6GyqWy`!7Us?HU<)K0uEYTMqKGk5_HWATzX8nA_MZ>@h$ZtsN0V5qquL z)BH54_2CM@Iz=h}yw1z!0;ha7Kp^Cx~~8ZS^w4U+V%a zO65qD#*RDyT9?s(d+I7U-q}WuWEn*;+-KFgvyKGq5r|)DsN^)7ghnj2Fx|k#u3@0ReD96>v}l#lY5J zGr31N)&USxMr~8UF|aCy;jp(OOBs2yvM@{|XVPe!?8&EIsBym7C2$dw$dS@m_b2or z|C#A%(S24!|B%#enXsTJa&%~I8yuW6ekeS|2iO9K8$@75@>rhv8cuq^d*3dcVXq$w zh#aX0n_i;!JUgY20{o3+GbQaE3$YT4k4xX^j&*-e9~ZL)rJNswH3htbjr+Npa>UPq zejR;xTM5sPiJHK1IpFN^9bqZu2us#(s8UA4ZtL;VHYPjF@%$WnOtKlrZF*TCm0Qzq zti>~CURWlYPeOsb3))K^|5C?OZOEImnR){S#@U+Iq8b53nGbya_^EJ1VHavTVlgy& zp_43NL{)d7OK23!dfWVL@3v;`+7cGq)$hStM8kCI)K4wF|_P>eqnOmepl)jLELl zS|DoQZWBUqo7XQ2oYyoV5)T*bppuxye_FqxqbpH)k`&vH%ssx-X zm^rsBfN$-U&@Br23eR)=3dfi9SB7dSL%bw*sg{w?fpqvKz-HhmrB9cUYH{fAbyBAd zYVL@s`M$Xn7QDIl&4>y)K|&E*GfVJoi)6yvjxlWfV?P~U4;}s-8%eVO&abre)A09n zCR2B`Ms!`cbI1JK)_MkaXVD?u$6cO(HZHgT2ufn#6&@9r3a zm$RW}k{D_#5Uc*wPk#l)l~~7D;F>?_m_2AI1x%FPx3fIN^jTJqEli#)-cwO)W$FJo zJzqwO6Gru@nS8T^_<*}ATWKb{*!BR}ihSvgbc2qg)d5R2#R;*WqG&4c9MpLGIfi@x zET&3^+Knr;fxV?+rLYEb=X?BGFZOBqr7% zzpvkoq_<1hzTho={@*rreKAr>%^C`kQrA$bX9@!#sE=<5SMt+@NH^!sqmt8SC3(<- z#j;?5Yjox+ZV*j5VF84=VvZWD2UshUfu(vj`_MqHg6%G@Mc3{Nh#ZT&YAR)(FH?c? zHiR_CrM6`kZ&}ztzTd`TFUtwgvI8^S%Lmongo-K|u{3b!OoVovu@QVAJ-6J|8m|xspU@aH5hJP9iDm_LoT@k+dDB)D{8}G$)m$k>_Yl(eVY`YpGJl*Fa{A zAb(N7rao_$Fk71>C{Bf-ON&26^QAg1?lr^@279Olye#meo@a(2k zFQ4CQ;i+eHFF7y@HJoPz9rlKPH)SiRo7QjW>leGnOcEp9v-JkBDW>pcn+tCsB`-~) zfE3fjcKn1%>`F@67a>pk83=?`764(bjW}oufU7XK12fxs-_)5(rSGBnN1#4pq;gRw z^CohXOVC$@*A-1S=j%n{x_|{p3;!H?WwJ!v=gO%fGH|f2|$xWsJ*`jwdofjtmCx)Kv9=s25Al+#NZYWBQ+*Zx4S?(h5ZO2RF zn4Sqj*_2Ff2uFW58A{Rm&BbsKM}W^bp%o-K-O{K-!m%d9G9H?pK~mnD@s@Y{mLy< zhx&;Y*??xndc5%+Dv`2Z;gsUJjNbbPenJy5bK9ZbcsmcMQR*#OFtM2=sFA043!)La z@|RkG1GUWz*FG75)^kw0syq7&wU-ki?WP8JcT)o=`hUjPJVkvmiIOI5Q3w^&gULdk zVmjeOQQN^>%4U7uUNazaa|b(v12c$L3{L*ssvLZPuuGPEFxXb6Z|E)H+#Kq@_uA>J z_iuqU1yY9(Q1gBa9}O}kCfQrewY)f=FySDeXS3?E!Tg9mjHmSezSkCr)`5ud4NYKb z_B>B&u9ZyhQl(cVZnioDX6uRMkDG4gyNQ)ADRM|D1FBg0X{*{h}b?IAs{d?@1+;C{m;_ap^#Lx zIuKU?4o;?A$d$&h23FL1$}6c~P$zhvImXN{ET7^kwGa1spS8|B#{H6=4uU>X*>VzvG{HHT9Ym7ZRQr|D(?` z18^>~Jf=X@H`w#$XXYHJ4Y^ojugVe|eg2Dik+wzy1t|mL=*35D?eQ0Q@imv7O_i)2 zRk7_zFQzDFKpM3ocz3)(+vLfg+v??3=79uxkl&N-6sc>r8W6_(tIMG=1b8ir^ehth zc0f{xdJ>!ax=PdmdK?UQQx9GF@p1Siu}%hsn^I5~W_Xs@nnO;Q@ypfc9qf8gvd|eG z$CFOz8#f3h74$_tg-LFxa9lNAtonBczfp0(xniI2wp>5krE2mYic4P|`=_X=akKU5 zPMJWqG`E+zM>auC_u}?LW)H!FbmzxZN!6LT9>|%Jjd#PT;ghM&(jbeAV|+d}{i-i& ztzz z=sKVsZ&y9v=mS(^mI8Ge!$7|O2g`FPs7AoF6^=wghVN=igcs)!#Lw6Ne#fmt$6kw& zkdR*=eo%_p^`3>%9|U&`axZsx4~L-t?Sfn>rw+wi_h{z*>=vnAj5a0X)rlt$}z2;H5h!7>FGk$) zjlKNyJ?Q^tP4#6RhFj7kTMmG_jnBz{eFmTE^hr6AyekzEt?8ck!Tdv_TEPns_GE^} z43TG|tn9jz`<5>W0+u*^WAqBu&*gXUcT4s!KM{I4bavpw)WT#J-Rk|y#uL@SpxUZ? zsJ+Ur?b-5VUMZenE1Mv^;gLaLrwab?4|i~la%NBJ;d+ZNRGup~p(#SSt!4sC@-jbm z>FBV`Zi$5Wxr)(xurVa?T&uHVbuR-j`ZoF^A-sY}gpAfsKNs2oA ze*&cR3r~@O=_UUGcrTEURgc=(*wE*<(R)Ts;&B4b+h5e+olm^$m3@rjPuLCGIjg-p z90@=plXokMeH00N?bciPH)0pb4j5074GL>fP?`G3NxffFl9^jX{ChW<5fKiQ0Un*w zk56)%$<#!YT?$^fhr{Co!($Lesn3BD0^UouCsiS}LcYToeNYKF@1XKhW-KQD8iLEHMZffp|f;_tk?l_8%rHx^{yw#&> zROX%|jRr7+0_*1Fck1X>AESgDv~k0C@>JF-ogS)sAz3 zi+Th$RZCsDfoE*z_W%42hJNm>(5jva@bVUM)R;oSQXnRfBe%QMhim)!7d@|-eQza_ z<#be%5%t<}@g54Zq*mqFMZH&Mgh4dCVI#kN@>U>CFF~KJprcIX3oSuNKDHS%r?i+-bu?8pdxPL+7>o3>yModFaNGP2$uRha|LK=$S8Sj zQ+ss1ukyG2n=BsKRHY z%AbQ-BhPh8(#?@KCzrsC!faRYE3nHK)>)mX)uFSy1U&>uXv~$zL}3}lp6lSqQ`J%n zF++J%Ja;-DB%>+KK!jlsXU6!*k71qgP(^hN7vkqeMo}@8p*`?E6Z<^iszJ@>DAOz3a^cS0V z@pHYGNPa8%6DZ3bd91svypo$t(hZIEZn&^PoLBB|(x`^qQ@v<8v){U}@l&t9y6Nd> zvm)44BPZ@om|IZ3eju=umLGX%az4G+3lluuf3QYQ#pAiR8EyPJ?LmP-#w<8_!N+|n zK3V%N9Jw2K*7-}7+8pP(;&=&23-q80ukQcX z2tpT9-qr_zQ|}frP6~Fd5+OI1t0&LhpR7bDFo+}6)D1>Iru~^-s)pS)phUSG*s+kd zyX$wouESK2NIJ5)2GMCkdIrjJT_^Ho-F(akFS-rVxy*1gI7N7*g=t+yQ@llCjALpa zmaykFrcZt-)$1l+UcSfL|0^l9CWEKU7iZ^hY7=lNoE=pC>+ShVc?o)(c=Ny;`nxbK;iCjlc|`(G|J4C zy(=On-tu%E;)xW5U=-#IlEo9*^DWD=NIO}Ec<+hk1V7q%knJkYWUI2%v4O3;yJ4aA z?}OX(B5#3T89_tKJh^Uz?Kh|C-43MLtyKje z>;mNnQW2?F4C#6<89@qf1{yFk>ORAtr(N%{{z2}M{5s5d=asjZjx1ZGfWv>PI zQ^B;+RAxr+S67jGW8fjYV(Q&f;k^TuV1_1WGcWcV>= zm`Tl5YE#ZghKzr6f{$jP*?)TGo4wj`>bLYhGy;c8yp~@}eZFxZxM%WcQs5WVy4O?{ zKkIeGS;f2iSMx@vP?3(%X*1}Ac_59bYs$Y=WEPR6s8Irv4D%clqlJo?QAB(DDyOQj zwv=g0CByhLk^1l2FXBW%a#c12NG2*9bs+~HiwB}sW`tYe;(0cto| z*ilpQFHB7l#P-X6DMj;w+4H2KG<$)L(fTW^DxXy>C}XrNp}KCk9%{w_CbVo@?3aA< zO}T4(ic?%(LMxHi5FwQ3Uz4JS#o1GRAl)b2g@)DamaRtDp9xwZkq9jCx29I0md@+dyeF`0N=3Oy1C;kN_>n-09k%l@6%d`%Sua^&W__|OpDckjWL zuQ3k!vy}x^kS-Ok+ZkR5CcQdOVhu7z{$M%9gw0INe|r?vQn=BWU}Q*ss@+iY6ejdj zW|X`_)=jIGFZ-zBfC$@QxYLJ9pTENYYN7h+uThS+NZWr4k3H4!Su_)s5rJdPnh{1* zC+F5MU12pwXjqFZFfs+hDHvgw{N|qElb!rswv0-Eql;E#PGa0h^q~lHpgJ_wd+lc1 z5Lb^fr(ERzlHc;r<=l{DlKfA2j^_acXv@~JjhC<0Jl+9a!zbILj{W@d@_j1k^O+WJ z;(UWIMhaU!mhOAdN`Dg%pYFr?9{l;Eza#b(W^X6_CpupFHlJyEDf=HL6a>_3wfG@$ zz(c;P+YgcCxeU0BU}G){Di;IfcGqaTJhw4(pz3b>xZ(jo>Q)+bpEpxGj3BDaM~YNMI6Ldhjxz$ z)SdjdkkDaFIU(E}Xwlm4qBsa@IMs^3u-tG3Xo0`G-TAW5p#`19=?w%mcouDUWWOLo zgKu!|Snc1S3xgi(r^DQ4pK$ufTef?Wc6!v87CyYrhuh@tP5z`3yh z#(kxGEdYV#M2Bx*%_-C@d;t5jzv6IUC^6Vl?GT-voC!ydo;I_g&Bp@O#^beIJVta*pioOXJ@xOULu!|8qShfGX#$Keo7s)8HJ1v_5lM{ z8r*p}jV++Z)b1U6Sv^skTV86HCUd1#jo^ueEAv}(fvzJu{tJ=y?d;;Y^e4c4pp75x z8y55B`GYzlTez+7F7>etduAeZoK<#FKb8YpwQ8CHz} z7{KYn{s}~Spc63cL-=)I_>b=}Qaq6G`S%+EqPB(9vvO|Rg?<_IWN+lKuguTu)Viwh zf^-EDg4Kd2$4|~RaHct^Jads)>CS0sAak1Lc{dGSM$W1az5SM-o`qor_1|76acX|eU=I0~{5Gzi4Y@5-7 zJ2MDCPRfxxlN{av<92ewk@E$Lq3?i^72jiksTzfblOkoDp%EFroHjC$u4I=GFn;Rb zfH{-X`kV0A%3ezW&M=o0U@(UnECj}34Cf6nh@!<;p6Mm?za9bMzVW`7_*PWoxB7PE zst27Pv@kS2yV`X?`>CPd&VB!jIT!sfq{Te5kmRPaS@)1nKxb6~iRAF}2iiVlPL6a1 z=%w%P21GN;Am08uAkyaN-fMBjdKgJ_sM9f_4`mGl37_y`%Rw{Z_^}+te9%kM$&!|N?L*)`qxB|=fF=MFO@LbDOHwBhB^sxh zAp;0*N8@}jAbzi&O;R`{YVF=8k>AD&11LocY>z_X*V&5aOEm*^=sYufW|@3}ygf&R z-#1hnSPp3PrE->YL~Sou4P64JzmRL^&H&R9*bnVBZ9`Yi+sgs4>YU6unJ@Fq{+W{* zhoO7aYSRq&IsE3l7R(vUp+wzg=M~Nbsd>O=d8=$n+P}xIFm!;ZU#Dj4!`BO-g+E7c zkD85|$9zMvha=?vM?3c#)Wj7905}Y-)Ini7iqsK}<7k!gh~g6;iM2W%M9f%Jv_xYo z))|o^K_N&s(PE`dS`d7o5)8hAT5Gfz1I-IcQG)Usv=G1qLSj&aBugYA*<^b*t)1!P zfBU2Pl|Osu+`DJb-rb$^0k~ph=R)k%lg|me$F4o)OZh-w%Bz)qPlynj$(+xp2uZD> z1hR9w;rj+k5G=B5LxBwJl!1lkbQ6KK5hcW0D8Erotpii(ikm*8iAyjurQY5O{7RMO zXg)tlZM^|uylkX#!QtY0rjd#2V)hRA8KipeO%|UIF%=q}D5M-RyInrV;x};lx|x7O zKF-xF6m47SY~|XL$*R;P2pC7vK6r<6Xd*(&GU~Bs`TX7Za9^sDjC<1~2t7qAoCPcO zk5*1iage(A24Hf?b!Sb5H)V;3%yksPR6 zG=`riCcnBs0xx-617eU$SbGGz_o}Q5Nb&A^vR6=_&G00c9@k2Wt`D_bW44BE)9h|nEwm_Oals}h|^?bjpo$1_X^T#p! z&BIekOQO(sS##03YiF#BUPo;!fNV`NvSqMN@>%UNa8YhYpx!lv7PbD?ve0tM;ro;L z%;h6@wDuBp7So3JjK1A*Bw*KN!*WX-e;KO+wcxecOl!i%t|B0uO2hOMAQSIt>_d>k z`%8Z(oL*|_Dg}G&2|aov4gwWTX*sG@RE@)s;FX&dt}m0C#D3)wIDz~Wc;uBZW!WCX zvEcz3RVl3d0SZ9(tOrZuVWv?rNH)y0dzNKzXh@V;;9s+zn&l|@tV4%5%0yA-$@PP( z5pCs^B4)Oxp1`b{L{IVF$h|t3h_ZKPg1{;374H8%pkm4Tb1&H2p$Zrw1se^G)(W z0iKrih2zSDRbsYg;f{bAM@$&Dg34gvEj*<&uSJ*G9M138<7xFxJaf-Q#yLAD#OuCZPD^kgc1%jMGeK7+S*aLr%fs9ubT-n+*)z42yCMwy0=3zndTXF zXxt&;(30NIbp+FxSZA_si;!!X`!F_G-q#Q)=wN0a${5O&M4CzSL$W#cVMCNmzrBa! z@|cAA({pU?0y3ytf|6D75pF8vj?w6x4;@Ra7=!}}S>51Xs`uV8FooXL00N>apA^^> zmE9BTp7K~e3|g4^6AHU4q&}E2^3p&g%!NNIgczzV42A7OXk&G7?${IkBKOukMn4a1 zPQ(cXh4~x6A3R5$2$Eu#5rOxl>p- z5s(JD6mBD-Dpop>lZcpYuI|8m2#uFcMb}0vUi74T^|?&~eh1fG*%)irUW(-PwM*vh zv|$whARYw(5h}CzeKEZh0K{=Q>Kx)+olB-!pKmwWqxM=NRjAbcu)5GLj8I=>4<~-g z=Gb6QNp=4LNN#}&bXo9lMrNGt?1!`)ip5Y5rS2JLm0TV)*7GGg_v(xmk$1bTO|IHT z>~n*-I>cHTr{FSqO;0M*8^XA?TTG1OSJKY3#9)ni6G}`Z&&&sLnuACBil5(wx_&%f z=g5P)o;7?ibJJ$&HB9QjHraYVxHNv(O@i5TUMTTakE9Jyy1FBHbtg0cJD&KL_P3u5 z+Y2A&%0(&5cN9BFp4k@n`%sNuKP38lYF_^*{nxwT{)3Hq|Az#Z$nU>@_ha7c7Y(HH ct(!1>+E1JZ?A4Exh|uEJZCrb6&8{zg1>l3Tvj6}9 literal 0 HcmV?d00001 diff --git a/docs/cugraph/source/wholegraph/imgs/general_wholememory.png b/docs/cugraph/source/wholegraph/imgs/general_wholememory.png new file mode 100644 index 0000000000000000000000000000000000000000..3ece02b007b6f4b0ad6e97b868dedb2fb0996d98 GIT binary patch literal 14073 zcmeI3cT|(f`{-X56$>cnDowz$impl#5u~b!C@MunKnztuN+BM)iN9yGkz&{ef7cEW#l|9?1!NMx<6BkYZ zKuz)nf%|H(yzb^1r(gis93uXa81Yhz0Dz-OXHT8D6yd@k!*Rca!_>YqgS|bahl1J_ z36q0IO<^`q@0i(KJD-2`;<&>0o%Xi2DGIi>Pb_6F{QOp9>dwW*Q(B_WM=!=)g5E!+ zy!FZ{<(R4S_Y2&DeV4AEl2tH!^c(N$*NjZoCN1IIf=QGP$H=vJwr7?_CV%`$Jf~!= z!UGb$Jpyz8ub_x z)bbhjuYlWAPC!<6PYh2OGT7e}H8!~+XPGtKF^Sb+d^D)IP4FmH1ad_K(!g(6n?Oc>ncb#y_&ch7<}s!)uW&ry?tnLc;O+7aw+l9 zIXnQ+t-)U*HIFj$io$?cb^~r$0&v`mEuA6+G z^KvTF*0bwnZ&Rbv2B178N>xBBnH#T~jSj#4*@9BhnSG1U!91G=X$%&bCT2txypIVZsaBLI zFWqf`qj$0qCvG7vTo}OXHXO-NrNopgA^MOSOYDUNDWRF}kqmKfRj2S@z198VtwB?| zy|)~Y)~%NWq+Q>NO%SNO1sBbnbWIam(RW{=yb*u0@2O^!Nfw-AeMxMNm-N?@eO=fv zD38|+w@o}d-HIg^=|SfI@GZ##TR)lr{+jX_`DR%B3zE21cocl{c^SNV?AHNjrcje$ zV_7AL675=kXUbzv&**R*z=rc4EDNk>S`4xxmK@Pw9~R?DrVI58?{`zqNpY56-Ow)v z?T!bH51`YFy-G?|05{eQIC+40_pfa@XKbVy@3jii-fTZ4dRFtfYs@qp84tD^_4cJh zVPMWu2zoKEDmlJodD*(y`engGzpqqM}q8W4#{d~;i4{=436>m z?*;C-{cT42Ob*;p8KU~%s0A)WOaFGE3(!FkK*M+VvOYxviyDXNaa^N=>ZpZ;{ zwOO!E;X=giBd~E1?A8$hS*qnvY!C(Sy9RKR0tfQm+!=p!dbP-RfVq7&@ECE)5CGbB z-WYnta4HvzqEwrp1ETU`WA)`3^FU>`Byb8M%RX`c_6bmeI$N=uPUxCF%ek5K6QEFB zA-0##<2D>A30_A#H|?!~-kXJ8#QClThC*!RPr)ABfi*6GUFRk=dEIAi>9GZX@j9`4 zYsqM&bC);;(+3u#pDS64LsBJ863{QKFtGva99xmJ*5rGgl!i*e9MPuMLWMb!S|m1W`7be1EV&gSkI?UWb%G{JYFn+t6n!AuyIS&l zO+ufp1NGmV-E2?ki2-8wu1HDZuASTm#0F)Cp41r!EhFwhJtm?YR7iPomIGS?m|6I` zd($PDAzuQ>T$xAE$=%Ni0HD)na2@`UB%rlD4(mz%=si=#1As8zsOO^nfB*0-xRGc- zGGpkKJ?eW2$R$f|f;Z4eA2$K-pNl=|hN0RhXjCn$TC1B~=K+NxP_(9P(_M?!ac5^|>kwwlA_iFl z0M>;8plD*F*4lt`t$KQuPG0fY4qfzt%0`c_Hb2%5PO5W2sf+Q)MxOzQzyw_>)j5Zz zQhGh~;H08#E1XEbCYyHM9~)ka2vyO7lrIWcmBf0sw9Vz{s?&M_uJSGPi+blXPm>9h zaR~i!1eC?}u0gaw?X1;P-M5FD!C!mJ(UoIwc~G{J8<(#NtnlwEyzWy75LPpSTuzUj z>x2dmYPWiHgwHfGKTn}!q^!Zl@VqS5DyVSzU>K_bN}GOt_|cZ`tt~lqsx(TMpe71R z$_kYRS`V-DI@voQ^^oWi#VEndbt)q%Ge5UPv5~IEw;6Tu4}0cmn@Q1VIGQIKF(4&O%ueBi6@LoV>fA#6p!arlm(O&%ky!eK5s7Tm8T6^ zc`8fhO6!`1L;T{=s_AvA^P!&=BEo8VXwU{IM2|R!_V)zn#~JS-#NiWB?eniu3iK){ z30gU>K{^BgGx2VF7v4ONrv{DHf3UH=XpG0IJst_^zW~l?;?>{_1c%e*l8LIU(uQ%L z!Rv;SQ%LN9OsH8HrBr#)Vqi^SgI(-}BCmA(F26Nht+ffEgP|ly8dN%$Ik2(z!^nHn zdz;iMd3%u!&TK^hNC4-Iv78hUDx*Slg)f(2xKu>Wlt<%!wd@Jqsz|(D(N@T#UN$%O zNae{2q64YPy6>v+Ed6*v*NjJ$Pe2!&gKBPGK9yEAS(*3?VC3TzYDTIX7GjzF^P%m$ z`=O4!YpSh#?R0Qn;EH6zVZ7M*Ou*EjT=p>TGN>TQ)7YPW&dFhK&@!!W_pj4i=>KE4 zMYdydO#ot2eR!}2oa?|r+}m9tHkB{6jX$)Z^&@2)RYXbn*@-rHHfe557a~l=SS?OS z(Cnyt;)B`zL9*EV7x3I?Ua5;XS3?FHMtCMVn-l4SUxbpve7RvLT;$y=wO%%Q)mHP& z_mz|awn@OB(Wd%mg~L<%sc-Oy@t*>OsIz1(r(mIH#^t<p=$Zy1Ld#EJ+JVX4Ud*sTv(1B+n$D-@LB(Kc)i#avI z0U1Ec9}EWv2AlFKE<=Q_P^tCgJRP9&>jp;YvgQ&(vN=xJwi^+xqJg3L0izP{cQ z^D+iEP6yHxGu^`y$`voaoFziaF@mw4qlJ>%0OB-5%a9>MXY`%ZtN^2=ThJ;U_uuDR zJ&){K%&WmNrJmz42f|)@gkP@JfJY}eTn*~j=lFicBU-4K+%G^}`Sjc2ctroB#Rrv7 zqOmNDOAqo~?#cGet^V^cOIKPvHfNQC-a<>j+8ULP2QB8&rvb!XN=}I)+9?FDMW64Q zY(!+{ImsWXzn~X?Zk8zTr1Pl1>2ocUq;#Az!QWbMrZ=oSSkNp10GH2dZMnP;9vz%! z2vj$A#o=}-t)X?^X}r?FNJ`c=K|3jJra^ZVsf6d^s#Ya|@|+q!C4dEEcb`8N80T!F z2~kR%C(eXL0`XR4EwxZH5iS1es>7a9v+WN|V|Uu<;Jn}Hnl)*lgZdLkOZJR4hx`2j zu0jcUVd=e({qE(}>ES#EcN_$_3p>HdF<>+_(@)3&qfG_RYPw&-hh>R3CkNHg=O+`l z&nXT`U!my{s^f? zR59IsN!DRA5h6`&srn+t>wiry$m~xPHjR2_W1Q-A!(K0(qrJf2m;$CMt3u5v)_0Jg z2?BuJ?YiyJ+aNj<{Lew%EmN60IpdMdvbv}229DNkT8(^ac zRNqEi)j8GtIA1nQ#3NYCFf)f_X<~0P(I{X&Ynb(DFq`~L!4*>n8Dn$ou4ev%b z`vf`)y^S;?z!Xp6TCZ2}58}uuNuj{abb&Du1_y zPr51@0*-37yC*`;oLhHr@OwCjJ>UATmiCRG_TUv-2{rS*?sJj#kO9LUao1XV8*BAi zc{YY21Vp5H!0B=2H_9nXaLVP0+y&Ww28Wu$IN zLq-<(ZIi-tT?+SIoIhP*0>c6aI}g+1)59oMf0ihRS&)J1$N}ku3dMYvbnlM@>G2MO z8`yAvSK8LtFSdp^0@J&G^}FYo^PvL`4QS9dt-%79_h6M-Oa`h_7{lxiH_*bMFSFoH zPT;mi9qp2*AR`wlKl=91F}IKrNujylNQHZYLtf{62O5bfLK)t(+H8p57f(k&U780zt@!QR#R&He!_P}oc^62>Gm7nXW* z*!o)>E0%I9{OXsc;Ss^0xgJg=M>?h&M0_;*oUGB=(KZ#Jyk%RV#u-BqWTBnMA9CC8ZE}Ep!S|=XC=;!V4XrL?zB#LF76QXOE+`5}r~FD= z1H7}Axu5hm}SPhD-2R}tek4); z-nzbq<)>QGMIq)9(R2II3bP66U0&l>ICyDR#7HcW{mvIazg;ABmn&zoPnxK~o(Jsn z{38^MmnlpnXzN30Fa3}y8$!4xNhSXlIQ_I*OxW*D7cV(+ak&pVMi{)tme-+I$?=-u1)bDyP|`biJQL zY(*D{#NGp%*Sa8u898U(Dxc#O1PB${o%cVDAHFFE#6)AJ!sIl)tTv=|5X! zk1!K2j97nG+*0Vi%=!3C9|!1Q5m_<+0xM8#(W0iEJCv?aGbXV<&Vl#?T9${rv9o)1 zN4~UlH|z(pEy-G0X!pfbdoYsfgO*Bl(wR`l?V35fs=HFr+Dt z=(vvAMmW0B>S_%SN|u_RZ^P;-^|telHG*v&wnv~)XefGDsD0vY+nFz8Wq6+xO-=5< zxN?qmM;M*4i?R11c0LP2M<*d=bKF_)NRB<^9>0<5!#;c*oQS$0@%0id{_TKFLe2ee zHad^Ot=Nd1eUMa_na2q3p=$wQ77VQnPJD;sHHOx#RY5%png>PvG=)TY;>oj`Q@V117~P6P=X7 z?sV50%$Z0zguFf1K4tu2CF;xU{C(#2w^dDKsN@xs=B1hW&@H4>kNg5KP?9OY7?2q+ zRo;dSAp@;owy)FK9RFBKfB2-7Z~WZ6&jec@z!Zlm1cF8GQ;YG&rRx&)FL z)@lF@Fz=Fe&4kUwC(L#P_Go!?=vV(TQi6+r!++B&GN&uVO|d0I!qX3aVvfq zO2XD4BL~~+U@#$n%Ht~e%Q&JP+TABuu7ibtZ>p$yv5Cr+cKc&My1Z6^i5k@4oP%!T zLb9weetH2e!FU*y6jOtwMumS=r{L8wCO-iPc-USq;S^Et~ zS^sH`JSR_IbJ?7H91Blp8&)&;Ci0X-F8@5uLkUWu*E5?L%$_=1O^vz;++E?(+UYzu;qG{F28`sBwE}17MaL z5bPN{%&nEb-ok`TaW^hbAONg?<>V1U@lAJSl-z_Fgg z0$H=gbHk3BwK(f0la`e6U)T^;Op`^E7i2cKu^j4~+8gXN&$v0rw(0ge7iO)s<@PVw zgX4f!*ykXvAx@$EczyyUFfcILrRAX}=L#+uXJOn6o#!(0Uj{QT+Gw1_8bxZ`!7sn0 z&Uf-21oZKo>K&7)C=|)!_W@GS!N6d2y@DFu^C{_2t89sK_VWP8>~GJ-fyPe9$cLZ{ z`enf%Mu7qc9iy{Z64gq+_te3iSoY?6vqPv3$63cg>JdLS`T@J<9b;SW-E*4Fbu`1m zLpO+@m>uD6(IRts%F|FH3a+pSxeD(+o#tm%JKMN9pj-->FVHz$S8FB(XaOutF?3Myw`Cn`VON?olJVEB1;mV8pdyV z+G3H$*tv;ZSw>ZPQIVoHXlMFCH;mDh6Mb38PTS=B0f@bmscQ!AhXJTK73KhL8QiwL zkU6!lO!0Pt8{uHy9LjL9-aK{;w(Iq={bz8^MXV+?U8;P8` zw?erlZ%rpJikyw!s_1~HwUw7!?5=0Jhu&{%>18&`yT35M!j4ukz74AF`b9^LP78&q zQhGE_X>0?;C8xzJ)zH}r5fd}{8WDglrO_L%``fXiv%4Q_hNm&|6yM<*0^iBvR()?d zRwmU{saey#$sF&XdKIy4%JAE6gUEH97|&yAK|bjQWok$?`GjA&8jkr`#1k|!jcywF zM7?P*QT4?F0G z&VKPOwAPA;uFyOuOy@*nWVoo9i8p_gCHMWk$;1uoPp|ds+nwleoxplpRke055rc7i zluXh4BGGtpta;ay{UOIN*Z_L>6VoDNN`M*53OSql((!NFIE`MldUwg-HXItvFjbpl=e`iguCsNPJwSjG) z&e3O4IrUS9Edd5?=f7JFaNGh755vXXfrdwxOZv_Y3)l(km8iTUu5)WAjh-p8wk5x+ z#27tMtV*_--1R!^Fk-CPqxp4oC5?4O3Pv7u*h4Ydv~tE>Uw!nDuQT=Me>Qy+_ibwJ zx#J64`%Im^j%kun#1OA4!w#=khY|TAHLdUaeh+b?e%Fo3Tw#?Ke_xd;t}`OkK5?u zz2^IB70cJlDfw^6Kg17+fF?Tx_7`IoI*h;+8uR!SDXoHpE;k~?Xg2bLdkg9&l!Djk zjRfkEXY+dVDBK#l=&~KQ`E-QtrqHNI+_As4Y)z1O1uEvZy2{bAv!j)_d9`oUa!i=y z&0uUf2%h@|uMMDUwD>o>I;gvgNvB%1TkIyvIt1tWMw3~SKJ)^gr=&~k3WW{x(??oN zjs7_ef8Q0c%^lN38Xi_6f2cY1 z+?$RI(Rhzf;F&5Ja4i?16Go$5&b@8V)efAC;qs7^$U~nSXDl9c8BLD1Ea5uw!GcPp z(M_WO*j*E--5wkm#}Gdca}-!#hK7^gEXH2t&sCIZ7Sc7<2J$6<)nF1m7ffmogExDz zm0%{W_24z-`)-7$@J)|ws9?^qEN^bto1^EZ4i=H3m+Ka%4VH~E>!!H0UtguN9(H9l zkezCe;hiRR?u6Cem1nUx4^#cJBj1@81kJtJ%rXiXP7zw9)?Z*yS><24wF5{#nTzlP zMAXJf$VRWpr2nMo-v`T;JhX8j+SoT~2)#1Fgpix#DoiKXo1iBqA-k#)jceDhy(@=# zSnYB!P(G;mouei+y!>iWB=se-aom>cSz2v>%e@Jq*!dVFK5LcVAWgF)fh4D>=FzlmnP-R@&2S=zpULmK4?oa>fL2h5Y*qpc1QfWpQ=ke=Fl5_ z(2gV;Ji?0Ji0dlANyH{j%z1#9(&DS51K@d0Ys1Gij@Elrp)%dq5D=B{|3p|IJ-;g4 z;jRWaUta!*F+fqHI2Ug+)glPugMswwkI>P2`sJ5Y=hMlN*s+HP%Cu9yR0S`d;2lIU z{i>Q#O1Bz__CLZmaP-}1`#lTOagFrQ<2o(vZg)vEz&oble7M^4WDn&eD3?$b6RJyn zVs>w%J?vn@53-|LdzXKP)FDs(SuKT~ICR2IXVse<^eEB61gT|NcifPdr*cmNW5`7l z%+>ndzb^M|pNxOr86(0BKg~I;u=+u%XiYvjn$3R`R4qhm!;CVf!I)ZtgZ2U|)%o&MHu$ z?@x=b6vbCFUOP$P4IuT$yaN6OQT|#C{>fjwsE1u7n64$pL?gi)wIk71 zx~|^}89#K*RWK&sf6kTH=X?J`xL|3nE|7u+{|#L{7gI|!G$*$bTZHU|JMvArN**a$ z`a!$0V4RNxS;TB@&a(gQjsq^XGBpndr0cj7>ZTbnyka<7$)n!~o?o=QE>seVWmg0i zZqR51D~sRU#;|5abQzZIJZi5)y1D!%?jaNa)MX6P-+J)}LaQtn$>BL$_B?80!kFhA zAffgKlh1<7wEfErXB&5u!${EScL%D(Phy*O8w*RjT>m%m;{y-5Yuozi|MJu)M-Hf|SPLbQS zw6U;im5yS0qY{@x3bbw!@mpLI-BTbDqrVr#eUfjUGAAe;B}kbrwJoT_=hdg%)r2>} zzMm4M*K_b3B7B)RO^jg>`zjhnu7LHLM`Z#PRxjy33Z5CN`2eXJ3n2Mx=t@F4vt)#7 z#OL2as2SduJ%=voeYzRPQP0j@(XIyQi6P`rVxjmZ1AI7~bE8jq>c-$k+M4v_N3w0< z`(X9|B3|Dd<6ms;dw}>?5&usq;_m_4vmeG?bV*B6GtG7p@rVwdL)VzMiI_eFML=DN z%hCuM7^*MwK24gWjVyKQiJhJj#ce6&S#jr_DfaA|2iRpsVDU;As|bYoiKAgLoL4;E z`kUnRpXW46!Qg52O#hF7#?knNr0}K!FrZ!3c~6A~MmeKr(e%FQ;vAPmcRTW}?nA>l z5qCK112T2oIMb!L5bl?>;z?5x7Yu7pIqI4D%hL@p^oBlygDWJ_eT#0b4AIr3WN6L7 z#BYITPx@Q8X1=cdCZMj7+F4fY(IW93#wk?>(mR|-=2I6e;0qSuQgTLXOPKZP`=1S; zV3k6&>$0cpr>(()4~x^9CiZ~6%~WD$Ehlr}F}kmd_&9Hq zl$d#S_zNYEEv1S#wj@_!!Ha0HZU#8SI@7#>Ts)(JlI^mPFK=(=Z@{n~0LT62kKJH9 zmfs~h-?2eVuXwEmvjVwdmTt7H!_Wzak9)n2JVnCJ)h}un!gO^eN7f6%0DGW|X>s#JLfvm7t z{q&8Id*v;P2BmkH z<(?r(0oDCx$)BA657KkTf?dEiGmAz`1Y3~#Iz)?OE`L@g*eJ1);KB8o z*u8eq;`G(z#sn)I*e{U`Sw#?Nj94R&`&1h`ob;+=YbkKN2XYXI)mY)y@Vp&8*7_C_ z!1+Ik8~+Fb=|AtAZMd6m6h9>PiICP4UTmStKcTw{@JE+y$TxWdBB|dYITxMR+;uCM z+_0FLOkbmji|*hCT27j-u-ZWsH6frns=P4hGCdc#HAUvYmjNow>1W=!rwL)gTYC0v z(YqQP=GRDT7x#HKh`PeQb_bS%JnDrlp~nX#>%Fpa!79Gu!)gL}A9?|lasVyHOsi*> zvr@{0Z66mO$rtHZ8>*9In280W7^`R#82@!~e|Inc%Lgr>T2Z((c%WZi#N;FMo>Zcj zj14PK*@MKKn7RznuK31`e$6onH|HUk(YYWmDZlB{V~u0I9lw7V|Fg*FyTj4;8T7Dqm93^d-wro;fqe99b(e- zgFXnr&Kk_UTo7tPvV>GaQGrrQ-*3ULh>@?GC=N2lPeD5#wf4hTSDE2&o&?8(xR(8$ zCf4Z{lbMc3ccm-3#Eh7dK=5`k=4;|Y21Xcs+sHUmW2q*`GHUUeiP9#Zyg@v(cnN_5 z%))P)&Dv!Cd&*@`Ysl}zg>7cyR^6}SZNxkRY8cRfijUH%mg`&Q17H_}A1KG=J;ZcorMG&{H3F%j1o~JZ1 z?Fi}W3{(2`g3jb+@0NpE>^j!zWVvJXN31@Or7v-j0LuG7y8hWLd}%ghY}9jFs7B68 z%369=QZo*^Z;8v?^9{X`_6AE$4aJiD%JNQ}_~=d-WA`C7xbMV${X@`Tj@HVGxN?|q zh~IaC0{X>d8>vhiXqA1&$4O3u4wJ3%QG_IA|4iHvrW!AXy(|@j#OAf^Z2EBFbv6i$U|{mHMf+D@by+5WH~(Wjdi%J37a( zu#iy`6Sf7Uq@1PEQc`kPZx>3f)MYg|(2Ki-A9KH(+LBrZncq7i+4|{o~ocU!VE5r2jK(@ju0Q|Joh^ l|9)fXyDI*v2)X^saMZ8L4NhOfvc(@bYi@O_@}%3H{{>*QD{5u`&wW*d{Qhet z*;twE5ji3P0)h5iF*ULSfqnu0ZTsWrpMW#(&f6UU{OOPQf71fjh!~+eU9nhl4=&5m$^Z*@wF`Mqsm^6ekG|yB>2oI;pio z1b0c}{?8{a8?|5i>(%3eKmIx=_f$T1$V^~!qP#8PZzQ8d@!p%^JAa*X@X_BUeZTza zwL-fSrnx^nc1eHTbuV7F{inDb+2bBnhy0G?WADJAJ|FJnug!*dbv61jYdG)`H9V^J zJU_1nwc2=7-;22x+=mNa7!Z=Oeq3)K{k*zs>)T6bk+QANvAtkZAvNnqMZSUJ*5~Vs zVs_iMzPu>CFJ#pE;iwyDzxBCwMxp43tuNQCWe#n9ei`v!T0&Jyw=Z@3OAu;h);AX- z_$KYI$7(_iMJ-$7Uh(XAqt?bINe~{%Uj(F;R*$1~IA{d(F z>n{nq_&C9n5wY5(2`~IVTG=rMsyn|F5p@K*(&7H>?b6F9;ZdSN7eG_Lm7=nuL|e6A ziO0AC8$3KV70npWMXkMK!lK8yYp)`Af=a$N;TDf-to;)yn4C1hb198s&(lfUa%ACA zI@-r#6(016n&Wm}PoChvwK3gsIvBr{8}pJ^23h_#2}LbgaQA{NdUoP@y1CM8UqxZj z6BpMaxO*?D?!3NtVxBQ^2Q>M+6%`5WLGm4!ZwuT2y7tX$ZIOm{H=}ZoG4Zpwc?y*~ z*A>i+AWXJE_;puq1Oa0^p07~TR(Z~BcOOpn*lg^rr-?LrIsUW}Alx2t&w z+VutekryEkf!&J0qx1oS!7`=N!o}?0aS6ruMXtxYuXg(8uFaIjz}Nu-s&xp{PlFjw zchS%n7)lSV45Z>);8Xq0#gXEmZaP+?r7id;mT3BpWew@{VHGdL(ZdDx0+biG2U!xeQw;`ldefa zDl5NBNq_!sZWx|jIdTznS_Ku!Am!56Rv<9FW$FA-#9AsdnlSlVpZ~Sjt@S+()S0T| zeyVi)Y0aSJaF#A8SdPjB6s@&$rBt7%hT_pk)@%LL7T&3Td`sjGP!H!UdZl}kASjLL zX^CL@=B)9HrDxnL@4D`7b$vfhe$cZmM@hTo3BVNB_uQsPb|s-P5Oh*C|Fs}yg<^dN zt&zJXz^`<7#jsHb6RrrTgvk`7;v0G8nl0XJy7byJQa?JHgC#-21b6Xlm27mj6(HR1 zzI;M3!0_OcrS<8|Z8>Lo(P4ms-5e8-VT%IX#S8RDUYx>51%CXgRT9D7nNMk+LPvE? zI>gX>N&WcM1k^|rpFi5dYt_WjS&Mc6-O8_>+2K}JJFovTv2X(*-268xbJB7I53(4x zUj1a!ggHgN-j6s93O)>1W;s2WYAAj{+UA&iesVr9Z*Qwzg>I=RuoFiSUbGHiuxI}X z#&hPo2zv96Nb_OMiz>Me(PKN+2-~iA%c6|kcco9sQdKemCVi?C-#zxKNgV3e6-0n%jk3`;ef)jGl67asZu%q?2XQC*U}jBMp+V#ZF);^80C!a+h=P z@;;taY;ii1AXM_ebpxj^t?JLY!!-0m#{q5Zcr$vk<@LiK@&U`a-_LT_V$;5 zn3Kn~0rfxO0=ANtr2;3&lYF%hXvX@orPaIYg?`I_Hz;eoA*|)u(MS!ZN*uuJ2P(Q7 zy!Nx)*y@A#T-e4ndTZyBTDZP$ueQC^VXh9oRa_xHjbC=cXx`a zrraR@m2+pCgq(-CUgd%WS<~Bd_P5btn%A%7L**di_#r)+vD4#mAUnZ z-2p}4!$?HY59Kc$(EjdG^X;U9$xH|Lh81NI`?^CVZMBAD?5Jql*#mU-5MC&xDYWuv zf4>^|*EsP;%-cl7(;S~>Nd>Qy2^@zo`rl9V4Zug(A59OO=02?DF!c^~;K zIYe5HDhQI!;Ga3<`*CivD>pk1j0p(#mY#wsbC@!b;JXcgDQybjk@NnmVtE;H^2HlB zGc;u@?Q$!*)j#^YE(zh(4K5&SQHEe*>}}S7?QUEYYvok*F!_0Rnev@S&N8+*J%T*Z z_FJ$kS${1`^QV9PtUE;wH;}5kA8CBCzD&Gv0=an6KESmBLy?Yi)AS!6aCLRnIorms zG`28zkT7mE`7nsd%PcM2=<~wKWdFaS344+*~z7a5>=d^ z+rstZb4qmj5O(Q~ofoBej)#ZZgFGb8l4a@xsB<0pS|$OvBA$D6eHTtER8(4Twj)2s z?9zMtex(UrjbrrvQI!7pM{HLusqB*5GtbN6#py)PrnVwH%z1)em*cJsbrc}dIhROkS8pBU*VK_2 zUf1`OZa7vE)?_>zw_!gSuCZ z1Hq6>4$UZWaC>8Ylt~IKh1VyNTliFw%4wD_yINr=CCgQXuNQSIVmFil(?q5(-}^V-{8NIN>jCiq_8WPgK=f54gYf8!u# z6B=gkDkLMs8S)%27^-x2wnXKnQ4qW-swqKg|GtrxQrIk*vQzR}ZP~4UkneuDqFaC4 zhwOy08ZG_032K4sVIQ7_Rd!EUdW^IRjUTQG+x_K(^RByNQAc>sHEJiGH=C3m?{4zO zd0I8Y**7&qqj?+#+YujxH;yi+J%{bMtYZ9olg{BlN*$wyK?}L#2Nm;r;tVN}s7M7& zGpAsoxH0shuc-x!T089E+T0#tv3l7V7wuW!l5jc!aoV`-ShnUipHG%9v4F!RXiv)h z;354^J>lVyMys>Y-lUb`Ah(A_E0Lx_jd{3~CHKK)#2sLd*Q>^KT$GNNKPfHkgW6O? zCXS(?I(=&5l<34Co&~KL@8{x(V-Be+px~{u3#);OI9HQ&8&}gJ z8FvD|p>NltM5U>#xsaUY)Y_<+R9^9w22s|k&EsXgoI=(h8v_{^SMyF>eUh>(h%e#z zLqk;F(soERIP_i=KY??plbZTwwOy|Q3yUpIX1fz@8!`fjk{L=6(exhL0yHtyP7tWa z@t&pLadW2-XjmPPb-VW2J%j^O(+Y=dY5JU@Qj-#x>?vdaBTWoU^!&(g@y(_x5N5Da zFkPJstx}`%^>!e21>|k5f5QZ9R5T)%AbtfYi&yStdZFS(f2HR|Z}3GXHZK7z7%U0F z2?c_@V=+c8U_SFa3T>FUz>ht}3LdSRYFQcnb^JWr&64@Vut9aoSv1YMieiPd2c4FqsjW!Iq_@nD zg;3C^$JjY@HX}17ymr>ZA@uFi- zTi~F0y?cT+AIfO($Q{V0>69*{d~QfY9A-q6Gj@KF7#nuT=gv{_)QnQ-W0UNdhZlEi zyIE09d@csYK)>;C4W5l(-K9KI5PUV|)y^Z-{bLu6_1Yp6nx${wO6X>$BR}n>alg!rgUSwcpXkY4qTGRScv$`Fs*VW`c?x>xgjz?RjZLlb#$yMor=L zG5~dQYv+E1F}U~jEw<;}4W8`PIyLn+|CoyFj92Q zYmG@q@UmrY-`d;Vnx#?5hK3^y+4!8Wwkh!>8F%CpFBGe=vibQ@Iee?XiV>+am!7)o z`p5)pq-ePBv09o;n%nYSPZg?g^Xf637ChxiHz5H@U-!lQmbw(xB04A-iw}rW`;h-9 zRvvA_Y0z*!sEt?#(oQd$wo-RX!Ye|A4>W>9W}ExstkenJOd7I?pJwc|TQ`B}?MW_~ zzmT-r#0=YcU2}c&CCemy*_y069uBWKnxZ0}gMEX_(b&_VE=~))s}WzMf#1jWTQ!DR zR#@Yn@Y;t}S#X92H9R&L@QI}0evDLqPvffR+%#!iCMcMc+MVOIa_KyZ`#~q{5Zx{s zqV*XaIBxs1AbeMaC44kr1khytl4>mr!Op&?v&I*?*pjgJwhgs}#4ZV}X?Y z3Y07hME==%cesq!WTB1+8{QG_G%H5%$=g$_LW+zAhV4BI1jMC_zW|XuG9BHPrE+I zC5$&`lgt>=KsmeQd6T94=tjx#R}xF zGTq}J58jxdzB5!fRopCw%50da%}vnQjK(X#SjU36SjHELnVW9JkbA>;n7N6#K4M zRq~VgOpZyTjakV;LW3iD zXws{@L5KDlJp3vJrf-z5#H9}8|K>~7pQs!CtUD#`x=l+*yCpR*+7XimKbit~{~upo zONWF`KRM;mot`T<)e%IH2o}{MGfPM7f{wBI12THZMPmDMiA>Te`0$^Z> zg<~$fcd>|Q?OmY-(R_~K@URMXP2NuWFfby~m=R$505 zrUDdv#z`{Eo4@2dSTXRR?s|IX>;3qkXsCZxM@0J$x8>g3aLMryI!3p|kr1R?V)NyE zrtH~R!~3KJ+SCdnV6>e>YtV6^w?j=QR+q+oh~uj5MVr)O%CKs&!SB(%a{4sGW^q(~{#o__BG?n{vBUO@@AXcEbt8?3U11GLzxK54`7k8i4UP8B`D^GxM z$pI0iGNY(uNBqi9L4I!wlFL{;&WPksjz=1rEpd>v`NY@(!5LB5#lxNfsrylar%$Hl z!*vf~(s5Y&1jPtV?v#vXqKFdr8Ot`_hlg?5|2I&)G3YXx!9-Pr{@3L)ZRr>zng#Ilx7n$ z@SxO}fvg84!o-wYojOwc8hNKDz;Tm*SfW+?^=aQmp5sPUtGzR zK(go#KJ48dR2sDgRLE(Ugc@cv{JZ&B$)IH9p?3k!{@}yEVRa0?en2kL!K1fQbFm0b zb@tJQ;8)uoB|m?_%0?4jIW@uZJqw010`6?z?PQ>;#%ABwVNOx>j604EXOAkTGa?nZuBF`iLxTh1U2a!*dGBF(kp zSlNwy&Qm)n(yd%X+>f3VU5;gVEc7u{6-}c$QgQ6iL^4soTNoBg!+*4?>I|riv6(wt zYu#P8(tUuU-5ataC&f8%p}pO>hqvfz><|>ubbfMy#lNLo+IJHKy1qxK8TZ<{Ub94B zPRGNB7m^L~UnOP5haW>dsi`kw^%uzabM{mQ@Dj$dj$(NlQfy3g11tVEMm&JkSQ^J}08D8h$+gkn?W>&L9qZ!`8I z-lYk&ORD7?SL8f+PYPFb%3$0Kks=ZkDmB0JCF-3ob~Hj(+-P);l#Zq2zz=#*z<~Q_sGZSf3m4d0N1<3LLa%PB7;MD zg!eWs{Ve2d%o8kgTk(o$1`xbh$aIW=$4x#10i5nLqHBI#SIp#C$F@m19L(>ke ztOgouY5b2zP&LUpjg!&-2BQMsYr$fq zu09O%qzbg_2hy6K@WvHd1!Kb~mQmgsp9Oy;N20$2bb+?8ub&Oy_A9XKw)XK|cTGmL zKnL&4Lsl$+?y=4?Rd7Y~ClIN6BI*>-I=Q`S_+xz0A1G>egK+_UH@4#pM^NO4gd`I>sorCmRH0Z)j^JF)(E|( za1v|O?j3;D41U_cYTox{^d!9Ff|MUcQ7;21Ra=Xuxd)#8o*V)Ny_AGUli2^s?+0sm z0@#RzurxTx1z^i6V1v;#C^~fGI({UT)8B}ptpfl$^)&5(qzM4x_>LRs<}$Ox)C4vF zM=gd$qrT+|aSkE4)a4t)$(#cs8)O3eF5o>k=AYG(fuNPv1YE;W04Q|KI)j|*N1Gon z-TuT}Z)t^Tf&{W(^kgP=8d!bBNGc^NYXa_HCd8aPQa4r?^CfSrZh(_!cGXMKJGSM# z6pb2h(m+Q~V!Hy?zIv%a)@A})HetXK>F_z$c~ECN1F}+|yT*xu541#lO8qB>Ye>R} z3)%V?b_0ex@NW98kfp!ls6g*A_E+A{m?;+xynwlOi#_87X^L7ELW*6H0%E_PV2--< zA{)K!`pj6r8ZV9#wpKZ%vBuv;?Qe0`14z|&iCjn5Iv?eYUIz5JdxAgWrV~+dbm#S- z#vD#N2&UCwg7GQbhnAYbsBbQ#HE`Og=-ceMFdSBhYtw?!2p&s&7B2pVx-=K&C$&-| zJ=(mc%bXo`+6mmgA)P=6bZYXlm9#Z*cmE0g>U6ru^_w^j1Jc4vXRgq5k%+Og=(Pir zJu$4}F0uf!79BhvzM{`7;rij0RRVBP(x7g}I&2)XLZgnZyj#1?Wlcg4;G=o^kQKpI z_g6wZH%>>-0)RDDCl*HDbAhi>FZL|t`3q&?d-?b! zLJU3<>knXqU>PLS1Xx#bVqshe(TeZ{uK;uD999!dO8Z66Wa~xnQY1h*)uPetNC2{Z zE8Vzq;*_;;$A9YJo&cDtz@3{#DfN|Z>;|=z!HU+FKBG*0&8QR`e>anyU zt_cwkA)#PUYaoEp2HQ%vYsmH_amD?-QqNrH~7}-WLvP zYg;&6tc*yRR)e^Ec)`)gF#yDz?ikXIT6zh5;Z32K9r^P5{Y?k9s5TU{Q=m^Yms807?G;rKV+lq z))2{^oBa69BzsND*{O4jhSp|S-H*%R_~wL)Z>ck`$VH$mSU;|7i`fNg zH0g|@XMZV2YknUQk}#C%H8Or}v+&_gVBmb`?eUJ($^2Y{&Ql&l`0lP#mf8CLY7u`D zr_$pQsfcC>&kDhHsCpGQ2;7VEO>DSn0~mT2Rq`!52H*KN9lgLu#m*dq#rYE zrdBI&E+2ayYDPG`A zaYQ?=tTL<_l@?;LrE#rC6^b6;mk%mCG;f5q^nPa?Xd(9T^Dqw46sAX6u*@DMCm`C# zS6RG?IJ;uuE}^R&r}}|xZ&W7x6~Hwb9@9D0+_S*Dtysx8VZ^ty*y5frGY!p%Sn@vR zqq`%sa7jU2S5-G+;ahd$sXbgKHKyeElanr-F|oKaJeJ3Dl%Bd2JB{nxa-n#dMQgh= zy&9$Ucb|J1QT7E36D?B-)F+3%QSEwG>?Fh?^h3v+%_zM1rv=_VV-Lb%`opvvl`Tck zCrT6`x!gFAx))x`bq1aBpzKw5#YeZ~j)TMIhT%6K^SR70)aP~`)^sp*s`nJ+awj{7 zwOGkO4a7%1hS9-ucOM5mmT=}?ZP2S^hBA&(6b~ht6cI^V(6`W z71i&t+Wphr*QeKHx)~LBb^K3D>?TQ0u;33-dw#lm%y}~qfByk>e^Wv znFp9R^ttC5@JyQ(8r>JFulm3%sJOY|hMEw(wJqs_G&BZdh5u(4{zK}vmtx%v`d7#8 z$`9H6X6zZhq~t_@+zO38}aLr$6f-bGj;o>GUDuI zfWX(7Wo)j(_o|Gu$5;Y`22GM=WQ?*BwGZdX?CF^Dc|TPFHH86&WXafdMOL`UG2bjH zJfpe0df^x|7>hS*UOF;A=8}k~ot*ulXeCeZdjydKUMFYFw@n{G#cCwj)fIzlsWM~j z{fXjKgO!KVu5ZZ+CVa~es>p1ml#;?PCq)8+)Gec?lgIMpe|-En3c+S5NYgb51m-ap zF`ITwsz=#Cm}lg>l;>M287`*_kf(@$J1`m%A8~jH>s3_UU*_y8WV(n1e*3Iz zm)W9I>^WCW?f)!yb#U(k+?(1D^|OefC07;Rx(xgPOD>yLO#(L6y!5!ZqdvdiDIP}P ze`cVMZ*a0tQZ1Vdxe!m2i=6x*_$z$={8)RvV^H%-XTbut7+Oa?yg`%)V;(2hnIKhm zST!Obc`P?v(%s$hVBxEi$+t^n`_4lM;YMzQ!F_D9J~eeeQMP;_#ujww6jf`^`}Ucw z0ZNGylk)v`an~g%E@T_BK@;a*2HIMitcR5!=v`$R(Qt_&q<(A&wW8RhTpjk* zbC6#<-;fNG3ZCs(#isWmLg$G7fP~so0x(%in5mG-%EfS0`Rd|+HQeRqrNh4WvPn~F z$KF<(l>b3wsGZtW7m2k`=4x&NCH)#SjJjAdAh^4!ViC>0OS}(qy+3BOC>#Gt*6_5p z?5}d8cf=PlO+rh;8=?e6X!wTO3H85C{|Nx4N}AEbS`@Hr zTlV6!NsQ4%i;D4Ai@|YwRXx(KTLw;TXT|U-&Nq563yW3fOKfI8OX)=or0z-acwVFv zUyS>!+HLle6kr-&GJ-6?0o(w?T18O>KDpKkNdN z@?Vk+@?6DuZ80VC{z)<&z_5kt^s&v&m`lx0>z*fbxv(d#wU8@dVj{i-VfwQ z6z)5rc59kGsF3X9PsNScoA&f?w;Ke1ybo)U$9B>k2^Semw+Q14#z=b^{i^fWkbuF` zveeqzTo`?y-A9Lp;USIwzJTXl-6IdkPK5gcNS3bM3Gj>AxqtaHhidw#Yy!Dga6ss6 z0Dmdyqp!21?$}#s9cs%AyXIME6YSgx<@4j7258qD10B}xHV>V#|AAm6H9%;7yWv|j_K1lsDm4(YMvUlM_(GKS+-WW zW{tFOE!Xx+?cw^Cmt6}@+3;G}Hh$)8V+Mgg%&^{c#W5G2Kx(?F157B`Cq{d7jkfUq zAxSeI^N6g&8^NY0`9E8rhS6^ia38R`02%kFu218uW}$A&Y21b(2oM3*4UEe2?b~6> z(_TCm_+Rt(|K4!wkvsI#r`u_~R`A$Iab53)60Oz-&6$0pQm9r2w~fbT-yXF~L~8 z-lpqLoe$!z6hj?3rkNHEtF%pDb6^}2FUG-vU>N=kMvQ3vfhO#O(QQTX+*fg3k6l1la zB+BPU&#(N4vxd_Rd&6; zw_2qcx1J}g2lDkiVd4A0AE+Wv35o+Iha}>lCzC z*KVXKjLD+P=l>~h(-g7jgH2RyW)YRLf$=v7*j7?--Rath+|}}lwEXzF(UwT<=-uAh zmyq^7cYr#GWKebu$o{FW4MD|`?Ri}ryX>ncms}fsvdOAqLcm3IjWUA3cV(z;sayc> zJYATpMLgEpP(-0#kD+kFohnd1DGV{chnV+wgqy|c+5r*sz>aef63%EXjwdh|+H8!m z->oiXC^mcoK$i`Acm5Df?fSAs zadi`rVXgD$2sOgp>({ztOmIE^!KN&Og=R7>Q6g3LK)V*drALx#`oZ!D*mMq~etpA0 zCKlPu{F#j*0&=}wk|F--r791DG^j6-EnBkKzxA9*E2{#?cfu{DnSsJ;iEU%65;KWQI5O`38LA+MgkaF&(9xZhC;Kj=g5Y8iq z+O^{xx&jw+i*dEmPukFjA?N+Xx^o;DB7PX9JocIF1b#RazFa)}S*hvZR_tB6JA>rx;nBz#(S3ov8o18M1oW(cboArGEER;*X(J1bS{pbCuJx)sfkU8(94cHO1S@cuMw$WgPNx^CZFExwEjbm{YR z)S-ypp^$~SYgP`F?y@J+B~jgv3qN;&k@oNHm72^qV^rCE^ob@_PbvK{K9C=5nk%ZC z5yXpHJoXk`i~3$%sQ_WVbATBEzO}ZVga97Yd)?t&)wQGTI0jcUvIRXzkG`24& z-ghe5(mDCLf%2*3YoWF-WLAin$q8>=pZ1IU#Zff+0lp?RY-1`q@vkR#olSN35FTOs z`6Xh6<5o9q&()^j-jwcRW|KO(2-P94Jd1?MOz8wirt{G>t`9&TqazC1K%1gPd0g}u z8jAX)6VxTwxc;mj6Mi`#b1y0nE&K z(xs;!(NtkrZ9S_;%+CrkY|%VX$&eomL1h=dQc(Nn1(p;w@}`P(q$%~Q4F$d%qYHLM zm6;Rpv+0VDoIS&&H;L|1XLH@xGK`FeN#dJ>I|Wk^WXtkweIX1z*E%SLHu zur5UJW$16F9K{5F+G*(*)9H#*+p>)=Sty7YJZS#%%s#rVGSMej{`7mAuIDj3a}22t z@%a==17*_Cxl#k;B#oGeVcL^k)UDcHKh$o99nBBD>hCou<($cQIs5U2>r`_K(zd&F zwyTmn;7;B0qE@i;fES&?+0czki%n2LkNZZ$nthV|fw9dZny`5W(21-HR50WfmB@Bpq~WkV=q0Y37_33DozY zCK&tKyDZ1v=XJ5{Ud|OU-4USY<_ydxH*u46i>ebUoLziizCitiWhi9lw$)zY z!5e1Xh72vWQI6fJ&Ky!%n|kh$wlC4{s2X<=FuV4R%-4C0-%VEr!<)$?f~_2-vD zE2*n`8pWLKOsJcPFw7DVm;RB{v~RWLoW@R$h7~3%a(V{} z&@Yuokv|m31J!4&f!H=u8Lwa!@2{SNDG;Grf zTJ2OCP0A6G&qNZwT~fSyFCi$}*7R&$_6z8uDVEeY>zlo30oa!@N$~51zVn`u#K`a? zO$bWXlNg+}sC(0jbWXLxlbXrju+Lz8zkjbXmihqdUNXW_jjNfPx&GsE8jEL6I1PCI z&H`aowjSLl!g-Gjr+v){et02Y0Q%a8#t|ktNMo+))x);N>1L$+q+L!1gLm!4GH0L7 z(iF48s@%&$-Leb3IfGf?xdg2=N|reCU}t8_+0;WM66KbdW+vh3RT3_Fto(Qu$^`o#@ovuLZX$pa4PH5 zg@l|*AYum$s9~CibV!JS##GP9-IYCc6;Rk(efO!Nx`BGDDUHnFKG!0R(MEE&+n~y3 ziFH7q)7;&R{a>%0QW)toKrTBna1d+oRL|4e(DR0^*u9Cur^iK%3%{EDc?HNT13mvV zw-iGerC(8<6rZQ*kj8QiQ$orXiHAs1L%8btl6~!3SEz#bG=2z>9u;1fzd|?@?|6B) zy1S2)d#BQvFWtSn=33&lx|AG<9{L>q&TgVN8frQ_ zl-cz1*;EA)y$;Ub2Px?<77yEVG_D>bb(c=lM-)Cs!hK$+OaF}`7u;)rq% zFoI5EDm6=Fu(Cz!h2Oj=ffjYx;_8vwfjWT-U7YNw zua(_VYkbC&2+y$1+AK>7)lCojdf}e#D8WC5vKe#Ew*!&MLF(;>!EKn~G~lPlY@G3Q z%d*v}&1BPVT$pUiMiwaoZ40E&QMqKjAUN7sCfTg;>B*85gNJs_((NixkynN9j(ZS2 z6C5u$ORtrM4(4M#_#RG_zEkl79nR^Nk4-EN_GI%Fx}IG#1-!H)FyvEh#;fj-BqvV0 ztewL<&ebg})owxEC|KHx^!T_smMNCk~2f0I%Hs!K=B8ndGoT>-}9C>7s8Bhm}m~14S$||Gi zKl+Q+kf;}{CpId}%PW=rWMKvcyBJUK)8Ddjz!`Y$EpARj8S+Ipv#;sR=I+*VvAsaH z`tQpT&8DFaHv=}?Rc-o@fc7K(0qa%%iOrVx!WiITl+ZPNi~mN~=l1W@{o8<|kNpMU zBo%lb!C7dN!W6mh05#0r>k)ArF~`|#<^sLXZnV=>z<-DpoX|C2cVux(-tA5CC{dtC zvPIJipbN3y#2VxP?WL=sr#8#)>GjO>f31E5x^G*{w9Wu{zvFp8M9wjm=QD|OK*_fr zb$ps3@2g04~Hvqh2z2ReJ z_76xo#r4%*mR4)fdm8(M{&wZ;l}=ogHOTQKb$JAf^U?(A5ny6c^QY6Ir*w?B`jB3CSO$v5zqGn)d2Q-WYeaah^#A1T|Bq2{lTOQq zi~_R)&3eiofp$(KhUfS;a7)XxOk{dv5h4aaL`6PNgZkpv#J^6%m+ zV^P2Z!|sBu*I2X$0Yr7at^}T>L_dV6?y(^c-9@)$ll{_GybE^gQ-shT1B))SfnXMx~FZS*j@))xrf6r+$1Nq0c;v_)AqBS z(yJu*oKN>tJhE^GS-`_p+-|R2-1H|t@3s$5V~>hH2l}bLN*!G-!H5%g|JIQ&p?J;Q z19A$Jx3d551g*U)J@FVuDS3S;qI7w3Kj~kvymG2R`TjyeexHom%&!l`C;t9+Y#Nw+ z$OZa(I6a|+A9-XstJ{cuo@hU(j?hC%AmIc)T-4&qV^xTig>W7KtXR(;!upOly1|;D zYcwBS)l^2NBDJ4);n<9pAEo<#^79~daunSAQQ4mM`{IZb>TZc-=unUpRzrfc<$xST z;_SV|j1Ax)EyZ4Bw8+KDMGj5@yTQCG8JfYp-t&<_Y^7y2R!dS z5Q9M!55&wnRVjZ^oVsX#+`eOy&(0teD8Hfm>dRmYh6GnW30@p>E?#nNh}z<-Ho(`* zp6ND~u_qijcJ_xCCWwFKmz5tj4WNf?=k*Mm`8DNa*Ft%Ph|tGPhGMo@`IPkK2okYZB*zLN89&sf>d^#7Ux)wc(wuMp zr_?5?t&SpPCku<0%I!Hni?v_&w6%192UgnsxO^Z@qR)g=2FwB;0QZtg zkN}?apzMFEy5lBaKJq7}lziu1Ig_E19v7i4K5=%KJ>FZla8!)5-zC;=Vi>0bRvNp4 zG_-iDok=1~zaC&Y;$1s^_Y!~?4A2U_s;Zx>#5WVR=8a-c*qWX{3rAj}MEpFXH1Wto zy*Fc`-C#bksfdP&K2Olw%Zp&hsvX{i$+tZoI#;N%C2FvpX~;>M+yrqn&4)l;6dg{zRLV@w% zPs>BriQI9=EQb69eIM|2SX46CBbPYVI8^It?G3y?pvr2?_)eb?DeJCKj!!bkl6Ooh z%$D$IR*p`}uZWL$=JSZqm8;o=@iI=g0j2@s&eEDa9PbXvE!7>;1I)WNQR72@G?CW9 zFZ;P%xlXZPrcb>+i|*zhBX%XzqlKuEF3((2|B=41#e1rHyro5g)iE7f?s3>q^k-bp)yC&TlXdwUNoSiH}nla8mKnMuhMtC$WyJY z_#WIEQA2NUk9)P)`=k@=bT*pP96^Ebd^1Y?!`_1Wj9jfjS z<`_$>){Jxy-qCFCtvgneQ&qQzr><7Kn5uDvRd?Zd3Ekz?mhO}_gp2haJaqv1^Tn@Y z*8}FY8b3d@6Wd$tA=~#_8`y$lvP-=`FtM!sC;I{j7)1oYyCV#iqyCnzPXf=%>^5z& zjf4H$kQh$P?h+Dc@%2iUK}+gtZ=l#R3hkTh17l9JK|4Dg^7 z3L9hA1e3zJskHT*EK`|7cUWU(`;cs~(N?baa<53)ZwK4K>chahC4#wgZpPm;Sf#-E zZy5iTZ(|tmQ?8P&?s+vJP;Bw55c)OEpIQ%Sp@B}aBrpiFb^HI1-hGAred|3cEULQf z$*pb?$Pt(--*pFgJ&Bh9HZPn)amzInh4Ap%k;j%9yU&$Zu!@_%fPt|B+XahzG5WC&VGxJtwX?V{)8d9 zN-1aqC)cghAoCKy7Ybj?;y+sk{Pjiq=j0d5Mg@JTM>SOw0BQ9n0GbF>A|`v=S*Uht zqP(apKBNGrP}P#I{u6t=FFH>I!Y#C^=g2E9eV%_uxj-d1iHcoy6;3PbH_9(-eI2`| zzzMXZMkB`xfUGVQo!=O}x;(j#ax^BkAo!s@&p=FIUTponEe_=59}4X|L)HYVq!?d% zhB;l_*ocxP&2A2&nQ>=Q_hOI9P}kHM!Wo_>6^y&>%PjF~?8=%va# zG2{hMR{WCJ5BRb)pzhgqV24J7yoT9w#Kjpp+{eY%R&i3HG{tk0WLfC=ym+c8$IJcr zXHC4LzgWWriJ7Z0=`ff32PHIv?K_gTPxsG7n7Npf@jTcmqxX~2RWTp|(Epbz^1-Iq z-#l$UV|{_KeSuI=GR31$>l7l5l0EX}cPHmZ7G^(}@X(pe7MC@~*wM%7KI{~(k8?>^ z3Nnbs2i(y>$VI!rYi(I{aowvYc#V^UXV&funbO~L!qC$DJ_Uv0r=ZK5pa3^o@Ic+NXLfs;badUW=qAc z1Jl!o6%R@7(H&J&JMQyHZ`S$ULu(2*=QUU{rQBKMJ+ZJ7M)~NRcbMg!5YhB6#}hkl zV~X8}K;9*##IR)Zf|frk*^Q2a8G+#qODkhzh28@iXs}WwLNT&&Cb+`=DN(j^Xnx_l z--;c#HO1QS#>ae+TzX^$wT?isto&`prVbX?RlqflmK27oR9&oZ!1XOV^aGeI!+P_) z`rS>+_%d-c$0yF=JkZ`-yLU{O=XU++3YP@NiCAw`{^j+wfY(LsN*6cHG}gcp_$^Cg zDcNiBZV^xsOkNIm;6^`9#jOFt8q@l>D^!6iFbwrzxEdIE!w!(>BJy0TD|f#5<6BKJ zk;_(>LT5`On+Lj}t6LjOGneW8Ap;q!B)wmpMVO|w|Bm$z_Wbjjds|IqxG#~&kc#;G zhTYs0RL337oR8+=IR9e9TR=B|$C^yFM`H8u`Sk47)lgM?C!(^$6m0JwX3lixJ>$Ho zldW}Yi=egGwVz=lBncn>wentT;mSMo5Jcc&9>yWqqhio#c12f<*=|7)jBmQTyC z|MJ6ROW()He=GI|XggWA>~YYZHdDOvWiPBudSD?Er0WouVH^THpC-lR%Hp84-&Z{W zpV%V|?BczBJ5!5m0{+uPgI|ND8~FPm~3 z*yY%o{MBvh6|P5ZUAJL@dZXWI)lBW`N~u?=T1%LKX3ZNbxAw8lpY9t6`gH%+&eU` zn<#^?U7|;00^c0Xv;_6+y$jJhjR$!u*1-Dtpnh>vr`O&)OQRwgh# z`tPPE_Z--5&|GAt{~WedJA%J8#DMW*y>i#`L%@>HOdq(0EpvNx?X&A_L jm=6OXjnimvzxn5Xf%44jPAzgJpwRGi^>bP0l+XkKx!Y~n literal 0 HcmV?d00001 diff --git a/docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step2.png b/docs/cugraph/source/wholegraph/imgs/host_mapped_wholememory_step2.png new file mode 100644 index 0000000000000000000000000000000000000000..20597f3e5154c7608d9113822b70567797530dbc GIT binary patch literal 32110 zcmeFZ2~^T++c!$P?XGRRY_M#X!!9eQgly2vfn8=+rk0i?N@eDV8kC9(w3|##$x^X2 z*=b5L;{1nSf&Z4F9Bqz(3R~1ifj2+< z9=1CS0u|#|NiHY>?|%(F?uG(^)?ZcpTh@13D*^<aCX9RBHiUf&N9t0I!N6;6$u9;RZYTwhO_cm;K0 zl?!~D9pX-}(`#gdeoJbg$7ig0yVewAIT!`H^HI={{Z09Lfi#6W)!MFt8}W69@}A-g zEk)rtj1&wyHQm9In&yW@iwQ3&-#HB2|42)P7DjM8_eelPRnzrdN3+x4hb$-xrSIkL|;X z$2+)tZ^PES-PSnJpiNpHdK?+41w3%g2?`4W5cr#@RYl%2hR4z2Rn_Uc)O4CbkexRnT#=E>)%Zt|-;?MM1+!S+8u-|qMCfY{IWKZ6KraH4T%`4!K4%7h)U9J>OPIIc~c zYo~4FPn!i%_)p+u5d|kyfm^y{k^(BsC+{X~`f1(QZ#F_ycgXrIsjU=>BA-*7cQ2+#JLy9_&8aJ16S+`) z_&M;I3g7_=fH0dTHN-TIf-*LkV$66Jpy3^Hg@fV?MIQ21Z-1KCAojk9hvC?|bTDz9 z3aBPNdL!B(HhL2p;XjB3wIrF#+yOZoK5kDuMp^Mq-MUcp7$VQ&JMb{}?>4@@fnkW{ zqA_ox_G`J!-bi1EzqsRJcakYiYB954h6QA}{I#?#&wp*OlhM zRn#89PvFiyp#2JJX*a1EAV%*Zu+hPV0EdSz#-h!2NQ!mnquzjA4fUtTnUiOti@qtJ z{|h^J6Z#s<4&DM9`fkCJ%-DCE(?6vEr<$?b5I@Z0d}1yGwX{A!i8ccgV~VDmKx;H> z7QTBy*2J0{8Dn6l%fWED0umqkjxT>}S~V-~i+;p7GFb639JLc5z{Iv`pkgn{#tih< z)FWVl1``IBl|Cpw07&(!X>F~@$20RLh5FO=X@ZK6trj4s&r&m2jJsd{XwSk)J9C;D;_AAaOrlz$u0M?^9DH><3 zkfi8uBxtc{geB$E7$G7F-5tYm!At=smv-}GU=r)dV3oqe!YG*=fYYmRKcYzoq`JBN z3v!qdj3YNXqKr%f%-Mmq{T2-LuJ(;cZvxpUHFHpj8~wQZ1{yZuY~eLq@hg3x!k?W^&5 zSgChcUw4GtU=+aEc-|Bo{dl5}SI;PdFMAt3DCBq(qa!b%zzVSlnq1(juWgfpt9PCq z$scDF6?sK`zfH>>wD&IX;UL{ATm_0>e>$EavoKI$KifHK@eRj;s1 z0%)r&)M}6Z<)+HH)m@2iEXTcEt?FqPE|*DlIGYjTxv2pir-B5JRiy`^yc$5`tgqOt z2B8EKo(CMJ`(S{rZ0_oro)@{6)MSOo4F=RXpGjMUUt`qQx|0UFP16*IFlMIZT3*b1S*xYdu;keztdbBxuHJ zfu8^ZLS3ou1(a_nEJL*MEw=WAlX;beU8ph|OPk-w+ih%gPu*$z3kxS3vx9^RSU6l{ z*~NU_Q+lhMPfm%`Aj)pD+YF0#-yLlwXL}&lBiz7Fl#I(x)PyYP(EQU2tOLNeD{gL)U-Ri(~?5Is2!Vhr%e;15mQ`cwCkKx+4|Z(I zTb;$p)@aFD9hnTyvWe>GHZMbksHuQv4}(JpA%ae&4ja6tMOXc{GUM~qEQgfv8_S#B z2J^0ZZD>2K8ON%dD&8qiWS5+#`64SbC~BX4M`wq;P*swcl369}N=@V@#0kI@29I z6b)_nfXB6L97T$Z`=flu1+9P*g2bma$NL}iavO@Tq!_%XXz3iZo~W9$Tx)&Oiq+aL zO}m)cQJqTpP+Ov{l`7n4sgtWmQ%4CVd-?eWq|Kh6?z%Lm0Pd3OtGN=dsy$nEjIQ(x zetiF+<0OG)_LO>kzV`;{R>r>F2YJKh9ANKw8g#Q-<0SZyy-Vi)V^x+|$y}o<=Slnq zXT)#Kf59y6B-=un@nqpFzU=UBmX=>Q0c^f}x3#*!F|jEp+G*%-m~eWtsi!^k)`$*U zD?ZyU8`FkH7bMV7=^4}kKgFGDR<}1hO&>nn zxi|a_w&7k+^lv_iE|)$A)rhRsuHSaWj0?;keQSqw;0N;mw5$=`mhq%UfC_K{AQ%rH z%lZ75lVPA5?dIC+? zkR-29j3e1sO)+Zz#9FF1%Z6rIAKiGZ`?`V9hw2Z9?H8nW^g(uFLfUcW?I^8$Ap1>cSE_h9Pyx$C85nNvT)z~jiB9U z(Z03$@Eeer2fbHGhb5LnOh%~GA3Y^Jfo2?s@T$CK(G3ozhlM>dUd!n_Y<$vP?EbGY z15y`_cig@vRD?l1`RF-cF@xADlEJPL8cbTNW;FNNzDhP4xyImuUP!30aLX zi5E-@{`WyL zPXn+k;r160>TF+7+Xt~)AUTJ z5@^-g%(FA0N7@0+l@umhNvDeVx6B50J_=QJo=ssX6f52wtXhkvM_`sTZz|3}c8NKfDh(?J`vSe)soRWrxm0kU#RSZ5v=qxv)`?0Lhl_9YOx3Im((nCEz@?b4=o89eK^2L3r`yU za#goxqX+wTLyY>By~sY}Y)GBRI?`QWyATNc{hd+SyE%iOsN=6eCxLaDi_P8i; z^Fn0J!&d?`_38MGm_xsQf?QqmmEr|cAt1;cBj_JRJpkAENwY6&IjQUdeu)@#`Tc5;YT{Qtcqc-*F1t){S{(I$TMi{%(B-OQy@;%7UXUPu^gBbU?iQWwnA4e|zj+?C z4>xc4)iy)7KGqw{)8MA)>@!rm_VDf}!Xx`rsN$mB-TU%3dT>zCa<3q0ho=Q7W;?;5 zoE}d;kfi28hW6RAc79QsG{-m4+2IMi$**=z+!a%Y}I zd1^kz=h8%izc8`b)N(sS@A)-ATm{l`$?TcA`d~e;%0qzm4apd(t*h^pNp5MAaTCu> z?AHw3@)^krK|d|!X9c?l=Y<_{@0+SR5foUFP{v>J&77Y}b6V3q9BCbFTT!ikA*(sP zVh+4cts(}AsV2&&iuwJ1K_0uyQ{jRuPhU$cuZU+kd;`$XQF}t0N}l8qS63_ z5TH%F)VXvx|}jt^cr`H?+#{=wq0E4I=!f{ptcV&U%`q_jt6ID_V-c~AZ7$g1pOj`$vfWb63F(>{!b50192Q#XYx$a$LYj_z zN4?4Q9Gaaojh%hsxPL{d0tNTSZg{d^<79{$$_jh<$NlltJKHk1+u!f6h1e@k$FrwU zknoc!;VlzL9WbcM?IM^Cxv#l*MUIW~sovS-1{%!EtJ5N7t}qF%t5Sn;S?>DSdvN*> zKvKoYth*Z{x0tT}wz+Ab&s3F~tVAj6pLmQwvWj`pce6N>sIwFCUOawbQ;#NenCU(= zU}X?Lb1$M!ecpl?K*+0OAKusf}3--VV1!}{EoXEz$GcJ4&7ndRq~oabP=r*vGOvMS;;r&^6j3*AVHdZ=ajq6V^5iy}?$VP%rYJ zO9gphFBAD1G_9$sy@E2;uE~A$^1IwK6`s@A=);X{GI^%L?qZaQA*n-c5H~cB*Ugsl zx83@luRcNi>Q%XQYBKMPqX=#4w;FsEr2p1-yW&qm%H4BhIJq=R8k0>KUyesDzUFt1Yv8@HyV6@?j@uzz?V+deBvkuxVj zWYh~Y346u>DY}A;6gG7DJ@gW9e*4rq(-38)o}C?Pf;_4Ey#C5QyS(S+&ikWw`y}4u zC-CzM0)wn73X5x!0vnD_P^P2f=j@XWrg1_~#EAGXqN=>Y9&=oXy+=;OkmKjptfQTf z8WTiTUAuguKEMEpzJ6OTX3&tjR?8847}XZ*tip@G5#!3@O=jBZ(65v@nw@EcB$Dz- z8`4~$XRaD#cfny?e91g-giZSNIzuEj~IRCB#ZmwlILedlO z+Q93d_ihe(?5NH)hTFQ)P$)pYML+{4(K{3VznB`7r$m;}j^HcCdiV!h!BjUmE z-*3Kcp0a>nJvHu%^&RE(8@;X2r>~&%JM{sbIpAaD5*(SBtS3-|q8n1i`*)RCYds|FDQzmBLoKE~z!2 zWo-kG1-oB%``j_Dbmh7c>Lc3+tNCzP%DXsxU$yO!oZPf3!AhLfe7FW4dM}K|FhXAL zOVu=7`KpQs+b7!-{=A=H^`MPC<2W-A4PPOkBIa!;+NR0f{xQ;Rsqv zy#}-0Bl8tMOh6+yg~b!6F>mmk z-SR(+M~_^=`d;N+TKf%_;SGl1y9$IA2Ux?2=vHH}=8);m**fF#o^ z+tl%3%nfH#X|+PmhQ(#jv&%zMqO$(h$<;mINQ!`9S&@iCc#E@1wl8gWo5^49&*4AW z4(})&hZVZy?*)^yRhdud4~GmG6+uKtO{#wsjsB8Qs&}tH+yrzi~HKGu#xjjfWYsxXW@RDvug^$UJ;%u}mn4+Y7JbYTKk#m?H$8CH96P4A=6x zn+HM6c=a;|mCA;udyn?rWveSf0K&zpi$p?}q5Ccn=%C3$GGV(fXZz@`b#0DsudbX@ z{~@{I6W?(SGc(dEYQF**xx>404PCqs3-`K(M}J9v7S8~LCR4WAos59p@3RhsCK{Wb zqbeOtM$y<)HqCp9hzCfI_~Z@RL!C_LLT{l>!(M2NI~YH!hw|a5q>(!^X8_r9Cjuh9U<_!l zH~dF)UEQbXCxJXZ!jtjlOSP>C6!y{gP4cp84ymnzv) zW6udZ0-Ph;ia<7DSB2^hr_!7(w;l?PDWw>}n-J-Pb#*4eJ+9P4g~+I;tp3ha^A)=k zDDVts`qh%OiJ2tWhr6WI>8UPu%Dg)uKWzy?T|#T3PXUg;W`DFLWdCx|^__1b__p!^ zejw=bWIt2(L6MD|Qjt6YoX<3Z>9Cn1MeK315U3OYw#R-eXb;Yoe(l)*Ps`z+YA{b2 zEbY{;HaEM)dj8w$YtZ-Y0}Xh)T@Och0|GYxIxjPKT)X%)=znipz>U>jXm$$Uz#RsI znLqN19L?cZ{cz`E1y#gR$gYQtmk!DG5;?nXng{QFbZR;l&5z`gJr zi2j{DOJc!+f;ta=o=TC6sW_?f^@`9S_!{w+^O@=_db)JQ9*0Ek#7*qad!UINQuuuJ zzBhI|A<>2VN7l#T)@+-6HLr*Z(w56Ww#~yrE}0%DdIbg)fBuwi;!+;M@&eMusPXX3 zjkiw-MRr{_Rkf*2oX3?fk)E2C(R0nl1S;!m3#?H<`_et3p6I5=n$;?V*NAXKD`d^0 zcK>ITyd-D@%UGQn zygC{)2D2PC&orwI%aur;u!K(ta?RKBuXoj0N2=$xOgXT)qp+wKIfFAs7G;OK2#Kw@ zhP!HTZLjE6Q!dk{F+edYKk;bsxqG>#3B~VIXLbg>C;{n>=6nla+<0E%*>ug+`Q5qb zmYXY7oCREfhp0YY*tyVlVy457J2SkxoE?gV`U8iyup(j5JEp`zehDIdR!}U4pHMu$ z-($Bag#e=@$Ct%)wv?(ky*r0oBZVyNAl|e=aJA5)-4Q8D%j`b?9HLPoC$EK2j#bZX zvc-q5VXeTcPSl?o|_C6Fd~>G0B}s0opnmE+6~v)-C~ zQJCl1&$3coQ}@@NxJoaqkj*z+lKPqEl`b03H+oqAMep>M9;sE7C9VzBC$wJt`)u## ztXF=1eWlgab2=T_>gYVey2AH>NM*XQI@l@W>*{>H5tx;1EYTJo!U;iLUDNOq^(OoB z$2^0&y7;1X5K5h6KS`6J;@=Y9K2w4y5Q|-G*yfoEH6BZ8VT`o7$VpjggB11K~>J*h@Z^r&C=+uwPFy08N<9bst z8%v+DbIv?D!qtXY7rTL#^W2m1%I0Ck8<*%DZd~c2Ganz0I|6^IW_vG&kCF| z?RlUki1}5D8IhI8E>h$8AJot=5zR6uK#&q5rVMMLPv|RMh6_l>e$qmQlbjwqFMp3K z>dgQVTUThm8)B(EwfUV>RT!sR)Q(O&w4IS{gYPIROs;dj68ggKz?nat({f?c7(msCT7cO-EJd) z;XS+YjkirK5<<^`Q9)Zhy3Iq1UzVxUUSzml)VNrPSF8_c}gxioT5K@|Q&)N-gOO2-3(pZx~m)bi}D2 zfg?_SvF4!_B2Z>(sJ#Z^UetH&jdDRY%+7y0^_s?(f*0ya`pU^oXP!2-lHHjX$7DVG z-=90qG%Bq|n#H7U$Sq`L)RMXR4q7Mi;a`fPpj0R&m0d8l}k_w82zIkrMe&PpM` zr}N};%}vR~Lj(wWvePPPI2RM~QPwt(FbgU!sa_Q0!)Q(18Yy&9!z&sV&OhRq>hNHI zwmSd$?lJU_Z|_9yPVDEW&iri1&eviqggHCfkbx-N_4zFq(O z_pw`!?H4w0HPSWNpXIpOy*`&7`~FeHS1;OeyQe;bq-|BT!4L3*s!cuwq9>CKqgAF( zpZdKukF>7PJ$c;hE7pk-V&C#Tt$je_K;EY{o})F3;=C4jfVi$OF`Iy}MfV&}SpZ!M zbF!+=S#cnAxxV^oO0Z7{MrJ zd@AvA$*0HyRkelVvW?X`R7iZE!F@66!)%gb#$<~dRVZq2al^H{oD31>B}qnD6lc`E zf3_22w#Qye&JGMt2*IrXMa3G$>F`zOT zkNo<)w5H6s^O=ulcvoiTu{ZkSi+j($-E_w?DxI7ek2-jnA7S(9bUNQzFLS}VJ+5d7 zIaH|Cwm{{bSwWr%zI_poSXaU)k~A)wIbrR!UDW?u1`$B3k@^Iny6=(|G3=W4R-ZVu zoxBsWKF($!rOt;@=;=Y-4xm;m_Kfbv6~oc{%TVS|fv{=*)oG#)d*b>+SH&t@C7~o2t&T1;d+qi|1`>+e(*4(-|sYI26Ad zQfIYQ%$~Jv9Q6k9=PBjtM;UWTPc8aPqqP8Jx_IW>gQI6osWxP!1C`SPAf=>c$!1a< zeExEfJvsxb1#~mOX_f3Bho=xx=R~9`S;Hv6?`jrcFlUYWPa~dq%sL?k^Q<<`C2RZd zbn2e%)P|K60v5xx!zQ;7w@4gta8EvH9I&$Y)B`VWe?64{274Su$qit zv|T?C)wqbLk7h5JzCZT`C8x#C`z!IL_0+lkc!GQu+gov2&lGdY0r6_I$_uXm8-=q2 zh{KYxQ4VTK%FJ3x@_Cl8+nKjFyZW~)^ zAIll5e^s0Bh)XJ+pm3ppoo`om$-17SdfVX4=MC0I{#&-RkKETj zmXf8btL%TW?3#sj@i|@QKFH7G)~c@TtWW*lHe1^o)s5?pHKk-{S&N5^^a_InkMn5y z!vzVE=Vm{QEcbU?l8xb(l`c>8Y)%Cv9ZT7=<#)Yqqb`A;m6f*VYE$Rpa}UUvbdDt; zKHHR&>f^El#{0fZ<`qn{Rm$YK*H-2od@1SaHfD5iJH;~9MS%9dmH&)!a_Yf?!;uyk z?GWDEA}oCDMJ{L_AWh;HOzAO6o1m>&`gO^*ikbaxB%*I#@{JP#e!v_Md-zpeDpIP# zswv>xa6$arAL+h~m8+h)v_VPVD5bw<4x{?EwtC&`QKPFd>xo;(*N(3rKl`x5WXLS+ zom~J`)294nmR**#38MieaCheQIzb!j=u@&}viR$%OyAK(`Jg@qe6xmD8aXAr zJ~gpdesa+rX`HGVTI91Hi8_F8RDTYr1fAn1{!7Pnux z;KKY<%QO%@cRpM|XS=S;faesypp7}Tf~yS9|BV)CN#sCVe*dm~-eFN*tPmX@IS~Nt z&xawJtTxZ`1z_3Q%_5EA{$c4GdW&ts?s85EO%q^O793Z_H)^36Dhb8OOMTGRF!S{n zANZkTa*ZXY7XE2?-U(mA0g&INTL#xHOaaZMhkyNTG{>>}KjNQ%JeXNZ#m_T&-kbg> zCZ7`TNeE5Ju#&G@WM<2h#gjmXvg8cQT$1tTG2564s^Q}Kei`_mo$sL-vw%Apm5#8* z_-ZGGU>2}w>D>i_Vi9ZYB4E`12Vmr4ToXf6W_skv#YZPjK`3+mY-$ZWywGjopo2t( z3~#y0qHs+9FAutMnk+auwJ2cC7XP$`+)1vLfNHl$#)JNU;JMr62k%)T09f;(t!y+J z(1Q>lCLIs2$rR!27j^W&f{rRCGUi{O0cRH!_1|{Qxs#m(hib#ta6Wj?5RDdSJ|fWo zjB^DLX^w^0Xo_(2aZ7yBBadp-oaN^4|1>wTt`Iak2t?@?eP^5$Y;)GLTq4!5Z$Xt- zv!|T+RNzwuvfQN(Om=DmA$V|iOw0sy=?otL#*EH%UEpr=@YL{U`OQUGi&}W*SaU5d zV?WSvXgeNWu15PvP>iX}zkb{IzCH8k_b;m;)uBK5pW?HCN`85yOgPMdB6JGEkzfvH zjUtoM4n$Fz{;a{FPwDx9J)5YOkI+ojUq@DF?<(thYEWac=+*ZZ4FM}E+pn-s zA8)^y|AR+^z-3^{x3PUobPRu3=@Ruo|AY5nmRP`rQ!H=Im;$hcsk}Dc;dA($qOGqm zGitJe?~w8O--3vW2E*aPLioj&SvMp$OUv2=ITN%bccGT|Ld#a<>6>>o zd~~wn(f7VThijTu-&&);aY*mYWBmV{-e{Prl5tM;w>v(K0*q>H!bYb$uBykfMFwZ6 zX~Q_WnmVqQN5Jj$j@N7S}pT}K*_#*9Fo!xq?9~>m~0G_?q6XN+ADX`s;tGQMs?l!eADTzb_(=mE z>t2uBUO1=nAa}09_<=J}&btdC{7$pQmoj$@ee!--Iw1+4n!aC(C`;2D-#1g?Z|WTu z`(*6h%@yM=%tnS$frliWs?}yZt~pKMtaYk}UUNH9$|)Qtq@F4CkW_yPTLR{Z^mT<{ zYnA@U`=%dPpy)KL#7aeM`h})So2MrczTCVPC*0Uqo-F~ONfzDVf z?N!!iPI~RxxBu}OQ#b`%(e2CV2=^O&inQ=Xv*|*fw?)<;d9dL$phH!^yny@?nSRk? zzzECsomgL$aMtee*6tPFP3+$|+-}+LV^;os9X<&K;RVFi3FaxO9v(&R)Wp{QCSd@x zi#hdl*xUIF5Uh{3F#8PZ7X4o%!Q53CcbnRRGl&?ISzeVh7h!p~-h7g=%3G!Q3ER^A zvOI4I_J}RMxALbuoVJ!?8RK~h5L=9`7SO<>X%&=rCL!ze4z)Tcu;5RxF#|>6U$DyAw2I z;2m;E`nBU>vAx!&NT=YS{m-A3Y3*c^(Ry)z^NzfAr#3STgT!u5$qn+3>AaOg-Q!DY0%~Icc z3&v|Lu3@E%Zz0Jz4G1@4WD98mAO*Z>AqCQX#5@uV_QmV&agd?2%Bb+{EomF7_)2Ktzwc*z{Z38MOPcmHNZ}8oj;~uW9R2+`9{XO8-V2 zQ{A$)H}9OPK3n^0d#YQ@zDtBoj2_eI_bpvPPSE^DZ({`frGvgDjk(Lx08Zz<_7H`~ zBTbpzv(qY18Q#v53G#mV1%9K0^d^lVr$XgAF7O|w)NvkP@0rZ~U!4L9p(+?R%8<*( zsM9)f347e6S{7c26+(q9U@##b$S0lFgr1YzE=XU(oS6fVI&5YGg98;-gEMyGANQrC z`79fdFh)kJ2+>;=4d9WKf0s(z75DxLtYGEkwv~qu6l9A_UnIP^da7!ZNBhyK(-odm zrdRgFO7|LnX$2RH*JF0BiC{-gWRK|jcX7}7ZDcAkGrzDYFE-w0H_56I#ZEN&s#&gK zNVHTjiZTTkCMYUpcylTn9rEwS4M0PuDDdCE2fR3|x4Hh9hQ7@{eVgxHGvaODM4}w5 z9O8Z~M4yq}(PucF+;pvLXzI?HTh4(GbrUP21i@v`1KYy5>#H2tcRk7|U%MGdAu^EZ zgbiLly^GDI(fe|tUz2IE^sZyJRqD1AuM<`B2(lkkKEVaY`2Y6_A!QuppjS8Z5|(lPPbtz|Fx z`@&;;&rzrWL{oB6%)Po(T{_-0J+amJNKg*OkfhTZY@zWwpl@2L<%$ugV1MxV;Zg|o zGCwoCWzj(5-4r8uwT=}~#V9b`<|j!pQ6i)e0Oj_VN1 z$(lxROQhpnVbe&_7z zjrdOn%;=HnFNb2%hJmS@B7eeMb@yJHaZqo43G_|}+M}bmLsmfhH4n0;pe>J&Z0;VU zON3e;QX7bz3W3Y>Su;>qshRw}e1yj@&atHAeTvLeh5VyjRSwkgMupTlKR#E&zyT^j ztPqmh1(HVGERWwaSRug)Y-Df8a-d|7DVeb`hwCk%F6OFSk&--=5Ex?%W|><28xJ}Z zD0Qw!iPAJ%zJE9kKXgfsSW5FO*7^c~!U!-wWl(nG={EL9YoBF~s;PrnKXxde?1vs# z&8k%`(k|S6D=&6?ite$-I^K2)?6TXl*CEB=vNM$a*A3(%b*bLbR{OnQEi0-$!(*T5 zE5|=5g?Pu&@9KFG+$|xg8>h;yK?rUJt_EW3-4M95yI1!Lb~SqIt}{Q*wv_v81TtBb z8EKA^&xcxvwz6b9$6fN<@zEpM<1V03*}}+SnLFkK@K@LaP&Alc{yV6p>%Y$JQD@(H zTf78jH3VRWww*~xA55O>sj1cWAx4Ektym9jK+=Rwc^Nt#h~pJ~R!$YokCARpb+ha# zZ>N_IU6vP`_6Be~(u<7SEu-R6rD?fo;in8Qx0U)!Jvtz%X98bub1FysX^?jNzT^@e zUotD^dZ4lzybDWYL1@dSN~ic$4d*6ZfL?;VpS%yH$Y(<^6DNY>%}v6AYF!y71ZXVC zMV}(QDmBlfp(0B^fCI#BKwH77>9m$QU}7>2GbBc|i&B8EKWf1pQWDDav|)(TiNPtF z>rCnZqJX01VD=xkH)C{<31fx9o3fp5N65c_hbV}faV_HS z)2Uw~b9`!5r|Y89PxTHxz1CBj5dk&yZPG=3#SEBb_MMydkM8_t7T zoPUE$>;%|LiqGUCmK@?7BCdhIlKE#Y4X(Nj&B#T!<)dTieNzsFIAA=qcqYRVm=rz_ z#*VZ{%WVe5I*NqlpGk9y(f$tTli*|nP-l!e9vH)Y|7#{N$qL(Tgdc9okWmE0%U@T4 zI@`Z2OtV%*5ToV4DF#p#M$4OThGm}HvM^!~k#w>uqgMm7%siLD;W~`8f&%=)iM*US9p3pD)1D+K$di;L)X`QQ%qzkq&5tWn(#& zPY@c!R&Np_^neld?w>&|@4n1W41UXK1lq2nKrL>P)xz7rV7+4OSuxuBu1{Q8gnjTE zZpsCR;x|x-iwZ1(@qacQ^!fEim!$ITxZAWP%(k6ZHn2V80?2aGpX4CX)i zFVom@)NMMzLhj4Xf2YV0RA0X0rhpD$V3{TOL2{PY<77CvVhUZ@qBR#wi%vPJ86QnP zTL#la{0y>q@Gq0$vdgksl&^#9tUuzWsS+G8 zQ13ZZ^wHbaWr*jetri3f@S9T}0h~9z-xdzcK!>jB52>3n88}M86~s3HG2diMU0vOa zf%fGQ^+3LjHOsB37D2EQ?Qx=WemGnnN4u~ME*K*(d#jf-Tss`z7Cml84dO4CH+|we z{0ua6X*LeD0W6$3ZqI@MEYxfiaDYC*iAG>C`425zB!3boFS0~E5VRCUw6Rb05o$)K zJGh{?S1^McsFCO!f&nN$R&jGc1WN#se{bi`A%5X}0*uQypFe+} z2P=u&0CYZgJ~Y*MEt?QX=f?Wrvd7|wgf@Q2kqh-TzOnfq12Z zGQTVxiNQ@G(?;+Zo)b{Vs{OSuiX!CX2U5A8fJM(kHKiPUQJ>(Zmb>)F3fyIBONv58 z2v%db|Mi6x&;G-+7m~Tl-`k?F?+Y zS`ka>x<7V82Z!ZrYl%kKvAryb(4ml3vz~q%_s^GaRQcA1Sbhwz!jLg?Fa{@D}hMgQ!L^fleie!>h6YMOT zstDl%E!04^d|Rqa;G!i@VYy-^2=YD=%b`Hl0+DqC8`1HSRt%X1*B}<0;>0asz&QUa zLv%F+xbT4app6~_T&zJJ6b}Hy6K{9SLnKBJ8PWKk??@r2{U-mvctJ`Kr$~~3_gZZx zd`fh~%y4BQktZl^Xx{CL`5><^$2E=-#mhrHsWc-1&L&Q3a&H1URX8{!YZL%)7j7Y$ zr~@EkL%w3p-D{2QqdO$YTv7BGkM*B!mk~Cv6&UTGN@x9W%ZJYSz)KfyZGdGw01$UA zS^~_I02j=(4O;<7eK#6s4gk^8YfDYA6teUNv=tElr8ocQ4x#_A-?XHLhnHb{pTx(< z^9`;~f1cX5)GyX~b8=OoDg-qA-e{?-YKX3_w3J=`-+Wb+-%OK1(MxzMa2rE^BTyuP z6tI>O^an$f3+*ovE+%`F2V2A@Fl;v=2<2>J&br%Yg?pQ%Nm!gz#HZnw2yq1~xnz_L z6LDDSSQ8CGt;V)b0zI8TkH`F6`n4QMP3z+&c}}%`x6x*UwzYNf7s{$sMd1a=p^ow3 zz!eLCcdob?0C2qOA|aUCy^CMD-lzAbUpFgFSF_P&lL^Plw5c8FnCvJrj^F+Y=2pG& zq7d+ctD`mKsl;-Xsps$g(h6EDpKC_3?riM# z^vxD|&Yc_T_D^9YzUomw!zmdhARTNA0d>+z(_k|*x%`Cwj&9OZ1im)wMXXZVAwJ zMoU&Yzij@^d+Sq0vcI#)-%Z}XFjUd?Wss@ zO-FiXAF=c7xG&k4 z(sHi_b2b3&hHDnN^Itn_!D`?S|I$-i4a3Ts{%bew1W*+IH>L3dRzBb9xj#MbIP>)V zd<(581)rOu_FyO57*-XAkOttg-uk7#FLerZ8@4J{Tcq!81<>ys`z-Fm+B#FGIM;ro zU@Ps7b{aK$jSLcOsFUlifx4BC{5h95&Ex!&+# z=-I(P11eZv26eWJM_xEJS?QU&7f-B|FCnwg z+e

Ms=1BZJjIqmW~bulz2A8B5ZRLW&4aBTglpn|D(C{j%qU7`ag3=?)BoRg9a%I z41ySt-UOvMAibH;k!B(-5`iEHNO2fx0fQn05J!d}y%|c7B8*-bG(mckPy`JqMI;b3 zhVq^SqBG9icipvqcYf=A{%|dXJkQB_&e><5y+7Z5Xq0a}U6?E>-Axc9I=Jf*{R1*N zM6E`LT4QbAL0S+CKR8)3inj#e;{GiQWx__yz@XnoCzR_#FOMlu{us2!FqtDYd-pz% zcHW-9N7^Y}^X6YLuk0x;>Cz*G4MM0DVWi5SFA@>C(CIg(kRND-1n8^S)36<_)d;;o zY&oK(kel=OcnZ+G!^Kld7NYCke)kAyV`mW&{t{`U^B{WuR&_ZZ`ac+Y1WP4k%xPnOJot=x6};Bk`JL)35PCOliTq4&ym{!C&6r7P(o4~jY&l|Z(V zHtTJ`N>yriiS;7cObUll?gZ8>Wcd7RZBXHgO_Fht;BAsDuvo4}c)y`pEq^6K7*^Al z1U+De7$ca-4Z;0tr98zGBAc!qmf^06GMulIsy)vhSvB|Yp8or9QR~6-S=RfZ`HkZo}C+fod~l>j?gX&PHxUs%nIOFd{&(>hxFL%Kdyl6yTl&`*GaFEf~5qzAPR5(_+63S}fu57f+8f)wwCzB3yNiDBd@ zRZpj-sEwd<%l40*F2;?wmzC$fB?E+M1uo;ZomdIF?ypm!y7vp+$kws}>}qPzYc_(- zNtkQ7MR~c$ee9KB`E&5CB!A*c=mh^qK-FhqL07~)-Q}eZdD@Q+03ujICDAikKqXyL zB{tV{!NzTLzoEGewj6h4uz1wroDD0v))$*Z`}GGBY0agk?@Y8Qg^!p+=_(gU%Gb@L zWN;VN0NnP(JcPJb>XO`?CdR@GHl6mJ$N^7EjUG=}vA}o?svbL7F2wYEV(iOk#WyV3)e9-Y32enJK|?OWQyV!!q&7_*dm_oemJQQfNQQ@;n||urZUwnox&ntn=Ni*aU z>^052zQuZM^YaS;;*~YaF9@oWLtgH1cNKW;7GAt~>`$3I@GYJdG**X~vf*V8R5AK; zEVxnb|J9^@!QruA$sl;lZ<92Qb0JP&q+Zxv)vId(d`v*u#ccbSK}~BN&EGKc4OS;O zm>v)}y;*9OT=^q6i*xGxt0v&@UcR@qoopPi3gvgBH<%1-YS^3n412|fue|s*iO7UgD3*+8_-L&=_LE0s-i=#|%x5fc(VCWr%omvL)UL_DKJ_vtX zKQp-a*1g6Qn7gR-g%i`=k#R4z3X%4ELE`|yy5h{S+}obP&_Y#O*joR(^)?F`{W0nU z3HTKke8k1_Hj}v^&Z6d!mwy3(aB?@4I~5 zXwqJd`4YQkyQ#U_QJl>t`|%=6m+u~;(&)>zzT#q4n>AeFAuc25c?Mm3e++9+uxvYH zii1bR8ta~R&&2gx0jL3eb1{cV@_|u@YWWA?EK8^N&U(9+w9aLRB#`7;HknmXzQVPe zU!N0ay}l8y=f#j_Ed4>6q-y;IH~?VI<>L&Nqcvw`?;F@7XeTSpl+Qr#$X=c^& zM066fkCd*w6y%%(-64{Sw_%OejOK>0Q_HqO5OUXZL&$ge8!+B+LgsV%wxj zT7#O&Af2&4pcX`9mBJ(}<&L%$dKTnRK3G!i&!&8EMV~JsqGcx(hme5OL!oOIpblZ~ z3(S9cT3h-mB( zaehU(9)OgxyIxEMy)^fmzvY^a&ZxYipsPx%hqRfp(LY+=gLKroU(sR#ZM-=WQ~-CW zzmr+#Vuc<<#YS+uHs0%6seHcKyKiH^EAB3kQ^7vINrZ*cY1d?yXL7PFM9&r9X&D@+ zX`ky0k88T&)faw^9z^dFVvR`yL15ne*Y)@i6AiG*XJ#4X>|aKYtm0Qu>rIvg8`1v^ zYxHB)$M2gPT#~3g>Fa(GRqZCr-qLcmK>u4n=SwaY?EWZUCi6LKer+t&hm zvrBg?b?MaQ`+=^XWgw7uddb?n$4VCkqlBmN8Y0ohR~s$C#NtQq{|Cuu6JBF_XpJBw zkG;NnL|0Y(D_MV*i}_{)M>%`H>SsS{6kf3~-#r$;z~u%tAs|J4uJ4H@ZRhs})Eht> zibp>~l3^+2F8OHpzN(6H>aFY(4R$IAGW=Qnh$>ga<#c}=-?Aqc{oMCHmoD(~)1npG zQlmOG^3KMhU*aT2%G;6lm2kZa=xQ4Wpy=3&Rkd2{R0NHGAuV8_RN-cj<>e^+PEgXjcl|S+rrR&84`x)QyMH6%itYACr zuzw>!?-<@s=N0IanpiqTG-Ug}Y9)fmLyK}7amlC9P4R$;(Dk?e9neJ!1- zQe~|bV+V;#zP_&}g}(wsSSBsdm-65g4a?_m&->}TKQi(G1iC9_)Gj<glgr4Q$-}1n7@b0~SVV zLD-!ov?R;tlUjA-bs9N@ngKnN6hpxDmkh63=GiUmYgA;;i0MOYXk*Q!kNxG=Z8kdb2X<_aB+*A2HG@c??l~IFW1jriOocJ}WgDP#o1c+H(dL0bC znX)il+b?5Ynry!)BzYNHbAZey#J zfTV-rqvyjj>5^9mM!fo(JJa(F(<6P!tLc=INI;n5o6l1J0eV3Zk+aZy84niqKM|Qx zQt1zF%5H~WqL)Cl>{wTI{K_!fBj-eTgCgdLq8h?~Dk#}afJzDSLSgTEz>fW>A<<-? zp&WCk9_;G=SWVhYY(mlm8h#)K3|u+I&Cz>uAaB}YyC%=D(qd1t zSDpmVwy5MTbCya0C~X-kItS`3w#^v23-8Y}F^rIrSJhQ?z$w||Le%>+dUc-+7~L0{ zZOk52-gsu|WMx7Tc8=ce%c};ekL`YhzJ0Rmd_kEF1eeAu7Z$Pb-cu1C@e*3kzr^pBsDcf2)r~~&$G2cADb@Keec_+d&t?X-2I+s zEKof}Z7ipG#u0pRr)*Z)Gfg)uXttUQM?i`p0w7*ynuV8 z(GR0&-ILgK^3B>*bVhx7O{6-l;hJL(=2C$~?T2o7LbdsPCgA=J5NBWVXfl@L26d9l zqgRqJ(S@EgN|I+9%cIDrg4Cf@gH!e&-@H{!G0p4XPPIcoi=hEE9v2p|Ls@9e_H?Tt znxVroyDShJp;9i8%zJz*_kCRn_h}#qQYjoL!*>>}1A)R+m2f+Xw!B8U^>9C__`3eM zm9KE|Pbi=VLHW2_G0QOS@gpB+*7MrB6WoifrDz)?H52cN&(8U174KfH&si;xM-FQJ zRgTg3$xWa}qGdK;4C-)!r|?n~Gt76x)&t(GCygL?;be3r+wd)wzFCEB1q{B?Jl-W7XeW&F%W0Tat@39jj$#3a=)DU z7m60ejGUH#r)6=7@^bi-v<~z^j8*=R=g(|C<0(s%XIVl};YaN~O76f+q~?4zZ4N%z ze3Yl-*5b}W+RYfAJFNh1s{~7MlymN1ciei(3aj4anSQ$TQ)G7v&vZL+eFa2U zej+bwfb2$CK*x4U^(vEHReAe5B`FDrphV<>?VmekCe8ov`Fho0>ud7nJUiyTN|LN`jmsN`Ld%b7%;3f5Gp@!S6f! zQ7OCn6venX-xtTJNmaT`ytd9w%HI*>bug|npEQAxUlC^e%6l!ycfU$3~ntYNfRya9`CZoUMGA&`HJa;*8qSZ5zT$m_Z9gn zG>e2>W%we(W$1}hr9upD#Dj;F`vUu8;^9(|PsiTDP5$QOgG@8!*@0AS1v_#y0IE(Y zy0^D1ebQM{VF7KAv_s!{tj}r^ZoICaxqC&WtxH)4)O1&mb%0$G$n%$PTW7sjq)KdX zXTHZ3z6!@{&?YRb5(5h8JAbNoa8(2RB3Xzbd%4V5l-nappzghwtXRf-Hix00pl~P!x3?(aX zcw?9}{40j_@}W-Z#r}B%ON;DPf2WxNOx(|3BhK zxP{Mt_6F^j!n&hkwUkS2Zew68Dz@)1pqkesnxBfAd$?W>PJW_**p(7u5jUugw>wbI zsq*FE@GEvC)n`sh_h|IWr0%K#rRf}MQfuDmuanYjv-AsDX-J+G3i~2%w87r zk_9bf{6Qdcy`+LP)&oR#JFRSt>&es;?uC&H@s?C!IMGf4pB3%)v7fhZf?-&K%Fy64ntcv!1A>Mi-|ib_$ovGK-R*QI|7WxpIRCXZ0i)xy~%9;&l|S zlZ>uIl=44@e?9$WpJhAjxz51$mg$}ik3x+E~&%0`c_jSLvcuH#7MOy^& z!{{|Pw;c{aMeAn{%(XqS&n5-l!h?ti*OqhxlX!HWB#&j*uW7MHwe6I2a!(TdaSm~U zA%`s?w<{Y-IJ;Ru2;JU(mkJJq5zmxI(j%2t9Un!Ekq%z~0oumkeDuQ#Wc3Y-wVqL<7_ zM7M?%!hWUeKls3O;bU^AE(VODqI{E5TbvTa(A0Cu((RqivF~Q>{ERJ>;ls*6s~3@0 zO1~K=p{OY$dkJhSp1hCLA`g-f4-xB##tV7LZC$i0+?&mudL-~8VfZLE4aKVmGuh9p zt`i23AJvp|$ui?f3kkm2`Zca;lGoGQVCAt^GbC}uN#sKmuN`r>FLzRE8uL{M2^g9o zdG#I&u%T@x9D&w4|c!F5+F7Mab;b@$JuDjf4fLe@)AqBlp}w zk=v=)b$S&t_@XdjlLxv)!pl6yy~@!!)R`#guDXz$g6W3DEPd`i+~Xn`C8wPFdsWO} zTKiy5LN!nO+aE2Q!1UH*BX8B&)m(A=R8*sf+WLYv#%yYK+@{b7@q z$#6A$wf!v=q7VZk>phd=Jh1#q=9}NK$lC67rD41HXb5Ur4 zI|`JH9t~LZW4%x*2a;pbKe)aH^Ks6>x=kAtLj95QWEY8U+zYmk1Q0$fcb7MJ>cDgE zi4te!8b1F=+(+w#_egjAI`kDN5MqDq2KGH7n|!!wq>1+A!sW&B@=k8jG(ZlFk$ZBt z6Js)S)+T)d%b^EK|ar~5nF1^tR(Gb>MG_luIq}>n@>njA9Oy-Z85ST^iVp# zH&o0rq@c7@zYY1W_lX6}BMpml{Cm3`508h!+tK>!DBfQVX;y%EwuSe#W12sp0kO7y zgI@sB79(j2LR5f|e!ls8#L7YJC%(4pimJJKy08pklaJW8hTX{Ocv2&VNo1VzFB6kRUR@>|GIs|+N zJ1@`&Rv&gjM2Amt7NxH_USg#ER3O^BVfL55X?cN>7Zf7+%HH@=6Vx64{ZVrs|TlwW%AB(|N0kj z%tqVK^uVj~v9`1#M=W>n@?`Gkb+D*O__%2 zQ&|Q{3CUMfkxva53gOiB^ge<$=)5%0WmFUR>`bHa%^qMZhV*oTm#3c%0QYD}Z@_S# zT@gU+`%rM&%)oM&9;g!F1oKJ5HiO()DAOPP6UKL)FznMe)2@=kDO^!mTe+|^jMd%* zGAIW}@s5>HKgzCg=Z)wzmS5j0QRm+b?GnPbM+FT{pLV}CHIQ^d%j%jL>d_*WvY0c{ zFx!5WeKvBciqay^ZV?wD|9X|9r{*tL>6d-JFC>j}VGQ@JWYTW5IRt5V)7GhJK!t1v zi9bTrZnNiht#E-6&d|a123VB;Bk`jc{FFSC&%TLwT>veL^r_#V`~b zLGQUm=D(w_HEWZcmL_;$w}3#pq?{o?Y(e=7vFU@W(-}3j5!-#dR_xn*O4zMwtNA-g z*r{(^4~KvPDJ ziWkdG)Nxz>jZHwCQL&QmJTWHh5sb2SjvA8_F`nCgP|U_DVgMdsf8a8D1UGn?bffE} zMmWCxpWMaP0?dmcq3~~9-K5e4=hBu7VR$B>;x-Of%3gH8+;LpTzOwFdapd!wF?H6(qzFBimZ-?}*p z4L4$572d{&0T+Hl=zCBx^W|lxOPT;DO^$ zZ%QsN0PA8crX^^28Y%f*o306%=$ejdYNQURF1wD%$HHR2iS@qsxaa^A$=hz^ zLpFxXiq{DS!UEEgVv3ggG;c@+Z4Jbng(+>AvPZPV@BSMcMi8glXu%*TUci<#y08+dD zIxvQN2lgg$5Kinacw9z*?|IH40g)d%>I{+Cef{ zoyVFZudH+TSy$1a_rB|}a5|q3fb#&xXi!&q^wjWgq7M+4M;)!3*Il%Yl#EDlzhGw^ zw_u&pUcL$suc=KVh*K=P{vUfcO&Pj_{~R7-pUx@yIg}I@I`+|htFho;jWhpABu7E4 z1-1UM43xMobN}j2ysFM9x9_s5A3OqAH2?qr literal 0 HcmV?d00001 diff --git a/docs/cugraph/source/wholegraph/imgs/wholememory_tensor.png b/docs/cugraph/source/wholegraph/imgs/wholememory_tensor.png new file mode 100644 index 0000000000000000000000000000000000000000..e725d6c28edffbfa9879e42cb9b36fb6a15e2516 GIT binary patch literal 40367 zcmeFZXIPWl(>Ho63c3~0Z2_d&vC*Uogn$T&bdaJ*lTB|TJrF=qP!Ui;K$Y#DL!d{MzQBfAum{*uggqKJ2hl)>4L`lBhjvRy)Dx z-FL6u^njqhJsJOP8?_bmg`lJ>>Q|I+_?pj*>?uBJ>?gEbEBxf}(I;_7p8um%d~|n0 z*tVUd{V#23zR#h-525_utXRGIt($ode8?BNo2`u7&*~HqQD2SNmQl-=Nc_J&5@ zlG@BiviKAd?YcIwXuT}i9!e?Fwze~kI_L8<%U z*=@f+1;5>G@b7S{2Qqj3{uk)!Y0ce#J{;8gA3_lHKdfUa#Q#5(p`yg^28;vZqjgQP zV7T1YuhCxNHuG0EkodycwXt^2P5QUVby_+UY$Iny$RL(Qcjjv&`*^_fM4@$0+OS!i zvAnc)zMxbPkrn#UpxDRm$oO4fCbgrS1zNC3ab&Z-3U-ZS|H#59aTTX1XP@f(EzO@) z4K<#-hA5-9CX3@C~!hR@3CmWLkCtqR0I|Gm~I|P`AiB4%DZ{J?1IC-)gyRVOf@rR+j z7#!^XgCsBqBmUJ@`CPZrxvz*0al7PILP(x!hXNU0iJBPov-Ki&NU$2efS^{cI*EYI zQ8#0s(HgIWyadV`Nx6t!CTO#Wbd`_#4Be1Fw4Q%HkE$JH1GBrwplcSM&!mewJhDX> z6{{yN7Q~=C9UZhp$>lWLJWuP3H4vbU6Tg=haZ75?AZDs*w;{_{!-)Pk)`&y>!*^3x zKOignJSx}Pswssc0nKyxO|xuX$P(6$a;;iSXfQ-BYiwpF552X8f-y2t6``r#P%urh zXz2dueVIE_Wj86bO_bccPlK6`Ato~7K(0ozmyOrgLAL^Jxv_30v60PhFbqa!P0fqe zkJp2htysb;bIlhQWDMc{qyEI4xjuHNc*9mtCg6vC`%iU-G|F%Ol19T#EXRdNAPrSN zik)atVFNjxcEb=xSKq8cOn59B+a)c&L;#*^yP~KP7SV7_m|5wuuKs z>K3k!nygc~0#oH&re4$dAj^?YOKTb(0V-Zobuz{>BP)X+YH_n0*S(h3@dX|7>xV+7 zrGa41bQYt=lGDAKs#Tb(6=NqarJ_QHpo!O+-j;LVC^+-69KTjT?5y7)0Xj)Fq(d0E z{|7;=0n~`1cbmZtkfaVB<4XKG=EFi|{n9j1TnLtiSS1AB5lhmSfs_V^h@n?gmY0ds zQSf?KT?z06h26^z_CSoVV;E4iIE)WvWyliIM>%*Q3NX)!!cfHZ?kT*kj*HoJ&i(CB z3-ztU#-O`#VBmB`2&PwHsfW0LR z)Oyl`i6OnsJ&+Tr3rI7{5L2465S5og@5UEoLBTy&`1jG2P81w?0C;ZyESI67O4hec zRdnScC_<#1rS*j7?#5fptyS-b7GfiS9i45ylDHn?4yE4K5NbAa$*6MFYu$1!<|?cjH9k|x(Yy_MEAn{t=#{#kA38VpfcLx3 z6*1pWsI`5JKeML24cgmlBwri=wpM-t$&GFgGcx2#L8^N7?2amAELzoUA5EFiim%HR zFpz;DbSo(|6mag^gp|sf+L+H@kY#+u(ESN;)>{e#{a2!WHiheh{o>(cW*)T-a#kmW zP}YBDA~Z)JNI~Be$y~Jqa$keqa?*B)`&1GC@u|jpz;7syecQ#bXlw+ACBXo5B+tzT z*rY#CLMJk z%c@#!T`=V)g~<#dj5Run1g!2^CqB4(D| zq2L>)xCe`@fT3mDT@n-XOrAuNUf;_)uedGP+b=2A6@o%nKGaqA{ofb!Ao*;ZPW<1& zQ#*fCtVA@yMbko_|MUpkvwprN7y3{jYrL?RicI^xJL_S3G7DF0xk?(g#V0M# zKD8QHy1gc>7*xYeGvjH>z{ky1Ev!VxPgc9UnfSEInQm*6pE+6{we4`;%9;y(ud8BM zO&}}U8W-EWH##tL>OaXkhJ6lSkEgiNxw#zF=5HaFVO^rJZD zmi2~`B^;|3dNkNb;+pk=3{useH`5oDH{O_q4zT527>U)R+nS?hsP1T<3lC8D4YOQ- zh?rrskAT=Xp%3yYVsouKs&c=K)(@vbceH;sqP-*Z526H9cZN95=ncW}{ zmS!dQYAFQ98h4v{VGKiPk7?7TDq$QLYeTQkhm8BIOZU@{7Ha=VLa&odsr9P!b+8Mh0Y-<|(>BZy$R!jfy9#XKYPLg=L6`$MZ zQUp}nQ*GnJWVPpy>df>GG-&av;1sbPP%(0d_jJ1VkE{?jPQ_Y5YkX#dP8r#{oaJY6 zy5K>m*xNIt{hS=)Vr2nt&nG;={i@Ce@e;XbA`;su=koV)l#0z(wMUhx771qbv9G7uU5A2`zZ5MfApVxAx;OiAk7)i> zS$G4QVCzMCZ0k3-8c`u|Vb!w6*gx_B{i4O>1vjnt9rJl1OS2=UQ80ZzyPO51Gtje4 z*Z4$K8vahoXZMkQ0-NQbcXcPFfS87JI3A;&IRfd)>{20btd6vo9B|p2P8=Dj@#(iV zov{f1>fjciIF#1V%wOq8hW0VSJH%1=HrFF(Vsj|$UDK;=gopj=3yb3sE`%fJjnc&T zrn>d9M;pj2;F_D8-76`>7I^I5;OX&7waB9rRUT!21`^pk6?q!ZrylGP$wos19Axly z`F!bemM%5m(x$S_3(Bg|CM(Y%;ZHRE#S<}@!W{KV_z^9Jtx)DZ%6})YpEY^mQpAsG z@yuSEX3{6I^DBs{It^FjxP1?sj|P9u;XMtuTF}V8?_sDySmrjdTN}u~Q%YJn^jR5; zOpfHcq#xxnO`l`!k^yRymBc zLKr<;5!bvokC$xx^gcZcUAzChxlrg$$upm*y|m=X<<|ap8Y(&)@eY-2mLn4>(Zo&hLt{3q@@lk7~csm~%$1qwh|Lw=yb^!Ame2PRe zq$2Cz!0V3RGl%hdk8IRh9Y3P=<`@?{huM)sZ&)9D4yMOeo0W)6Q>|BeZ^IHOrR4a} z7a{Cxl+`_AsP*a%nG9(ene}UIV%955mo2h+`u0*|!W8vZACii3)3zJG0pY>7+Ba7FVAd(CT=FxVL7oZE%L8Xq@svPJ93052+a4^z7H#b1k& zY>h-+`D}h$VZ_k0V~Z}R`Vu!>#r`wVK*mZ@p0Mf21JjckqJs0Rq0-Zbq9Z1Ez+zvd zEG$m5Q9bFI(`F~|ZtnAZDx=E+l&{&*KSYEEnq2F|TdG_VY;o3tj%OefBrz0t^QPd; zwjr&KybQv>Q;8GAVfH$P+F^N7k4K28S}V_vzE<1>5+zdMVuu7`vv&XM(JIO*r{> z1UHFGXhb}NUCEDgckAEmJPs_XKYHnV8rQOU!CnPtbd248+GU2kY|mbn$x)=;V%2@1 z+Q;e0R^F8VX@kK*L|gOfr;Y8qfZqbaUZyB<959w^e)YIrtTpth3D#SuXG*w^j4m_w zZ(dYAp(&9uR2aa2>d3LT;>VM@C{`Pa+NfH;$H=(N#(gOBvvQ)XfyiLPOjmBLkzypW zfEOA_F0A8tN4va%lo4h~QD9#88lN>%z8*4qu^8Ij$ARmQTMV5X(25!yu-N`e2}ER& z$@`nqVxAbu%Nat`;-6||?%CuwegjYzfb*r3uSG zJ(5l=8N_L`TnjYa2Soxap$#U`3X~^4$tKA-ho>T&=dgjql>`-3)SZ+C8aIo{&emmD zJ$Yw6`3psLX~Md@kAh>i_@5P*qw{&hsk1QksI<k3hPp!t|W#?o1EK9T_MZBdCGK!Aa?3(0oT0O zG_O>X^M!QVyZ&y$gFT~N5kpqeAlTho+GZ{ILx^c3T8j!b-S(GUKRx~Et~^e*)zoh9 z0rtOg+~Kkl><8!86qg=!0tK3`h*klnPh9mc$fSR1boU${iA@=`ko_lsJ(BTpKZ zaiI&tW^K!pR_b}Z_@;t9cAim^Ap#X*?=`(K0yVB((Sc9<+%S10n1$J!XTu|<66jA^ zIGhG)UiELXcMO}RmDC;M{I;63-K0j$W=G@ky;#G56`4*RDao<-Uds<2sPS!f|9sB% zN-yIR)cCz@{mcH8ZoQyq2U4b5mxo#yEfLu?#fk51JdwkzW+2msRsi;MGEaEl{ste2J4%qh*JGJXljStlUV(T!W1EG9_J<-fy2h#aY~R(9EzZy0~^L1c8G^3|5wR*&!E z;-A5UdB~*QvF(rHlk<| zGzZiEfzSXmYkxEn|H&f(A&iGpV}K=I%aCu7b01Lm!+HBuuhZU+8&_%5LvGgB`cmeR z2hdmAoJRXog?0yj&E_4-;hl8CGIZ+@KEY+`)|^@IT1WRHKVOhWXa8ZX_PfvCjK%wQ zQVI9=D<2N-v3Il2@{TX^>*FYS=G*y(AdTVUTv7bGB%_<%F6F+U;vylP#e-@|k@_5B zx()B6v&O2AG@fg1usF>>R1d$r8t>4+rN6S|-k(PDQANQ+T}-)F5Z>amHTC4Wr+)3v zJ9V;f6{pteRZC|!U6XD7R^HqC1!PoE1sZGLNreDk?`4p;j@+lE9xC|K^BG5ZyfLAEirRqQw>2*vD!<^owauG}iPkrh%U|N8rlHnUS4&JeQc&w+ z88YnQf_QI$%=kQIyimp2j%#{@yTZk6oHY2kibAhgIyWh$q6klN#lYXzMu&c?St@5f{4<880gY}$uZ8k%)G_HMmm7*7-c`o@t%5?a~-YCj*-x2jIF0b%!@5C=ScB;CW z)<>=wK5Z)OaS2tMNm-LVw(>sq?4lEjzjsvaL@XC>*hppSdleaR0s4837)k&dS)Ym$5tSsxpY|Mn(Ek#Tb>`-8h!ObB+IWRDohG}JU00mPuL+V zz}tBfYloyO1sbD%;Yv26 zH1|N#KYwmxQa2gzt35oE9`@xX-n=HS!t{g=PTocVzCC?KxL!p9&i^*9L*zTQqh!W6 zU}KFNRp^5XENSa2p71gA<}Obb4|-|ysKVy6*!26Jspffo@uY_44ALd-Qj4Dc$7zG2 zU>9nnBrFk+9QzjV`VZMtHwS^WJ$vE`5A2GF!RYzPG?g%7$;RWOW;=cymyBB?^7M*J z-bblF@Vpwy$Fs((T<<$uJR#FM`}Z<0pxHor|K_eW?)}G;tw%BkXH5cePHmT|N7@6w zO3_bki0t@t+f)oY&y{)G+byP=8RD!TP_ErIQ2yv==##tV{krBCDt9Y+ixb^T64kLD zF(#YpA0u$Pk2lxzm&07@VznKIoyPAZWbF?7bFUTSIPozC`6J?@*N2^4eb%O#nfsD3( zspejDr7(UL7v&buy3hhs1-k*%#P<@dFXC8-;rEs4!q)~DH$FSPqn|37MrJ>2;-+!3Z`P1s8a*MAs%G|0CrPY&XJyWFMJ8;4O7MMiElBoIl$*KD->i$E2?(NRJ z7MnQoRLSzlxo4?nal!b}I)74$a2N-FE;=;)xvjuYjq-Ju+m_;-tdpU2<^p}sipQ(G zTK!)r+#A0JlfGs8{GQSLKBu_;md)g_>woSExwYf(v6FF?MFO}UsRgdFkCI;oPCbvq zPu6M)n6ypYN0WDK!)%3(a4u!;)WvzPFDivi}1=op@Omcb6!^jJsOqk5c?S z^0-y)E+bcGXHNmtVw>y}kvikaI#Fp7e_9K9hg?mqt@Z8B*VXDQxxMk@<&%eg{^5>^ z&&L>p7XO>K|LH2!GA)YAJBqMS2w47kV$tH^IZ@P)x60Y~`dsp;rXfD$_LBCCOI900 zy~ElHc%k3ap|S&+i6ssn-Kc+W`%LEOefYP~p(3xJ-#%cIkDuV?uCsNm%=-7{m0KHLk0!zGSkr%QN#$^0}So{;)azkX6gsvYo4k2cxaI zm@+fcd3f$b*C5tlg6S6v`~A50ve>3RDwgMQyM_PDB~kojK+2!Fbpg)))va{w5UVua%nhwCn7-tSdhn2axMI%zdfTiH=R@*hJKfgz4P zi1GU({2-7z+g(g5$OFR|uNdLYCg_2k$*`nFV*#ZZ6}QM0xw4%;9)4#eFO-I9{hmUw z;c?AN_jl!&?>w6>^kHprsPy^Lr-Aay&n9uTK20L`Ym4*EFC-x4TdH*J>l@}%|0yKb zC8FuFITfbmQGbOt<5zuK{r(;7nC9yzjpjT4qFPy7TkkV*+M_=^Qd61yreICFy&HEY zU*kxAYZmW{+-C$@`V+xpZq1gLAcbNTwLAMDs$H&`lttnp^&bvsThcMzB_7Mw*|~HR zHxJS=x1)qD1VW!#i()U0L{1`GLt?0ZE&b8${Je!KFnMRJg(HsfZ5%b|TAGb0UrQp>*svwm>G3|v(n zP9HC>c&ubrt5!QIyZ?EsYO2WP7iP&Rx70WzqVoyaKk?obetMLNj5|rE{Y391SmV%- zO#i5$Y_gaQ+kX9U$YQV7PuCas%dOH-CPlU*Y6Ut9@XOwFRTa;zN}u^g*gTy3PCI=Q zEmC(znx60CCq8nah%(k>Qt8=H=Auz*aVO!=knzJq!;`#_KI#K2gkol!tIUL;u2wnb8(se;oy-4->{M&GmTyM&# zS?BB97fx(1W8>hwrF3_4z-tTZgbSH+{n(|R&Y&+3W$Vp0& zpSXn=)F{pHaYTCFFz?K)7}|MqO}kqVOhS9~{6xmsr;4LC@uxMbTQ;?N*2k){PCAs7 zxZCT%)N<)=NBlJ!}0*AIW?PMcgUss|-`kd3ctt68~~k9iP<{#%auD?rQ~; zG8cHD`RYjK-s#1I3l_;ACybH$75GgqxPIMSgj;^rnL)v|4}v#3gEpvnQ!t7jjqf^b zgc4YZp!nlW>sH#`3a516*93m=Czj+QW-RLK1k$f?DlS{nNDUh#3~qF>g3zIL)}T!N zO8mu+9LBW_Ms#yt)Huq2m_WP~bEbd?Tuf~K6x>juwHVjGM-25Q8+E4LkQu(KLdyzT z%^FJK?%KGbEAyN_fL1huZOla#iBx@ifZ%x7vvK9zP)Y}EX_$!Lqz74mw?4+K9|IaT zUU%IarVvI`dwuSbCJ+0D|Dv&v$DH`hNy*F-H#e|$+GpD}k z@AEi=>91cb|M03oX4Kaj7WUF1Bh?*`Tr!mRnavkK{5Tddc!Lr!C7);NTKx{uUo&dh znf8y4UyDI55RtZJ?p;4q5Nc67;SGpR|Irf%KcxEPRMt)i|5<>4Z^Gz}B;?|t$gCRu z0J?kQN)zSadP({Q7vdQE6}9D(@LhjZ>^yry@=L7b6}kK=?cci~Sij&l2(LZ5ohzW|@qavWHF*E)j&2ai;zllh5t)4kCbKgMg|{Y9&mT|=tS1y@5p7-^!8^m&c@ZP>>`%|7ziM#60s7qRq%-ZGsNR zUfPg$2=?NcPnP5A#+~^+aN{|XCt^0HKk{~We%@ENzq#wjJ^v}mgIujor({~gr^~n+ z#LoHhY^(RxK4D>A8o6@On*TKWx6c=QT-;)l>I1EBr@r64OZv{>3IhFHR6bdPq-29b zc(bdi>BLZfMBLLSEc)R2x|1R-L{Va*EnRFzxy(urxlRk5ENAw>}H5y@XS73h=<8sl(+4%0@+>`Tl)vDi2(!ht+i+mdc!~{pShNSFOaHiw_d_XWafQaM(J?)hFKYxmfHUKOSl=*0*ZeH&*xK-NnnVnh?3q zbO~=&pG$D+t0{?LJv=<9V^#wQQH^|!nNBjtZ$eNp(MUvd@UiYhaq_tneN|sf1E9r@Xq_z) z=X{w-KfOIW$FNq|4}ale)t=vPG|m(R#J?}ehY0ss*oY;{* z=1|Yx+7-55VYaf-vE-IwcK5?h8u#zFm8!AejJtr`lddZI#Lq3-&n?F9Zb?+->a0rC zYcoWmGY)U|AZ@O;3bqEW;7+(6;Qak&;~fSgx7($EbTND27A__d&t0c9%eX3jQ!v1` zLT<+))Sun8k%NB>He*3jiJvcW28ftC+s(A2?Vo`KT=vi3tbGuKM8UidJZ+DvvE{C4Sfb!9qS*T@q6l8| zOMgmMgFgc+^DB7S|HEm!t5&0^%!MnmoMD`Th3JZmZMB~K)`4GCLgp$9l~DmWs!i5B z?Yio&GBTwk-*Zs>6QHJsb85dF=s#ogXb-^)S0TQrEK#-&8|@dV(_fsx{xo~=Zv=FT zVbcWO^Pbx7l79IiRqR6i!0GMI`xn|moL%bpT#aW!+S}VpD(^;u>)F%f{rAl2GMvY~ z$t51x(|&N?cKXm~?^yVrm5mvV>;u0!XF15^T+Oa^Rj8mqrfwHYbJvcc=104x(sqAg zH*JU)#C3s_0PnkYQn}f8t*ZZVgU---I|W6oa5BC^ar%ADL&>Zc^?@FFWShq>Gl;f# z6enYs`z@-1mEnP~-&BI@Z-J?w_-LV3e3Ae7UhbwUCDV;WkvFMU63;F=t`A{_1yV#B z!rhC+wUk7XQ`{)|u_Kt1bJ#&`T-~kzo*lXSGBbR7M$dBc%SkT#j9j@^Htk0R$5}|w)HFEvXz}J}x25*G z`kKOixtNW|fupJc;vl{?-uq2!hcO^9wF5C>$lNadj+(eH3$%snyCK+8om0j3SC?P? zn$Mn7n(A&+FzL+7CUdX-&b2?@hO-jFcj<>_8n^r)d%UPTzy?)}@iZ72;sfZ|qc?u# zkc!2CziG6C{ls1FIsK^xjI<}mJHTC?iTvkB&B2~r88pg>2!e+`s9?I{{P!j=2W z@|kd3T}Kq`zA-ymSCd@2dc3dd@ll({=CGY-w^@F^l6n!5?YztI*NWPb#B8EI3%{0; zkr5O)e5jF>FaTozQAOqJkcqNg19@4=X5a5|<9V&Fn^-@Hfr>vK*X60ZG5PR!&~K{^ zE_k8UbCjfur+!agOoDNR*Ja!-F|ntpP$=}@VVEV}3paP1yGqj?{PTFAf)RAOLngfb ziuHO?uLvoLx`uG%9oV+69b%n_^PQ>XMaTlw^7rsOUbotkWRVre>5@h~PNo-(rxgZ{{=Nh3QRRECUZ(a7X&s z6N5HZF#e;|WFn(TU`jXFsjxZ|(P}!Xk0ctTFkA9+BslMvODc8!VGx z8(F!;N5*dJzYMn0dB#<>$yKe#`p3ag)7R2>z|9(uh4RIkh#fk|0xQj<>mF) z&3a*+fOCuJcv>~)Vp>U@J^moNb6ILzi#YIfiwwCe0^Gd_5i(ehGdDrWW>B@k_{$L= zJQCB9B%c>1fXE&i(A9~ZO`-(Yd9?~x-k<=t7x?TmXLbCpdyRPY8|wHE^)j4zHz!@0 zyBjN0#uUF!|J*$yRjJ7;Ifd1eZ-~Nfe&LHktVR{sJ#rh%XNRc`Csu?1GR7E0^jC z2WVF=KmN_7Nd`kx-Hu;-jUP4r^VF63RA>I^yM4bv1&zaut8t|iN#*OuekHhqAMIz7 zT29b-c;=V2iB)3i^S4Ny8yJZ|tv>_qW)8@CT3s7xKKGP%5Zk!<@Lf(g7+U z)ty3_m5_QeP!}fLV(mLoB4tJMceD+PAY%^m9ytpI>j51J7I}Ca3N8eR15aT4_hrs- zf{l&UzvY~`z_sbbNE6JzD!S3>*Wm%8cW-LDjImv^pDj#v`mOs%7q4AQo8z12Q8~QX zxjF&g4`6iG`*c9)KRa7PDHNc_qx%eEW_nXLJZU{~^<~L#Z!i;Rw}aMcA*S=nAc|I@ zA5b1qnq`A8D|@w^%>rKsS%Vyz+A(&fG<5XNS+#6{!Ujv>v(j6${9luBQT()qKY9(R z{tdRs@1Os{)G#>l-vN$fq9Np*cq7UX!@>!Q{QjeO*uhjqjstq`=w#Y83Tl?4vAc(O zW0{q}olu-1mOb^ivl|9m5BCg2{-d+ong17}=fi&bIN4|@{RJ3v(;vBTf35Dh_* zas9g)+2HtS9p8ZehqrEb#w=BTKLFYP{(h2Bl719F{J%~Ok$Ia@bBxtTd~R;{+3@<$ z3T1_bu8{ww+B(&)7$%pcHz_a&3~b>E%T- zvpDjM1(JNL_*m}PJ68xAx`O;U;$N?)V;JHt1#0Uxa7Vt&{M7&uf9d1pD&u@x75O~? z`Vng<@u$$$We6mb4;-oi4_IG{k>$yHXDffAt|gmh8a zd)_MACsIFYkiSLg|kkGRNO^fpW3q>cEg8&Q|0cVpsQOU-7lApWeW)$i)V@H9M4Vv|eJSWRaLqfDmrMb; zsq4H=$}WTLx(~D9vqHHwlucJ!0R@Q=L()eq3TA7UCW%*l*4r$PKqhYP@9b6y&8^8B znZy(YI+@nTfb-~96f#-fBCXDZU?>f~^}ZGo6EZZ`z-ungtn)uesO`1T8(FR+PLsEnz~A4~z{Pey%O`8;}`f zlEMES8IT)q91|kaBB4cY{0_V(-59QY8)pJF~Ti zOG*QGK*gGz3Z8CETQ=fHX~!&4=-yt`q#&4mV$}*g$PE5H-cDh|1~bb*&G@XICtU*d z$WSBk);~-@6sG*g7Pt#j{%q*}3h-!CeN*aI)-h>rwdp8YA&GPeYLpwG4N!AOI&0q} z=3W;SO=;=?6(>1hnZm~QDr^CDu)Vd8Y?J>LiC1;&WZ`jux6@hZ*jFc==(C!aOYL0! zjF_DPX2Oyt{FZ@ZY9yN8VCoC_;~#S43=pK!WLi5LEBhgf>h(_65qJDbQ-NzW0na8w zDH7ZXUFhA-D`c3J6flW(DI^A^P%}~lBz0#7WJ?13*1*82o2$A^A#SerF!x_q5HgoP zo}Ur9{Q60?mnR1OiQA#4CmBxrub=;az%Ks>RSd-W-@ya=t+xLIF=uk(zZW{6H{mLV z05!(cFEzOt%8afI5-VyNZjUg4zD*-bc=HkZx1~IP!GR;vd11t>EK0AXbNMkH2Xv6W zzHLu*a=l@T!vF!AD6zbBRoma4Ru!FM1VGWOOiD7tnh}pu{JPB>2GR%j@7;q6=!nF1? z1x{PmenyqN&r79`lVl_>mIzW%&P_5KW1_4Pe~S;_>*rz|rV0uxC7sBs58rR<)K(?e z$brMZST^rA!?s(x8`TvY15jQ|cWjz>DoFc5@r`zFffimrcq!b&hppp1l06;DNiiOI zW>J^yy4aV2cOi#WUEZB#!~mZP?uKj-_S6pqS78RQ*JT88E>tieXLX)zjEM@!0Eg_X zo%zkW23mSZ^OFjkbz;DGhmiBS ztj2F=5SZ?1GzOe86@*c?*Z&?5Nuy6bZg_BuZ|NSm+_=Pw zuWNS^HqK!Z`JR8Ty$gV68^7nTQVu$&0PZ~d>Ep9z<3(d{&_j;Ja$-Cx5>eE%7;5sG zM@Z-s51k-Y7#9>DCOWcc!5VQmrC~xRmx8A+F`N$+$u3VLc?AoQ!<$bK9rFyDD@I$( znJ>7$C5`z4eR88GkZkH_*wUs7?S*JVH3K+{^{4XT~02Q1+LDI6h$l(u%A+2u?5)INpWJc z1tAr55rC;*6QdM2Ly452pN|k7u|W%EK~naKvMy6M8OUoM;B#gwTYt`~rv>$SR47cu z2o{*rYnKd}`0ozpQIB8MXCC$V)xNEx-VK?zb<|&5BjbZoSDPA;NLr%`SO>^s>L6Xb zzLpl|cgRlc1Qb#^>(@g%4UL1qI`9EgQN&AzUf&~=LPDQ%DbS>7i7OzH(nvJ1-Ex&m zW`4O!-C;xb2SK_acnE{{we$LU-mM_(;3cT|oZ?avTwnqFeB;Z{r&DWcG=t6n4VaWa z!{gHm-|83eCiYjqfH$$AUw}XG`0nZ7R!izOUXeB5ffY4`FIl7O@ph^}C-YqNn7h;ncYMneHp0DiVw+^dtKZkrook*n z3~sYFIhWy~2Hbr&m9B%h3}b^#b}lLW=6`6WCrD>DnY z!89=nO1T|)c~<*f*p&f{on!^z)YM;rNUChm{AOJV!#j^gl0eHo1!0=%y@C5OG_8v8`Nt9ln z-Wnp4ycJ)@IMkH0gagcsl6c`pvs)iXw|va1uRwe-urL5MA0(0bB?7&{oyLwPK47*_ z=s&p?I!b}O4KRF4)fEQv#lDQ?jIEl&Zz0F9oEQT`=H^91!TK5aO^d1tm5|lc2oT)< zVA_D-!Yz(!>i>#0$GT)j8NCYNh@pmD=9UMUhNsILQF5s-uxtRm3U$3>s0d`XwR_?- zytjbw{vZTGVq_??7rF5z?sv*@m}E#dKbz&Pz%D((D@o58wRI6&ThIQP6Z1Go?SAKN z=?BcGiT^Z)FHV74zzAn^{|)aOhz4NCN~&QP)-YAPka>6otHps10pJj^EI_W;^xvo) zrM>&GoFkv|oq^KV-Rg_5+zem)Y zdSB92*3DdymH9t5R=9P`Q^BWy0^F+WqRK?Jx+!#>{?!ekf~j86&|Z*!VYXqwRyJ%_ z6Ju-YWx3%ipL!Wv@$-MAUsC8DY#2~8nODnXt;_tIV}a>E<#z&Xy(zho3Ei*gF||va z0xLY~F*;q83zBNRGVADHeGqQRNN%N$c>YZtv9tnBY8;ZA#qtQl5fEgf;g&Y%gQwqs zWr9SHKn^)J5is>=9KNuZau8}f<4PN0bYjVWCm*<6KuqxA%gGuFT7aMy@6keQ_?=;T z;}SE?^z)j-NEbr^s>^=&d|-4;z|+SniDzR~nQ6b?q?7_M!$cvoTV#c~GQe2N)~`Z} zuDU|b*j`4ir|b!8tj)mU#_v>_6WBzIUImF|-n(Q@|4AQ$6dPEnWB=JHtTkvB!P~d{ zYY{Kl$RLBL?W+lTOGL>{mjDqo&I!0OQd8&8|^N$FGFfbnRXR@!}|APV;1+7P(K-v#GzmUJMm%r@2ReHjv7Sq^Wzm5WD!-& zmJ40Vw}gfhh7W~IFw_0O>Of}WUW`T}fEF7i7ab|c23;#JXD%yLo%B!=G{pesvN$$! zk3n+ki|vrTO!k4EI6(Y0 zwclLS9Kt@21YI!xgZEqMVhF$xg&E67D zqnjw<^u_jC&%r4|q=X4jr9ADCE;CfV9{p%gyGRJiYVmu#BX;-d(E^@6_WZZY0a28^ z1&N@(u8DvGvR^tAa9)F&C5fPg89gAp_N8P+2lhwMUG4olvX#lH-UEfQu08z%(%Jk! z_IRlT{df%^@OIx@K81^*vWH6TbYNKtzH1kX-qsLv^H@pyNxgQ=+WlPO)ANPdXxkHu z%D0ogjNfVSx)f?WA>b&*u%0KhSX5_wG(-y)LA?#&)biM|OzXg0k~%^`)WQfie>p|x zc)H$cZBG>(D(a={W;QZ4?an>-T1kqAePku5ehCW;fcv-%K;O8xrg37?3RlA3xcF(x z5fO>CukW0vRNYX7Jq67z#5mm($CZ-@vPyyyq(du^`AOR>>18sSYUC~*P|6YD3vaI1 z3QDoZp&i-2SzU#nfh@^CSQ89A7yfZIt>k(Pifkg2*DxMK?^^k31GawC1Zz32)@y+4fHzPMfCu;(SeN<4x;P7Pt&#i{t69lcp&EeN_c(;nsVJ|GE{T5ET-klhJ#AR;M z>A)2fg>ENe_qWe~r}4VLOcp+P7cKG$E!xz#-5h=MPtt@j8wH>W1pG5(JM^aubRmG2 z7Q5u7((qX^oPa7qb2gs4?yv61rx;LWz`rkQkZ)}f#=F&P!dhsETMbjgtShS+Xn>-7 zN5w9cs{}>JyHba?9R;QYT2KnyJ}jww!2r*j+Dm(l-dz;^gZz~j*DPg}f7zN_2}Eg( zE((IT$Cn_N6p&Z5K!ii1zJ>g+lZj13c%U3E3D?1;Zo>(jnFP*Sp-md{PKjvpUya0q^4 zb5$qG?}SmUHb{HxKAy@V!P@8IiZtRvlh?$1T{#<3t#Ba>`$#tLs#U)pVK&dIg#;ufxMJ7PIqTZTK{AxD+Q-6#$7^8w^Csy5Qb7pS@mhrR3~Jr4 z>5AkVNfnVJd??9tlI;7Y-(MB>sJUa-GlibdszylT?Qr7ZHc-BVSJQYOJ9;6<^_q*T z>*@;*%jS8?qk3i|g|nP~*`g4NDaa;q&{ANAiI{rp4Ltyna^p){d0J&BdzS+zFI6@t zGI~h_CrJJN*VlcUYWt2N==P!Ut{bCYbUM&a%-{4(H2aR4>^Bb8&MmlqJQl8D7WrC{ z-x$hMV(3?sBREysv{#z~HjTjHDf+J9?kzr}%2fb<((TB>2$vnq6)Rj0 z>`6#DC~E5QK)#jsG)13yHo8FaiTl^ zxvM|oBi?Y-7N6tNdNo$d;Nj7nTg=}uThThTd!F=?RDP1~)7G9Hx~oF~#6U*U2> z_cTMs0@H{dexaUrbqmFD_Ps7W=TGyt??zlW0iP!Ka!yV!YW3Er)#Ja9Uka_AgN;Um zh7xAl#bF}|>lk;ObbAF_u}>E&UhQKC5Y!3~5VT-vHK#V~HNr&T6?vd*Mr$t5cW#Y{ zzY-v^F`rL~ zBE-&E*`Y&1)9qrSNT<|m{zu{`pQ7ErdvoEckBDPMwNcjBc!TE!{*FC$mBE2xwz-=_hh4M5<#NME+=&QG6pkkPrKmT>oqcytgEv6L5)1J#VuX+b; z%Rf$LPtW16C=w;Jx0fUcl$JE?ICp=mmqMwt<&Rv4hu3s^tRMTiZ@=Wi(pW1gv{^c2 z*S)KEub+x)fApm9ZPOE1@~tHnJ`}SSGYbbxE0b8CuT>m{_;f}O%i7<{wsEFTW=F{e-+czzZr!@mS5*_O5?_uP6NbC z0hN?@bQ^)nj>qIqoj2$@u5P6l$+ywe1lCm9-Js@wHrxbVrhFRP;1tp(yTa*h<`3VN zD$fvU=wQ4ZqpiR=zSjkw-)V45bez;9;}|+i3&29S{t=Odl zymUw-*23)d=5O}wR_ssVd=%00irVgWBzoa;e9Ev-w?Y<)Pv*ESIr1GW+^;AOkbbGL4j_HKMO3##=9$5( zcWKDB$pgv~C(M15j78*{o+_41Y+8SNb(U1x^o#J9`|w7LxS5ZJT!~4Ob);E;)!X*Ty!g3gq01p(cnoCeMk|N!T{(DzJb1FHD64 zOa+h~33Mt&?MRBM_#&Fu??j4N7BiE=#NQPSI{`Y3w1;@i9h-pA zH?_Rjk9UAJe{<>tgZ|>k_D$zXelwrV$j7sk4H3UKbkv$iSMWGT6xFZy0I=H7;mb=U0w2esXy7tJdnWghQC-@;|sd>y=8jqVyvk`MFS_ zVpkiLU)Xjk(fP!Bh)o@OGo-3{Vt6`FS#2x3R}WF413xVsp_Trc`QgVoy8Drk=C21=rUC#pt%| z?X*zPbBr7%uV2+u7F~O`Ft|6QJk77L)=;+7(Ahhp zGRWBYt@5LiDyTi&e3s-&960pj=x)$m%QXYpQTTA;mN-FtWUVI@)7&{0rl*VAT?G+3 zDyfr+E3&-EyY|?OLJ(8`VU2yMXm}RjLani3$s1$IGc}AgA(t6GQ!LZ!6AO9 z{F5%I;RujT{94tiRun4r_^SpD!*6(2_k@N}G?^_Qk6W*{wQ^&^iu00q@2 z3<<{Zz21{Even45)cXi(#ESpMMvnO;lwuln-Dtmb+Qplq^^n^FxMF`l=~*Ep_l4rJ zF2~i%c`~!Pi1Icq8TF6zyog-xqpt6uhO)xl?Kc*N%|N;xUzG@k7ZCkK3qo5EG|OgepN=Xc1L_gWKO8&8;NpK10s}3W(7^--qOUS z!YTcud-(Qfgl#}oYI1Nl!47WM8nNf;Iw`fv2d})~( zx1_~>klhKx6^0tr_u<_qO=^8haACxJ@Zt_|w|oyWA5X3Xd6jB=_Ful{+j7I1W2B_K zGqF*bdTzy&1`YE?Gkj(eM!V-5jw2f07_>i5Z1EpUG!%39zXjZ0#$^dvCuQ>VmeyIr zB-f!_{|U?2lut_)37zwJm;N~XisiSch>47{WnDgyX7wO&RsQf4t29mAHT>89K;ae0#|1C$eTIv%##wAJK69J-2zdak0@RB8#82$y{*z0%j&` zqnPqMp^fsjbRCds`0TUzjEFhj4YkGo!s%qTIqC2Ts^OB%xDpLZDvue%pQDDN@6#tZ za(3T)G89tX@N9Q;{E}baHg|cyTUApPOKEXtX*fiJhwe0QwU7g#QSG{A=X)(V}g)kwB(7zhAJ0=?TbSs`;_t<>d;foU3Y zmj|-{)y@F_g-bR6x$SfIV8>Wk{^Vc!avIqP?p^)03zJ$G|^{Pifz>><;`K6$t^M$Cn z)Qt677YTOMomB_t3kXu+{~LdH#J*Rz_%?^&uim~yle+|>fuYZXn_^Tp z+s4Q!ez`T@q~@006zU2^+5Z59M`8B$K}x9pLx2yay*7(l)}^MQ`(oaK#*SWkwSWbd z3`@@52nq#d`OPn<^r|J-N&~Ej;zrl-vNgd%s+RQwjsTlg6H}y=9$Ka7~ug1))(+eJzFtPj1jpAWDwVBVTje2-T&!XMgg@ct^Sj{ddQ?o0W- ziiE?`QNw#u(++UK?77A0e~Zc!0+NU2D7-$C^z(@AV5ldx?80q7;(?me*sq5yMi;C; zz%xsW+ss8D&s%($kAF;TxQvPZg=;7j{Tn(+>j&Rs&zh~L&hsTEjRmcPpJZYDA@j+q zr+e)H6Gocdw!lC<>T~yiuIq>AU7Dp1ELNfisll!CTy#k@m0;eY`DOGh7o*m1hI^B% ztCj$>J9V=ZSnm*m6LaG%K-ifQQ#|F}o8cIv@_;*ufH_C><*u zP|eqRkePgGaNT)rX6j4X63mbVY1@d24$9pI3rs6ySivg+yEGrE@;bR(>J(Z7M*}n!z1&1RS zjI`#ialW_L8C0`cy&J zv-wZwAdsLPS99*%LN!8@9`@O%0KS_mFGU&8ea}ADFErQcpZQ|yWb6^`%%lt_$@}{H zW>$87UO_(j_=t(^rfB*^^{mD6r-6oB)na#+bgP7F3)VSLA1Bn8eH;kuZuIs5tL#ml ze_Z$lzZc6AqrJVR4N(#H4Ed6aJ0#uJqdnKXo34yp4Lw=oi-~;1RNPll)NO(QI1k3p zTq#&;iW0XI5n2(rQ@sZ=wEUNl5ZmV?FmnL#BAiNOzq|2f*UnO$X786L^88{l1)5FV zg>j}QKdopiRIA{kf^B8*^xLX#p5^0xB_E_sI@&M&V8_^U z-2HjNAIIzXMp0D)H>I3K4y2pJc6SQ^O4(0z+aBE8ot9zQQUQuOYUVL-8U&=EWAozz z(2n1u$v}#h)2z9kmX6!zSSMC*Zwq9h6)n~bC+{)`fMh*Ux4m**e zyJxqX3Z-laGtCSIFJ~*B{B+v#x3cMx2zi<{yy22LXA=Afk=wuDf=tQZB`427PGi|3 zkYal9E>;-wKT%(J5j^PPEb+GO_U}AP7&ji}n)QZNaODd^f@JwUNZDLuYm$jDFK_@jtAx~&W(I}OweNzp(tWtb8xxm|G=b;2aNtz^uYB8-spyK80g-%Ah z40qO2n6S9IG+GH<@+0u;M*Nr@WP`4tGS>G72l{KfH2WR@ks4j6B_cMGnf9bqyu){% z^YGQ5=V{?dBCmx*6zGc{-0*);HK(AO$dgYKddUf+fY!}Pd7JWXF7;Tf#k=cUN6&pZ zU-doTD762);N@yhu^<{o28L5-^=EpV&=bwi!~(i1Ja9R7nDw z#3TnaDUIm6Z7*i}98>QIkAgK*M^w(13AQ5Se< zeApR1#}X6&s%??>?L!=&X#%(%6MjY!WLpt9VbVz>xhH3iPXCNz3LFuo)g-$m39`DU z(sL4ThoR>_TgJiSU+Hj;It-|eSq58ks}M%b#IATt{nAlRVT&#mXSmv;=;7T6s`<{I zeDjX(x`Fl5b+5Fn(jw{-JNH919K`HtTFx;Y`dm+ccVEAqB%2C z^`*;?cG6;P^6J=-S)L_o72VV8f zm$y|(o8-x7(Y2s172TIx90k~Tmqw@uR|Ti9Yc^tfsR<}4PXO%}DT($K+D)|!nkn@; z6m?#!cN2PxDDs^PZYx0awv%{7Aayw9v67$Rgxz5Wunlw)dxrTOq%GujzCM5cJQRVA zdP$q`u?~01-3pzllLo&aZIG_RNxF!$%~B*_{*^xh-AoHERuC$#Ua#?Iz&gZp7sQR< z;NR$aGi18yoB&E)iOvi@(hoK?{)uHHCJs~>ji!8Q8!3cH^=~l9X8n1l(97~?6Q9Fn z?yA`S!9Cm{riNdG`lWtOvF^E2_v$xi44NXYnkI?*XYyW(woNeS_H9uuP2$li_$isW z!Pq$wxE|X*k$GwE+A4qq9dVkg)~Gg5@#c)&{Mfnuz(qTL)A&KL&{~)KMYhy|i4kY* zU8FCj=LtPZE{-7`S0mjOex=hqV9&bSQK(Vz<-7c&9UewmFX;q~f1VgDY_H97y6EE4 zoi}CCRlm3OTyWaYXKzQ#R4}~*^Ya!!quM)_q#VQC3{sK>WtxGT!pdu`AB66q4DkIw+4fo&5jiyrTEmqN|JCQolQPCID}wZ7v>zx;7vh!^KR?K25p~Y zhk=Xdw!p6>ZGjup#GlxsomfAD9OPku`K2K63xXdT&Z%W4RoAwbjW}n5j!8kE*qblY zbZ7O9*yMaH1gYF%0GGkZ1PW8SXZ#|=ZB z>UvCiJn_tn)~gR#sABTOF_W(X8JfILFhrD=I#tyMUX)DQ;%{;bFwXp@f)i(B@}EO+ zJ;;jh&WhWy9t-^JxEG;{YFT_Xs%~>P$9UFdjFI zF)m|y8TirZr}M{nTrI-gX=I(Xxzl;(5p$>c8+lZ%C?mOirhGcXoA`lkCUT$3;in=a zcQZju%KqPSAdVdRi?+df{DVo-pw0Uml}Y-0TXg4M@UKZmma#8M9G*NHvr{SfdsoAd3d$DCI{gYFZ-&1a zHjJ}Ysd=(B%yV&BE$dLT=nXIzw+Z#I^`|afRg2+$gQ0zC(K+E~P#|}x9I3b7XJZPv)2b)h@5~#do7E3TxN_|NN;Spb6Ndwo& zot+VyG7{lI7u$`#3CS?yx(}=DsrHp#&U3lDH=KYsvJ#-2u}ChyR!*R^F~+^VuI-N8 zno2g8Vyg>7lorb?dvc~-%X)j=nm-uBQK!l)E^_beQ|=hyH-RiX`XZqd>E4GTkMXI+ zNTsgxFas1y{|Tdr&RK(c)~bgoG}=6Nqwe=Qo$sCyS1TLHIn3I-S>*AWGD(pr07X9D z62r1K*AKUACy&JLwAJ)V(p88SCcY<6bwo5Q@l+Vmj;wSjQxx{u+y;B+ev;xba2X6m zo^cb{b#<&qR83-B9cBZ$TL46!piJf*YJK2oi^RI;_I#NXn3)FNvXZsh_J%NC<%4-K z&q4{M>!QS*U%MlWV4pN})c zxyILH?~)CLWT6=AKBlM<3uv7ic<;nw^d$H00$($x*s*iF_ts**>f8ohnR}C4S#4Tx zX&CJN2s^et_Pj&lZ$Sq`UWXcr;xGzB{g~dx1T5$m!#6vN^&WR-1{>W)tyy9#x)ho~ zF7a@unztgmS?%C*>l%w^Jh3q6B@!_;{-;uWOKcCS`uq}C8 zkrs7I&u0wN9m=Csv?O+DOX*~)-=W*Tc}J|t3V^VB(|fsAMFW-b*~p3CA#LVCay{*@ zbmA_gY)QgA#?rS&o>d(%p$5w{yk^XX0u4GrbQx7j-?l!Q7-)jGGjj+mb$FfWhsRqgOxN|_LU zG1VHC(MJv)Y2L!4t5rjf$Qab%!VbM^4L~j{us!_TTiQY>1QSH(8Ps2I`tIjsI-Cb+FtQ^5q}oz5ZcRr5(t%db|Ws;I%vZFsk#zC*fvXM5u9K$KOwju0HT zN>Hd_eyW3>obU_Y%|*9n-f7GVX`X0Med*i(wbN19uF)cC^bu;iUy)DHO}mfTO<#Y= z6ESjN;D>8VP$l_2&wE>eXgWN2vGU3Hs;(n8vpxA!>G>lPDVA2cS-17c)I%qXXtTSe z^@@u$(Aa>kTZ>q~b(;i?4r-g}>T-TnAlLP9qi=itDO(ZqW0&IzW0a0Y0st`?bN;8) zL{DyZ_O5LwDDWLV@D&}Ujc*FHR&dZ@(7-?~&c$fmadVJeuraaFBsEyZaL`Mp%G zH)art%`g=_uj^=Sl{PZryS#{7-0D5Eg9UKhz8?Ck_UerObL&7Tx2NPF*G1f-r~d*G z`vH;tDi)L67DPMA^^c`&c~QD_sgCPOYtd0d zW#-cUld-2rk;X)ZwHIZ~a-twUdQ^EbLi5ZZ3yUy1 zAZWIztdmkVj|r}Y;2vzbzUYdmaf&XB40)Mou|jAuI7LDHZyA=zq&Rori?D$HQ@E(rIA|ovH>*-$GnGhHE9FMPLgVD{qrNGe(IZU zXwn>|r0I23_xvivbe`1Rm^&(sah`g6pfPOSQTF1Ta_9F`KTO)tE59C}p;ef4cizE{ z+mT|{?(FHv{xvYHpDMm1P;H;}hgqU*>*fpIsC}YS(OW?W?^t|ge!IqlT0*z#RGBSDwpV6h+ zx68~&b;<9^;aWARVmYmS=aJ2(d<4H7#IuUj$mnZgb;)K!iqEbAy07NRahABL=^};1 zVPQ`cP?^EjzE;0CQmhp)m^$A`k3puM;=Hex;kpfs-vQ%yNd20^S<)iH$%!`f=<=>WD=YMK}QS}^c4-WRl(aAf8OE))r_YAs@neJ z3SAZ*8;#U*Ax!@Mn|kWO-(Ljb-4v<7ZbBR&!np!?30SP+frO}o?mw0O@TVClecj`) zW&r{a%;W9kpJoBL3G6I#>h0EF4MXo@`PS3<&^YW{s}nEQ8b z`lG*ZysrEEYVCigA&-)hm7Da0=Exv~tfi%wW9KZGNrr0+6A7zv79i2Xq$TSQ)yJe$ zr4sZ6Gtg5Hmns-xSkhi#gh5_X+bB$X$_Qf=VI2FVsds0LwmVXOAz~B>OM2D>s&=q= zmZBq8;b5YMAdkga=1 zsil7B&dWM}>aOq&9#lA5-~B))#2K zP&h(ms%lzAFz0SFw+RjI$3n{w_FdM&m_V}LFQ)e)=!RI3c>DLN*-LCpC*WRGmcz{b#bY z+*$jqYncotnxlrB;qzbx!B1Y>bH%)UuIv5im@5neW4U;Jbl2JE*0@U6l|p@8ERzyC@GDF#EmqOv1c zL9iakPdj<+54V|R1*PmH{%{uud=K6oh$-nbh_UF*_QiK9pm<6NOyMEEA~Rp&_Sjz^ z*a|iX-FOJ$#Y~z3Zc_uA-hX~qfld8vsuIUVc-yJ~u@y`zCS+;2(v&dVR#xb(`XBZe z&W(Q|R;U#6=}%d5eA(C6>P-R?-oIlxkwoa1G$|ydbl8a@cda=ExnsPa74`@>;pdAA z`0mO6Wx2)b;HNY8Pvw_E<%H+|t_UQwx-V;)Y%>)I@@?`TR)D`%%S}&WP}xYC86jpz z_|^_ek!a=TN$V6klU_McqF|HxHEywG8LlNtP&f<@_o@NPkX zIf_McI^d*3?hjS70*RsD{znzEqk>-C`|AailM*}nhr%Ex2@4mTV9pSq{-cJp-U4%4 zo}eihM0}xz@sH}#s)>L98545Fn;;fp`&FEVgyr!+gsB7xri-#bBmBGH`C>;pw z*hpyo5ssEoZ)EK+gn5(p!B>%Q^YCCE z>^PxOB1YB*a!4O{3J=~qn77N&uKRMQuO3d!ZWNnKqKYO}C!7=6y@OW9fnS9#F#qU* zCjwRk-Q`Yr{vDl&@Y#P@`9kO~qzDpwl;*545E+biP9?OQkbFmcfk-{+P;Fy4KU=!? zw<}TLB5HGH#drgjfsHu9`j!ADxLKButS>e2I1=aMCX`^91WHcE2YOfg()c`E9tRcO&!l6uCu$oF zlwzjl)}W-mtQ~G#*9i{kE6^EQc{=VRPI%vR?|k;|4U!hgZ3`%(As800EBW4 zgOE7`AQVi!8TY-@zDxGdl)7?KGsJ8=k?=~zT-*xh8=D;Ag^%uey6i5;C<{E>?0<$y zXv;2hT}(3#jwzli;%fXLz^-lE^m^TE$7Lm+<%Hf=dsxNFR0n7YY0wE=K7JqaMDDu*4E{sPj6b#Ro zx0DvTfx`k5BNC1GY+fka$z5fyfWrA6f|z>-%bWXRY@;@Ek%FJuY3MmnFo3xNMHaf= zGN~6F3g29w@Bu+ENqLPH%$G|8HAAhA(YohY12-tfjgCBrO?tTnm3!H_usA8@xgd)9-6mJqVkTP+S3*-DEQ)Q`py)n3+Wo zHpJvPh;euZ69sc6A0q%u!%&d)2@j%;Jz$%U!Dp?B&4}f)_KZp9?n%*7z}{MQc_rQv zBbG*c+X?k;MS=ye%W*)2^*1BFvPga;Rt#`B!VKW6_rIun1^xcnB!J95xx^Y7@#Oi) z<_NlvYL=Of=tIHAT=qA0?}eptO1EtBJ71({9Oz=^*|FC}QWj{p3#<-wF^`p_9t&8~)>yyaLH~olHg-M&;Pt zA|^+!qI*I`#5$btgg#jtaZu65hV_vH23rRX`CxY6Y#i%3h}jx+E;4WytY$g{S*WYm zJ0O^psOtMU<7xf49Y0tkbfJqG6Z{T%{#gihyOk*QP#tY>A77rePaW$`y1~#Xh3?wa z{_{pEHp|kZmqWl2b}3H>!L0Eh158$$BE=z@`zMx??&`F5_nUH~tFd&T)REMFPSVcI z_g->QTnB9PtPIo$+eLa$3sv=cq(MC-^rNn}WxhL5W`ZCtJ$cf<=)O>zPVdv&pz#}- zzSL$vqjEL>%{S*pPuwUlW0;fcebXUZ)2s8>oI$3n#G{gaPOY|(E0&8o%Ee;3SCmF) zl|COR2y{Rac`ys07YyOW_X?)Jfs1$?KKwiwStB&EKss|r1szTSLG)bkpTtF$(v@PV_JkWBd=oQJ9o|iI*>PMER6}rbF{%#$jqbz!!#;$65WPL*9A6vHUX=Laaj~Wv6RMxn*ZLe z|GTB;f7WF|;2!^B@%*obz?A)`b!v}N!J!xtSnM%*;z1Gvz9>dj_)%C&dT5!k2|)N?wjQe`HK-gW>$4SYtnoMn)h`fn@P{~*{+=U zHKfLuC&0)yf$o>m$TtIb4p;Ae95<_Plri{Gwrwm%I{c$W_e=oe$LQv|&ZIk$6(EF} z+PEb;3P^o$Jziy|PH{zIm~I3(JdF>Oq^IW9WgKsc1P7c=NA`9Z45s}m^B2oXB`KqK z(nS79LH7;0)@@R`i`%XQA?iSf0w<=&LAc>D zfRu2&x6le$+2gB~yk5ORQECh=5e{Y=g(HbQ;mvr#>^jXdK)<`3JJC+(neB$#0#~ij zBnWt%3zS&XZ~_rrk(#Mn&04QKbl;f|JKZ1RIrQ-Kw6&-KtX_XkC;YHw}nVQOGr>@hA^xHWUnM;-c> zntAmGn!)1|RynvjiQ+~6CUUC0r6PH6c&2CLr>51hLhZ#oLO=U=qb>|*?YYQ1E%lnd zNQbUW7zS|XhGIk>1}$Y1oOxRQ}+l3K!Ovr;oRZ2*ETv5mC-eJ-l!Sa)G8go_I0}NK13) z)6*5C+c8RP01QPH55Uq|=l7XC`7Vh-@(GYaP(Qs-hV-16V`CqQqRv6(F453!bKir( zPCWQsBAD}j$7IT(+gO;i>t9U?81pKziB-R$fVqCwel+8}>SKBspeJjvlOM&X|*;0ThsNF!pd1Cme#qE?o(|fB9BCWwhI3+|!|jhAA?be%<%V zte?}KOMA3w;A5m=fA+R|U~(R%o10y_F$|XaCf~ab34mlGY-TA#Y{~7sA#HRDtkMaq z$YQ}LuZ5qbS;;41dkV`MEn$A$)pU8TZ2&ST%e)Wb)i_Y{q zfb4lypD&ZnVA%Y;85n4>Ryus{_Vi48o-o)AxGwGPg_`c3@(|wo49NSH44a(A2T^)R80?vI>dI}uL2W0Q|E3uK%_*qKo2mIk zQS4x{2a(G^=8uyq6~zikKHUgesHEA=6e5PbvcHy2@kxWX1e+cR!6y}^;t{G+k{=3G z4c^@VFl4PaQ9aL_fpm|}>b_0{OG(zIIlT%Y*MM&zP&yCfbjH;i07N)8S`M@YQh=rP zxM-M^XK-dBF2AJ3h+sBnu+KGUPQT$P#Lo>1kGb~X>O=Ea;O2Mf&YOf%&fBKyo}27Z zdJKEhL(2vo0x_xnz<*thJc~YZH=U|k!c`NI4wgH0dLLdYwe6uyf&=(pE|qDL8%W5l zfj5PN@8r(8Y=z-)D^6W{uK!5a;7f9#kLDSkL&SeYY#xVCOnRFY&*TjYwp2Y>A>ngGTIas2HQ%i zYdIv_wPi3t02Vt~?Y=%*w&yiw#ga)n4I!14*5JXy&gvQI;0GF%1po~cA@ucJ>$Sdy zibPVWx6Ri#>Zw>Ligh=2!6I#S+VZ*=uFbohLlfm(rhN?-cu5eApRH%~E%-ioi2D~h z=tJOEK_Q*KRum@9n)eu3ByIRyHe)g=vRJkr&oaU$rLtIdA0wW#?iRCDi=9OiH9-&$ zx;1#f>9f34n*03;dyOVngpjs!my)KEz07%w;EIt~@*GOn^O)EWHN;_LMw-zO-AvS*epyKV7LaeqUL+|9ysAzc`G+fT(TX= zVJk@+VbNnp-VADQ$Pdf-3Mb_)LQCmxW{*c^-2EA_PeHiPI{8Gi!hB8-=!v95cPG}> zyrX!@abqSc0RobYZhJcGp2BzFJWVhd>g;arFqg7RNx@aM;gV-8B*G;V!*)W}Ai;i; zViph8*vfLz2v5$^?&0Y@&)2|S@Q5$nY7zF_E5^Df=223Aw7d3o?Nf23mJ&)3)Y)$V zI!~&%n~bd-z&Ip)pmba25%x3>-BA@YXR;@PtrYx7g>d#-doSAVS8NqI11ipL5{5D6 zKl(0TvBH^G(X2;?8=Elq(IpuKaopij&ekHf&0hJUuvj=^a9MbmG;l9+Au@ZKmwK`@My__jX1haeDpJhog=r(?T_^5N~JLBu&LXV9%aE=311JB0i6F;m$b3H$~ z4;VhsHnl(3sCWjmqYt6OqDzL`vtLyfchZ!TKAbCCMek9neRRGp@_4)gs=R{hsH4}N z7MDxnTX|)Q0y51X^AZYhofY>7-W&HoK;r!wT6ns!E419_d#(LN84j`bkj5W?p}<`c zUs#Fpo}b%tiJ-gkSpT8M?pma|@0pIIMuj|B`6GyvBCJPxTR`Fyo%ly%AMK(ADudOL z-pa_Nm{8-3fna8el!f~#j%Sfi#4LUnB!zvcq|nt4QBJdL3RHkrn(-rJV#r^G4W+92~&kOIv3KBmU7;3IH)LNyW?WgMHO!DQ4 zk1XRYx{S>t$d6ZS4Z=ldf5jP1|1hX8|G9D#R@d?NyKu#KVYf5n%*=jKSc!j$*W13eywl z2wbssP7fi49w;7IqlcAuRnPK|e7eyWJ>A4!#?`bY3HH?&T`BzPK0&$dv-DOZP8;4D z4hny1Z8+*i9ql}}oj-fk3{f6f+V8$mZ8ZCRl%F;r0pKA=6ITR}7KBDfxauOb{G6&X zk_T++FAM`!=1$<89^eMFQTUEEsR)qW2Y9@ba%oJ|%o$AM#`oY)#lgxFp0noQdSJpo zFb{6PcMtyiKXCX>sQX`*Bz`9TXW|C_|C+j*>l=I57*q0IF@3|IY~GSpx{-NZ|HXd+ DoB#6Y literal 0 HcmV?d00001 diff --git a/docs/cugraph/source/wholegraph/index.rst b/docs/cugraph/source/wholegraph/index.rst new file mode 100644 index 00000000000..2a69544b4c9 --- /dev/null +++ b/docs/cugraph/source/wholegraph/index.rst @@ -0,0 +1,14 @@ +WholeGraph +========== +RAPIDS WholeGraph has following package: + +* pylibwholegraph: shared memory-based GPU-accelerated GNN training + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + basics/index + installation/index + diff --git a/docs/cugraph/source/wholegraph/installation/container.md b/docs/cugraph/source/wholegraph/installation/container.md new file mode 100644 index 00000000000..3a2c627c56a --- /dev/null +++ b/docs/cugraph/source/wholegraph/installation/container.md @@ -0,0 +1,29 @@ +# Build Container for WholeGraph +To run WholeGraph or build WholeGraph from source, set up the environment first. +We recommend using Docker images. +For example, to build the WholeGraph base image from the NGC pytorch 22.10 image, you can follow `Dockerfile`: +```dockerfile +FROM nvcr.io/nvidia/pytorch:22.10-py3 + +RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y lsb-core software-properties-common wget libspdlog-dev + +#RUN remove old cmake to update +RUN conda remove --force -y cmake +RUN rm -rf /usr/local/bin/cmake && rm -rf /usr/local/lib/cmake && rm -rf /usr/lib/cmake + +RUN apt-key adv --fetch-keys https://apt.kitware.com/keys/kitware-archive-latest.asc && \ + export LSB_CODENAME=$(lsb_release -cs) && \ + apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ ${LSB_CODENAME} main" && \ + apt update && apt install -y cmake + +# update py for pytest +RUN pip3 install -U py +RUN pip3 install Cython setuputils3 scikit-build nanobind pytest-forked pytest +``` + +To run GNN applications, you may also need cuGraphOps, DGL and/or PyG libraries to run the GNN layers. +You may refer to [DGL](https://www.dgl.ai/pages/start.html) or [PyG](https://pytorch-geometric.readthedocs.io/en/latest/notes/installation.html) +For example, to install DGL, you may need to add: +```dockerfile +RUN pip3 install dgl -f https://data.dgl.ai/wheels/cu118/repo.html +``` diff --git a/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md b/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md new file mode 100644 index 00000000000..5b2072b0523 --- /dev/null +++ b/docs/cugraph/source/wholegraph/installation/getting_wholegraph.md @@ -0,0 +1,48 @@ + +# Getting the WholeGraph Packages + +Start by reading the [RAPIDS Instalation guide](https://docs.rapids.ai/install) +and checkout the [RAPIDS install selector](https://rapids.ai/start.html) for a pick list of install options. + + +There are 4 ways to get WholeGraph packages: +1. [Quick start with Docker Repo](#docker) +2. [Conda Installation](#conda) +3. [Pip Installation](#pip) +4. [Build from Source](./source_build.md) + + +
+ +## Docker +The RAPIDS Docker containers (as of Release 23.10) contain all RAPIDS packages, including WholeGraph, as well as all required supporting packages. To download a container, please see the [Docker Repository](https://hub.docker.com/r/rapidsai/rapidsai/), choosing a tag based on the NVIDIA CUDA version you’re running. This provides a ready to run Docker container with example notebooks and data, showcasing how you can utilize all of the RAPIDS libraries. + +
+ + +## Conda +It is easy to install WholeGraph using conda. You can get a minimal conda installation with [Miniconda](https://conda.io/miniconda.html) or get the full installation with [Anaconda](https://www.anaconda.com/download). + +WholeGraph conda packages + * libwholegraph + * pylibwholegraph + +Replace the package name in the example below to the one you want to install. + + +Install and update WholeGraph using the conda command: + +```bash +conda install -c rapidsai -c conda-forge -c nvidia wholegraph cudatoolkit=11.8 +``` + +
+ +## PIP +wholegraph, and all of RAPIDS, is available via pip. + +``` +pip install wholegraph-cu11 --extra-index-url=https://pypi.nvidia.com +``` + +
diff --git a/docs/cugraph/source/wholegraph/installation/index.rst b/docs/cugraph/source/wholegraph/installation/index.rst new file mode 100644 index 00000000000..09f1cb44a24 --- /dev/null +++ b/docs/cugraph/source/wholegraph/installation/index.rst @@ -0,0 +1,9 @@ +Installation +============ + +.. toctree:: + :maxdepth: 2 + + getting_wholegraph + container + source_build diff --git a/docs/cugraph/source/wholegraph/installation/source_build.md b/docs/cugraph/source/wholegraph/installation/source_build.md new file mode 100644 index 00000000000..c468048c351 --- /dev/null +++ b/docs/cugraph/source/wholegraph/installation/source_build.md @@ -0,0 +1,187 @@ +# Building from Source + +The following instructions are for users wishing to build wholegraph from source code. These instructions are tested on supported distributions of Linux,CUDA, +and Python - See [RAPIDS Getting Started](https://rapids.ai/start.html) for a list of supported environments. +Other operating systems _might be_ compatible, but are not currently tested. + +The wholegraph package includes both a C/C++ CUDA portion and a python portion. Both libraries need to be installed in order for cuGraph to operate correctly. +The C/C++ CUDA library is `libwholegraph` and the python library is `pylibwholegraph`. + +## Prerequisites + +__Compiler__: +* `gcc` version 11.0+ +* `nvcc` version 11.0+ +* `cmake` version 3.26.4+ + +__CUDA__: +* CUDA 11.8+ +* NVIDIA driver 450.80.02+ +* Pascal architecture or better + +You can obtain CUDA from [https://developer.nvidia.com/cuda-downloads](https://developer.nvidia.com/cuda-downloads). + +__Other Packages__: +* ninja +* nccl +* cython +* setuputils3 +* scikit-learn +* scikit-build +* nanobind>=0.2.0 + +## Building wholegraph +To install wholegraph from source, ensure the dependencies are met. + +### Clone Repo and Configure Conda Environment +__GIT clone a version of the repository__ + + ```bash + # Set the location to wholegraph in an environment variable WHOLEGRAPH_HOME + export WHOLEGRAPH_HOME=$(pwd)/wholegraph + + # Download the wholegraph repo - if you have a forked version, use that path here instead + git clone https://github.com/rapidsai/wholegraph.git $WHOLEGRAPH_HOME + + cd $WHOLEGRAPH_HOME + ``` + +__Create the conda development environment__ + +```bash +# create the conda environment (assuming in base `wholegraph` directory) + +# for CUDA 11.x +conda env create --name wholegraph_dev --file conda/environments/all_cuda-118_arch-x86_64.yaml + +# activate the environment +conda activate wholegraph_dev + +# to deactivate an environment +conda deactivate +``` + + - The environment can be updated as development includes/changes the dependencies. To do so, run: + + +```bash + +# Where XXX is the CUDA version +conda env update --name wholegraph_dev --file conda/environments/all_cuda-XXX_arch-x86_64.yaml + +conda activate wholegraph_dev +``` + + +### Build and Install Using the `build.sh` Script +Using the `build.sh` script make compiling and installing wholegraph a +breeze. To build and install, simply do: + +```bash +$ cd $WHOLEGRAPH_HOME +$ ./build.sh clean +$ ./build.sh libwholegraph +$ ./build.sh pylibwholegraph +``` + +There are several other options available on the build script for advanced users. +`build.sh` options: +```bash +build.sh [ ...] [ ...] + where is: + clean - remove all existing build artifacts and configuration (start over). + uninstall - uninstall libwholegraph and pylibwholegraph from a prior build/install (see also -n) + libwholegraph - build the libwholegraph C++ library. + pylibwholegraph - build the pylibwholegraph Python package. + tests - build the C++ (OPG) tests. + benchmarks - build benchmarks. + docs - build the docs + and is: + -v - verbose build mode + -g - build for debug + -n - no install step + --allgpuarch - build for all supported GPU architectures + --cmake-args=\\\"\\\" - add arbitrary CMake arguments to any cmake call + --compile-cmd - only output compile commands (invoke CMake without build) + --clean - clean an individual target (note: to do a complete rebuild, use the clean target described above) + -h | --h[elp] - print this text + + default action (no args) is to build and install 'libwholegraph' then 'pylibwholegraph' targets + +examples: +$ ./build.sh clean # remove prior build artifacts (start over) +$ ./build.sh + +# make parallelism options can also be defined: Example build jobs using 4 threads (make -j4) +$ PARALLEL_LEVEL=4 ./build.sh libwholegraph + +Note that the libraries will be installed to the location set in `$PREFIX` if set (i.e. `export PREFIX=/install/path`), otherwise to `$CONDA_PREFIX`. +``` + + +## Building each section independently +### Build and Install the C++/CUDA `libwholegraph` Library +CMake depends on the `nvcc` executable being on your path or defined in `$CUDACXX`. + +This project uses cmake for building the C/C++ library. To configure cmake, run: + + ```bash + # Set the location to wholegraph in an environment variable WHOLEGRAPH_HOME + export WHOLEGRAPH_HOME=$(pwd)/wholegraph + + cd $WHOLEGRAPH_HOME + cd cpp # enter cpp directory + mkdir build # create build directory + cd build # enter the build directory + cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX + + # now build the code + make -j # "-j" starts multiple threads + make install # install the libraries + ``` +The default installation locations are `$CMAKE_INSTALL_PREFIX/lib` and `$CMAKE_INSTALL_PREFIX/include/wholegraph` respectively. + +### Building and installing the Python package + +Build and Install the Python packages to your Python path: + +```bash +cd $WHOLEGRAPH_HOME +cd python +cd pylibwholegraph +python setup.py build_ext --inplace +python setup.py install # install pylibwholegraph +``` + +## Run tests + +Run either the C++ or the Python tests with datasets + + - **Python tests with datasets** + + ```bash + cd $WHOLEGRAPH_HOME + cd python + pytest + ``` + + - **C++ stand alone tests** + + From the build directory : + + ```bash + # Run the tests + cd $WHOLEGRAPH_HOME + cd cpp/build + gtests/PARALLEL_UTILS_TESTS # this is an executable file + ``` + + +Note: This conda installation only applies to Linux and Python versions 3.8/3.10. + +## Creating documentation + +Python API documentation can be generated from _./docs/wholegraph directory_. Or through using "./build.sh docs" + +## Attribution +Portions adopted from https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md From 332676e78bbbd209b77caf81ab756fa6bff2a07b Mon Sep 17 00:00:00 2001 From: Huiyu Xie Date: Tue, 21 Nov 2023 11:40:36 -0800 Subject: [PATCH 4/6] [DOC]: Fix invalid links and add materials to notebooks (#4002) I'm new to CuGraph repository and would like to contribute further in the future. So I start with minor tasks with docs to familiarize myself with the contribution process in this repository. Here is what I have done in this PR: - Fixed some invalid links in the documentation. - Corrected and added formulas related to centrality. - Added more references to the centrality to ensure consistency. Also, the images located at https://github.com/rapidsai/cugraph/tree/main/python/cugraph cannot be displayed because it is symbolic link to the repository's README file. But it's not a major issue. Authors: - Huiyu Xie (https://github.com/huiyuxie) - Ralph Liu (https://github.com/nv-rliu) - Naim (https://github.com/naimnv) - Brad Rees (https://github.com/BradReesWork) Approvers: - Don Acosta (https://github.com/acostadon) - Brad Rees (https://github.com/BradReesWork) URL: https://github.com/rapidsai/cugraph/pull/4002 --- .../graph_support/algorithms/Centrality.md | 14 +++++++----- .../source/installation/getting_cugraph.md | 2 +- .../algorithms/centrality/Centrality.ipynb | 22 ++++++++++++++----- notebooks/algorithms/centrality/Degree.ipynb | 2 +- notebooks/algorithms/centrality/Katz.ipynb | 7 ------ readme_pages/CONTRIBUTING.md | 4 ++-- 6 files changed, 29 insertions(+), 22 deletions(-) diff --git a/docs/cugraph/source/graph_support/algorithms/Centrality.md b/docs/cugraph/source/graph_support/algorithms/Centrality.md index f82a1fe123b..8119e655236 100644 --- a/docs/cugraph/source/graph_support/algorithms/Centrality.md +++ b/docs/cugraph/source/graph_support/algorithms/Centrality.md @@ -15,13 +15,15 @@ But which vertices are most important? The answer depends on which measure/algor |Algorithm |Notebooks Containing |Description | | --------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -|[Degree Centrality](./degree_centrality.html)| [Centrality](./Centrality.ipynb), [Degree](./Degree.ipynb) |Measure based on counting direct connections for each vertex| -|[Betweenness Centrality](./betweenness_centrality.html)| [Centrality](./Centrality.ipynb), [Betweenness](./Betweenness.ipynb) |Number of shortest paths through the vertex| -|[Eigenvector Centrality](./eigenvector_centrality.html)|[Centrality](./Centrality.ipynb), [Eigenvector](./Eigenvector.ipynb)|Measure of connectivity to other important vertices (which also have high connectivity) often referred to as the influence measure of a vertex| -|[Katz Centrality](./katz_centrality.html)|[Centrality](./Centrality.ipynb), [Katz](./Katz.ipynb) |Similar to Eigenvector but has tweaks to measure more weakly connected graph | -|Pagerank|[Centrality](./Centrality.ipynb), [Pagerank](../../link_analysis/Pagerank.ipynb) |Classified as both a link analysis and centrality measure by quantifying incoming links from central vertices. | +|[Degree Centrality](./degree_centrality.md)| [Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Degree](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Degree.ipynb) |Measure based on counting direct connections for each vertex| +|[Betweenness Centrality](./betweenness_centrality.md)| [Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Betweenness](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Betweenness.ipynb) |Number of shortest paths through the vertex| +|[Eigenvector Centrality](./eigenvector_centrality.md)|[Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Eigenvector](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Eigenvector.ipynb)|Measure of connectivity to other important vertices (which also have high connectivity) often referred to as the influence measure of a vertex| +|[Katz Centrality](./katz_centrality.md)|[Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Katz](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Katz.ipynb) |Similar to Eigenvector but has tweaks to measure more weakly connected graph | +|Pagerank|[Centrality](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/centrality/Centrality.ipynb), [Pagerank](https://github.com/rapidsai/cugraph/blob/main/notebooks/algorithms/link_analysis/Pagerank.ipynb) |Classified as both a link analysis and centrality measure by quantifying incoming links from central vertices. | + +[System Requirements](https://github.com/rapidsai/cugraph/blob/main/notebooks/README.md#requirements) + -[System Requirements](../../README.html#requirements) | Author Credit | Date | Update | cuGraph Version | Test Hardware | | --------------|------------|------------------|-----------------|----------------| diff --git a/docs/cugraph/source/installation/getting_cugraph.md b/docs/cugraph/source/installation/getting_cugraph.md index 509508c5283..625f2c64c27 100644 --- a/docs/cugraph/source/installation/getting_cugraph.md +++ b/docs/cugraph/source/installation/getting_cugraph.md @@ -9,7 +9,7 @@ There are 4 ways to get cuGraph packages: 1. [Quick start with Docker Repo](#docker) 2. [Conda Installation](#conda) 3. [Pip Installation](#pip) -4. [Build from Source](#SOURCE) +4. [Build from Source](./source_build.md)
diff --git a/notebooks/algorithms/centrality/Centrality.ipynb b/notebooks/algorithms/centrality/Centrality.ipynb index d19dd646b15..e470febb975 100644 --- a/notebooks/algorithms/centrality/Centrality.ipynb +++ b/notebooks/algorithms/centrality/Centrality.ipynb @@ -36,22 +36,34 @@ "__Degree Centrality__
\n", "Degree centrality is based on the notion that whoever has the most connections must be important. \n", "\n", - "$C_d(v) = \\frac{{\\text{{degree of vertex }} v}}{{\\text{{total number of vertices}} - 1}}$\n", - "\n", + "$C_{degree}(v) = \\frac{{\\text{degree of vertex} \\ v}}{{\\text{total number of vertices} - 1}}$\n", "\n", + "See:\n", + "* [Degree (graph theory) on Wikipedia](https://en.wikipedia.org/wiki/Degree_(graph_theory)) for more details on the algorithm.\n", + "* [Learn more about Degree Centrality](https://www.sci.unich.it/~francesc/teaching/network/degree.html)\n", "\n", - "___Closeness centrality – coming soon___
\n", + "__Closeness Centrality__
\n", "Closeness is a measure of the shortest path to every other node in the graph. A node that is close to every other node, can reach over other node in the fewest number of hops, means that it has greater influence on the network versus a node that is not close.\n", "\n", + "$C_{closeness}(v)=\\frac{n-1}{\\sum_{t} d(v,t)}$\n", + "\n", + "See:\n", + "* [Closeness Centrality on Wikipedia](https://en.wikipedia.org/wiki/Closeness_centrality) for more details on the algorithm.\n", + "* [Learn more about Closeness Centrality](https://www.sci.unich.it/~francesc/teaching/network/closeness.html)\n", + "\n", "__Betweenness Centrality__
\n", "Betweenness is a measure of the number of shortest paths that cross through a node, or over an edge. A node with high betweenness means that it had a greater influence on the flow of information. \n", "\n", "Betweenness centrality of a node 𝑣 is the sum of the fraction of all-pairs shortest paths that pass through 𝑣\n", "\n", - "$C_{betweenness}=\\sum_{s \\neq v \\neq t} \\frac{\\sigma_{st}(v)}{\\sigma_{st}}$\n", + "$C_{betweenness}(v)=\\sum_{s \\neq v \\neq t} \\frac{\\sigma_{st}(v)}{\\sigma_{st}}$\n", "\n", "To speedup runtime of betweenness centrailty, the metric can be computed on a limited number of nodes (randomly selected) and then used to estimate the other scores. For this example, the graphs are relatively small (under 5,000 nodes) so betweenness on every node will be computed.\n", "\n", + "See:\n", + "* [Betweenness Centrality on Wikipedia](https://en.wikipedia.org/wiki/Betweenness_centrality) for more details on the algorithm.\n", + "* [Learn more about Betweenness Centrality](https://www.sci.unich.it/~francesc/teaching/network/betweeness.html)\n", + "\n", "__Katz Centrality__
\n", "Katz is a variant of degree centrality and of eigenvector centrality. \n", "Katz centrality is a measure of the relative importance of a node within the graph based on measuring the influence across the total number of walks between vertex pairs.\n", @@ -60,7 +72,7 @@ "\n", "\n", "See:\n", - "* [Katz on Wikipedia](https://en.wikipedia.org/wiki/Katz_centrality) for more details on the algorithm.\n", + "* [Katz Centrality on Wikipedia](https://en.wikipedia.org/wiki/Katz_centrality) for more details on the algorithm.\n", "* [Learn more about Katz Centrality](https://www.sci.unich.it/~francesc/teaching/network/katz.html)\n", "\n", "__Eigenvector Centrality__
\n", diff --git a/notebooks/algorithms/centrality/Degree.ipynb b/notebooks/algorithms/centrality/Degree.ipynb index e7535420b65..5a5213a904f 100644 --- a/notebooks/algorithms/centrality/Degree.ipynb +++ b/notebooks/algorithms/centrality/Degree.ipynb @@ -27,7 +27,7 @@ "\n", "See [Degree Centrality on Wikipedia](https://en.wikipedia.org/wiki/Degree_centrality) for more details on the algorithm.\n", "\n", - "$C_d(v) = \\frac{{\\text{{degree of vertex }} v}}{{\\text{{number of vertices in graph}} - 1}}$" + "$C_d(v) = \\frac{{\\text{degree of vertex } \\ v}}{{\\text{number of vertices in graph} - 1}}$" ] }, { diff --git a/notebooks/algorithms/centrality/Katz.ipynb b/notebooks/algorithms/centrality/Katz.ipynb index c94a14bb14a..08ee42df788 100755 --- a/notebooks/algorithms/centrality/Katz.ipynb +++ b/notebooks/algorithms/centrality/Katz.ipynb @@ -333,13 +333,6 @@ "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n", "___" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/readme_pages/CONTRIBUTING.md b/readme_pages/CONTRIBUTING.md index 4b736b25155..ffe1ef1831b 100644 --- a/readme_pages/CONTRIBUTING.md +++ b/readme_pages/CONTRIBUTING.md @@ -68,7 +68,7 @@ If you need more context on a particular issue, please ask. # So you want to contribute code **TL;DR General Development Process** -1. Read the documentation on [building from source](./SOURCEBUILD.md) to learn how to setup, and validate, the development environment +1. Read the documentation on [building from source](../docs/cugraph/source/installation/source_build.md) to learn how to setup, and validate, the development environment 2. Read the RAPIDS [Code of Conduct](https://docs.rapids.ai/resources/conduct/) 3. Find or submit an issue to work on (include a comment that you are working issue) 4. Fork the cuGraph [repo](#fork) and Code (make sure to add unit tests)! @@ -99,7 +99,7 @@ The RAPIDS cuGraph repo cannot directly be modified. Contributions must come in ```git clone https://github.com//cugraph.git``` -Read the section on [building cuGraph from source](./SOURCEBUILD.md) to validate that the environment is correct. +Read the section on [building cuGraph from source](../docs/cugraph/source/installation/source_build.md) to validate that the environment is correct. **Pro Tip** add an upstream remote repository so that you can keep your forked repo in sync ```git remote add upstream https://github.com/rapidsai/cugraph.git``` From f2df81d27fbf72c569c850504fd055196899b40c Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 21 Nov 2023 20:44:54 -0600 Subject: [PATCH 5/6] nx-cugraph: add `eigenvector_centrality`, `katz_centrality`, `hits`, `pagerank` (#3968) Add `eigenvector_centrality`, `katz_centrality`, and `hits`. I may add pagerank next. Authors: - Erik Welch (https://github.com/eriknw) - Ralph Liu (https://github.com/nv-rliu) - Naim (https://github.com/naimnv) - Rick Ratzel (https://github.com/rlratzel) Approvers: - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/3968 --- python/nx-cugraph/_nx_cugraph/__init__.py | 21 ++++ python/nx-cugraph/lint.yaml | 2 +- .../nx_cugraph/algorithms/__init__.py | 10 +- .../algorithms/centrality/__init__.py | 2 + .../algorithms/centrality/eigenvector.py | 64 ++++++++++ .../nx_cugraph/algorithms/centrality/katz.py | 100 ++++++++++++++++ .../algorithms/community/louvain.py | 34 ++++-- .../algorithms/link_analysis/__init__.py | 14 +++ .../algorithms/link_analysis/hits_alg.py | 81 +++++++++++++ .../algorithms/link_analysis/pagerank_alg.py | 112 ++++++++++++++++++ python/nx-cugraph/nx_cugraph/classes/graph.py | 50 ++++---- python/nx-cugraph/nx_cugraph/interface.py | 21 +++- .../nx_cugraph/tests/test_ktruss.py | 2 +- python/nx-cugraph/nx_cugraph/utils/misc.py | 34 +++++- 14 files changed, 509 insertions(+), 38 deletions(-) create mode 100644 python/nx-cugraph/nx_cugraph/algorithms/centrality/eigenvector.py create mode 100644 python/nx-cugraph/nx_cugraph/algorithms/centrality/katz.py create mode 100644 python/nx-cugraph/nx_cugraph/algorithms/link_analysis/__init__.py create mode 100644 python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py create mode 100644 python/nx-cugraph/nx_cugraph/algorithms/link_analysis/pagerank_alg.py diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index 910db1bc379..ef5f8f3fc23 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -47,12 +47,14 @@ "diamond_graph", "dodecahedral_graph", "edge_betweenness_centrality", + "eigenvector_centrality", "empty_graph", "florentine_families_graph", "from_pandas_edgelist", "from_scipy_sparse_array", "frucht_graph", "heawood_graph", + "hits", "house_graph", "house_x_graph", "icosahedral_graph", @@ -62,6 +64,7 @@ "isolates", "k_truss", "karate_club_graph", + "katz_centrality", "krackhardt_kite_graph", "ladder_graph", "les_miserables_graph", @@ -75,6 +78,7 @@ "number_of_selfloops", "octahedral_graph", "out_degree_centrality", + "pagerank", "pappus_graph", "path_graph", "petersen_graph", @@ -96,19 +100,36 @@ # BEGIN: extra_docstrings "betweenness_centrality": "`weight` parameter is not yet supported.", "edge_betweenness_centrality": "`weight` parameter is not yet supported.", + "eigenvector_centrality": "`nstart` parameter is not used, but it is checked for validity.", "from_pandas_edgelist": "cudf.DataFrame inputs also supported.", "k_truss": ( "Currently raises `NotImplementedError` for graphs with more than one connected\n" "component when k >= 3. We expect to fix this soon." ), + "katz_centrality": "`nstart` isn't used (but is checked), and `normalized=False` is not supported.", "louvain_communities": "`seed` parameter is currently ignored.", + "pagerank": "`dangling` parameter is not supported, but it is checked for validity.", # END: extra_docstrings }, "extra_parameters": { # BEGIN: extra_parameters + "eigenvector_centrality": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", + }, + "hits": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", + 'weight : string or None, optional (default="weight")': "The edge attribute to use as the edge weight.", + }, + "katz_centrality": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", + }, "louvain_communities": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", "max_level : int, optional": "Upper limit of the number of macro-iterations (max: 500).", }, + "pagerank": { + "dtype : dtype or None, optional": "The data type (np.float32, np.float64, or None) to use for the edge weights in the algorithm. If None, then dtype is determined by the edge values.", + }, # END: extra_parameters }, } diff --git a/python/nx-cugraph/lint.yaml b/python/nx-cugraph/lint.yaml index 01a806e6162..a94aa9f0448 100644 --- a/python/nx-cugraph/lint.yaml +++ b/python/nx-cugraph/lint.yaml @@ -53,7 +53,7 @@ repos: rev: v0.1.3 hooks: - id: ruff - args: [--fix-only, --show-fixes] + args: [--fix-only, --show-fixes] # --unsafe-fixes] - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: diff --git a/python/nx-cugraph/nx_cugraph/algorithms/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/__init__.py index 32cd6f31a47..63841b15bd5 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/__init__.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/__init__.py @@ -10,10 +10,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from . import bipartite, centrality, community, components, shortest_paths +from . import ( + bipartite, + centrality, + community, + components, + shortest_paths, + link_analysis, +) from .bipartite import complete_bipartite_graph from .centrality import * from .components import * from .core import * from .isolate import * from .shortest_paths import * +from .link_analysis import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/__init__.py index af91f227843..496dc6aff81 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/centrality/__init__.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/__init__.py @@ -12,3 +12,5 @@ # limitations under the License. from .betweenness import * from .degree_alg import * +from .eigenvector import * +from .katz import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/eigenvector.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/eigenvector.py new file mode 100644 index 00000000000..c0f02a6258e --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/eigenvector.py @@ -0,0 +1,64 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import ( + _dtype_param, + _get_float_dtype, + networkx_algorithm, + not_implemented_for, +) + +__all__ = ["eigenvector_centrality"] + + +@not_implemented_for("multigraph") +@networkx_algorithm(extra_params=_dtype_param) +def eigenvector_centrality( + G, max_iter=100, tol=1.0e-6, nstart=None, weight=None, *, dtype=None +): + """`nstart` parameter is not used, but it is checked for validity.""" + G = _to_graph(G, weight, np.float32) + if len(G) == 0: + raise nx.NetworkXPointlessConcept( + "cannot compute centrality for the null graph" + ) + if dtype is not None: + dtype = _get_float_dtype(dtype) + elif weight in G.edge_values: + dtype = _get_float_dtype(G.edge_values[weight].dtype) + else: + dtype = np.float32 + if nstart is not None: + # Check if given nstart is valid even though we don't use it + nstart = G._dict_to_nodearray(nstart, dtype=dtype) + if (nstart == 0).all(): + raise nx.NetworkXError("initial vector cannot have all zero values") + if nstart.sum() == 0: + raise ZeroDivisionError + # nstart /= total # Uncomment (and assign total) when nstart is used below + try: + node_ids, values = plc.eigenvector_centrality( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(weight, 1, dtype, store_transposed=True), + epsilon=tol, + max_iterations=max_iter, + do_expensive_check=False, + ) + except RuntimeError as exc: + # Errors from PLC are sometimes a little scary and not very helpful + raise nx.PowerIterationFailedConvergence(max_iter) from exc + return G._nodearrays_to_dict(node_ids, values) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/centrality/katz.py b/python/nx-cugraph/nx_cugraph/algorithms/centrality/katz.py new file mode 100644 index 00000000000..b61b811b8fa --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/centrality/katz.py @@ -0,0 +1,100 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import ( + _dtype_param, + _get_float_dtype, + networkx_algorithm, + not_implemented_for, +) + +__all__ = ["katz_centrality"] + + +@not_implemented_for("multigraph") +@networkx_algorithm(extra_params=_dtype_param) +def katz_centrality( + G, + alpha=0.1, + beta=1.0, + max_iter=1000, + tol=1.0e-6, + nstart=None, + normalized=True, + weight=None, + *, + dtype=None, +): + """`nstart` isn't used (but is checked), and `normalized=False` is not supported.""" + if not normalized: + # Redundant with the `_can_run` check below when being dispatched by NetworkX, + # but we raise here in case this funcion is called directly. + raise NotImplementedError("normalized=False is not supported.") + G = _to_graph(G, weight, np.float32) + if (N := len(G)) == 0: + return {} + if dtype is not None: + dtype = _get_float_dtype(dtype) + elif weight in G.edge_values: + dtype = _get_float_dtype(G.edge_values[weight].dtype) + else: + dtype = np.float32 + if nstart is not None: + # Check if given nstart is valid even though we don't use it + nstart = G._dict_to_nodearray(nstart, 0, dtype) + b = bs = None + try: + b = float(beta) + except (TypeError, ValueError) as exc: + try: + bs = G._dict_to_nodearray(beta, dtype=dtype) + b = 1.0 # float value must be given to PLC (and will be ignored) + except (KeyError, ValueError): + raise nx.NetworkXError( + "beta dictionary must have a value for every node" + ) from exc + try: + node_ids, values = plc.katz_centrality( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(weight, 1, dtype, store_transposed=True), + betas=bs, + alpha=alpha, + beta=b, + epsilon=N * tol, + max_iterations=max_iter, + do_expensive_check=False, + ) + except RuntimeError as exc: + # Errors from PLC are sometimes a little scary and not very helpful + raise nx.PowerIterationFailedConvergence(max_iter) from exc + return G._nodearrays_to_dict(node_ids, values) + + +@katz_centrality._can_run +def _( + G, + alpha=0.1, + beta=1.0, + max_iter=1000, + tol=1.0e-6, + nstart=None, + normalized=True, + weight=None, + *, + dtype=None, +): + return normalized diff --git a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py index 45a3429d2ee..936d837dacd 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py @@ -16,6 +16,7 @@ from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( + _dtype_param, _groupby, _seed_to_int, networkx_algorithm, @@ -32,11 +33,19 @@ extra_params={ "max_level : int, optional": ( "Upper limit of the number of macro-iterations (max: 500)." - ) + ), + **_dtype_param, } ) def louvain_communities( - G, weight="weight", resolution=1, threshold=0.0000001, seed=None, *, max_level=None + G, + weight="weight", + resolution=1, + threshold=0.0000001, + seed=None, + *, + max_level=None, + dtype=None, ): """`seed` parameter is currently ignored.""" # NetworkX allows both directed and undirected, but cugraph only allows undirected. @@ -54,20 +63,20 @@ def louvain_communities( stacklevel=2, ) max_level = 500 - vertices, clusters, modularity = plc.louvain( + node_ids, clusters, modularity = plc.louvain( resource_handle=plc.ResourceHandle(), - graph=G._get_plc_graph(), + graph=G._get_plc_graph(weight, 1, dtype), max_level=max_level, # TODO: add this parameter to NetworkX threshold=threshold, resolution=resolution, do_expensive_check=False, ) - groups = _groupby(clusters, vertices, groups_are_canonical=True) - rv = [set(G._nodearray_to_list(node_ids)) for node_ids in groups.values()] - # TODO: PLC doesn't handle isolated vertices yet, so this is a temporary fix + groups = _groupby(clusters, node_ids, groups_are_canonical=True) + rv = [set(G._nodearray_to_list(ids)) for ids in groups.values()] + # TODO: PLC doesn't handle isolated node_ids yet, so this is a temporary fix isolates = _isolates(G) if isolates.size > 0: - isolates = isolates[isolates > vertices.max()] + isolates = isolates[isolates > node_ids.max()] if isolates.size > 0: rv.extend({node} for node in G._nodearray_to_list(isolates)) return rv @@ -75,7 +84,14 @@ def louvain_communities( @louvain_communities._can_run def _( - G, weight="weight", resolution=1, threshold=0.0000001, seed=None, *, max_level=None + G, + weight="weight", + resolution=1, + threshold=0.0000001, + seed=None, + *, + max_level=None, + dtype=None, ): # NetworkX allows both directed and undirected, but cugraph only allows undirected. return not G.is_directed() diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/__init__.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/__init__.py new file mode 100644 index 00000000000..a68d6940d02 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .hits_alg import * +from .pagerank_alg import * diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py new file mode 100644 index 00000000000..1c8a47c24b1 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py @@ -0,0 +1,81 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import ( + _dtype_param, + _get_float_dtype, + index_dtype, + networkx_algorithm, +) + +__all__ = ["hits"] + + +@networkx_algorithm( + extra_params={ + 'weight : string or None, optional (default="weight")': ( + "The edge attribute to use as the edge weight." + ), + **_dtype_param, + } +) +def hits( + G, + max_iter=100, + tol=1.0e-8, + nstart=None, + normalized=True, + *, + weight="weight", + dtype=None, +): + G = _to_graph(G, weight, np.float32) + if (N := len(G)) == 0: + return {}, {} + if dtype is not None: + dtype = _get_float_dtype(dtype) + elif weight in G.edge_values: + dtype = _get_float_dtype(G.edge_values[weight].dtype) + else: + dtype = np.float32 + if nstart is not None: + nstart = G._dict_to_nodearray(nstart, 0, dtype) + if max_iter <= 0: + if nx.__version__[:3] <= "3.2": + raise ValueError("`maxiter` must be a positive integer.") + raise nx.PowerIterationFailedConvergence(max_iter) + try: + node_ids, hubs, authorities = plc.hits( + resource_handle=plc.ResourceHandle(), + graph=G._get_plc_graph(weight, 1, dtype, store_transposed=True), + tol=tol, + initial_hubs_guess_vertices=None + if nstart is None + else cp.arange(N, dtype=index_dtype), + initial_hubs_guess_values=nstart, + max_iter=max_iter, + normalized=normalized, + do_expensive_check=False, + ) + except RuntimeError as exc: + # Errors from PLC are sometimes a little scary and not very helpful + raise nx.PowerIterationFailedConvergence(max_iter) from exc + return ( + G._nodearrays_to_dict(node_ids, hubs), + G._nodearrays_to_dict(node_ids, authorities), + ) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/pagerank_alg.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/pagerank_alg.py new file mode 100644 index 00000000000..63f6e89c33a --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/pagerank_alg.py @@ -0,0 +1,112 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import cupy as cp +import networkx as nx +import numpy as np +import pylibcugraph as plc + +from nx_cugraph.convert import _to_graph +from nx_cugraph.utils import ( + _dtype_param, + _get_float_dtype, + index_dtype, + networkx_algorithm, +) + +__all__ = ["pagerank"] + + +@networkx_algorithm(extra_params=_dtype_param) +def pagerank( + G, + alpha=0.85, + personalization=None, + max_iter=100, + tol=1.0e-6, + nstart=None, + weight="weight", + dangling=None, + *, + dtype=None, +): + """`dangling` parameter is not supported, but it is checked for validity.""" + G = _to_graph(G, weight, 1, np.float32) + if (N := len(G)) == 0: + return {} + if dtype is not None: + dtype = _get_float_dtype(dtype) + elif weight in G.edge_values: + dtype = _get_float_dtype(G.edge_values[weight].dtype) + else: + dtype = np.float32 + if nstart is not None: + nstart = G._dict_to_nodearray(nstart, 0, dtype=dtype) + if (total := nstart.sum()) == 0: + raise ZeroDivisionError + nstart /= total + if personalization is not None: + personalization = G._dict_to_nodearray(personalization, 0, dtype=dtype) + if (total := personalization.sum()) == 0: + raise ZeroDivisionError + personalization /= total + if dangling is not None: + # Check if given dangling is valid even though we don't use it + dangling = G._dict_to_nodearray(dangling, 0) # Check validity + if dangling.sum() == 0: + raise ZeroDivisionError + if (G._out_degrees_array() == 0).any(): + raise NotImplementedError("custom dangling weights is not supported") + if max_iter <= 0: + raise nx.PowerIterationFailedConvergence(max_iter) + kwargs = { + "resource_handle": plc.ResourceHandle(), + "graph": G._get_plc_graph(weight, 1, dtype, store_transposed=True), + "precomputed_vertex_out_weight_vertices": None, + "precomputed_vertex_out_weight_sums": None, + "initial_guess_vertices": None + if nstart is None + else cp.arange(N, dtype=index_dtype), + "initial_guess_values": nstart, + "alpha": alpha, + "epsilon": N * tol, + "max_iterations": max_iter, + "do_expensive_check": False, + "fail_on_nonconvergence": False, + } + if personalization is None: + node_ids, values, is_converged = plc.pagerank(**kwargs) + else: + node_ids, values, is_converged = plc.personalized_pagerank( + personalization_vertices=cp.arange(N, dtype=index_dtype), # Why? + personalization_values=personalization, + **kwargs, + ) + if not is_converged: + raise nx.PowerIterationFailedConvergence(max_iter) + return G._nodearrays_to_dict(node_ids, values) + + +@pagerank._can_run +def pagerank( + G, + alpha=0.85, + personalization=None, + max_iter=100, + tol=1.0e-6, + nstart=None, + weight="weight", + dangling=None, + *, + dtype=None, +): + return dangling is None diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index fea318e036e..e32f93d8bfe 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -79,7 +79,7 @@ class Graph: np.dtype(np.uint32): np.dtype(np.float64), np.dtype(np.uint64): np.dtype(np.float64), # raise if x > 2**53 # other - np.dtype(np.bool_): np.dtype(np.float16), + np.dtype(np.bool_): np.dtype(np.float32), np.dtype(np.float16): np.dtype(np.float32), } _plc_allowed_edge_types: ClassVar[set[np.dtype]] = { @@ -562,12 +562,13 @@ def _get_plc_graph( switch_indices: bool = False, edge_array: cp.ndarray[EdgeValue] | None = None, ): - if edge_array is not None: + if edge_array is not None or edge_attr is None: pass - elif edge_attr is None: - edge_array = None elif edge_attr not in self.edge_values: - raise KeyError("Graph has no edge attribute {edge_attr!r}") + if edge_default is None: + raise KeyError("Graph has no edge attribute {edge_attr!r}") + # If we were given a default edge value, then it's probably okay to + # use None for the edge_array if we don't have this edge attribute. elif edge_attr not in self.edge_masks: edge_array = self.edge_values[edge_attr] elif not self.edge_masks[edge_attr].all(): @@ -685,6 +686,9 @@ def _degrees_array(self): degrees += cp.bincount(self.dst_indices, minlength=self._N) return degrees + _in_degrees_array = _degrees_array + _out_degrees_array = _degrees_array + # Data conversions def _nodeiter_to_iter(self, node_ids: Iterable[IndexValue]) -> Iterable[NodeKey]: """Convert an iterable of node IDs to an iterable of node keys.""" @@ -748,20 +752,22 @@ def _dict_to_nodearrays( values = cp.fromiter(d.values(), dtype) return node_ids, values - # def _dict_to_nodearray( - # self, - # d: dict[NodeKey, NodeValue] | cp.ndarray[NodeValue], - # default: NodeValue | None = None, - # dtype: Dtype | None = None, - # ) -> cp.ndarray[NodeValue]: - # if isinstance(d, cp.ndarray): - # if d.shape[0] != len(self): - # raise ValueError - # return d - # if default is None: - # val_iter = map(d.__getitem__, self) - # else: - # val_iter = (d.get(node, default) for node in self) - # if dtype is None: - # return cp.array(list(val_iter)) - # return cp.fromiter(val_iter, dtype) + def _dict_to_nodearray( + self, + d: dict[NodeKey, NodeValue] | cp.ndarray[NodeValue], + default: NodeValue | None = None, + dtype: Dtype | None = None, + ) -> cp.ndarray[NodeValue]: + if isinstance(d, cp.ndarray): + if d.shape[0] != len(self): + raise ValueError + if dtype is not None and d.dtype != dtype: + return d.astype(dtype) + return d + if default is None: + val_iter = map(d.__getitem__, self) + else: + val_iter = (d.get(node, default) for node in self) + if dtype is None: + return cp.array(list(val_iter)) + return cp.fromiter(val_iter, dtype) diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index 8903fdc541e..be6b3596030 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -72,12 +72,29 @@ def key(testpath): from packaging.version import parse nxver = parse(nx.__version__) - if nxver.major == 3 and nxver.minor in {0, 1}: + + if nxver.major == 3 and nxver.minor <= 2: + # Networkx versions prior to 3.2.1 have tests written to expect + # sp.sparse.linalg.ArpackNoConvergence exceptions raised on no + # convergence in HITS. Newer versions since the merge of + # https://github.com/networkx/networkx/pull/7084 expect + # nx.PowerIterationFailedConvergence, which is what nx_cugraph.hits + # raises, so we mark them as xfail for previous versions of NX. + xfail.update( + { + key( + "test_hits.py:TestHITS.test_hits_not_convergent" + ): "nx_cugraph.hits raises updated exceptions not caught in " + "these tests", + } + ) + + if nxver.major == 3 and nxver.minor <= 1: # MAINT: networkx 3.0, 3.1 # NetworkX 3.2 added the ability to "fallback to nx" if backend algorithms # raise NotImplementedError or `can_run` returns False. The tests below # exercise behavior we have not implemented yet, so we mark them as xfail - # for previous versions of NetworkX. + # for previous versions of NX. xfail.update( { key( diff --git a/python/nx-cugraph/nx_cugraph/tests/test_ktruss.py b/python/nx-cugraph/nx_cugraph/tests/test_ktruss.py index a3e4cee3124..92fe2360688 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_ktruss.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_ktruss.py @@ -22,7 +22,7 @@ def test_k_truss(get_graph): Gnx = get_graph() Gcg = nxcg.from_networkx(Gnx, preserve_all_attrs=True) - for k in range(10): + for k in range(6): Hnx = nx.k_truss(Gnx, k) Hcg = nxcg.k_truss(Gcg, k) assert nx.utils.graphs_equal(Hnx, nxcg.to_networkx(Hcg)) diff --git a/python/nx-cugraph/nx_cugraph/utils/misc.py b/python/nx-cugraph/nx_cugraph/utils/misc.py index 26f023bdcec..e303375918d 100644 --- a/python/nx-cugraph/nx_cugraph/utils/misc.py +++ b/python/nx-cugraph/nx_cugraph/utils/misc.py @@ -16,11 +16,14 @@ import operator as op import sys from random import Random -from typing import SupportsIndex +from typing import TYPE_CHECKING, SupportsIndex import cupy as cp import numpy as np +if TYPE_CHECKING: + from ..typing import Dtype + try: from itertools import pairwise # Python >=3.10 except ImportError: @@ -33,11 +36,26 @@ def pairwise(it): prev = cur -__all__ = ["index_dtype", "_groupby", "_seed_to_int", "_get_int_dtype"] +__all__ = [ + "index_dtype", + "_groupby", + "_seed_to_int", + "_get_int_dtype", + "_get_float_dtype", + "_dtype_param", +] # This may switch to np.uint32 at some point index_dtype = np.int32 +# To add to `extra_params=` of `networkx_algorithm` +_dtype_param = { + "dtype : dtype or None, optional": ( + "The data type (np.float32, np.float64, or None) to use for the edge weights " + "in the algorithm. If None, then dtype is determined by the edge values." + ), +} + def _groupby( groups: cp.ndarray, values: cp.ndarray, groups_are_canonical: bool = False @@ -144,3 +162,15 @@ def _get_int_dtype( return np.dtype(dtype_string) except TypeError as exc: raise ValueError("Value is too large to store as integer: {val}") from exc + + +def _get_float_dtype(dtype: Dtype): + """Promote dtype to float32 or float64 as appropriate.""" + if dtype is None: + return np.dtype(np.float32) + rv = np.promote_types(dtype, np.float32) + if np.float32 != rv != np.float64: + raise TypeError( + f"Dtype {dtype} cannot be safely promoted to float32 or float64" + ) + return rv From 4b1618b7c6e474420e8ad07cefebb45064476475 Mon Sep 17 00:00:00 2001 From: Rick Ratzel <3039903+rlratzel@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:12:50 -0600 Subject: [PATCH 6/6] Adds missing connected_components algo to table. (#4019) --- python/nx-cugraph/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/python/nx-cugraph/README.md b/python/nx-cugraph/README.md index 273a6112d77..f6a9aac1088 100644 --- a/python/nx-cugraph/README.md +++ b/python/nx-cugraph/README.md @@ -101,6 +101,7 @@ supported in nx-cugraph. | balanced_cut_clustering | ? | | betweenness_centrality | 23.10 | | bfs | ? | +| connected_components | 23.12 | | core_number | ? | | degree_centrality | 23.12 | | ecg | ? |