Forward-merge branch-23.12 to branch-24.02

.github/workflows/build.yaml

@@ -22,7 +22,7 @@ on:
         default: nightly
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
 jobs:

ci/build_wheel.sh

@@ -40,8 +40,11 @@ for dep in rmm cudf raft-dask pylibcugraph pylibraft ucx-py; do
     sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
 done
 
-# dask-cuda doesn't get a suffix, but it does get an alpha spec.
-sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file}
+# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec.
+for dep in dask-cuda rapids-dask-dependency; do
+    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
+done
+
 
 if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
     sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}

ci/release/update-version.sh

@@ -88,6 +88,7 @@ DEPENDENCIES=(
   raft-dask
   rmm
   ucx-py
+  rapids-dask-dependency
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do

ci/test_python.sh

@@ -197,27 +197,26 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
     conda activate test_cugraph_pyg
     set -u
 
-    # Install pytorch
+    # Will automatically install built dependencies of cuGraph-PyG
     rapids-mamba-retry install \
-      --force-reinstall \
-      --channel pyg \
+      --channel "${CPP_CHANNEL}" \
+      --channel "${PYTHON_CHANNEL}" \
       --channel pytorch \
       --channel nvidia \
-      'pyg=2.3' \
-      'pytorch=2.0.0' \
-      'pytorch-cuda=11.8'
+      --channel pyg \
+      --channel rapidsai-nightly \
+      "cugraph-pyg" \
+      "pytorch>=2.0,<2.1" \
+      "pytorch-cuda=11.8"
 
     # Install pyg dependencies (which requires pip)
-    pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.0.0+cu118.html
-
-    rapids-mamba-retry install \
-      --channel "${CPP_CHANNEL}" \
-      --channel "${PYTHON_CHANNEL}" \
-      libcugraph \
-      pylibcugraph \
-      pylibcugraphops \
-      cugraph \
-      cugraph-pyg
+    pip install \
+        pyg_lib \
+        torch_scatter \
+        torch_sparse \
+        torch_cluster \
+        torch_spline_conv \
+      -f https://data.pyg.org/whl/torch-2.0.0+cu118.html
 
     rapids-print-env
 

ci/test_wheel_cugraph.sh

@@ -8,7 +8,4 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep
 python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl
 
-# Always install latest dask for testing
-python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main
-
 ./ci/test_wheel.sh cugraph python/cugraph

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -20,11 +20,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core>=2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
-- dask>=2023.7.1
-- distributed>=2023.7.1
 - doxygen
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
@@ -62,6 +59,7 @@ dependencies:
 - pytest-xdist
 - python-louvain
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - requests
 - rmm==23.12.*

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -20,11 +20,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core>=2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
-- dask>=2023.7.1
-- distributed>=2023.7.1
 - doxygen
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
@@ -61,6 +58,7 @@ dependencies:
 - pytest-xdist
 - python-louvain
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - requests
 - rmm==23.12.*

conda/recipes/cugraph-dgl/meta.yaml

@@ -26,7 +26,7 @@ requirements:
     - dgl >=1.1.0.cu*
     - numba >=0.57
     - numpy >=1.21
-    - pylibcugraphops ={{ version }}
+    - pylibcugraphops ={{ minor_version }}
     - python
     - pytorch
 

conda/recipes/cugraph-pyg/meta.yaml

@@ -26,15 +26,15 @@ requirements:
     - python
     - scikit-build >=0.13.1
   run:
-    - distributed >=2023.9.2
+    - rapids-dask-dependency ={{ minor_version }}
     - numba >=0.57
     - numpy >=1.21
     - python
     - pytorch >=2.0
     - cupy >=12.0.0
     - cugraph ={{ version }}
-    - pylibcugraphops ={{ version }}
-    - pyg >=2.3,<2.4
+    - pylibcugraphops ={{ minor_version }}
+    - pyg >=2.3,<2.5
 
 tests:
   imports:

conda/recipes/cugraph-service/meta.yaml

@@ -59,10 +59,10 @@ outputs:
         - cupy >=12.0.0
         - dask-cuda ={{ minor_version }}
         - dask-cudf ={{ minor_version }}
-        - distributed >=2023.9.2
         - numba >=0.57
         - numpy >=1.21
         - python
+        - rapids-dask-dependency ={{ minor_version }}
         - thriftpy2 >=0.4.15
         - ucx-py {{ ucx_py_version }}
 

conda/recipes/cugraph/meta.yaml

@@ -76,15 +76,13 @@ requirements:
     - cupy >=12.0.0
     - dask-cuda ={{ minor_version }}
     - dask-cudf ={{ minor_version }}
-    - dask >=2023.9.2
-    - dask-core >=2023.9.2
-    - distributed >=2023.9.2
     - fsspec>=0.6.0
     - libcugraph ={{ version }}
     - pylibcugraph ={{ version }}
     - pylibraft ={{ minor_version }}
     - python
     - raft-dask ={{ minor_version }}
+    - rapids-dask-dependency ={{ minor_version }}
     - requests
     - ucx-proc=*=gpu
     - ucx-py {{ ucx_py_version }}

cpp/CMakeLists.txt

@@ -153,6 +153,11 @@ rapids_cpm_init()
 # lags behind.
 ###
 
+# Need to make sure rmm is found before cuco so that rmm patches the libcudacxx
+# directory to be found by cuco.
+include(${rapids-cmake-dir}/cpm/rmm.cmake)
+rapids_cpm_rmm(BUILD_EXPORT_SET cugraph-exports
+               INSTALL_EXPORT_SET  cugraph-exports)
 # Putting this before raft to override RAFT from pulling them in.
 include(cmake/thirdparty/get_libcudacxx.cmake)
 include(${rapids-cmake-dir}/cpm/cuco.cmake)

cpp/include/cugraph/algorithms.hpp

@@ -464,51 +464,6 @@ k_truss_subgraph(raft::handle_t const& handle,
                  size_t number_of_vertices,
                  int k);
 
-// FIXME: Internally distances is of int (signed 32-bit) data type, but current
-// template uses data from VT, ET, WT from the legacy::GraphCSR View even if weights
-// are not considered
-/**
- * @Synopsis   Performs a breadth first search traversal of a graph starting from a vertex.
- *
- * @throws     cugraph::logic_error with a custom message when an error occurs.
- *
- * @tparam VT                        Type of vertex identifiers. Supported value : int (signed,
- * 32-bit)
- * @tparam ET                        Type of edge identifiers.  Supported value : int (signed,
- * 32-bit)
- * @tparam WT                        Type of edge weights. Supported values : int (signed, 32-bit)
- *
- * @param[in] handle                 Library handle (RAFT). If a communicator is set in the handle,
- the multi GPU version will be selected.
- * @param[in] graph                  cuGraph graph descriptor, should contain the connectivity
- * information as a CSR
- *
- * @param[out] distances             If set to a valid pointer, this is populated by distance of
- * every vertex in the graph from the starting vertex
- *
- * @param[out] predecessors          If set to a valid pointer, this is populated by bfs traversal
- * predecessor of every vertex
- *
- * @param[out] sp_counters           If set to a valid pointer, this is populated by bfs traversal
- * shortest_path counter of every vertex
- *
- * @param[in] start_vertex           The starting vertex for breadth first search traversal
- *
- * @param[in] directed               Treat the input graph as directed
- *
- * @param[in] mg_batch               If set to true use SG BFS path when comms are initialized.
- *
- */
-template <typename VT, typename ET, typename WT>
-void bfs(raft::handle_t const& handle,
-         legacy::GraphCSRView<VT, ET, WT> const& graph,
-         VT* distances,
-         VT* predecessors,
-         double* sp_counters,
-         const VT start_vertex,
-         bool directed = true,
-         bool mg_batch = false);
-
 /**
  * @brief      Compute Hungarian algorithm on a weighted bipartite graph
  *

cpp/include/cugraph/utilities/device_comm.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -806,9 +806,6 @@ device_sendrecv(raft::comms::comms_t const& comm,
   size_t constexpr tuple_size =
     thrust::tuple_size<typename thrust::iterator_traits<InputIterator>::value_type>::value;
 
-  // FIXME: NCCL 2.7 supports only one ncclSend and one ncclRecv for a source rank and destination
-  // rank inside ncclGroupStart/ncclGroupEnd, so we cannot place this inside
-  // ncclGroupStart/ncclGroupEnd, this restriction will be lifted in NCCL 2.8
   detail::device_sendrecv_tuple_iterator_element_impl<InputIterator,
                                                       OutputIterator,
                                                       size_t{0},
@@ -866,9 +863,6 @@ device_multicast_sendrecv(raft::comms::comms_t const& comm,
   size_t constexpr tuple_size =
     thrust::tuple_size<typename thrust::iterator_traits<InputIterator>::value_type>::value;
 
-  // FIXME: NCCL 2.7 supports only one ncclSend and one ncclRecv for a source rank and destination
-  // rank inside ncclGroupStart/ncclGroupEnd, so we cannot place this inside
-  // ncclGroupStart/ncclGroupEnd, this restriction will be lifted in NCCL 2.8
   detail::device_multicast_sendrecv_tuple_iterator_element_impl<InputIterator,
                                                                 OutputIterator,
                                                                 size_t{0},