Merge branch 'branch-23.12' into docs_c_api

build.sh

@@ -33,6 +33,7 @@ VALIDARGS="
    cugraph-dgl
    nx-cugraph
    cpp-mgtests
+   cpp-mtmgtests
    docs
    all
    -v
@@ -61,6 +62,7 @@ HELP="$0 [<target> ...] [<flag> ...]
    cugraph-dgl                - build the cugraph-dgl extensions for DGL
    nx-cugraph                 - build the nx-cugraph Python package
    cpp-mgtests                - build libcugraph and libcugraph_etl MG tests. Builds MPI communicator, adding MPI as a dependency.
+   cpp-mtmgtests              - build libcugraph MTMG tests. Adds UCX as a dependency (temporary).
    docs                       - build the docs
    all                        - build everything
  and <flag> is:
@@ -107,6 +109,7 @@ BUILD_TYPE=Release
 INSTALL_TARGET="--target install"
 BUILD_CPP_TESTS=ON
 BUILD_CPP_MG_TESTS=OFF
+BUILD_CPP_MTMG_TESTS=OFF
 BUILD_ALL_GPU_ARCH=0
 BUILD_WITH_CUGRAPHOPS=ON
 CMAKE_GENERATOR_OPTION="-G Ninja"
@@ -174,6 +177,9 @@ fi
 if hasArg --without_cugraphops; then
     BUILD_WITH_CUGRAPHOPS=OFF
 fi
+if hasArg cpp-mtmgtests; then
+    BUILD_CPP_MTMG_TESTS=ON
+fi
 if hasArg cpp-mgtests || hasArg all; then
     BUILD_CPP_MG_TESTS=ON
 fi
@@ -266,6 +272,7 @@ if buildDefault || hasArg libcugraph || hasArg all; then
               -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
               -DBUILD_TESTS=${BUILD_CPP_TESTS} \
               -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \
+	      -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \
 	      -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \
               ${CMAKE_GENERATOR_OPTION} \
               ${CMAKE_VERBOSE_OPTION}
@@ -296,6 +303,7 @@ if buildDefault || hasArg libcugraph_etl || hasArg all; then
               -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
               -DBUILD_TESTS=${BUILD_CPP_TESTS} \
               -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \
+              -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \
               -DCMAKE_PREFIX_PATH=${LIBCUGRAPH_BUILD_DIR} \
               ${CMAKE_GENERATOR_OPTION} \
               ${CMAKE_VERBOSE_OPTION} \

ci/build_wheel.sh

@@ -40,8 +40,11 @@ for dep in rmm cudf raft-dask pylibcugraph pylibraft ucx-py; do
     sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
 done
 
-# dask-cuda doesn't get a suffix, but it does get an alpha spec.
-sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file}
+# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec.
+for dep in dask-cuda rapids-dask-dependency; do
+    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
+done
+
 
 if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
     sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}

ci/release/update-version.sh

@@ -88,10 +88,12 @@ DEPENDENCIES=(
   raft-dask
   rmm
   ucx-py
+  rapids-dask-dependency
 )
 for DEP in "${DEPENDENCIES[@]}"; do
-  for FILE in dependencies.yaml conda/environments/*.yaml; do
+  for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do
     sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE}
+    sed_runner "/-.* ${DEP}-cu[0-9][0-9]==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE}
     sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*/g" ${FILE}
   done
   for FILE in python/**/pyproject.toml python/**/**/pyproject.toml; do
@@ -108,6 +110,11 @@ sed_runner "/^ucx_py_version:$/ {n;s/.*/  - \"${NEXT_UCX_PY_VERSION}.*\"/}" cond
 sed_runner "/^ucx_py_version:$/ {n;s/.*/  - \"${NEXT_UCX_PY_VERSION}.*\"/}" conda/recipes/cugraph-service/conda_build_config.yaml
 sed_runner "/^ucx_py_version:$/ {n;s/.*/  - \"${NEXT_UCX_PY_VERSION}.*\"/}" conda/recipes/pylibcugraph/conda_build_config.yaml
 
+# nx-cugraph NetworkX entry-point meta-data
+sed_runner "s@branch-[0-9][0-9].[0-9][0-9]@branch-${NEXT_SHORT_TAG}@g" python/nx-cugraph/_nx_cugraph/__init__.py
+# FIXME: can this use the standard VERSION file and update mechanism?
+sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/_nx_cugraph/__init__.py
+
 # CI files
 for FILE in .github/workflows/*.yaml; do
   sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"

ci/test_wheel_cugraph.sh

@@ -8,7 +8,4 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep
 python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl
 
-# Always install latest dask for testing
-python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main
-
 ./ci/test_wheel.sh cugraph python/cugraph

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -20,11 +20,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core>=2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
-- dask>=2023.9.2
-- distributed>=2023.9.2
 - doxygen
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
@@ -62,6 +59,7 @@ dependencies:
 - pytest-xdist
 - python-louvain
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - requests
 - rmm==23.12.*

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -20,11 +20,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core>=2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
-- dask>=2023.9.2
-- distributed>=2023.9.2
 - doxygen
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
@@ -61,6 +58,7 @@ dependencies:
 - pytest-xdist
 - python-louvain
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - requests
 - rmm==23.12.*

conda/recipes/cugraph-pyg/meta.yaml

@@ -26,7 +26,7 @@ requirements:
     - python
     - scikit-build >=0.13.1
   run:
-    - distributed >=2023.9.2
+    - rapids-dask-dependency ={{ version }}
     - numba >=0.57
     - numpy >=1.21
     - python

conda/recipes/cugraph-service/meta.yaml

@@ -59,10 +59,10 @@ outputs:
         - cupy >=12.0.0
         - dask-cuda ={{ minor_version }}
         - dask-cudf ={{ minor_version }}
-        - distributed >=2023.9.2
         - numba >=0.57
         - numpy >=1.21
         - python
+        - rapids-dask-dependency ={{ minor_version }}
         - thriftpy2 >=0.4.15
         - ucx-py {{ ucx_py_version }}
 

conda/recipes/cugraph/meta.yaml

@@ -76,15 +76,13 @@ requirements:
     - cupy >=12.0.0
     - dask-cuda ={{ minor_version }}
     - dask-cudf ={{ minor_version }}
-    - dask >=2023.9.2
-    - dask-core >=2023.9.2
-    - distributed >=2023.9.2
     - fsspec>=0.6.0
     - libcugraph ={{ version }}
     - pylibcugraph ={{ version }}
     - pylibraft ={{ minor_version }}
     - python
     - raft-dask ={{ minor_version }}
+    - rapids-dask-dependency ={{ minor_version }}
     - requests
     - ucx-proc=*=gpu
     - ucx-py {{ ucx_py_version }}

cpp/CMakeLists.txt

@@ -166,7 +166,10 @@ endif()
 
 include(cmake/thirdparty/get_nccl.cmake)
 include(cmake/thirdparty/get_cuhornet.cmake)
-include(cmake/thirdparty/get_ucp.cmake)
+
+if (BUILD_CUGRAPH_MTMG_TESTS)
+  include(cmake/thirdparty/get_ucp.cmake)
+endif()
 
 if(BUILD_TESTS)
   include(cmake/thirdparty/get_gtest.cmake)

cpp/tests/CMakeLists.txt

@@ -415,13 +415,6 @@ ConfigureTest(K_HOP_NBRS_TEST traversal/k_hop_nbrs_test.cpp)
 # - install tests ---------------------------------------------------------------------------------
 rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcugraph)
 
-###################################################################################################
-# - MTMG tests -------------------------------------------------------------------------
-ConfigureTest(MTMG_TEST mtmg/threaded_test.cu)
-target_link_libraries(MTMG_TEST
-                      PRIVATE
-                      UCP::UCP
-                    )
 
 ###################################################################################################
 # - MG tests --------------------------------------------------------------------------------------
@@ -681,15 +674,6 @@ if(BUILD_CUGRAPH_MG_TESTS)
 
     rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_mg DESTINATION bin/gtests/libcugraph_mg)
 
-    ###############################################################################################
-    # - Multi-node MTMG tests ---------------------------------------------------------------------
-    ConfigureTest(MTMG_MULTINODE_TEST mtmg/multi_node_threaded_test.cu utilities/mg_utilities.cpp)
-    target_link_libraries(MTMG_MULTINODE_TEST
-                          PRIVATE
-                          cugraphmgtestutil
-                          UCP::UCP
-                         )
-
 endif()
 
 ###################################################################################################
@@ -749,4 +733,25 @@ ConfigureCTest(CAPI_EGONET_TEST c_api/egonet_test.c)
 ConfigureCTest(CAPI_TWO_HOP_NEIGHBORS_TEST c_api/two_hop_neighbors_test.c)
 ConfigureCTest(CAPI_LEGACY_K_TRUSS_TEST c_api/legacy_k_truss_test.c)
 
+if (BUILD_CUGRAPH_MTMG_TESTS)
+    ###################################################################################################
+    # - MTMG tests -------------------------------------------------------------------------
+    ConfigureTest(MTMG_TEST mtmg/threaded_test.cu)
+    target_link_libraries(MTMG_TEST
+                        PRIVATE
+                        UCP::UCP
+                        )
+
+    if(BUILD_CUGRAPH_MG_TESTS)
+        ###############################################################################################
+        # - Multi-node MTMG tests ---------------------------------------------------------------------
+        ConfigureTest(MTMG_MULTINODE_TEST mtmg/multi_node_threaded_test.cu utilities/mg_utilities.cpp)
+        target_link_libraries(MTMG_MULTINODE_TEST
+                              PRIVATE
+                              cugraphmgtestutil
+                              UCP::UCP
+                             )
+    endif(BUILD_CUGRAPH_MG_TESTS)
+endif(BUILD_CUGRAPH_MTMG_TESTS)
+
 rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_c DESTINATION bin/gtests/libcugraph_c)

dependencies.yaml

@@ -377,15 +377,13 @@ dependencies:
     common:
       - output_types: [conda, pyproject]
         packages:
-          - &dask dask>=2023.9.2
-          - &distributed distributed>=2023.9.2
+          - &dask rapids-dask-dependency==23.12.*
           - &dask_cuda dask-cuda==23.12.*
           - &numba numba>=0.57
           - &ucx_py ucx-py==0.35.*
       - output_types: conda
         packages:
           - aiohttp
-          - &dask-core_conda dask-core>=2023.9.2
           - fsspec>=0.6.0
           - libcudf==23.12.*
           - requests
@@ -431,14 +429,10 @@ dependencies:
         packages:
           - *dask
           - *dask_cuda
-          - *distributed
           - *numba
           - *numpy
           - *thrift
           - *ucx_py
-      - output_types: conda
-        packages:
-          - *dask-core_conda
       - output_types: pyproject
         packages:
           - *cugraph

python/cugraph-service/server/pyproject.toml

@@ -25,10 +25,9 @@ dependencies = [
     "cupy-cuda11x>=12.0.0",
     "dask-cuda==23.12.*",
     "dask-cudf==23.12.*",
-    "dask>=2023.9.2",
-    "distributed>=2023.9.2",
     "numba>=0.57",
     "numpy>=1.21",
+    "rapids-dask-dependency==23.12.*",
     "rmm==23.12.*",
     "thriftpy2",
     "ucx-py==0.35.*",

python/cugraph/cugraph/structure/property_graph.py

@@ -800,15 +800,9 @@ def add_vertex_data(
             tmp_df.index = tmp_df.index.rename(self.vertex_col_name)
 
         # FIXME: handle case of a type_name column already being in tmp_df
-        if self.__series_type is cudf.Series:
-            # cudf does not yet support initialization with a scalar
-            tmp_df[TCN] = cudf.Series(
-                cudf.Series([type_name], dtype=cat_dtype).repeat(len(tmp_df)),
-                index=tmp_df.index,
-            )
-        else:
-            # pandas is oddly slow if dtype is passed to the constructor here
-            tmp_df[TCN] = pd.Series(type_name, index=tmp_df.index).astype(cat_dtype)
+        tmp_df[TCN] = self.__series_type(type_name, index=tmp_df.index).astype(
+            cat_dtype
+        )
 
         if property_columns:
             # all columns
@@ -1207,15 +1201,9 @@ def add_edge_data(
         tmp_df[self.src_col_name] = tmp_df[vertex_col_names[0]]
         tmp_df[self.dst_col_name] = tmp_df[vertex_col_names[1]]
 
-        if self.__series_type is cudf.Series:
-            # cudf does not yet support initialization with a scalar
-            tmp_df[TCN] = cudf.Series(
-                cudf.Series([type_name], dtype=cat_dtype).repeat(len(tmp_df)),
-                index=tmp_df.index,
-            )
-        else:
-            # pandas is oddly slow if dtype is passed to the constructor here
-            tmp_df[TCN] = pd.Series(type_name, index=tmp_df.index).astype(cat_dtype)
+        tmp_df[TCN] = self.__series_type(type_name, index=tmp_df.index).astype(
+            cat_dtype
+        )
 
         # Add unique edge IDs to the new rows. This is just a count for each
         # row starting from the last edge ID value, with initial edge ID 0.

python/cugraph/pyproject.toml

@@ -33,12 +33,11 @@ dependencies = [
     "cupy-cuda11x>=12.0.0",
     "dask-cuda==23.12.*",
     "dask-cudf==23.12.*",
-    "dask>=2023.9.2",
-    "distributed>=2023.9.2",
     "fsspec[http]>=0.6.0",
     "numba>=0.57",
     "pylibcugraph==23.12.*",
     "raft-dask==23.12.*",
+    "rapids-dask-dependency==23.12.*",
     "rmm==23.12.*",
     "ucx-py==0.35.*",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

python/nx-cugraph/_nx_cugraph/__init__.py

@@ -136,6 +136,7 @@ def get_info():
     return d
 
 
+# FIXME: can this use the standard VERSION file and update mechanism?
 __version__ = "23.12.00"
 
 if __name__ == "__main__":