From ff943332512e6b5bd3bdbad4893ce46987fd9b1b Mon Sep 17 00:00:00 2001 From: Rick Ratzel <3039903+rlratzel@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:29:03 -0600 Subject: [PATCH 1/3] Adds update-version.sh support for proper versioning to cu version suffix pacakges and nx-cugraph meta-data files. (#3994) Adds `update-version.sh` support for: * nx-cugraph meta-data files * `cugraph-pyg`, `cugraph-dgl` env yaml files * `*-cu11`, `*-cu12` wheel dependencies in dependencies.yaml Authors: - Rick Ratzel (https://github.com/rlratzel) Approvers: - Alex Barghi (https://github.com/alexbarghi-nv) - Brad Rees (https://github.com/BradReesWork) - Ray Douglass (https://github.com/raydouglass) --- ci/release/update-version.sh | 8 +++++++- python/nx-cugraph/_nx_cugraph/__init__.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d3dbed6ae46..0f2d4a3b914 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -90,8 +90,9 @@ DEPENDENCIES=( ucx-py ) for DEP in "${DEPENDENCIES[@]}"; do - for FILE in dependencies.yaml conda/environments/*.yaml; do + for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE} + sed_runner "/-.* ${DEP}-cu[0-9][0-9]==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE} sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*/g" ${FILE} done for FILE in python/**/pyproject.toml python/**/**/pyproject.toml; do @@ -108,6 +109,11 @@ sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" cond sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" conda/recipes/cugraph-service/conda_build_config.yaml sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" conda/recipes/pylibcugraph/conda_build_config.yaml +# nx-cugraph NetworkX entry-point meta-data +sed_runner "s@branch-[0-9][0-9].[0-9][0-9]@branch-${NEXT_SHORT_TAG}@g" python/nx-cugraph/_nx_cugraph/__init__.py +# FIXME: can this use the standard VERSION file and update mechanism? +sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/_nx_cugraph/__init__.py + # CI files for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index 8ef976aabf1..26638d1e735 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -136,6 +136,7 @@ def get_info(): return d +# FIXME: can this use the standard VERSION file and update mechanism? __version__ = "23.12.00" if __name__ == "__main__": From 96a7a6eaf12e4550256c5835a0f0486c78db539e Mon Sep 17 00:00:00 2001 From: Naim <110031745+naimnv@users.noreply.github.com> Date: Tue, 14 Nov 2023 18:25:39 +0100 Subject: [PATCH 2/3] Move MTMG_TEST to MG tests block (#3993) * Move MTMG_TEST to MG tests block * Remove extra newline * remove reference to get_ucp.cmake * make building MTMG a separate build flag * Updates PropertyGraph for latest cudf changes. * style fixes. --------- Co-authored-by: Naim Co-authored-by: Charles Hastings Co-authored-by: Rick Ratzel --- build.sh | 8 ++++ cpp/CMakeLists.txt | 5 ++- cpp/tests/CMakeLists.txt | 37 +++++++++++-------- .../cugraph/structure/property_graph.py | 24 +++--------- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/build.sh b/build.sh index 99082fa96fb..1723e750978 100755 --- a/build.sh +++ b/build.sh @@ -31,6 +31,7 @@ VALIDARGS=" cugraph-dgl nx-cugraph cpp-mgtests + cpp-mtmgtests docs all -v @@ -59,6 +60,7 @@ HELP="$0 [ ...] [ ...] cugraph-dgl - build the cugraph-dgl extensions for DGL nx-cugraph - build the nx-cugraph Python package cpp-mgtests - build libcugraph and libcugraph_etl MG tests. Builds MPI communicator, adding MPI as a dependency. + cpp-mtmgtests - build libcugraph MTMG tests. Adds UCX as a dependency (temporary). docs - build the docs all - build everything and is: @@ -105,6 +107,7 @@ BUILD_TYPE=Release INSTALL_TARGET="--target install" BUILD_CPP_TESTS=ON BUILD_CPP_MG_TESTS=OFF +BUILD_CPP_MTMG_TESTS=OFF BUILD_ALL_GPU_ARCH=0 BUILD_WITH_CUGRAPHOPS=ON CMAKE_GENERATOR_OPTION="-G Ninja" @@ -172,6 +175,9 @@ fi if hasArg --without_cugraphops; then BUILD_WITH_CUGRAPHOPS=OFF fi +if hasArg cpp-mtmgtests; then + BUILD_CPP_MTMG_TESTS=ON +fi if hasArg cpp-mgtests || hasArg all; then BUILD_CPP_MG_TESTS=ON fi @@ -264,6 +270,7 @@ if buildDefault || hasArg libcugraph || hasArg all; then -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DBUILD_TESTS=${BUILD_CPP_TESTS} \ -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \ + -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \ -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \ ${CMAKE_GENERATOR_OPTION} \ ${CMAKE_VERBOSE_OPTION} @@ -294,6 +301,7 @@ if buildDefault || hasArg libcugraph_etl || hasArg all; then -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DBUILD_TESTS=${BUILD_CPP_TESTS} \ -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \ + -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \ -DCMAKE_PREFIX_PATH=${LIBCUGRAPH_BUILD_DIR} \ ${CMAKE_GENERATOR_OPTION} \ ${CMAKE_VERBOSE_OPTION} \ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 41870cbc92b..360165e688d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -166,7 +166,10 @@ endif() include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_cuhornet.cmake) -include(cmake/thirdparty/get_ucp.cmake) + +if (BUILD_CUGRAPH_MTMG_TESTS) + include(cmake/thirdparty/get_ucp.cmake) +endif() if(BUILD_TESTS) include(cmake/thirdparty/get_gtest.cmake) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 2f69cf9cb0d..6530a25d178 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -415,13 +415,6 @@ ConfigureTest(K_HOP_NBRS_TEST traversal/k_hop_nbrs_test.cpp) # - install tests --------------------------------------------------------------------------------- rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcugraph) -################################################################################################### -# - MTMG tests ------------------------------------------------------------------------- -ConfigureTest(MTMG_TEST mtmg/threaded_test.cu) -target_link_libraries(MTMG_TEST - PRIVATE - UCP::UCP - ) ################################################################################################### # - MG tests -------------------------------------------------------------------------------------- @@ -681,15 +674,6 @@ if(BUILD_CUGRAPH_MG_TESTS) rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_mg DESTINATION bin/gtests/libcugraph_mg) - ############################################################################################### - # - Multi-node MTMG tests --------------------------------------------------------------------- - ConfigureTest(MTMG_MULTINODE_TEST mtmg/multi_node_threaded_test.cu utilities/mg_utilities.cpp) - target_link_libraries(MTMG_MULTINODE_TEST - PRIVATE - cugraphmgtestutil - UCP::UCP - ) - endif() ################################################################################################### @@ -749,4 +733,25 @@ ConfigureCTest(CAPI_EGONET_TEST c_api/egonet_test.c) ConfigureCTest(CAPI_TWO_HOP_NEIGHBORS_TEST c_api/two_hop_neighbors_test.c) ConfigureCTest(CAPI_LEGACY_K_TRUSS_TEST c_api/legacy_k_truss_test.c) +if (BUILD_CUGRAPH_MTMG_TESTS) + ################################################################################################### + # - MTMG tests ------------------------------------------------------------------------- + ConfigureTest(MTMG_TEST mtmg/threaded_test.cu) + target_link_libraries(MTMG_TEST + PRIVATE + UCP::UCP + ) + + if(BUILD_CUGRAPH_MG_TESTS) + ############################################################################################### + # - Multi-node MTMG tests --------------------------------------------------------------------- + ConfigureTest(MTMG_MULTINODE_TEST mtmg/multi_node_threaded_test.cu utilities/mg_utilities.cpp) + target_link_libraries(MTMG_MULTINODE_TEST + PRIVATE + cugraphmgtestutil + UCP::UCP + ) + endif(BUILD_CUGRAPH_MG_TESTS) +endif(BUILD_CUGRAPH_MTMG_TESTS) + rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_c DESTINATION bin/gtests/libcugraph_c) diff --git a/python/cugraph/cugraph/structure/property_graph.py b/python/cugraph/cugraph/structure/property_graph.py index 36ce5baa212..513798f35f9 100644 --- a/python/cugraph/cugraph/structure/property_graph.py +++ b/python/cugraph/cugraph/structure/property_graph.py @@ -800,15 +800,9 @@ def add_vertex_data( tmp_df.index = tmp_df.index.rename(self.vertex_col_name) # FIXME: handle case of a type_name column already being in tmp_df - if self.__series_type is cudf.Series: - # cudf does not yet support initialization with a scalar - tmp_df[TCN] = cudf.Series( - cudf.Series([type_name], dtype=cat_dtype).repeat(len(tmp_df)), - index=tmp_df.index, - ) - else: - # pandas is oddly slow if dtype is passed to the constructor here - tmp_df[TCN] = pd.Series(type_name, index=tmp_df.index).astype(cat_dtype) + tmp_df[TCN] = self.__series_type(type_name, index=tmp_df.index).astype( + cat_dtype + ) if property_columns: # all columns @@ -1207,15 +1201,9 @@ def add_edge_data( tmp_df[self.src_col_name] = tmp_df[vertex_col_names[0]] tmp_df[self.dst_col_name] = tmp_df[vertex_col_names[1]] - if self.__series_type is cudf.Series: - # cudf does not yet support initialization with a scalar - tmp_df[TCN] = cudf.Series( - cudf.Series([type_name], dtype=cat_dtype).repeat(len(tmp_df)), - index=tmp_df.index, - ) - else: - # pandas is oddly slow if dtype is passed to the constructor here - tmp_df[TCN] = pd.Series(type_name, index=tmp_df.index).astype(cat_dtype) + tmp_df[TCN] = self.__series_type(type_name, index=tmp_df.index).astype( + cat_dtype + ) # Add unique edge IDs to the new rows. This is just a count for each # row starting from the last edge ID value, with initial edge ID 0. From b90e1a53e4c2ae6fdde73258c2d366dde596a154 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 14 Nov 2023 14:35:20 -0600 Subject: [PATCH 3/3] Use new `rapids-dask-dependency` metapackage for managing `dask` versions (#3991) Currently dask versions are pinned as part of every release cycle and then unpinned for the next development cycle across all of RAPIDS. This introduces a great deal of churn. To centralize the dependency, we have created a metapackage to manage the required dask version and this PR introduces that metapackage as a dependency of cugraph. xref: https://github.com/rapidsai/cudf/pull/14364 Authors: - Chuck Hastings (https://github.com/ChuckHastings) - Rick Ratzel (https://github.com/rlratzel) - Vyas Ramasubramani (https://github.com/vyasr) - GALI PREM SAGAR (https://github.com/galipremsagar) - Naim (naim@uib.no) Approvers: - Jake Awe (https://github.com/AyodeAwe) --- ci/build_wheel.sh | 7 +++++-- ci/release/update-version.sh | 1 + ci/test_wheel_cugraph.sh | 3 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 4 +--- conda/environments/all_cuda-120_arch-x86_64.yaml | 4 +--- conda/recipes/cugraph-pyg/meta.yaml | 2 +- conda/recipes/cugraph-service/meta.yaml | 2 +- conda/recipes/cugraph/meta.yaml | 4 +--- dependencies.yaml | 8 +------- python/cugraph-service/server/pyproject.toml | 3 +-- python/cugraph/pyproject.toml | 3 +-- 11 files changed, 14 insertions(+), 27 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c888c908056..163520ea1da 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -40,8 +40,11 @@ for dep in rmm cudf raft-dask pylibcugraph pylibraft ucx-py; do sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} done -# dask-cuda doesn't get a suffix, but it does get an alpha spec. -sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} +# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec. +for dep in dask-cuda rapids-dask-dependency; do + sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} +done + if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 0f2d4a3b914..69eb085e7ed 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -88,6 +88,7 @@ DEPENDENCIES=( raft-dask rmm ucx-py + rapids-dask-dependency ) for DEP in "${DEPENDENCIES[@]}"; do for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh index f9e2aa6d8da..d351ea21624 100755 --- a/ci/test_wheel_cugraph.sh +++ b/ci/test_wheel_cugraph.sh @@ -8,7 +8,4 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl -# Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main - ./ci/test_wheel.sh cugraph python/cugraph diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2f3a9c988cf..aa38defcd7c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,11 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* - dask-cudf==23.12.* -- dask>=2023.7.1 -- distributed>=2023.7.1 - doxygen - fsspec>=0.6.0 - gcc_linux-64=11.* @@ -62,6 +59,7 @@ dependencies: - pytest-xdist - python-louvain - raft-dask==23.12.* +- rapids-dask-dependency==23.12.* - recommonmark - requests - rmm==23.12.* diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 31ff503e682..a9f793b15f5 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -20,11 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* - dask-cudf==23.12.* -- dask>=2023.7.1 -- distributed>=2023.7.1 - doxygen - fsspec>=0.6.0 - gcc_linux-64=11.* @@ -61,6 +58,7 @@ dependencies: - pytest-xdist - python-louvain - raft-dask==23.12.* +- rapids-dask-dependency==23.12.* - recommonmark - requests - rmm==23.12.* diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 2714dcfa55a..a6744a9f340 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -26,7 +26,7 @@ requirements: - python - scikit-build >=0.13.1 run: - - distributed >=2023.9.2 + - rapids-dask-dependency ={{ version }} - numba >=0.57 - numpy >=1.21 - python diff --git a/conda/recipes/cugraph-service/meta.yaml b/conda/recipes/cugraph-service/meta.yaml index ae8074ba7d3..d52a004db05 100644 --- a/conda/recipes/cugraph-service/meta.yaml +++ b/conda/recipes/cugraph-service/meta.yaml @@ -59,10 +59,10 @@ outputs: - cupy >=12.0.0 - dask-cuda ={{ minor_version }} - dask-cudf ={{ minor_version }} - - distributed >=2023.9.2 - numba >=0.57 - numpy >=1.21 - python + - rapids-dask-dependency ={{ minor_version }} - thriftpy2 >=0.4.15 - ucx-py {{ ucx_py_version }} diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml index 65403bc8d73..58b9ea220d4 100644 --- a/conda/recipes/cugraph/meta.yaml +++ b/conda/recipes/cugraph/meta.yaml @@ -76,15 +76,13 @@ requirements: - cupy >=12.0.0 - dask-cuda ={{ minor_version }} - dask-cudf ={{ minor_version }} - - dask >=2023.9.2 - - dask-core >=2023.9.2 - - distributed >=2023.9.2 - fsspec>=0.6.0 - libcugraph ={{ version }} - pylibcugraph ={{ version }} - pylibraft ={{ minor_version }} - python - raft-dask ={{ minor_version }} + - rapids-dask-dependency ={{ minor_version }} - requests - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} diff --git a/dependencies.yaml b/dependencies.yaml index b127d9bd29e..13f100610cf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -377,15 +377,13 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - &dask dask>=2023.7.1 - - &distributed distributed>=2023.7.1 + - &dask rapids-dask-dependency==23.12.* - &dask_cuda dask-cuda==23.12.* - &numba numba>=0.57 - &ucx_py ucx-py==0.35.* - output_types: conda packages: - aiohttp - - &dask-core_conda dask-core>=2023.9.2 - fsspec>=0.6.0 - libcudf==23.12.* - requests @@ -431,14 +429,10 @@ dependencies: packages: - *dask - *dask_cuda - - *distributed - *numba - *numpy - *thrift - *ucx_py - - output_types: conda - packages: - - *dask-core_conda - output_types: pyproject packages: - *cugraph diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index f50b33b3f15..d68f8055ded 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -25,10 +25,9 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "dask-cuda==23.12.*", "dask-cudf==23.12.*", - "dask>=2023.7.1", - "distributed>=2023.7.1", "numba>=0.57", "numpy>=1.21", + "rapids-dask-dependency==23.12.*", "rmm==23.12.*", "thriftpy2", "ucx-py==0.35.*", diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index aaa301fa05f..319900b3de3 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -33,12 +33,11 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "dask-cuda==23.12.*", "dask-cudf==23.12.*", - "dask>=2023.7.1", - "distributed>=2023.7.1", "fsspec[http]>=0.6.0", "numba>=0.57", "pylibcugraph==23.12.*", "raft-dask==23.12.*", + "rapids-dask-dependency==23.12.*", "rmm==23.12.*", "ucx-py==0.35.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.