diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ccfdb826812..0f490283795 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,7 +22,7 @@ on: default: nightly concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: diff --git a/build.sh b/build.sh index 99082fa96fb..1723e750978 100755 --- a/build.sh +++ b/build.sh @@ -31,6 +31,7 @@ VALIDARGS=" cugraph-dgl nx-cugraph cpp-mgtests + cpp-mtmgtests docs all -v @@ -59,6 +60,7 @@ HELP="$0 [ ...] [ ...] cugraph-dgl - build the cugraph-dgl extensions for DGL nx-cugraph - build the nx-cugraph Python package cpp-mgtests - build libcugraph and libcugraph_etl MG tests. Builds MPI communicator, adding MPI as a dependency. + cpp-mtmgtests - build libcugraph MTMG tests. Adds UCX as a dependency (temporary). docs - build the docs all - build everything and is: @@ -105,6 +107,7 @@ BUILD_TYPE=Release INSTALL_TARGET="--target install" BUILD_CPP_TESTS=ON BUILD_CPP_MG_TESTS=OFF +BUILD_CPP_MTMG_TESTS=OFF BUILD_ALL_GPU_ARCH=0 BUILD_WITH_CUGRAPHOPS=ON CMAKE_GENERATOR_OPTION="-G Ninja" @@ -172,6 +175,9 @@ fi if hasArg --without_cugraphops; then BUILD_WITH_CUGRAPHOPS=OFF fi +if hasArg cpp-mtmgtests; then + BUILD_CPP_MTMG_TESTS=ON +fi if hasArg cpp-mgtests || hasArg all; then BUILD_CPP_MG_TESTS=ON fi @@ -264,6 +270,7 @@ if buildDefault || hasArg libcugraph || hasArg all; then -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DBUILD_TESTS=${BUILD_CPP_TESTS} \ -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \ + -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \ -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \ ${CMAKE_GENERATOR_OPTION} \ ${CMAKE_VERBOSE_OPTION} @@ -294,6 +301,7 @@ if buildDefault || hasArg libcugraph_etl || hasArg all; then -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DBUILD_TESTS=${BUILD_CPP_TESTS} \ -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \ + -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \ -DCMAKE_PREFIX_PATH=${LIBCUGRAPH_BUILD_DIR} \ ${CMAKE_GENERATOR_OPTION} \ ${CMAKE_VERBOSE_OPTION} \ diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c888c908056..163520ea1da 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -40,8 +40,11 @@ for dep in rmm cudf raft-dask pylibcugraph pylibraft ucx-py; do sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} done -# dask-cuda doesn't get a suffix, but it does get an alpha spec. -sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} +# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec. +for dep in dask-cuda rapids-dask-dependency; do + sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} +done + if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d3dbed6ae46..69eb085e7ed 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -88,10 +88,12 @@ DEPENDENCIES=( raft-dask rmm ucx-py + rapids-dask-dependency ) for DEP in "${DEPENDENCIES[@]}"; do - for FILE in dependencies.yaml conda/environments/*.yaml; do + for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE} + sed_runner "/-.* ${DEP}-cu[0-9][0-9]==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE} sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*/g" ${FILE} done for FILE in python/**/pyproject.toml python/**/**/pyproject.toml; do @@ -108,6 +110,11 @@ sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" cond sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" conda/recipes/cugraph-service/conda_build_config.yaml sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}.*\"/}" conda/recipes/pylibcugraph/conda_build_config.yaml +# nx-cugraph NetworkX entry-point meta-data +sed_runner "s@branch-[0-9][0-9].[0-9][0-9]@branch-${NEXT_SHORT_TAG}@g" python/nx-cugraph/_nx_cugraph/__init__.py +# FIXME: can this use the standard VERSION file and update mechanism? +sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/nx-cugraph/_nx_cugraph/__init__.py + # CI files for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" diff --git a/ci/test_python.sh b/ci/test_python.sh index 1690ce2f15b..273d3c93482 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -197,27 +197,26 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then conda activate test_cugraph_pyg set -u - # Install pytorch + # Will automatically install built dependencies of cuGraph-PyG rapids-mamba-retry install \ - --force-reinstall \ - --channel pyg \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ --channel pytorch \ --channel nvidia \ - 'pyg=2.3' \ - 'pytorch=2.0.0' \ - 'pytorch-cuda=11.8' + --channel pyg \ + --channel rapidsai-nightly \ + "cugraph-pyg" \ + "pytorch>=2.0,<2.1" \ + "pytorch-cuda=11.8" # Install pyg dependencies (which requires pip) - pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.0.0+cu118.html - - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - libcugraph \ - pylibcugraph \ - pylibcugraphops \ - cugraph \ - cugraph-pyg + pip install \ + pyg_lib \ + torch_scatter \ + torch_sparse \ + torch_cluster \ + torch_spline_conv \ + -f https://data.pyg.org/whl/torch-2.0.0+cu118.html rapids-print-env diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh index f9e2aa6d8da..d351ea21624 100755 --- a/ci/test_wheel_cugraph.sh +++ b/ci/test_wheel_cugraph.sh @@ -8,7 +8,4 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl -# Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main - ./ci/test_wheel.sh cugraph python/cugraph diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2f3a9c988cf..aa38defcd7c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,11 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* - dask-cudf==23.12.* -- dask>=2023.7.1 -- distributed>=2023.7.1 - doxygen - fsspec>=0.6.0 - gcc_linux-64=11.* @@ -62,6 +59,7 @@ dependencies: - pytest-xdist - python-louvain - raft-dask==23.12.* +- rapids-dask-dependency==23.12.* - recommonmark - requests - rmm==23.12.* diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 31ff503e682..a9f793b15f5 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -20,11 +20,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* - dask-cudf==23.12.* -- dask>=2023.7.1 -- distributed>=2023.7.1 - doxygen - fsspec>=0.6.0 - gcc_linux-64=11.* @@ -61,6 +58,7 @@ dependencies: - pytest-xdist - python-louvain - raft-dask==23.12.* +- rapids-dask-dependency==23.12.* - recommonmark - requests - rmm==23.12.* diff --git a/conda/recipes/cugraph-dgl/meta.yaml b/conda/recipes/cugraph-dgl/meta.yaml index bb85734098a..aaa1cd8a936 100644 --- a/conda/recipes/cugraph-dgl/meta.yaml +++ b/conda/recipes/cugraph-dgl/meta.yaml @@ -26,7 +26,7 @@ requirements: - dgl >=1.1.0.cu* - numba >=0.57 - numpy >=1.21 - - pylibcugraphops ={{ version }} + - pylibcugraphops ={{ minor_version }} - python - pytorch diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 2714dcfa55a..a2a02a1d9f6 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -26,15 +26,15 @@ requirements: - python - scikit-build >=0.13.1 run: - - distributed >=2023.9.2 + - rapids-dask-dependency ={{ minor_version }} - numba >=0.57 - numpy >=1.21 - python - pytorch >=2.0 - cupy >=12.0.0 - cugraph ={{ version }} - - pylibcugraphops ={{ version }} - - pyg >=2.3,<2.4 + - pylibcugraphops ={{ minor_version }} + - pyg >=2.3,<2.5 tests: imports: diff --git a/conda/recipes/cugraph-service/meta.yaml b/conda/recipes/cugraph-service/meta.yaml index ae8074ba7d3..d52a004db05 100644 --- a/conda/recipes/cugraph-service/meta.yaml +++ b/conda/recipes/cugraph-service/meta.yaml @@ -59,10 +59,10 @@ outputs: - cupy >=12.0.0 - dask-cuda ={{ minor_version }} - dask-cudf ={{ minor_version }} - - distributed >=2023.9.2 - numba >=0.57 - numpy >=1.21 - python + - rapids-dask-dependency ={{ minor_version }} - thriftpy2 >=0.4.15 - ucx-py {{ ucx_py_version }} diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml index 65403bc8d73..58b9ea220d4 100644 --- a/conda/recipes/cugraph/meta.yaml +++ b/conda/recipes/cugraph/meta.yaml @@ -76,15 +76,13 @@ requirements: - cupy >=12.0.0 - dask-cuda ={{ minor_version }} - dask-cudf ={{ minor_version }} - - dask >=2023.9.2 - - dask-core >=2023.9.2 - - distributed >=2023.9.2 - fsspec>=0.6.0 - libcugraph ={{ version }} - pylibcugraph ={{ version }} - pylibraft ={{ minor_version }} - python - raft-dask ={{ minor_version }} + - rapids-dask-dependency ={{ minor_version }} - requests - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b6efae8d2fe..626d62cffa5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -153,6 +153,11 @@ rapids_cpm_init() # lags behind. ### +# Need to make sure rmm is found before cuco so that rmm patches the libcudacxx +# directory to be found by cuco. +include(${rapids-cmake-dir}/cpm/rmm.cmake) +rapids_cpm_rmm(BUILD_EXPORT_SET cugraph-exports + INSTALL_EXPORT_SET cugraph-exports) # Putting this before raft to override RAFT from pulling them in. include(cmake/thirdparty/get_libcudacxx.cmake) include(${rapids-cmake-dir}/cpm/cuco.cmake) @@ -166,7 +171,10 @@ endif() include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_cuhornet.cmake) -include(cmake/thirdparty/get_ucp.cmake) + +if (BUILD_CUGRAPH_MTMG_TESTS) + include(cmake/thirdparty/get_ucp.cmake) +endif() if(BUILD_TESTS) include(cmake/thirdparty/get_gtest.cmake) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 2f69cf9cb0d..6530a25d178 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -415,13 +415,6 @@ ConfigureTest(K_HOP_NBRS_TEST traversal/k_hop_nbrs_test.cpp) # - install tests --------------------------------------------------------------------------------- rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcugraph) -################################################################################################### -# - MTMG tests ------------------------------------------------------------------------- -ConfigureTest(MTMG_TEST mtmg/threaded_test.cu) -target_link_libraries(MTMG_TEST - PRIVATE - UCP::UCP - ) ################################################################################################### # - MG tests -------------------------------------------------------------------------------------- @@ -681,15 +674,6 @@ if(BUILD_CUGRAPH_MG_TESTS) rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_mg DESTINATION bin/gtests/libcugraph_mg) - ############################################################################################### - # - Multi-node MTMG tests --------------------------------------------------------------------- - ConfigureTest(MTMG_MULTINODE_TEST mtmg/multi_node_threaded_test.cu utilities/mg_utilities.cpp) - target_link_libraries(MTMG_MULTINODE_TEST - PRIVATE - cugraphmgtestutil - UCP::UCP - ) - endif() ################################################################################################### @@ -749,4 +733,25 @@ ConfigureCTest(CAPI_EGONET_TEST c_api/egonet_test.c) ConfigureCTest(CAPI_TWO_HOP_NEIGHBORS_TEST c_api/two_hop_neighbors_test.c) ConfigureCTest(CAPI_LEGACY_K_TRUSS_TEST c_api/legacy_k_truss_test.c) +if (BUILD_CUGRAPH_MTMG_TESTS) + ################################################################################################### + # - MTMG tests ------------------------------------------------------------------------- + ConfigureTest(MTMG_TEST mtmg/threaded_test.cu) + target_link_libraries(MTMG_TEST + PRIVATE + UCP::UCP + ) + + if(BUILD_CUGRAPH_MG_TESTS) + ############################################################################################### + # - Multi-node MTMG tests --------------------------------------------------------------------- + ConfigureTest(MTMG_MULTINODE_TEST mtmg/multi_node_threaded_test.cu utilities/mg_utilities.cpp) + target_link_libraries(MTMG_MULTINODE_TEST + PRIVATE + cugraphmgtestutil + UCP::UCP + ) + endif(BUILD_CUGRAPH_MG_TESTS) +endif(BUILD_CUGRAPH_MTMG_TESTS) + rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_c DESTINATION bin/gtests/libcugraph_c) diff --git a/dependencies.yaml b/dependencies.yaml index b127d9bd29e..a89acd9288b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -377,15 +377,13 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - &dask dask>=2023.7.1 - - &distributed distributed>=2023.7.1 + - &dask rapids-dask-dependency==23.12.* - &dask_cuda dask-cuda==23.12.* - &numba numba>=0.57 - &ucx_py ucx-py==0.35.* - output_types: conda packages: - aiohttp - - &dask-core_conda dask-core>=2023.9.2 - fsspec>=0.6.0 - libcudf==23.12.* - requests @@ -431,14 +429,10 @@ dependencies: packages: - *dask - *dask_cuda - - *distributed - *numba - *numpy - *thrift - *ucx_py - - output_types: conda - packages: - - *dask-core_conda - output_types: pyproject packages: - *cugraph @@ -503,9 +497,9 @@ dependencies: - output_types: [conda] packages: - cugraph==23.12.* - - pytorch==2.0 + - pytorch>=2.0 - pytorch-cuda==11.8 - - pyg=2.3.1=*torch_2.0.0*cu118* + - pyg>=2.4.0 depends_on_rmm: common: diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index f98eab430ba..71d1c7e389c 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -13,13 +13,13 @@ dependencies: - cugraph==23.12.* - pandas - pre-commit -- pyg=2.3.1=*torch_2.0.0*cu118* +- pyg>=2.4.0 - pylibcugraphops==23.12.* - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda==11.8 -- pytorch==2.0 +- pytorch>=2.0 - scipy name: cugraph_pyg_dev_cuda-118 diff --git a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py index d1b24543956..edeeface4c4 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py @@ -210,7 +210,10 @@ class EXPERIMENTAL__CuGraphStore: def __init__( self, F: cugraph.gnn.FeatureStore, - G: Union[Dict[str, Tuple[TensorType]], Dict[str, int]], + G: Union[ + Dict[Tuple[str, str, str], Tuple[TensorType]], + Dict[Tuple[str, str, str], int], + ], num_nodes_dict: Dict[str, int], *, multi_gpu: bool = False, @@ -744,7 +747,7 @@ def _subgraph(self, edge_types: List[tuple] = None) -> cugraph.MultiGraph: def _get_vertex_groups_from_sample( self, nodes_of_interest: TensorType, is_sorted: bool = False - ) -> dict: + ) -> Dict[str, torch.Tensor]: """ Given a tensor of nodes of interest, this method a single dictionary, noi_index. @@ -808,7 +811,10 @@ def _get_sample_from_vertex_groups( def _get_renumbered_edge_groups_from_sample( self, sampling_results: cudf.DataFrame, noi_index: dict - ) -> Tuple[dict, dict]: + ) -> Tuple[ + Dict[Tuple[str, str, str], torch.Tensor], + Tuple[Dict[Tuple[str, str, str], torch.Tensor]], + ]: """ Given a cudf (NOT dask_cudf) DataFrame of sampling results and a dictionary of non-renumbered vertex ids grouped by vertex type, this method diff --git a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py index 8552e7412e0..ad8d22e255e 100644 --- a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py @@ -15,6 +15,7 @@ import os import re +import warnings import cupy import cudf @@ -159,23 +160,34 @@ def __init__( if batch_size is None or batch_size < 1: raise ValueError("Batch size must be >= 1") - self.__directory = tempfile.TemporaryDirectory(dir=directory) + self.__directory = ( + tempfile.TemporaryDirectory() if directory is None else directory + ) if isinstance(num_neighbors, dict): raise ValueError("num_neighbors dict is currently unsupported!") - renumber = ( - True - if ( - (len(self.__graph_store.node_types) == 1) - and (len(self.__graph_store.edge_types) == 1) + if "renumber" in kwargs: + warnings.warn( + "Setting renumbering manually could result in invalid output," + " please ensure you intended to do this." + ) + renumber = kwargs.pop("renumber") + else: + renumber = ( + True + if ( + (len(self.__graph_store.node_types) == 1) + and (len(self.__graph_store.edge_types) == 1) + ) + else False ) - else False - ) bulk_sampler = BulkSampler( batch_size, - self.__directory.name, + self.__directory + if isinstance(self.__directory, str) + else self.__directory.name, self.__graph_store._subgraph(edge_types), fanout_vals=num_neighbors, with_replacement=replace, @@ -219,7 +231,13 @@ def __init__( ) bulk_sampler.flush() - self.__input_files = iter(os.listdir(self.__directory.name)) + self.__input_files = iter( + os.listdir( + self.__directory + if isinstance(self.__directory, str) + else self.__directory.name + ) + ) def __next__(self): from time import perf_counter @@ -423,9 +441,6 @@ def __next__(self): sampler_output.edge, ) else: - if self.__graph_store.order == "CSR": - raise ValueError("CSR format incompatible with CSC output") - out = filter_cugraph_store_csc( self.__feature_store, self.__graph_store, @@ -437,11 +452,8 @@ def __next__(self): # Account for CSR format in cuGraph vs. CSC format in PyG if self.__coo and self.__graph_store.order == "CSC": - for node_type in out.edge_index_dict: - out[node_type].edge_index[0], out[node_type].edge_index[1] = ( - out[node_type].edge_index[1], - out[node_type].edge_index[0], - ) + for edge_type in out.edge_index_dict: + out[edge_type].edge_index = out[edge_type].edge_index.flip(dims=[0]) out.set_value_dict("num_sampled_nodes", sampler_output.num_sampled_nodes) out.set_value_dict("num_sampled_edges", sampler_output.num_sampled_edges) diff --git a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py index ed7f70034e2..13c9c90c7c2 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py @@ -120,7 +120,7 @@ def test_get_edge_index(graph, edge_index_type, dask_client): G[et][0] = dask_cudf.from_cudf(cudf.Series(G[et][0]), npartitions=1) G[et][1] = dask_cudf.from_cudf(cudf.Series(G[et][1]), npartitions=1) - cugraph_store = CuGraphStore(F, G, N, multi_gpu=True) + cugraph_store = CuGraphStore(F, G, N, order="CSC", multi_gpu=True) for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py index 853836dc2a6..27b73bf7d35 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py @@ -18,6 +18,7 @@ import cudf import cupy +import numpy as np from cugraph_pyg.loader import CuGraphNeighborLoader from cugraph_pyg.loader import BulkSampleLoader @@ -27,6 +28,8 @@ from cugraph.gnn import FeatureStore from cugraph.utilities.utils import import_optional, MissingModule +from typing import Dict, Tuple + torch = import_optional("torch") torch_geometric = import_optional("torch_geometric") trim_to_layer = import_optional("torch_geometric.utils.trim_to_layer") @@ -40,7 +43,11 @@ @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") -def test_cugraph_loader_basic(karate_gnn): +def test_cugraph_loader_basic( + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ] +): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") loader = CuGraphNeighborLoader( @@ -66,7 +73,11 @@ def test_cugraph_loader_basic(karate_gnn): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") -def test_cugraph_loader_hetero(karate_gnn): +def test_cugraph_loader_hetero( + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ] +): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") loader = CuGraphNeighborLoader( @@ -342,7 +353,7 @@ def test_cugraph_loader_e2e_coo(): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.skipif(not HAS_TORCH_SPARSE, reason="torch-sparse not available") @pytest.mark.parametrize("framework", ["pyg", "cugraph-ops"]) -def test_cugraph_loader_e2e_csc(framework): +def test_cugraph_loader_e2e_csc(framework: str): m = [2, 9, 99, 82, 9, 3, 18, 1, 12] x = torch.randint(3000, (256, 256)).to(torch.float32) F = FeatureStore() @@ -442,3 +453,40 @@ def test_cugraph_loader_e2e_csc(framework): x = x.narrow(dim=0, start=0, length=s - num_sampled_nodes[1]) assert list(x.shape) == [1, 1] + + +@pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +@pytest.mark.parametrize("directory", ["local", "temp"]) +def test_load_directory( + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ], + directory: str, +): + if directory == "local": + local_dir = tempfile.TemporaryDirectory(dir=".") + + cugraph_store = CuGraphStore(*karate_gnn) + cugraph_loader = CuGraphNeighborLoader( + (cugraph_store, cugraph_store), + torch.arange(8, dtype=torch.int64), + 2, + num_neighbors=[8, 4, 2], + random_state=62, + replace=False, + directory=None if directory == "temp" else local_dir.name, + batches_per_partition=1, + ) + + it = iter(cugraph_loader) + next_batch = next(it) + assert next_batch is not None + + if directory == "local": + assert len(os.listdir(local_dir.name)) == 4 + + count = 1 + while next(it, None) is not None: + count += 1 + + assert count == 4 diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py index da3043760d4..b39ebad8254 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py @@ -113,7 +113,7 @@ def test_get_edge_index(graph, edge_index_type): G[et][0] = cudf.Series(G[et][0]) G[et][1] = cudf.Series(G[et][1]) - cugraph_store = CuGraphStore(F, G, N) + cugraph_store = CuGraphStore(F, G, N, order="CSC") for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index f50b33b3f15..d68f8055ded 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -25,10 +25,9 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "dask-cuda==23.12.*", "dask-cudf==23.12.*", - "dask>=2023.7.1", - "distributed>=2023.7.1", "numba>=0.57", "numpy>=1.21", + "rapids-dask-dependency==23.12.*", "rmm==23.12.*", "thriftpy2", "ucx-py==0.35.*", diff --git a/python/cugraph/cugraph/structure/property_graph.py b/python/cugraph/cugraph/structure/property_graph.py index 36ce5baa212..513798f35f9 100644 --- a/python/cugraph/cugraph/structure/property_graph.py +++ b/python/cugraph/cugraph/structure/property_graph.py @@ -800,15 +800,9 @@ def add_vertex_data( tmp_df.index = tmp_df.index.rename(self.vertex_col_name) # FIXME: handle case of a type_name column already being in tmp_df - if self.__series_type is cudf.Series: - # cudf does not yet support initialization with a scalar - tmp_df[TCN] = cudf.Series( - cudf.Series([type_name], dtype=cat_dtype).repeat(len(tmp_df)), - index=tmp_df.index, - ) - else: - # pandas is oddly slow if dtype is passed to the constructor here - tmp_df[TCN] = pd.Series(type_name, index=tmp_df.index).astype(cat_dtype) + tmp_df[TCN] = self.__series_type(type_name, index=tmp_df.index).astype( + cat_dtype + ) if property_columns: # all columns @@ -1207,15 +1201,9 @@ def add_edge_data( tmp_df[self.src_col_name] = tmp_df[vertex_col_names[0]] tmp_df[self.dst_col_name] = tmp_df[vertex_col_names[1]] - if self.__series_type is cudf.Series: - # cudf does not yet support initialization with a scalar - tmp_df[TCN] = cudf.Series( - cudf.Series([type_name], dtype=cat_dtype).repeat(len(tmp_df)), - index=tmp_df.index, - ) - else: - # pandas is oddly slow if dtype is passed to the constructor here - tmp_df[TCN] = pd.Series(type_name, index=tmp_df.index).astype(cat_dtype) + tmp_df[TCN] = self.__series_type(type_name, index=tmp_df.index).astype( + cat_dtype + ) # Add unique edge IDs to the new rows. This is just a count for each # row starting from the last edge ID value, with initial edge ID 0. diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index aaa301fa05f..319900b3de3 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -33,12 +33,11 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "dask-cuda==23.12.*", "dask-cudf==23.12.*", - "dask>=2023.7.1", - "distributed>=2023.7.1", "fsspec[http]>=0.6.0", "numba>=0.57", "pylibcugraph==23.12.*", "raft-dask==23.12.*", + "rapids-dask-dependency==23.12.*", "rmm==23.12.*", "ucx-py==0.35.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index 8ef976aabf1..26638d1e735 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -136,6 +136,7 @@ def get_info(): return d +# FIXME: can this use the standard VERSION file and update mechanism? __version__ = "23.12.00" if __name__ == "__main__":