Merge branch 'branch-24.08' into testing-on-demand-jobs

.pre-commit-config.yaml

@@ -41,13 +41,8 @@ repos:
       - id: clang-format
         types_or: [c, c++, cuda]
         args: ["-fallback-style=none", "-style=file", "-i"]
-  - repo: https://github.com/rapidsai/dependency-file-generator
-    rev: v1.8.0
-    hooks:
-        - id: rapids-dependency-file-generator
-          args: ["--clean"]
   - repo: https://github.com/rapidsai/pre-commit-hooks
-    rev: v0.0.3
+    rev: v0.2.0
     hooks:
       - id: verify-copyright
         files: |
@@ -58,6 +53,12 @@ repos:
               [.]flake8[.]cython$|
               meta[.]yaml$|
               setup[.]cfg$
+      - id: verify-alpha-spec
+  - repo: https://github.com/rapidsai/dependency-file-generator
+    rev: v1.13.11
+    hooks:
+        - id: rapids-dependency-file-generator
+          args: ["--clean"]
   - repo: local
     hooks:
       - id: nx-cugraph-meta-data-update

benchmarks/cugraph/pytest-based/bench_algos.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2023, NVIDIA CORPORATION.
+# Copyright (c) 2020-2024, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -268,6 +268,20 @@ def is_graph_distributed(graph):
     return isinstance(graph.edgelist.edgelist_df, dask_cudf.DataFrame)
 
 
+def get_vertex_pairs(G, num_vertices=10):
+    """
+    Return a DateFrame containing two-hop vertex pairs randomly sampled from
+    a Graph.
+    """
+    random_vertices = G.select_random_vertices(num_vertices=num_vertices)
+
+    if isinstance(random_vertices, dask_cudf.Series):
+        random_vertices = random_vertices.compute()
+
+    vertices = random_vertices.to_arrow().to_pylist()
+    return G.get_two_hop_neighbors(start_vertices=vertices)
+
+
 ###############################################################################
 # Benchmarks
 def bench_create_graph(gpubenchmark, edgelist):
@@ -315,12 +329,6 @@ def bench_bfs(gpubenchmark, graph):
     gpubenchmark(bfs, graph, start)
 
 
-def bench_force_atlas2(gpubenchmark, graph):
-    if is_graph_distributed(graph):
-        pytest.skip("distributed graphs are not supported")
-    gpubenchmark(cugraph.force_atlas2, graph, max_iter=50)
-
-
 def bench_sssp(gpubenchmark, graph):
     sssp = dask_cugraph.sssp if is_graph_distributed(graph) else cugraph.sssp
     start = graph.edgelist.edgelist_df["src"][0]
@@ -329,8 +337,20 @@ def bench_sssp(gpubenchmark, graph):
 
 def bench_jaccard(gpubenchmark, unweighted_graph):
     G = unweighted_graph
+    # algo cannot compute neighbors on all nodes without running into OOM
+    # this is why we will call jaccard on a subset of nodes
+    vert_pairs = get_vertex_pairs(G)
     jaccard = dask_cugraph.jaccard if is_graph_distributed(G) else cugraph.jaccard
-    gpubenchmark(jaccard, G)
+    gpubenchmark(jaccard, G, vert_pairs)
+
+
+def bench_sorensen(gpubenchmark, unweighted_graph):
+    G = unweighted_graph
+    # algo cannot compute neighbors on all nodes without running into OOM
+    # this is why we will call sorensen on a subset of nodes
+    vert_pairs = get_vertex_pairs(G)
+    sorensen = dask_cugraph.sorensen if is_graph_distributed(G) else cugraph.sorensen
+    gpubenchmark(sorensen, G, vert_pairs)
 
 
 @pytest.mark.skipif(
@@ -353,8 +373,11 @@ def bench_weakly_connected_components(gpubenchmark, graph):
 
 def bench_overlap(gpubenchmark, unweighted_graph):
     G = unweighted_graph
+    # algo cannot compute neighbors on all nodes without running into OOM
+    # this is why we will call sorensen on a subset of nodes
+    vertex_pairs = get_vertex_pairs(G)
     overlap = dask_cugraph.overlap if is_graph_distributed(G) else cugraph.overlap
-    gpubenchmark(overlap, G)
+    gpubenchmark(overlap, G, vertex_pairs)
 
 
 def bench_triangle_count(gpubenchmark, graph):

build.sh

@@ -109,7 +109,7 @@ BUILD_CPP_MTMG_TESTS=OFF
 BUILD_ALL_GPU_ARCH=0
 BUILD_WITH_CUGRAPHOPS=ON
 CMAKE_GENERATOR_OPTION="-G Ninja"
-PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps"
+PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true"
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if PREFIX is not set, check CONDA_PREFIX, but there is no fallback

ci/build_cpp.sh

@@ -13,10 +13,8 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
-version=$(rapids-generate-version)
-
 rapids-logger "Begin cpp build"
 
-RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcugraph
+RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcugraph
 
 rapids-upload-conda-to-s3 cpp

ci/build_docs.sh

@@ -8,7 +8,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key docs \
+  --file-key docs \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n docs
@@ -72,10 +72,8 @@ pushd docs/cugraph
 # type of failure well.
 python -c "import cugraph; print(f'Using cugraph: {cugraph}')"
 sphinx-build -b dirhtml source _html
-sphinx-build -b text source _text
-mkdir -p "${RAPIDS_DOCS_DIR}/cugraph/"{html,txt}
+mkdir -p "${RAPIDS_DOCS_DIR}/cugraph/html"
 mv _html/* "${RAPIDS_DOCS_DIR}/cugraph/html"
-mv _text/* "${RAPIDS_DOCS_DIR}/cugraph/txt"
 popd
 
 rapids-upload-docs

ci/build_python.sh

@@ -15,20 +15,11 @@ rapids-print-env
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
-export RAPIDS_PACKAGE_VERSION=${version}
-echo "${version}" > VERSION
+rapids-generate-version > ./VERSION
+export RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION)
 
 rapids-logger "Begin py build"
 
-package_dir="python"
-for package_name in pylibcugraph cugraph cugraph-pyg cugraph-dgl; do
-  underscore_package_name=$(echo "${package_name}" | tr "-" "_")
-  sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py"
-done
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/nx-cugraph/_nx_cugraph/_version.py"
-
 # TODO: Remove `--no-test` flags once importing on a CPU
 # node works correctly
 rapids-conda-retry mambabuild \
@@ -56,10 +47,6 @@ rapids-conda-retry mambabuild \
 # built on each CUDA platform to ensure they are included in each set of
 # artifacts, since test scripts only install from one set of artifacts based on
 # the CUDA version used for the test run.
-version_file_cugraph_service_client="python/cugraph-service/client/cugraph_service_client/_version.py"
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_cugraph_service_client}
-version_file_cugraph_service_server="python/cugraph-service/server/cugraph_service_server/_version.py"
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_cugraph_service_server}
 rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \

ci/build_wheel.sh

@@ -5,54 +5,13 @@ set -euo pipefail
 
 package_name=$1
 package_dir=$2
-underscore_package_name=$(echo "${package_name}" | tr "-" "_")
 
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
-
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# This is the version of the suffix with a preceding hyphen. It's used
-# everywhere except in the final wheel name.
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
-
-# Patch project metadata files to include the CUDA version suffix and version override.
-version_package_name="$underscore_package_name"
-if [[ "${version_package_name}" = "nx_cugraph" ]]; then
-    version_package_name="_nx_cugraph"
-fi
-pyproject_file="${package_dir}/pyproject.toml"
-version_file="${package_dir}/${version_package_name}/_version.py"
-
-sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-echo "${version}" > VERSION
-sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file}
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
-for dep in rmm cudf cugraph raft-dask pylibcugraph pylibcugraphops pylibwholegraph pylibraft ucx-py; do
-    sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
-done
-
-# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec.
-for dep in dask-cuda rapids-dask-dependency; do
-    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
-done
-
-
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
-fi
+rapids-generate-version > ./VERSION
 
 cd "${package_dir}"
 

ci/build_wheel_cugraph.sh

@@ -6,11 +6,16 @@ set -euo pipefail
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 # Download the pylibcugraph wheel built in the previous step and make it
-# available for pip to find. We must use PIP_FIND_LINKS because the package
-# must be made available to the isolated build step, and there is no way to
-# manually install it into that environment.
-RAPIDS_PY_WHEEL_NAME=pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibcugraph
-export PIP_FIND_LINKS=$(pwd)/local-pylibcugraph
+# available for pip to find.
+#
+# ensure 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run
+#
+# using env variable PIP_CONSTRAINT is necessary to ensure the constraints
+# are used when creating the isolated build environment
+CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcugraph_dist)
+
+echo "pylibcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/pylibcugraph_*.whl)" > ./constraints.txt
+export PIP_CONSTRAINT="${PWD}/constraints.txt"
 
 PARALLEL_LEVEL=$(python -c \
   "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))")

ci/check_style.sh

@@ -8,7 +8,7 @@ rapids-logger "Create checks conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key checks \
+  --file-key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n checks

ci/release/update-version.sh

@@ -74,12 +74,12 @@ DEPENDENCIES=(
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}"
-    sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
+    sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*,>=0.0.0a0/g" "${FILE}"
   done
   for FILE in python/**/pyproject.toml python/**/**/pyproject.toml; do
-    sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" "${FILE}"
-    sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_VERSION}.*\"/g" "${FILE}"
+    sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" "${FILE}"
+    sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_VERSION}.*,>=0.0.0a0\"/g" "${FILE}"
   done
 done
 

ci/test_cpp.sh

@@ -11,7 +11,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_cpp \
+  --file-key test_cpp \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
@@ -43,6 +43,9 @@ popd
 
 export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/
 
+# Skip benchmark tests inside of CI
+export GTEST_FILTER="-*benchmark*"
+
 # Run libcugraph gtests from libcugraph-tests package
 rapids-logger "Run gtests"
 ./ci/run_ctests.sh -j10 && EXITCODE=$? || EXITCODE=$?;

ci/test_notebooks.sh

@@ -8,7 +8,7 @@ set -Eeuo pipefail
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_notebooks \
+  --file-key test_notebooks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

ci/test_python.sh

@@ -11,7 +11,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
+  --file-key test_python \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test