Merge branch 'branch-24.08' into c_api_lookup_src_dst

rapidsai · Jun 30, 2024 · 7f46d9e · 7f46d9e
2 parents cd592c6 + f0590ef
commit 7f46d9e
Show file tree

Hide file tree

Showing 470 changed files with 19,053 additions and 11,308 deletions.
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -72,10 +72,8 @@ pushd docs/cugraph
 # type of failure well.
 python -c "import cugraph; print(f'Using cugraph: {cugraph}')"
 sphinx-build -b dirhtml source _html
-sphinx-build -b text source _text
-mkdir -p "${RAPIDS_DOCS_DIR}/cugraph/"{html,txt}
+mkdir -p "${RAPIDS_DOCS_DIR}/cugraph/html"
 mv _html/* "${RAPIDS_DOCS_DIR}/cugraph/html"
-mv _text/* "${RAPIDS_DOCS_DIR}/cugraph/txt"
 popd
 
 rapids-upload-docs
diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -56,7 +56,13 @@ fi
 
 cd "${package_dir}"
 
-python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+python -m pip wheel \
+    -w dist \
+    -vvv \
+    --no-deps \
+    --disable-pip-version-check \
+    --extra-index-url https://pypi.nvidia.com \
+    .
 
 # pure-python packages should be marked as pure, and not have auditwheel run on them.
 if [[ ${package_name} == "nx-cugraph" ]] || \

diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -12,6 +12,10 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME=pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibcugraph
 export PIP_FIND_LINKS=$(pwd)/local-pylibcugraph
 
+PARALLEL_LEVEL=$(python -c \
+  "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))")
+
 export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/"
+export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}"
 
 ./ci/build_wheel.sh cugraph python/cugraph
diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh
@@ -1,8 +1,12 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
+PARALLEL_LEVEL=$(python -c \
+  "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))")
+
 export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/"
+export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}"
 
 ./ci/build_wheel.sh pylibcugraph python/pylibcugraph
diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -44,6 +44,8 @@ rapids-mamba-retry install \
 rapids-logger "Check GPU usage"
 nvidia-smi
 
+export LD_PRELOAD="${CONDA_PREFIX}/lib/libgomp.so.1"
+
 # RAPIDS_DATASET_ROOT_DIR is used by test scripts
 export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
 pushd "${RAPIDS_DATASET_ROOT_DIR}"
@@ -191,6 +193,8 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
     conda activate test_cugraph_pyg
     set -u
 
+    rapids-print-env
+
     # TODO re-enable logic once CUDA 12 is testable
     #if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
     CONDA_CUDA_VERSION="11.8"
@@ -204,18 +208,9 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
     rapids-mamba-retry install \
       --channel "${CPP_CHANNEL}" \
       --channel "${PYTHON_CHANNEL}" \
-      --channel pytorch \
       --channel pyg \
-      --channel nvidia \
       "cugraph-pyg" \
-      "pytorch=2.1.0" \
-      "pytorch-cuda=${CONDA_CUDA_VERSION}"
-
-    # Install pyg dependencies (which requires pip)
-
-    pip install \
-      ogb \
-      tensordict
+      "ogb"
 
     pip install \
         pyg_lib \

diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh
@@ -42,7 +42,6 @@ rapids-retry python -m pip install \
   pyg_lib \
   torch_scatter \
   torch_sparse \
-  tensordict \
   -f ${PYG_URL}
 
 rapids-logger "pytest cugraph-pyg (single GPU)"

diff --git a/ci/test_wheel_nx-cugraph.sh b/ci/test_wheel_nx-cugraph.sh
@@ -1,6 +1,11 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -eoxu pipefail
 
+# Download wheels built during this job.
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
+python -m pip install ./local-deps/*.whl
+
 ./ci/test_wheel.sh nx-cugraph python/nx-cugraph
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -56,6 +56,7 @@ dependencies:
 - pytest-mpl
 - pytest-xdist
 - python-louvain
+- pytorch>=2.0,<2.2.0a0
 - raft-dask==24.8.*
 - rapids-dask-dependency==24.8.*
 - recommonmark
@@ -69,6 +70,7 @@ dependencies:
 - sphinx-markdown-tables
 - sphinx<6
 - sphinxcontrib-websupport
+- thriftpy2<=0.5.0
 - ucx-proc=*=gpu
 - ucx-py==0.39.*
 - wget

diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml
@@ -61,6 +61,7 @@ dependencies:
 - pytest-mpl
 - pytest-xdist
 - python-louvain
+- pytorch>=2.0,<2.2.0a0
 - raft-dask==24.8.*
 - rapids-dask-dependency==24.8.*
 - recommonmark
@@ -74,6 +75,7 @@ dependencies:
 - sphinx-markdown-tables
 - sphinx<6
 - sphinxcontrib-websupport
+- thriftpy2<=0.5.0
 - ucx-proc=*=gpu
 - ucx-py==0.39.*
 - wget

diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml
@@ -42,7 +42,7 @@ requirements:
     - {{ compiler('cxx') }}
     - cmake {{ cmake_version }}
     - ninja
-    - openmpi # Required for building cpp-mgtests (multi-GPU tests)
+    - openmpi<5.0.3 # Required for building cpp-mgtests (multi-GPU tests)
     - {{ stdlib("c") }}
   host:
     {% if cuda_major == "11" %}