From 946b239c72a4ebd8dc6b03bdfb6a393e7db91006 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 10:56:11 -0700 Subject: [PATCH 01/52] add ci files --- ci/build_cpp.sh | 22 ++ ci/build_python.sh | 55 +--- ci/build_wheel.sh | 11 +- ci/run_ctests.sh | 9 + ci/run_pylibwholegraph_pytests.sh | 9 + ci/test_cpp.sh | 22 +- ci/test_python.sh | 264 +++++++++--------- ci/wheel_smoke_test_cugraph-dgl.py | 17 ++ ci/wheel_smoke_test_cugraph-pyg.py | 17 ++ ci/wheel_smoke_test_cugraph.py | 37 --- ci/wheel_smoke_test_pylibwholegraph.py | 17 ++ .../all_cuda-124_arch-x86_64.yaml | 63 +++++ dependencies.yaml | 20 +- .../conda/cugraph_dgl_dev_cuda-118.yaml | 2 +- .../conda/cugraph_pyg_dev_cuda-118.yaml | 2 +- 15 files changed, 330 insertions(+), 237 deletions(-) create mode 100644 ci/build_cpp.sh create mode 100644 ci/run_ctests.sh create mode 100644 ci/run_pylibwholegraph_pytests.sh create mode 100644 ci/wheel_smoke_test_cugraph-dgl.py create mode 100644 ci/wheel_smoke_test_cugraph-pyg.py delete mode 100644 ci/wheel_smoke_test_cugraph.py create mode 100644 ci/wheel_smoke_test_pylibwholegraph.py create mode 100644 conda/environments/all_cuda-124_arch-x86_64.yaml diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh new file mode 100644 index 0000000..5049639 --- /dev/null +++ b/ci/build_cpp.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) 2022-2024, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-configure-conda-channels + +source rapids-configure-sccache + +source rapids-date-string + +export CMAKE_GENERATOR=Ninja + +rapids-print-env + +version=$(rapids-generate-version) + +rapids-logger "Begin cpp build" + +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libwholegraph + +rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 62154fd..bfd890d 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -23,7 +23,7 @@ echo "${version}" > VERSION rapids-logger "Begin py build" package_dir="python" -for package_name in pylibcugraph cugraph cugraph-pyg cugraph-dgl; do +for package_name in pylibwholegraph cugraph-pyg cugraph-dgl; do underscore_package_name=$(echo "${package_name}" | tr "-" "_") sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" done @@ -34,65 +34,20 @@ sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/nx-cugra rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ - conda/recipes/pylibcugraph + conda/recipes/pylibwholegraph -rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cugraph +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -# NOTE: nothing in nx-cugraph is CUDA-specific, but it is built on each CUDA -# platform to ensure it is included in each set of artifacts, since test -# scripts only install from one set of artifacts based on the CUDA version used -# for the test run. rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/nx-cugraph + conda/recipes/cugraph-pyg -# NOTE: nothing in the cugraph-service packages are CUDA-specific, but they are -# built on each CUDA platform to ensure they are included in each set of -# artifacts, since test scripts only install from one set of artifacts based on -# the CUDA version used for the test run. -version_file_cugraph_service_client="python/cugraph-service/client/cugraph_service_client/_version.py" -sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_cugraph_service_client} -version_file_cugraph_service_server="python/cugraph-service/server/cugraph_service_server/_version.py" -sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_cugraph_service_server} rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cugraph-service - -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" - -if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then - # Only CUDA 11 is supported right now due to PyTorch requirement. - rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel pyg \ - --channel pytorch \ - --channel pytorch-nightly \ - conda/recipes/cugraph-pyg - - # Only CUDA 11 is supported right now due to PyTorch requirement. - rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel dglteam \ - --channel pytorch \ - --channel pytorch-nightly \ - conda/recipes/cugraph-dgl -fi - -rapids-conda-retry mambabuild \ - --no-test \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cugraph-equivariant + conda/recipes/cugraph-dgl rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index a143830..f9b2489 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -61,8 +61,15 @@ if [[ ${package_name} == "cugraph-dgl" ]] || \ [[ ${package_name} == "cugraph-pyg" ]]; then RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist else - # presumably WholeGraph when we add it + # Hardcode the output dir + SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DBUILD_SHARED_LIBS=OFF;-DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE;-DCUDA_STATIC_RUNTIME=ON;-DWHOLEGRAPH_BUILD_WHEELS=ON" \ + python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + mkdir -p final_dist - python -m auditwheel repair -w final_dist dist/* + python -m auditwheel repair \ + --exclude libcuda.so.1 \ + --exclude libnvidia-ml.so.1 \ + -w final_dist dist/* + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist fi diff --git a/ci/run_ctests.sh b/ci/run_ctests.sh new file mode 100644 index 0000000..1ebbf4c --- /dev/null +++ b/ci/run_ctests.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Support customizing the ctests' install location +cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libwholegraph/" + +find . -type f -executable -print0 | xargs -0 -r -t -n1 -P1 sh -c 'exec "$0"'; diff --git a/ci/run_pylibwholegraph_pytests.sh b/ci/run_pylibwholegraph_pytests.sh new file mode 100644 index 0000000..b21cbf9 --- /dev/null +++ b/ci/run_pylibwholegraph_pytests.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Support invoking run_pytests.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/pylibwholegraph/pylibwholegraph/ + +pytest --cache-clear --forked --import-mode=append "$@" tests diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index a5f7fee..09ecf7a 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" + rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -22,30 +24,22 @@ conda activate test set -u CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) - RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" rapids-print-env rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - libcugraph libcugraph_etl libcugraph-tests + --channel "${CPP_CHANNEL}" \ + "libwholegraph=${RAPIDS_VERSION}" \ + "libwholegraph-tests=${RAPIDS_VERSION}" rapids-logger "Check GPU usage" nvidia-smi -# RAPIDS_DATASET_ROOT_DIR is used by test scripts -export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" -pushd "${RAPIDS_DATASET_ROOT_DIR}" -./get_test_data.sh --subset -popd - -export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/ - -# Run libcugraph gtests from libcugraph-tests package -rapids-logger "Run gtests" -./ci/run_ctests.sh -j10 && EXITCODE=$? || EXITCODE=$?; +# Run libwholegraph tests from libwholegraph-tests package +rapids-logger "Run tests" +./ci/run_ctests.sh && EXITCODE=$? || EXITCODE=$? rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_python.sh b/ci/test_python.sh index d9f64e4..7e1be3a 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -14,13 +14,6 @@ rapids-dependency-file-generator \ --file-key test_python \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n test - -# Temporarily allow unbound variables for conda activation. -set +u -conda activate test -set -u - rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) @@ -29,21 +22,6 @@ RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" -rapids-print-env - -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - libcugraph \ - pylibcugraph \ - cugraph \ - nx-cugraph \ - cugraph-service-server \ - cugraph-service-client - -rapids-logger "Check GPU usage" -nvidia-smi - # RAPIDS_DATASET_ROOT_DIR is used by test scripts export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" pushd "${RAPIDS_DATASET_ROOT_DIR}" @@ -65,115 +43,147 @@ set +e # # FIXME: TEMPORARILY disable MG PropertyGraph tests (experimental) tests and # bulk sampler IO tests (hangs in CI) -if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - # we are only testing in a single cuda version - # because of pytorch and rapids compatibilty problems - rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_dgl - - # activate test_cugraph_dgl environment for dgl - set +u - conda activate test_cugraph_dgl - set -u - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - --channel conda-forge \ - --channel dglteam/label/cu118 \ - --channel nvidia \ - libcugraph \ - pylibcugraph \ - pylibcugraphops \ - cugraph \ - cugraph-dgl \ - 'dgl>=1.1.0.cu*,<=2.0.0.cu*' \ - 'pytorch>=2.0' \ - 'cuda-version=11.8' - - rapids-print-env - - rapids-logger "pytest cugraph_dgl (single GPU)" - ./ci/run_cugraph_dgl_pytests.sh \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-dgl.xml" \ - --cov-config=../../.coveragerc \ - --cov=cugraph_dgl \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-dgl-coverage.xml" \ - --cov-report=term - - # Reactivate the test environment back - set +u - conda deactivate - conda activate test - set -u - else - rapids-logger "skipping cugraph_dgl pytest on ARM64" - fi + +if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then + rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_dgl + + # activate test_cugraph_dgl environment for dgl + set +u + conda activate test_cugraph_dgl + set -u + + rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + --channel pytorch \ + --channel conda-forge \ + --channel dglteam/label/th23_cu118 \ + --channel nvidia \ + "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "pylibcugraphops=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "cugraph-dgl=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + 'pytorch::pytorch>=2.3,<2.4' \ + 'cuda-version=11.8' \ + "ogb" + + rapids-print-env + + rapids-logger "Check GPU usage" + nvidia-smi + + rapids-logger "pytest cugraph_dgl (single GPU)" + ./ci/run_cugraph_dgl_pytests.sh \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-dgl.xml" \ + --cov-config=../../.coveragerc \ + --cov=cugraph_dgl \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-dgl-coverage.xml" \ + --cov-report=term + + # Reactivate the test environment back + set +u + conda deactivate + conda activate test + set -u +else + rapids-logger "skipping cugraph_dgl pytest on ARM64" +fi + +if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then + rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_pyg + + # Temporarily allow unbound variables for conda activation. + set +u + conda activate test_cugraph_pyg + set -u + + # TODO re-enable logic once CUDA 12 is testable + #if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then + CONDA_CUDA_VERSION="11.8" + PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html" + #else + # CONDA_CUDA_VERSION="12.1" + # PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html" + #fi + + # Will automatically install built dependencies of cuGraph-PyG + rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + --channel pytorch \ + "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "pylibcugraphops=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "cugraph-pyg=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "pytorch::pytorch>=2.3,<2.4" \ + "ogb" + + rapids-print-env + + rapids-logger "Check GPU usage" + nvidia-smi + + rapids-logger "pytest cugraph_pyg (single GPU)" + ./ci/run_cugraph_pyg_pytests.sh \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-pyg.xml" \ + --cov-config=../../.coveragerc \ + --cov=cugraph_pyg \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-pyg-coverage.xml" \ + --cov-report=term + + # Reactivate the test environment back + set +u + conda deactivate + conda activate test + set -u else - rapids-logger "skipping cugraph_dgl pytest on CUDA!=11.8" + rapids-logger "skipping cugraph_pyg pytest on ARM64" fi -if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_pyg - - # Temporarily allow unbound variables for conda activation. - set +u - conda activate test_cugraph_pyg - set -u - - # TODO re-enable logic once CUDA 12 is testable - #if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - CONDA_CUDA_VERSION="11.8" - PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html" - #else - # CONDA_CUDA_VERSION="12.1" - # PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html" - #fi - - # Will automatically install built dependencies of cuGraph-PyG - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - --channel pytorch \ - --channel pyg \ - --channel nvidia \ - "cugraph-pyg" \ - "pytorch=2.1.0" \ - "pytorch-cuda=${CONDA_CUDA_VERSION}" - - # Install pyg dependencies (which requires pip) - - pip install \ - ogb \ - tensordict - - pip install \ - pyg_lib \ - torch_scatter \ - torch_sparse \ - -f ${PYG_URL} - - rapids-print-env - - rapids-logger "pytest cugraph_pyg (single GPU)" - # rmat is not tested because of multi-GPU testing - ./ci/run_cugraph_pyg_pytests.sh \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-pyg.xml" \ - --cov-config=../../.coveragerc \ - --cov=cugraph_pyg \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-pyg-coverage.xml" \ - --cov-report=term - - # Reactivate the test environment back - set +u - conda deactivate - conda activate test - set -u - else - rapids-logger "skipping cugraph_pyg pytest on ARM64" - fi +if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then + rapids-mamba-retry env create --yes -f env.yaml -n test_pylibwholegraph + + # Temporarily allow unbound variables for conda activation. + set +u + conda activate test_pylibwholegraph + set -u + + # TODO re-enable logic once CUDA 12 is testable + #if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then + CONDA_CUDA_VERSION="11.8" + #else + # CONDA_CUDA_VERSION="12.1" + #fi + + # Will automatically install built dependencies of pylibwholegraph + rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + --channel pytorch \ + "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "pytorch::pytorch>=2.3,<2.4" \ + "ogb" + + rapids-print-env + + rapids-logger "Check GPU usage" + nvidia-smi + + rapids-logger "pytest cugraph_pyg (single GPU)" + ./ci/run_cugraph_pyg_pytests.sh \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-pyg.xml" \ + --cov-config=../../.coveragerc \ + --cov=cugraph_pyg \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-pyg-coverage.xml" \ + --cov-report=term + + # Reactivate the test environment back + set +u + conda deactivate + conda activate test + set -u else - rapids-logger "skipping cugraph_pyg pytest on CUDA!=11.8" + rapids-logger "skipping cugraph_pyg pytest on ARM64" fi diff --git a/ci/wheel_smoke_test_cugraph-dgl.py b/ci/wheel_smoke_test_cugraph-dgl.py new file mode 100644 index 0000000..8b2f933 --- /dev/null +++ b/ci/wheel_smoke_test_cugraph-dgl.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cugraph_dgl + +if __name__ == "__main__": + print(cugraph_dgl.__version__) diff --git a/ci/wheel_smoke_test_cugraph-pyg.py b/ci/wheel_smoke_test_cugraph-pyg.py new file mode 100644 index 0000000..d05e510 --- /dev/null +++ b/ci/wheel_smoke_test_cugraph-pyg.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cugraph_pyg + +if __name__ == "__main__": + print(cugraph_pyg.__version__) diff --git a/ci/wheel_smoke_test_cugraph.py b/ci/wheel_smoke_test_cugraph.py deleted file mode 100644 index fb88dcd..0000000 --- a/ci/wheel_smoke_test_cugraph.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import cudf -import cugraph - -if __name__ == "__main__": - edgelist = cudf.DataFrame({"source": ["a", "b", "c"], "destination": ["b", "c", "d"]}) - - # directed graph - G = cugraph.Graph(directed=True) - G.from_cudf_edgelist(edgelist, store_transposed=True) - result_df = cugraph.pagerank(G) - - assert(result_df["pagerank"].sum() == 1.0) - assert(result_df.sort_values(by="pagerank")["vertex"].values_host.tolist() - == ["a", "b", "c", "d"]) - - # undirected graph - G = cugraph.Graph(directed=False) - G.from_cudf_edgelist(edgelist, store_transposed=True) - result_df = cugraph.pagerank(G) - - assert(result_df["pagerank"].sum() == 1.0) - result_df.set_index("vertex", inplace=True) - assert(result_df.loc["a", "pagerank"] == result_df.loc["d", "pagerank"]) - assert(result_df.loc["b", "pagerank"] == result_df.loc["c", "pagerank"]) diff --git a/ci/wheel_smoke_test_pylibwholegraph.py b/ci/wheel_smoke_test_pylibwholegraph.py new file mode 100644 index 0000000..43d56f8 --- /dev/null +++ b/ci/wheel_smoke_test_pylibwholegraph.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pylibwholegraph + +if __name__ == "__main__": + print(pylibwholegraph.__version__) diff --git a/conda/environments/all_cuda-124_arch-x86_64.yaml b/conda/environments/all_cuda-124_arch-x86_64.yaml new file mode 100644 index 0000000..f0ac1bb --- /dev/null +++ b/conda/environments/all_cuda-124_arch-x86_64.yaml @@ -0,0 +1,63 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- dask/label/dev +- pyg +- conda-forge +- nvidia +dependencies: +- breathe +- cmake>=3.26.4,!=3.30.0 +- cuda-cudart-dev +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=12.4 +- cudf==24.12.* +- cugraph==24.12.* +- cupy>=12.0.0 +- cython>=3.0.0 +- dask-cudf==24.12.* +- dglteam/label/th21_cu124::dgl +- doxygen +- graphviz +- ipython +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- nbsphinx +- ninja +- notebook>=0.5.0 +- numba>=0.57 +- numpy>=1.23,<3.0a0 +- numpydoc +- pandas +- pre-commit +- pydantic +- pydata-sphinx-theme +- pyg::pyg +- pylibcugraphops==24.12.* +- pylibraft==24.12.* +- pylibwholegraph==24.12.* +- pytest +- pytest-benchmark +- pytest-cov +- pytest-xdist +- pytorch-cuda=12.4 +- pytorch::pytorch>=2.0,<2.2.0a0 +- raft-dask==24.12.* +- recommonmark +- rmm==24.12.* +- scikit-build-core>=0.10.0 +- scipy +- setuptools>=61.0.0 +- sphinx-copybutton +- sphinx-markdown-tables +- sphinx<6 +- sphinxcontrib-websupport +- torchdata +- wget +- wheel +name: all_cuda-124_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 269b1c3..94543f4 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: [conda] matrix: - cuda: ["11.8", "12.1"] + cuda: ["11.8", "12.1", "12.4"] arch: [x86_64] includes: - checks @@ -185,6 +185,10 @@ dependencies: cuda: "12.2" packages: - cuda-version=12.2 + - matrix: + cuda: "12.4" + packages: + - cuda-version=12.4 cuda: specific: - output_types: [conda] @@ -306,7 +310,7 @@ dependencies: packages: - cugraph==24.12.* - pytorch::pytorch>=2.0 - - dgl>=1.1.0.cu* + - dgl>=2.4.0.cu* - &tensordict tensordict>=0.1.2 cugraph_pyg_dev: common: @@ -315,7 +319,7 @@ dependencies: - cugraph==24.12.* - pytorch::pytorch>=2.0 - *tensordict - - pyg>=2.5,<2.6 + - pytorch_geometric>=2.5,<2.6 depends_on_pytorch: common: @@ -347,9 +351,12 @@ dependencies: - {matrix: null, packages: [*pytorch_pip, *tensordict]} - output_types: [conda] matrices: - - matrix: {cuda: "12.*"} + - matrix: {cuda: "12.1"} packages: - pytorch-cuda=12.1 + - matrix: {cuda: "12.4"} + packages: + - pytorch-cuda=12.4 - matrix: {cuda: "11.*"} packages: - pytorch-cuda=11.8 @@ -358,9 +365,12 @@ dependencies: specific: - output_types: [conda] matrices: - - matrix: {cuda: "12.*"} + - matrix: {cuda: "12.1"} packages: - "dglteam/label/th21_cu121::dgl" + - matrix: {cuda: "12.4"} + packages: + - "dglteam/label/th21_cu124::dgl" - matrix: {cuda: "11.*"} packages: - "dglteam/label/th21_cu118::dgl" diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index c1aa00c..2618be1 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -9,7 +9,7 @@ channels: - nvidia dependencies: - cugraph==24.12.* -- dgl>=1.1.0.cu* +- dgl>=2.4.0.cu* - dglteam/label/th21_cu118::dgl - pandas - pre-commit diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index 6fed8da..266a3e5 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -12,7 +12,6 @@ dependencies: - pandas - pre-commit - pydantic -- pyg>=2.5,<2.6 - pylibcugraphops==24.12.* - pytest - pytest-benchmark @@ -21,6 +20,7 @@ dependencies: - pytorch-cuda=11.8 - pytorch::pytorch>=2.0 - pytorch::pytorch>=2.0,<2.2.0a0 +- pytorch_geometric>=2.5,<2.6 - scipy - tensordict>=0.1.2 - torchdata From db06ee256bef06ec5b5fe9f5caa06ec852535f70 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:09:04 -0700 Subject: [PATCH 02/52] update dependencies file --- .../all_cuda-118_arch-x86_64.yaml | 19 +- .../all_cuda-121_arch-x86_64.yaml | 19 +- .../all_cuda-124_arch-x86_64.yaml | 19 +- dependencies.yaml | 166 +++++++++--------- .../conda/cugraph_dgl_dev_cuda-118.yaml | 8 +- python/cugraph-dgl/pyproject.toml | 3 +- .../conda/cugraph_pyg_dev_cuda-118.yaml | 6 +- python/cugraph-pyg/pyproject.toml | 3 +- 8 files changed, 113 insertions(+), 130 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 10fbcc9..4380b82 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -13,12 +13,11 @@ dependencies: - cuda-nvtx - cuda-version=11.8 - cudatoolkit -- cudf==24.12.* -- cugraph==24.12.* +- cudf==24.12.*,>=0.0.0a0 +- cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 - cython>=3.0.0 -- dask-cudf==24.12.* -- dglteam/label/th21_cu118::dgl +- dask-cudf==24.12.*,>=0.0.0a0 - doxygen - graphviz - ipython @@ -32,19 +31,17 @@ dependencies: - pre-commit - pydantic - pydata-sphinx-theme -- pyg::pyg -- pylibcugraphops==24.12.* -- pylibraft==24.12.* -- pylibwholegraph==24.12.* +- pylibcugraphops==24.12.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda=11.8 -- pytorch::pytorch>=2.0,<2.2.0a0 -- raft-dask==24.12.* +- pytorch>=2.3,<2.4.0a0 +- raft-dask==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.12.* +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy - setuptools>=61.0.0 diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index f58fb71..a868c5c 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -14,12 +14,11 @@ dependencies: - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.1 -- cudf==24.12.* -- cugraph==24.12.* +- cudf==24.12.*,>=0.0.0a0 +- cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 - cython>=3.0.0 -- dask-cudf==24.12.* -- dglteam/label/th21_cu121::dgl +- dask-cudf==24.12.*,>=0.0.0a0 - doxygen - graphviz - ipython @@ -37,19 +36,17 @@ dependencies: - pre-commit - pydantic - pydata-sphinx-theme -- pyg::pyg -- pylibcugraphops==24.12.* -- pylibraft==24.12.* -- pylibwholegraph==24.12.* +- pylibcugraphops==24.12.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda=12.1 -- pytorch::pytorch>=2.0,<2.2.0a0 -- raft-dask==24.12.* +- pytorch>=2.3,<2.4.0a0 +- raft-dask==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.12.* +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy - setuptools>=61.0.0 diff --git a/conda/environments/all_cuda-124_arch-x86_64.yaml b/conda/environments/all_cuda-124_arch-x86_64.yaml index f0ac1bb..20dcef6 100644 --- a/conda/environments/all_cuda-124_arch-x86_64.yaml +++ b/conda/environments/all_cuda-124_arch-x86_64.yaml @@ -14,12 +14,11 @@ dependencies: - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.4 -- cudf==24.12.* -- cugraph==24.12.* +- cudf==24.12.*,>=0.0.0a0 +- cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 - cython>=3.0.0 -- dask-cudf==24.12.* -- dglteam/label/th21_cu124::dgl +- dask-cudf==24.12.*,>=0.0.0a0 - doxygen - graphviz - ipython @@ -37,19 +36,17 @@ dependencies: - pre-commit - pydantic - pydata-sphinx-theme -- pyg::pyg -- pylibcugraphops==24.12.* -- pylibraft==24.12.* -- pylibwholegraph==24.12.* +- pylibcugraphops==24.12.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda=12.4 -- pytorch::pytorch>=2.0,<2.2.0a0 -- raft-dask==24.12.* +- pytorch>=2.3,<2.4.0a0 +- raft-dask==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.12.* +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy - setuptools>=61.0.0 diff --git a/dependencies.yaml b/dependencies.yaml index 94543f4..29ceca6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -20,11 +20,8 @@ files: - depends_on_pylibraft - depends_on_raft_dask - depends_on_pylibcugraphops - - depends_on_pylibwholegraph - depends_on_cupy - depends_on_pytorch - - depends_on_dgl - - depends_on_pyg - python_run_cugraph_dgl - python_run_cugraph_pyg - test_notebook @@ -60,7 +57,6 @@ files: - cuda_version - depends_on_cugraph - depends_on_cudf - - depends_on_pylibwholegraph - py_version - test_python_common - test_python_pylibcugraph @@ -88,7 +84,6 @@ files: key: test includes: - test_python_common - - depends_on_pylibwholegraph py_build_cugraph_pyg: output: pyproject pyproject_dir: python/cugraph-pyg @@ -112,7 +107,6 @@ files: key: test includes: - test_python_common - - depends_on_pylibwholegraph cugraph_dgl_dev: @@ -309,7 +303,7 @@ dependencies: - output_types: [conda] packages: - cugraph==24.12.* - - pytorch::pytorch>=2.0 + - &pytorch_conda pytorch>=2.3,<2.4.0a0 - dgl>=2.4.0.cu* - &tensordict tensordict>=0.1.2 cugraph_pyg_dev: @@ -317,7 +311,7 @@ dependencies: - output_types: [conda] packages: - cugraph==24.12.* - - pytorch::pytorch>=2.0 + - *pytorch_conda - *tensordict - pytorch_geometric>=2.5,<2.6 @@ -325,7 +319,7 @@ dependencies: common: - output_types: [conda] packages: - - pytorch::pytorch>=2.0,<2.2.0a0 + - *pytorch_conda - torchdata - pydantic specific: @@ -342,7 +336,7 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - &pytorch_pip torch>=2.0,<2.2.0a0 + - &pytorch_pip torch>=2.3,<2.4.0a0 - *tensordict - matrix: {cuda: "11.*"} packages: @@ -361,63 +355,11 @@ dependencies: packages: - pytorch-cuda=11.8 - depends_on_dgl: - specific: - - output_types: [conda] - matrices: - - matrix: {cuda: "12.1"} - packages: - - "dglteam/label/th21_cu121::dgl" - - matrix: {cuda: "12.4"} - packages: - - "dglteam/label/th21_cu124::dgl" - - matrix: {cuda: "11.*"} - packages: - - "dglteam/label/th21_cu118::dgl" - - {matrix: null, packages: ["dglteam/label/th23_cu121::dgl"]} - - depends_on_pyg: - specific: - - output_types: [conda] - matrices: - - matrix: {cuda: "12.*"} - packages: - - "pyg::pyg" - - matrix: {cuda: "11.*"} - packages: - - "pyg::pyg" - - {matrix: null, packages: ["pyg::pyg"]} - - depends_on_pylibwholegraph: - common: - - output_types: conda - packages: - - &pylibwholegraph_unsuffixed pylibwholegraph==24.12.* - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - specific: - - output_types: [requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - pylibwholegraph-cu12==24.12.* - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - pylibwholegraph-cu11==24.12.* - - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} - depends_on_rmm: common: - output_types: conda packages: - - &rmm_unsuffixed rmm==24.12.* + - &rmm_unsuffixed rmm==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -430,19 +372,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - rmm-cu12==24.12.* + - rmm-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - rmm-cu11==24.12.* + - rmm-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_unsuffixed]} depends_on_cugraph: common: - output_types: conda packages: - - &cugraph_unsuffixed cugraph==24.12.* + - &cugraph_unsuffixed cugraph==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -455,19 +397,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - cugraph-cu12==24.12.* + - cugraph-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - cugraph-cu11==24.12.* + - cugraph-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*cugraph_unsuffixed]} depends_on_cudf: common: - output_types: conda packages: - - &cudf_unsuffixed cudf==24.12.* + - &cudf_unsuffixed cudf==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -480,19 +422,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - cudf-cu12==24.12.* + - cudf-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - cudf-cu11==24.12.* + - cudf-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*cudf_unsuffixed]} depends_on_dask_cudf: common: - output_types: conda packages: - - &dask_cudf_unsuffixed dask-cudf==24.12.* + - &dask_cudf_unsuffixed dask-cudf==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -505,19 +447,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - dask-cudf-cu12==24.12.* + - dask-cudf-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - dask-cudf-cu11==24.12.* + - dask-cudf-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*dask_cudf_unsuffixed]} depends_on_pylibraft: common: - output_types: conda packages: - - &pylibraft_unsuffixed pylibraft==24.12.* + - &pylibraft_unsuffixed pylibraft==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -530,19 +472,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibraft-cu12==24.12.* + - pylibraft-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibraft-cu11==24.12.* + - pylibraft-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*pylibraft_unsuffixed]} depends_on_raft_dask: common: - output_types: conda packages: - - &raft_dask_unsuffixed raft-dask==24.12.* + - &raft_dask_unsuffixed raft-dask==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -555,19 +497,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - raft-dask-cu12==24.12.* + - raft-dask-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - raft-dask-cu11==24.12.* + - raft-dask-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*raft_dask_unsuffixed]} depends_on_pylibcugraph: common: - output_types: conda packages: - - &pylibcugraph_unsuffixed pylibcugraph==24.12.* + - &pylibcugraph_unsuffixed pylibcugraph==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -580,19 +522,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibcugraph-cu12==24.12.* + - pylibcugraph-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibcugraph-cu11==24.12.* + - pylibcugraph-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*pylibcugraph_unsuffixed]} depends_on_pylibcugraphops: common: - output_types: conda packages: - - &pylibcugraphops_unsuffixed pylibcugraphops==24.12.* + - &pylibcugraphops_unsuffixed pylibcugraphops==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -605,12 +547,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibcugraphops-cu12==24.12.* + - pylibcugraphops-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibcugraphops-cu11==24.12.* + - pylibcugraphops-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*pylibcugraphops_unsuffixed]} depends_on_cupy: @@ -631,3 +573,55 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} + + depends_on_dgl: + specific: + - output_types: [conda] + matrices: + - matrix: {cuda: "12.1"} + packages: + - "dglteam/label/th23_cu121::dgl" + - matrix: {cuda: "12.4"} + packages: + - "dglteam/label/th23_cu124::dgl" + - matrix: {cuda: "11.*"} + packages: + - "dglteam/label/th23_cu118::dgl" + - {matrix: null, packages: ["dglteam/label/th23_cu121::dgl"]} + + depends_on_pyg: + specific: + - output_types: [conda] + matrices: + - matrix: {cuda: "12.*"} + packages: + - "pyg::pyg" + - matrix: {cuda: "11.*"} + packages: + - "pyg::pyg" + - {matrix: null, packages: ["pyg::pyg"]} + + depends_on_pylibwholegraph: + common: + - output_types: conda + packages: + - &pylibwholegraph_unsuffixed pylibwholegraph==24.12.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - pylibwholegraph-cu12==24.12.* + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - pylibwholegraph-cu11==24.12.* + - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index 2618be1..a8a14b1 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -9,19 +9,19 @@ channels: - nvidia dependencies: - cugraph==24.12.* +- cugraph==24.12.*,>=0.0.0a0 - dgl>=2.4.0.cu* -- dglteam/label/th21_cu118::dgl +- dglteam/label/th23_cu118::dgl - pandas - pre-commit - pydantic -- pylibcugraphops==24.12.* +- pylibcugraphops==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda=11.8 -- pytorch::pytorch>=2.0 -- pytorch::pytorch>=2.0,<2.2.0a0 +- pytorch>=2.3,<2.4.0a0 - scipy - tensordict>=0.1.2 - torchdata diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 685367e..a9937fd 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -26,13 +26,12 @@ dependencies = [ "cugraph==24.12.*", "numba>=0.57", "numpy>=1.23,<3.0a0", - "pylibcugraphops==24.12.*", + "pylibcugraphops==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.optional-dependencies] test = [ "pandas", - "pylibwholegraph==24.12.*", "pytest", "pytest-benchmark", "pytest-cov", diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index 266a3e5..b8cafee 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -9,17 +9,17 @@ channels: - nvidia dependencies: - cugraph==24.12.* +- cugraph==24.12.*,>=0.0.0a0 - pandas - pre-commit - pydantic -- pylibcugraphops==24.12.* +- pylibcugraphops==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda=11.8 -- pytorch::pytorch>=2.0 -- pytorch::pytorch>=2.0,<2.2.0a0 +- pytorch>=2.3,<2.4.0a0 - pytorch_geometric>=2.5,<2.6 - scipy - tensordict>=0.1.2 diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index c769d4e..078e2ff 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "cugraph==24.12.*", "numba>=0.57", "numpy>=1.23,<3.0a0", - "pylibcugraphops==24.12.*", + "pylibcugraphops==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] @@ -40,7 +40,6 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [project.optional-dependencies] test = [ "pandas", - "pylibwholegraph==24.12.*", "pytest", "pytest-benchmark", "pytest-cov", From c5813fe663add3881f8b8ce6cc6347a0037e7a02 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:09:51 -0700 Subject: [PATCH 03/52] add github files --- .github/workflows/build.yaml | 114 ++++++++++++++++++++++ .github/workflows/pr.yaml | 184 +++++++++++++++++++++++++++++++++++ .github/workflows/test.yaml | 71 ++++++++++++++ 3 files changed, 369 insertions(+) create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/pr.yaml create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..e5cb6b7 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,114 @@ +name: build + +on: + push: + branches: + - "branch-*" + tags: + - v[0-9][0-9].[0-9][0-9].[0-9][0-9] + workflow_dispatch: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + build_type: + type: string + default: nightly + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + cpp-build: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: cpu32 + python-build: + needs: [cpp-build] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + upload-conda: + needs: [cpp-build, python-build] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + wheel-build-cugraph-dgl: + needs: wheel-publish-cugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cugraph-dgl.sh + wheel-publish-cugraph-dgl: + needs: wheel-build-cugraph-dgl + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cugraph-dgl + wheel-build-cugraph-pyg: + needs: wheel-publish-cugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cugraph-pyg.sh + wheel-publish-cugraph-pyg: + needs: wheel-build-cugraph-pyg + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cugraph-pyg + wheel-build-pylibwholegraph: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_pylibwholegraph.sh + wheel-publish-pylibwholegraph: + needs: wheel-build-pylibwholegraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: pylibwholegraph \ No newline at end of file diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000..c604db2 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,184 @@ +name: pr + +on: + push: + branches: + - "pull-request/[0-9]+" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pr-builder: + needs: + - changed-files + - checks + - conda-cpp-build + - conda-cpp-tests + - conda-cpp-checks + - conda-notebook-tests + - conda-python-build + - conda-python-tests + - wheel-build-pylibwholegraph + - wheel-tests-pylibwholegraph + - wheel-build-cugraph-dgl + - wheel-tests-cugraph-dgl + - wheel-build-cugraph-pyg + - wheel-tests-cugraph-pyg + - devcontainer + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 + if: always() + with: + needs: ${{ toJSON(needs) }} + changed-files: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + with: + files_yaml: | + test_cpp: + - '**' + - '!.devcontainers/**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!mg_utils/**' + - '!notebooks/**' + - '!python/**' + - '!readme_pages/**' + # TODO: Remove this before merging + - '!.github/**' + test_notebooks: + - '**' + - '!.devcontainers/**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + # TODO: Remove this before merging + - '!.github/**' + test_python: + - '**' + - '!.devcontainers/**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + # TODO: Remove this before merging + - '!.github/**' + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 + with: + enable_check_generated_files: false + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 + with: + build_type: pull-request + node_type: cpu32 + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + with: + build_type: pull-request + enable_check_symbols: true + symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 + with: + build_type: pull-request + conda-python-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + conda-notebook-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + run_script: "ci/test_notebooks.sh" + wheel-build-pylibwholegraph: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: pull-request + script: ci/build_wheel_pylibcugraph.sh + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-24.12 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + node_type: cpu32 + wheel-tests-pylibwholegraph: + needs: [wheel-build-pylibcugraph, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_pylibcugraph.sh + wheel-build-cugraph-dgl: + needs: wheel-tests-cugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: pull-request + script: ci/build_wheel_cugraph-dgl.sh + wheel-tests-cugraph-dgl: + needs: [wheel-build-cugraph-dgl, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph-dgl.sh + matrix_filter: map(select(.ARCH == "amd64")) + wheel-build-cugraph-pyg: + needs: wheel-tests-cugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: pull-request + script: ci/build_wheel_cugraph-pyg.sh + wheel-tests-cugraph-pyg: + needs: [wheel-build-cugraph-pyg, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph-pyg.sh + matrix_filter: map(select(.ARCH == "amd64")) + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + node_type: cpu32 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + build_command: | + sccache -z; + build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; + sccache -s; \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..7cfee6f --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,71 @@ +name: test + +on: + workflow_dispatch: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + +jobs: + conda-cpp-checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + enable_check_symbols: true + symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) + conda-cpp-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + conda-python-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + wheel-tests-pylibcugraph: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel_pylibwholegraph.sh + wheel-tests-cugraph-dgl: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel_cugraph-dgl.sh + matrix_filter: map(select(.ARCH == "amd64")) + wheel-tests-cugraph-pyg: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel_cugraph-pyg.sh + matrix_filter: map(select(.ARCH == "amd64")) \ No newline at end of file From 4e29cacba34547cec7ece2e29ed942acc1862fac Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:12:08 -0700 Subject: [PATCH 04/52] add copy pr bot --- .github/copy-pr-bot.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/copy-pr-bot.yaml diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml new file mode 100644 index 0000000..935857f --- /dev/null +++ b/.github/copy-pr-bot.yaml @@ -0,0 +1,4 @@ +# Configuration file for `copy-pr-bot` GitHub App +# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/ + +enabled: true \ No newline at end of file From 1fe2c11f7cc1721c65e11b5d74ec5861441bc737 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:13:05 -0700 Subject: [PATCH 05/52] fix newline --- .github/copy-pr-bot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml index 935857f..895ba83 100644 --- a/.github/copy-pr-bot.yaml +++ b/.github/copy-pr-bot.yaml @@ -1,4 +1,4 @@ # Configuration file for `copy-pr-bot` GitHub App # https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/ -enabled: true \ No newline at end of file +enabled: true From e978611d1b648669ad5ff6f1fb0245f00512c76e Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:13:55 -0700 Subject: [PATCH 06/52] add ops bot --- .github/ops-bot.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/ops-bot.yaml diff --git a/.github/ops-bot.yaml b/.github/ops-bot.yaml new file mode 100644 index 0000000..2ed5231 --- /dev/null +++ b/.github/ops-bot.yaml @@ -0,0 +1,9 @@ +# This file controls which features from the `ops-bot` repository below are enabled. +# - https://github.com/rapidsai/ops-bot + +auto_merger: true +branch_checker: true +label_checker: true +release_drafter: true +recently_updated: true +forward_merger: true From d34ea2fe4eec7546ef18037e716ad9b30e556dc0 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:38:44 -0700 Subject: [PATCH 07/52] fix pr.yaml --- .github/workflows/pr.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c604db2..ed9fb14 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -125,21 +125,21 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request - script: ci/build_wheel_pylibcugraph.sh + script: ci/build_wheel_pylibwholegraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 wheel-tests-pylibwholegraph: - needs: [wheel-build-pylibcugraph, changed-files] + needs: [wheel-build-pylibwholegraph, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request - script: ci/test_wheel_pylibcugraph.sh + script: ci/test_wheel_pylibwholegraph.sh wheel-build-cugraph-dgl: - needs: wheel-tests-cugraph + needs: wheel-tests-pylibwholegraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: @@ -155,7 +155,7 @@ jobs: script: ci/test_wheel_cugraph-dgl.sh matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-pyg: - needs: wheel-tests-cugraph + needs: wheel-tests-pylibwholegraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: From 2a0827d8d489cb51e4e87b2691648c675fe37259 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:38:56 -0700 Subject: [PATCH 08/52] add wheel test script --- ci/build_wheel_pylibwholegraph.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 ci/build_wheel_pylibwholegraph.sh diff --git a/ci/build_wheel_pylibwholegraph.sh b/ci/build_wheel_pylibwholegraph.sh new file mode 100644 index 0000000..1def7d3 --- /dev/null +++ b/ci/build_wheel_pylibwholegraph.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +./ci/build_wheel.sh pylibwholegraph python/pylibwholegraph From 64c60a4832c46dcfda06cf4639e21b2e0a790c72 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:42:32 -0700 Subject: [PATCH 09/52] fix style ' --- .github/workflows/build.yaml | 3 ++- .github/workflows/pr.yaml | 3 ++- .github/workflows/test.yaml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e5cb6b7..c350f13 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -111,4 +111,5 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: pylibwholegraph \ No newline at end of file + package-name: pylibwholegraph + \ No newline at end of file diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ed9fb14..945ee9b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -181,4 +181,5 @@ jobs: build_command: | sccache -z; build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; - sccache -s; \ No newline at end of file + sccache -s; + \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7cfee6f..27ec807 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -68,4 +68,5 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_cugraph-pyg.sh - matrix_filter: map(select(.ARCH == "amd64")) \ No newline at end of file + matrix_filter: map(select(.ARCH == "amd64")) + \ No newline at end of file From 80370eb6b581eea2268f9f14270a8a4cba26b137 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:46:58 -0700 Subject: [PATCH 10/52] fix newlines (again) --- .github/workflows/build.yaml | 2 +- .github/workflows/pr.yaml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c350f13..dc6e635 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -112,4 +112,4 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: pylibwholegraph - \ No newline at end of file + diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 945ee9b..adb520d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -182,4 +182,4 @@ jobs: sccache -z; build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; sccache -s; - \ No newline at end of file + diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 27ec807..4d15947 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,4 +69,4 @@ jobs: sha: ${{ inputs.sha }} script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) - \ No newline at end of file + From 314461c3552c6eb0528348d89fc73591d224ea00 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:52:24 -0700 Subject: [PATCH 11/52] fix yamls --- .github/workflows/build.yaml | 5 ++--- .github/workflows/pr.yaml | 1 - .github/workflows/test.yaml | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dc6e635..03ded27 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -54,7 +54,7 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} wheel-build-cugraph-dgl: - needs: wheel-publish-cugraph + needs: wheel-publish-pylibwholegraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: @@ -74,7 +74,7 @@ jobs: date: ${{ inputs.date }} package-name: cugraph-dgl wheel-build-cugraph-pyg: - needs: wheel-publish-cugraph + needs: wheel-publish-pylibwholegraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: @@ -112,4 +112,3 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: pylibwholegraph - diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index adb520d..0cbe71a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -182,4 +182,3 @@ jobs: sccache -z; build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; sccache -s; - diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4d15947..13c0d44 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,4 +69,3 @@ jobs: sha: ${{ inputs.sha }} script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) - From 5d13fd8be7dc8eda943e424a2dd4850323955e5f Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 11:57:42 -0700 Subject: [PATCH 12/52] change perms --- ci/build_cpp.sh | 0 ci/build_wheel_pylibwholegraph.sh | 0 ci/run_ctests.sh | 0 ci/run_pylibwholegraph_pytests.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/build_cpp.sh mode change 100644 => 100755 ci/build_wheel_pylibwholegraph.sh mode change 100644 => 100755 ci/run_ctests.sh mode change 100644 => 100755 ci/run_pylibwholegraph_pytests.sh diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh old mode 100644 new mode 100755 diff --git a/ci/build_wheel_pylibwholegraph.sh b/ci/build_wheel_pylibwholegraph.sh old mode 100644 new mode 100755 diff --git a/ci/run_ctests.sh b/ci/run_ctests.sh old mode 100644 new mode 100755 diff --git a/ci/run_pylibwholegraph_pytests.sh b/ci/run_pylibwholegraph_pytests.sh old mode 100644 new mode 100755 From 1adfe9252915a5cd4818ae2e8fa9648773bda5ad Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 12:46:50 -0700 Subject: [PATCH 13/52] fix build.sh --- build.sh | 161 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 109 insertions(+), 52 deletions(-) diff --git a/build.sh b/build.sh index 44c4740..2b18808 100755 --- a/build.sh +++ b/build.sh @@ -26,13 +26,17 @@ VALIDARGS=" uninstall cugraph-pyg cugraph-dgl - docs + pylibwholegraph + libwholegraph + tests + benchmarks all -v -g -n --pydevelop --allgpuarch + --compile-cmd --clean -h --help @@ -43,27 +47,36 @@ HELP="$0 [ ...] [ ...] clean - remove all existing build artifacts and configuration (start over) uninstall - uninstall libcugraph and cugraph from a prior build/install (see also -n) cugraph-pyg - build the cugraph-pyg Python package - cugraph-dgl - build the cugraph-dgl extensions for DGL - docs - build the docs + cugraph-dgl - build the cugraph-dgl Python package + pylibwholegraph - build the pylibwholegraph Python package + libwholegraph - build the libwholegraph library + test - build the C++ tests + benchmarks - build benchmarks. all - build everything and is: - -v - verbose build mode - -g - build for debug - -n - do not install after a successful build (does not affect Python packages) - --pydevelop - install the Python packages in editable mode - --allgpuarch - build for all supported GPU architectures - --clean - clean an individual target (note: to do a complete rebuild, use the clean target described above) - -h - print this text - - default action (no args) is to build and install 'cugraph-pyg' then 'cugraph-dgl' then 'wholegraph' targets + -v - verbose build mode + -g - build for debug + -n - do not install after a successful build (does not affect Python packages) + --pydevelop - install the Python packages in editable mode + --allgpuarch - build for all supported GPU architectures + --enable-nvshmem - build with nvshmem support (beta). + --compile-cmd - only output compile commands (invoke CMake without build) + --clean - clean an individual target (note: to do a complete rebuild, use the clean target described above) + -h - print this text + + default action (no args) is to build and install 'libwholegraph' then 'pylibwholegraph' then 'cugraph-pyg' then 'cugraph-dgl' " CUGRAPH_PYG_BUILD_DIR=${REPODIR}/python/cugraph-pyg/build CUGRAPH_DGL_BUILD_DIR=${REPODIR}/python/cugraph-dgl/build +PYLIBWHOLEGRAPH_BUILD_DIR=${REPODIR}/python/pylibwholegraph/build +LIBWHOLEGRAPH_BUILD_DIR=${REPODIR}/cpp/build BUILD_DIRS="${CUGRAPH_PYG_BUILD_DIR} ${CUGRAPH_DGL_BUILD_DIR} + ${PYLIBWHOLEGRAPH_BUILD_DIR} + ${LIBWHOLEGRAPH_BUILD_DIR} " # Set defaults for vars modified by flags to this script @@ -131,14 +144,40 @@ fi if hasArg --pydevelop; then PYTHON_ARGS_FOR_INSTALL="${PYTHON_ARGS_FOR_INSTALL} -e" fi +if hasArg --enable-nvshmem; then + BUILD_WITH_NVSHMEM=ON +else + BUILD_WITH_NVSHMEM=OFF +fi +if hasArg tests; then + BUILD_TESTS=ON +else + BUILD_TESTS=OFF +fi +if hasArg benchmarks; then + BUILD_BENCHMARKS=ON +else + BUILD_BENCHMARKS=OFF +fi # If clean or uninstall targets given, run them prior to any other steps if hasArg uninstall; then - # uninstall cugraph and pylibcugraph installed from a prior install + if [[ "$INSTALL_PREFIX" != "" ]]; then + rm -rf ${INSTALL_PREFIX}/include/wholememory + rm -f ${INSTALL_PREFIX}/lib/libwholegraph.so + rm -rf ${INSTALL_PREFIX}/lib/cmake/wholegraph + fi + # This may be redundant given the above, but can also be used in case + # there are other installed files outside of the locations above. + if [ -e ${LIBWHOLEGRAPH_BUILD_DIR}/install_manifest.txt ]; then + xargs rm -f < ${LIBWHOLEGRAPH_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1 + fi + + # uninstall cugraph-dgl/cugraph-pyg/wholegraph installed from a prior install # FIXME: if multiple versions of these packages are installed, this only # removes the latest one and leaves the others installed. build.sh uninstall # can be run multiple times to remove all of them, but that is not obvious. - pip uninstall -y cugraph-dgl cugraph-pyg + pip uninstall -y cugraph-dgl cugraph-pyg pylibwholegraph libwholegraph fi if hasArg clean; then @@ -164,6 +203,61 @@ if hasArg clean; then fi ################################################################################ + +# Build and install the libwholegraph library +if hasArg libwholegraph || buildDefault || hasArg all ; then + + # set values based on flags + if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then + WHOLEGRAPH_CMAKE_CUDA_ARCHITECTURES="${WHOLEGRAPH_CMAKE_CUDA_ARCHITECTURES:=NATIVE}" + echo "Building for the architecture of the GPU in the system..." + else + WHOLEGRAPH_CMAKE_CUDA_ARCHITECTURES="70-real;75-real;80-real;86-real;90" + echo "Building for *ALL* supported GPU architectures..." + fi + + cmake -S ${REPODIR}/cpp -B ${LIBWHOLEGRAPH_BUILD_DIR} \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + -DCMAKE_CUDA_ARCHITECTURES=${WHOLEGRAPH_CMAKE_CUDA_ARCHITECTURES} \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} \ + -DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE \ + -DBUILD_TESTS=${BUILD_TESTS} \ + -DBUILD_WITH_NVSHMEM=${BUILD_WITH_NVSHMEM} + + cd ${LIBWHOLEGRAPH_BUILD_DIR} + + if ! hasArg --compile-cmd; then + ## Build and (optionally) install library + tests + cmake --build . -j${PARALLEL_LEVEL} ${INSTALL_TARGET} ${VERBOSE_FLAG} + fi +fi + +# Build and install the pylibwholegraph Python package +if hasArg pylibwholegraph || buildDefault || hasArg all; then + if hasArg --clean; then + cleanPythonDir ${REPODIR}/python/pylibwholegraph + fi + + # setup.py and cmake reference an env var LIBWHOLEGRAPH_DIR to find the + # libwholegraph package (cmake). + # If not set by the user, set it to LIBWHOLEGRAPH_BUILD_DIR + LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR:=${LIBWHOLEGRAPH_BUILD_DIR}} + if ! hasArg --compile-cmd; then + cd ${REPODIR}/python/pylibwholegraph + env LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR} \ + SKBUILD_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}" python ${PYTHON_ARGS_FOR_INSTALL} \ + --config-settings rapidsai.disable-cuda=true \ + . + + else + # just invoke cmake without going through scikit-build-core + env LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR} \ + cmake -S ${REPODIR}/python/pylibwholegraph -B ${REPODIR}/python/pylibwholegraph/build \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} + fi +fi + # Build and install the cugraph-pyg Python package if hasArg cugraph-pyg || buildDefault || hasArg all; then if hasArg --clean; then @@ -173,7 +267,7 @@ if hasArg cugraph-pyg || buildDefault || hasArg all; then fi fi -# Install the cugraph-dgl extensions for DGL +# Build and install the cugraph-dgl Python package if hasArg cugraph-dgl || buildDefault ||hasArg all; then if hasArg --clean; then cleanPythonDir ${REPODIR}/python/cugraph-dgl @@ -181,40 +275,3 @@ if hasArg cugraph-dgl || buildDefault ||hasArg all; then python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-dgl fi fi - -# Build the docs -if hasArg docs || hasArg all; then - if [ ! -d ${LIBCUGRAPH_BUILD_DIR} ]; then - mkdir -p ${LIBCUGRAPH_BUILD_DIR} - cd ${LIBCUGRAPH_BUILD_DIR} - cmake -B "${LIBCUGRAPH_BUILD_DIR}" -S "${REPODIR}/cpp" \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - ${CMAKE_GENERATOR_OPTION} \ - ${CMAKE_VERBOSE_OPTION} - fi - - for PROJECT in libcugraphops libwholegraph; do - XML_DIR="${REPODIR}/docs/cugraph/${PROJECT}" - rm -rf "${XML_DIR}" - mkdir -p "${XML_DIR}" - export XML_DIR_${PROJECT^^}="$XML_DIR" - - echo "downloading xml for ${PROJECT} into ${XML_DIR}. Environment variable XML_DIR_${PROJECT^^} is set to ${XML_DIR}" - curl -O "https://d1664dvumjb44w.cloudfront.net/${PROJECT}/xml_tar/${RAPIDS_VERSION}/xml.tar.gz" - tar -xzf xml.tar.gz -C "${XML_DIR}" - rm "./xml.tar.gz" - done - - cd ${LIBCUGRAPH_BUILD_DIR} - cmake --build "${LIBCUGRAPH_BUILD_DIR}" -j${PARALLEL_LEVEL} --target docs_cugraph ${VERBOSE_FLAG} - - echo "making libcugraph doc dir" - rm -rf ${REPODIR}/docs/cugraph/libcugraph - mkdir -p ${REPODIR}/docs/cugraph/libcugraph - - export XML_DIR_LIBCUGRAPH="${REPODIR}/cpp/doxygen/xml" - - cd ${REPODIR}/docs/cugraph - make html -fi From c57490d7b54454d2c81614f4b1415b731465e27d Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 13:07:22 -0700 Subject: [PATCH 14/52] remove extra repos --- .github/workflows/pr.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0cbe71a..88f860d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -126,9 +126,6 @@ jobs: with: build_type: pull-request script: ci/build_wheel_pylibwholegraph.sh - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-24.12 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 wheel-tests-pylibwholegraph: needs: [wheel-build-pylibwholegraph, changed-files] @@ -177,8 +174,7 @@ jobs: arch: '["amd64"]' cuda: '["12.5"]' node_type: cpu32 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY build_command: | sccache -z; - build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; + build-all --verbose -j$(nproc --ignore=1); sccache -s; From d5571c690497fba31a86b44cea866be184e8925a Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 13:19:38 -0700 Subject: [PATCH 15/52] fixes to dependencies.yaml --- .../all_cuda-118_arch-x86_64.yaml | 8 ++++++- .../all_cuda-121_arch-x86_64.yaml | 8 ++++++- .../all_cuda-124_arch-x86_64.yaml | 8 ++++++- dependencies.yaml | 22 ++++++++++++++++--- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 4380b82..a006610 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -9,20 +9,26 @@ channels: - nvidia dependencies: - breathe +- c-compiler - cmake>=3.26.4,!=3.30.0 - cuda-nvtx - cuda-version=11.8 - cudatoolkit - cudf==24.12.*,>=0.0.0a0 +- cudnn=8.8 - cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 +- cxx-compiler - cython>=3.0.0 - dask-cudf==24.12.*,>=0.0.0a0 - doxygen - graphviz - ipython +- libraft-headers==24.12.*,>=0.0.0a0 +- librmm==24.12.*,>=0.0.0a0 +- nanobind>=0.2.0 - nbsphinx -- ninja +- nccl>=2.19 - notebook>=0.5.0 - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index a868c5c..8fd6284 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -9,14 +9,17 @@ channels: - nvidia dependencies: - breathe +- c-compiler - cmake>=3.26.4,!=3.30.0 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.1 - cudf==24.12.*,>=0.0.0a0 +- cudnn=8.8 - cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 +- cxx-compiler - cython>=3.0.0 - dask-cudf==24.12.*,>=0.0.0a0 - doxygen @@ -26,8 +29,11 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev +- libraft-headers==24.12.*,>=0.0.0a0 +- librmm==24.12.*,>=0.0.0a0 +- nanobind>=0.2.0 - nbsphinx -- ninja +- nccl>=2.19 - notebook>=0.5.0 - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/conda/environments/all_cuda-124_arch-x86_64.yaml b/conda/environments/all_cuda-124_arch-x86_64.yaml index 20dcef6..32ea733 100644 --- a/conda/environments/all_cuda-124_arch-x86_64.yaml +++ b/conda/environments/all_cuda-124_arch-x86_64.yaml @@ -9,14 +9,17 @@ channels: - nvidia dependencies: - breathe +- c-compiler - cmake>=3.26.4,!=3.30.0 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.4 - cudf==24.12.*,>=0.0.0a0 +- cudnn=8.8 - cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 +- cxx-compiler - cython>=3.0.0 - dask-cudf==24.12.*,>=0.0.0a0 - doxygen @@ -26,8 +29,11 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev +- libraft-headers==24.12.*,>=0.0.0a0 +- librmm==24.12.*,>=0.0.0a0 +- nanobind>=0.2.0 - nbsphinx -- ninja +- nccl>=2.19 - notebook>=0.5.0 - numba>=0.57 - numpy>=1.23,<3.0a0 diff --git a/dependencies.yaml b/dependencies.yaml index 29ceca6..f349ec8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -204,10 +204,20 @@ dependencies: - cuda-nvtx common_build: common: - - output_types: [conda, pyproject] + - output_types: [requirements, pyproject] packages: - - &cmake_ver cmake>=3.26.4,!=3.30.0 - ninja + - output_types: [conda] + packages: + - &cmake_ver cmake>=3.26.4,!=3.30.0 + - cudnn=8.8 + - c-compiler + - cxx-compiler + - libraft-headers==24.12.*,>=0.0.0a0 + - librmm==24.12.*,>=0.0.0a0 + - nanobind>=0.2.0 + - &cython cython>=3.0.0 + - &nccl nccl>=2.19 docs: common: @@ -254,7 +264,7 @@ dependencies: common: - output_types: [conda, pyproject, requirements] packages: - - cython>=3.0.0 + - *cython - output_types: conda packages: - scikit-build-core>=0.10.0 @@ -288,6 +298,12 @@ dependencies: - output_types: [conda] packages: - wget + test_cpp: + common: + - output_types: conda + packages: + - *cmake_ver + - *nccl test_python_common: common: - output_types: [conda, pyproject] From b99122d956518370b1e00998773501df7f9af8eb Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 13:22:27 -0700 Subject: [PATCH 16/52] fix --- ci/build_python.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index bfd890d..7c6cb10 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -27,7 +27,6 @@ for package_name in pylibwholegraph cugraph-pyg cugraph-dgl; do underscore_package_name=$(echo "${package_name}" | tr "-" "_") sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" done -sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/nx-cugraph/_nx_cugraph/_version.py" # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly From 164d1a2c3c9be114e6ea8d9c0015ef47b0f8dbc7 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 13:22:49 -0700 Subject: [PATCH 17/52] update gitignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 2fea102..9480c26 100644 --- a/.gitignore +++ b/.gitignore @@ -78,9 +78,6 @@ datasets/* !datasets/karate-disjoint.csv !datasets/netscience.csv -# nx-cugraph side effects -python/nx-cugraph/objects.inv - .pydevproject # Jupyter Notebooks From 8f303cccf79e6bfe387b26ce04a768adc4c36afc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 11 Oct 2024 15:49:02 -0500 Subject: [PATCH 18/52] bump NCCL floor to 2.19 --- conda/recipes/libwholegraph/conda_build_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libwholegraph/conda_build_config.yaml b/conda/recipes/libwholegraph/conda_build_config.yaml index 35b1d6b..ebb154c 100644 --- a/conda/recipes/libwholegraph/conda_build_config.yaml +++ b/conda/recipes/libwholegraph/conda_build_config.yaml @@ -17,7 +17,7 @@ doxygen_version: - ">=1.8.11" nccl_version: - - ">=2.9.9" + - ">=2.19" c_stdlib: - sysroot From 96684a03c8ed640ae543106b8cd35e3197737eb9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 11 Oct 2024 15:49:35 -0500 Subject: [PATCH 19/52] package license --- conda/recipes/pylibwholegraph/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/pylibwholegraph/meta.yaml b/conda/recipes/pylibwholegraph/meta.yaml index d3f9a49..8149fdc 100644 --- a/conda/recipes/pylibwholegraph/meta.yaml +++ b/conda/recipes/pylibwholegraph/meta.yaml @@ -75,4 +75,6 @@ requirements: about: home: https://rapids.ai/ + license: Apache-2.0 + license_file: ../../../LICENSE summary: pylibwholegraph library From 58806fd475da1a5311afd03cf6c60bd23088e355 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 11 Oct 2024 15:53:39 -0500 Subject: [PATCH 20/52] pytorch pins, some CI script cleanup --- ci/test_notebooks.sh | 6 +++++- ci/test_python.sh | 20 +++++++++++--------- ci/test_wheel.sh | 1 - ci/test_wheel_cugraph-dgl.sh | 1 - ci/test_wheel_cugraph-pyg.sh | 1 - conda/recipes/cugraph-dgl/meta.yaml | 4 ++-- conda/recipes/cugraph-pyg/meta.yaml | 4 ++-- 7 files changed, 20 insertions(+), 17 deletions(-) diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 31ec560..59b60e4 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -5,6 +5,8 @@ set -Eeuo pipefail . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" + rapids-logger "Generate notebook testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -27,7 +29,9 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libcugraph pylibcugraph cugraph + "libcugraph=${RAPIDS_VERSION}" \ + "pylibcugraph=${RAPIDS_VERSION}" \ + "cugraph=${RAPIDS_VERSION}" NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" NOTEBOOK_LIST="$(realpath "$(dirname "$0")/notebook_list.py")" diff --git a/ci/test_python.sh b/ci/test_python.sh index 7e1be3a..9fec454 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" + rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -59,10 +61,10 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then --channel conda-forge \ --channel dglteam/label/th23_cu118 \ --channel nvidia \ - "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "pylibcugraphops=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "cugraph-dgl=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "pylibwholegraph=${RAPIDS_VERSION}.*" \ + "pylibcugraphops=${RAPIDS_VERSION}.*" \ + "cugraph=${RAPIDS_VERSION}.*" \ + "cugraph-dgl=${RAPIDS_VERSION}.*" \ 'pytorch::pytorch>=2.3,<2.4' \ 'cuda-version=11.8' \ "ogb" @@ -111,10 +113,10 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ --channel pytorch \ - "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "pylibcugraphops=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "cugraph-pyg=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "pylibwholegraph=${RAPIDS_VERSION}.*" \ + "pylibcugraphops=${RAPIDS_VERSION}.*" \ + "cugraph=${RAPIDS_VERSION}.*" \ + "cugraph-pyg=${RAPIDS_VERSION}.*" \ "pytorch::pytorch>=2.3,<2.4" \ "ogb" @@ -160,7 +162,7 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ --channel pytorch \ - "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ + "pylibwholegraph=${RAPIDS_VERSION}.*" \ "pytorch::pytorch>=2.3,<2.4" \ "ogb" diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 39d2c1f..25a59fd 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -4,7 +4,6 @@ set -eoxu pipefail package_name=$1 -package_dir=$2 python_package_name=$(echo ${package_name}|sed 's/-/_/g') diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 564b46c..1dbf44f 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -4,7 +4,6 @@ set -eoxu pipefail package_name="cugraph-dgl" -package_dir="python/cugraph-dgl" python_package_name=$(echo ${package_name}|sed 's/-/_/g') diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 1004063..1dc94bb 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -4,7 +4,6 @@ set -eoxu pipefail package_name="cugraph-pyg" -package_dir="python/cugraph-pyg" python_package_name=$(echo ${package_name}|sed 's/-/_/g') diff --git a/conda/recipes/cugraph-dgl/meta.yaml b/conda/recipes/cugraph-dgl/meta.yaml index 3c90075..86bbd4a 100644 --- a/conda/recipes/cugraph-dgl/meta.yaml +++ b/conda/recipes/cugraph-dgl/meta.yaml @@ -23,13 +23,13 @@ requirements: - python run: - cugraph ={{ version }} - - dgl >=1.1.0.cu* + - dgl >=2.4.0.th23.cu* - numba >=0.57 - numpy >=1.23,<3.0a0 - pylibcugraphops ={{ minor_version }} - tensordict >=0.1.2 - python - - pytorch >=2.0 + - pytorch >=2.3,<2.4.0a0 - cupy >=12.0.0 tests: diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 5bce780..27ccaa4 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -30,12 +30,12 @@ requirements: - numba >=0.57 - numpy >=1.23,<3.0a0 - python - - pytorch >=2.0 + - pytorch >=2.3,<2.4.0a0 - cupy >=12.0.0 - cugraph ={{ version }} - pylibcugraphops ={{ minor_version }} - tensordict >=0.1.2 - - pyg >=2.5,<2.6 + - pytorch_geometric >=2.5,<2.6 tests: imports: From 66bcde6b09b4e0bdcaab5ffa262218f2650093cc Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 13:54:09 -0700 Subject: [PATCH 21/52] set cuda version 12.5 --- ci/build_wheel.sh | 5 +++-- dependencies.yaml | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index f9b2489..82ea83c 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -54,16 +54,17 @@ fi cd "${package_dir}" -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +WHEEL_BUILD_ARGS = "-m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check" # pure-python packages should be marked as pure, and not have auditwheel run on them. if [[ ${package_name} == "cugraph-dgl" ]] || \ [[ ${package_name} == "cugraph-pyg" ]]; then + python ${WHEEL_BUILD_ARGS} RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist else # Hardcode the output dir SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DBUILD_SHARED_LIBS=OFF;-DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE;-DCUDA_STATIC_RUNTIME=ON;-DWHOLEGRAPH_BUILD_WHEELS=ON" \ - python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + python ${WHEEL_BUILD_ARGS} mkdir -p final_dist python -m auditwheel repair \ diff --git a/dependencies.yaml b/dependencies.yaml index f349ec8..0859e39 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -183,6 +183,10 @@ dependencies: cuda: "12.4" packages: - cuda-version=12.4 + - matrix: + cuda: "12.5" + packages: + - cuda-version=12.5 cuda: specific: - output_types: [conda] From 5db48ab2b2ef3b00cb58443bbcb320db944e0274 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 11 Oct 2024 15:55:50 -0500 Subject: [PATCH 22/52] update cugraph-dgl and cugrpah-pyg wheel-testing scripts to match what is in the cugraph repo --- ci/test_wheel_cugraph-dgl.sh | 38 ++++++++++----------------- ci/test_wheel_cugraph-pyg.sh | 51 ++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 50 deletions(-) diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 1dbf44f..d7558d4 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -5,22 +5,15 @@ set -eoxu pipefail package_name="cugraph-dgl" -python_package_name=$(echo ${package_name}|sed 's/-/_/g') - mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download wheels built during this job. +# Download the pylibcugraph, cugraph, and cugraph-dgl built in the previous step RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -python -m pip install ./local-deps/*.whl - -# use 'ls' to expand wildcard before adding `[extra]` requires for pip RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] - +# determine pytorch and DGL sources PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then @@ -29,20 +22,17 @@ else PYTORCH_CUDA_VER=$PKG_CUDA_VER fi PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}" -DGL_URL="https://data.dgl.ai/wheels/cu${PYTORCH_CUDA_VER}/repo.html" - -# Starting from 2.2, PyTorch wheels depend on nvidia-nccl-cuxx>=2.19 wheel and -# dynamically link to NCCL. RAPIDS CUDA 11 CI images have an older NCCL version that -# might shadow the newer NCCL required by PyTorch during import (when importing -# `cupy` before `torch`). -if [[ "${NCCL_VERSION}" < "2.19" ]]; then - PYTORCH_VER="2.1.0" -else - PYTORCH_VER="2.3.0" -fi - -rapids-logger "Installing PyTorch and DGL" -rapids-retry python -m pip install "torch==${PYTORCH_VER}" --index-url ${PYTORCH_URL} -rapids-retry python -m pip install dgl==2.0.0 --find-links ${DGL_URL} +DGL_URL="https://data.dgl.ai/wheels/torch-2.3/cu${PYTORCH_CUDA_VER}/repo.html" + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + -v \ + --extra-index-url "${PYTORCH_URL}" \ + --find-links "${DGL_URL}" \ + "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + 'dgl==2.4.0' \ + 'torch>=2.3.0,<2.4' python -m pytest python/cugraph-dgl/tests diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 1dc94bb..2f508ee 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -5,44 +5,43 @@ set -eoxu pipefail package_name="cugraph-pyg" -python_package_name=$(echo ${package_name}|sed 's/-/_/g') - mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download wheels built during this job. +# Download the pylibcugraph, cugraph, and cugraph-pyg built in the previous step RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -python -m pip install ./local-deps/*.whl - -# use 'ls' to expand wildcard before adding `[extra]` requires for pip RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] - -# RAPIDS_DATASET_ROOT_DIR is used by test scripts -export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" - -# Used to skip certain examples in CI due to memory limitations -export CI_RUN=1 +# determine pytorch and pyg sources if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then PYTORCH_URL="https://download.pytorch.org/whl/cu118" - PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html" + PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html" else PYTORCH_URL="https://download.pytorch.org/whl/cu121" - PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html" + PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html" fi -rapids-logger "Installing PyTorch and PyG dependencies" -rapids-retry python -m pip install torch==2.1.0 --index-url ${PYTORCH_URL} -rapids-retry python -m pip install "torch-geometric>=2.5,<2.6" -rapids-retry python -m pip install \ - ogb \ - pyg_lib \ - torch_scatter \ - torch_sparse \ - tensordict \ - -f ${PYG_URL} + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + -v \ + --extra-index-url "${PYTORCH_URL}" \ + --find-links "${PYG_URL}" \ + "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + 'ogb' \ + 'pyg_lib' \ + 'torch>=2.3.0,<2.4' \ + 'torch-geometric>=2.5,<2.6' \ + 'torch_scatter' \ + 'torch_sparse' + +# RAPIDS_DATASET_ROOT_DIR is used by test scripts +export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" + +# Used to skip certain examples in CI due to memory limitations +export CI_RUN=1 rapids-logger "pytest cugraph-pyg (single GPU)" pushd python/cugraph-pyg/cugraph_pyg From b851de4b0384ce6a6824523d51c052c176c4a848 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 14:09:20 -0700 Subject: [PATCH 23/52] fix wheel build args --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 82ea83c..5be8d63 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -54,7 +54,7 @@ fi cd "${package_dir}" -WHEEL_BUILD_ARGS = "-m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check" +WHEEL_BUILD_ARGS="-m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check" # pure-python packages should be marked as pure, and not have auditwheel run on them. if [[ ${package_name} == "cugraph-dgl" ]] || \ From 8ecc9ba425e6d9972e81853ee6d66c1fb141f156 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 14:36:55 -0700 Subject: [PATCH 24/52] remove old steps from wheel build --- ci/build_wheel.sh | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 5be8d63..08b4d54 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -10,48 +10,10 @@ underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) +rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -version_package_name="$underscore_package_name" - -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${version_package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -for dep in rmm cudf cugraph raft-dask pylibcugraph pylibcugraphops pylibwholegraph pylibraft ucx-py; do - sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -done - -# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec. -for dep in dask-cuda rapids-dask-dependency; do - sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} -done - - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi - cd "${package_dir}" WHEEL_BUILD_ARGS="-m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check" From 7676b5d4c8e3c00132193e25c99dff5b3807f9dc Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 14:42:37 -0700 Subject: [PATCH 25/52] remove unused cmake args --- conda/recipes/pylibwholegraph/build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conda/recipes/pylibwholegraph/build.sh b/conda/recipes/pylibwholegraph/build.sh index cd61f90..3a7eb7c 100644 --- a/conda/recipes/pylibwholegraph/build.sh +++ b/conda/recipes/pylibwholegraph/build.sh @@ -1,6 +1,4 @@ #!/usr/bin/env bash # Copyright (c) 2022-2024, NVIDIA CORPORATION. -CMAKE_EXTRA_ARGS="--cmake-args=\"-DBUILD_OPS_WITH_TORCH_C10_API=OFF\"" - -./build.sh pylibwholegraph --allgpuarch -v ${CMAKE_EXTRA_ARGS} +./build.sh pylibwholegraph --allgpuarch -v From afef793b061382004a875d5e0dbe1cbe62350298 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 15:00:16 -0700 Subject: [PATCH 26/52] pull the right branch of raft --- cpp/cmake/thirdparty/get_raft.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index e542e7f..7855603 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -14,8 +14,8 @@ # limitations under the License. #============================================================================= -set(WHOLEGRAPH_MIN_VERSION_raft "${RAPIDS_VERSION}.00") -set(WHOLEGRAPH_BRANCH_VERSION_raft "${RAPIDS_VERSION}") +set(WHOLEGRAPH_MIN_VERSION_raft "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}.00") +set(WHOLEGRAPH_BRANCH_VERSION_raft "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}") function(find_and_configure_raft) From 201aee96d9f327b7ec8b965ba89970831f092a20 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 15:55:37 -0700 Subject: [PATCH 27/52] install pylibwholegraph --- ci/test_wheel_pylibwholegraph.sh | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 ci/test_wheel_pylibwholegraph.sh diff --git a/ci/test_wheel_pylibwholegraph.sh b/ci/test_wheel_pylibwholegraph.sh new file mode 100644 index 0000000..bc44ac9 --- /dev/null +++ b/ci/test_wheel_pylibwholegraph.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -e # abort the script on error +set -o pipefail # piped commands propagate their error +set -E # ERR traps are inherited by subcommands + +mkdir -p ./dist +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps + +PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" +PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} +if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then + INDEX_URL="https://download.pytorch.org/whl/cu121" +else + INDEX_URL="https://download.pytorch.org/whl/cu${PKG_CUDA_VER}" +fi +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} +RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} +mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" + +# echo to expand wildcard before adding `[extra]` requires for pip +rapids-logger "Installing Packages" +rapids-retry python -m pip install \ + --extra-index-url ${INDEX_URL} \ + $(echo ./dist/pylibwholegraph*.whl) \ + pytest \ + pytest-forked \ + numpy + +rapids-logger "pytest pylibwholegraph" +cd python/pylibwholegraph/pylibwholegraph/tests +python -m pytest \ + --cache-clear \ + --forked \ + --import-mode=append \ + . From 0fbe438827572a7898a15224afd78868e5ed90cf Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 15:56:48 -0700 Subject: [PATCH 28/52] remove pylibcugraph reference --- dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 0859e39..8a11b4e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -59,7 +59,6 @@ files: - depends_on_cudf - py_version - test_python_common - - test_python_pylibcugraph py_build_cugraph_dgl: output: pyproject @@ -211,6 +210,7 @@ dependencies: - output_types: [requirements, pyproject] packages: - ninja + - output_types: [conda] packages: - &cmake_ver cmake>=3.26.4,!=3.30.0 From 1d04ad0d549a2426d8765962fe91a14f902e6d63 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 15:59:26 -0700 Subject: [PATCH 29/52] add smoke test notebook --- notebooks/notebook_smoke_test.ipynb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 notebooks/notebook_smoke_test.ipynb diff --git a/notebooks/notebook_smoke_test.ipynb b/notebooks/notebook_smoke_test.ipynb new file mode 100644 index 0000000..8d455fc --- /dev/null +++ b/notebooks/notebook_smoke_test.ipynb @@ -0,0 +1,20 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import cugraph" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From d4809465fff5501af3a4d0d9d05af691708e2082 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 11 Oct 2024 16:02:02 -0700 Subject: [PATCH 30/52] disable cpp checks --- .github/workflows/pr.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 88f860d..f65d846 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,7 +16,7 @@ jobs: - checks - conda-cpp-build - conda-cpp-tests - - conda-cpp-checks + # - conda-cpp-checks - conda-notebook-tests - conda-python-build - conda-python-tests @@ -87,14 +87,14 @@ jobs: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) + # conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + # with: + # build_type: pull-request + # enable_check_symbols: true + # symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) conda-python-build: needs: conda-cpp-build secrets: inherit From 738f6c44e495db2773c6791989fea4c39ab35fab Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 08:37:31 -0700 Subject: [PATCH 31/52] fix tests --- ci/test_python.sh | 1 + ci/test_wheel_pylibwholegraph.sh | 0 2 files changed, 1 insertion(+) mode change 100644 => 100755 ci/test_wheel_pylibwholegraph.sh diff --git a/ci/test_python.sh b/ci/test_python.sh index 9fec454..65baf36 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -26,6 +26,7 @@ mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" # RAPIDS_DATASET_ROOT_DIR is used by test scripts export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" +mkdir -p "${RAPIDS_DATASET_ROOT_DIR}" pushd "${RAPIDS_DATASET_ROOT_DIR}" ./get_test_data.sh --benchmark popd diff --git a/ci/test_wheel_pylibwholegraph.sh b/ci/test_wheel_pylibwholegraph.sh old mode 100644 new mode 100755 From b6e61c2c4869365504a2c1b20c27057f35f91e23 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 08:40:24 -0700 Subject: [PATCH 32/52] move notebook to inner directory --- notebooks/debug/notebook_smoke_test.ipynb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 notebooks/debug/notebook_smoke_test.ipynb diff --git a/notebooks/debug/notebook_smoke_test.ipynb b/notebooks/debug/notebook_smoke_test.ipynb new file mode 100644 index 0000000..8d455fc --- /dev/null +++ b/notebooks/debug/notebook_smoke_test.ipynb @@ -0,0 +1,20 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import cugraph" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From c278d9726ac5f13fd47b9192e3b91e4067e50800 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 10:16:27 -0700 Subject: [PATCH 33/52] add dataset script --- ci/test_wheel_pylibwholegraph.sh | 2 +- datasets/get_test_data.sh | 128 ++++++++++++++++++++++++++++ notebooks/notebook_smoke_test.ipynb | 20 ----- 3 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 datasets/get_test_data.sh delete mode 100644 notebooks/notebook_smoke_test.ipynb diff --git a/ci/test_wheel_pylibwholegraph.sh b/ci/test_wheel_pylibwholegraph.sh index bc44ac9..52cb706 100755 --- a/ci/test_wheel_pylibwholegraph.sh +++ b/ci/test_wheel_pylibwholegraph.sh @@ -7,7 +7,7 @@ set -E # ERR traps are inherited by subcommands mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps +RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} diff --git a/datasets/get_test_data.sh b/datasets/get_test_data.sh new file mode 100644 index 0000000..6778166 --- /dev/null +++ b/datasets/get_test_data.sh @@ -0,0 +1,128 @@ +# Copyright (c) 2021-2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -e +set -o pipefail + +# Ensure we're in the cugraph/datasets dir +cd "$( cd "$( dirname "$(realpath -m "${BASH_SOURCE[0]}")" )" && pwd )"; + +# Update this to add/remove/change a dataset, using the following format: +# +# comment about the dataset +# dataset download URL +# destination dir to untar to +# blank line separator +# +# FIXME: some test data needs to be extracted to "benchmarks", which is +# confusing now that there's dedicated datasets for benchmarks. +CPP_CI_DATASET_DATA=" +# ~10s download +https://data.rapids.ai/cugraph/test/cpp_ci_datasets.tgz +test +" + +BASE_DATASET_DATA=" +# ~22s download +https://data.rapids.ai/cugraph/test/datasets.tgz +test + +# ~14s download +https://data.rapids.ai/cugraph/test/ref/pagerank.tgz +test/ref + +# ~1s download +https://data.rapids.ai/cugraph/test/ref/sssp.tgz +test/ref + +# ~15s download +https://data.rapids.ai/cugraph/benchmark/hibench/hibench_1_large.tgz +benchmark + +# ~1s download +https://data.rapids.ai/cugraph/benchmark/hibench/hibench_1_small.tgz +benchmark + +# ~0.6s download +https://data.rapids.ai/cugraph/test/tsplib/datasets.tar.gz +tsplib +" + +EXTENDED_DATASET_DATA=" +# ~42s download - tests using this dataset are currently not run in test.sh with --quick +https://data.rapids.ai/cugraph/benchmark/hibench/hibench_1_huge.tgz +benchmark +" + +BENCHMARK_DATASET_DATA=" +# ~90s download - these are used for benchmarks runs (code in /benchmarks) +https://data.rapids.ai/cugraph/benchmark/benchmark_csv_data.tgz +csv +" + +SELF_LOOPS_DATASET_DATA=" +# ~1s download +https://data.rapids.ai/cugraph/benchmark/benchmark_csv_data_self_loops.tgz +self_loops +" +################################################################################ +# Do not change the script below this line if only adding/updating a dataset + +NUMARGS=$# +ARGS=$* +function hasArg { + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +if hasArg -h || hasArg --help; then + echo "$0 [--subset | --benchmark | --self_loops]" + exit 0 +fi + +# Select the datasets to install +if hasArg "--benchmark"; then + DATASET_DATA="${BENCHMARK_DATASET_DATA}" +elif hasArg "--subset"; then + DATASET_DATA="${BASE_DATASET_DATA}" +elif hasArg "--cpp_ci_subset"; then + DATASET_DATA="${CPP_CI_DATASET_DATA}" +elif hasArg "--self_loops"; then + DATASET_DATA="${SELF_LOOPS_DATASET_DATA}" +# Do not include benchmark datasets by default - too big +else + DATASET_DATA="${BASE_DATASET_DATA} ${EXTENDED_DATASET_DATA}" +fi + +URLS=($(echo "$DATASET_DATA"|awk '{if (NR%4 == 3) print $0}')) # extract 3rd fields to a bash array +DESTDIRS=($(echo "$DATASET_DATA"|awk '{if (NR%4 == 0) print $0}')) # extract 4th fields to a bash array + +echo Downloading ... + +# Download all tarfiles to a tmp dir +mkdir -p tmp +cd tmp +for url in ${URLS[*]}; do + time wget -N --progress=dot:giga ${url} +done +cd .. + +# create the destination dirs +mkdir -p "${DESTDIRS[@]}" + +# Iterate over the arrays and untar the nth tarfile to the nth dest directory. +# The tarfile name is derived from the download url. +echo Decompressing ... +for index in ${!DESTDIRS[*]}; do + echo "tmp/$(basename "${URLS[$index]}") -C ${DESTDIRS[$index]}" | tr '\n' '\0' +done | xargs -0 -t -r -n1 -P$(nproc --all) sh -c 'tar -xzvf $0 --overwrite' diff --git a/notebooks/notebook_smoke_test.ipynb b/notebooks/notebook_smoke_test.ipynb deleted file mode 100644 index 8d455fc..0000000 --- a/notebooks/notebook_smoke_test.ipynb +++ /dev/null @@ -1,20 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import cugraph" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From a87bebdc0b1b410a0318601e2f80bda9e066cd87 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 10:42:39 -0700 Subject: [PATCH 34/52] set perms on script --- datasets/get_test_data.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 datasets/get_test_data.sh diff --git a/datasets/get_test_data.sh b/datasets/get_test_data.sh old mode 100644 new mode 100755 From 09cc676e7603df96da0518c35e0f41c86f4538c2 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 10:43:45 -0700 Subject: [PATCH 35/52] add torch to test script --- ci/test_wheel_pylibwholegraph.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/test_wheel_pylibwholegraph.sh b/ci/test_wheel_pylibwholegraph.sh index 52cb706..60598bc 100755 --- a/ci/test_wheel_pylibwholegraph.sh +++ b/ci/test_wheel_pylibwholegraph.sh @@ -27,7 +27,8 @@ rapids-retry python -m pip install \ $(echo ./dist/pylibwholegraph*.whl) \ pytest \ pytest-forked \ - numpy + numpy \ + 'torch>=2.3.0,<2.4' rapids-logger "pytest pylibwholegraph" cd python/pylibwholegraph/pylibwholegraph/tests From c915472b2d9bfcf93fa822a5bde0ab12a584d9a8 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 11:24:16 -0700 Subject: [PATCH 36/52] bump pytorch to 12.4 --- ci/test_wheel_cugraph-dgl.sh | 2 +- ci/test_wheel_cugraph-pyg.sh | 4 ++-- ci/test_wheel_pylibwholegraph.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index d7558d4..6483840 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -17,7 +17,7 @@ RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_ PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then - PYTORCH_CUDA_VER="121" + PYTORCH_CUDA_VER="124" else PYTORCH_CUDA_VER=$PKG_CUDA_VER fi diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 2f508ee..972a761 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -18,8 +18,8 @@ if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then PYTORCH_URL="https://download.pytorch.org/whl/cu118" PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html" else - PYTORCH_URL="https://download.pytorch.org/whl/cu121" - PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html" + PYTORCH_URL="https://download.pytorch.org/whl/cu124" + PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu124.html" fi # echo to expand wildcard before adding `[extra]` requires for pip diff --git a/ci/test_wheel_pylibwholegraph.sh b/ci/test_wheel_pylibwholegraph.sh index 60598bc..518da45 100755 --- a/ci/test_wheel_pylibwholegraph.sh +++ b/ci/test_wheel_pylibwholegraph.sh @@ -12,7 +12,7 @@ RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download- PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then - INDEX_URL="https://download.pytorch.org/whl/cu121" + INDEX_URL="https://download.pytorch.org/whl/cu124" else INDEX_URL="https://download.pytorch.org/whl/cu${PKG_CUDA_VER}" fi From a78dc8d062ccf7bcb5a74b4916f0c28b83b6cb9d Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 12:25:28 -0700 Subject: [PATCH 37/52] fix git config, coverage --- .github/workflows/test.yaml | 2 +- ci/test_python.sh | 27 ++++----------------------- python/.coveragerc | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 24 deletions(-) create mode 100644 python/.coveragerc diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 13c0d44..cacb96b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -40,7 +40,7 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - wheel-tests-pylibcugraph: + wheel-tests-pylibwholegraph: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: diff --git a/ci/test_python.sh b/ci/test_python.sh index 65baf36..d7cb28d 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -86,7 +86,6 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then # Reactivate the test environment back set +u conda deactivate - conda activate test set -u else rapids-logger "skipping cugraph_dgl pytest on ARM64" @@ -100,15 +99,6 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then conda activate test_cugraph_pyg set -u - # TODO re-enable logic once CUDA 12 is testable - #if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - CONDA_CUDA_VERSION="11.8" - PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html" - #else - # CONDA_CUDA_VERSION="12.1" - # PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html" - #fi - # Will automatically install built dependencies of cuGraph-PyG rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ @@ -137,7 +127,6 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then # Reactivate the test environment back set +u conda deactivate - conda activate test set -u else rapids-logger "skipping cugraph_pyg pytest on ARM64" @@ -151,13 +140,6 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then conda activate test_pylibwholegraph set -u - # TODO re-enable logic once CUDA 12 is testable - #if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - CONDA_CUDA_VERSION="11.8" - #else - # CONDA_CUDA_VERSION="12.1" - #fi - # Will automatically install built dependencies of pylibwholegraph rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ @@ -172,18 +154,17 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then rapids-logger "Check GPU usage" nvidia-smi - rapids-logger "pytest cugraph_pyg (single GPU)" - ./ci/run_cugraph_pyg_pytests.sh \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-pyg.xml" \ + rapids-logger "pytest pylibwholegraph (single GPU)" + ./ci/run_pylibwholegraph_pytests.sh \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-pylibwholegraph.xml" \ --cov-config=../../.coveragerc \ --cov=cugraph_pyg \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-pyg-coverage.xml" \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/pylibwholegraph-coverage.xml" \ --cov-report=term # Reactivate the test environment back set +u conda deactivate - conda activate test set -u else rapids-logger "skipping cugraph_pyg pytest on ARM64" diff --git a/python/.coveragerc b/python/.coveragerc new file mode 100644 index 0000000..d24d634 --- /dev/null +++ b/python/.coveragerc @@ -0,0 +1,21 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration file for Python coverage tests +[run] +include = cugraph-dgl/cugraph_dgl/* + cugraph-pyg/cugraph_pyg/* + pylibwholegraph/pylibwholegraph/* +omit = cugraph-dgl/cugraph_dgl/tests/* + cugraph-pyg/cugraph_pyg/tests/* + pylibwholegraph/pylibwholegraph/tests/* From 46f0323d8e3fe1ec433aec63b784667992d04dc3 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 12:40:08 -0700 Subject: [PATCH 38/52] update matrix filter for pylibwholegraph wheel tests --- .github/workflows/pr.yaml | 1 + .github/workflows/test.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f65d846..ceed525 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -135,6 +135,7 @@ jobs: with: build_type: pull-request script: ci/test_wheel_pylibwholegraph.sh + matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-dgl: needs: wheel-tests-pylibwholegraph secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cacb96b..187f4b4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -49,6 +49,7 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_pylibwholegraph.sh + matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-dgl: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 From 4c4d3c0f1c5dab4ae5f59368cc0d93abeb84a42e Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 14 Oct 2024 14:43:52 -0700 Subject: [PATCH 39/52] only install build wheels, add karate file --- ci/test_wheel_cugraph-dgl.sh | 8 +- ci/test_wheel_cugraph-pyg.sh | 8 +- datasets/karate.csv | 156 +++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 datasets/karate.csv diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 6483840..733b851 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -8,9 +8,8 @@ package_name="cugraph-dgl" mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the pylibcugraph, cugraph, and cugraph-dgl built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps +# Download the pylibwholegraph and cugraph-dgl built in the previous step +RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # determine pytorch and DGL sources @@ -29,8 +28,7 @@ python -m pip install \ -v \ --extra-index-url "${PYTORCH_URL}" \ --find-links "${DGL_URL}" \ - "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ 'dgl==2.4.0' \ 'torch>=2.3.0,<2.4' diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 972a761..a3a424c 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -8,9 +8,8 @@ package_name="cugraph-pyg" mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the pylibcugraph, cugraph, and cugraph-pyg built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps +# Download the pylibwholegraph and cugraph-pyg built in the previous step +RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # determine pytorch and pyg sources @@ -27,8 +26,7 @@ python -m pip install \ -v \ --extra-index-url "${PYTORCH_URL}" \ --find-links "${PYG_URL}" \ - "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ 'ogb' \ 'pyg_lib' \ diff --git a/datasets/karate.csv b/datasets/karate.csv new file mode 100644 index 0000000..4ed9f43 --- /dev/null +++ b/datasets/karate.csv @@ -0,0 +1,156 @@ +1 0 1.0 +2 0 1.0 +3 0 1.0 +4 0 1.0 +5 0 1.0 +6 0 1.0 +7 0 1.0 +8 0 1.0 +10 0 1.0 +11 0 1.0 +12 0 1.0 +13 0 1.0 +17 0 1.0 +19 0 1.0 +21 0 1.0 +31 0 1.0 +2 1 1.0 +3 1 1.0 +7 1 1.0 +13 1 1.0 +17 1 1.0 +19 1 1.0 +21 1 1.0 +30 1 1.0 +3 2 1.0 +7 2 1.0 +8 2 1.0 +9 2 1.0 +13 2 1.0 +27 2 1.0 +28 2 1.0 +32 2 1.0 +7 3 1.0 +12 3 1.0 +13 3 1.0 +6 4 1.0 +10 4 1.0 +6 5 1.0 +10 5 1.0 +16 5 1.0 +16 6 1.0 +30 8 1.0 +32 8 1.0 +33 8 1.0 +33 9 1.0 +33 13 1.0 +32 14 1.0 +33 14 1.0 +32 15 1.0 +33 15 1.0 +32 18 1.0 +33 18 1.0 +33 19 1.0 +32 20 1.0 +33 20 1.0 +32 22 1.0 +33 22 1.0 +25 23 1.0 +27 23 1.0 +29 23 1.0 +32 23 1.0 +33 23 1.0 +25 24 1.0 +27 24 1.0 +31 24 1.0 +31 25 1.0 +29 26 1.0 +33 26 1.0 +33 27 1.0 +31 28 1.0 +33 28 1.0 +32 29 1.0 +33 29 1.0 +32 30 1.0 +33 30 1.0 +32 31 1.0 +33 31 1.0 +33 32 1.0 +0 1 1.0 +0 2 1.0 +0 3 1.0 +0 4 1.0 +0 5 1.0 +0 6 1.0 +0 7 1.0 +0 8 1.0 +0 10 1.0 +0 11 1.0 +0 12 1.0 +0 13 1.0 +0 17 1.0 +0 19 1.0 +0 21 1.0 +0 31 1.0 +1 2 1.0 +1 3 1.0 +1 7 1.0 +1 13 1.0 +1 17 1.0 +1 19 1.0 +1 21 1.0 +1 30 1.0 +2 3 1.0 +2 7 1.0 +2 8 1.0 +2 9 1.0 +2 13 1.0 +2 27 1.0 +2 28 1.0 +2 32 1.0 +3 7 1.0 +3 12 1.0 +3 13 1.0 +4 6 1.0 +4 10 1.0 +5 6 1.0 +5 10 1.0 +5 16 1.0 +6 16 1.0 +8 30 1.0 +8 32 1.0 +8 33 1.0 +9 33 1.0 +13 33 1.0 +14 32 1.0 +14 33 1.0 +15 32 1.0 +15 33 1.0 +18 32 1.0 +18 33 1.0 +19 33 1.0 +20 32 1.0 +20 33 1.0 +22 32 1.0 +22 33 1.0 +23 25 1.0 +23 27 1.0 +23 29 1.0 +23 32 1.0 +23 33 1.0 +24 25 1.0 +24 27 1.0 +24 31 1.0 +25 31 1.0 +26 29 1.0 +26 33 1.0 +27 33 1.0 +28 31 1.0 +28 33 1.0 +29 32 1.0 +29 33 1.0 +30 32 1.0 +30 33 1.0 +31 32 1.0 +31 33 1.0 +32 33 1.0 From 065c5df2eda3900069ebacd297fd7846012fc8cc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 12:13:23 -0500 Subject: [PATCH 40/52] add pytest-forked, try to fix wheel-testing scripts --- ci/build_wheel.sh | 11 ++++++----- ci/test_wheel.sh | 6 +++--- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-121_arch-x86_64.yaml | 1 + conda/environments/all_cuda-124_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + .../cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml | 1 + python/cugraph-dgl/pyproject.toml | 1 + .../cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml | 1 + python/cugraph-pyg/pyproject.toml | 1 + 10 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 08b4d54..bc20be7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -16,13 +16,13 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" cd "${package_dir}" -WHEEL_BUILD_ARGS="-m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check" +WHEEL_BUILD_ARGS="-m pip wheel . -w dist -v --no-deps --disable-pip-version-check" # pure-python packages should be marked as pure, and not have auditwheel run on them. if [[ ${package_name} == "cugraph-dgl" ]] || \ [[ ${package_name} == "cugraph-pyg" ]]; then python ${WHEEL_BUILD_ARGS} - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist else # Hardcode the output dir SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DBUILD_SHARED_LIBS=OFF;-DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE;-DCUDA_STATIC_RUNTIME=ON;-DWHOLEGRAPH_BUILD_WHEELS=ON" \ @@ -30,9 +30,10 @@ else mkdir -p final_dist python -m auditwheel repair \ - --exclude libcuda.so.1 \ - --exclude libnvidia-ml.so.1 \ - -w final_dist dist/* + --exclude libcuda.so.1 \ + --exclude libnvidia-ml.so.1 \ + -w final_dist \ + dist/* RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist fi diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 25a59fd..38dad93 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -11,9 +11,9 @@ mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# use 'ls' to expand wildcard before adding `[extra]` requires for pip -# pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + "$(echo ./dist/${python_package_name}*.whl)[test]" # Run smoke tests for aarch64 pull requests arch=$(uname -m) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index a006610..22b2ce3 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -42,6 +42,7 @@ dependencies: - pytest - pytest-benchmark - pytest-cov +- pytest-forked - pytest-xdist - pytorch-cuda=11.8 - pytorch>=2.3,<2.4.0a0 diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index 8fd6284..45951ad 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -47,6 +47,7 @@ dependencies: - pytest - pytest-benchmark - pytest-cov +- pytest-forked - pytest-xdist - pytorch-cuda=12.1 - pytorch>=2.3,<2.4.0a0 diff --git a/conda/environments/all_cuda-124_arch-x86_64.yaml b/conda/environments/all_cuda-124_arch-x86_64.yaml index 32ea733..1dd928b 100644 --- a/conda/environments/all_cuda-124_arch-x86_64.yaml +++ b/conda/environments/all_cuda-124_arch-x86_64.yaml @@ -47,6 +47,7 @@ dependencies: - pytest - pytest-benchmark - pytest-cov +- pytest-forked - pytest-xdist - pytorch-cuda=12.4 - pytorch>=2.3,<2.4.0a0 diff --git a/dependencies.yaml b/dependencies.yaml index 8a11b4e..fe267ea 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -316,6 +316,7 @@ dependencies: - pytest - pytest-benchmark - pytest-cov + - pytest-forked - pytest-xdist - scipy cugraph_dgl_dev: diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index a8a14b1..2ff055f 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -19,6 +19,7 @@ dependencies: - pytest - pytest-benchmark - pytest-cov +- pytest-forked - pytest-xdist - pytorch-cuda=11.8 - pytorch>=2.3,<2.4.0a0 diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index a9937fd..1d028e7 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -35,6 +35,7 @@ test = [ "pytest", "pytest-benchmark", "pytest-cov", + "pytest-forked", "pytest-xdist", "scipy", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index b8cafee..3c67da1 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -17,6 +17,7 @@ dependencies: - pytest - pytest-benchmark - pytest-cov +- pytest-forked - pytest-xdist - pytorch-cuda=11.8 - pytorch>=2.3,<2.4.0a0 diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index 078e2ff..7e7d282 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -43,6 +43,7 @@ test = [ "pytest", "pytest-benchmark", "pytest-cov", + "pytest-forked", "pytest-xdist", "scipy", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 45ca78931f064960bf8c102e5732f7fde1e497e7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 12:26:29 -0500 Subject: [PATCH 41/52] use rapids-build-backend --- .pre-commit-config.yaml | 12 ++++++------ conda/recipes/cugraph-dgl/meta.yaml | 4 +++- .../cugraph-pyg/conda_build_config.yaml | 2 +- conda/recipes/cugraph-pyg/meta.yaml | 3 ++- python/cugraph-dgl/cugraph_dgl/_version.py | 19 ++++++++++++------- .../cugraph_dgl/tests/test_version.py | 12 ++++++++++++ python/cugraph-dgl/pyproject.toml | 14 +++++++++++--- .../cugraph_pyg/tests/test_version.py | 12 ++++++++++++ python/cugraph-pyg/pyproject.toml | 13 +++++++++++-- 9 files changed, 70 insertions(+), 21 deletions(-) create mode 100644 python/cugraph-dgl/cugraph_dgl/tests/test_version.py create mode 100644 python/cugraph-pyg/cugraph_pyg/tests/test_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cdde470..96c110a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,13 +40,8 @@ repos: - id: clang-format types_or: [c, c++, cuda] args: ["-fallback-style=none", "-style=file", "-i"] - - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.13.11 - hooks: - - id: rapids-dependency-file-generator - args: ["--clean"] - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.0.3 + rev: v0.4.0 hooks: - id: verify-copyright files: | @@ -57,3 +52,8 @@ repos: [.]flake8[.]cython$| meta[.]yaml$| setup[.]cfg$ + - repo: https://github.com/rapidsai/dependency-file-generator + rev: v1.15.1 + hooks: + - id: rapids-dependency-file-generator + args: ["--clean"] diff --git a/conda/recipes/cugraph-dgl/meta.yaml b/conda/recipes/cugraph-dgl/meta.yaml index 86bbd4a..f08c831 100644 --- a/conda/recipes/cugraph-dgl/meta.yaml +++ b/conda/recipes/cugraph-dgl/meta.yaml @@ -21,6 +21,8 @@ build: requirements: host: - python + - rapids-build-backend>=0.3.1,<0.4.0.dev0 + - setuptools>=61.0.0 run: - cugraph ={{ version }} - dgl >=2.4.0.th23.cu* @@ -41,4 +43,4 @@ about: dev_url: https://github.com/rapidsai/cugraph license: Apache-2.0 license_file: ../../../LICENSE - summary: cuGraph library + summary: cuGraph-dgl library diff --git a/conda/recipes/cugraph-pyg/conda_build_config.yaml b/conda/recipes/cugraph-pyg/conda_build_config.yaml index 2d7aff1..47d98b4 100644 --- a/conda/recipes/cugraph-pyg/conda_build_config.yaml +++ b/conda/recipes/cugraph-pyg/conda_build_config.yaml @@ -10,7 +10,7 @@ cuda_compiler: - nvcc cmake_version: - - ">=3.26.4" + - ">=3.26.4,!=3.30.0" c_stdlib: - sysroot diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 27ccaa4..7d3e503 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -24,7 +24,8 @@ requirements: host: - cython >=3.0.0 - python - - scikit-build-core >=0.7.0 + - rapids-build-backend>=0.3.1,<0.4.0.dev0 + - setuptools>=61.0.0 run: - rapids-dask-dependency ={{ minor_version }} - numba >=0.57 diff --git a/python/cugraph-dgl/cugraph_dgl/_version.py b/python/cugraph-dgl/cugraph_dgl/_version.py index f2210fd..e8adcc3 100644 --- a/python/cugraph-dgl/cugraph_dgl/_version.py +++ b/python/cugraph-dgl/cugraph_dgl/_version.py @@ -15,12 +15,17 @@ import importlib.resources -# Read VERSION file from the module that is symlinked to VERSION file -# in the root of the repo at build time or copied to the moudle at -# installation. VERSION is a separate file that allows CI build-time scripts -# to update version info (including commit hashes) without modifying -# source files. __version__ = ( - importlib.resources.files("cugraph_dgl").joinpath("VERSION").read_text().strip() + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/cugraph-dgl/cugraph_dgl/tests/test_version.py b/python/cugraph-dgl/cugraph_dgl/tests/test_version.py new file mode 100644 index 0000000..343e4fb --- /dev/null +++ b/python/cugraph-dgl/cugraph_dgl/tests/test_version.py @@ -0,0 +1,12 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import cugraph_dgl + + +def test_version_constants_are_populated(): + # __git_commit__ will only be non-empty in a built distribution + assert isinstance(cugraph_dgl.__git_commit__, str) + + # __version__ should always be non-empty + assert isinstance(cugraph_dgl.__version__, str) + assert len(cugraph_dgl.__version__) > 0 diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 1d028e7..e3e1221 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -3,10 +3,11 @@ [build-system] requires = [ + "rapids-build-backend>=0.3.1,<0.4.0.dev0", "setuptools>=61.0.0", "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -build-backend = "setuptools.build_meta" +build-backend = "rapids_build_backend.build" [project] name = "cugraph-dgl" @@ -23,7 +24,7 @@ classifiers = [ "Programming Language :: Python", ] dependencies = [ - "cugraph==24.12.*", + "cugraph==24.12.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<3.0a0", "pylibcugraphops==24.12.*,>=0.0.0a0", @@ -32,12 +33,14 @@ dependencies = [ [project.optional-dependencies] test = [ "pandas", + "pylibwholegraph==24.12.*,>=0.0.0a0", "pytest", "pytest-benchmark", "pytest-cov", - "pytest-forked", "pytest-xdist", "scipy", + "tensordict>=0.1.2", + "torch>=2.3,<2.4.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] @@ -54,3 +57,8 @@ version = {file = "cugraph_dgl/VERSION"} include = [ "cugraph_dgl*", ] + +[tool.rapids-build-backend] +build-backend = "setuptools.build_meta" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_version.py b/python/cugraph-pyg/cugraph_pyg/tests/test_version.py new file mode 100644 index 0000000..4ea0f98 --- /dev/null +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_version.py @@ -0,0 +1,12 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import cugraph_pyg + + +def test_version_constants_are_populated(): + # __git_commit__ will only be non-empty in a built distribution + assert isinstance(cugraph_pyg.__git_commit__, str) + + # __version__ should always be non-empty + assert isinstance(cugraph_pyg.__version__, str) + assert len(cugraph_pyg.__version__) > 0 diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index 7e7d282..e157f36 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -3,9 +3,11 @@ [build-system] requires = [ + "rapids-build-backend>=0.3.1,<0.4.0.dev0", "setuptools>=61.0.0", "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "rapids_build_backend.build" [tool.pytest.ini_options] testpaths = ["cugraph_pyg/tests"] @@ -27,7 +29,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "cugraph==24.12.*", + "cugraph==24.12.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<3.0a0", "pylibcugraphops==24.12.*,>=0.0.0a0", @@ -40,12 +42,14 @@ Documentation = "https://docs.rapids.ai/api/cugraph/stable/" [project.optional-dependencies] test = [ "pandas", + "pylibwholegraph==24.12.*,>=0.0.0a0", "pytest", "pytest-benchmark", "pytest-cov", - "pytest-forked", "pytest-xdist", "scipy", + "tensordict>=0.1.2", + "torch>=2.3,<2.4.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.setuptools] @@ -59,3 +63,8 @@ include = [ "cugraph_pyg*", "cugraph_pyg.*", ] + +[tool.rapids-build-backend] +build-backend = "setuptools.build_meta" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" From afb87d4706e624d46693a1ab9b1227caa02ec018 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 12:32:45 -0500 Subject: [PATCH 42/52] more dependencies.yaml changes --- dependencies.yaml | 28 +++++++++++++------ .../conda/cugraph_dgl_dev_cuda-118.yaml | 1 - python/cugraph-dgl/pyproject.toml | 1 + .../conda/cugraph_pyg_dev_cuda-118.yaml | 1 - python/cugraph-pyg/pyproject.toml | 1 + 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index fe267ea..c44a39b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -59,13 +59,13 @@ files: - depends_on_cudf - py_version - test_python_common - py_build_cugraph_dgl: output: pyproject pyproject_dir: python/cugraph-dgl extras: table: build-system includes: + - python_build_rapids - python_build_wheel py_run_cugraph_dgl: output: pyproject @@ -83,12 +83,15 @@ files: key: test includes: - test_python_common + - depends_on_pylibwholegraph + - depends_on_pytorch py_build_cugraph_pyg: output: pyproject pyproject_dir: python/cugraph-pyg extras: table: build-system includes: + - python_build_rapids - python_build_wheel py_run_cugraph_pyg: output: pyproject @@ -106,6 +109,8 @@ files: key: test includes: - test_python_common + - depends_on_pylibwholegraph + - depends_on_pytorch cugraph_dgl_dev: @@ -258,6 +263,11 @@ dependencies: - matrix: packages: - python>=3.10,<3.13 + python_build_rapids: + common: + - output_types: [conda, pyproject, requirements] + packages: + - rapids-build-backend>=0.3.1,<0.4.0.dev0 python_build_wheel: common: - output_types: [conda, pyproject, requirements] @@ -281,9 +291,9 @@ dependencies: packages: - &numba numba>=0.57 - &numpy numpy>=1.23,<3.0a0 - - output_types: [pyproject] + - output_types: [pyproject, requirements] packages: - - &cugraph cugraph==24.12.* + - &cugraph cugraph==24.12.*,>=0.0.0a0 python_run_cugraph_pyg: common: - output_types: [conda, pyproject] @@ -323,7 +333,7 @@ dependencies: common: - output_types: [conda] packages: - - cugraph==24.12.* + - &cugraph_unsuffixed cugraph==24.12.*,>=0.0.0a0 - &pytorch_conda pytorch>=2.3,<2.4.0a0 - dgl>=2.4.0.cu* - &tensordict tensordict>=0.1.2 @@ -331,7 +341,7 @@ dependencies: common: - output_types: [conda] packages: - - cugraph==24.12.* + - *cugraph_unsuffixed - *pytorch_conda - *tensordict - pytorch_geometric>=2.5,<2.6 @@ -405,7 +415,7 @@ dependencies: common: - output_types: conda packages: - - &cugraph_unsuffixed cugraph==24.12.*,>=0.0.0a0 + - *cugraph_unsuffixed - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -626,7 +636,7 @@ dependencies: common: - output_types: conda packages: - - &pylibwholegraph_unsuffixed pylibwholegraph==24.12.* + - &pylibwholegraph_unsuffixed pylibwholegraph==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -639,10 +649,10 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibwholegraph-cu12==24.12.* + - pylibwholegraph-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibwholegraph-cu11==24.12.* + - pylibwholegraph-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index 2ff055f..4911005 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -8,7 +8,6 @@ channels: - conda-forge - nvidia dependencies: -- cugraph==24.12.* - cugraph==24.12.*,>=0.0.0a0 - dgl>=2.4.0.cu* - dglteam/label/th23_cu118::dgl diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index e3e1221..1f654e7 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -37,6 +37,7 @@ test = [ "pytest", "pytest-benchmark", "pytest-cov", + "pytest-forked", "pytest-xdist", "scipy", "tensordict>=0.1.2", diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index 3c67da1..fd2bcf9 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -8,7 +8,6 @@ channels: - conda-forge - nvidia dependencies: -- cugraph==24.12.* - cugraph==24.12.*,>=0.0.0a0 - pandas - pre-commit diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index e157f36..d071c03 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -46,6 +46,7 @@ test = [ "pytest", "pytest-benchmark", "pytest-cov", + "pytest-forked", "pytest-xdist", "scipy", "tensordict>=0.1.2", From dce3e1e9eff008e552eae32f9d0b3733365356ab Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 13:09:09 -0500 Subject: [PATCH 43/52] skip rapids-build-backend cuda versioning for conda builds --- build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 2b18808..8e21576 100755 --- a/build.sh +++ b/build.sh @@ -84,7 +84,7 @@ VERBOSE_FLAG="" BUILD_TYPE=Release INSTALL_TARGET="--target install" BUILD_ALL_GPU_ARCH=0 -PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps" +PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true" # Set defaults for vars that may not have been defined externally # FIXME: if PREFIX is not set, check CONDA_PREFIX, but there is no fallback @@ -247,9 +247,7 @@ if hasArg pylibwholegraph || buildDefault || hasArg all; then cd ${REPODIR}/python/pylibwholegraph env LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR} \ SKBUILD_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}" python ${PYTHON_ARGS_FOR_INSTALL} \ - --config-settings rapidsai.disable-cuda=true \ . - else # just invoke cmake without going through scikit-build-core env LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR} \ From 8a3fda807da8898b7f439b3587b20e18f6e2f76f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 13:50:57 -0500 Subject: [PATCH 44/52] add same mkl pin that wholegraph CI uses --- ci/test_python.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index d7cb28d..8950cb7 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -62,10 +62,10 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then --channel conda-forge \ --channel dglteam/label/th23_cu118 \ --channel nvidia \ - "pylibwholegraph=${RAPIDS_VERSION}.*" \ - "pylibcugraphops=${RAPIDS_VERSION}.*" \ - "cugraph=${RAPIDS_VERSION}.*" \ - "cugraph-dgl=${RAPIDS_VERSION}.*" \ + "pylibwholegraph=${RAPIDS_VERSION}" \ + "pylibcugraphops=${RAPIDS_VERSION}" \ + "cugraph=${RAPIDS_VERSION}" \ + "cugraph-dgl=${RAPIDS_VERSION}" \ 'pytorch::pytorch>=2.3,<2.4' \ 'cuda-version=11.8' \ "ogb" @@ -104,10 +104,10 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ --channel pytorch \ - "pylibwholegraph=${RAPIDS_VERSION}.*" \ - "pylibcugraphops=${RAPIDS_VERSION}.*" \ - "cugraph=${RAPIDS_VERSION}.*" \ - "cugraph-pyg=${RAPIDS_VERSION}.*" \ + "pylibwholegraph=${RAPIDS_VERSION}" \ + "pylibcugraphops=${RAPIDS_VERSION}" \ + "cugraph=${RAPIDS_VERSION}" \ + "cugraph-pyg=${RAPIDS_VERSION}" \ "pytorch::pytorch>=2.3,<2.4" \ "ogb" @@ -145,7 +145,8 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ --channel pytorch \ - "pylibwholegraph=${RAPIDS_VERSION}.*" \ + 'mkl<2024.1.0' \ + "pylibwholegraph=${RAPIDS_VERSION}" \ "pytorch::pytorch>=2.3,<2.4" \ "ogb" From 4263e515f558c3e6a75fc9fb3c50bc70ded0e752 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 14:13:31 -0500 Subject: [PATCH 45/52] add pytorch to test environment (wholegraph does not declare a dependency on it) --- ci/test_wheel_cugraph-pyg.sh | 4 ++-- dependencies.yaml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index a3a424c..3a4bf5f 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -17,8 +17,8 @@ if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then PYTORCH_URL="https://download.pytorch.org/whl/cu118" PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html" else - PYTORCH_URL="https://download.pytorch.org/whl/cu124" - PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu124.html" + PYTORCH_URL="https://download.pytorch.org/whl/cu121" + PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html" fi # echo to expand wildcard before adding `[extra]` requires for pip diff --git a/dependencies.yaml b/dependencies.yaml index c44a39b..a94a0a9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -26,7 +26,6 @@ files: - python_run_cugraph_pyg - test_notebook - test_python_common - checks: output: none includes: @@ -48,6 +47,7 @@ files: output: none includes: - cuda_version + - depends_on_pytorch - py_version - test_notebook - test_python_common @@ -57,6 +57,7 @@ files: - cuda_version - depends_on_cugraph - depends_on_cudf + - depends_on_pytorch - py_version - test_python_common py_build_cugraph_dgl: From 15f913b9cf6db812f5c300cebd07b46b45c8de8c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 14:38:47 -0500 Subject: [PATCH 46/52] remove pytorch-cuda --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - conda/environments/all_cuda-121_arch-x86_64.yaml | 1 - conda/environments/all_cuda-124_arch-x86_64.yaml | 1 - dependencies.yaml | 11 ----------- .../cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml | 1 - .../cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml | 1 - 6 files changed, 16 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 22b2ce3..d15e5a2 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -44,7 +44,6 @@ dependencies: - pytest-cov - pytest-forked - pytest-xdist -- pytorch-cuda=11.8 - pytorch>=2.3,<2.4.0a0 - raft-dask==24.12.*,>=0.0.0a0 - recommonmark diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index 45951ad..74a07f2 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -49,7 +49,6 @@ dependencies: - pytest-cov - pytest-forked - pytest-xdist -- pytorch-cuda=12.1 - pytorch>=2.3,<2.4.0a0 - raft-dask==24.12.*,>=0.0.0a0 - recommonmark diff --git a/conda/environments/all_cuda-124_arch-x86_64.yaml b/conda/environments/all_cuda-124_arch-x86_64.yaml index 1dd928b..002aa33 100644 --- a/conda/environments/all_cuda-124_arch-x86_64.yaml +++ b/conda/environments/all_cuda-124_arch-x86_64.yaml @@ -49,7 +49,6 @@ dependencies: - pytest-cov - pytest-forked - pytest-xdist -- pytorch-cuda=12.4 - pytorch>=2.3,<2.4.0a0 - raft-dask==24.12.*,>=0.0.0a0 - recommonmark diff --git a/dependencies.yaml b/dependencies.yaml index a94a0a9..4d60123 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -375,17 +375,6 @@ dependencies: - *pytorch_pip - *tensordict - {matrix: null, packages: [*pytorch_pip, *tensordict]} - - output_types: [conda] - matrices: - - matrix: {cuda: "12.1"} - packages: - - pytorch-cuda=12.1 - - matrix: {cuda: "12.4"} - packages: - - pytorch-cuda=12.4 - - matrix: {cuda: "11.*"} - packages: - - pytorch-cuda=11.8 depends_on_rmm: common: diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index 4911005..acb32ad 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -20,7 +20,6 @@ dependencies: - pytest-cov - pytest-forked - pytest-xdist -- pytorch-cuda=11.8 - pytorch>=2.3,<2.4.0a0 - scipy - tensordict>=0.1.2 diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index fd2bcf9..82935b2 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -18,7 +18,6 @@ dependencies: - pytest-cov - pytest-forked - pytest-xdist -- pytorch-cuda=11.8 - pytorch>=2.3,<2.4.0a0 - pytorch_geometric>=2.5,<2.6 - scipy From 4d66d77bdbea8cbc12871071dac966f3b131e9fa Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 15:37:40 -0500 Subject: [PATCH 47/52] consolidate 'cugraph' dependency, avoid unsuffixed use for pyproject/requirements --- ci/test_wheel_cugraph-dgl.sh | 2 +- ci/test_wheel_pylibwholegraph.sh | 2 +- dependencies.yaml | 13 +++---------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 733b851..3eaa098 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -16,7 +16,7 @@ RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_ PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then - PYTORCH_CUDA_VER="124" + PYTORCH_CUDA_VER="121" else PYTORCH_CUDA_VER=$PKG_CUDA_VER fi diff --git a/ci/test_wheel_pylibwholegraph.sh b/ci/test_wheel_pylibwholegraph.sh index 518da45..60598bc 100755 --- a/ci/test_wheel_pylibwholegraph.sh +++ b/ci/test_wheel_pylibwholegraph.sh @@ -12,7 +12,7 @@ RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download- PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then - INDEX_URL="https://download.pytorch.org/whl/cu124" + INDEX_URL="https://download.pytorch.org/whl/cu121" else INDEX_URL="https://download.pytorch.org/whl/cu${PKG_CUDA_VER}" fi diff --git a/dependencies.yaml b/dependencies.yaml index 4d60123..f12ace6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -75,6 +75,7 @@ files: table: project includes: - python_run_cugraph_dgl + - depends_on_cugraph - depends_on_pylibcugraphops py_test_cugraph_dgl: output: pyproject @@ -101,6 +102,7 @@ files: table: project includes: - python_run_cugraph_pyg + - depends_on_cugraph - depends_on_pylibcugraphops py_test_cugraph_pyg: output: pyproject @@ -216,7 +218,6 @@ dependencies: - output_types: [requirements, pyproject] packages: - ninja - - output_types: [conda] packages: - &cmake_ver cmake>=3.26.4,!=3.30.0 @@ -292,18 +293,12 @@ dependencies: packages: - &numba numba>=0.57 - &numpy numpy>=1.23,<3.0a0 - - output_types: [pyproject, requirements] - packages: - - &cugraph cugraph==24.12.*,>=0.0.0a0 python_run_cugraph_pyg: common: - output_types: [conda, pyproject] packages: - *numba - *numpy - - output_types: [pyproject] - packages: - - *cugraph test_notebook: common: - output_types: [conda, requirements] @@ -334,7 +329,6 @@ dependencies: common: - output_types: [conda] packages: - - &cugraph_unsuffixed cugraph==24.12.*,>=0.0.0a0 - &pytorch_conda pytorch>=2.3,<2.4.0a0 - dgl>=2.4.0.cu* - &tensordict tensordict>=0.1.2 @@ -342,7 +336,6 @@ dependencies: common: - output_types: [conda] packages: - - *cugraph_unsuffixed - *pytorch_conda - *tensordict - pytorch_geometric>=2.5,<2.6 @@ -405,7 +398,7 @@ dependencies: common: - output_types: conda packages: - - *cugraph_unsuffixed + - &cugraph_unsuffixed cugraph==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file From 87a840f8990507d0e9a2dd48c2dbbbd9d8b9f491 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 16:16:52 -0500 Subject: [PATCH 48/52] always generate a .py script for notebook tests, pin version of nbconvert explicitly --- ci/utils/nbtest.sh | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-121_arch-x86_64.yaml | 1 + conda/environments/all_cuda-124_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 91af633..faf7d28 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -60,7 +60,7 @@ for nb in $*; do echo -------------------------------------------------------------------------------- echo STARTING: ${NBNAME} echo -------------------------------------------------------------------------------- - jupyter nbconvert --to script ${NBFILENAME} --output ${NBTMPDIR}/${NBNAME}-test + jupyter nbconvert --to python ${NBFILENAME} --output ${NBTMPDIR}/${NBNAME}-test echo "${MAGIC_OVERRIDE_CODE}" > ${NBTMPDIR}/tmpfile cat ${NBTESTSCRIPT} >> ${NBTMPDIR}/tmpfile mv ${NBTMPDIR}/tmpfile ${NBTESTSCRIPT} diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d15e5a2..5a42a1b 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -27,6 +27,7 @@ dependencies: - libraft-headers==24.12.*,>=0.0.0a0 - librmm==24.12.*,>=0.0.0a0 - nanobind>=0.2.0 +- nbconvert>=7.16 - nbsphinx - nccl>=2.19 - notebook>=0.5.0 diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index 74a07f2..1da7397 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -32,6 +32,7 @@ dependencies: - libraft-headers==24.12.*,>=0.0.0a0 - librmm==24.12.*,>=0.0.0a0 - nanobind>=0.2.0 +- nbconvert>=7.16 - nbsphinx - nccl>=2.19 - notebook>=0.5.0 diff --git a/conda/environments/all_cuda-124_arch-x86_64.yaml b/conda/environments/all_cuda-124_arch-x86_64.yaml index 002aa33..76711ac 100644 --- a/conda/environments/all_cuda-124_arch-x86_64.yaml +++ b/conda/environments/all_cuda-124_arch-x86_64.yaml @@ -32,6 +32,7 @@ dependencies: - libraft-headers==24.12.*,>=0.0.0a0 - librmm==24.12.*,>=0.0.0a0 - nanobind>=0.2.0 +- nbconvert>=7.16 - nbsphinx - nccl>=2.19 - notebook>=0.5.0 diff --git a/dependencies.yaml b/dependencies.yaml index f12ace6..530d295 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -304,6 +304,7 @@ dependencies: - output_types: [conda, requirements] packages: - ipython + - nbconvert>=7.16 - notebook>=0.5.0 - output_types: [conda] packages: From 08c40797cd10fe5bec325eb923c3da010bef8c5a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 16 Oct 2024 09:25:31 -0500 Subject: [PATCH 49/52] fix test paths --- ci/run_cugraph_dgl_pytests.sh | 2 +- ci/test_python.sh | 8 ++++---- ci/test_wheel_cugraph-dgl.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/run_cugraph_dgl_pytests.sh b/ci/run_cugraph_dgl_pytests.sh index 83c26a5..e85eb04 100755 --- a/ci/run_cugraph_dgl_pytests.sh +++ b/ci/run_cugraph_dgl_pytests.sh @@ -4,6 +4,6 @@ set -euo pipefail # Support invoking run_cugraph_dgl_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-dgl/tests +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-dgl/cugraph_dgl pytest --cache-clear --ignore=mg "$@" . diff --git a/ci/test_python.sh b/ci/test_python.sh index 8950cb7..5e3cfaa 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -108,8 +108,8 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then "pylibcugraphops=${RAPIDS_VERSION}" \ "cugraph=${RAPIDS_VERSION}" \ "cugraph-pyg=${RAPIDS_VERSION}" \ - "pytorch::pytorch>=2.3,<2.4" \ - "ogb" + 'pytorch::pytorch>=2.3,<2.4' \ + 'ogb' rapids-print-env @@ -147,8 +147,8 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then --channel pytorch \ 'mkl<2024.1.0' \ "pylibwholegraph=${RAPIDS_VERSION}" \ - "pytorch::pytorch>=2.3,<2.4" \ - "ogb" + 'pytorch::pytorch>=2.3,<2.4' \ + 'ogb' rapids-print-env diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 3eaa098..d709145 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -33,4 +33,4 @@ python -m pip install \ 'dgl==2.4.0' \ 'torch>=2.3.0,<2.4' -python -m pytest python/cugraph-dgl/tests +python -m pytest python/cugraph-dgl/cugraph_dgl/tests From e7abf779ac1dd49bdd0aa6e5e556563bc5df6085 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 21 Oct 2024 12:34:35 -0500 Subject: [PATCH 50/52] setuptools --- ci/build_python.sh | 6 ++---- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-121_arch-x86_64.yaml | 1 + conda/environments/all_cuda-124_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + python/cugraph-dgl/pyproject.toml | 1 + python/cugraph-pyg/pyproject.toml | 1 + python/pylibwholegraph/pyproject.toml | 1 + 8 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 1ce96dd..9f42948 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -25,15 +25,13 @@ RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ conda/recipes/pylibwholegraph -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" - -rapids-conda-retry mambabuild \ +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cugraph-pyg -rapids-conda-retry mambabuild \ +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index bd16184..56438bf 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -52,6 +52,7 @@ dependencies: - rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy +- setuptools - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index 4acb275..95aba20 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -57,6 +57,7 @@ dependencies: - rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy +- setuptools - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 diff --git a/conda/environments/all_cuda-124_arch-x86_64.yaml b/conda/environments/all_cuda-124_arch-x86_64.yaml index d37125f..214688f 100644 --- a/conda/environments/all_cuda-124_arch-x86_64.yaml +++ b/conda/environments/all_cuda-124_arch-x86_64.yaml @@ -57,6 +57,7 @@ dependencies: - rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy +- setuptools - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 diff --git a/dependencies.yaml b/dependencies.yaml index 870395a..021e749 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -321,6 +321,7 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cython>=3.0.0 - ninja + - setuptools python_build_cythonize: common: - output_types: [conda, pyproject, requirements] diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 1718ed3..8af2524 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -7,6 +7,7 @@ requires = [ "cython>=3.0.0", "ninja", "rapids-build-backend>=0.3.1,<0.4.0.dev0", + "setuptools", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "rapids_build_backend.build" diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index 872e377..f096b58 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -7,6 +7,7 @@ requires = [ "cython>=3.0.0", "ninja", "rapids-build-backend>=0.3.1,<0.4.0.dev0", + "setuptools", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "rapids_build_backend.build" diff --git a/python/pylibwholegraph/pyproject.toml b/python/pylibwholegraph/pyproject.toml index 3eed9dc..f1bcfb3 100644 --- a/python/pylibwholegraph/pyproject.toml +++ b/python/pylibwholegraph/pyproject.toml @@ -58,6 +58,7 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0", "ninja", + "setuptools", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.scikit-build] From 237563c9aa75707426faf8b824607ce99648bb03 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 22 Oct 2024 13:47:21 -0700 Subject: [PATCH 51/52] add missing files, update tests --- .../dataloading/neighbor_sampler.py | 6 +-- .../cugraph_dgl/tests/test_graph.py | 9 ++-- python/cugraph-dgl/cugraph_dgl/view.py | 43 +++++++++++++++++++ 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py b/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py index 4ec513c..ecc5100 100644 --- a/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py +++ b/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py @@ -197,10 +197,8 @@ def sample( if g.is_homogeneous: indices = torch.concat(list(indices)) - ds.sample_from_nodes(indices.long(), batch_size=batch_size) - return HomogeneousSampleReader( - ds.get_reader(), self.output_format, self.edge_dir - ) + reader = ds.sample_from_nodes(indices.long(), batch_size=batch_size) + return HomogeneousSampleReader(reader, self.output_format, self.edge_dir) raise ValueError( "Sampling heterogeneous graphs is currently" diff --git a/python/cugraph-dgl/cugraph_dgl/tests/test_graph.py b/python/cugraph-dgl/cugraph_dgl/tests/test_graph.py index a60db97..2b9f46d 100644 --- a/python/cugraph-dgl/cugraph_dgl/tests/test_graph.py +++ b/python/cugraph-dgl/cugraph_dgl/tests/test_graph.py @@ -53,10 +53,11 @@ def test_graph_make_homogeneous_graph(direction): graph.nodes() == torch.arange(num_nodes, dtype=torch.int64, device="cuda") ).all() - assert graph.nodes[None]["x"] is not None - assert (graph.nodes[None]["x"] == torch.as_tensor(node_x, device="cuda")).all() + emb = graph.nodes[None]["x"] + assert emb is not None + assert (emb() == torch.as_tensor(node_x, device="cuda")).all() assert ( - graph.nodes[None]["num"] + graph.nodes[None]["num"]() == torch.arange(num_nodes, dtype=torch.int64, device="cuda") ).all() @@ -64,7 +65,7 @@ def test_graph_make_homogeneous_graph(direction): graph.edges("eid", device="cuda") == torch.arange(len(df), dtype=torch.int64, device="cuda") ).all() - assert (graph.edges[None]["weight"] == torch.as_tensor(wgt, device="cuda")).all() + assert (graph.edges[None]["weight"]() == torch.as_tensor(wgt, device="cuda")).all() plc_expected_graph = pylibcugraph.SGGraph( pylibcugraph.ResourceHandle(), diff --git a/python/cugraph-dgl/cugraph_dgl/view.py b/python/cugraph-dgl/cugraph_dgl/view.py index dbc53e7..7c4d95f 100644 --- a/python/cugraph-dgl/cugraph_dgl/view.py +++ b/python/cugraph-dgl/cugraph_dgl/view.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import warnings from collections import defaultdict from collections.abc import MutableMapping @@ -20,11 +21,53 @@ import cugraph_dgl from cugraph_dgl.typing import TensorType +from cugraph_dgl.utils.cugraph_conversion_utils import _cast_to_torch_tensor torch = import_optional("torch") dgl = import_optional("dgl") +class EmbeddingView: + def __init__(self, storage: "dgl.storages.base.FeatureStorage", ld: int): + self.__ld = ld + self.__storage = storage + + def __getitem__(self, u: TensorType) -> "torch.Tensor": + u = _cast_to_torch_tensor(u) + try: + return self.__storage.fetch( + u, + "cuda", + ) + except RuntimeError as ex: + warnings.warn( + "Got error accessing data, trying again with index on device: " + + str(ex) + ) + return self.__storage.fetch( + u.cuda(), + "cuda", + ) + + def __call__(self): + warnings.warn( + "Getting an entire embedding tensor is not recommended " + " as it wastes memory. Consider indexing to get only the " + "required elements of the embedding tensor." + ) + return self[torch.arange(self.__ld, dtype=torch.int64)] + + @property + def shape(self) -> "torch.Size": + try: + f = self.__storage.fetch(torch.tensor([0]), "cpu") + except RuntimeError: + f = self.__storage.fetch(torch.tensor([0], device="cuda"), "cuda") + sz = [s for s in f.shape] + sz[0] = self.__ld + return torch.Size(tuple(sz)) + + class HeteroEdgeDataView(MutableMapping): """ Duck-typed version of DGL's HeteroEdgeDataView. From be6fc5a5a501f30371ffa032c529997c3e23f8c4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 10:31:50 -0500 Subject: [PATCH 52/52] remove smoke-testing scripts --- ci/wheel_smoke_test_cugraph-dgl.py | 17 ----------------- ci/wheel_smoke_test_cugraph-pyg.py | 17 ----------------- ci/wheel_smoke_test_pylibwholegraph.py | 17 ----------------- notebooks/debug/notebook_smoke_test.ipynb | 20 -------------------- 4 files changed, 71 deletions(-) delete mode 100644 ci/wheel_smoke_test_cugraph-dgl.py delete mode 100644 ci/wheel_smoke_test_cugraph-pyg.py delete mode 100644 ci/wheel_smoke_test_pylibwholegraph.py delete mode 100644 notebooks/debug/notebook_smoke_test.ipynb diff --git a/ci/wheel_smoke_test_cugraph-dgl.py b/ci/wheel_smoke_test_cugraph-dgl.py deleted file mode 100644 index 8b2f933..0000000 --- a/ci/wheel_smoke_test_cugraph-dgl.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import cugraph_dgl - -if __name__ == "__main__": - print(cugraph_dgl.__version__) diff --git a/ci/wheel_smoke_test_cugraph-pyg.py b/ci/wheel_smoke_test_cugraph-pyg.py deleted file mode 100644 index d05e510..0000000 --- a/ci/wheel_smoke_test_cugraph-pyg.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import cugraph_pyg - -if __name__ == "__main__": - print(cugraph_pyg.__version__) diff --git a/ci/wheel_smoke_test_pylibwholegraph.py b/ci/wheel_smoke_test_pylibwholegraph.py deleted file mode 100644 index 43d56f8..0000000 --- a/ci/wheel_smoke_test_pylibwholegraph.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pylibwholegraph - -if __name__ == "__main__": - print(pylibwholegraph.__version__) diff --git a/notebooks/debug/notebook_smoke_test.ipynb b/notebooks/debug/notebook_smoke_test.ipynb deleted file mode 100644 index 8d455fc..0000000 --- a/notebooks/debug/notebook_smoke_test.ipynb +++ /dev/null @@ -1,20 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import cugraph" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}