From cd5fc6f6510056804af39f9deaad8885e632458a Mon Sep 17 00:00:00 2001 From: Tingyu Wang Date: Tue, 9 Jan 2024 15:19:27 -0500 Subject: [PATCH] build wheels for `cugraph-dgl` and `cugraph-pyg` (#4075) Closes #4061 Authors: - Tingyu Wang (https://github.com/tingyu66) Approvers: - Alex Barghi (https://github.com/alexbarghi-nv) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cugraph/pull/4075 --- .github/workflows/build.yaml | 40 ++++++++++++++++++++++++++++ .github/workflows/pr.yaml | 34 ++++++++++++++++++++++++ .github/workflows/test.yaml | 18 +++++++++++++ ci/build_wheel.sh | 8 +++--- ci/build_wheel_cugraph-dgl.sh | 6 +++++ ci/build_wheel_cugraph-pyg.sh | 6 +++++ ci/test_wheel_cugraph-dgl.sh | 34 ++++++++++++++++++++++++ ci/test_wheel_cugraph-pyg.sh | 43 +++++++++++++++++++++++++++++++ dependencies.yaml | 18 +++++++++++++ python/cugraph-dgl/pyproject.toml | 21 +++++++++++---- python/cugraph-pyg/pyproject.toml | 13 +++++++++- 11 files changed, 232 insertions(+), 9 deletions(-) create mode 100755 ci/build_wheel_cugraph-dgl.sh create mode 100755 ci/build_wheel_cugraph-pyg.sh create mode 100755 ci/test_wheel_cugraph-dgl.sh create mode 100755 ci/test_wheel_cugraph-pyg.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 85ac682daf4..273a8902eae 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -133,3 +133,43 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: nx-cugraph + wheel-build-cugraph-dgl: + needs: wheel-publish-cugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cugraph-dgl.sh + wheel-publish-cugraph-dgl: + needs: wheel-build-cugraph-dgl + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cugraph-dgl + wheel-build-cugraph-pyg: + needs: wheel-publish-cugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cugraph-pyg.sh + wheel-publish-cugraph-pyg: + needs: wheel-build-cugraph-pyg + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cugraph-pyg diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 82c71efffdb..84d22f8e896 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,10 @@ jobs: - wheel-tests-cugraph - wheel-build-nx-cugraph - wheel-tests-nx-cugraph + - wheel-build-cugraph-dgl + - wheel-tests-cugraph-dgl + - wheel-build-cugraph-pyg + - wheel-tests-cugraph-pyg - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 @@ -127,6 +131,36 @@ jobs: with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh + wheel-build-cugraph-dgl: + needs: wheel-tests-cugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 + with: + build_type: pull-request + script: ci/build_wheel_cugraph-dgl.sh + wheel-tests-cugraph-dgl: + needs: wheel-build-cugraph-dgl + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.02 + with: + build_type: pull-request + script: ci/test_wheel_cugraph-dgl.sh + matrix_filter: map(select(.ARCH == "amd64")) + wheel-build-cugraph-pyg: + needs: wheel-tests-cugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 + with: + build_type: pull-request + script: ci/build_wheel_cugraph-pyg.sh + wheel-tests-cugraph-pyg: + needs: wheel-build-cugraph-pyg + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.02 + with: + build_type: pull-request + script: ci/test_wheel_cugraph-pyg.sh + matrix_filter: map(select(.ARCH == "amd64" and .CUDA_VER == "11.8.0")) devcontainer: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.02 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0d9f4d291c3..773358ede8d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -57,3 +57,21 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_nx-cugraph.sh + wheel-tests-cugraph-dgl: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.02 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel_cugraph-dgl.sh + wheel-tests-cugraph-pyg: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.02 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel_cugraph-pyg.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 163520ea1da..828d8948143 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail @@ -36,7 +36,7 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi -for dep in rmm cudf raft-dask pylibcugraph pylibraft ucx-py; do +for dep in rmm cudf cugraph raft-dask pylibcugraph pylibcugraphops pylibraft ucx-py; do sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} done @@ -55,7 +55,9 @@ cd "${package_dir}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # pure-python packages should not have auditwheel run on them. -if [[ ${package_name} == "nx-cugraph" ]]; then +if [[ ${package_name} == "nx-cugraph" ]] || \ + [[ ${package_name} == "cugraph-dgl" ]] || \ + [[ ${package_name} == "cugraph-pyg" ]]; then RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 dist else mkdir -p final_dist diff --git a/ci/build_wheel_cugraph-dgl.sh b/ci/build_wheel_cugraph-dgl.sh new file mode 100755 index 00000000000..d62f810cba4 --- /dev/null +++ b/ci/build_wheel_cugraph-dgl.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +./ci/build_wheel.sh cugraph-dgl python/cugraph-dgl diff --git a/ci/build_wheel_cugraph-pyg.sh b/ci/build_wheel_cugraph-pyg.sh new file mode 100755 index 00000000000..97baa243f73 --- /dev/null +++ b/ci/build_wheel_cugraph-pyg.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +./ci/build_wheel.sh cugraph-pyg python/cugraph-pyg diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh new file mode 100755 index 00000000000..90c86af95fe --- /dev/null +++ b/ci/test_wheel_cugraph-dgl.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -eoxu pipefail + +package_name="cugraph-dgl" +package_dir="python/cugraph-dgl" + +python_package_name=$(echo ${package_name}|sed 's/-/_/g') + +mkdir -p ./dist +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# use 'ls' to expand wildcard before adding `[extra]` requires for pip +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +# pip creates wheels using python package names +python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] + + +PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" +PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} +if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then + PYTORCH_CUDA_VER="121" +else + PYTORCH_CUDA_VER=$PKG_CUDA_VER +fi +PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}" +DGL_URL="https://data.dgl.ai/wheels/cu${PYTORCH_CUDA_VER}/repo.html" + +rapids-logger "Installing PyTorch and DGL" +rapids-retry python -m pip install torch --index-url ${PYTORCH_URL} +rapids-retry python -m pip install dgl --find-links ${DGL_URL} + +python -m pytest python/cugraph-dgl/tests diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh new file mode 100755 index 00000000000..9a211c81886 --- /dev/null +++ b/ci/test_wheel_cugraph-pyg.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -eoxu pipefail + +package_name="cugraph-pyg" +package_dir="python/cugraph-pyg" + +python_package_name=$(echo ${package_name}|sed 's/-/_/g') + +mkdir -p ./dist +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# use 'ls' to expand wildcard before adding `[extra]` requires for pip +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +# pip creates wheels using python package names +python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] + +# RAPIDS_DATASET_ROOT_DIR is used by test scripts +export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" + +if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then + rapids-logger "Installing PyTorch and PyG dependencies" + PYTORCH_URL="https://download.pytorch.org/whl/cu118" + rapids-retry python -m pip install torch==2.1.0 --index-url ${PYTORCH_URL} + rapids-retry python -m pip install torch-geometric==2.4.0 + rapids-retry python -m pip install \ + pyg_lib \ + torch_scatter \ + torch_sparse \ + torch_cluster \ + torch_spline_conv \ + -f https://data.pyg.org/whl/torch-2.1.0+cu118.html + + rapids-logger "pytest cugraph-pyg (single GPU)" + python -m pytest \ + --cache-clear \ + --ignore=tests/int \ + --ignore=tests/mg \ + python/cugraph-pyg/cugraph_pyg/tests +else + rapids-logger "skipping cugraph-pyg wheel test on CUDA!=11.8" +fi diff --git a/dependencies.yaml b/dependencies.yaml index 579acec3996..3eed525bfe4 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -165,6 +165,15 @@ files: table: project includes: - python_run_cugraph_dgl + - depends_on_pylibcugraphops + py_test_cugraph_dgl: + output: pyproject + pyproject_dir: python/cugraph-dgl + extras: + table: project.optional-dependencies + key: test + includes: + - test_python_common py_build_cugraph_pyg: output: pyproject pyproject_dir: python/cugraph-pyg @@ -179,6 +188,15 @@ files: table: project includes: - python_run_cugraph_pyg + - depends_on_pylibcugraphops + py_test_cugraph_pyg: + output: pyproject + pyproject_dir: python/cugraph-pyg + extras: + table: project.optional-dependencies + key: test + includes: + - test_python_common py_build_cugraph_service_client: output: pyproject pyproject_dir: python/cugraph-service/client diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 62fa8ab6368..65ee414da44 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. [build-system] @@ -18,15 +18,26 @@ authors = [ ] license = { text = "Apache 2.0" } requires-python = ">=3.9" +classifiers = [ + "Intended Audience :: Developers", + "Programming Language :: Python", +] dependencies = [ "cugraph==24.2.*", "numba>=0.57", "numpy>=1.21", + "pylibcugraphops==24.2.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.optional-dependencies] +test = [ + "pandas", + "pytest", + "pytest-benchmark", + "pytest-cov", + "pytest-xdist", + "scipy", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -classifiers = [ - "Intended Audience :: Developers", - "Programming Language :: Python", -] [project.urls] Homepage = "https://github.com/rapidsai/cugraph" diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index b0671644982..c4bd00bb86c 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. [build-system] @@ -29,12 +29,23 @@ dependencies = [ "cugraph==24.2.*", "numba>=0.57", "numpy>=1.21", + "pylibcugraphops==24.2.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/cugraph" Documentation = "https://docs.rapids.ai/api/cugraph/stable/" +[project.optional-dependencies] +test = [ + "pandas", + "pytest", + "pytest-benchmark", + "pytest-cov", + "pytest-xdist", + "scipy", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + [tool.setuptools] license-files = ["LICENSE"]