From 9ad7389398a7abdbbee070a220b2d638c29bc876 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 7 Feb 2024 18:46:45 -0600 Subject: [PATCH 1/2] handle more RAPIDS version formats in update-version.sh, refactor dependencies.yaml (#4109) Contributes to https://github.com/rapidsai/build-planning/issues/13. Updates `update-version.sh` to correctly handle RAPIDS dependencies like `cudf-cu12==24.2.*`. This also pulls in some dependency refactoring originally added in #4088, which allows greater use of dependencies.yaml globs (and therefore less maintenance effort to support new CUDA versions). Authors: - James Lamb (https://github.com/jameslamb) - Brad Rees (https://github.com/BradReesWork) - Ray Douglass (https://github.com/raydouglass) - Bradley Dice (https://github.com/bdice) Approvers: - Jake Awe (https://github.com/AyodeAwe) - Bradley Dice (https://github.com/bdice) - Brad Rees (https://github.com/BradReesWork) - Rick Ratzel (https://github.com/rlratzel) URL: https://github.com/rapidsai/cugraph/pull/4109 --- ci/release/update-version.sh | 11 ++-- dependencies.yaml | 124 ++++++++++------------------------- 2 files changed, 38 insertions(+), 97 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index c091bd1ed33..9e284f49b5b 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -96,13 +96,12 @@ DEPENDENCIES=( ) for DEP in "${DEPENDENCIES[@]}"; do for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do - sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE} - sed_runner "/-.* ${DEP}-cu[0-9][0-9]==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE} - sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*/g" ${FILE} + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}" + sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*/g" "${FILE}" done for FILE in python/**/pyproject.toml python/**/**/pyproject.toml; do - sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" ${FILE} - sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_VERSION}.*\"/g" ${FILE} + sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" "${FILE}" + sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_VERSION}.*\"/g" "${FILE}" done done diff --git a/dependencies.yaml b/dependencies.yaml index 9d8ba3e5340..cfefe3b9ff9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -567,17 +567,12 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.2"} - packages: &rmm_packages_pip_cu12 + - matrix: {cuda: "12.*"} + packages: - rmm-cu12==24.4.* - - {matrix: {cuda: "12.1"}, packages: *rmm_packages_pip_cu12} - - {matrix: {cuda: "12.0"}, packages: *rmm_packages_pip_cu12} - - matrix: {cuda: "11.8"} - packages: &rmm_packages_pip_cu11 + - matrix: {cuda: "11.*"} + packages: - rmm-cu11==24.4.* - - {matrix: {cuda: "11.5"}, packages: *rmm_packages_pip_cu11} - - {matrix: {cuda: "11.4"}, packages: *rmm_packages_pip_cu11} - - {matrix: {cuda: "11.2"}, packages: *rmm_packages_pip_cu11} - {matrix: null, packages: [*rmm_conda]} depends_on_cudf: @@ -593,17 +588,12 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.2"} - packages: &cudf_packages_pip_cu12 + - matrix: {cuda: "12.*"} + packages: - cudf-cu12==24.4.* - - {matrix: {cuda: "12.1"}, packages: *cudf_packages_pip_cu12} - - {matrix: {cuda: "12.0"}, packages: *cudf_packages_pip_cu12} - - matrix: {cuda: "11.8"} - packages: &cudf_packages_pip_cu11 + - matrix: {cuda: "11.*"} + packages: - cudf-cu11==24.4.* - - {matrix: {cuda: "11.5"}, packages: *cudf_packages_pip_cu11} - - {matrix: {cuda: "11.4"}, packages: *cudf_packages_pip_cu11} - - {matrix: {cuda: "11.2"}, packages: *cudf_packages_pip_cu11} - {matrix: null, packages: [*cudf_conda]} depends_on_dask_cudf: @@ -619,17 +609,12 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.2"} - packages: &dask_cudf_packages_pip_cu12 + - matrix: {cuda: "12.*"} + packages: - dask-cudf-cu12==24.4.* - - {matrix: {cuda: "12.1"}, packages: *dask_cudf_packages_pip_cu12} - - {matrix: {cuda: "12.0"}, packages: *dask_cudf_packages_pip_cu12} - - matrix: {cuda: "11.8"} - packages: &dask_cudf_packages_pip_cu11 + - matrix: {cuda: "11.*"} + packages: - dask-cudf-cu11==24.4.* - - {matrix: {cuda: "11.5"}, packages: *dask_cudf_packages_pip_cu11} - - {matrix: {cuda: "11.4"}, packages: *dask_cudf_packages_pip_cu11} - - {matrix: {cuda: "11.2"}, packages: *dask_cudf_packages_pip_cu11} - {matrix: null, packages: [*dask_cudf_conda]} depends_on_pylibraft: @@ -645,17 +630,12 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.2"} - packages: &pylibraft_packages_pip_cu12 + - matrix: {cuda: "12.*"} + packages: - pylibraft-cu12==24.4.* - - {matrix: {cuda: "12.1"}, packages: *pylibraft_packages_pip_cu12} - - {matrix: {cuda: "12.0"}, packages: *pylibraft_packages_pip_cu12} - - matrix: {cuda: "11.8"} - packages: &pylibraft_packages_pip_cu11 + - matrix: {cuda: "11.*"} + packages: - pylibraft-cu11==24.4.* - - {matrix: {cuda: "11.5"}, packages: *pylibraft_packages_pip_cu11} - - {matrix: {cuda: "11.4"}, packages: *pylibraft_packages_pip_cu11} - - {matrix: {cuda: "11.2"}, packages: *pylibraft_packages_pip_cu11} - {matrix: null, packages: [*pylibraft_conda]} depends_on_raft_dask: @@ -671,17 +651,12 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.2"} - packages: &raft_dask_packages_pip_cu12 + - matrix: {cuda: "12.*"} + packages: - raft-dask-cu12==24.4.* - - {matrix: {cuda: "12.1"}, packages: *raft_dask_packages_pip_cu12} - - {matrix: {cuda: "12.0"}, packages: *raft_dask_packages_pip_cu12} - - matrix: {cuda: "11.8"} - packages: &raft_dask_packages_pip_cu11 + - matrix: {cuda: "11.*"} + packages: - raft-dask-cu11==24.4.* - - {matrix: {cuda: "11.5"}, packages: *raft_dask_packages_pip_cu11} - - {matrix: {cuda: "11.4"}, packages: *raft_dask_packages_pip_cu11} - - {matrix: {cuda: "11.2"}, packages: *raft_dask_packages_pip_cu11} - {matrix: null, packages: [*raft_dask_conda]} depends_on_pylibcugraph: @@ -697,17 +672,12 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.2"} - packages: &pylibcugraph_packages_pip_cu12 + - matrix: {cuda: "12.*"} + packages: - pylibcugraph-cu12==24.4.* - - {matrix: {cuda: "12.1"}, packages: *pylibcugraph_packages_pip_cu12} - - {matrix: {cuda: "12.0"}, packages: *pylibcugraph_packages_pip_cu12} - - matrix: {cuda: "11.8"} - packages: &pylibcugraph_packages_pip_cu11 + - matrix: {cuda: "11.*"} + packages: - pylibcugraph-cu11==24.4.* - - {matrix: {cuda: "11.5"}, packages: *pylibcugraph_packages_pip_cu11} - - {matrix: {cuda: "11.4"}, packages: *pylibcugraph_packages_pip_cu11} - - {matrix: {cuda: "11.2"}, packages: *pylibcugraph_packages_pip_cu11} - {matrix: null, packages: [*pylibcugraph_conda]} depends_on_pylibcugraphops: @@ -723,17 +693,12 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.2"} - packages: &pylibcugraphops_packages_pip_cu12 + - matrix: {cuda: "12.*"} + packages: - pylibcugraphops-cu12==24.4.* - - {matrix: {cuda: "12.1"}, packages: *pylibcugraphops_packages_pip_cu12} - - {matrix: {cuda: "12.0"}, packages: *pylibcugraphops_packages_pip_cu12} - - matrix: {cuda: "11.8"} - packages: &pylibcugraphops_packages_pip_cu11 + - matrix: {cuda: "11.*"} + packages: - pylibcugraphops-cu11==24.4.* - - {matrix: {cuda: "11.5"}, packages: *pylibcugraphops_packages_pip_cu11} - - {matrix: {cuda: "11.4"}, packages: *pylibcugraphops_packages_pip_cu11} - - {matrix: {cuda: "11.2"}, packages: *pylibcugraphops_packages_pip_cu11} - {matrix: null, packages: [*pylibcugraphops_conda]} depends_on_cupy: @@ -744,33 +709,10 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - # All CUDA 12 + x86_64 versions - - matrix: {cuda: "12.2", arch: x86_64} - packages: &cupy_packages_cu12_x86_64 + - matrix: {cuda: "12.*"} + packages: - cupy-cuda12x>=12.0.0 - - {matrix: {cuda: "12.1", arch: x86_64}, packages: *cupy_packages_cu12_x86_64} - - {matrix: {cuda: "12.0", arch: x86_64}, packages: *cupy_packages_cu12_x86_64} - - # All CUDA 12 + aarch64 versions - - matrix: {cuda: "12.2", arch: aarch64} - packages: &cupy_packages_cu12_aarch64 - - cupy-cuda12x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. - - {matrix: {cuda: "12.1", arch: aarch64}, packages: *cupy_packages_cu12_aarch64} - - {matrix: {cuda: "12.0", arch: aarch64}, packages: *cupy_packages_cu12_aarch64} - - # All CUDA 11 + x86_64 versions - - matrix: {cuda: "11.8", arch: x86_64} - packages: &cupy_packages_cu11_x86_64 + - matrix: {cuda: "11.*"} + packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - - {matrix: {cuda: "11.5", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} - - {matrix: {cuda: "11.4", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} - - {matrix: {cuda: "11.2", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} - - # All CUDA 11 + aarch64 versions - - matrix: {cuda: "11.8", arch: aarch64} - packages: &cupy_packages_cu11_aarch64 - - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. - - {matrix: {cuda: "11.5", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} - - {matrix: {cuda: "11.4", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} - - {matrix: {cuda: "11.2", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} - - {matrix: null, packages: [cupy-cuda11x>=12.0.0]} + - {matrix: null, packages: *cupy_packages_cu11} From bb5bee24a2e86a7b979a481d65b802e7ed28e1c8 Mon Sep 17 00:00:00 2001 From: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Date: Thu, 8 Feb 2024 17:15:59 -0500 Subject: [PATCH 2/2] Revert "Exclude tests from builds (#4147)" (#4157) This reverts commit 5c6c4d3e716c49932ba46642a4d0ad9cb82f0036. --- python/cugraph-dgl/pyproject.toml | 2 -- python/cugraph-equivariant/pyproject.toml | 1 - python/cugraph-equivariant/setup.py | 20 ++++++++++++++++++++ python/cugraph-pyg/pyproject.toml | 2 -- python/cugraph-service/client/pyproject.toml | 1 - python/cugraph-service/server/pyproject.toml | 1 - python/cugraph/pyproject.toml | 1 - python/nx-cugraph/pyproject.toml | 1 - python/pylibcugraph/pyproject.toml | 1 - 9 files changed, 20 insertions(+), 10 deletions(-) create mode 100644 python/cugraph-equivariant/setup.py diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index b575559bee5..65ee414da44 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -53,5 +53,3 @@ version = {file = "cugraph_dgl/VERSION"} include = [ "cugraph_dgl*", ] -exclude = ["*tests*"] - diff --git a/python/cugraph-equivariant/pyproject.toml b/python/cugraph-equivariant/pyproject.toml index 3cb0183e3a2..f261b0e3535 100644 --- a/python/cugraph-equivariant/pyproject.toml +++ b/python/cugraph-equivariant/pyproject.toml @@ -62,4 +62,3 @@ include = [ "cugraph_equivariant*", "cugraph_equivariant.*", ] -exclude = ["*tests*"] diff --git a/python/cugraph-equivariant/setup.py b/python/cugraph-equivariant/setup.py new file mode 100644 index 00000000000..acd0df3f717 --- /dev/null +++ b/python/cugraph-equivariant/setup.py @@ -0,0 +1,20 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import find_packages, setup + +if __name__ == "__main__": + packages = find_packages(include=["cugraph_equivariant*"]) + setup( + package_data={key: ["VERSION"] for key in packages}, + ) diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index 2284b454888..b8666c0d806 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -57,5 +57,3 @@ include = [ "cugraph_pyg*", "cugraph_pyg.*", ] -exclude = ["*tests*"] - diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml index ad4a98da507..59539693877 100644 --- a/python/cugraph-service/client/pyproject.toml +++ b/python/cugraph-service/client/pyproject.toml @@ -42,4 +42,3 @@ version = {file = "cugraph_service_client/VERSION"} include = [ "cugraph_service_client", ] -exclude = ["*tests*"] diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index 2d1c00d027e..84c0358668b 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -71,4 +71,3 @@ include = [ "cugraph_service_server", "cugraph_service_server.*" ] -exclude = ["*tests*"] diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index ff7fc1a8d89..6a9d88bf5c8 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -72,7 +72,6 @@ build-dir = "build/{wheel_tag}" cmake.build-type = "Release" cmake.minimum-version = "3.26.4" ninja.make-fallback = true -sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["cugraph"] diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index d8adeec0b38..63ac115918f 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -79,7 +79,6 @@ include = [ "_nx_cugraph*", "_nx_cugraph.*", ] -exclude = ["*tests*"] [tool.black] line-length = 88 diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 97c94c1b9db..1d27d952af1 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -59,7 +59,6 @@ build-dir = "build/{wheel_tag}" cmake.build-type = "Release" cmake.minimum-version = "3.26.4" ninja.make-fallback = true -sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["pylibcugraph"]