From e5bea87dd3d5f46571a0bd658874517d6c94bc83 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 13 Nov 2023 10:27:53 -0600 Subject: [PATCH] Use new `rapids-dask-dependency` metapackage for managing dask versions (#1968) Currently dask versions are pinned as part of every release cycle and then unpinned for the next development cycle across all of RAPIDS. This introduces a great deal of churn. To centralize the dependency, we have created a metapackage to manage the required dask version and this PR introduces that metapackage as a dependency of raft_dask. xref: https://github.com/rapidsai/cudf/pull/14364 Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Divye Gala (https://github.com/divyegala) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/raft/pull/1968 --- ci/build_wheel.sh | 2 ++ ci/release/update-version.sh | 6 +----- ci/test_wheel_raft_dask.sh | 3 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 4 +--- conda/environments/all_cuda-118_arch-x86_64.yaml | 4 +--- conda/environments/all_cuda-120_arch-aarch64.yaml | 4 +--- conda/environments/all_cuda-120_arch-x86_64.yaml | 4 +--- conda/recipes/raft-dask/meta.yaml | 4 +--- dependencies.yaml | 4 +--- python/raft-dask/pyproject.toml | 3 +-- 10 files changed, 10 insertions(+), 28 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 3dc7740a1e..5d06e46303 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -39,6 +39,8 @@ fi if [[ ${package_name} == "raft-dask" ]]; then sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} else sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} fi diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 471987eb40..0be1f1ad82 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -57,6 +57,7 @@ DEPENDENCIES=( dask-cuda pylibraft rmm + rapids-dask-dependency # ucx-py is handled separately below ) for FILE in dependencies.yaml conda/environments/*.yaml; do @@ -75,11 +76,6 @@ done sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/raft-dask/conda_build_config.yaml -# Wheel builds install dask-cuda from source, update its branch -for FILE in .github/workflows/*.yaml; do - sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE}; -done - for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" done diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index a20e950313..b70563b7a1 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -11,9 +11,6 @@ RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl -# Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.12 - # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/raft_dask*.whl)[test] diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 762d436028..c28f1961e6 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -20,10 +20,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - gmock>=1.13.0 @@ -49,6 +46,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov +- rapids-dask-dependency==23.12.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 48cd34c6ca..9b7c110bc3 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,10 +20,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 @@ -49,6 +46,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov +- rapids-dask-dependency==23.12.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-aarch64.yaml b/conda/environments/all_cuda-120_arch-aarch64.yaml index 9108fb6215..8d614d3c2c 100644 --- a/conda/environments/all_cuda-120_arch-aarch64.yaml +++ b/conda/environments/all_cuda-120_arch-aarch64.yaml @@ -21,10 +21,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - gmock>=1.13.0 @@ -45,6 +42,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov +- rapids-dask-dependency==23.12.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 8f1fbf6744..f9d65cee39 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -21,10 +21,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 @@ -45,6 +42,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov +- rapids-dask-dependency==23.12.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index db4fc860df..eae5a6affe 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -60,10 +60,8 @@ requirements: - cudatoolkit {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - - dask >=2023.9.2 - - dask-core >=2023.9.2 - dask-cuda ={{ minor_version }} - - distributed >=2023.9.2 + - rapids-dask-dependency ={{ minor_version }} - joblib >=0.11 - nccl >=2.9.9 - pylibraft {{ version }} diff --git a/dependencies.yaml b/dependencies.yaml index aba81d7ed9..419cd0c22e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -436,16 +436,14 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask>=2023.9.2 - dask-cuda==23.12.* - - distributed>=2023.9.2 - joblib>=0.11 - numba>=0.57 - *numpy + - rapids-dask-dependency==23.12.* - ucx-py==0.35.* - output_types: conda packages: - - dask-core>=2023.9.2 - ucx>=1.13.0 - ucx-proc=*=gpu - &ucx_py_conda ucx-py==0.35.* diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 7abb6231a1..be030f839d 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -35,12 +35,11 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "dask-cuda==23.12.*", - "dask>=2023.9.2", - "distributed>=2023.9.2", "joblib>=0.11", "numba>=0.57", "numpy>=1.21", "pylibraft==23.12.*", + "rapids-dask-dependency==23.12.*", "ucx-py==0.35.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [