From e2902831d307bb7ad59046ed9c9646a15807f3e7 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Wed, 6 Sep 2023 13:19:16 -0400 Subject: [PATCH 1/2] Simplify wheel build scripts and allow alphas of RAPIDS dependencies (#1804) This PR: 1. Removes `ci/apply_wheel_modifications.sh` and uses it inline in wheel build scripts 2. Allows for specifying alpha versioned dependencies of RAPIDS projects Authors: - Divye Gala (https://github.com/divyegala) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/raft/pull/1804 --- ci/build_wheel.sh | 33 ++++++++++++++++++++++--- ci/build_wheel_raft_dask.sh | 5 ---- ci/release/apply_wheel_modifications.sh | 25 ------------------- 3 files changed, 30 insertions(+), 33 deletions(-) delete mode 100755 ci/release/apply_wheel_modifications.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index a9f7f64294..af66de7f8b 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -15,9 +15,36 @@ version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" -echo "The package name and/or version was modified in the package source. The git diff is:" -git diff +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" + +sed -i "s/^version = .*/version = \"${version_override}\"/g" ${pyproject_file} +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +if [[ ${package_name} == "raft_dask" ]]; then + sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -i "s/ucx-py/ucx-py${PACKAGE_CUDA_SUFFIX}/g" python/raft-dask/pyproject.toml +else + sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +fi + +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} +fi cd "${package_dir}" diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index f0204d45c0..e90412fb80 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -6,9 +6,4 @@ set -euo pipefail # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CONFIGURE_OPTIONS="-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -RAPIDS_PY_WHEEL_NAME=pylibraft_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibraft -python -m pip install --no-deps ./local-pylibraft/pylibraft*.whl - ci/build_wheel.sh raft_dask python/raft-dask diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh deleted file mode 100755 index fd6c2f929e..0000000000 --- a/ci/release/apply_wheel_modifications.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Usage: bash apply_wheel_modifications.sh - -VERSION=${1} -CUDA_SUFFIX=${2} - -# pyproject.toml versions -sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/pylibraft/pyproject.toml -sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/raft-dask/pyproject.toml - -# pylibraft pyproject.toml cuda suffixes -sed -i "s/^name = \"pylibraft\"/name = \"pylibraft${CUDA_SUFFIX}\"/g" python/pylibraft/pyproject.toml -sed -i "s/rmm/rmm${CUDA_SUFFIX}/g" python/pylibraft/pyproject.toml - -# raft-dask pyproject.toml cuda suffixes -sed -i "s/^name = \"raft-dask\"/name = \"raft-dask${CUDA_SUFFIX}\"/g" python/raft-dask/pyproject.toml -sed -i "s/pylibraft/pylibraft${CUDA_SUFFIX}/g" python/raft-dask/pyproject.toml -sed -i "s/ucx-py/ucx-py${CUDA_SUFFIX}/g" python/raft-dask/pyproject.toml - -if [[ $CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" python/pylibraft/pyproject.toml - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/pylibraft/pyproject.toml -fi From f691fc9651dc83a2e6b3ce7a8de524f4c2a57c44 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Wed, 6 Sep 2023 17:40:35 -0400 Subject: [PATCH 2/2] Fix `raft-dask` naming in wheel builds (#1805) Before https://github.com/rapidsai/raft/pull/1804/files, we were using `raft-dask` and `raft_dask` interchangeably because they were hardcoded. Since the package name is now capture as a variable, it's a bug to use `raft_dask` as the pyproject name does not get updated correctly and we end up publishing `raft-dask` wheels without `cu{version}` suffixes Authors: - Divye Gala (https://github.com/divyegala) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/raft/pull/1805 --- ci/build_wheel.sh | 5 +++-- ci/build_wheel_raft_dask.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index af66de7f8b..662a11ad0e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string @@ -34,7 +35,7 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi -if [[ ${package_name} == "raft_dask" ]]; then +if [[ ${package_name} == "raft-dask" ]]; then sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} sed -i "s/ucx-py/ucx-py${PACKAGE_CUDA_SUFFIX}/g" python/raft-dask/pyproject.toml else @@ -54,4 +55,4 @@ python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index e90412fb80..ff89f4da23 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -6,4 +6,4 @@ set -euo pipefail # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CONFIGURE_OPTIONS="-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" -ci/build_wheel.sh raft_dask python/raft-dask +ci/build_wheel.sh raft-dask python/raft-dask