From 120fc29ed8b4d407e3ce99fd40a67d772edb58d5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 25 Jul 2023 13:30:02 -0700 Subject: [PATCH] Switch to new wheel building pipeline (#13723) Moves the wheel build and test logic out of the workflow into the repo. This matches conda tests more closely and allows each repo to manage its own wheels more easily. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Divye Gala (https://github.com/divyegala) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/13723 --- .github/workflows/build.yaml | 16 +++++++--------- .github/workflows/pr.yaml | 27 ++++++++++----------------- .github/workflows/test.yaml | 13 +++++-------- ci/build_wheel_cudf.sh | 26 ++++++++++++++++++++++++++ ci/build_wheel_dask_cudf.sh | 24 ++++++++++++++++++++++++ ci/test_wheel_cudf.sh | 17 +++++++++++++++++ ci/test_wheel_dask_cudf.sh | 19 +++++++++++++++++++ 7 files changed, 108 insertions(+), 34 deletions(-) create mode 100755 ci/build_wheel_cudf.sh create mode 100755 ci/build_wheel_dask_cudf.sh create mode 100755 ci/test_wheel_cudf.sh create mode 100755 ci/test_wheel_dask_cudf.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 15f3269efaf..fd782da424a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -69,19 +69,17 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: cudf - package-dir: python/cudf - skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" + script: ci/build_wheel_cudf.sh wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -91,18 +89,18 @@ jobs: wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: dask_cudf - package-dir: python/dask_cudf + script: ci/build_wheel_dask_cudf.sh wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f157a3b04cb..6c7b19df4bf 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -98,37 +98,30 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: pull-request - package-name: cudf - package-dir: python/cudf - skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" + script: "ci/build_wheel_cudf.sh" wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: pull-request - package-name: cudf - test-unittest: "python -m pytest -n 8 ./python/cudf/cudf/tests" - test-smoketest: "python ./ci/wheel_smoke_test_cudf.py" + script: ci/test_wheel_cudf.sh wheel-build-dask-cudf: needs: wheel-tests-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) build_type: pull-request - package-name: dask_cudf - package-dir: python/dask_cudf - before-wheel: "RAPIDS_PY_WHEEL_NAME=cudf_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-cudf && python -m pip install --no-deps ./local-cudf/cudf*.whl" + script: "ci/build_wheel_dask_cudf.sh" wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) build_type: pull-request - package-name: dask_cudf - # Install the cudf we just built, and also test against latest dask/distributed/dask-cuda. - test-before: "RAPIDS_PY_WHEEL_NAME=cudf_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-cudf-dep && python -m pip install --no-deps ./local-cudf-dep/cudf*.whl && python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-unittest: "python -m pytest -n 8 ./python/dask_cudf/dask_cudf/tests" + script: ci/test_wheel_dask_cudf.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b5563d79466..9428f9539ad 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -79,23 +79,20 @@ jobs: run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: cudf - test-unittest: "python -m pytest -n 8 ./python/cudf/cudf/tests" + script: ci/test_wheel_cudf.sh wheel-tests-dask-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yml@branch-23.08 with: + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: dask_cudf - # Test against latest dask/distributed/dask-cuda. - test-before: "python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-unittest: "python -m pytest -n 8 ./python/dask_cudf/dask_cudf/tests" + script: ci/test_wheel_dask_cudf.sh diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh new file mode 100755 index 00000000000..c20a5162788 --- /dev/null +++ b/ci/build_wheel_cudf.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-configure-sccache +source rapids-date-string + +# Use gha-tools rapids-pip-wheel-version to generate wheel version then +# update the necessary files +version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" +echo "The package name and/or version was modified in the package source. The git diff is:" +git diff + +cd python/cudf + +SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_dask_cudf.sh b/ci/build_wheel_dask_cudf.sh new file mode 100755 index 00000000000..9af90d3a863 --- /dev/null +++ b/ci/build_wheel_dask_cudf.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-configure-sccache +source rapids-date-string + +# Use gha-tools rapids-pip-wheel-version to generate wheel version then +# update the necessary files +version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" +echo "The package name and/or version was modified in the package source. The git diff is:" +git diff + +cd python/dask_cudf + +# Hardcode the output dir +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 dist diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh new file mode 100755 index 00000000000..83e24ab3ff1 --- /dev/null +++ b/ci/test_wheel_cudf.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eou pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/cudf*.whl)[test] + +# Run smoke tests for aarch64 pull requests +if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then + python ./ci/wheel_smoke_test_cudf.py +else + python -m pytest -n 8 ./python/cudf/cudf/tests +fi diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh new file mode 100755 index 00000000000..4527e91248d --- /dev/null +++ b/ci/test_wheel_dask_cudf.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eou pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + +# Download the cudf built in the previous step +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep +python -m pip install --no-deps ./local-cudf-dep/cudf*.whl + +# Always install latest dask for testing +python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08 + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/dask_cudf*.whl)[test] + +python -m pytest -n 8 ./python/dask_cudf/dask_cudf/tests