diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index de9caa0fabe..4354ca40327 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,27 +68,20 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: pylibcugraph - package-dir: python/pylibcugraph - - # Note that this approach to cloning repos obviates any modification to - # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as is. + script: ci/build_wheel_pylibcugraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/" wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -98,28 +91,20 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: cugraph - package-dir: python/cugraph - - # Note that this approach to cloning repos obviates any modification to - # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as is. + script: ci/build_wheel_cugraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse" - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/" wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4d52cd26de4..17d7ac48907 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -78,47 +78,34 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: pull-request - package-name: pylibcugraph - package-dir: python/pylibcugraph + script: ci/build_wheel_pylibcugraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: pull-request - package-name: pylibcugraph - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" + script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: pull-request - package-name: cugraph - package-dir: python/cugraph + script: ci/build_wheel_cugraph.sh extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse" - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: pull-request - package-name: cugraph - # Always want to test against latest dask/distributed. - test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests. - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_cugraph.py" + script: ci/test_wheel_cugraph.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d697b8f1649..1a579a87a7f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,24 +32,19 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: pylibcugraph - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests" + script: ci/test_wheel_pylibcugraph.sh wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: cugraph - # Always want to test against latest dask/distributed. - test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests" + script: ci/test_wheel_cugraph.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 00000000000..fbf1e6fce2d --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +package_name=$1 +package_dir=$2 + +source rapids-configure-sccache +source rapids-date-string + +# Use gha-tools rapids-pip-wheel-version to generate wheel version then +# update the necessary files +version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +bash ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" +echo "The package name and/or version was modified in the package source. The git diff is:" +git diff + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh new file mode 100755 index 00000000000..657220d372b --- /dev/null +++ b/ci/build_wheel_cugraph.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +RAPIDS_PY_WHEEL_NAME=pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibcugraph +python -m pip install --no-deps ./local-pylibcugraph/pylibcugraph*.whl + +export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/" + +./ci/build_wheel.sh cugraph python/cugraph diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh new file mode 100755 index 00000000000..8d365bc250b --- /dev/null +++ b/ci/build_wheel_pylibcugraph.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/" + +./ci/build_wheel.sh pylibcugraph python/pylibcugraph diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100755 index 00000000000..146186ae2e7 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eoxu pipefail + +package_name=$1 +package_dir=$2 + +mkdir -p ./dist +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# echo to expand wildcard before adding `[extra]` requires for pip +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +python -m pip install $(echo ./dist/${package_name}*.whl)[test] + +# Run smoke tests for aarch64 pull requests +arch=$(uname -m) +if [[ "${arch}" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then + python ./ci/wheel_smoke_test_${package_name}.py +else + RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets python -m pytest ./python/${package_name}/${package_name}/tests +fi diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh new file mode 100755 index 00000000000..90af5313619 --- /dev/null +++ b/ci/test_wheel_cugraph.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eoxu pipefail + +# Download the pylibcugraph built in the previous step +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep +python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl + +# Always install latest dask for testing +python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08 + +# Only download test data for x86 +arch=$(uname -m) +if [[ "${arch}" == "x86_64" ]]; then + pushd ./datasets + bash ./get_test_data.sh + popd +fi + +./ci/test_wheel.sh cugraph python/cugraph diff --git a/ci/test_wheel_pylibcugraph.sh b/ci/test_wheel_pylibcugraph.sh new file mode 100755 index 00000000000..d04cb358d21 --- /dev/null +++ b/ci/test_wheel_pylibcugraph.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eoxu pipefail + +./ci/test_wheel.sh pylibcugraph python/pylibcugraph