From 4e79b623a9e248d3ea090d771c94e2d69d15d070 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 8 Oct 2024 16:47:03 -0500 Subject: [PATCH 1/5] make package installations in CI stricter --- ci/build_docs.sh | 14 +++++++------- ci/test_cpp.sh | 6 +++++- ci/test_python.sh | 7 ++++++- ci/test_wheel_raft_dask.sh | 5 +++-- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index a2447f5f06..4ce5d0bb48 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,6 +6,9 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +export RAPIDS_VERSION_NUMBER="$RAPIDS_VERSION_MAJOR_MINOR" + rapids-dependency-file-generator \ --output conda \ --file-key docs \ @@ -23,14 +26,11 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libraft \ - libraft-headers \ - pylibraft \ - raft-dask + "libraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ + "libraft-headers=${RAPIDS_VERSION_MAJOR_MINOR}" \ + "pylibraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ + "raft-dask=${RAPIDS_VERSION_MAJOR_MINOR}" -export RAPIDS_VERSION="$(rapids-version)" -export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" -export RAPIDS_VERSION_NUMBER="$RAPIDS_VERSION_MAJOR_MINOR" export RAPIDS_DOCS_DIR="$(mktemp -d)" rapids-logger "Build CPP docs" diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 05323e4f5d..482e0bc043 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" + rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -29,7 +31,9 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - libraft-headers libraft libraft-tests + "libraft-headers=${RAPIDS_VERSION_MAJOR_MINOR}" \ + "libraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ + "libraft-tests=${RAPIDS_VERSION_MAJOR_MINOR}" rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_python.sh b/ci/test_python.sh index 01e5ac9456..ae7f712273 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" + rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -34,7 +36,10 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libraft libraft-headers pylibraft raft-dask + "libraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ + "libraft-headers=${RAPIDS_VERSION_MAJOR_MINOR}" \ + "pylibraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ + "raft-dask=${RAPIDS_VERSION_MAJOR_MINOR}" rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index 9b1187592d..a778a3ec51 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -9,10 +9,11 @@ RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels # Download the pylibraft built in the previous step RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep -python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install -v "$(echo ./dist/raft_dask_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" +python -m pip install -v \ + ./local-pylibraft-dep/pylibraft*.whl \ + "$(echo ./dist/raft_dask_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" test_dir="python/raft-dask/raft_dask/test" From 08c72c78a8d631fe2d305c10ecda98b46568aafb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 9 Oct 2024 11:05:01 -0500 Subject: [PATCH 2/5] make version constraint even stricter --- ci/build_docs.sh | 13 ++++++------- ci/test_cpp.sh | 8 ++++---- ci/test_python.sh | 10 +++++----- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 4ce5d0bb48..a5f87685dc 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,8 +6,7 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" -export RAPIDS_VERSION_NUMBER="$RAPIDS_VERSION_MAJOR_MINOR" +RAPIDS_VERSION="$(rapids-version)" rapids-dependency-file-generator \ --output conda \ @@ -26,10 +25,10 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - "libraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ - "libraft-headers=${RAPIDS_VERSION_MAJOR_MINOR}" \ - "pylibraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ - "raft-dask=${RAPIDS_VERSION_MAJOR_MINOR}" + "libraft=${RAPIDS_VERSION}" \ + "libraft-headers=${RAPIDS_VERSION}" \ + "pylibraft=${RAPIDS_VERSION}" \ + "raft-dask=${RAPIDS_VERSION}" export RAPIDS_DOCS_DIR="$(mktemp -d)" @@ -45,4 +44,4 @@ mkdir -p "${RAPIDS_DOCS_DIR}/raft/"html mv _html/* "${RAPIDS_DOCS_DIR}/raft/html" popd -rapids-upload-docs +RAPIDS_VERSION_NUMBER="$(rapids-version-major-minor)" rapids-upload-docs diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 482e0bc043..9d0edc6b21 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,7 +8,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +RAPIDS_VERSION="$(rapids-version)" rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ @@ -31,9 +31,9 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - "libraft-headers=${RAPIDS_VERSION_MAJOR_MINOR}" \ - "libraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ - "libraft-tests=${RAPIDS_VERSION_MAJOR_MINOR}" + "libraft-headers=${RAPIDS_VERSION}" \ + "libraft=${RAPIDS_VERSION}" \ + "libraft-tests=${RAPIDS_VERSION}" rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_python.sh b/ci/test_python.sh index ae7f712273..af93d2e04b 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,7 +8,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +RAPIDS_VERSION="$(rapids-version)" rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ @@ -36,10 +36,10 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - "libraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ - "libraft-headers=${RAPIDS_VERSION_MAJOR_MINOR}" \ - "pylibraft=${RAPIDS_VERSION_MAJOR_MINOR}" \ - "raft-dask=${RAPIDS_VERSION_MAJOR_MINOR}" + "libraft=${RAPIDS_VERSION}" \ + "libraft-headers=${RAPIDS_VERSION}" \ + "pylibraft=${RAPIDS_VERSION}" \ + "raft-dask=${RAPIDS_VERSION}" rapids-logger "Check GPU usage" nvidia-smi From aa2755c1af493c9fc1502d66fe4018c240519f41 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 9 Oct 2024 16:02:45 -0500 Subject: [PATCH 3/5] rapids-generate-version, not rapids-version --- ci/build_docs.sh | 2 +- ci/test_cpp.sh | 2 +- ci/test_python.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index a5f87685dc..0accc17f1e 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,7 +6,7 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +RAPIDS_VERSION="$(rapids-generate-version)" rapids-dependency-file-generator \ --output conda \ diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 9d0edc6b21..dd9d7ef76d 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,7 +8,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +RAPIDS_VERSION="$(rapids-generate-version)" rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ diff --git a/ci/test_python.sh b/ci/test_python.sh index af93d2e04b..73a8def6df 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,7 +8,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +RAPIDS_VERSION="$(rapids-generate-version)" rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ From 090a0c0c17d722230223ad489051fb38398a2d47 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 9 Oct 2024 16:45:48 -0500 Subject: [PATCH 4/5] Revert "rapids-generate-version, not rapids-version" This reverts commit aa2755c1af493c9fc1502d66fe4018c240519f41. --- ci/build_docs.sh | 2 +- ci/test_cpp.sh | 2 +- ci/test_python.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 0accc17f1e..a5f87685dc 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,7 +6,7 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-generate-version)" +RAPIDS_VERSION="$(rapids-version)" rapids-dependency-file-generator \ --output conda \ diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index dd9d7ef76d..9d0edc6b21 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,7 +8,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-generate-version)" +RAPIDS_VERSION="$(rapids-version)" rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ diff --git a/ci/test_python.sh b/ci/test_python.sh index 73a8def6df..af93d2e04b 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,7 +8,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-generate-version)" +RAPIDS_VERSION="$(rapids-version)" rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ From 47797def86e07583c55f4e1263b4b54490f488eb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 10 Oct 2024 09:25:38 -0500 Subject: [PATCH 5/5] RAPIDS_VERSION_MAJOR_MINOR is needed for Doxygen --- ci/build_docs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index a5f87685dc..aff7674892 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -7,6 +7,7 @@ rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh RAPIDS_VERSION="$(rapids-version)" +export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" rapids-dependency-file-generator \ --output conda \ @@ -44,4 +45,4 @@ mkdir -p "${RAPIDS_DOCS_DIR}/raft/"html mv _html/* "${RAPIDS_DOCS_DIR}/raft/html" popd -RAPIDS_VERSION_NUMBER="$(rapids-version-major-minor)" rapids-upload-docs +RAPIDS_VERSION_NUMBER="${RAPIDS_VERSION_MAJOR_MINOR}" rapids-upload-docs