From d3478f729ab8479a14c4e4197faae87d55bb01bd Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Fri, 8 Mar 2024 11:15:34 -0600 Subject: [PATCH] Fix 'cuvs' conda package to only include Python artifacts and clean build.sh (#40) PR does the following: - [x] Stops the inclusion of C/C++ artifacts in the `cuvs` conda package - [x] Cleans `build.sh` from code that was copied from RAFT - [x] Adds `dlpack` to development environments - [x] Other small fixes and cleanups Authors: - Dante Gama Dessavre (https://github.com/dantegd) - Bradley Dice (https://github.com/bdice) - Ben Frederickson (https://github.com/benfred) Approvers: - Ben Frederickson (https://github.com/benfred) - Corey J. Nolet (https://github.com/cjnolet) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cuvs/pull/40 --- build.sh | 52 +++----------- ci/build_wheel_cuvs.sh | 2 +- .../all_cuda-118_arch-aarch64.yaml | 1 + .../all_cuda-118_arch-x86_64.yaml | 1 + .../all_cuda-122_arch-aarch64.yaml | 1 + .../all_cuda-122_arch-x86_64.yaml | 1 + conda/recipes/cuvs/meta.yaml | 6 ++ conda/recipes/libcuvs/meta.yaml | 72 ++++++++++++++++--- dependencies.yaml | 7 +- python/cuvs/CMakeLists.txt | 13 +++- python/cuvs/pyproject.toml | 3 +- 11 files changed, 96 insertions(+), 63 deletions(-) diff --git a/build.sh b/build.sh index c6e09c5bf..d53f7c354 100755 --- a/build.sh +++ b/build.sh @@ -48,13 +48,13 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= /dev/null 2>&1 - fi - - # Try to uninstall via pip if it is installed - if [ -x "$(command -v pip)" ]; then - echo "Using pip to uninstall cuvs-dask" - pip uninstall -y cuvs-dask - - # Otherwise, try to uninstall through conda if that's where things are installed - elif [ -x "$(command -v conda)" ] && [ "$INSTALL_PREFIX" == "$CONDA_PREFIX" ]; then - echo "Using conda to uninstall cuvs-dask" - conda uninstall -y cuvs-dask - - # Otherwise, fail - else - echo "Could not uninstall cuvs-dask from pip or conda. cuvs-dask package will need to be manually uninstalled." - fi - fi exit 0 fi @@ -261,24 +239,14 @@ if hasArg --allgpuarch; then BUILD_ALL_GPU_ARCH=1 fi - -# Append `-DFIND_CUVS_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUVS_CPP"* ]]; then - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUVS_CPP=ON" -fi - if hasArg tests || (( ${NUMARGS} == 0 )); then BUILD_TESTS=ON CMAKE_TARGET="${CMAKE_TARGET};${TEST_TARGETS}" # Force compile library when needed test targets are specified - if [[ $CMAKE_TARGET == *"CLUSTER_TEST"* || \ - $CMAKE_TARGET == *"DISTANCE_TEST"* || \ - $CMAKE_TARGET == *"NEIGHBORS_ANN_CAGRA_TEST"* || \ - $CMAKE_TARGET == *"NEIGHBORS_ANN_IVF_TEST"* || \ - $CMAKE_TARGET == *"NEIGHBORS_ANN_NN_DESCENT_TEST"* || \ - $CMAKE_TARGET == *"NEIGHBORS_TEST"* || \ - $CMAKE_TARGET == *"STATS_TEST"* ]]; then + if [[ $CMAKE_TARGET == *"CAGRA_C_TEST"* || \ + $CMAKE_TARGET == *"INTEROP_TEST"* || \ + $CMAKE_TARGET == *"NEIGHBORS_ANN_CAGRA_TEST"* ]]; then echo "-- Enabling compiled lib for gtests" COMPILE_LIBRARY=ON fi @@ -304,13 +272,10 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi -# Append `-DFIND_CUVS_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. - - SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUVS_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_CUVS_CPP=ON" + SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DFIND_CUVS_CPP=ON" fi # If clean given, run it prior to any other steps @@ -353,7 +318,6 @@ if (( ${NUMARGS} == 0 )) || hasArg libcuvs || hasArg docs || hasArg tests || has -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${CUVS_CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DCUVS_COMPILE_LIBRARY=${COMPILE_LIBRARY} \ -DBUILD_C_LIBRARY=${COMPILE_LIBRARY} \ -DCUVS_NVTX=${NVTX} \ -DCUDA_LOG_COMPILE_TIME=${LOG_COMPILE_TIME} \ @@ -422,9 +386,9 @@ fi # Build and (optionally) install the cuvs Python package if (( ${NUMARGS} == 0 )) || hasArg python; then - SKBUILD_CONFIGURE_OPTIONS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ + SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL}" \ - python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/cuvs + python -m pip install --no-build-isolation --no-deps -vvv ${REPODIR}/python/cuvs fi if hasArg docs; then diff --git a/ci/build_wheel_cuvs.sh b/ci/build_wheel_cuvs.sh index 238483b6e..0fe28c2f7 100755 --- a/ci/build_wheel_cuvs.sh +++ b/ci/build_wheel_cuvs.sh @@ -4,6 +4,6 @@ set -euo pipefail # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DFIND_CUVS_CPP=OFF" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF -DFIND_CUVS_CPP=OFF" ci/build_wheel.sh cuvs python/cuvs diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index e4f922378..d711fc555 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -20,6 +20,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 +- dlpack>=0.8,<1.0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - gmock>=1.13.0 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index a26314b22..b4c2c6857 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,6 +20,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 +- dlpack>=0.8,<1.0 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 diff --git a/conda/environments/all_cuda-122_arch-aarch64.yaml b/conda/environments/all_cuda-122_arch-aarch64.yaml index 91d55917a..85d96e5c9 100644 --- a/conda/environments/all_cuda-122_arch-aarch64.yaml +++ b/conda/environments/all_cuda-122_arch-aarch64.yaml @@ -21,6 +21,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 +- dlpack>=0.8,<1.0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - gmock>=1.13.0 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index f27d131ff..16cc655c6 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -21,6 +21,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 +- dlpack>=0.8,<1.0 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 diff --git a/conda/recipes/cuvs/meta.yaml b/conda/recipes/cuvs/meta.yaml index 0902dd15a..b2a807400 100644 --- a/conda/recipes/cuvs/meta.yaml +++ b/conda/recipes/cuvs/meta.yaml @@ -22,6 +22,9 @@ build: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-cudart-dev {% endif %} requirements: @@ -43,6 +46,7 @@ requirements: - cudatoolkit {% else %} - cuda-python >=12.0,<13.0a0 + - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} - cython >=3.0.0 @@ -57,6 +61,8 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit + {% else %} + - cuda-cudart {% endif %} - pylibraft {{ minor_version }} - libcuvs {{ version }} diff --git a/conda/recipes/libcuvs/meta.yaml b/conda/recipes/libcuvs/meta.yaml index 4b6ff87e9..4ad3618c3 100644 --- a/conda/recipes/libcuvs/meta.yaml +++ b/conda/recipes/libcuvs/meta.yaml @@ -1,13 +1,12 @@ # Copyright (c) 2022-2024, NVIDIA CORPORATION. # Usage: - # conda build . -c conda-forge -c nvidia -c rapidsai - {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} - {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} - {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} - {% set cuda_major = cuda_version.split('.')[0] %} - {% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 - {% set date_string = environ['RAPIDS_DATE_STRING'] %} +# conda build . -c conda-forge -c nvidia -c rapidsai +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} +{% set date_string = environ['RAPIDS_DATE_STRING'] %} package: name: libcuvs-split @@ -42,6 +41,13 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev {% endif %} requirements: build: @@ -70,6 +76,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} + - cuda-cudart-dev - cuda-profiler-api - libcublas-dev - libcurand-dev @@ -79,6 +86,13 @@ outputs: run: - libraft ={{ minor_version }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + {% if cuda_major != "11" %} + - cuda-cudart + - libcublas + - libcurand + - libcusolver + - libcusparse + {% endif %} about: home: https://rapids.ai/ license: Apache-2.0 @@ -93,6 +107,13 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev {% endif %} requirements: build: @@ -121,6 +142,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} + - cuda-cudart-dev - cuda-profiler-api - libcublas-dev - libcurand-dev @@ -130,6 +152,13 @@ outputs: run: - libraft ={{ minor_version }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + {% if cuda_major != "11" %} + - cuda-cudart + - libcublas + - libcurand + - libcusolver + - libcusparse + {% endif %} about: home: https://rapids.ai/ license: Apache-2.0 @@ -144,6 +173,13 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev {% endif %} requirements: build: @@ -160,7 +196,7 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - libraft ={{ minor_version }} - - libcuvs ={{ minor_version }} + - {{ pin_subpackage('libcuvs', exact=True) }} - cuda-version ={{ cuda_version }} {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} @@ -186,9 +222,15 @@ outputs: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit + {% else %} + - cuda-cudart + - libcublas + - libcurand + - libcusolver + - libcusparse {% endif %} - libraft ={{ minor_version }} - - libcuvs ={{ minor_version }} + - {{ pin_subpackage('libcuvs', exact=True) }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} about: @@ -205,6 +247,10 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcublas-dev {% endif %} requirements: build: @@ -221,13 +267,14 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - libraft ={{ minor_version }} - - libcuvs ={{ minor_version }} + - {{ pin_subpackage('libcuvs', exact=True) }} - cuda-version ={{ cuda_version }} {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} {% else %} + - cuda-cudart-dev - cuda-profiler-api - libcublas-dev {% endif %} @@ -235,9 +282,12 @@ outputs: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit + {% else %} + - cuda-cudart + - libcublas {% endif %} - libraft ={{ minor_version }} - - libcuvs ={{ minor_version }} + - {{ pin_subpackage('libcuvs', exact=True) }} about: home: https://rapids.ai/ license: Apache-2.0 diff --git a/dependencies.yaml b/dependencies.yaml index d7562ce57..7fff7811a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -47,6 +47,7 @@ files: rust: output: none includes: + - cuda_version - build - cuda - rust @@ -138,6 +139,7 @@ dependencies: packages: - &rmm_conda rmm==24.4.* - &pylibraft_conda pylibraft==24.4.* + - dlpack>=0.8,<1.0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -320,7 +322,7 @@ dependencies: - make - rust # clang/liblclang only needed for bindgen support - - clang + - clang - libclang build_wheels: common: @@ -380,7 +382,7 @@ dependencies: - matrix: {cuda: "11.*"} packages: - *pylibraft_cu11 - - {matrix: null, packages: [*rmm_conda]} + - {matrix: null, packages: [*pylibraft_conda]} test_python_common: common: - output_types: [conda, requirements, pyproject] @@ -391,5 +393,4 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - *pylibraft_conda - scikit-learn diff --git a/python/cuvs/CMakeLists.txt b/python/cuvs/CMakeLists.txt index 5a5efe8f9..33a9420e7 100644 --- a/python/cuvs/CMakeLists.txt +++ b/python/cuvs/CMakeLists.txt @@ -33,11 +33,17 @@ project( C CXX CUDA ) +################################################################################ +# - User Options -------------------------------------------------------------- + option(FIND_CUVS_CPP "Search for existing CUVS C++ installations before defaulting to local files" OFF ) -message("- FIND_CUVS_CPP: ${FIND_CUVS_CPP}") +message("CUVS_PY: Searching for existing cuVS C/C++ installations before defaulting to local files: ${FIND_CUVS_CPP}") + +################################################################################ +# - Process User Options ------------------------------------------------------ include(../../fetch_rapids.cmake) include(rapids-cmake) @@ -50,7 +56,7 @@ rapids_cpm_init() # If the user requested it we attempt to find CUVS. if(FIND_CUVS_CPP) - find_package(cuvs ${cuvs_version}) + find_package(cuvs ${cuvs_version} REQUIRED) include(../../cpp/cmake/thirdparty/get_dlpack.cmake) else() set(cuvs_FOUND OFF) @@ -70,6 +76,9 @@ if(NOT cuvs_FOUND) install(TARGETS cuvs cuvs_c DESTINATION ${cython_lib_dir}) endif() +################################################################################ +# - Build Cython artifacts ----------------------------------------------------- + rapids_cython_init() add_subdirectory(cuvs/common) diff --git a/python/cuvs/pyproject.toml b/python/cuvs/pyproject.toml index 664cf2969..07578abf3 100644 --- a/python/cuvs/pyproject.toml +++ b/python/cuvs/pyproject.toml @@ -38,7 +38,7 @@ requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", "numpy>=1.23", - "rmm==24.4.*", + "pylibraft==24.4.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -50,7 +50,6 @@ classifiers = [ [project.optional-dependencies] test = [ "cupy-cuda11x>=12.0.0", - "pylibraft==24.4.*", "pytest-cov", "pytest==7.*", "scikit-learn",