From ea54c792b2b415b238cecf238f418aabb37836a2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 10 May 2024 09:07:31 -0700 Subject: [PATCH] Revert "Build C++ wheel (#2264)" (#2305) This reverts commit b760453221c6eb5b6986ceb2153bb0a0284697e2. The various issues around symbol visibility caused by raft currently supporting a mix of compiled library and header-only usage make this wheel unsafe to use in various environments at the moment. We will revisit this work once all of the compiled components of raft are fully split out into cuvs. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/raft/pull/2305 --- .github/workflows/build.yaml | 28 +++---- .github/workflows/pr.yaml | 27 +++--- build.sh | 12 ++- ci/build_wheel.sh | 67 +++++++++++++++ ci/build_wheel_cpp.sh | 46 ---------- ci/build_wheel_pylibraft.sh | 9 ++ ci/build_wheel_python.sh | 84 ------------------- ci/build_wheel_raft_dask.sh | 9 ++ ci/test_wheel_pylibraft.sh | 11 ++- ci/test_wheel_raft_dask.sh | 10 ++- cpp/CMakeLists.txt | 69 ++++++--------- dependencies.yaml | 67 +-------------- python/libraft/CMakeLists.txt | 45 ---------- python/libraft/LICENSE | 1 - python/libraft/README.md | 1 - python/libraft/libraft/VERSION | 1 - python/libraft/libraft/__init__.py | 17 ---- python/libraft/libraft/_version.py | 25 ------ python/libraft/libraft/load.py | 48 ----------- python/libraft/pyproject.toml | 65 -------------- python/pylibraft/CMakeLists.txt | 39 ++++++++- python/pylibraft/pylibraft/__init__.py | 10 --- .../pylibraft/cluster/CMakeLists.txt | 2 +- .../pylibraft/pylibraft/common/CMakeLists.txt | 2 +- .../pylibraft/distance/CMakeLists.txt | 2 +- .../pylibraft/pylibraft/matrix/CMakeLists.txt | 2 +- .../pylibraft/neighbors/CMakeLists.txt | 2 +- .../pylibraft/neighbors/cagra/CMakeLists.txt | 2 +- .../neighbors/ivf_flat/CMakeLists.txt | 2 +- .../pylibraft/neighbors/ivf_pq/CMakeLists.txt | 2 +- .../pylibraft/pylibraft/random/CMakeLists.txt | 2 +- python/pylibraft/pyproject.toml | 2 - python/raft-dask/CMakeLists.txt | 28 ++++++- python/raft-dask/pyproject.toml | 2 - python/raft-dask/raft_dask/__init__.py | 13 +-- .../raft-dask/raft_dask/common/CMakeLists.txt | 2 +- .../raft_dask/include_test/CMakeLists.txt | 2 +- 37 files changed, 237 insertions(+), 521 deletions(-) create mode 100755 ci/build_wheel.sh delete mode 100755 ci/build_wheel_cpp.sh create mode 100755 ci/build_wheel_pylibraft.sh delete mode 100755 ci/build_wheel_python.sh create mode 100755 ci/build_wheel_raft_dask.sh delete mode 100644 python/libraft/CMakeLists.txt delete mode 120000 python/libraft/LICENSE delete mode 120000 python/libraft/README.md delete mode 120000 python/libraft/libraft/VERSION delete mode 100644 python/libraft/libraft/__init__.py delete mode 100644 python/libraft/libraft/_version.py delete mode 100644 python/libraft/libraft/load.py delete mode 100644 python/libraft/pyproject.toml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9944dd5198..e013d4f1c5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,39 +67,36 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} - wheel-build-cpp: + wheel-build-pylibraft: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_cpp.sh - wheel-build-python: - needs: wheel-build-cpp + script: ci/build_wheel_pylibraft.sh + wheel-publish-pylibraft: + needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_python.sh - wheel-publish-cpp: - needs: wheel-build-cpp + package-name: pylibraft + wheel-build-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: raft - package-type: cpp - wheel-publish-python: - needs: wheel-build-python + script: ci/build_wheel_raft_dask.sh + wheel-publish-raft-dask: + needs: wheel-build-raft-dask secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: @@ -107,5 +104,4 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: raft - package-type: python + package-name: raft_dask diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5d0368e3f7..c2d9556859 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,9 +19,9 @@ jobs: - conda-python-build - conda-python-tests - docs-build - - wheel-build-cpp - - wheel-build-python + - wheel-build-pylibraft - wheel-tests-pylibraft + - wheel-build-raft-dask - wheel-tests-raft-dask - devcontainer secrets: inherit @@ -74,30 +74,29 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cpp: + wheel-build-pylibraft: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request - script: ci/build_wheel_cpp.sh - wheel-build-python: - needs: wheel-build-cpp - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - with: - build_type: pull-request - script: ci/build_wheel_python.sh + script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: - needs: wheel-build-python + needs: wheel-build-pylibraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_pylibraft.sh + wheel-build-raft-dask: + needs: wheel-tests-pylibraft + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + build_type: pull-request + script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: - needs: wheel-build-python + needs: wheel-build-raft-dask secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: diff --git a/build.sh b/build.sh index ee16bb4b1b..da5efa5183 100755 --- a/build.sh +++ b/build.sh @@ -381,6 +381,14 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi +# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. +SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" +if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then + SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_RAFT_CPP=ON" +fi +# Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS +SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') + # If clean given, run it prior to any other steps if (( ${CLEAN} == 1 )); then # If the dirs to clean are mounted dirs in a container, the @@ -487,13 +495,13 @@ fi # Build and (optionally) install the pylibraft Python package if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then - CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ + SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/pylibraft fi # Build and (optionally) install the raft-dask Python package if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then - CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ + SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/raft-dask fi diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 0000000000..e3e7ce9c89 --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name=$1 +package_dir=$2 +underscore_package_name=$(echo "${package_name}" | tr "-" "_") + +# Clear out system ucx files to ensure that we're getting ucx from the wheel. +rm -rf /usr/lib64/ucx +rm -rf /usr/lib64/libuc* + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${underscore_package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +if [[ ${package_name} == "raft-dask" ]]; then + sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/distributed-ucxx==(.*)\"/distributed-ucxx${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +else + sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +fi + +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} +fi + +cd "${package_dir}" + +# Hardcode the output dir +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist --exclude "libucp.so.0" dist/* + +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh deleted file mode 100755 index 0e0417dc35..0000000000 --- a/ci/build_wheel_cpp.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -package_name="libraft" -package_dir="python/libraft" - -source rapids-configure-sccache -source rapids-date-string - -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} - -cd "${package_dir}" - -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check - -mkdir -p final_dist -python -m auditwheel repair -w final_dist dist/* - -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh new file mode 100755 index 0000000000..895c311f46 --- /dev/null +++ b/ci/build_wheel_pylibraft.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Set up skbuild options. Enable sccache in skbuild config options +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" + +ci/build_wheel.sh pylibraft python/pylibraft diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh deleted file mode 100755 index ae15d3734b..0000000000 --- a/ci/build_wheel_python.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -# Clear out system ucx files to ensure that we're getting ucx from the wheel -# when building raft-dask. -rm -rf /usr/lib64/ucx -rm -rf /usr/lib64/libuc* - -source rapids-configure-sccache -source rapids-date-string - -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -echo "${version}" > VERSION - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist) -PYTHON_WHEELHOUSE="${PWD}/dist/" -PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" -WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${CPP_WHEELHOUSE}") -mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" - -FIND_LINKS="" -# Iterate over the array -for wheelhouse in "${WHEELHOUSES[@]}"; do - FIND_LINKS+="--find-links ${wheelhouse} " -done - - -build_wheel () { - local package_name="${1}" - local underscore_package_name=$(echo "${package_name}" | tr "-" "_") - - local package_dir="python/${package_name}" - local pyproject_file="${package_dir}/pyproject.toml" - local version_file="${package_dir}/${underscore_package_name}/_version.py" - - sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} - sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - - sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} - - for dep in rmm libraft pylibraft ucx-py distributed-ucxx; do - sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - done - - # dask-cuda & rapids-dask-dependency don't get a suffix, but they do get an alpha spec. - for dep in dask-cuda rapids-dask-dependency; do - sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} - done - - if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} - fi - - pushd "${package_dir}" - - python -m pip wheel . -w "${PYTHON_WHEELHOUSE}" -vvv --no-deps --disable-pip-version-check ${FIND_LINKS} - popd -} - -build_wheel pylibraft -build_wheel raft-dask - -python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libraft.so --exclude "libucp.so.0" "${PYTHON_WHEELHOUSE}"/* -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${PYTHON_AUDITED_WHEELHOUSE}" diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh new file mode 100755 index 0000000000..feba2d7a5b --- /dev/null +++ b/ci/build_wheel_raft_dask.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Set up skbuild options. Enable sccache in skbuild config options +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" + +ci/build_wheel.sh raft-dask python/raft-dask diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 230889ae82..b38f5a690b 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -3,12 +3,11 @@ set -euo pipefail +mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -WHEELHOUSE="${PWD}/dist/" -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/pylibraft*.whl)[test] -python -m pip install "pylibraft-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" - -python -m pytest python/pylibraft/pylibraft/test +python -m pytest ./python/pylibraft/pylibraft/test diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index bc3af96688..b8ca54f6e8 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -3,13 +3,15 @@ set -euo pipefail +mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -WHEELHOUSE="${PWD}/dist/" -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" +# Download the pylibraft built in the previous step +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep +python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl -python -m pip install "raft-dask-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" +python -m pip install "raft_dask-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links dist/ test_dir="python/raft-dask/raft_dask/test" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f7d1c9b119..7270c5a12b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -75,12 +75,9 @@ if((BUILD_TESTS ) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() -option(RAFT_COMPILE_LIBRARY "Enable building raft library instantiations" +option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) -option(RAFT_COMPILE_DYNAMIC_ONLY "Only build the static library and skip the -static library. Has no effect if RAFT_COMPILE_LIBRARY is OFF" OFF -) if(BUILD_CPU_ONLY) set(BUILD_SHARED_LIBS OFF) @@ -585,23 +582,17 @@ if(RAFT_COMPILE_LIBRARY) ) add_library(raft_lib SHARED $) - - set(raft_lib_targets raft_lib) - if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - add_library(raft_lib_static STATIC $) - list(APPEND raft_lib_targets raft_lib_static) - endif() + add_library(raft_lib_static STATIC $) set_target_properties( - ${raft_lib_targets} + raft_lib raft_lib_static PROPERTIES OUTPUT_NAME raft BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) - list(APPEND raft_lib_targets raft_objs) - foreach(target IN LISTS raft_lib_targets) + foreach(target raft_lib raft_lib_static raft_objs) target_link_libraries( ${target} PUBLIC raft::raft @@ -626,23 +617,21 @@ target_link_libraries(raft_compiled INTERFACE raft::raft $ - ) +if(TARGET raft_lib_static AND (NOT TARGET raft::raft_lib_static)) + add_library(raft::raft_lib_static ALIAS raft_lib_static) endif() +target_link_libraries( + raft_compiled_static INTERFACE raft::raft $ +) + # ################################################################################################## # * raft_distributed ------------------------------------------------------------------------------- add_library(raft_distributed INTERFACE) @@ -690,12 +679,8 @@ install( EXPORT raft-exports ) -set(raft_compiled_install_targets raft_compiled) -if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - list(APPEND raft_compiled_install_targets raft_compiled_static) -endif() install( - TARGETS ${raft_compiled_install_targets} + TARGETS raft_compiled raft_compiled_static DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -708,14 +693,12 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - install( - TARGETS raft_lib_static - DESTINATION ${lib_dir} - COMPONENT compiled-static - EXPORT raft-compiled-static-lib-exports - ) - endif() + install( + TARGETS raft_lib_static + DESTINATION ${lib_dir} + COMPONENT compiled-static + EXPORT raft-compiled-static-lib-exports + ) install( DIRECTORY include/raft_runtime DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -786,12 +769,8 @@ endif() set(raft_components compiled distributed) set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) - list(APPEND raft_components compiled) - list(APPEND raft_export_sets raft-compiled-lib-exports) - if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - list(APPEND raft_components compiled-static) - list(APPEND raft_export_sets raft-compiled-static-lib-exports) - endif() + list(APPEND raft_components compiled compiled-static) + list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) endif() string( diff --git a/dependencies.yaml b/dependencies.yaml index 3daa400444..6ed26a3b14 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -8,7 +8,6 @@ files: includes: - build - build_pylibraft - - cython_build - cuda - cuda_version - depends_on_cupy @@ -29,7 +28,6 @@ files: arch: [x86_64, aarch64] includes: - build - - cython_build - cuda - cuda_version - develop @@ -62,25 +60,6 @@ files: - docs - py_version - test_pylibraft - py_build_libraft: - output: pyproject - pyproject_dir: python/libraft - extras: - table: build-system - includes: - - build - - librmm - py_run_libraft: - output: pyproject - pyproject_dir: python/libraft - extras: - table: project - includes: - # This is really a build requirement for anything using libraft to build - # against, but is not required when _running_ with libraft. There isn't a - # great way to express that without separating libraft into libraft and - # libraft-dev packages, though. - - librmm py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -89,7 +68,6 @@ files: includes: - build - build_pylibraft - - cython_build py_run_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -114,7 +92,6 @@ files: table: build-system includes: - build - - cython_build - depends_on_ucx_build py_run_raft_dask: output: pyproject @@ -159,6 +136,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4 + - cython>=3.0.0 - ninja - output_types: [conda] packages: @@ -203,33 +181,7 @@ dependencies: packages: [nvcc_linux-64=11.2] - matrix: {cuda: "11.2", arch: aarch64} packages: [nvcc_linux-aarch64=11.2] - cython_build: - common: - - output_types: [conda, requirements, pyproject] - packages: - - cython>=3.0.0 - - output_types: [requirements, pyproject] - packages: - - libraft==24.6.* - librmm: - common: - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - specific: - - output_types: [requirements, pyproject] - matrices: - - matrix: {cuda: "12.*"} - packages: - - librmm-cu12==24.6.* - - matrix: {cuda: "11.*"} - packages: - - librmm-cu11==24.6.* - - matrix: null - packages: - - librmm==24.6.* + build_pylibraft: common: - output_types: [conda] @@ -479,15 +431,10 @@ dependencies: - matrix: {cuda: "12.*"} packages: - *rmm_cu12 - - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - *rmm_cu11 - - libraft-cu11==24.6.* - - matrix: null - packages: - - *rmm_conda - - libraft==24.6.* + - {matrix: null, packages: [*rmm_conda]} run_raft_dask: common: - output_types: [conda, pyproject] @@ -519,17 +466,11 @@ dependencies: packages: - &pylibraft_cu12 pylibraft-cu12==24.6.* - &ucx_py_cu12 ucx-py-cu12==0.38.* - - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - &pylibraft_cu11 pylibraft-cu11==24.6.* - &ucx_py_cu11 ucx-py-cu11==0.38.* - - libraft-cu11==24.6.* - - matrix: null - packages: - - *pylibraft_conda - - *ucx_py_conda - - libraft==24.6.* + - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} test_python_common: common: - output_types: [conda, requirements, pyproject] diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt deleted file mode 100644 index cc3b48d033..0000000000 --- a/python/libraft/CMakeLists.txt +++ /dev/null @@ -1,45 +0,0 @@ -# ============================================================================= -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) - -include(../../rapids_config.cmake) - -include(rapids-cuda) -rapids_cuda_init_architectures(libraft-python) - -project( - libraft-python - VERSION "${RAPIDS_VERSION}" - LANGUAGES CXX CUDA -) - -# Check if raft is already available. If so, it is the user's responsibility to ensure that the -# CMake package is also available at build time of the Python raft package. -find_package(raft "${RAPIDS_VERSION}") - -if(raft_FOUND) - return() -endif() - -unset(raft_FOUND) - -set(BUILD_TESTS OFF) -set(BUILD_PRIMS_BENCH OFF) -set(BUILD_ANN_BENCH OFF) -set(RAFT_COMPILE_LIBRARY ON) -set(RAFT_COMPILE_DYNAMIC_ONLY ON) -set(CUDA_STATIC_RUNTIME ON) - -add_subdirectory(../../cpp raft-cpp) diff --git a/python/libraft/LICENSE b/python/libraft/LICENSE deleted file mode 120000 index 30cff7403d..0000000000 --- a/python/libraft/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../LICENSE \ No newline at end of file diff --git a/python/libraft/README.md b/python/libraft/README.md deleted file mode 120000 index fe84005413..0000000000 --- a/python/libraft/README.md +++ /dev/null @@ -1 +0,0 @@ -../../README.md \ No newline at end of file diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION deleted file mode 120000 index d62dc733ef..0000000000 --- a/python/libraft/libraft/VERSION +++ /dev/null @@ -1 +0,0 @@ -../../../VERSION \ No newline at end of file diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py deleted file mode 100644 index 2ba8e06d56..0000000000 --- a/python/libraft/libraft/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from libraft._version import __git_commit__, __version__ -from libraft.load import load_library diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py deleted file mode 100644 index 3e3792a85c..0000000000 --- a/python/libraft/libraft/_version.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import importlib.resources - -__version__ = ( - importlib.resources.files("libraft") - .joinpath("VERSION") - .read_text() - .strip() -) -__git_commit__ = "" diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py deleted file mode 100644 index fb7bf35274..0000000000 --- a/python/libraft/libraft/load.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import ctypes -import os - - -def load_library(): - # Dynamically load libraft.so. Prefer a system library if one is present to - # avoid clobbering symbols that other packages might expect, but if no - # other library is present use the one in the wheel. - libraft_lib = None - try: - libraft_lib = ctypes.CDLL("libraft.so", ctypes.RTLD_GLOBAL) - except OSError: - # If neither of these directories contain the library, we assume we are - # in an environment where the C++ library is already installed - # somewhere else and the CMake build of the libraft Python - # package was a no-op. Note that this approach won't work for - # real editable installs of the libraft package, but that's not a use - # case I think we need to support. scikit-build-core has limited - # support for importlib.resources so there isn't a clean way to support - # that case yet. - for lib_dir in ("lib", "lib64"): - if os.path.isfile( - lib := os.path.join( - os.path.dirname(__file__), lib_dir, "libraft.so" - ) - ): - libraft_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) - break - - # The caller almost never needs to do anything with this library, but no - # harm in offering the option since this object at least provides a handle - # to inspect where libraft was loaded from. - return libraft_lib diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml deleted file mode 100644 index c5a76e363a..0000000000 --- a/python/libraft/pyproject.toml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[build-system] - -requires = [ - "cmake>=3.26.4", - "librmm==24.6.*", - "ninja", - "scikit-build-core[pyproject]>=0.7.0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -build-backend = "scikit_build_core.build" - -[project] -name = "libraft" -dynamic = ["version"] -description = "RAFT: Reusable Algorithms Functions and other Tools" -readme = { file = "README.md", content-type = "text/markdown" } -authors = [ - { name = "NVIDIA Corporation" }, -] -license = { text = "Apache 2.0" } -requires-python = ">=3.9" -classifiers = [ - "Intended Audience :: Developers", - "Programming Language :: C++", - "Environment :: GPU :: NVIDIA CUDA", -] -dependencies = [ - "librmm==24.6.*", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - -[project.urls] -Homepage = "https://github.com/rapidsai/raft" -Documentation = "https://docs.rapids.ai/api/raft/stable/" - -[project.entry-points."cmake.prefix"] -libraft = "libraft" - -[tool.scikit-build] -build-dir = "build/{wheel_tag}" -cmake.build-type = "Release" -cmake.minimum-version = "3.26.4" -ninja.make-fallback = true -sdist.exclude = ["*tests*"] -sdist.reproducible = true -wheel.packages = ["libraft"] -wheel.install-dir = "libraft" -wheel.py-api = "py3" - -[tool.scikit-build.metadata.version] -provider = "scikit_build_core.metadata.regex" -input = "libraft/VERSION" -regex = "(?P.*)" diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index bba5549f62..7a2d77041d 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,9 +27,42 @@ project( LANGUAGES CXX CUDA ) -find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) +option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" + ON +) + +# If the user requested it we attempt to find RAFT. +if(FIND_RAFT_CPP) + find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) + if(NOT TARGET raft::raft_lib) + message( + FATAL_ERROR + "Building against a preexisting libraft library requires the compiled libraft to have been built!" + ) + + endif() +else() + set(raft_FOUND OFF) +endif() include(rapids-cython-core) + +if(NOT raft_FOUND) + set(BUILD_TESTS OFF) + set(BUILD_PRIMS_BENCH OFF) + set(BUILD_ANN_BENCH OFF) + set(RAFT_COMPILE_LIBRARY ON) + set(CUDA_STATIC_RUNTIME ON) + + add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) + + # When building the C++ libraries from source we must copy libraft.so alongside the + # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft + # library, we shouldn't need this + set(cython_lib_dir pylibraft) + install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) +endif() + rapids_cython_init() add_subdirectory(pylibraft/common) @@ -38,3 +71,7 @@ add_subdirectory(pylibraft/matrix) add_subdirectory(pylibraft/neighbors) add_subdirectory(pylibraft/random) add_subdirectory(pylibraft/cluster) + +if(DEFINED cython_lib_dir) + rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") +endif() diff --git a/python/pylibraft/pylibraft/__init__.py b/python/pylibraft/pylibraft/__init__.py index 8aac8f93da..b0869501f3 100644 --- a/python/pylibraft/pylibraft/__init__.py +++ b/python/pylibraft/pylibraft/__init__.py @@ -14,13 +14,3 @@ # from pylibraft._version import __git_commit__, __version__ - -# If libraft was installed as a wheel, we must request it to load the library symbols. -# Otherwise, we assume that the library was installed in a system path that ld can find. -try: - import libraft -except ModuleNotFoundError: - pass -else: - libraft.load_library() - del libraft diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index 06a639436a..562cff5098 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX cluster_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ ) diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index d1c1acb3aa..53279bfaf7 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX common_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ ) diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index ffcef45c32..2530e07a98 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX distance_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ ) diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt index 07d35325a5..5b7803db00 100644 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ b/python/pylibraft/pylibraft/matrix/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX matrix_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ ) diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt index 2a954183d3..069038a0e8 100644 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt @@ -20,7 +20,7 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ ) add_subdirectory(cagra) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt index 2df03c7b0b..0939d7c5b3 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_cagra_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt index f50051ba23..37c57c45db 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfflat_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt index e57798fcc6..af431adb16 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfpq_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ ) diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index 7d61855111..10ff776471 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX random_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ ) diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 8feb600f11..df1001538e 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -18,7 +18,6 @@ requires = [ "cmake>=3.26.4", "cuda-python>=11.7.1,<12.0a0", "cython>=3.0.0", - "libraft==24.6.*", "ninja", "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", @@ -37,7 +36,6 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", - "libraft==24.6.*", "numpy>=1.23,<2.0a0", "rmm==24.6.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 3218ea4d32..2c629f3b73 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -25,12 +25,38 @@ project( LANGUAGES CXX CUDA ) +option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" + OFF +) + rapids_cpm_init() # Once https://github.com/rapidsai/ucxx/issues/173 is resolved we can remove this. find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) -find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) +# If the user requested it we attempt to find RAFT. +if(FIND_RAFT_CPP) + find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) +else() + set(raft_FOUND OFF) +endif() + +if(NOT raft_FOUND) + # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all + # library compilation and we don't need to install anything here. + set(BUILD_TESTS OFF) + set(BUILD_ANN_BENCH OFF) + set(BUILD_PRIMS_BENCH OFF) + set(RAFT_COMPILE_LIBRARIES OFF) + set(RAFT_COMPILE_DIST_LIBRARY OFF) + set(RAFT_COMPILE_NN_LIBRARY OFF) + set(CUDA_STATIC_RUNTIME ON) + set(RAFT_DASK_UCXX_STATIC ON) + + add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) + list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) + find_package(NCCL REQUIRED) +endif() include(rapids-cython-core) rapids_cython_init() diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 37691cc01b..4718d51a35 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -18,7 +18,6 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.0", - "libraft==24.6.*", "libucx==1.15.0", "ninja", "scikit-build-core[pyproject]>=0.7.0", @@ -37,7 +36,6 @@ requires-python = ">=3.9" dependencies = [ "dask-cuda==24.6.*", "joblib>=0.11", - "libraft==24.6.*", "libucx>=1.15.0", "numba>=0.57", "numpy>=1.23,<2.0a0", diff --git a/python/raft-dask/raft_dask/__init__.py b/python/raft-dask/raft_dask/__init__.py index 13b9a96154..19a037ae75 100644 --- a/python/raft-dask/raft_dask/__init__.py +++ b/python/raft-dask/raft_dask/__init__.py @@ -15,17 +15,8 @@ from raft_dask._version import __git_commit__, __version__ -# If libraft or libucx was installed as a wheel, we must request that those packages -# load the library symbols. Otherwise, we assume that the libraries were installed in a -# system path that ld can find. -try: - import libraft -except ModuleNotFoundError: - pass -else: - libraft.load_library() - del libraft - +# If libucx was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. try: import libucx except ModuleNotFoundError: diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 49dee15d8f..65d5f06577 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft raft::distributed) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" CXX ) diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index 8475bcaa93..2ff1cd9150 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" CXX )