diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9944dd5198..e013d4f1c5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,39 +67,36 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} - wheel-build-cpp: + wheel-build-pylibraft: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_cpp.sh - wheel-build-python: - needs: wheel-build-cpp + script: ci/build_wheel_pylibraft.sh + wheel-publish-pylibraft: + needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_python.sh - wheel-publish-cpp: - needs: wheel-build-cpp + package-name: pylibraft + wheel-build-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: raft - package-type: cpp - wheel-publish-python: - needs: wheel-build-python + script: ci/build_wheel_raft_dask.sh + wheel-publish-raft-dask: + needs: wheel-build-raft-dask secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: @@ -107,5 +104,4 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: raft - package-type: python + package-name: raft_dask diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5d0368e3f7..c2d9556859 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,9 +19,9 @@ jobs: - conda-python-build - conda-python-tests - docs-build - - wheel-build-cpp - - wheel-build-python + - wheel-build-pylibraft - wheel-tests-pylibraft + - wheel-build-raft-dask - wheel-tests-raft-dask - devcontainer secrets: inherit @@ -74,30 +74,29 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cpp: + wheel-build-pylibraft: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request - script: ci/build_wheel_cpp.sh - wheel-build-python: - needs: wheel-build-cpp - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - with: - build_type: pull-request - script: ci/build_wheel_python.sh + script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: - needs: wheel-build-python + needs: wheel-build-pylibraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_pylibraft.sh + wheel-build-raft-dask: + needs: wheel-tests-pylibraft + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + build_type: pull-request + script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: - needs: wheel-build-python + needs: wheel-build-raft-dask secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: diff --git a/build.sh b/build.sh index ee16bb4b1b..da5efa5183 100755 --- a/build.sh +++ b/build.sh @@ -381,6 +381,14 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi +# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. +SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" +if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then + SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_RAFT_CPP=ON" +fi +# Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS +SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') + # If clean given, run it prior to any other steps if (( ${CLEAN} == 1 )); then # If the dirs to clean are mounted dirs in a container, the @@ -487,13 +495,13 @@ fi # Build and (optionally) install the pylibraft Python package if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then - CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ + SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/pylibraft fi # Build and (optionally) install the raft-dask Python package if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then - CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ + SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/raft-dask fi diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 0000000000..e3e7ce9c89 --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name=$1 +package_dir=$2 +underscore_package_name=$(echo "${package_name}" | tr "-" "_") + +# Clear out system ucx files to ensure that we're getting ucx from the wheel. +rm -rf /usr/lib64/ucx +rm -rf /usr/lib64/libuc* + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${underscore_package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +if [[ ${package_name} == "raft-dask" ]]; then + sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/distributed-ucxx==(.*)\"/distributed-ucxx${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +else + sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +fi + +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} +fi + +cd "${package_dir}" + +# Hardcode the output dir +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist --exclude "libucp.so.0" dist/* + +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh deleted file mode 100755 index 0e0417dc35..0000000000 --- a/ci/build_wheel_cpp.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -package_name="libraft" -package_dir="python/libraft" - -source rapids-configure-sccache -source rapids-date-string - -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} - -cd "${package_dir}" - -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check - -mkdir -p final_dist -python -m auditwheel repair -w final_dist dist/* - -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh new file mode 100755 index 0000000000..895c311f46 --- /dev/null +++ b/ci/build_wheel_pylibraft.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Set up skbuild options. Enable sccache in skbuild config options +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" + +ci/build_wheel.sh pylibraft python/pylibraft diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh deleted file mode 100755 index ae15d3734b..0000000000 --- a/ci/build_wheel_python.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -# Clear out system ucx files to ensure that we're getting ucx from the wheel -# when building raft-dask. -rm -rf /usr/lib64/ucx -rm -rf /usr/lib64/libuc* - -source rapids-configure-sccache -source rapids-date-string - -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -echo "${version}" > VERSION - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist) -PYTHON_WHEELHOUSE="${PWD}/dist/" -PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" -WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${CPP_WHEELHOUSE}") -mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" - -FIND_LINKS="" -# Iterate over the array -for wheelhouse in "${WHEELHOUSES[@]}"; do - FIND_LINKS+="--find-links ${wheelhouse} " -done - - -build_wheel () { - local package_name="${1}" - local underscore_package_name=$(echo "${package_name}" | tr "-" "_") - - local package_dir="python/${package_name}" - local pyproject_file="${package_dir}/pyproject.toml" - local version_file="${package_dir}/${underscore_package_name}/_version.py" - - sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} - sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - - sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} - - for dep in rmm libraft pylibraft ucx-py distributed-ucxx; do - sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - done - - # dask-cuda & rapids-dask-dependency don't get a suffix, but they do get an alpha spec. - for dep in dask-cuda rapids-dask-dependency; do - sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} - done - - if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} - fi - - pushd "${package_dir}" - - python -m pip wheel . -w "${PYTHON_WHEELHOUSE}" -vvv --no-deps --disable-pip-version-check ${FIND_LINKS} - popd -} - -build_wheel pylibraft -build_wheel raft-dask - -python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libraft.so --exclude "libucp.so.0" "${PYTHON_WHEELHOUSE}"/* -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${PYTHON_AUDITED_WHEELHOUSE}" diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh new file mode 100755 index 0000000000..feba2d7a5b --- /dev/null +++ b/ci/build_wheel_raft_dask.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Set up skbuild options. Enable sccache in skbuild config options +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" + +ci/build_wheel.sh raft-dask python/raft-dask diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 230889ae82..b38f5a690b 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -3,12 +3,11 @@ set -euo pipefail +mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -WHEELHOUSE="${PWD}/dist/" -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/pylibraft*.whl)[test] -python -m pip install "pylibraft-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" - -python -m pytest python/pylibraft/pylibraft/test +python -m pytest ./python/pylibraft/pylibraft/test diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index bc3af96688..b8ca54f6e8 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -3,13 +3,15 @@ set -euo pipefail +mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -WHEELHOUSE="${PWD}/dist/" -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" -RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" +# Download the pylibraft built in the previous step +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep +python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl -python -m pip install "raft-dask-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" +python -m pip install "raft_dask-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links dist/ test_dir="python/raft-dask/raft_dask/test" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f7d1c9b119..7270c5a12b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -75,12 +75,9 @@ if((BUILD_TESTS ) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() -option(RAFT_COMPILE_LIBRARY "Enable building raft library instantiations" +option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) -option(RAFT_COMPILE_DYNAMIC_ONLY "Only build the static library and skip the -static library. Has no effect if RAFT_COMPILE_LIBRARY is OFF" OFF -) if(BUILD_CPU_ONLY) set(BUILD_SHARED_LIBS OFF) @@ -585,23 +582,17 @@ if(RAFT_COMPILE_LIBRARY) ) add_library(raft_lib SHARED $) - - set(raft_lib_targets raft_lib) - if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - add_library(raft_lib_static STATIC $) - list(APPEND raft_lib_targets raft_lib_static) - endif() + add_library(raft_lib_static STATIC $) set_target_properties( - ${raft_lib_targets} + raft_lib raft_lib_static PROPERTIES OUTPUT_NAME raft BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) - list(APPEND raft_lib_targets raft_objs) - foreach(target IN LISTS raft_lib_targets) + foreach(target raft_lib raft_lib_static raft_objs) target_link_libraries( ${target} PUBLIC raft::raft @@ -626,23 +617,21 @@ target_link_libraries(raft_compiled INTERFACE raft::raft $ - ) +if(TARGET raft_lib_static AND (NOT TARGET raft::raft_lib_static)) + add_library(raft::raft_lib_static ALIAS raft_lib_static) endif() +target_link_libraries( + raft_compiled_static INTERFACE raft::raft $ +) + # ################################################################################################## # * raft_distributed ------------------------------------------------------------------------------- add_library(raft_distributed INTERFACE) @@ -690,12 +679,8 @@ install( EXPORT raft-exports ) -set(raft_compiled_install_targets raft_compiled) -if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - list(APPEND raft_compiled_install_targets raft_compiled_static) -endif() install( - TARGETS ${raft_compiled_install_targets} + TARGETS raft_compiled raft_compiled_static DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -708,14 +693,12 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - install( - TARGETS raft_lib_static - DESTINATION ${lib_dir} - COMPONENT compiled-static - EXPORT raft-compiled-static-lib-exports - ) - endif() + install( + TARGETS raft_lib_static + DESTINATION ${lib_dir} + COMPONENT compiled-static + EXPORT raft-compiled-static-lib-exports + ) install( DIRECTORY include/raft_runtime DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -786,12 +769,8 @@ endif() set(raft_components compiled distributed) set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) - list(APPEND raft_components compiled) - list(APPEND raft_export_sets raft-compiled-lib-exports) - if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - list(APPEND raft_components compiled-static) - list(APPEND raft_export_sets raft-compiled-static-lib-exports) - endif() + list(APPEND raft_components compiled compiled-static) + list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) endif() string( diff --git a/dependencies.yaml b/dependencies.yaml index 3daa400444..6ed26a3b14 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -8,7 +8,6 @@ files: includes: - build - build_pylibraft - - cython_build - cuda - cuda_version - depends_on_cupy @@ -29,7 +28,6 @@ files: arch: [x86_64, aarch64] includes: - build - - cython_build - cuda - cuda_version - develop @@ -62,25 +60,6 @@ files: - docs - py_version - test_pylibraft - py_build_libraft: - output: pyproject - pyproject_dir: python/libraft - extras: - table: build-system - includes: - - build - - librmm - py_run_libraft: - output: pyproject - pyproject_dir: python/libraft - extras: - table: project - includes: - # This is really a build requirement for anything using libraft to build - # against, but is not required when _running_ with libraft. There isn't a - # great way to express that without separating libraft into libraft and - # libraft-dev packages, though. - - librmm py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -89,7 +68,6 @@ files: includes: - build - build_pylibraft - - cython_build py_run_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -114,7 +92,6 @@ files: table: build-system includes: - build - - cython_build - depends_on_ucx_build py_run_raft_dask: output: pyproject @@ -159,6 +136,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4 + - cython>=3.0.0 - ninja - output_types: [conda] packages: @@ -203,33 +181,7 @@ dependencies: packages: [nvcc_linux-64=11.2] - matrix: {cuda: "11.2", arch: aarch64} packages: [nvcc_linux-aarch64=11.2] - cython_build: - common: - - output_types: [conda, requirements, pyproject] - packages: - - cython>=3.0.0 - - output_types: [requirements, pyproject] - packages: - - libraft==24.6.* - librmm: - common: - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - specific: - - output_types: [requirements, pyproject] - matrices: - - matrix: {cuda: "12.*"} - packages: - - librmm-cu12==24.6.* - - matrix: {cuda: "11.*"} - packages: - - librmm-cu11==24.6.* - - matrix: null - packages: - - librmm==24.6.* + build_pylibraft: common: - output_types: [conda] @@ -479,15 +431,10 @@ dependencies: - matrix: {cuda: "12.*"} packages: - *rmm_cu12 - - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - *rmm_cu11 - - libraft-cu11==24.6.* - - matrix: null - packages: - - *rmm_conda - - libraft==24.6.* + - {matrix: null, packages: [*rmm_conda]} run_raft_dask: common: - output_types: [conda, pyproject] @@ -519,17 +466,11 @@ dependencies: packages: - &pylibraft_cu12 pylibraft-cu12==24.6.* - &ucx_py_cu12 ucx-py-cu12==0.38.* - - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - &pylibraft_cu11 pylibraft-cu11==24.6.* - &ucx_py_cu11 ucx-py-cu11==0.38.* - - libraft-cu11==24.6.* - - matrix: null - packages: - - *pylibraft_conda - - *ucx_py_conda - - libraft==24.6.* + - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} test_python_common: common: - output_types: [conda, requirements, pyproject] diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt deleted file mode 100644 index cc3b48d033..0000000000 --- a/python/libraft/CMakeLists.txt +++ /dev/null @@ -1,45 +0,0 @@ -# ============================================================================= -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) - -include(../../rapids_config.cmake) - -include(rapids-cuda) -rapids_cuda_init_architectures(libraft-python) - -project( - libraft-python - VERSION "${RAPIDS_VERSION}" - LANGUAGES CXX CUDA -) - -# Check if raft is already available. If so, it is the user's responsibility to ensure that the -# CMake package is also available at build time of the Python raft package. -find_package(raft "${RAPIDS_VERSION}") - -if(raft_FOUND) - return() -endif() - -unset(raft_FOUND) - -set(BUILD_TESTS OFF) -set(BUILD_PRIMS_BENCH OFF) -set(BUILD_ANN_BENCH OFF) -set(RAFT_COMPILE_LIBRARY ON) -set(RAFT_COMPILE_DYNAMIC_ONLY ON) -set(CUDA_STATIC_RUNTIME ON) - -add_subdirectory(../../cpp raft-cpp) diff --git a/python/libraft/LICENSE b/python/libraft/LICENSE deleted file mode 120000 index 30cff7403d..0000000000 --- a/python/libraft/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../LICENSE \ No newline at end of file diff --git a/python/libraft/README.md b/python/libraft/README.md deleted file mode 120000 index fe84005413..0000000000 --- a/python/libraft/README.md +++ /dev/null @@ -1 +0,0 @@ -../../README.md \ No newline at end of file diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION deleted file mode 120000 index d62dc733ef..0000000000 --- a/python/libraft/libraft/VERSION +++ /dev/null @@ -1 +0,0 @@ -../../../VERSION \ No newline at end of file diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py deleted file mode 100644 index 2ba8e06d56..0000000000 --- a/python/libraft/libraft/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from libraft._version import __git_commit__, __version__ -from libraft.load import load_library diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py deleted file mode 100644 index 3e3792a85c..0000000000 --- a/python/libraft/libraft/_version.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import importlib.resources - -__version__ = ( - importlib.resources.files("libraft") - .joinpath("VERSION") - .read_text() - .strip() -) -__git_commit__ = "" diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py deleted file mode 100644 index fb7bf35274..0000000000 --- a/python/libraft/libraft/load.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import ctypes -import os - - -def load_library(): - # Dynamically load libraft.so. Prefer a system library if one is present to - # avoid clobbering symbols that other packages might expect, but if no - # other library is present use the one in the wheel. - libraft_lib = None - try: - libraft_lib = ctypes.CDLL("libraft.so", ctypes.RTLD_GLOBAL) - except OSError: - # If neither of these directories contain the library, we assume we are - # in an environment where the C++ library is already installed - # somewhere else and the CMake build of the libraft Python - # package was a no-op. Note that this approach won't work for - # real editable installs of the libraft package, but that's not a use - # case I think we need to support. scikit-build-core has limited - # support for importlib.resources so there isn't a clean way to support - # that case yet. - for lib_dir in ("lib", "lib64"): - if os.path.isfile( - lib := os.path.join( - os.path.dirname(__file__), lib_dir, "libraft.so" - ) - ): - libraft_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) - break - - # The caller almost never needs to do anything with this library, but no - # harm in offering the option since this object at least provides a handle - # to inspect where libraft was loaded from. - return libraft_lib diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml deleted file mode 100644 index c5a76e363a..0000000000 --- a/python/libraft/pyproject.toml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[build-system] - -requires = [ - "cmake>=3.26.4", - "librmm==24.6.*", - "ninja", - "scikit-build-core[pyproject]>=0.7.0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -build-backend = "scikit_build_core.build" - -[project] -name = "libraft" -dynamic = ["version"] -description = "RAFT: Reusable Algorithms Functions and other Tools" -readme = { file = "README.md", content-type = "text/markdown" } -authors = [ - { name = "NVIDIA Corporation" }, -] -license = { text = "Apache 2.0" } -requires-python = ">=3.9" -classifiers = [ - "Intended Audience :: Developers", - "Programming Language :: C++", - "Environment :: GPU :: NVIDIA CUDA", -] -dependencies = [ - "librmm==24.6.*", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - -[project.urls] -Homepage = "https://github.com/rapidsai/raft" -Documentation = "https://docs.rapids.ai/api/raft/stable/" - -[project.entry-points."cmake.prefix"] -libraft = "libraft" - -[tool.scikit-build] -build-dir = "build/{wheel_tag}" -cmake.build-type = "Release" -cmake.minimum-version = "3.26.4" -ninja.make-fallback = true -sdist.exclude = ["*tests*"] -sdist.reproducible = true -wheel.packages = ["libraft"] -wheel.install-dir = "libraft" -wheel.py-api = "py3" - -[tool.scikit-build.metadata.version] -provider = "scikit_build_core.metadata.regex" -input = "libraft/VERSION" -regex = "(?P.*)" diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index bba5549f62..7a2d77041d 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,9 +27,42 @@ project( LANGUAGES CXX CUDA ) -find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) +option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" + ON +) + +# If the user requested it we attempt to find RAFT. +if(FIND_RAFT_CPP) + find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) + if(NOT TARGET raft::raft_lib) + message( + FATAL_ERROR + "Building against a preexisting libraft library requires the compiled libraft to have been built!" + ) + + endif() +else() + set(raft_FOUND OFF) +endif() include(rapids-cython-core) + +if(NOT raft_FOUND) + set(BUILD_TESTS OFF) + set(BUILD_PRIMS_BENCH OFF) + set(BUILD_ANN_BENCH OFF) + set(RAFT_COMPILE_LIBRARY ON) + set(CUDA_STATIC_RUNTIME ON) + + add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) + + # When building the C++ libraries from source we must copy libraft.so alongside the + # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft + # library, we shouldn't need this + set(cython_lib_dir pylibraft) + install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) +endif() + rapids_cython_init() add_subdirectory(pylibraft/common) @@ -38,3 +71,7 @@ add_subdirectory(pylibraft/matrix) add_subdirectory(pylibraft/neighbors) add_subdirectory(pylibraft/random) add_subdirectory(pylibraft/cluster) + +if(DEFINED cython_lib_dir) + rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") +endif() diff --git a/python/pylibraft/pylibraft/__init__.py b/python/pylibraft/pylibraft/__init__.py index 8aac8f93da..b0869501f3 100644 --- a/python/pylibraft/pylibraft/__init__.py +++ b/python/pylibraft/pylibraft/__init__.py @@ -14,13 +14,3 @@ # from pylibraft._version import __git_commit__, __version__ - -# If libraft was installed as a wheel, we must request it to load the library symbols. -# Otherwise, we assume that the library was installed in a system path that ld can find. -try: - import libraft -except ModuleNotFoundError: - pass -else: - libraft.load_library() - del libraft diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index 06a639436a..562cff5098 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX cluster_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ ) diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index d1c1acb3aa..53279bfaf7 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX common_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ ) diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index ffcef45c32..2530e07a98 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX distance_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ ) diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt index 07d35325a5..5b7803db00 100644 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ b/python/pylibraft/pylibraft/matrix/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX matrix_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ ) diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt index 2a954183d3..069038a0e8 100644 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt @@ -20,7 +20,7 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ ) add_subdirectory(cagra) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt index 2df03c7b0b..0939d7c5b3 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_cagra_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt index f50051ba23..37c57c45db 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfflat_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt index e57798fcc6..af431adb16 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfpq_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ ) diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index 7d61855111..10ff776471 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX random_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ ) diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 8feb600f11..df1001538e 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -18,7 +18,6 @@ requires = [ "cmake>=3.26.4", "cuda-python>=11.7.1,<12.0a0", "cython>=3.0.0", - "libraft==24.6.*", "ninja", "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", @@ -37,7 +36,6 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", - "libraft==24.6.*", "numpy>=1.23,<2.0a0", "rmm==24.6.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 3218ea4d32..2c629f3b73 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -25,12 +25,38 @@ project( LANGUAGES CXX CUDA ) +option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" + OFF +) + rapids_cpm_init() # Once https://github.com/rapidsai/ucxx/issues/173 is resolved we can remove this. find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) -find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) +# If the user requested it we attempt to find RAFT. +if(FIND_RAFT_CPP) + find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) +else() + set(raft_FOUND OFF) +endif() + +if(NOT raft_FOUND) + # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all + # library compilation and we don't need to install anything here. + set(BUILD_TESTS OFF) + set(BUILD_ANN_BENCH OFF) + set(BUILD_PRIMS_BENCH OFF) + set(RAFT_COMPILE_LIBRARIES OFF) + set(RAFT_COMPILE_DIST_LIBRARY OFF) + set(RAFT_COMPILE_NN_LIBRARY OFF) + set(CUDA_STATIC_RUNTIME ON) + set(RAFT_DASK_UCXX_STATIC ON) + + add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) + list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) + find_package(NCCL REQUIRED) +endif() include(rapids-cython-core) rapids_cython_init() diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 37691cc01b..4718d51a35 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -18,7 +18,6 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.0", - "libraft==24.6.*", "libucx==1.15.0", "ninja", "scikit-build-core[pyproject]>=0.7.0", @@ -37,7 +36,6 @@ requires-python = ">=3.9" dependencies = [ "dask-cuda==24.6.*", "joblib>=0.11", - "libraft==24.6.*", "libucx>=1.15.0", "numba>=0.57", "numpy>=1.23,<2.0a0", diff --git a/python/raft-dask/raft_dask/__init__.py b/python/raft-dask/raft_dask/__init__.py index 13b9a96154..19a037ae75 100644 --- a/python/raft-dask/raft_dask/__init__.py +++ b/python/raft-dask/raft_dask/__init__.py @@ -15,17 +15,8 @@ from raft_dask._version import __git_commit__, __version__ -# If libraft or libucx was installed as a wheel, we must request that those packages -# load the library symbols. Otherwise, we assume that the libraries were installed in a -# system path that ld can find. -try: - import libraft -except ModuleNotFoundError: - pass -else: - libraft.load_library() - del libraft - +# If libucx was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. try: import libucx except ModuleNotFoundError: diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 49dee15d8f..65d5f06577 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft raft::distributed) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" CXX ) diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index 8475bcaa93..2ff1cd9150 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" CXX )