diff --git a/.gitignore b/.gitignore index 11b7bc3eba..3d6c84a83f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ log dask-worker-space/ *.egg-info/ *.bin -bench/ann/data temporary_*.json ## scikit-build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a5342a74e..d8ccf92ce5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: entry: ./cpp/scripts/run-cmake-format.sh cmake-format language: python types: [cmake] - exclude: .*/thirdparty/.*|.*FindAVX.cmake.* + exclude: .*/thirdparty/.* # Note that pre-commit autoupdate does not update the versions # of dependencies, so we'll have to update this manually. additional_dependencies: @@ -114,7 +114,6 @@ repos: cpp/include/raft/neighbors/detail/faiss_select/| cpp/include/raft/thirdparty/| docs/source/sphinxext/github_link[.]py| - cpp/cmake/modules/FindAVX[.]cmake - id: verify-alpha-spec - repo: https://github.com/rapidsai/dependency-file-generator rev: v1.16.0 diff --git a/build.sh b/build.sh index a95cb8ee23..8b3aad2580 100755 --- a/build.sh +++ b/build.sh @@ -29,6 +29,10 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=>>>>>> branch-24.12 and is: -v - verbose build mode diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index bc0ff1fae7..4857f12cd1 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -19,21 +19,6 @@ c_stdlib_version: cmake_version: - ">=3.26.4,!=3.30.0" -nccl_version: - - ">=2.19" - -glog_version: - - ">=0.6.0" - -faiss_version: - - ">=1.7.1" - -h5py_version: - - ">=3.8.0" - -nlohmann_json_version: - - ">=3.11.2" - # The CTK libraries below are missing from the conda-forge::cudatoolkit package # for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages # and the "*_run_*" version specifiers correspond to `11.x` packages. diff --git a/cpp/cmake/config.json b/cpp/cmake/config.json index f7cc50e513..3c568d9766 100644 --- a/cpp/cmake/config.json +++ b/cpp/cmake/config.json @@ -9,7 +9,7 @@ "VERSION": "?", "GIT_SHALLOW": "?", "OPTIONS": "*", - "FIND_PACKAGE_ARGUMENTS": "*" + "FIND_PACKAGE_ARGUMENTS": "*" } }, "ConfigureTest": { diff --git a/cpp/cmake/modules/FindAVX.cmake b/cpp/cmake/modules/FindAVX.cmake deleted file mode 100644 index 7f3b2dfc76..0000000000 --- a/cpp/cmake/modules/FindAVX.cmake +++ /dev/null @@ -1,110 +0,0 @@ -# ============================================================================= -# Copyright (c) 2016- Facebook, Inc (Adam Paszke) -# Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -# Copyright (c) 2011-2013 NYU (Clement Farabet) -# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) -# Copyright (c) 2006 Idiap Research Institute (Samy Bengio) -# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) -# -# Note: This file was copied from PyTorch and modified for use in the RAFT library. -# Refer to thirdparty/LICENSES/LICENSE.pytorch for license and additional -# copyright information. -# ============================================================================= - -INCLUDE(CheckCXXSourceRuns) - -SET(AVX_CODE - " - #include - - int main() - { - __m256 a; - a = _mm256_set1_ps(0); - return 0; - } -" -) - -SET(AVX512_CODE - " - #include - - int main() - { - __m512i a = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0); - __m512i b = a; - __mmask64 equality_mask = _mm512_cmp_epi8_mask(a, b, _MM_CMPINT_EQ); - return 0; - } -" -) - -SET(AVX2_CODE - " - #include - - int main() - { - __m256i a = {0}; - a = _mm256_abs_epi16(a); - __m256i x; - _mm256_extract_epi64(x, 0); // we rely on this in our AVX2 code - return 0; - } -" -) - -MACRO(CHECK_SSE lang type flags) - SET(__FLAG_I 1) - SET(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) - FOREACH(__FLAG ${flags}) - IF(NOT ${lang}_${type}_FOUND) - SET(CMAKE_REQUIRED_FLAGS ${__FLAG}) - CHECK_CXX_SOURCE_RUNS("${${type}_CODE}" ${lang}_HAS_${type}_${__FLAG_I}) - IF(${lang}_HAS_${type}_${__FLAG_I}) - SET(${lang}_${type}_FOUND - TRUE - CACHE BOOL "${lang} ${type} support" - ) - SET(${lang}_${type}_FLAGS - "${__FLAG}" - CACHE STRING "${lang} ${type} flags" - ) - ENDIF() - MATH(EXPR __FLAG_I "${__FLAG_I}+1") - ENDIF() - ENDFOREACH() - SET(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) - - IF(NOT ${lang}_${type}_FOUND) - SET(${lang}_${type}_FOUND - FALSE - CACHE BOOL "${lang} ${type} support" - ) - SET(${lang}_${type}_FLAGS - "" - CACHE STRING "${lang} ${type} flags" - ) - ENDIF() - - MARK_AS_ADVANCED(${lang}_${type}_FOUND ${lang}_${type}_FLAGS) - -ENDMACRO() - -# CHECK_SSE(C "AVX" " ;-mavx;/arch:AVX") CHECK_SSE(C "AVX2" " ;-mavx2 -mfma;/arch:AVX2") CHECK_SSE(C -# "AVX512" " ;-mavx512f -mavx512dq -mavx512vl -mavx512bw -mfma;/arch:AVX512") -# -CHECK_SSE(CXX "AVX" " ;-mavx;/arch:AVX") -CHECK_SSE(CXX "AVX2" " ;-mavx2 -mfma;/arch:AVX2") -CHECK_SSE(CXX "AVX512" " ;-mavx512f -mavx512dq -mavx512vl -mavx512bw -mfma;/arch:AVX512") diff --git a/cpp/cmake/thirdparty/get_fmt.cmake b/cpp/cmake/thirdparty/get_fmt.cmake deleted file mode 100644 index c06f8a78bb..0000000000 --- a/cpp/cmake/thirdparty/get_fmt.cmake +++ /dev/null @@ -1,22 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Use CPM to find or clone fmt -function(find_and_configure_fmt) - - include(${rapids-cmake-dir}/cpm/fmt.cmake) - rapids_cpm_fmt(INSTALL_EXPORT_SET rmm-exports BUILD_EXPORT_SET rmm-exports) -endfunction() - -find_and_configure_fmt() \ No newline at end of file diff --git a/cpp/cmake/thirdparty/get_glog.cmake b/cpp/cmake/thirdparty/get_glog.cmake deleted file mode 100644 index 35a9170f99..0000000000 --- a/cpp/cmake/thirdparty/get_glog.cmake +++ /dev/null @@ -1,48 +0,0 @@ -#============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_glog) - set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - rapids_cpm_find(glog ${PKG_VERSION} - GLOBAL_TARGETS glog::glog - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/glog.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL} - ) - - if(glog_ADDED) - message(VERBOSE "RAFT: Using glog located in ${glog_SOURCE_DIR}") - else() - message(VERBOSE "RAFT: Using glog located in ${glog_DIR}") - endif() - - -endfunction() - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_glog_SOURCE=/path/to/local/glog -find_and_configure_glog(VERSION 0.6.0 - FORK google - PINNED_TAG v0.6.0 - EXCLUDE_FROM_ALL ON - ) diff --git a/cpp/template/cmake/thirdparty/get_raft.cmake b/cpp/template/cmake/thirdparty/get_raft.cmake new file mode 100644 index 0000000000..4474fd2875 --- /dev/null +++ b/cpp/template/cmake/thirdparty/get_raft.cmake @@ -0,0 +1,67 @@ +# ============================================================================= +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. + +# Use RAPIDS_VERSION from cmake/thirdparty/fetch_rapids.cmake +set(RAFT_VERSION "${RAPIDS_VERSION}") +set(RAFT_FORK "rapidsai") +set(RAFT_PINNED_TAG "branch-${RAPIDS_VERSION}") + +function(find_and_configure_raft) + set(oneValueArgs VERSION FORK PINNED_TAG COMPILE_LIBRARY ENABLE_NVTX ENABLE_MNMG_DEPENDENCIES) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + set(RAFT_COMPONENTS "") + if(PKG_COMPILE_LIBRARY) + string(APPEND RAFT_COMPONENTS " compiled") + endif() + + if(PKG_ENABLE_MNMG_DEPENDENCIES) + string(APPEND RAFT_COMPONENTS " distributed") + endif() + + #----------------------------------------------------- + # Invoke CPM find_package() + #----------------------------------------------------- + # Since the RAFT_NVTX option is used by targets generated by + # find_package(RAFT_NVTX) and when building from source we want to + # make `RAFT_NVTX` a cache variable so we get consistent + # behavior + # + set(RAFT_NVTX ${PKG_ENABLE_NVTX} CACHE BOOL "Enable raft nvtx logging" FORCE) + rapids_cpm_find(raft ${PKG_VERSION} + GLOBAL_TARGETS raft::raft + BUILD_EXPORT_SET raft-template-exports + INSTALL_EXPORT_SET raft-template-exports + COMPONENTS ${RAFT_COMPONENTS} + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_TESTS OFF" + "BUILD_PRIMS_BENCH OFF" + "RAFT_COMPILE_LIBRARY ${PKG_COMPILE_LIBRARY}" + ) +endfunction() + +# Change pinned tag here to test a commit in CI +# To use a different RAFT locally, set the CMake variable +# CPM_raft_SOURCE=/path/to/local/raft +find_and_configure_raft(VERSION ${RAFT_VERSION}.00 + FORK ${RAFT_FORK} + PINNED_TAG ${RAFT_PINNED_TAG} + COMPILE_LIBRARY ON + ENABLE_MNMG_DEPENDENCIES OFF + ENABLE_NVTX OFF +)