diff --git a/ci/run_ctests.sh b/ci/run_ctests.sh index f6d0ef26ef..3086c39e8c 100755 --- a/ci/run_ctests.sh +++ b/ci/run_ctests.sh @@ -6,4 +6,4 @@ set -euo pipefail # Support customizing the ctests' install location cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libraft/" -ctest --output-on-failure "$@" +ctest --output-on-failure --no-tests=error "$@" diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 613b44dfb7..40b031d677 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -39,7 +39,7 @@ dependencies: - nccl>=2.9.9 - ninja - numba>=0.57 -- numpy>=1.21 +- numpy>=1.23 - numpydoc - nvcc_linux-aarch64=11.8 - pre-commit diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 69a7a17fc0..5485d09a37 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -39,7 +39,7 @@ dependencies: - nccl>=2.9.9 - ninja - numba>=0.57 -- numpy>=1.21 +- numpy>=1.23 - numpydoc - nvcc_linux-64=11.8 - pre-commit diff --git a/conda/environments/all_cuda-122_arch-aarch64.yaml b/conda/environments/all_cuda-122_arch-aarch64.yaml index 1c924bae46..b688bf3952 100644 --- a/conda/environments/all_cuda-122_arch-aarch64.yaml +++ b/conda/environments/all_cuda-122_arch-aarch64.yaml @@ -36,7 +36,7 @@ dependencies: - nccl>=2.9.9 - ninja - numba>=0.57 -- numpy>=1.21 +- numpy>=1.23 - numpydoc - pre-commit - pydata-sphinx-theme diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 09da75016a..013f852aee 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -36,7 +36,7 @@ dependencies: - nccl>=2.9.9 - ninja - numba>=0.57 -- numpy>=1.21 +- numpy>=1.23 - numpydoc - pre-commit - pydata-sphinx-theme diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index d2b709172e..5c2829d297 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -65,7 +65,7 @@ requirements: {% endif %} - libraft {{ version }} - libraft-headers {{ version }} - - numpy >=1.21 + - numpy >=1.23 - python x.x - rmm ={{ minor_version }} diff --git a/cpp/.clang-format b/cpp/.clang-format index 18f376d660..e939dc3c16 100644 --- a/cpp/.clang-format +++ b/cpp/.clang-format @@ -27,7 +27,7 @@ AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: Yes -BinPackArguments: false +BinPackArguments: false BinPackParameters: false BraceWrapping: AfterClass: false @@ -71,8 +71,28 @@ ForEachMacros: - foreach - Q_FOREACH - BOOST_FOREACH -IncludeBlocks: Preserve -IncludeIsMainRegex: '([-_](test|unittest))?$' +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^"' # quoted includes + Priority: 1 + - Regex: '^<(common|benchmarks|tests)/' # benchmark/test includes + Priority: 2 + #- Regex: '^<(cuspatial_test|cuproj_test)/' # cuSpatial/cuProj test includes + # Priority: 3 + - Regex: '^ #define JSON_DIAGNOSTICS 1 #include +#include #include #include -#include -#include - namespace raft::bench::ann { struct lib_handle { diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp index 851993712c..d7bcd17a00 100644 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -20,9 +20,11 @@ #include "dataset.hpp" #include "util.hpp" -#include #include +#include +#include + #include #include #include @@ -34,7 +36,6 @@ #include #include #include -#include #include namespace raft::bench::ann { diff --git a/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp b/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp index ef071c7026..27be26dfe9 100644 --- a/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp +++ b/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp @@ -15,12 +15,15 @@ */ #pragma once +#include + #include -#include #include + #include #include +#include namespace raft::mr { /** diff --git a/cpp/bench/ann/src/common/cuda_pinned_resource.hpp b/cpp/bench/ann/src/common/cuda_pinned_resource.hpp index a86a5df809..3256fc293c 100644 --- a/cpp/bench/ann/src/common/cuda_pinned_resource.hpp +++ b/cpp/bench/ann/src/common/cuda_pinned_resource.hpp @@ -15,10 +15,9 @@ */ #pragma once -#include - #include #include +#include #include diff --git a/cpp/bench/ann/src/common/cuda_stub.hpp b/cpp/bench/ann/src/common/cuda_stub.hpp index 512ccbce34..5ed138a86d 100644 --- a/cpp/bench/ann/src/common/cuda_stub.hpp +++ b/cpp/bench/ann/src/common/cuda_stub.hpp @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#pragma once /* The content of this header is governed by two preprocessor definitions: @@ -21,26 +20,30 @@ The content of this header is governed by two preprocessor definitions: - BUILD_CPU_ONLY - whether none of the CUDA functions are used. - ANN_BENCH_LINK_CUDART - dynamically link against this string if defined. -______________________________________________________________________________ +___________________________________________________________________________________ |BUILD_CPU_ONLY | ANN_BENCH_LINK_CUDART | cudart | cuda_runtime_api.h | -| | | found | needed | included | -|---------|-----------------------|-----------|---------|--------------------| -| ON | | false | false | NO | -| ON | "cudart.so.xx.xx" | false | false | NO | -| OFF | | true | true | YES | -| OFF | "cudart.so.xx.xx" | | true | YES | ------------------------------------------------------------------------------- +| | | found | needed | included | +|---------------|-----------------------|-----------|---------|--------------------| +| ON | | false | false | NO | +| ON | "cudart.so.xx.xx" | false | false | NO | +| OFF | | true | true | YES | +| OFF | "cudart.so.xx.xx" | | true | YES | +------------------------------------------------------------------------------------ */ +#pragma once + #ifndef BUILD_CPU_ONLY #include #include #ifdef ANN_BENCH_LINK_CUDART -#include #include + +#include #endif #else #include + typedef void* cudaStream_t; typedef void* cudaEvent_t; typedef uint16_t half; diff --git a/cpp/bench/ann/src/common/thread_pool.hpp b/cpp/bench/ann/src/common/thread_pool.hpp index c01fa2c32c..4a5684ecb3 100644 --- a/cpp/bench/ann/src/common/thread_pool.hpp +++ b/cpp/bench/ann/src/common/thread_pool.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #pragma once +#include + #include #include #include #include -#include #include #include #include diff --git a/cpp/bench/ann/src/common/util.hpp b/cpp/bench/ann/src/common/util.hpp index 8f6468befa..6cdff316e9 100644 --- a/cpp/bench/ann/src/common/util.hpp +++ b/cpp/bench/ann/src/common/util.hpp @@ -25,21 +25,20 @@ #include #include +#include #include +#include #include #include #include +#include +#include #include +#include #include #include #include -#include -#include -#include -#include -#include - namespace raft::bench::ann { /** diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp index e3e25a99a2..965522d929 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp +++ b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,12 @@ * limitations under the License. */ +#include "../common/ann_types.hpp" +#include "faiss_cpu_wrapper.h" + +#define JSON_DIAGNOSTICS 1 +#include + #include #include #include @@ -22,11 +28,6 @@ #include #include -#include "../common/ann_types.hpp" -#include "faiss_cpu_wrapper.h" -#define JSON_DIAGNOSTICS 1 -#include - namespace raft::bench::ann { template diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu b/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu index a9388531cc..c5056cb364 100644 --- a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu +++ b/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,14 @@ * limitations under the License. */ +#include "../common/ann_types.hpp" + +#undef WARP_SIZE +#include "faiss_gpu_wrapper.h" + +#define JSON_DIAGNOSTICS 1 +#include + #include #include #include @@ -22,12 +30,6 @@ #include #include -#include "../common/ann_types.hpp" -#undef WARP_SIZE -#include "faiss_gpu_wrapper.h" -#define JSON_DIAGNOSTICS 1 -#include - namespace raft::bench::ann { template diff --git a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu b/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu index 884476ea12..48d41388d4 100644 --- a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu +++ b/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu @@ -14,6 +14,12 @@ * limitations under the License. */ +#include "../common/ann_types.hpp" +#include "ggnn_wrapper.cuh" + +#define JSON_DIAGNOSTICS 1 +#include + #include #include #include @@ -22,11 +28,6 @@ #include #include -#include "../common/ann_types.hpp" -#include "ggnn_wrapper.cuh" -#define JSON_DIAGNOSTICS 1 -#include - namespace raft::bench::ann { template diff --git a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh b/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh index 5f5013ef7f..c89f02d974 100644 --- a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh +++ b/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh @@ -19,9 +19,10 @@ #include "../common/ann_types.hpp" #include "../common/util.hpp" -#include #include +#include + #include #include diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp b/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp index 1af19a22cb..df82c68830 100644 --- a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp +++ b/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ */ #include "../common/ann_types.hpp" +#include "hnswlib_wrapper.h" + +#define JSON_DIAGNOSTICS 1 +#include #include #include @@ -24,10 +28,6 @@ #include #include -#include "hnswlib_wrapper.h" -#define JSON_DIAGNOSTICS 1 -#include - namespace raft::bench::ann { template diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h index 45f7698a2d..a8f7dd824f 100644 --- a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h +++ b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h @@ -15,6 +15,11 @@ */ #pragma once +#include "../common/ann_types.hpp" +#include "../common/thread_pool.hpp" + +#include + #include #include #include @@ -31,10 +36,6 @@ #include #include -#include "../common/ann_types.hpp" -#include "../common/thread_pool.hpp" -#include - namespace raft::bench::ann { template diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h b/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h index 2018815c1e..ed9c120ed4 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h +++ b/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h @@ -17,6 +17,7 @@ #include "../hnswlib/hnswlib_wrapper.h" #include "raft_cagra_wrapper.h" + #include namespace raft::bench::ann { diff --git a/cpp/bench/ann/src/raft/raft_cagra_wrapper.h b/cpp/bench/ann/src/raft/raft_cagra_wrapper.h old mode 100755 new mode 100644 index ba4936976e..25f7f93777 --- a/cpp/bench/ann/src/raft/raft_cagra_wrapper.h +++ b/cpp/bench/ann/src/raft/raft_cagra_wrapper.h @@ -15,11 +15,11 @@ */ #pragma once -#include -#include -#include -#include -#include +#include "../common/ann_types.hpp" +#include "../common/cuda_huge_page_resource.hpp" +#include "../common/cuda_pinned_resource.hpp" +#include "raft_ann_bench_utils.h" + #include #include #include @@ -33,21 +33,19 @@ #include #include #include + #include +#include + +#include +#include +#include +#include +#include #include #include #include -#include "../common/ann_types.hpp" -#include "raft_ann_bench_utils.h" -#include - -#include "../common/cuda_huge_page_resource.hpp" -#include "../common/cuda_pinned_resource.hpp" - -#include -#include - namespace raft::bench::ann { enum class AllocatorType { HostPinned, HostHugePage, Device }; diff --git a/cpp/bench/ann/src/raft/raft_wrapper.h b/cpp/bench/ann/src/raft/raft_wrapper.h index 6fcb869d38..586b81ae06 100644 --- a/cpp/bench/ann/src/raft/raft_wrapper.h +++ b/cpp/bench/ann/src/raft/raft_wrapper.h @@ -15,20 +15,21 @@ */ #pragma once -#include -#include +#include "../common/ann_types.hpp" +#include "raft_ann_bench_utils.h" + #include #include #include #include #include + +#include +#include #include #include #include -#include "../common/ann_types.hpp" -#include "raft_ann_bench_utils.h" - namespace raft_temp { inline raft::distance::DistanceType parse_metric_type(raft::bench::ann::Metric metric) diff --git a/cpp/bench/prims/cluster/kmeans.cu b/cpp/bench/prims/cluster/kmeans.cu index 3147960f72..6387211135 100644 --- a/cpp/bench/prims/cluster/kmeans.cu +++ b/cpp/bench/prims/cluster/kmeans.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include diff --git a/cpp/bench/prims/cluster/kmeans_balanced.cu b/cpp/bench/prims/cluster/kmeans_balanced.cu index 129578c303..dc05783989 100644 --- a/cpp/bench/prims/cluster/kmeans_balanced.cu +++ b/cpp/bench/prims/cluster/kmeans_balanced.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include #include diff --git a/cpp/bench/prims/common/benchmark.hpp b/cpp/bench/prims/common/benchmark.hpp index c924d8b50a..4ecad6df3d 100644 --- a/cpp/bench/prims/common/benchmark.hpp +++ b/cpp/bench/prims/common/benchmark.hpp @@ -16,18 +16,14 @@ #pragma once -#include -#include - #include #include #include #include +#include #include #include -#include - #include #include #include @@ -35,6 +31,10 @@ #include #include +#include + +#include + namespace raft::bench { /** diff --git a/cpp/bench/prims/core/bitset.cu b/cpp/bench/prims/core/bitset.cu index ce3136bcd5..166f7aa6dc 100644 --- a/cpp/bench/prims/core/bitset.cu +++ b/cpp/bench/prims/core/bitset.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,10 @@ */ #include + #include #include + #include namespace raft::bench::core { diff --git a/cpp/bench/prims/core/copy.cu b/cpp/bench/prims/core/copy.cu index 31ee83b924..4898f560cb 100644 --- a/cpp/bench/prims/core/copy.cu +++ b/cpp/bench/prims/core/copy.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include -#include + #include #include #include @@ -25,6 +25,8 @@ #include #include +#include + namespace raft::bench::core { template diff --git a/cpp/bench/prims/distance/distance_common.cuh b/cpp/bench/prims/distance/distance_common.cuh index dff3401b62..8368062168 100644 --- a/cpp/bench/prims/distance/distance_common.cuh +++ b/cpp/bench/prims/distance/distance_common.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,10 @@ */ #include + #include #include + #include namespace raft::bench::distance { diff --git a/cpp/bench/prims/distance/fused_l2_nn.cu b/cpp/bench/prims/distance/fused_l2_nn.cu index c0ebd60458..a263bef6ba 100644 --- a/cpp/bench/prims/distance/fused_l2_nn.cu +++ b/cpp/bench/prims/distance/fused_l2_nn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,12 @@ */ #include + #include #include #include #include + #include namespace raft::bench::distance { diff --git a/cpp/bench/prims/distance/kernels.cu b/cpp/bench/prims/distance/kernels.cu index 3f74759665..eb86330637 100644 --- a/cpp/bench/prims/distance/kernels.cu +++ b/cpp/bench/prims/distance/kernels.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,14 @@ * limitations under the License. */ #include -#include + #include #include #include #include #include + +#include #include #include #include diff --git a/cpp/bench/prims/distance/masked_nn.cu b/cpp/bench/prims/distance/masked_nn.cu index 19d78f4cd9..979d438b67 100644 --- a/cpp/bench/prims/distance/masked_nn.cu +++ b/cpp/bench/prims/distance/masked_nn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,8 @@ * limitations under the License. */ -#include -#include -#include -#include -#include - #include -#include + #include #include #include @@ -30,6 +24,13 @@ #include #include +#include +#include +#include +#include +#include +#include + namespace raft::bench::distance::masked_nn { // Introduce various sparsity patterns diff --git a/cpp/bench/prims/distance/tune_pairwise/bench.cu b/cpp/bench/prims/distance/tune_pairwise/bench.cu index 87159ab1b1..81105cdefe 100644 --- a/cpp/bench/prims/distance/tune_pairwise/bench.cu +++ b/cpp/bench/prims/distance/tune_pairwise/bench.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,12 +38,16 @@ // maximal throughput of a kernel. Measuring other things, like performance on // skinny or wide matrices is not yet implemented. -#include "kernel.cuh" // launch_kernel -#include // std::min -#include // RAFT_BENCH_REGISTER +#include "kernel.cuh" // launch_kernel + +#include // RAFT_BENCH_REGISTER + #include // pairwise_matrix_params -#include // rmm::device_uvector -#include // std::vector + +#include // rmm::device_uvector + +#include // std::min +#include // std::vector namespace raft::bench::distance::tune { diff --git a/cpp/bench/prims/distance/tune_pairwise/kernel.cu b/cpp/bench/prims/distance/tune_pairwise/kernel.cu index 3112e1ea9a..42173c51f5 100644 --- a/cpp/bench/prims/distance/tune_pairwise/kernel.cu +++ b/cpp/bench/prims/distance/tune_pairwise/kernel.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "kernel.cuh" + #include // pairwise_matrix_sm60_wrapper #include // raft::linalg::Policy4x4 #include // raft::util::arch::SM_compute_arch diff --git a/cpp/bench/prims/linalg/add.cu b/cpp/bench/prims/linalg/add.cu index 456214ad7b..f861e24d44 100644 --- a/cpp/bench/prims/linalg/add.cu +++ b/cpp/bench/prims/linalg/add.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,9 @@ */ #include + #include + #include namespace raft::bench::linalg { diff --git a/cpp/bench/prims/linalg/map_then_reduce.cu b/cpp/bench/prims/linalg/map_then_reduce.cu index 84aebd85bf..3ca775a1da 100644 --- a/cpp/bench/prims/linalg/map_then_reduce.cu +++ b/cpp/bench/prims/linalg/map_then_reduce.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,9 @@ */ #include + #include + #include namespace raft::bench::linalg { diff --git a/cpp/bench/prims/linalg/matrix_vector_op.cu b/cpp/bench/prims/linalg/matrix_vector_op.cu index d1fbaee79b..c95efed91b 100644 --- a/cpp/bench/prims/linalg/matrix_vector_op.cu +++ b/cpp/bench/prims/linalg/matrix_vector_op.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,9 @@ */ #include + #include + #include namespace raft::bench::linalg { diff --git a/cpp/bench/prims/linalg/norm.cu b/cpp/bench/prims/linalg/norm.cu index 1db23e4ca4..7d8a07e932 100644 --- a/cpp/bench/prims/linalg/norm.cu +++ b/cpp/bench/prims/linalg/norm.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include #include diff --git a/cpp/bench/prims/linalg/normalize.cu b/cpp/bench/prims/linalg/normalize.cu index 91319e774c..7d2a725914 100644 --- a/cpp/bench/prims/linalg/normalize.cu +++ b/cpp/bench/prims/linalg/normalize.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include #include diff --git a/cpp/bench/prims/linalg/reduce.cu b/cpp/bench/prims/linalg/reduce.cu index cf41c5916a..f0fd13529d 100644 --- a/cpp/bench/prims/linalg/reduce.cu +++ b/cpp/bench/prims/linalg/reduce.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include diff --git a/cpp/bench/prims/linalg/reduce_cols_by_key.cu b/cpp/bench/prims/linalg/reduce_cols_by_key.cu index 1b584e80c8..f936a04ba1 100644 --- a/cpp/bench/prims/linalg/reduce_cols_by_key.cu +++ b/cpp/bench/prims/linalg/reduce_cols_by_key.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include #include diff --git a/cpp/bench/prims/linalg/reduce_rows_by_key.cu b/cpp/bench/prims/linalg/reduce_rows_by_key.cu index b68cefc274..a63b679bb6 100644 --- a/cpp/bench/prims/linalg/reduce_rows_by_key.cu +++ b/cpp/bench/prims/linalg/reduce_rows_by_key.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include diff --git a/cpp/bench/prims/linalg/sddmm.cu b/cpp/bench/prims/linalg/sddmm.cu index 139a2b838d..cd4259106d 100644 --- a/cpp/bench/prims/linalg/sddmm.cu +++ b/cpp/bench/prims/linalg/sddmm.cu @@ -14,17 +14,18 @@ * limitations under the License. */ #include -#include + #include #include +#include +#include #include #include #include #include #include -#include -#include +#include #include #include diff --git a/cpp/bench/prims/matrix/argmin.cu b/cpp/bench/prims/matrix/argmin.cu index afee81aa00..29655998e1 100644 --- a/cpp/bench/prims/matrix/argmin.cu +++ b/cpp/bench/prims/matrix/argmin.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include #include diff --git a/cpp/bench/prims/matrix/gather.cu b/cpp/bench/prims/matrix/gather.cu index 00a145ffa9..e6f26ba925 100644 --- a/cpp/bench/prims/matrix/gather.cu +++ b/cpp/bench/prims/matrix/gather.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include #include diff --git a/cpp/bench/prims/matrix/main.cpp b/cpp/bench/prims/matrix/main.cpp index 9cdb1c2546..b59e152971 100644 --- a/cpp/bench/prims/matrix/main.cpp +++ b/cpp/bench/prims/matrix/main.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include namespace raft::matrix { diff --git a/cpp/bench/prims/matrix/select_k.cu b/cpp/bench/prims/matrix/select_k.cu index a8e65d3e03..ff04e6f8a8 100644 --- a/cpp/bench/prims/matrix/select_k.cu +++ b/cpp/bench/prims/matrix/select_k.cu @@ -14,19 +14,18 @@ * limitations under the License. */ -#include - #include #include #include +#include +#include +#include #include #include #include -#include -#include -#include +#include #include #include diff --git a/cpp/bench/prims/neighbors/cagra_bench.cuh b/cpp/bench/prims/neighbors/cagra_bench.cuh index 07e93a3473..acbeba375a 100644 --- a/cpp/bench/prims/neighbors/cagra_bench.cuh +++ b/cpp/bench/prims/neighbors/cagra_bench.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,12 @@ #pragma once #include + #include #include #include #include + #include #include diff --git a/cpp/bench/prims/neighbors/knn.cuh b/cpp/bench/prims/neighbors/knn.cuh index 55865b577a..aea7168142 100644 --- a/cpp/bench/prims/neighbors/knn.cuh +++ b/cpp/bench/prims/neighbors/knn.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,20 +17,18 @@ #pragma once #include -#include - -#include #include +#include #include #include #include +#include #include #include #include #include - #include #include diff --git a/cpp/bench/prims/neighbors/refine.cuh b/cpp/bench/prims/neighbors/refine.cuh index 7cda59b57e..0360babd82 100644 --- a/cpp/bench/prims/neighbors/refine.cuh +++ b/cpp/bench/prims/neighbors/refine.cuh @@ -16,8 +16,6 @@ #pragma once -#include - #include #include @@ -27,6 +25,8 @@ #include #include +#include + #include #include #include diff --git a/cpp/bench/prims/neighbors/refine_float_int64_t.cu b/cpp/bench/prims/neighbors/refine_float_int64_t.cu index bbedc1ae64..d69a157eca 100644 --- a/cpp/bench/prims/neighbors/refine_float_int64_t.cu +++ b/cpp/bench/prims/neighbors/refine_float_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "refine.cuh" + #include using namespace raft::neighbors; diff --git a/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu b/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu index 4952361f03..9da536b6c7 100644 --- a/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu +++ b/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "refine.cuh" + #include using namespace raft::neighbors; diff --git a/cpp/bench/prims/random/make_blobs.cu b/cpp/bench/prims/random/make_blobs.cu index f43d914cf2..75ff861926 100644 --- a/cpp/bench/prims/random/make_blobs.cu +++ b/cpp/bench/prims/random/make_blobs.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,11 @@ */ #include + #include + #include + #include namespace raft::bench::random { diff --git a/cpp/bench/prims/random/permute.cu b/cpp/bench/prims/random/permute.cu index 829cf42720..eb6ebc3d73 100644 --- a/cpp/bench/prims/random/permute.cu +++ b/cpp/bench/prims/random/permute.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include #include diff --git a/cpp/bench/prims/random/rng.cu b/cpp/bench/prims/random/rng.cu index d15c9441d7..037c69e0ea 100644 --- a/cpp/bench/prims/random/rng.cu +++ b/cpp/bench/prims/random/rng.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include #include diff --git a/cpp/bench/prims/sparse/convert_csr.cu b/cpp/bench/prims/sparse/convert_csr.cu index 634c749a54..687e7850b5 100644 --- a/cpp/bench/prims/sparse/convert_csr.cu +++ b/cpp/bench/prims/sparse/convert_csr.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,14 @@ * limitations under the License. */ -#include - #include + #include + #include +#include + namespace raft::bench::sparse { template diff --git a/cpp/include/raft/cluster/detail/connectivities.cuh b/cpp/include/raft/cluster/detail/connectivities.cuh index 49ac6ae704..c527b754c3 100644 --- a/cpp/include/raft/cluster/detail/connectivities.cuh +++ b/cpp/include/raft/cluster/detail/connectivities.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,21 +16,20 @@ #pragma once +#include #include #include #include -#include -#include - -#include -#include - -#include #include #include +#include #include #include #include +#include +#include + +#include #include #include diff --git a/cpp/include/raft/cluster/detail/kmeans.cuh b/cpp/include/raft/cluster/detail/kmeans.cuh index e647e33734..4efeedcbaa 100644 --- a/cpp/include/raft/cluster/detail/kmeans.cuh +++ b/cpp/include/raft/cluster/detail/kmeans.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,19 +15,6 @@ */ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - #include #include #include @@ -38,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -48,9 +37,21 @@ #include #include #include + #include #include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + namespace raft { namespace cluster { namespace detail { diff --git a/cpp/include/raft/cluster/detail/kmeans_auto_find_k.cuh b/cpp/include/raft/cluster/detail/kmeans_auto_find_k.cuh index f6bdb191cd..97755351c4 100644 --- a/cpp/include/raft/cluster/detail/kmeans_auto_find_k.cuh +++ b/cpp/include/raft/cluster/detail/kmeans_auto_find_k.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,21 +14,18 @@ * limitations under the License. */ +#include #include #include +#include #include -#include -#include - #include - -#include - -#include - +#include #include #include +#include + namespace raft::cluster::detail { template diff --git a/cpp/include/raft/cluster/detail/kmeans_balanced.cuh b/cpp/include/raft/cluster/detail/kmeans_balanced.cuh index 3b5d3ff02a..6d3f430e88 100644 --- a/cpp/include/raft/cluster/detail/kmeans_balanced.cuh +++ b/cpp/include/raft/cluster/detail/kmeans_balanced.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,15 @@ #pragma once -#include -#include -#include -#include -#include - #include #include #include #include #include #include +#include +#include +#include #include #include #include @@ -44,7 +41,6 @@ #include #include -#include #include #include #include @@ -55,7 +51,9 @@ #include #include +#include #include +#include namespace raft::cluster::detail { diff --git a/cpp/include/raft/cluster/detail/kmeans_common.cuh b/cpp/include/raft/cluster/detail/kmeans_common.cuh index 5d56a1d081..8263aa4615 100644 --- a/cpp/include/raft/cluster/detail/kmeans_common.cuh +++ b/cpp/include/raft/cluster/detail/kmeans_common.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,20 +15,6 @@ */ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - #include #include #include @@ -36,6 +22,8 @@ #include #include #include +#include +#include #include #include #include @@ -51,6 +39,18 @@ #include #include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + namespace raft { namespace cluster { namespace detail { diff --git a/cpp/include/raft/cluster/detail/kmeans_deprecated.cuh b/cpp/include/raft/cluster/detail/kmeans_deprecated.cuh index 0b5dec4e19..e89f5480e3 100644 --- a/cpp/include/raft/cluster/detail/kmeans_deprecated.cuh +++ b/cpp/include/raft/cluster/detail/kmeans_deprecated.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,13 +21,16 @@ #pragma once -#include -#include -#include -#include #include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -45,13 +48,10 @@ #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include namespace raft { namespace cluster { diff --git a/cpp/include/raft/cluster/detail/mst.cuh b/cpp/include/raft/cluster/detail/mst.cuh index a962d4b7c6..55becc8e15 100644 --- a/cpp/include/raft/cluster/detail/mst.cuh +++ b/cpp/include/raft/cluster/detail/mst.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,12 @@ #pragma once #include -#include -#include - #include #include #include +#include +#include + #include #include diff --git a/cpp/include/raft/cluster/detail/single_linkage.cuh b/cpp/include/raft/cluster/detail/single_linkage.cuh index 848ca0357e..ccc6472684 100644 --- a/cpp/include/raft/cluster/detail/single_linkage.cuh +++ b/cpp/include/raft/cluster/detail/single_linkage.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,14 @@ #pragma once -#include -#include -#include - #include #include #include #include +#include +#include + +#include namespace raft::cluster::detail { diff --git a/cpp/include/raft/cluster/kmeans.cuh b/cpp/include/raft/cluster/kmeans.cuh index d63413e82e..eb28cc1626 100644 --- a/cpp/include/raft/cluster/kmeans.cuh +++ b/cpp/include/raft/cluster/kmeans.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ */ #pragma once -#include #include #include #include @@ -24,6 +23,8 @@ #include #include +#include + namespace raft::cluster::kmeans { /** diff --git a/cpp/include/raft/cluster/kmeans_balanced.cuh b/cpp/include/raft/cluster/kmeans_balanced.cuh index 5c59f1393c..8cd7730814 100644 --- a/cpp/include/raft/cluster/kmeans_balanced.cuh +++ b/cpp/include/raft/cluster/kmeans_balanced.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,13 @@ #pragma once -#include -#include - #include #include +#include #include +#include + namespace raft::cluster::kmeans_balanced { /** diff --git a/cpp/include/raft/comms/comms_test.hpp b/cpp/include/raft/comms/comms_test.hpp index 3ceb2942a8..c2c07ebc8e 100644 --- a/cpp/include/raft/comms/comms_test.hpp +++ b/cpp/include/raft/comms/comms_test.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ #include #include - #include namespace raft { diff --git a/cpp/include/raft/comms/detail/mpi_comms.hpp b/cpp/include/raft/comms/detail/mpi_comms.hpp index 3342fec973..a07146052e 100644 --- a/cpp/include/raft/comms/detail/mpi_comms.hpp +++ b/cpp/include/raft/comms/detail/mpi_comms.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,24 +16,24 @@ #pragma once -#include -#include - -#include -#include -#include - -#include -#include - #include #include #include #include #include + #include #include +#include +#include + +#include +#include +#include +#include +#include + #define RAFT_MPI_TRY(call) \ do { \ int status = call; \ diff --git a/cpp/include/raft/comms/detail/std_comms.hpp b/cpp/include/raft/comms/detail/std_comms.hpp index 323e408cab..6e7ff7106f 100644 --- a/cpp/include/raft/comms/detail/std_comms.hpp +++ b/cpp/include/raft/comms/detail/std_comms.hpp @@ -19,36 +19,31 @@ #include #include #include - -#include -#include -#include - #include - +#include #include -#include +#include +#include #include +#include +#include +#include +#include #include #include -#include - -#include -#include -#include - #include #include #include #include #include -#include #include -#include +#include +#include +#include namespace raft { namespace comms { diff --git a/cpp/include/raft/comms/detail/test.hpp b/cpp/include/raft/comms/detail/test.hpp index 876a17de1a..601c1858a3 100644 --- a/cpp/include/raft/comms/detail/test.hpp +++ b/cpp/include/raft/comms/detail/test.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,15 +20,13 @@ #include #include #include + #include #include #include #include -#include -#include - namespace raft { namespace comms { namespace detail { diff --git a/cpp/include/raft/comms/detail/ucp_helper.hpp b/cpp/include/raft/comms/detail/ucp_helper.hpp index 9479bc24f9..5896248c1d 100644 --- a/cpp/include/raft/comms/detail/ucp_helper.hpp +++ b/cpp/include/raft/comms/detail/ucp_helper.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include diff --git a/cpp/include/raft/comms/detail/util.hpp b/cpp/include/raft/comms/detail/util.hpp index 85084f0eae..c8cc9bf0bf 100644 --- a/cpp/include/raft/comms/detail/util.hpp +++ b/cpp/include/raft/comms/detail/util.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,11 @@ #pragma once +#include #include #include -#include + #include /** diff --git a/cpp/include/raft/comms/std_comms.hpp b/cpp/include/raft/comms/std_comms.hpp index df1687f93d..c81b19c9ba 100644 --- a/cpp/include/raft/comms/std_comms.hpp +++ b/cpp/include/raft/comms/std_comms.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,17 +16,17 @@ #pragma once +#include +#include #include #include #include -#include -#include - -#include #include #include +#include + namespace raft { namespace comms { diff --git a/cpp/include/raft/core/bitset.cuh b/cpp/include/raft/core/bitset.cuh index 552c2e9ac5..53fd586ed2 100644 --- a/cpp/include/raft/core/bitset.cuh +++ b/cpp/include/raft/core/bitset.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ #include #include #include + #include namespace raft::core { diff --git a/cpp/include/raft/core/comms.hpp b/cpp/include/raft/core/comms.hpp index 463c17f2f6..7dc4800ed6 100644 --- a/cpp/include/raft/core/comms.hpp +++ b/cpp/include/raft/core/comms.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,11 @@ #pragma once +#include + #include + #include -#include #include namespace raft { diff --git a/cpp/include/raft/core/cublas_macros.hpp b/cpp/include/raft/core/cublas_macros.hpp index c782c8ded1..b69b121161 100644 --- a/cpp/include/raft/core/cublas_macros.hpp +++ b/cpp/include/raft/core/cublas_macros.hpp @@ -19,9 +19,10 @@ #pragma once -#include #include +#include + ///@todo: enable this once we have logger enabled // #include diff --git a/cpp/include/raft/core/cusolver_macros.hpp b/cpp/include/raft/core/cusolver_macros.hpp index 4477d32118..74a8b7c36c 100644 --- a/cpp/include/raft/core/cusolver_macros.hpp +++ b/cpp/include/raft/core/cusolver_macros.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ ///@todo: enable this once logging is enabled // #include #include + #include #define _CUSOLVER_ERR_TO_STR(err) \ diff --git a/cpp/include/raft/core/cusparse_macros.hpp b/cpp/include/raft/core/cusparse_macros.hpp index 21a25ae28c..9058f4847d 100644 --- a/cpp/include/raft/core/cusparse_macros.hpp +++ b/cpp/include/raft/core/cusparse_macros.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,9 @@ #pragma once -#include #include + +#include ///@todo: enable this once logging is enabled // #include diff --git a/cpp/include/raft/core/detail/copy.hpp b/cpp/include/raft/core/detail/copy.hpp index 1a9a4d004d..04e74c4e58 100644 --- a/cpp/include/raft/core/detail/copy.hpp +++ b/cpp/include/raft/core/detail/copy.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ */ #pragma once -#include #include #include #include @@ -24,6 +23,8 @@ #include #include #include + +#include #include #ifndef RAFT_DISABLE_CUDA #include diff --git a/cpp/include/raft/core/detail/fail_container_policy.hpp b/cpp/include/raft/core/detail/fail_container_policy.hpp index ff36659f04..cf9d0887dd 100644 --- a/cpp/include/raft/core/detail/fail_container_policy.hpp +++ b/cpp/include/raft/core/detail/fail_container_policy.hpp @@ -19,6 +19,7 @@ #include #include #include + #include namespace raft { diff --git a/cpp/include/raft/core/detail/mdspan_util.cuh b/cpp/include/raft/core/detail/mdspan_util.cuh index 6b2c90abcc..ded95c2f31 100644 --- a/cpp/include/raft/core/detail/mdspan_util.cuh +++ b/cpp/include/raft/core/detail/mdspan_util.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,9 @@ */ #pragma once +#include #include -#include #include #include diff --git a/cpp/include/raft/core/detail/nvtx.hpp b/cpp/include/raft/core/detail/nvtx.hpp index f077a49b77..8afd1f16c6 100644 --- a/cpp/include/raft/core/detail/nvtx.hpp +++ b/cpp/include/raft/core/detail/nvtx.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,11 @@ #ifdef NVTX_ENABLED +#include + #include #include #include -#include #include #include #include diff --git a/cpp/include/raft/core/detail/span.hpp b/cpp/include/raft/core/detail/span.hpp index e6ccb8535c..5719fdfd2b 100644 --- a/cpp/include/raft/core/detail/span.hpp +++ b/cpp/include/raft/core/detail/span.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,10 @@ */ #pragma once -#include // numeric_limits #include #include + +#include // numeric_limits #include namespace raft { diff --git a/cpp/include/raft/core/device_container_policy.hpp b/cpp/include/raft/core/device_container_policy.hpp index e8717d4c5e..8c6eff582b 100644 --- a/cpp/include/raft/core/device_container_policy.hpp +++ b/cpp/include/raft/core/device_container_policy.hpp @@ -22,14 +22,13 @@ */ #pragma once #ifndef RAFT_DISABLE_CUDA -#include -#include - #include // dynamic_extent +#include #include - #include #include +#include + #include #include #include diff --git a/cpp/include/raft/core/device_csr_matrix.hpp b/cpp/include/raft/core/device_csr_matrix.hpp index da4ac117b1..1d23c8912d 100644 --- a/cpp/include/raft/core/device_csr_matrix.hpp +++ b/cpp/include/raft/core/device_csr_matrix.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ #include #include #include + #include namespace raft { diff --git a/cpp/include/raft/core/device_mdarray.hpp b/cpp/include/raft/core/device_mdarray.hpp index fe543c97dd..855642cd76 100644 --- a/cpp/include/raft/core/device_mdarray.hpp +++ b/cpp/include/raft/core/device_mdarray.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,13 @@ #pragma once -#include #include #include #include #include +#include + namespace raft { /** diff --git a/cpp/include/raft/core/device_mdspan.hpp b/cpp/include/raft/core/device_mdspan.hpp index 7988bd3f6f..b4e9f8d1d7 100644 --- a/cpp/include/raft/core/device_mdspan.hpp +++ b/cpp/include/raft/core/device_mdspan.hpp @@ -16,11 +16,12 @@ #pragma once -#include #include #include #include +#include + namespace raft { template diff --git a/cpp/include/raft/core/device_resources.hpp b/cpp/include/raft/core/device_resources.hpp index cf06920a8c..366e387fdd 100644 --- a/cpp/include/raft/core/device_resources.hpp +++ b/cpp/include/raft/core/device_resources.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,24 +19,7 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - #include -#include -#include - #include #include #include @@ -52,6 +35,24 @@ #include #include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + namespace raft { /** diff --git a/cpp/include/raft/core/device_resources_manager.hpp b/cpp/include/raft/core/device_resources_manager.hpp index 75c9428ff7..1c4bee15a6 100644 --- a/cpp/include/raft/core/device_resources_manager.hpp +++ b/cpp/include/raft/core/device_resources_manager.hpp @@ -15,17 +15,19 @@ */ #pragma once -#include -#include -#include #include #include + #include #include #include #include #include +#include +#include +#include + namespace raft { /** diff --git a/cpp/include/raft/core/device_setter.hpp b/cpp/include/raft/core/device_setter.hpp index 42049102aa..76ac65de8e 100644 --- a/cpp/include/raft/core/device_setter.hpp +++ b/cpp/include/raft/core/device_setter.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,10 @@ */ #pragma once -#include #include #include + +#include namespace raft { /** diff --git a/cpp/include/raft/core/error.hpp b/cpp/include/raft/core/error.hpp index 6533e9c3c6..63b7d5e626 100644 --- a/cpp/include/raft/core/error.hpp +++ b/cpp/include/raft/core/error.hpp @@ -33,6 +33,7 @@ #ifdef ENABLE_COLLECT_CALLSTACK #include #include + #include #endif diff --git a/cpp/include/raft/core/host_container_policy.hpp b/cpp/include/raft/core/host_container_policy.hpp index 0192436934..fa44f1b138 100644 --- a/cpp/include/raft/core/host_container_policy.hpp +++ b/cpp/include/raft/core/host_container_policy.hpp @@ -23,6 +23,7 @@ #pragma once #include #include + #include namespace raft { diff --git a/cpp/include/raft/core/host_csr_matrix.hpp b/cpp/include/raft/core/host_csr_matrix.hpp index f32ff1dc00..e3cea3cd27 100644 --- a/cpp/include/raft/core/host_csr_matrix.hpp +++ b/cpp/include/raft/core/host_csr_matrix.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include #include #include + #include namespace raft { diff --git a/cpp/include/raft/core/host_device_accessor.hpp b/cpp/include/raft/core/host_device_accessor.hpp index 7cb2aaf487..cbcfa976b2 100644 --- a/cpp/include/raft/core/host_device_accessor.hpp +++ b/cpp/include/raft/core/host_device_accessor.hpp @@ -16,6 +16,7 @@ #pragma once #include + #include namespace raft { diff --git a/cpp/include/raft/core/host_mdarray.hpp b/cpp/include/raft/core/host_mdarray.hpp index 02c8914ca1..3020cde32d 100644 --- a/cpp/include/raft/core/host_mdarray.hpp +++ b/cpp/include/raft/core/host_mdarray.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,12 @@ #pragma once -#include +#include #include +#include #include -#include -#include +#include namespace raft { /** diff --git a/cpp/include/raft/core/host_mdspan.hpp b/cpp/include/raft/core/host_mdspan.hpp index d5f431f4a2..ef0a9a2c0f 100644 --- a/cpp/include/raft/core/host_mdspan.hpp +++ b/cpp/include/raft/core/host_mdspan.hpp @@ -16,11 +16,11 @@ #pragma once -#include +#include #include #include -#include +#include namespace raft { diff --git a/cpp/include/raft/core/interruptible.hpp b/cpp/include/raft/core/interruptible.hpp index 10ab22f820..4ec1067e00 100644 --- a/cpp/include/raft/core/interruptible.hpp +++ b/cpp/include/raft/core/interruptible.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,14 @@ #pragma once -#include -#include -#include #include #include + #include + +#include +#include +#include #include #include diff --git a/cpp/include/raft/core/kvp.hpp b/cpp/include/raft/core/kvp.hpp index 610945a76c..13686c0c9b 100644 --- a/cpp/include/raft/core/kvp.hpp +++ b/cpp/include/raft/core/kvp.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,8 +19,9 @@ #include #ifdef _RAFT_HAS_CUDA -#include #include // raft::shfl_xor + +#include #endif namespace raft { /** diff --git a/cpp/include/raft/core/logger-ext.hpp b/cpp/include/raft/core/logger-ext.hpp index 04a6a4d060..73fe463aba 100644 --- a/cpp/include/raft/core/logger-ext.hpp +++ b/cpp/include/raft/core/logger-ext.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,11 @@ */ #pragma once -#include // std::unique_ptr #include // RAFT_INLINE_CONDITIONAL -#include // std::string -#include // std::unordered_map + +#include // std::unique_ptr +#include // std::string +#include // std::unordered_map namespace raft { diff --git a/cpp/include/raft/core/logger-inl.hpp b/cpp/include/raft/core/logger-inl.hpp index fcfa1f1333..ea5f4ea26e 100644 --- a/cpp/include/raft/core/logger-inl.hpp +++ b/cpp/include/raft/core/logger-inl.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,19 +15,16 @@ */ #pragma once +#include "logger-macros.hpp" + #include #include - #include #include #include #include #include - -#include - -#include "logger-macros.hpp" // The logger-ext.hpp file contains the class declaration of the logger class. // In this case, it is okay to include the logger-ext.hpp file because it // contains no RAFT_EXPLICIT template instantiations. @@ -35,7 +32,8 @@ #define SPDLOG_HEADER_ONLY #include -#include // RAFT_INLINE_CONDITIONAL +#include // RAFT_INLINE_CONDITIONAL + #include // NOLINT #include // NOLINT diff --git a/cpp/include/raft/core/logger.hpp b/cpp/include/raft/core/logger.hpp index 59968ff5e5..e64a0db257 100644 --- a/cpp/include/raft/core/logger.hpp +++ b/cpp/include/raft/core/logger.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,8 @@ */ #pragma once -#include "logger-macros.hpp" - #include "logger-ext.hpp" +#include "logger-macros.hpp" #if !defined(RAFT_COMPILED) #include "logger-inl.hpp" diff --git a/cpp/include/raft/core/managed_container_policy.hpp b/cpp/include/raft/core/managed_container_policy.hpp index f4e26c6ef1..889e9383e6 100644 --- a/cpp/include/raft/core/managed_container_policy.hpp +++ b/cpp/include/raft/core/managed_container_policy.hpp @@ -15,15 +15,14 @@ */ #pragma once #ifndef RAFT_DISABLE_CUDA +#include // dynamic_extent #include #include -#include - -#include // dynamic_extent #include - #include #include +#include + #include namespace raft { diff --git a/cpp/include/raft/core/managed_mdarray.hpp b/cpp/include/raft/core/managed_mdarray.hpp index c1438d941d..021fa11321 100644 --- a/cpp/include/raft/core/managed_mdarray.hpp +++ b/cpp/include/raft/core/managed_mdarray.hpp @@ -16,12 +16,13 @@ #pragma once -#include #include #include #include #include +#include + namespace raft { /** diff --git a/cpp/include/raft/core/managed_mdspan.hpp b/cpp/include/raft/core/managed_mdspan.hpp index 9c2976ec6b..b421982df4 100644 --- a/cpp/include/raft/core/managed_mdspan.hpp +++ b/cpp/include/raft/core/managed_mdspan.hpp @@ -16,11 +16,12 @@ #pragma once -#include #include #include #include +#include + namespace raft { template diff --git a/cpp/include/raft/core/math.hpp b/cpp/include/raft/core/math.hpp index 809b2948e7..9ce768bf40 100644 --- a/cpp/include/raft/core/math.hpp +++ b/cpp/include/raft/core/math.hpp @@ -16,12 +16,12 @@ #pragma once +#include + #include #include #include -#include - #if defined(_RAFT_HAS_CUDA) #include #include diff --git a/cpp/include/raft/core/mdarray.hpp b/cpp/include/raft/core/mdarray.hpp index 2cdeb36fc8..0fa8e98df9 100644 --- a/cpp/include/raft/core/mdarray.hpp +++ b/cpp/include/raft/core/mdarray.hpp @@ -6,7 +6,7 @@ */ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,8 +22,6 @@ */ #pragma once -#include - #include #include #include @@ -31,6 +29,8 @@ #include #include +#include + namespace raft { /** diff --git a/cpp/include/raft/core/mdbuffer.cuh b/cpp/include/raft/core/mdbuffer.cuh index 18533ce882..d9557f8c55 100644 --- a/cpp/include/raft/core/mdbuffer.cuh +++ b/cpp/include/raft/core/mdbuffer.cuh @@ -15,9 +15,6 @@ */ #pragma once -#include -#include -#include #include #include #include @@ -31,12 +28,17 @@ #include #include #include + +#include +#include +#include #include #include #include #ifndef RAFT_DISABLE_CUDA #include #include + #include #else #include diff --git a/cpp/include/raft/core/mdspan.hpp b/cpp/include/raft/core/mdspan.hpp index 15414b9af3..45bebaec2f 100644 --- a/cpp/include/raft/core/mdspan.hpp +++ b/cpp/include/raft/core/mdspan.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,12 @@ */ #pragma once -#include -#include -#include - #include #include +#include #include - +#include +#include #include namespace raft { diff --git a/cpp/include/raft/core/memory_type.hpp b/cpp/include/raft/core/memory_type.hpp index 7849cd67ab..0334cb5059 100644 --- a/cpp/include/raft/core/memory_type.hpp +++ b/cpp/include/raft/core/memory_type.hpp @@ -17,8 +17,10 @@ #include #include #ifndef RAFT_DISABLE_CUDA -#include #include + +#include + #include #else #include diff --git a/cpp/include/raft/core/nvtx.hpp b/cpp/include/raft/core/nvtx.hpp index 57338c32c7..26e3a383f2 100644 --- a/cpp/include/raft/core/nvtx.hpp +++ b/cpp/include/raft/core/nvtx.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,10 @@ #pragma once -#include #include +#include + /** * \section Usage * diff --git a/cpp/include/raft/core/operators.hpp b/cpp/include/raft/core/operators.hpp index 7acc907c49..e42801fe32 100644 --- a/cpp/include/raft/core/operators.hpp +++ b/cpp/include/raft/core/operators.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,15 @@ #pragma once +#include +#include + #include #include #include #include #include -#include -#include - namespace raft { /** diff --git a/cpp/include/raft/core/pinned_container_policy.hpp b/cpp/include/raft/core/pinned_container_policy.hpp index 51451deadb..4870e2c5dc 100644 --- a/cpp/include/raft/core/pinned_container_policy.hpp +++ b/cpp/include/raft/core/pinned_container_policy.hpp @@ -14,9 +14,10 @@ * limitations under the License. */ #pragma once -#include #include #include + +#include #ifndef RAFT_DISABLE_CUDA #include #include diff --git a/cpp/include/raft/core/pinned_mdarray.hpp b/cpp/include/raft/core/pinned_mdarray.hpp index 72b8d52e0d..f289062a02 100644 --- a/cpp/include/raft/core/pinned_mdarray.hpp +++ b/cpp/include/raft/core/pinned_mdarray.hpp @@ -16,12 +16,13 @@ #pragma once -#include #include #include #include #include +#include + namespace raft { /** diff --git a/cpp/include/raft/core/pinned_mdspan.hpp b/cpp/include/raft/core/pinned_mdspan.hpp index e764101d1c..d03433a8db 100644 --- a/cpp/include/raft/core/pinned_mdspan.hpp +++ b/cpp/include/raft/core/pinned_mdspan.hpp @@ -16,11 +16,12 @@ #pragma once -#include #include #include #include +#include + namespace raft { template diff --git a/cpp/include/raft/core/resource/cublas_handle.hpp b/cpp/include/raft/core/resource/cublas_handle.hpp index 306a641f34..1c86f3e93f 100644 --- a/cpp/include/raft/core/resource/cublas_handle.hpp +++ b/cpp/include/raft/core/resource/cublas_handle.hpp @@ -15,12 +15,13 @@ */ #pragma once -#include #include #include #include #include +#include + namespace raft::resource { class cublas_resource : public resource { diff --git a/cpp/include/raft/core/resource/cublaslt_handle.hpp b/cpp/include/raft/core/resource/cublaslt_handle.hpp index a61e56bd91..4653a55e16 100644 --- a/cpp/include/raft/core/resource/cublaslt_handle.hpp +++ b/cpp/include/raft/core/resource/cublaslt_handle.hpp @@ -15,11 +15,12 @@ */ #pragma once -#include #include #include #include +#include + #include namespace raft::resource { diff --git a/cpp/include/raft/core/resource/cuda_event.hpp b/cpp/include/raft/core/resource/cuda_event.hpp index 4859d95ee9..db8e40925c 100644 --- a/cpp/include/raft/core/resource/cuda_event.hpp +++ b/cpp/include/raft/core/resource/cuda_event.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #pragma once -#include #include #include #include +#include + namespace raft::resource { class cuda_event_resource : public resource { diff --git a/cpp/include/raft/core/resource/cuda_stream.hpp b/cpp/include/raft/core/resource/cuda_stream.hpp index 6120a4b75c..68b5c055fe 100644 --- a/cpp/include/raft/core/resource/cuda_stream.hpp +++ b/cpp/include/raft/core/resource/cuda_stream.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,15 @@ */ #pragma once -#include #include #include #include #include + #include +#include + namespace raft::resource { class cuda_stream_resource : public resource { public: diff --git a/cpp/include/raft/core/resource/cuda_stream_pool.hpp b/cpp/include/raft/core/resource/cuda_stream_pool.hpp index 7ed356485c..c15a640909 100644 --- a/cpp/include/raft/core/resource/cuda_stream_pool.hpp +++ b/cpp/include/raft/core/resource/cuda_stream_pool.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,15 +15,16 @@ */ #pragma once -#include #include #include #include - #include #include + #include +#include + namespace raft::resource { class cuda_stream_pool_resource : public resource { diff --git a/cpp/include/raft/core/resource/cusolver_dn_handle.hpp b/cpp/include/raft/core/resource/cusolver_dn_handle.hpp index 7a33e2dd2a..a0698d9b82 100644 --- a/cpp/include/raft/core/resource/cusolver_dn_handle.hpp +++ b/cpp/include/raft/core/resource/cusolver_dn_handle.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,15 @@ #pragma once #include "cuda_stream.hpp" -#include + #include #include #include + #include +#include + namespace raft::resource { /** diff --git a/cpp/include/raft/core/resource/cusolver_sp_handle.hpp b/cpp/include/raft/core/resource/cusolver_sp_handle.hpp index 61fd95b44f..f28724632c 100644 --- a/cpp/include/raft/core/resource/cusolver_sp_handle.hpp +++ b/cpp/include/raft/core/resource/cusolver_sp_handle.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,13 @@ */ #pragma once -#include #include #include #include #include +#include + namespace raft::resource { /** diff --git a/cpp/include/raft/core/resource/cusparse_handle.hpp b/cpp/include/raft/core/resource/cusparse_handle.hpp index 9893ed2f86..f019b41a52 100644 --- a/cpp/include/raft/core/resource/cusparse_handle.hpp +++ b/cpp/include/raft/core/resource/cusparse_handle.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,13 @@ */ #pragma once -#include #include #include #include #include +#include + namespace raft::resource { class cusparse_resource : public resource { public: diff --git a/cpp/include/raft/core/resource/detail/stream_sync_event.hpp b/cpp/include/raft/core/resource/detail/stream_sync_event.hpp index 1d02fef20d..8429c7667d 100644 --- a/cpp/include/raft/core/resource/detail/stream_sync_event.hpp +++ b/cpp/include/raft/core/resource/detail/stream_sync_event.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,13 @@ */ #pragma once -#include #include #include #include #include +#include + namespace raft::resource::detail { /** diff --git a/cpp/include/raft/core/resource/device_id.hpp b/cpp/include/raft/core/resource/device_id.hpp index b55e56ca45..570d815780 100644 --- a/cpp/include/raft/core/resource/device_id.hpp +++ b/cpp/include/raft/core/resource/device_id.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #pragma once -#include #include #include #include +#include + namespace raft::resource { class device_id_resource : public resource { diff --git a/cpp/include/raft/core/resource/device_properties.hpp b/cpp/include/raft/core/resource/device_properties.hpp index c3b0b8f2b9..a87c29f709 100644 --- a/cpp/include/raft/core/resource/device_properties.hpp +++ b/cpp/include/raft/core/resource/device_properties.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,13 @@ */ #pragma once -#include #include #include #include #include +#include + namespace raft::resource { class device_properties_resource : public resource { diff --git a/cpp/include/raft/core/resource/thrust_policy.hpp b/cpp/include/raft/core/resource/thrust_policy.hpp index 78c04ce875..f81898be8a 100644 --- a/cpp/include/raft/core/resource/thrust_policy.hpp +++ b/cpp/include/raft/core/resource/thrust_policy.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ #include #include #include + #include namespace raft::resource { class thrust_policy_resource : public resource { diff --git a/cpp/include/raft/core/resources.hpp b/cpp/include/raft/core/resources.hpp index d5bd176d50..b0827d8e11 100644 --- a/cpp/include/raft/core/resources.hpp +++ b/cpp/include/raft/core/resources.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,12 @@ #pragma once #include "resource/resource_types.hpp" -#include -#include + #include // RAFT_EXPECTS #include + +#include +#include #include #include diff --git a/cpp/include/raft/core/span.hpp b/cpp/include/raft/core/span.hpp index d77e0fcb40..965600f490 100644 --- a/cpp/include/raft/core/span.hpp +++ b/cpp/include/raft/core/span.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,13 @@ */ #pragma once +#include +#include +#include + #include #include // size_t #include // std::byte -#include - -#include -#include // TODO (cjnolet): Remove thrust dependencies here so host_span can be used without CUDA Toolkit // being installed. Reference: https://github.com/rapidsai/raft/issues/812. @@ -29,6 +29,7 @@ #include #include // _RAFT_HOST_DEVICE #include + #include namespace raft { diff --git a/cpp/include/raft/core/stream_view.hpp b/cpp/include/raft/core/stream_view.hpp index 128050c414..3a42ee949e 100644 --- a/cpp/include/raft/core/stream_view.hpp +++ b/cpp/include/raft/core/stream_view.hpp @@ -19,6 +19,7 @@ #include #ifndef RAFT_DISABLE_CUDA #include + #include #endif diff --git a/cpp/include/raft/core/temporary_device_buffer.hpp b/cpp/include/raft/core/temporary_device_buffer.hpp index 358eeab861..8b07b51854 100644 --- a/cpp/include/raft/core/temporary_device_buffer.hpp +++ b/cpp/include/raft/core/temporary_device_buffer.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,8 +18,8 @@ #include "device_mdarray.hpp" #include "device_mdspan.hpp" -#include +#include #include #include diff --git a/cpp/include/raft/distance/detail/distance.cuh b/cpp/include/raft/distance/detail/distance.cuh index b6885808ce..a5c8c0ef4b 100644 --- a/cpp/include/raft/distance/detail/distance.cuh +++ b/cpp/include/raft/distance/detail/distance.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ #include #include #include + #include namespace raft { diff --git a/cpp/include/raft/distance/detail/fused_distance_nn/custom_epilogue_with_broadcast.h b/cpp/include/raft/distance/detail/fused_distance_nn/custom_epilogue_with_broadcast.h index ac20578083..186715851b 100644 --- a/cpp/include/raft/distance/detail/fused_distance_nn/custom_epilogue_with_broadcast.h +++ b/cpp/include/raft/distance/detail/fused_distance_nn/custom_epilogue_with_broadcast.h @@ -67,30 +67,26 @@ reduction #include #else #include + #include #endif #include #include #include +#include +#include #include #include +#include #include #include #include #include #include - -#include - #include #include -#include -#include - -#include - ///////////////////////////////////////////////////////////////////////////////////////////////// namespace cutlass { diff --git a/cpp/include/raft/distance/detail/fused_distance_nn/cutlass_base.cuh b/cpp/include/raft/distance/detail/fused_distance_nn/cutlass_base.cuh index be1c8891f5..3f359c2f54 100644 --- a/cpp/include/raft/distance/detail/fused_distance_nn/cutlass_base.cuh +++ b/cpp/include/raft/distance/detail/fused_distance_nn/cutlass_base.cuh @@ -43,6 +43,17 @@ #include // getMultiProcessorCount #include // RAFT_CUTLASS_TRY +#include + +#include +#include +#include +#include +#include +#include +#include +#include + namespace raft { namespace distance { namespace detail { diff --git a/cpp/include/raft/distance/detail/fused_distance_nn/epilogue.cuh b/cpp/include/raft/distance/detail/fused_distance_nn/epilogue.cuh index 8a0bea3469..22848d30b0 100644 --- a/cpp/include/raft/distance/detail/fused_distance_nn/epilogue.cuh +++ b/cpp/include/raft/distance/detail/fused_distance_nn/epilogue.cuh @@ -29,7 +29,7 @@ * **************************************************************************************************/ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -60,19 +60,17 @@ operation. #pragma once +#include +#include +#include + #include #include -#include - -#include - #include #include #include -#include - -#include -#include +#include +#include //////////////////////////////////////////////////////////////////////////////// diff --git a/cpp/include/raft/distance/detail/fused_distance_nn/epilogue_elementwise.cuh b/cpp/include/raft/distance/detail/fused_distance_nn/epilogue_elementwise.cuh index d65d2df4a4..993cf2f086 100644 --- a/cpp/include/raft/distance/detail/fused_distance_nn/epilogue_elementwise.cuh +++ b/cpp/include/raft/distance/detail/fused_distance_nn/epilogue_elementwise.cuh @@ -58,6 +58,7 @@ #include #include +#include #include #include #include diff --git a/cpp/include/raft/distance/detail/fused_distance_nn/gemm.h b/cpp/include/raft/distance/detail/fused_distance_nn/gemm.h index 3da8b3ee3d..42de4860a0 100644 --- a/cpp/include/raft/distance/detail/fused_distance_nn/gemm.h +++ b/cpp/include/raft/distance/detail/fused_distance_nn/gemm.h @@ -29,7 +29,7 @@ * **************************************************************************************************/ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,15 +46,14 @@ #pragma once -#include +#include +#include +#include #include #include #include -#include -#include - ///////////////////////////////////////////////////////////////////////////////////////////////// namespace cutlass { diff --git a/cpp/include/raft/distance/detail/fused_distance_nn/persistent_gemm.h b/cpp/include/raft/distance/detail/fused_distance_nn/persistent_gemm.h index 223af7eb58..f1a7c728e9 100644 --- a/cpp/include/raft/distance/detail/fused_distance_nn/persistent_gemm.h +++ b/cpp/include/raft/distance/detail/fused_distance_nn/persistent_gemm.h @@ -60,12 +60,11 @@ This file contains heavily customized version of GemmGrouped from CUTLASS 2.10.0 #include #include #include -#include -#include - #include #include #include +#include +#include #include ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cpp/include/raft/distance/detail/fused_distance_nn/predicated_tile_iterator_normvec_smem.h b/cpp/include/raft/distance/detail/fused_distance_nn/predicated_tile_iterator_normvec_smem.h index 14c09f6aef..794cd5eb63 100644 --- a/cpp/include/raft/distance/detail/fused_distance_nn/predicated_tile_iterator_normvec_smem.h +++ b/cpp/include/raft/distance/detail/fused_distance_nn/predicated_tile_iterator_normvec_smem.h @@ -29,7 +29,7 @@ * **************************************************************************************************/ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,6 +58,8 @@ This file contains a customized version of PredicatedTileIterator from CUTLASS 2 */ #pragma once +#include + #include #include #include @@ -71,8 +73,6 @@ This file contains a customized version of PredicatedTileIterator from CUTLASS 2 #include #include -#include - //////////////////////////////////////////////////////////////////////////////// namespace cutlass { diff --git a/cpp/include/raft/distance/detail/fused_distance_nn/predicated_tile_iterator_reduced_vec.h b/cpp/include/raft/distance/detail/fused_distance_nn/predicated_tile_iterator_reduced_vec.h index 5ceb0dbaf5..d61018593f 100644 --- a/cpp/include/raft/distance/detail/fused_distance_nn/predicated_tile_iterator_reduced_vec.h +++ b/cpp/include/raft/distance/detail/fused_distance_nn/predicated_tile_iterator_reduced_vec.h @@ -64,6 +64,7 @@ gmem. #include #include + #include #include #include diff --git a/cpp/include/raft/distance/detail/fused_l2_nn.cuh b/cpp/include/raft/distance/detail/fused_l2_nn.cuh new file mode 100644 index 0000000000..5e62045305 --- /dev/null +++ b/cpp/include/raft/distance/detail/fused_l2_nn.cuh @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2021-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include // raft::KeyValuePair +#include // raft::identity_op +#include // ops::l2_exp_distance_op +#include +#include // PairwiseDistances +#include // Policy +#include // raft::util::arch::SM_* +#include // raft::ceildiv, raft::shfl + +#include // size_t +#include // std::numeric_limits + +namespace raft { +namespace distance { + +namespace detail { + +template +struct KVPMinReduceImpl { + typedef raft::KeyValuePair KVP; + DI KVP operator()(LabelT rit, const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } + DI KVP operator()(const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } + +}; // KVPMinReduce + +template +struct MinAndDistanceReduceOpImpl { + typedef typename raft::KeyValuePair KVP; + DI void operator()(LabelT rid, KVP* out, const KVP& other) const + { + if (other.value < out->value) { + out->key = other.key; + out->value = other.value; + } + } + + DI void operator()(LabelT rid, DataT* out, const KVP& other) const + { + if (other.value < *out) { *out = other.value; } + } + + DI void operator()(LabelT rid, DataT* out, const DataT& other) const + { + if (other < *out) { *out = other; } + } + + DI void init(DataT* out, DataT maxVal) const { *out = maxVal; } + DI void init(KVP* out, DataT maxVal) const { out->value = maxVal; } + + DI void init_key(DataT& out, LabelT idx) const { return; } + DI void init_key(KVP& out, LabelT idx) const { out.key = idx; } + + DI DataT get_value(KVP& out) const + { + return out.value; + ; + } + DI DataT get_value(DataT& out) const { return out; } +}; + +template +struct MinReduceOpImpl { + typedef typename raft::KeyValuePair KVP; + DI void operator()(LabelT rid, DataT* out, const KVP& other) + { + if (other.value < *out) { *out = other.value; } + } + + DI void init(DataT* out, DataT maxVal) { *out = maxVal; } +}; + +template +RAFT_KERNEL initKernel(OutT* min, IdxT m, DataT maxVal, ReduceOpT redOp) +{ + auto tid = IdxT(blockIdx.x) * blockDim.x + threadIdx.x; + if (tid < m) { redOp.init(min + tid, maxVal); } +} + +template +void initialize(OutT* min, IdxT m, DataT maxVal, ReduceOpT redOp, cudaStream_t stream) +{ + auto blks = raft::ceildiv(m, 256); + initKernel<<>>(min, m, maxVal, redOp); +} + +// TODO: specialize this function for MinAndDistanceReduceOp +// with atomicCAS of 64 bit which will eliminate mutex and shfls +template +DI void updateReducedVal( + int* mutex, OutT* min, KVPair* val, ReduceOpT red_op, IdxT m, IdxT gridStrideY) +{ + const auto lid = threadIdx.x % raft::WarpSize; + const auto accrowid = threadIdx.x / P::AccThCols; + + // Update each output row in order within a warp. This will resolve hang + // issues with pre-Volta architectures +#pragma unroll + for (int j = 0; j < (raft::WarpSize / P::AccThCols); j++) { + if (lid == j * P::AccThCols) { +#pragma unroll + for (int i = 0; i < P::AccRowsPerTh; ++i) { + auto rid = gridStrideY + accrowid + i * P::AccThRows; + if (rid < m) { + auto value = val[i]; + while (atomicCAS(mutex + rid, 0, 1) == 1) + ; + __threadfence(); + red_op(rid, min + rid, value); + __threadfence(); + atomicCAS(mutex + rid, 1, 0); + } + } + } + } +} + +template +__launch_bounds__(P::Nthreads, 2) RAFT_KERNEL fusedL2NNkernel(OutT* min, + const DataT* x, + const DataT* y, + const DataT* xn, + const DataT* yn, + IdxT m, + IdxT n, + IdxT k, + DataT maxVal, + int* mutex, + ReduceOpT redOp, + KVPReduceOpT pairRedOp, + OpT distance_op, + FinalLambda fin_op) +{ +// compile only if below non-ampere arch. +#if __CUDA_ARCH__ < 800 + extern __shared__ char smem[]; + + typedef KeyValuePair KVPair; + KVPair val[P::AccRowsPerTh]; +#pragma unroll + for (int i = 0; i < P::AccRowsPerTh; ++i) { + val[i] = {0, maxVal}; + } + + // epilogue operation lambda for final value calculation + auto epilog_lambda = [n, pairRedOp, &val, maxVal] __device__( + DataT acc[P::AccRowsPerTh][P::AccColsPerTh], + DataT * regxn, + DataT * regyn, + IdxT gridStrideX, + IdxT gridStrideY) { + KVPReduceOpT pairRed_op(pairRedOp); + + // intra thread reduce + const auto acccolid = threadIdx.x % P::AccThCols; + const auto accrowid = threadIdx.x / P::AccThCols; +#pragma unroll + for (int i = 0; i < P::AccRowsPerTh; ++i) { +#pragma unroll + for (int j = 0; j < P::AccColsPerTh; ++j) { + auto tmpkey = acccolid + j * P::AccThCols + gridStrideX; + KVPair tmp = {tmpkey, acc[i][j]}; + if (tmpkey < n) { + val[i] = pairRed_op(accrowid + i * P::AccThRows + gridStrideY, tmp, val[i]); + } + } + } + }; + + auto rowEpilog_lambda = + [m, mutex, min, pairRedOp, redOp, &val, maxVal] __device__(IdxT gridStrideY) { + KVPReduceOpT pairRed_op(pairRedOp); + ReduceOpT red_op(redOp); + + const auto accrowid = threadIdx.x / P::AccThCols; + const auto lid = raft::laneId(); + + // reduce +#pragma unroll + for (int i = 0; i < P::AccRowsPerTh; ++i) { +#pragma unroll + for (int j = P::AccThCols / 2; j > 0; j >>= 1) { + // Actually, the srcLane (lid +j) should be (lid +j) % P:AccThCols, + // but the shfl op applies the modulo internally. + auto tmpkey = raft::shfl(val[i].key, lid + j, P::AccThCols); + auto tmpvalue = raft::shfl(val[i].value, lid + j, P::AccThCols); + KVPair tmp = {tmpkey, tmpvalue}; + val[i] = pairRed_op(accrowid + i * P::AccThRows + gridStrideY, tmp, val[i]); + } + } + + updateReducedVal(mutex, min, val, red_op, m, gridStrideY); + + // reset the val array. +#pragma unroll + for (int i = 0; i < P::AccRowsPerTh; ++i) { + val[i] = {0, maxVal}; + } + }; + + IdxT lda = k, ldb = k, ldd = n; + constexpr bool row_major = true; + constexpr bool write_out = false; + PairwiseDistances + obj(x, + y, + m, + n, + k, + lda, + ldb, + ldd, + xn, + yn, + nullptr, // Output pointer + smem, + distance_op, + epilog_lambda, + fin_op, + rowEpilog_lambda); + obj.run(); +#endif +} + +// cg::reduce functor for FusedDistanceNN used in its cutlass version +// to output the min distance value & key(loc id). +// This is used in fused_distance_nn/predicated_tile_iterator_reduced_vec.h +// store_with_byte_offset() passed to cg::reduce() & select_reduce. +template +struct kvp_cg_min_reduce_op { + typedef typename raft::KeyValuePair KVP; + + __host__ __device__ kvp_cg_min_reduce_op() noexcept {}; + + using AccTypeT = AccType; + using IndexT = Index; + // functor signature. + __host__ __device__ KVP operator()(KVP a, KVP b) const { return a.value < b.value ? a : b; } + + __host__ __device__ AccType operator()(AccType a, AccType b) const { return min(a, b); } + + __host__ __device__ bool isAmin(AccType a, AccType b) const { return a < b ? true : false; } +}; + +template +void fusedL2NNImpl(OutT* min, + const DataT* x, + const DataT* y, + const DataT* xn, + const DataT* yn, + IdxT m, + IdxT n, + IdxT k, + int* workspace, + ReduceOpT redOp, + KVPReduceOpT pairRedOp, + bool sqrt, + bool initOutBuffer, + cudaStream_t stream) +{ + // The kernel policy is determined by fusedL2NN. + typedef Policy P; + + dim3 blk(P::Nthreads); + auto nblks = raft::ceildiv(m, P::Nthreads); + constexpr auto maxVal = std::numeric_limits::max(); + typedef KeyValuePair KVPair; + + RAFT_CUDA_TRY(cudaMemsetAsync(workspace, 0, sizeof(int) * m, stream)); + if (initOutBuffer) { + initKernel + <<>>(min, m, maxVal, redOp); + RAFT_CUDA_TRY(cudaGetLastError()); + } + + namespace arch = raft::util::arch; + using AccT = DataT; + ops::l2_exp_distance_op distance_op{sqrt}; + + raft::identity_op fin_op{}; + + auto kernel = fusedL2NNkernel; + + // Get pointer to fp32 SIMT kernel to determine the best compute architecture + // out of all for which the kernel was compiled for that matches closely + // to the current device. Other methods to determine the architecture (that do not + // require a pointer) can be error prone. See: + // https://github.com/NVIDIA/cub/issues/545 + void* kernel_ptr = reinterpret_cast(kernel); + auto runtime_arch = arch::kernel_virtual_arch(kernel_ptr); + auto cutlass_range = arch::SM_range(arch::SM_80(), arch::SM_future()); + + if (cutlass_range.contains(runtime_arch)) { + // If device is SM_80 or later, use CUTLASS-based kernel. + using L2Op = raft::distance::detail::ops::l2_exp_cutlass_op; + using kvp_cg_min_reduce_op_ = kvp_cg_min_reduce_op; + kvp_cg_min_reduce_op_ cg_reduce_op; + L2Op L2_dist_op(sqrt); + + IdxT lda, ldb, ldd; + lda = k, ldb = k, ldd = n; + + cutlassFusedDistanceNN(x, + y, + xn, + yn, + m, + n, + k, + lda, + ldb, + ldd, + min, + workspace, + cg_reduce_op, + L2_dist_op, + redOp, + pairRedOp, + stream); + } else { + // If device less than SM_80, use fp32 SIMT kernel. + constexpr size_t shmemSize = P::SmemSize + ((P::Mblk + P::Nblk) * sizeof(DataT)); + dim3 grid = launchConfigGenerator

(m, n, shmemSize, kernel); + + kernel<<>>( + min, x, y, xn, yn, m, n, k, maxVal, workspace, redOp, pairRedOp, distance_op, fin_op); + RAFT_CUDA_TRY(cudaGetLastError()); + } +} + +} // namespace detail +} // namespace distance +} // namespace raft diff --git a/cpp/include/raft/distance/detail/kernels/gram_matrix.cuh b/cpp/include/raft/distance/detail/kernels/gram_matrix.cuh index 14b4ba12c6..4ce489bc05 100644 --- a/cpp/include/raft/distance/detail/kernels/gram_matrix.cuh +++ b/cpp/include/raft/distance/detail/kernels/gram_matrix.cuh @@ -22,11 +22,10 @@ #include #include // #include -#include -#include - #include #include +#include +#include namespace raft::distance::kernels::detail { diff --git a/cpp/include/raft/distance/detail/kernels/kernel_factory.cuh b/cpp/include/raft/distance/detail/kernels/kernel_factory.cuh index bb3ff1c2f5..a64f47cea4 100644 --- a/cpp/include/raft/distance/detail/kernels/kernel_factory.cuh +++ b/cpp/include/raft/distance/detail/kernels/kernel_factory.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ #include "gram_matrix.cuh" #include "kernel_matrices.cuh" + #include #include diff --git a/cpp/include/raft/distance/detail/kernels/kernel_matrices.cuh b/cpp/include/raft/distance/detail/kernels/kernel_matrices.cuh index 8d5b2c766e..0d64479d84 100644 --- a/cpp/include/raft/distance/detail/kernels/kernel_matrices.cuh +++ b/cpp/include/raft/distance/detail/kernels/kernel_matrices.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ #pragma once #include "gram_matrix.cuh" -#include +#include #include #include #include diff --git a/cpp/include/raft/distance/detail/masked_nn.cuh b/cpp/include/raft/distance/detail/masked_nn.cuh index 7bbb1ae789..3e3699766f 100644 --- a/cpp/include/raft/distance/detail/masked_nn.cuh +++ b/cpp/include/raft/distance/detail/masked_nn.cuh @@ -16,18 +16,20 @@ #pragma once -#include #include #include -#include - #include #include #include #include #include + #include +#include + +#include + namespace raft { namespace distance { namespace detail { diff --git a/cpp/include/raft/distance/detail/pairwise_distance_cutlass_base.cuh b/cpp/include/raft/distance/detail/pairwise_distance_cutlass_base.cuh index aeb862b06a..282097742c 100644 --- a/cpp/include/raft/distance/detail/pairwise_distance_cutlass_base.cuh +++ b/cpp/include/raft/distance/detail/pairwise_distance_cutlass_base.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,23 +26,23 @@ #define cutlass raft_cutlass #endif +#include "pairwise_distance_epilogue_elementwise.h" +#include "pairwise_distance_gemm.h" + +#include +#include + #include -#include #include #include #include - #include #include #include #include -#include -#include - -#include "./pairwise_distance_epilogue_elementwise.h" -#include "./pairwise_distance_gemm.h" +#include namespace raft { namespace distance { diff --git a/cpp/include/raft/distance/detail/pairwise_distance_epilogue.h b/cpp/include/raft/distance/detail/pairwise_distance_epilogue.h index 21e7d18854..6ead09ed16 100644 --- a/cpp/include/raft/distance/detail/pairwise_distance_epilogue.h +++ b/cpp/include/raft/distance/detail/pairwise_distance_epilogue.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,17 +30,16 @@ operation. #pragma once +#include "./predicated_tile_iterator_normvec.h" + #include #include -#include - -#include - -#include "./predicated_tile_iterator_normvec.h" #include #include #include #include +#include +#include //////////////////////////////////////////////////////////////////////////////// diff --git a/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h b/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h index 3e33f4d833..2b2c04b9d3 100644 --- a/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h +++ b/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,12 +27,12 @@ #include #include +#include #include +#include #include #include -#include - ///////////////////////////////////////////////////////////////////////////////////////////////// namespace cutlass { diff --git a/cpp/include/raft/distance/detail/pairwise_distance_gemm.h b/cpp/include/raft/distance/detail/pairwise_distance_gemm.h index 8dcccfc14f..aaf2689dab 100644 --- a/cpp/include/raft/distance/detail/pairwise_distance_gemm.h +++ b/cpp/include/raft/distance/detail/pairwise_distance_gemm.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,14 @@ #pragma once -#include +#include "./pairwise_distance_epilogue.h" +#include #include #include #include #include -#include "./pairwise_distance_epilogue.h" - ///////////////////////////////////////////////////////////////////////////////////////////////// namespace cutlass { diff --git a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch_layout.cuh b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch_layout.cuh index f2b0e59822..95926e5c9a 100644 --- a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch_layout.cuh +++ b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch_layout.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #pragma once -#include // std::min -#include // size_t #include // RAFT_EXPECTS #include // pairwise_matrix_params -#include // std::integral_constant + +#include // std::min +#include // size_t +#include // std::integral_constant namespace raft::distance::detail { /** diff --git a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch_sm60.cuh b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch_sm60.cuh index 2080fbe9cd..fc8f594275 100644 --- a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch_sm60.cuh +++ b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch_sm60.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #pragma once -#include // std::min #include // dispatch_layout #include // pairwise_matrix_sm60_wrapper #include // raft::linalg::Policy4x4 +#include // std::min + namespace raft::distance::detail { template // std::min #include // cutlassDistanceKernel #include // dispatch_layout +#include // std::min + namespace raft::distance::detail { template // assert #include // raft::void_op #include // PairwiseDistances #include // pairwise_matrix_params #include // raft::util::arch::SM_compute_arch +#include // assert + namespace raft::distance::detail { template // rbf_fin_op #include // raft::distance::DistanceType #include // RAFT_EXPLICIT -#include // rmm::device_uvector + +#include // rmm::device_uvector #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY diff --git a/cpp/include/raft/distance/distance-inl.cuh b/cpp/include/raft/distance/distance-inl.cuh index d17e5767b9..647c5b2908 100644 --- a/cpp/include/raft/distance/distance-inl.cuh +++ b/cpp/include/raft/distance/distance-inl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,15 @@ */ #pragma once +#include #include #include #include #include + #include -#include -#include +#include namespace raft { namespace distance { diff --git a/cpp/include/raft/distance/fused_l2_nn-ext.cuh b/cpp/include/raft/distance/fused_l2_nn-ext.cuh index 66e9960f1d..ffc6cccab5 100644 --- a/cpp/include/raft/distance/fused_l2_nn-ext.cuh +++ b/cpp/include/raft/distance/fused_l2_nn-ext.cuh @@ -16,11 +16,12 @@ #pragma once -#include // int64_t -#include // raft::KeyValuePair -#include // raft::resources +#include // raft::KeyValuePair +#include // raft::resources #include // include initialize and reduce operations -#include // RAFT_EXPLICIT +#include // RAFT_EXPLICIT + +#include // int64_t #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY diff --git a/cpp/include/raft/distance/fused_l2_nn-inl.cuh b/cpp/include/raft/distance/fused_l2_nn-inl.cuh index 10af89e051..bf9a49d813 100644 --- a/cpp/include/raft/distance/fused_l2_nn-inl.cuh +++ b/cpp/include/raft/distance/fused_l2_nn-inl.cuh @@ -19,14 +19,17 @@ #pragma once -#include -#include #include #include #include #include #include + +#include + #include + +#include #include namespace raft { diff --git a/cpp/include/raft/distance/kernels.cuh b/cpp/include/raft/distance/kernels.cuh index 86f9f82406..39a6ef8c6e 100644 --- a/cpp/include/raft/distance/kernels.cuh +++ b/cpp/include/raft/distance/kernels.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,10 +18,9 @@ #include #include -#include - #include #include +#include namespace raft::distance::kernels { diff --git a/cpp/include/raft/distance/masked_nn.cuh b/cpp/include/raft/distance/masked_nn.cuh index 33a6c0456d..5833a2e681 100644 --- a/cpp/include/raft/distance/masked_nn.cuh +++ b/cpp/include/raft/distance/masked_nn.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,13 +19,15 @@ #pragma once -#include #include #include #include #include + #include +#include + namespace raft { namespace distance { /** diff --git a/cpp/include/raft/label/detail/classlabels.cuh b/cpp/include/raft/label/detail/classlabels.cuh index 6e432e050c..3742b688a0 100644 --- a/cpp/include/raft/label/detail/classlabels.cuh +++ b/cpp/include/raft/label/detail/classlabels.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,16 @@ #pragma once -#include - #include #include #include #include + #include #include +#include + #include namespace raft { diff --git a/cpp/include/raft/label/detail/merge_labels.cuh b/cpp/include/raft/label/detail/merge_labels.cuh index 166bb2122a..5513f16b9f 100644 --- a/cpp/include/raft/label/detail/merge_labels.cuh +++ b/cpp/include/raft/label/detail/merge_labels.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,14 @@ #pragma once -#include -#include - #include #include #include +#include + +#include + namespace raft { namespace label { namespace detail { diff --git a/cpp/include/raft/linalg/add.cuh b/cpp/include/raft/linalg/add.cuh index 26b012956a..1e903a529e 100644 --- a/cpp/include/raft/linalg/add.cuh +++ b/cpp/include/raft/linalg/add.cuh @@ -19,10 +19,10 @@ #pragma once #include "detail/add.cuh" -#include #include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/axpy.cuh b/cpp/include/raft/linalg/axpy.cuh index 2c901b45da..c03b30dc21 100644 --- a/cpp/include/raft/linalg/axpy.cuh +++ b/cpp/include/raft/linalg/axpy.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ #pragma once #include "detail/axpy.cuh" -#include #include #include +#include namespace raft::linalg { diff --git a/cpp/include/raft/linalg/cholesky_r1_update.cuh b/cpp/include/raft/linalg/cholesky_r1_update.cuh index 5c345028f2..292140b4dc 100644 --- a/cpp/include/raft/linalg/cholesky_r1_update.cuh +++ b/cpp/include/raft/linalg/cholesky_r1_update.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #pragma once #include "detail/cholesky_r1_update.cuh" + #include namespace raft { diff --git a/cpp/include/raft/linalg/coalesced_reduction.cuh b/cpp/include/raft/linalg/coalesced_reduction.cuh index afa58d73fc..a4247e618f 100644 --- a/cpp/include/raft/linalg/coalesced_reduction.cuh +++ b/cpp/include/raft/linalg/coalesced_reduction.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ #pragma once #include "detail/coalesced_reduction.cuh" -#include #include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/detail/axpy.cuh b/cpp/include/raft/linalg/detail/axpy.cuh index 8dfeab1118..0a5aaa4cb8 100644 --- a/cpp/include/raft/linalg/detail/axpy.cuh +++ b/cpp/include/raft/linalg/detail/axpy.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,13 @@ #pragma once -#include -#include - #include "cublas_wrappers.hpp" +#include #include +#include + namespace raft::linalg::detail { template diff --git a/cpp/include/raft/linalg/detail/cholesky_r1_update.cuh b/cpp/include/raft/linalg/detail/cholesky_r1_update.cuh index 34d6bf01ee..f348543cf8 100644 --- a/cpp/include/raft/linalg/detail/cholesky_r1_update.cuh +++ b/cpp/include/raft/linalg/detail/cholesky_r1_update.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ #include "cublas_wrappers.hpp" #include "cusolver_wrappers.hpp" + #include #include #include diff --git a/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh b/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh index f3c150cbee..d580ea72c1 100644 --- a/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh +++ b/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,14 @@ #pragma once -#include #include #include #include + #include +#include + namespace raft { namespace linalg { namespace detail { diff --git a/cpp/include/raft/linalg/detail/cublas_wrappers.hpp b/cpp/include/raft/linalg/detail/cublas_wrappers.hpp index d15e343c9a..231a209d0a 100644 --- a/cpp/include/raft/linalg/detail/cublas_wrappers.hpp +++ b/cpp/include/raft/linalg/detail/cublas_wrappers.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,13 @@ #pragma once -#include #include #include -#include #include +#include + namespace raft { namespace linalg { namespace detail { diff --git a/cpp/include/raft/linalg/detail/cublaslt_wrappers.hpp b/cpp/include/raft/linalg/detail/cublaslt_wrappers.hpp index 423f5d29bf..ce0e45dc7a 100644 --- a/cpp/include/raft/linalg/detail/cublaslt_wrappers.hpp +++ b/cpp/include/raft/linalg/detail/cublaslt_wrappers.hpp @@ -24,8 +24,10 @@ #include #include -#include #include + +#include + #include namespace raft::linalg::detail { diff --git a/cpp/include/raft/linalg/detail/cusolver_wrappers.hpp b/cpp/include/raft/linalg/detail/cusolver_wrappers.hpp index 79fd869083..3b056e6960 100644 --- a/cpp/include/raft/linalg/detail/cusolver_wrappers.hpp +++ b/cpp/include/raft/linalg/detail/cusolver_wrappers.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,12 @@ #pragma once -#include -#include #include #include + +#include +#include + #include namespace raft { diff --git a/cpp/include/raft/linalg/detail/eig.cuh b/cpp/include/raft/linalg/detail/eig.cuh index c9f6c3c040..2a4cfd52ec 100644 --- a/cpp/include/raft/linalg/detail/eig.cuh +++ b/cpp/include/raft/linalg/detail/eig.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,14 +17,17 @@ #pragma once #include "cusolver_wrappers.hpp" -#include + #include #include #include #include + #include #include +#include + namespace raft { namespace linalg { namespace detail { diff --git a/cpp/include/raft/linalg/detail/gemv.hpp b/cpp/include/raft/linalg/detail/gemv.hpp index c75bb87515..0c1a5235f1 100644 --- a/cpp/include/raft/linalg/detail/gemv.hpp +++ b/cpp/include/raft/linalg/detail/gemv.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,13 @@ #pragma once -#include -#include - #include "cublas_wrappers.hpp" +#include #include +#include + namespace raft { namespace linalg { namespace detail { diff --git a/cpp/include/raft/linalg/detail/lanczos.cuh b/cpp/include/raft/linalg/detail/lanczos.cuh index 3ab020bfd4..134ef3ef36 100644 --- a/cpp/include/raft/linalg/detail/lanczos.cuh +++ b/cpp/include/raft/linalg/detail/lanczos.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,21 +19,23 @@ // for cmath: #define _USE_MATH_DEFINES -#include +#include "cublas_wrappers.hpp" + #include #include -#include - -#include -#include - -#include "cublas_wrappers.hpp" #include #include #include #include #include +#include + +#include + +#include +#include + namespace raft { namespace linalg { namespace detail { diff --git a/cpp/include/raft/linalg/detail/lstsq.cuh b/cpp/include/raft/linalg/detail/lstsq.cuh index 128757d1d8..c2842fda30 100644 --- a/cpp/include/raft/linalg/detail/lstsq.cuh +++ b/cpp/include/raft/linalg/detail/lstsq.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include @@ -32,6 +33,7 @@ #include #include #include + #include #include #include diff --git a/cpp/include/raft/linalg/detail/map_then_reduce.cuh b/cpp/include/raft/linalg/detail/map_then_reduce.cuh index d1e211f8d2..9ea100108f 100644 --- a/cpp/include/raft/linalg/detail/map_then_reduce.cuh +++ b/cpp/include/raft/linalg/detail/map_then_reduce.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,12 @@ #pragma once -#include #include #include #include +#include + namespace raft { namespace linalg { namespace detail { diff --git a/cpp/include/raft/linalg/detail/qr.cuh b/cpp/include/raft/linalg/detail/qr.cuh index 16a721dfd3..6233786812 100644 --- a/cpp/include/raft/linalg/detail/qr.cuh +++ b/cpp/include/raft/linalg/detail/qr.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,9 +18,11 @@ #include "cublas_wrappers.hpp" #include "cusolver_wrappers.hpp" + #include #include #include + #include #include diff --git a/cpp/include/raft/linalg/detail/reduce_cols_by_key.cuh b/cpp/include/raft/linalg/detail/reduce_cols_by_key.cuh index b726e3ea5a..4d6b82920b 100644 --- a/cpp/include/raft/linalg/detail/reduce_cols_by_key.cuh +++ b/cpp/include/raft/linalg/detail/reduce_cols_by_key.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,14 @@ #pragma once -#include -#include #include + +#include + #include +#include + namespace raft { namespace linalg { namespace detail { diff --git a/cpp/include/raft/linalg/detail/strided_reduction.cuh b/cpp/include/raft/linalg/detail/strided_reduction.cuh index aef346bd4b..617ac6d874 100644 --- a/cpp/include/raft/linalg/detail/strided_reduction.cuh +++ b/cpp/include/raft/linalg/detail/strided_reduction.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,12 @@ #pragma once -#include #include #include #include + +#include + #include namespace raft { diff --git a/cpp/include/raft/linalg/detail/svd.cuh b/cpp/include/raft/linalg/detail/svd.cuh index 5a4851bf6e..79b64c7447 100644 --- a/cpp/include/raft/linalg/detail/svd.cuh +++ b/cpp/include/raft/linalg/detail/svd.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,21 +18,22 @@ #include "cublas_wrappers.hpp" #include "cusolver_wrappers.hpp" + +#include #include #include #include +#include #include #include #include - -#include -#include #include #include #include #include #include #include + #include #include diff --git a/cpp/include/raft/linalg/detail/transpose.cuh b/cpp/include/raft/linalg/detail/transpose.cuh index bbd71a4cf1..999e7f1974 100644 --- a/cpp/include/raft/linalg/detail/transpose.cuh +++ b/cpp/include/raft/linalg/detail/transpose.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,9 @@ #include #include #include + #include + #include #include diff --git a/cpp/include/raft/linalg/divide.cuh b/cpp/include/raft/linalg/divide.cuh index 17ec5c3136..d23c7d60a6 100644 --- a/cpp/include/raft/linalg/divide.cuh +++ b/cpp/include/raft/linalg/divide.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,9 @@ #pragma once #include "detail/divide.cuh" -#include #include +#include #include #include diff --git a/cpp/include/raft/linalg/dot.cuh b/cpp/include/raft/linalg/dot.cuh index 9db9074c35..5a2ee75c4e 100644 --- a/cpp/include/raft/linalg/dot.cuh +++ b/cpp/include/raft/linalg/dot.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,13 +18,12 @@ #pragma once -#include -#include -#include - #include #include +#include +#include #include +#include namespace raft::linalg { diff --git a/cpp/include/raft/linalg/eig.cuh b/cpp/include/raft/linalg/eig.cuh index 57f3b61388..9f03f54f9a 100644 --- a/cpp/include/raft/linalg/eig.cuh +++ b/cpp/include/raft/linalg/eig.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,9 @@ #pragma once #include "detail/eig.cuh" -#include #include +#include namespace raft { namespace linalg { diff --git a/cpp/include/raft/linalg/gemv.cuh b/cpp/include/raft/linalg/gemv.cuh index 610ea07f96..31bad62930 100644 --- a/cpp/include/raft/linalg/gemv.cuh +++ b/cpp/include/raft/linalg/gemv.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,12 @@ #pragma once #include "detail/gemv.hpp" -#include #include #include #include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/map_reduce.cuh b/cpp/include/raft/linalg/map_reduce.cuh index f17caa478b..1886c941b9 100644 --- a/cpp/include/raft/linalg/map_reduce.cuh +++ b/cpp/include/raft/linalg/map_reduce.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,9 @@ #pragma once #include "detail/map_then_reduce.cuh" -#include #include +#include namespace raft::linalg { diff --git a/cpp/include/raft/linalg/matrix_vector_op.cuh b/cpp/include/raft/linalg/matrix_vector_op.cuh index a65f6ed390..1f6dfd785b 100644 --- a/cpp/include/raft/linalg/matrix_vector_op.cuh +++ b/cpp/include/raft/linalg/matrix_vector_op.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,9 +20,9 @@ #include "detail/matrix_vector_op.cuh" #include "linalg_types.hpp" -#include #include +#include #include #include diff --git a/cpp/include/raft/linalg/mean_squared_error.cuh b/cpp/include/raft/linalg/mean_squared_error.cuh index b59a0fcef7..9e753d3a31 100644 --- a/cpp/include/raft/linalg/mean_squared_error.cuh +++ b/cpp/include/raft/linalg/mean_squared_error.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #pragma once #include "detail/mean_squared_error.cuh" + #include #include diff --git a/cpp/include/raft/linalg/multiply.cuh b/cpp/include/raft/linalg/multiply.cuh index 9973a3cc6c..1a7668f8f2 100644 --- a/cpp/include/raft/linalg/multiply.cuh +++ b/cpp/include/raft/linalg/multiply.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ #pragma once #include "detail/multiply.cuh" -#include #include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/norm.cuh b/cpp/include/raft/linalg/norm.cuh index 0d472c5476..97a5d6135d 100644 --- a/cpp/include/raft/linalg/norm.cuh +++ b/cpp/include/raft/linalg/norm.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,11 @@ #include "detail/norm.cuh" #include "linalg_types.hpp" -#include -#include #include +#include #include +#include #include #include diff --git a/cpp/include/raft/linalg/normalize.cuh b/cpp/include/raft/linalg/normalize.cuh index 6169cf1dfa..1f60860c8c 100644 --- a/cpp/include/raft/linalg/normalize.cuh +++ b/cpp/include/raft/linalg/normalize.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,9 @@ #pragma once #include "detail/normalize.cuh" -#include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/qr.cuh b/cpp/include/raft/linalg/qr.cuh index 022c382e67..b032cbfa3a 100644 --- a/cpp/include/raft/linalg/qr.cuh +++ b/cpp/include/raft/linalg/qr.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #pragma once #include "detail/qr.cuh" + #include #include diff --git a/cpp/include/raft/linalg/reduce.cuh b/cpp/include/raft/linalg/reduce.cuh index 3181dd0224..a4523c6926 100644 --- a/cpp/include/raft/linalg/reduce.cuh +++ b/cpp/include/raft/linalg/reduce.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,10 @@ #include "detail/reduce.cuh" #include "linalg_types.hpp" -#include #include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/reduce_cols_by_key.cuh b/cpp/include/raft/linalg/reduce_cols_by_key.cuh index 5ed0fb7407..77ad8a9a80 100644 --- a/cpp/include/raft/linalg/reduce_cols_by_key.cuh +++ b/cpp/include/raft/linalg/reduce_cols_by_key.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,9 @@ #pragma once #include "detail/reduce_cols_by_key.cuh" -#include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/reduce_rows_by_key.cuh b/cpp/include/raft/linalg/reduce_rows_by_key.cuh index 7d93c3946f..2bb14729f4 100644 --- a/cpp/include/raft/linalg/reduce_rows_by_key.cuh +++ b/cpp/include/raft/linalg/reduce_rows_by_key.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,9 @@ #pragma once #include "detail/reduce_rows_by_key.cuh" -#include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/rsvd.cuh b/cpp/include/raft/linalg/rsvd.cuh index 163f360481..a90ba165ed 100644 --- a/cpp/include/raft/linalg/rsvd.cuh +++ b/cpp/include/raft/linalg/rsvd.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #pragma once #include "detail/rsvd.cuh" + #include #include diff --git a/cpp/include/raft/linalg/strided_reduction.cuh b/cpp/include/raft/linalg/strided_reduction.cuh index c7ff000e00..ac97c3cd68 100644 --- a/cpp/include/raft/linalg/strided_reduction.cuh +++ b/cpp/include/raft/linalg/strided_reduction.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,10 @@ #pragma once #include "detail/strided_reduction.cuh" -#include #include #include +#include #include #include diff --git a/cpp/include/raft/linalg/subtract.cuh b/cpp/include/raft/linalg/subtract.cuh index f4243f9dc1..e7b5c6b65a 100644 --- a/cpp/include/raft/linalg/subtract.cuh +++ b/cpp/include/raft/linalg/subtract.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,10 @@ #pragma once #include "detail/subtract.cuh" -#include #include #include +#include #include namespace raft { diff --git a/cpp/include/raft/linalg/svd.cuh b/cpp/include/raft/linalg/svd.cuh index f7071de75b..1c57515a47 100644 --- a/cpp/include/raft/linalg/svd.cuh +++ b/cpp/include/raft/linalg/svd.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #pragma once #include "detail/svd.cuh" + #include #include diff --git a/cpp/include/raft/linalg/transpose.cuh b/cpp/include/raft/linalg/transpose.cuh index 1b46082fbe..6f693d0e13 100644 --- a/cpp/include/raft/linalg/transpose.cuh +++ b/cpp/include/raft/linalg/transpose.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #pragma once #include "detail/transpose.cuh" + #include #include diff --git a/cpp/include/raft/matrix/detail/columnWiseSort.cuh b/cpp/include/raft/matrix/detail/columnWiseSort.cuh index 652c4fda0f..a02621d054 100644 --- a/cpp/include/raft/matrix/detail/columnWiseSort.cuh +++ b/cpp/include/raft/matrix/detail/columnWiseSort.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,13 @@ #pragma once -#include +#include + #include + +#include #include #include -#include #define INST_BLOCK_SORT(keyIn, keyOut, valueInOut, rows, columns, blockSize, elemPT, stream) \ devKeyValSortColumnPerRow<<>>( \ diff --git a/cpp/include/raft/matrix/detail/gather.cuh b/cpp/include/raft/matrix/detail/gather.cuh index da5fd8c103..651fec81c3 100644 --- a/cpp/include/raft/matrix/detail/gather.cuh +++ b/cpp/include/raft/matrix/detail/gather.cuh @@ -16,10 +16,11 @@ #pragma once -#include #include #include +#include + namespace raft { namespace matrix { namespace detail { diff --git a/cpp/include/raft/matrix/detail/gather_inplace.cuh b/cpp/include/raft/matrix/detail/gather_inplace.cuh index cc510e068b..a37ba550f9 100644 --- a/cpp/include/raft/matrix/detail/gather_inplace.cuh +++ b/cpp/include/raft/matrix/detail/gather_inplace.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #include #include #include + #include namespace raft { diff --git a/cpp/include/raft/matrix/detail/linewise_op.cuh b/cpp/include/raft/matrix/detail/linewise_op.cuh index 6061fe6aee..4a2e45b161 100644 --- a/cpp/include/raft/matrix/detail/linewise_op.cuh +++ b/cpp/include/raft/matrix/detail/linewise_op.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,9 +22,10 @@ #include #include -#include #include +#include + namespace raft { namespace matrix { namespace detail { diff --git a/cpp/include/raft/matrix/detail/math.cuh b/cpp/include/raft/matrix/detail/math.cuh index 9e9d7f8b3b..e1b6b2ef71 100644 --- a/cpp/include/raft/matrix/detail/math.cuh +++ b/cpp/include/raft/matrix/detail/math.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,19 @@ #pragma once -#include - -#include #include +#include #include #include #include #include #include + #include #include +#include + namespace raft { namespace matrix { namespace detail { diff --git a/cpp/include/raft/matrix/detail/matrix.cuh b/cpp/include/raft/matrix/detail/matrix.cuh index 2fa741fd96..6ed9ffa645 100644 --- a/cpp/include/raft/matrix/detail/matrix.cuh +++ b/cpp/include/raft/matrix/detail/matrix.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,21 +17,22 @@ #pragma once #include +#include +#include #include #include +#include #include +#include #include #include +#include + #include #include -#include -#include -#include -#include -#include namespace raft { namespace matrix { diff --git a/cpp/include/raft/matrix/detail/print.hpp b/cpp/include/raft/matrix/detail/print.hpp index 0b93819b97..92881d5b12 100644 --- a/cpp/include/raft/matrix/detail/print.hpp +++ b/cpp/include/raft/matrix/detail/print.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,21 +16,22 @@ #pragma once +#include +#include #include #include +#include #include +#include #include #include +#include + #include #include -#include -#include -#include -#include -#include namespace raft::matrix::detail { diff --git a/cpp/include/raft/matrix/detail/scatter_inplace.cuh b/cpp/include/raft/matrix/detail/scatter_inplace.cuh index 3a57c5478b..6a60e5006b 100644 --- a/cpp/include/raft/matrix/detail/scatter_inplace.cuh +++ b/cpp/include/raft/matrix/detail/scatter_inplace.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,16 @@ */ #pragma once -#include #include #include #include #include #include + #include +#include + namespace raft { namespace matrix { namespace detail { diff --git a/cpp/include/raft/matrix/detail/select_k-ext.cuh b/cpp/include/raft/matrix/detail/select_k-ext.cuh index e8db6827b5..6a7847d8a0 100644 --- a/cpp/include/raft/matrix/detail/select_k-ext.cuh +++ b/cpp/include/raft/matrix/detail/select_k-ext.cuh @@ -16,14 +16,17 @@ #pragma once -#include // uint32_t -#include // __half #include #include -#include // RAFT_EXPLICIT +#include // RAFT_EXPLICIT + #include // rmm:cuda_stream_view #include // rmm::mr::device_memory_resource +#include // __half + +#include // uint32_t + #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY namespace raft::matrix::detail { diff --git a/cpp/include/raft/matrix/detail/select_radix.cuh b/cpp/include/raft/matrix/detail/select_radix.cuh index 16b9ac0c6d..82983b7cd2 100644 --- a/cpp/include/raft/matrix/detail/select_radix.cuh +++ b/cpp/include/raft/matrix/detail/select_radix.cuh @@ -29,15 +29,15 @@ #include #include +#include +#include +#include + #include #include #include #include -#include -#include -#include - namespace raft::matrix::detail::select::radix { namespace impl { diff --git a/cpp/include/raft/matrix/detail/select_warpsort.cuh b/cpp/include/raft/matrix/detail/select_warpsort.cuh index 7cd43b030b..572558153d 100644 --- a/cpp/include/raft/matrix/detail/select_warpsort.cuh +++ b/cpp/include/raft/matrix/detail/select_warpsort.cuh @@ -27,13 +27,13 @@ #include #include +#include +#include + #include #include #include -#include -#include - /* Three APIs of different scopes are provided: 1. host function: select_k() diff --git a/cpp/include/raft/matrix/matrix.cuh b/cpp/include/raft/matrix/matrix.cuh index 63c33ff034..29cc6fb75b 100644 --- a/cpp/include/raft/matrix/matrix.cuh +++ b/cpp/include/raft/matrix/matrix.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,10 +30,10 @@ #include "detail/linewise_op.cuh" #include "detail/matrix.cuh" -#include -#include #include +#include +#include namespace raft { namespace matrix { diff --git a/cpp/include/raft/matrix/select_k.cuh b/cpp/include/raft/matrix/select_k.cuh index e2d94c67ae..2efa146495 100644 --- a/cpp/include/raft/matrix/select_k.cuh +++ b/cpp/include/raft/matrix/select_k.cuh @@ -17,10 +17,10 @@ #pragma once #include "detail/select_k.cuh" -#include #include #include +#include #include #include diff --git a/cpp/include/raft/neighbors/ball_cover-ext.cuh b/cpp/include/raft/neighbors/ball_cover-ext.cuh index 3d0b3c7858..2a83a57749 100644 --- a/cpp/include/raft/neighbors/ball_cover-ext.cuh +++ b/cpp/include/raft/neighbors/ball_cover-ext.cuh @@ -15,11 +15,12 @@ */ #pragma once -#include // uint32_t #include // raft::distance::DistanceType #include // BallCoverIndex #include // RAFT_EXPLICIT +#include // uint32_t + #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY namespace raft::neighbors::ball_cover { diff --git a/cpp/include/raft/neighbors/ball_cover-inl.cuh b/cpp/include/raft/neighbors/ball_cover-inl.cuh index cdf7c30e89..4af2a6772c 100644 --- a/cpp/include/raft/neighbors/ball_cover-inl.cuh +++ b/cpp/include/raft/neighbors/ball_cover-inl.cuh @@ -18,14 +18,15 @@ #pragma once -#include - #include #include #include #include + #include +#include + namespace raft::neighbors::ball_cover { /** diff --git a/cpp/include/raft/neighbors/ball_cover_types.hpp b/cpp/include/raft/neighbors/ball_cover_types.hpp index dc96f0d45b..a627a1d234 100644 --- a/cpp/include/raft/neighbors/ball_cover_types.hpp +++ b/cpp/include/raft/neighbors/ball_cover_types.hpp @@ -16,14 +16,16 @@ #pragma once -#include #include #include #include #include #include + #include +#include + namespace raft::neighbors::ball_cover { /** diff --git a/cpp/include/raft/neighbors/brute_force-ext.cuh b/cpp/include/raft/neighbors/brute_force-ext.cuh index e8c25f355b..4055c253c8 100644 --- a/cpp/include/raft/neighbors/brute_force-ext.cuh +++ b/cpp/include/raft/neighbors/brute_force-ext.cuh @@ -16,8 +16,6 @@ #pragma once -#include - #include // raft::device_matrix_view #include // raft::identity_op #include // raft::resources @@ -25,6 +23,8 @@ #include #include // RAFT_EXPLICIT +#include + #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY namespace raft::neighbors::brute_force { diff --git a/cpp/include/raft/neighbors/brute_force_types.hpp b/cpp/include/raft/neighbors/brute_force_types.hpp index 176b32a866..a8f073edc6 100644 --- a/cpp/include/raft/neighbors/brute_force_types.hpp +++ b/cpp/include/raft/neighbors/brute_force_types.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,19 +17,18 @@ #pragma once #include "ann_types.hpp" -#include #include #include #include #include +#include #include +#include #include #include #include -#include - namespace raft::neighbors::brute_force { /** * @addtogroup brute_force_knn diff --git a/cpp/include/raft/neighbors/cagra.cuh b/cpp/include/raft/neighbors/cagra.cuh index 384ed05e1f..b8258297e6 100644 --- a/cpp/include/raft/neighbors/cagra.cuh +++ b/cpp/include/raft/neighbors/cagra.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ #include #include #include + #include namespace raft::neighbors::cagra { diff --git a/cpp/include/raft/neighbors/cagra_types.hpp b/cpp/include/raft/neighbors/cagra_types.hpp index cc76d70fbf..0f574ae5bb 100644 --- a/cpp/include/raft/neighbors/cagra_types.hpp +++ b/cpp/include/raft/neighbors/cagra_types.hpp @@ -17,24 +17,24 @@ #pragma once #include "ann_types.hpp" -#include #include #include #include +#include #include +#include #include #include #include #include +#include + #include #include #include -#include #include - -#include namespace raft::neighbors::cagra { /** * @addtogroup cagra diff --git a/cpp/include/raft/neighbors/detail/cagra/bitonic.hpp b/cpp/include/raft/neighbors/detail/cagra/bitonic.hpp index 9fca7f8ebd..1d4c77af6f 100644 --- a/cpp/include/raft/neighbors/detail/cagra/bitonic.hpp +++ b/cpp/include/raft/neighbors/detail/cagra/bitonic.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,10 @@ */ #pragma once -#include #include +#include + namespace raft::neighbors::cagra::detail { namespace bitonic { diff --git a/cpp/include/raft/neighbors/detail/cagra/cagra_build.cuh b/cpp/include/raft/neighbors/detail/cagra/cagra_build.cuh old mode 100755 new mode 100644 index e7f7d96a13..08cc2beaeb --- a/cpp/include/raft/neighbors/detail/cagra/cagra_build.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/cagra_build.cuh @@ -17,10 +17,6 @@ #include "../../cagra_types.hpp" #include "graph_core.cuh" -#include -#include -#include -#include #include #include @@ -28,14 +24,18 @@ #include #include #include +#include #include -#include - #include #include #include #include #include +#include + +#include +#include +#include namespace raft::neighbors::cagra::detail { diff --git a/cpp/include/raft/neighbors/detail/cagra/cagra_search.cuh b/cpp/include/raft/neighbors/detail/cagra/cagra_search.cuh index 40cc7c76fb..0832e75633 100644 --- a/cpp/include/raft/neighbors/detail/cagra/cagra_search.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/cagra_search.cuh @@ -16,21 +16,22 @@ #pragma once -#include -#include -#include -#include +#include "factory.cuh" +#include "search_plan.cuh" +#include "search_single_cta.cuh" #include #include #include +#include #include #include -#include +#include +#include +#include +#include -#include "factory.cuh" -#include "search_plan.cuh" -#include "search_single_cta.cuh" +#include namespace raft::neighbors::cagra::detail { @@ -181,13 +182,14 @@ void search_main(raft::resources const& res, // and divide the values by kDivisor. Here we restore the original scale. constexpr float kScale = spatial::knn::detail::utils::config::kDivisor / spatial::knn::detail::utils::config::kDivisor; - ivf_pq::detail::postprocess_distances(dist_out, - dist_in, - index.metric(), - distances.extent(0), - distances.extent(1), - kScale, - resource::get_cuda_stream(res)); + ivf::detail::postprocess_distances(dist_out, + dist_in, + index.metric(), + distances.extent(0), + distances.extent(1), + kScale, + true, + resource::get_cuda_stream(res)); } /** @} */ // end group cagra diff --git a/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh b/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh index 84a9ce96cd..d7bd27222b 100644 --- a/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh @@ -16,8 +16,6 @@ #pragma once -#include -#include #include #include #include @@ -26,6 +24,8 @@ #include #include +#include +#include #include #include diff --git a/cpp/include/raft/neighbors/detail/cagra/compute_distance.hpp b/cpp/include/raft/neighbors/detail/cagra/compute_distance.hpp index 7f2e8b34cb..3732dcf3fe 100644 --- a/cpp/include/raft/neighbors/detail/cagra/compute_distance.hpp +++ b/cpp/include/raft/neighbors/detail/cagra/compute_distance.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #pragma once -#include - #include "device_common.hpp" #include "hashmap.hpp" #include "utils.hpp" + +#include + #include namespace raft::neighbors::cagra::detail { diff --git a/cpp/include/raft/neighbors/detail/cagra/device_common.hpp b/cpp/include/raft/neighbors/detail/cagra/device_common.hpp index b1a2207a4e..cd7469b55e 100644 --- a/cpp/include/raft/neighbors/detail/cagra/device_common.hpp +++ b/cpp/include/raft/neighbors/detail/cagra/device_common.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,13 @@ #pragma once #include "utils.hpp" + +#include + +#include + #include #include -#include -#include namespace raft::neighbors::cagra::detail { namespace device { diff --git a/cpp/include/raft/neighbors/detail/cagra/factory.cuh b/cpp/include/raft/neighbors/detail/cagra/factory.cuh index 0002dd8b2a..0aee912e25 100644 --- a/cpp/include/raft/neighbors/detail/cagra/factory.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/factory.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include "search_multi_kernel.cuh" #include "search_plan.cuh" #include "search_single_cta.cuh" + #include namespace raft::neighbors::cagra::detail { diff --git a/cpp/include/raft/neighbors/detail/cagra/graph_core.cuh b/cpp/include/raft/neighbors/detail/cagra/graph_core.cuh index 192850cf2d..e4e3ea3512 100644 --- a/cpp/include/raft/neighbors/detail/cagra/graph_core.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/graph_core.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,26 +15,28 @@ */ #pragma once -#include -#include -#include -#include -#include -#include -#include +#include "utils.hpp" + #include #include #include #include #include #include -#include -#include - #include #include -#include "utils.hpp" +#include + +#include +#include +#include + +#include +#include +#include +#include +#include namespace raft::neighbors::cagra::detail { namespace graph { diff --git a/cpp/include/raft/neighbors/detail/cagra/hashmap.hpp b/cpp/include/raft/neighbors/detail/cagra/hashmap.hpp index ed4763e475..034bca6178 100644 --- a/cpp/include/raft/neighbors/detail/cagra/hashmap.hpp +++ b/cpp/include/raft/neighbors/detail/cagra/hashmap.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,12 @@ #pragma once #include "utils.hpp" -#include + #include #include +#include + // #pragma GCC diagnostic push // #pragma GCC diagnostic ignored // #pragma GCC diagnostic pop diff --git a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh index 010b0a6f80..1fcd159959 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh @@ -15,20 +15,6 @@ */ #pragma once -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - #include "bitonic.hpp" #include "compute_distance.hpp" #include "device_common.hpp" @@ -37,10 +23,23 @@ #include "search_plan.cuh" #include "topk_for_cagra/topk_core.cuh" // TODO replace with raft topk if possible #include "utils.hpp" + +#include #include +#include +#include +#include +#include #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp +#include +#include +#include +#include +#include +#include + namespace raft::neighbors::cagra::detail { namespace multi_cta_search { diff --git a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-inl.cuh b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-inl.cuh index 8b394befd7..30f56780d6 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-inl.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-inl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,21 +15,6 @@ */ #pragma once -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - #include "bitonic.hpp" #include "compute_distance.hpp" #include "device_common.hpp" @@ -37,10 +22,24 @@ #include "search_plan.cuh" #include "topk_for_cagra/topk_core.cuh" // TODO replace with raft topk if possible #include "utils.hpp" + +#include #include +#include +#include +#include +#include +#include #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp +#include +#include +#include +#include +#include +#include + namespace raft::neighbors::cagra::detail { namespace multi_cta_search { diff --git a/cpp/include/raft/neighbors/detail/cagra/search_multi_kernel.cuh b/cpp/include/raft/neighbors/detail/cagra/search_multi_kernel.cuh index f9bf525503..e4a30675bb 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_multi_kernel.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_multi_kernel.cuh @@ -15,32 +15,33 @@ */ #pragma once -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "compute_distance.hpp" #include "device_common.hpp" #include "hashmap.hpp" #include "search_plan.cuh" #include "topk_for_cagra/topk_core.cuh" //todo replace with raft kernel #include "utils.hpp" + +#include #include +#include +#include #include +#include +#include #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp +#include +#include + +#include +#include +#include +#include +#include +#include + namespace raft::neighbors::cagra::detail { namespace multi_kernel_search { diff --git a/cpp/include/raft/neighbors/detail/cagra/search_plan.cuh b/cpp/include/raft/neighbors/detail/cagra/search_plan.cuh index 271a1f4955..11ef7e5211 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_plan.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_plan.cuh @@ -17,7 +17,9 @@ #pragma once #include "hashmap.hpp" + #include +#include // #include "search_single_cta.cuh" // #include "topk_for_cagra/topk_core.cuh" diff --git a/cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh b/cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh index 0b4fc2d47b..f1e74ee7a5 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,20 +15,6 @@ */ #pragma once -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "bitonic.hpp" #include "compute_distance.hpp" #include "device_common.hpp" @@ -38,10 +24,25 @@ #include "topk_by_radix.cuh" #include "topk_for_cagra/topk_core.cuh" // TODO replace with raft topk #include "utils.hpp" + +#include #include +#include +#include +#include +#include #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp +#include + +#include +#include +#include +#include +#include +#include + namespace raft::neighbors::cagra::detail { namespace single_cta_search { diff --git a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-inl.cuh b/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-inl.cuh index 80b5b343b2..652115928b 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-inl.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-inl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,21 +15,6 @@ */ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "bitonic.hpp" #include "compute_distance.hpp" #include "device_common.hpp" @@ -38,10 +23,27 @@ #include "topk_by_radix.cuh" #include "topk_for_cagra/topk_core.cuh" // TODO replace with raft topk #include "utils.hpp" + +#include #include +#include +#include +#include +#include +#include #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp +#include + +#include +#include +#include +#include +#include +#include +#include + namespace raft::neighbors::cagra::detail { namespace single_cta_search { diff --git a/cpp/include/raft/neighbors/detail/cagra/topk_for_cagra/topk.h b/cpp/include/raft/neighbors/detail/cagra/topk_for_cagra/topk.h index 92b9474047..ac1a746090 100644 --- a/cpp/include/raft/neighbors/detail/cagra/topk_for_cagra/topk.h +++ b/cpp/include/raft/neighbors/detail/cagra/topk_for_cagra/topk.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #pragma once #include + #include namespace raft::neighbors::cagra::detail { diff --git a/cpp/include/raft/neighbors/detail/cagra/topk_for_cagra/topk_core.cuh b/cpp/include/raft/neighbors/detail/cagra/topk_for_cagra/topk_core.cuh index 8e04a2eb0d..aedeb1be67 100644 --- a/cpp/include/raft/neighbors/detail/cagra/topk_for_cagra/topk_core.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/topk_for_cagra/topk_core.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,10 @@ */ #pragma once #include "topk.h" -#include + #include + +#include #include #include #include diff --git a/cpp/include/raft/neighbors/detail/cagra/utils.hpp b/cpp/include/raft/neighbors/detail/cagra/utils.hpp index 5e57a9589f..7e403abe91 100644 --- a/cpp/include/raft/neighbors/detail/cagra/utils.hpp +++ b/cpp/include/raft/neighbors/detail/cagra/utils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,15 +15,18 @@ */ #pragma once -#include -#include -#include -#include #include #include #include #include + #include + +#include +#include + +#include +#include #include namespace raft::neighbors::cagra::detail { diff --git a/cpp/include/raft/neighbors/detail/faiss_select/MergeNetworkBlock.cuh b/cpp/include/raft/neighbors/detail/faiss_select/MergeNetworkBlock.cuh index 1f4308fa2f..0677b0596c 100644 --- a/cpp/include/raft/neighbors/detail/faiss_select/MergeNetworkBlock.cuh +++ b/cpp/include/raft/neighbors/detail/faiss_select/MergeNetworkBlock.cuh @@ -7,10 +7,11 @@ #pragma once -#include #include #include +#include + namespace raft::neighbors::detail::faiss_select { // Merge pairs of lists smaller than blockDim.x (NumThreads) diff --git a/cpp/include/raft/neighbors/detail/faiss_select/MergeNetworkWarp.cuh b/cpp/include/raft/neighbors/detail/faiss_select/MergeNetworkWarp.cuh index 3d87e284ed..64b1bada48 100644 --- a/cpp/include/raft/neighbors/detail/faiss_select/MergeNetworkWarp.cuh +++ b/cpp/include/raft/neighbors/detail/faiss_select/MergeNetworkWarp.cuh @@ -9,7 +9,6 @@ #include #include - #include namespace raft::neighbors::detail::faiss_select { diff --git a/cpp/include/raft/neighbors/detail/faiss_select/Select.cuh b/cpp/include/raft/neighbors/detail/faiss_select/Select.cuh index fdb986ac69..445e638eaa 100644 --- a/cpp/include/raft/neighbors/detail/faiss_select/Select.cuh +++ b/cpp/include/raft/neighbors/detail/faiss_select/Select.cuh @@ -7,11 +7,10 @@ #pragma once +#include #include #include #include - -#include #include namespace raft::neighbors::detail::faiss_select { diff --git a/cpp/include/raft/neighbors/detail/hnsw.hpp b/cpp/include/raft/neighbors/detail/hnsw.hpp index 69478205a9..5deee3c2ba 100644 --- a/cpp/include/raft/neighbors/detail/hnsw.hpp +++ b/cpp/include/raft/neighbors/detail/hnsw.hpp @@ -18,13 +18,13 @@ #include "hnsw_types.hpp" -#include #include #include +#include #include -#include +#include namespace raft::neighbors::hnsw::detail { diff --git a/cpp/include/raft/neighbors/detail/hnsw_serialize.hpp b/cpp/include/raft/neighbors/detail/hnsw_serialize.hpp index 8103ffc5ab..4c3728f8fc 100644 --- a/cpp/include/raft/neighbors/detail/hnsw_serialize.hpp +++ b/cpp/include/raft/neighbors/detail/hnsw_serialize.hpp @@ -19,8 +19,6 @@ #include "../hnsw_types.hpp" #include "hnsw_types.hpp" -#include -#include #include #include #include @@ -29,6 +27,8 @@ #include #include +#include +#include #include #include diff --git a/cpp/include/raft/neighbors/detail/hnsw_types.hpp b/cpp/include/raft/neighbors/detail/hnsw_types.hpp index 94ade95965..9d35effd1a 100644 --- a/cpp/include/raft/neighbors/detail/hnsw_types.hpp +++ b/cpp/include/raft/neighbors/detail/hnsw_types.hpp @@ -17,13 +17,15 @@ #pragma once #include "../hnsw_types.hpp" -#include + #include #include -#include #include #include + +#include +#include #include namespace raft::neighbors::hnsw::detail { diff --git a/cpp/include/raft/neighbors/detail/ivf_common.cuh b/cpp/include/raft/neighbors/detail/ivf_common.cuh new file mode 100644 index 0000000000..ef7ae7c804 --- /dev/null +++ b/cpp/include/raft/neighbors/detail/ivf_common.cuh @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include // matrix::detail::select::warpsort::warp_sort_distributed + +#include + +namespace raft::neighbors::ivf::detail { + +/** + * Default value returned by `search` when the `n_probes` is too small and top-k is too large. + * One may encounter it if the combined size of probed clusters is smaller than the requested + * number of results per query. + */ +template +constexpr static IdxT kOutOfBoundsRecord = std::numeric_limits::max(); + +template +struct dummy_block_sort_t { + using queue_t = + matrix::detail::select::warpsort::warp_sort_distributed; + template + __device__ dummy_block_sort_t(int k, Args...){}; +}; + +/** + * For each query, we calculate a cumulative sum of the cluster sizes that we probe, and return that + * in chunk_indices. Essentially this is a segmented inclusive scan of the cluster sizes. The total + * number of samples per query (sum of the cluster sizes that we probe) is returned in n_samples. + */ +template +__launch_bounds__(BlockDim) RAFT_KERNEL + calc_chunk_indices_kernel(uint32_t n_probes, + const uint32_t* cluster_sizes, // [n_clusters] + const uint32_t* clusters_to_probe, // [n_queries, n_probes] + uint32_t* chunk_indices, // [n_queries, n_probes] + uint32_t* n_samples // [n_queries] + ) +{ + using block_scan = cub::BlockScan; + __shared__ typename block_scan::TempStorage shm; + + // locate the query data + clusters_to_probe += n_probes * blockIdx.x; + chunk_indices += n_probes * blockIdx.x; + + // block scan + const uint32_t n_probes_aligned = Pow2::roundUp(n_probes); + uint32_t total = 0; + for (uint32_t probe_ix = threadIdx.x; probe_ix < n_probes_aligned; probe_ix += BlockDim) { + auto label = probe_ix < n_probes ? clusters_to_probe[probe_ix] : 0u; + auto chunk = probe_ix < n_probes ? cluster_sizes[label] : 0u; + if (threadIdx.x == 0) { chunk += total; } + block_scan(shm).InclusiveSum(chunk, chunk, total); + __syncthreads(); + if (probe_ix < n_probes) { chunk_indices[probe_ix] = chunk; } + } + // save the total size + if (threadIdx.x == 0) { n_samples[blockIdx.x] = total; } +} + +struct calc_chunk_indices { + public: + struct configured { + void* kernel; + dim3 block_dim; + dim3 grid_dim; + uint32_t n_probes; + + inline void operator()(const uint32_t* cluster_sizes, + const uint32_t* clusters_to_probe, + uint32_t* chunk_indices, + uint32_t* n_samples, + rmm::cuda_stream_view stream) + { + void* args[] = // NOLINT + {&n_probes, &cluster_sizes, &clusters_to_probe, &chunk_indices, &n_samples}; + RAFT_CUDA_TRY(cudaLaunchKernel(kernel, grid_dim, block_dim, args, 0, stream)); + } + }; + + static inline auto configure(uint32_t n_probes, uint32_t n_queries) -> configured + { + return try_block_dim<1024>(n_probes, n_queries); + } + + private: + template + static auto try_block_dim(uint32_t n_probes, uint32_t n_queries) -> configured + { + if constexpr (BlockDim >= WarpSize * 2) { + if (BlockDim >= n_probes * 2) { return try_block_dim<(BlockDim / 2)>(n_probes, n_queries); } + } + return {reinterpret_cast(calc_chunk_indices_kernel), + dim3(BlockDim, 1, 1), + dim3(n_queries, 1, 1), + n_probes}; + } +}; + +/** + * Look up the chunk id corresponding to the sample index. + * + * Each query vector was compared to all the vectors from n_probes clusters, and sample_ix is an + * ordered number of one of such vectors. This function looks up to which chunk it belongs, + * and returns the index within the chunk (which is also an index within a cluster). + * + * @param[inout] sample_ix + * input: the offset of the sample in the batch; + * output: the offset inside the chunk (probe) / selected cluster. + * @param[in] n_probes number of probes + * @param[in] chunk_indices offsets of the chunks within the batch [n_probes] + * @return chunk index (== n_probes when the input index is not in the valid range, + * which can happen if there is not enough data to output in the selected clusters). + */ +__device__ inline auto find_chunk_ix(uint32_t& sample_ix, // NOLINT + uint32_t n_probes, + const uint32_t* chunk_indices) -> uint32_t +{ + uint32_t ix_min = 0; + uint32_t ix_max = n_probes; + do { + uint32_t i = (ix_min + ix_max) / 2; + if (chunk_indices[i] <= sample_ix) { + ix_min = i + 1; + } else { + ix_max = i; + } + } while (ix_min < ix_max); + if (ix_min > 0) { sample_ix -= chunk_indices[ix_min - 1]; } + return ix_min; +} + +template +__launch_bounds__(BlockDim) RAFT_KERNEL + postprocess_neighbors_kernel(IdxT1* neighbors_out, // [n_queries, topk] + const IdxT2* neighbors_in, // [n_queries, topk] + const IdxT1* const* db_indices, // [n_clusters][..] + const uint32_t* clusters_to_probe, // [n_queries, n_probes] + const uint32_t* chunk_indices, // [n_queries, n_probes] + uint32_t n_queries, + uint32_t n_probes, + uint32_t topk) +{ + const uint64_t i = threadIdx.x + BlockDim * uint64_t(blockIdx.x); + const uint32_t query_ix = i / uint64_t(topk); + if (query_ix >= n_queries) { return; } + const uint32_t k = i % uint64_t(topk); + neighbors_in += query_ix * topk; + neighbors_out += query_ix * topk; + chunk_indices += query_ix * n_probes; + clusters_to_probe += query_ix * n_probes; + uint32_t data_ix = neighbors_in[k]; + const uint32_t chunk_ix = find_chunk_ix(data_ix, n_probes, chunk_indices); + const bool valid = chunk_ix < n_probes; + neighbors_out[k] = + valid ? db_indices[clusters_to_probe[chunk_ix]][data_ix] : kOutOfBoundsRecord; +} + +/** + * Transform found sample indices into the corresponding database indices + * (as stored in index.indices()). + * The sample indices are the record indices as they appear in the database view formed by the + * probed clusters / defined by the `chunk_indices`. + * We assume the searched sample sizes (for a single query) fit into `uint32_t`. + */ +template +void postprocess_neighbors(IdxT1* neighbors_out, // [n_queries, topk] + const IdxT2* neighbors_in, // [n_queries, topk] + const IdxT1* const* db_indices, // [n_clusters][..] + const uint32_t* clusters_to_probe, // [n_queries, n_probes] + const uint32_t* chunk_indices, // [n_queries, n_probes] + uint32_t n_queries, + uint32_t n_probes, + uint32_t topk, + rmm::cuda_stream_view stream) +{ + constexpr int kPNThreads = 256; + const int pn_blocks = raft::div_rounding_up_unsafe(n_queries * topk, kPNThreads); + postprocess_neighbors_kernel + <<>>(neighbors_out, + neighbors_in, + db_indices, + clusters_to_probe, + chunk_indices, + n_queries, + n_probes, + topk); +} + +/** + * Post-process the scores depending on the metric type; + * translate the element type if necessary. + */ +template +void postprocess_distances(ScoreOutT* out, // [n_queries, topk] + const ScoreInT* in, // [n_queries, topk] + distance::DistanceType metric, + uint32_t n_queries, + uint32_t topk, + float scaling_factor, + bool account_for_max_close, + rmm::cuda_stream_view stream) +{ + constexpr bool needs_cast = !std::is_same::value; + const bool needs_copy = ((void*)in) != ((void*)out); + size_t len = size_t(n_queries) * size_t(topk); + switch (metric) { + case distance::DistanceType::L2Unexpanded: + case distance::DistanceType::L2Expanded: { + if (scaling_factor != 1.0) { + linalg::unaryOp( + out, + in, + len, + raft::compose_op(raft::mul_const_op{scaling_factor * scaling_factor}, + raft::cast_op{}), + stream); + } else if (needs_cast || needs_copy) { + linalg::unaryOp(out, in, len, raft::cast_op{}, stream); + } + } break; + case distance::DistanceType::L2SqrtUnexpanded: + case distance::DistanceType::L2SqrtExpanded: { + if (scaling_factor != 1.0) { + linalg::unaryOp(out, + in, + len, + raft::compose_op{raft::mul_const_op{scaling_factor}, + raft::sqrt_op{}, + raft::cast_op{}}, + stream); + } else if (needs_cast) { + linalg::unaryOp( + out, in, len, raft::compose_op{raft::sqrt_op{}, raft::cast_op{}}, stream); + } else { + linalg::unaryOp(out, in, len, raft::sqrt_op{}, stream); + } + } break; + case distance::DistanceType::InnerProduct: { + float factor = (account_for_max_close ? -1.0 : 1.0) * scaling_factor * scaling_factor; + if (factor != 1.0) { + linalg::unaryOp( + out, + in, + len, + raft::compose_op(raft::mul_const_op{factor}, raft::cast_op{}), + stream); + } else if (needs_cast || needs_copy) { + linalg::unaryOp(out, in, len, raft::cast_op{}, stream); + } + } break; + default: RAFT_FAIL("Unexpected metric."); + } +} + +/** Update the state of the dependent index members. */ +template +void recompute_internal_state(const raft::resources& res, Index& index) +{ + auto stream = resource::get_cuda_stream(res); + auto tmp_res = resource::get_workspace_resource(res); + rmm::device_uvector sorted_sizes(index.n_lists(), stream, tmp_res); + + // Actualize the list pointers + auto data_ptrs = index.data_ptrs(); + auto inds_ptrs = index.inds_ptrs(); + for (uint32_t label = 0; label < index.n_lists(); label++) { + auto& list = index.lists()[label]; + const auto data_ptr = list ? list->data.data_handle() : nullptr; + const auto inds_ptr = list ? list->indices.data_handle() : nullptr; + copy(&data_ptrs(label), &data_ptr, 1, stream); + copy(&inds_ptrs(label), &inds_ptr, 1, stream); + } + + // Sort the cluster sizes in the descending order. + int begin_bit = 0; + int end_bit = sizeof(uint32_t) * 8; + size_t cub_workspace_size = 0; + cub::DeviceRadixSort::SortKeysDescending(nullptr, + cub_workspace_size, + index.list_sizes().data_handle(), + sorted_sizes.data(), + index.n_lists(), + begin_bit, + end_bit, + stream); + rmm::device_buffer cub_workspace(cub_workspace_size, stream, tmp_res); + cub::DeviceRadixSort::SortKeysDescending(cub_workspace.data(), + cub_workspace_size, + index.list_sizes().data_handle(), + sorted_sizes.data(), + index.n_lists(), + begin_bit, + end_bit, + stream); + // copy the results to CPU + std::vector sorted_sizes_host(index.n_lists()); + copy(sorted_sizes_host.data(), sorted_sizes.data(), index.n_lists(), stream); + resource::sync_stream(res); + + // accumulate the sorted cluster sizes + auto accum_sorted_sizes = index.accum_sorted_sizes(); + accum_sorted_sizes(0) = 0; + for (uint32_t label = 0; label < sorted_sizes_host.size(); label++) { + accum_sorted_sizes(label + 1) = accum_sorted_sizes(label) + sorted_sizes_host[label]; + } +} + +} // namespace raft::neighbors::ivf::detail diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_build.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_build.cuh index fa11d9236f..55184cc615 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_build.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_build.cuh @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -75,7 +76,7 @@ auto clone(const raft::resources& res, const index& source) -> indexrecompute_internal_state(handle); + ivf::detail::recompute_internal_state(handle, *index); // Copy the old sizes, so we can start from the current state of the index; // we'll rebuild the `list_sizes_ptr` in the following kernel, using it as an atomic counter. raft::copy(list_sizes_ptr, old_list_sizes_dev.data_handle(), n_lists, stream); @@ -355,6 +356,8 @@ inline auto build(raft::resources const& handle, RAFT_EXPECTS(n_rows >= params.n_lists, "number of rows can't be less than n_lists"); index index(handle, params, dim); + utils::memzero( + index.accum_sorted_sizes().data_handle(), index.accum_sorted_sizes().size(), stream); utils::memzero(index.list_sizes().data_handle(), index.list_sizes().size(), stream); utils::memzero(index.data_ptrs().data_handle(), index.data_ptrs().size(), stream); utils::memzero(index.inds_ptrs().data_handle(), index.inds_ptrs().size(), stream); @@ -442,7 +445,7 @@ inline void fill_refinement_index(raft::resources const& handle, ivf::resize_list(handle, lists[label], list_device_spec, n_candidates, uint32_t(0)); } // Update the pointers and the sizes - refinement_index->recompute_internal_state(handle); + ivf::detail::recompute_internal_state(handle, *refinement_index); RAFT_CUDA_TRY(cudaMemsetAsync(list_sizes_ptr, 0, n_lists * sizeof(uint32_t), stream)); diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh index 58e94ee7aa..7c2d1d2157 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh @@ -16,18 +16,22 @@ #pragma once -#include // uintX_t #include // raft::neighbors::ivf_flat::index #include // none_ivf_sample_filter #include // RAFT_EXPLICIT -#include // rmm:cuda_stream_view + +#include // rmm:cuda_stream_view #include +#include // uintX_t + #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY namespace raft::neighbors::ivf_flat::detail { +auto RAFT_WEAK_FUNCTION is_local_topk_feasible(uint32_t k) -> bool; + template void ivfflat_interleaved_scan(const raft::neighbors::ivf_flat::index& index, const T* queries, @@ -37,6 +41,8 @@ void ivfflat_interleaved_scan(const raft::neighbors::ivf_flat::index& i const raft::distance::DistanceType metric, const uint32_t n_probes, const uint32_t k, + const uint32_t max_samples, + const uint32_t* chunk_indices, const bool select_min, IvfSampleFilterT sample_filter, IdxT* neighbors, @@ -60,6 +66,8 @@ void ivfflat_interleaved_scan(const raft::neighbors::ivf_flat::index& i const raft::distance::DistanceType metric, \ const uint32_t n_probes, \ const uint32_t k, \ + const uint32_t max_samples, \ + const uint32_t* chunk_indices, \ const bool select_min, \ IvfSampleFilterT sample_filter, \ IdxT* neighbors, \ diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-inl.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-inl.cuh index 51cd2876d8..6fc528e26b 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-inl.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-inl.cuh @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include + #include namespace raft::neighbors::ivf_flat::detail { @@ -36,6 +38,11 @@ using namespace raft::spatial::knn::detail; // NOLINT constexpr int kThreadsPerBlock = 128; +auto RAFT_WEAK_FUNCTION is_local_topk_feasible(uint32_t k) -> bool +{ + return k <= matrix::detail::select::warpsort::kMaxCapacity; +} + /** * @brief Copy `n` elements per block from one place to another. * @@ -627,6 +634,23 @@ struct loadAndComputeDist { } }; +// switch to dummy blocksort when Capacity is 0 this explicit dummy is chosen +// to support access to warpsort constants like ::queue_t::kDummy +template +struct flat_block_sort { + using type = matrix::detail::select::warpsort:: + block_sort; +}; + +template +struct flat_block_sort<0, Ascending, T, IdxT> + : ivf::detail::dummy_block_sort_t { + using type = ivf::detail::dummy_block_sort_t; +}; + +template +using block_sort_t = typename flat_block_sort::type; + /** * Scan clusters for nearest neighbors of the query vectors. * See `ivfflat_interleaved_scan` for more information. @@ -672,12 +696,15 @@ RAFT_KERNEL __launch_bounds__(kThreadsPerBlock) const uint32_t queries_offset, const uint32_t n_probes, const uint32_t k, + const uint32_t max_samples, + const uint32_t* chunk_indices, const uint32_t dim, IvfSampleFilterT sample_filter, IdxT* neighbors, float* distances) { extern __shared__ __align__(256) uint8_t interleaved_scan_kernel_smem[]; + constexpr bool kManageLocalTopK = Capacity > 0; // Using shared memory for the (part of the) query; // This allows to save on global memory bandwidth when reading index and query // data at the same time. @@ -687,8 +714,13 @@ RAFT_KERNEL __launch_bounds__(kThreadsPerBlock) { const int query_id = blockIdx.y; query += query_id * dim; - neighbors += query_id * k * gridDim.x + blockIdx.x * k; - distances += query_id * k * gridDim.x + blockIdx.x * k; + if constexpr (kManageLocalTopK) { + neighbors += query_id * k * gridDim.x + blockIdx.x * k; + distances += query_id * k * gridDim.x + blockIdx.x * k; + } else { + distances += query_id * uint64_t(max_samples); + chunk_indices += (n_probes * query_id); + } coarse_index += query_id * n_probes; } @@ -696,14 +728,8 @@ RAFT_KERNEL __launch_bounds__(kThreadsPerBlock) copy_vectorized(query_shared, query, std::min(dim, query_smem_elems)); __syncthreads(); - using block_sort_t = matrix::detail::select::warpsort::block_sort< - matrix::detail::select::warpsort::warp_sort_filtered, - Capacity, - Ascending, - float, - IdxT>; - block_sort_t queue(k); - + using local_topk_t = block_sort_t; + local_topk_t queue(k); { using align_warp = Pow2; const int lane_id = align_warp::mod(threadIdx.x); @@ -725,6 +751,13 @@ RAFT_KERNEL __launch_bounds__(kThreadsPerBlock) const uint32_t num_groups = align_warp::div(list_length + align_warp::Mask); // ceildiv by power of 2 + uint32_t sample_offset = 0; + if constexpr (!kManageLocalTopK) { + if (probe_id > 0) { sample_offset = chunk_indices[probe_id - 1]; } + assert(list_length == chunk_indices[probe_id] - sample_offset); + assert(sample_offset + list_length <= max_samples); + } + constexpr int kUnroll = WarpSize / Veclen; constexpr uint32_t kNumWarps = kThreadsPerBlock / WarpSize; // Every warp reads WarpSize vectors and computes the distances to them. @@ -771,17 +804,33 @@ RAFT_KERNEL __launch_bounds__(kThreadsPerBlock) } // Enqueue one element per thread - const float val = valid ? static_cast(dist) : block_sort_t::queue_t::kDummy; - const size_t idx = valid ? static_cast(list_indices_ptrs[list_id][vec_id]) : 0; - queue.add(val, idx); + const float val = valid ? static_cast(dist) : local_topk_t::queue_t::kDummy; + if constexpr (kManageLocalTopK) { + const size_t idx = valid ? static_cast(list_indices_ptrs[list_id][vec_id]) : 0; + queue.add(val, idx); + } else { + if (vec_id < list_length) distances[sample_offset + vec_id] = val; + } + } + + // fill up unused slots for current query + if constexpr (!kManageLocalTopK) { + if (probe_id + 1 == n_probes) { + for (uint32_t i = threadIdx.x + sample_offset + list_length; i < max_samples; + i += blockDim.x) { + distances[i] = local_topk_t::queue_t::kDummy; + } + } } } } // finalize and store selected neighbours - __syncthreads(); - queue.done(interleaved_scan_kernel_smem); - queue.store(distances, neighbors, post_process); + if constexpr (kManageLocalTopK) { + __syncthreads(); + queue.done(interleaved_scan_kernel_smem); + queue.store(distances, neighbors, post_process); + } } /** @@ -821,6 +870,8 @@ void launch_kernel(Lambda lambda, const uint32_t queries_offset, const uint32_t n_probes, const uint32_t k, + const uint32_t max_samples, + const uint32_t* chunk_indices, IvfSampleFilterT sample_filter, IdxT* neighbors, float* distances, @@ -841,12 +892,15 @@ void launch_kernel(Lambda lambda, const int max_query_smem = 16384; int query_smem_elems = std::min(max_query_smem / sizeof(T), Pow2::roundUp(index.dim())); - int smem_size = query_smem_elems * sizeof(T); - constexpr int kSubwarpSize = std::min(Capacity, WarpSize); - auto block_merge_mem = - raft::matrix::detail::select::warpsort::calc_smem_size_for_block_wide( - kThreadsPerBlock / kSubwarpSize, k); - smem_size += std::max(smem_size, block_merge_mem); + int smem_size = query_smem_elems * sizeof(T); + + if constexpr (Capacity > 0) { + constexpr int kSubwarpSize = std::min(Capacity, WarpSize); + auto block_merge_mem = + raft::matrix::detail::select::warpsort::calc_smem_size_for_block_wide( + kThreadsPerBlock / kSubwarpSize, k); + smem_size += std::max(smem_size, block_merge_mem); + } // power-of-two less than cuda limit (for better addr alignment) constexpr uint32_t kMaxGridY = 32768; @@ -879,13 +933,20 @@ void launch_kernel(Lambda lambda, queries_offset + query_offset, n_probes, k, + max_samples, + chunk_indices, index.dim(), sample_filter, neighbors, distances); queries += grid_dim_y * index.dim(); - neighbors += grid_dim_y * grid_dim_x * k; - distances += grid_dim_y * grid_dim_x * k; + if constexpr (Capacity > 0) { + neighbors += grid_dim_y * grid_dim_x * k; + distances += grid_dim_y * grid_dim_x * k; + } else { + distances += grid_dim_y * max_samples; + chunk_indices += grid_dim_y * n_probes; + } coarse_index += grid_dim_y * n_probes; } } @@ -1010,16 +1071,22 @@ struct select_interleaved_scan_kernel { * two parameters and ends with both values equal to 1. */ template - static inline void run(int capacity, int veclen, bool select_min, Args&&... args) + static inline void run(int k_max, int veclen, bool select_min, Args&&... args) { + if constexpr (Capacity > 0) { + if (k_max == 0 || k_max > Capacity) { + return select_interleaved_scan_kernel::run( + k_max, veclen, select_min, std::forward(args)...); + } + } if constexpr (Capacity > 1) { - if (capacity * 2 <= Capacity) { + if (k_max * 2 <= Capacity) { return select_interleaved_scan_kernel::run(capacity, + Veclen>::run(k_max, veclen, select_min, std::forward(args)...); @@ -1028,14 +1095,14 @@ struct select_interleaved_scan_kernel { if constexpr (Veclen > 1) { if (veclen % Veclen != 0) { return select_interleaved_scan_kernel::run( - capacity, 1, select_min, std::forward(args)...); + k_max, 1, select_min, std::forward(args)...); } } // NB: this is the limitation of the warpsort structures that use a huge number of // registers (used in the main kernel here). - RAFT_EXPECTS(capacity == Capacity, - "Capacity must be power-of-two not bigger than the maximum allowed size " - "matrix::detail::select::warpsort::kMaxCapacity (%d).", + RAFT_EXPECTS(Capacity == 0 || k_max == Capacity, + "Capacity must be either 0 or a power-of-two not bigger than the maximum " + "allowed size matrix::detail::select::warpsort::kMaxCapacity (%d).", matrix::detail::select::warpsort::kMaxCapacity); RAFT_EXPECTS( veclen == Veclen, @@ -1090,6 +1157,8 @@ void ivfflat_interleaved_scan(const index& index, const raft::distance::DistanceType metric, const uint32_t n_probes, const uint32_t k, + const uint32_t max_samples, + const uint32_t* chunk_indices, const bool select_min, IvfSampleFilterT sample_filter, IdxT* neighbors, @@ -1112,6 +1181,8 @@ void ivfflat_interleaved_scan(const index& index, queries_offset, n_probes, k, + max_samples, + chunk_indices, filter_adapter, neighbors, distances, diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh index d7dfe5c363..350b82ede7 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh @@ -16,13 +16,14 @@ #pragma once -#include // uintX_t #include // raft::neighbors::ivf_flat::index #include // none_ivf_sample_filter #include // RAFT_EXPLICIT #include +#include // uintX_t + #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY namespace raft::neighbors::ivf_flat::detail { diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_search-inl.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_search-inl.cuh index 29d521566d..98bdeda42f 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_search-inl.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_search-inl.cuh @@ -24,11 +24,13 @@ #include // raft::linalg::norm #include // raft::linalg::unary_op #include // matrix::detail::select_k +#include // raft::neighbors::detail::ivf #include // interleaved_scan #include // raft::neighbors::ivf_flat::index #include // none_ivf_sample_filter #include // utils::mapping -#include // rmm::device_memory_resource + +#include // rmm::device_memory_resource namespace raft::neighbors::ivf_flat::detail { @@ -42,6 +44,7 @@ void search_impl(raft::resources const& handle, uint32_t queries_offset, uint32_t k, uint32_t n_probes, + uint32_t max_samples, bool select_min, IdxT* neighbors, AccT* distances, @@ -49,18 +52,27 @@ void search_impl(raft::resources const& handle, IvfSampleFilterT sample_filter) { auto stream = resource::get_cuda_stream(handle); + + std::size_t n_queries_probes = std::size_t(n_queries) * std::size_t(n_probes); + // The norm of query rmm::device_uvector query_norm_dev(n_queries, stream, search_mr); // The distance value of cluster(list) and queries rmm::device_uvector distance_buffer_dev(n_queries * index.n_lists(), stream, search_mr); // The topk distance value of cluster(list) and queries - rmm::device_uvector coarse_distances_dev(n_queries * n_probes, stream, search_mr); + rmm::device_uvector coarse_distances_dev(n_queries_probes, stream, search_mr); // The topk index of cluster(list) and queries - rmm::device_uvector coarse_indices_dev(n_queries * n_probes, stream, search_mr); + rmm::device_uvector coarse_indices_dev(n_queries_probes, stream, search_mr); + + // Optional structures if postprocessing is required // The topk distance value of candidate vectors from each cluster(list) - rmm::device_uvector refined_distances_dev(n_queries * n_probes * k, stream, search_mr); + rmm::device_uvector distances_tmp_dev(0, stream, search_mr); // The topk index of candidate vectors from each cluster(list) - rmm::device_uvector refined_indices_dev(n_queries * n_probes * k, stream, search_mr); + rmm::device_uvector indices_tmp_dev(0, stream, search_mr); + // Number of samples for each query + rmm::device_uvector num_samples(0, stream, search_mr); + // Offsets per probe for each query + rmm::device_uvector chunk_index(0, stream, search_mr); size_t float_query_size; if constexpr (std::is_integral_v) { @@ -139,9 +151,6 @@ void search_impl(raft::resources const& handle, RAFT_LOG_TRACE_VEC(coarse_indices_dev.data(), n_probes); RAFT_LOG_TRACE_VEC(coarse_distances_dev.data(), n_probes); - auto distances_dev_ptr = refined_distances_dev.data(); - auto indices_dev_ptr = refined_indices_dev.data(); - uint32_t grid_dim_x = 0; if (n_probes > 1) { // query the gridDimX size to store probes topK output @@ -154,6 +163,8 @@ void search_impl(raft::resources const& handle, index.metric(), n_probes, k, + 0, + nullptr, select_min, sample_filter, nullptr, @@ -164,9 +175,30 @@ void search_impl(raft::resources const& handle, grid_dim_x = 1; } - if (grid_dim_x == 1) { - distances_dev_ptr = distances; - indices_dev_ptr = neighbors; + auto distances_dev_ptr = distances; + auto indices_dev_ptr = neighbors; + + bool manage_local_topk = is_local_topk_feasible(k); + if (!manage_local_topk || grid_dim_x > 1) { + if (!manage_local_topk) { + num_samples.resize(n_queries, stream); + chunk_index.resize(n_queries_probes, stream); + + ivf::detail::calc_chunk_indices::configure(n_probes, n_queries)( + index.list_sizes().data_handle(), + coarse_indices_dev.data(), + chunk_index.data(), + num_samples.data(), + stream); + } + + auto target_size = std::size_t(n_queries) * (manage_local_topk ? grid_dim_x * k : max_samples); + + distances_tmp_dev.resize(target_size, stream); + if (manage_local_topk) indices_tmp_dev.resize(target_size, stream); + + distances_dev_ptr = distances_tmp_dev.data(); + indices_dev_ptr = indices_tmp_dev.data(); } ivfflat_interleaved_scan::value_t, IdxT, IvfSampleFilterT>( @@ -178,6 +210,8 @@ void search_impl(raft::resources const& handle, index.metric(), n_probes, k, + max_samples, + chunk_index.data(), select_min, sample_filter, indices_dev_ptr, @@ -186,19 +220,34 @@ void search_impl(raft::resources const& handle, stream); RAFT_LOG_TRACE_VEC(distances_dev_ptr, 2 * k); - RAFT_LOG_TRACE_VEC(indices_dev_ptr, 2 * k); + if (indices_dev_ptr != nullptr) { RAFT_LOG_TRACE_VEC(indices_dev_ptr, 2 * k); } // Merge topk values from different blocks - if (grid_dim_x > 1) { + if (!manage_local_topk || grid_dim_x > 1) { matrix::detail::select_k(handle, - refined_distances_dev.data(), - refined_indices_dev.data(), + distances_tmp_dev.data(), + indices_tmp_dev.data(), n_queries, - k * grid_dim_x, + manage_local_topk ? (k * grid_dim_x) : max_samples, k, distances, neighbors, select_min); + + if (!manage_local_topk) { + // post process distances && neighbor IDs + ivf::detail::postprocess_distances( + distances, distances, index.metric(), n_queries, k, 1.0, false, stream); + ivf::detail::postprocess_neighbors(neighbors, + neighbors, + index.inds_ptrs().data_handle(), + coarse_indices_dev.data(), + chunk_index.data(), + n_queries, + n_probes, + k, + stream); + } } } @@ -223,14 +272,28 @@ inline void search(raft::resources const& handle, if (mr == nullptr) { mr = rmm::mr::get_current_device_resource(); } RAFT_EXPECTS(params.n_probes > 0, "n_probes (number of clusters to probe in the search) must be positive."); - auto n_probes = std::min(params.n_probes, index.n_lists()); + auto n_probes = std::min(params.n_probes, index.n_lists()); + bool manage_local_topk = is_local_topk_feasible(k); + + uint32_t max_samples = 0; + if (!manage_local_topk) { + IdxT ms = + Pow2<128 / sizeof(float)>::roundUp(std::max(index.accum_sorted_sizes()(n_probes), k)); + RAFT_EXPECTS(ms <= IdxT(std::numeric_limits::max()), + "The maximum sample size is too big."); + max_samples = ms; + } // a batch size heuristic: try to keep the workspace within the specified size - constexpr uint32_t kExpectedWsSize = 1024 * 1024 * 1024; + constexpr uint64_t kExpectedWsSize = 1024 * 1024 * 1024; + uint64_t max_ws_size = std::min(resource::get_workspace_free_bytes(handle), kExpectedWsSize); + + uint64_t ws_size_per_query = 4ull * (2 * n_probes + index.n_lists() + index.dim() + 1) + + (manage_local_topk ? ((sizeof(IdxT) + 4) * n_probes * k) + : (4ull * (max_samples + n_probes + 1))); + const uint32_t max_queries = - std::min(n_queries, - raft::div_rounding_up_safe( - kExpectedWsSize, 16ull * uint64_t{n_probes} * k + 4ull * index.dim())); + std::min(n_queries, raft::div_rounding_up_safe(max_ws_size, ws_size_per_query)); for (uint32_t offset_q = 0; offset_q < n_queries; offset_q += max_queries) { uint32_t queries_batch = min(max_queries, n_queries - offset_q); @@ -242,6 +305,7 @@ inline void search(raft::resources const& handle, offset_q, k, n_probes, + max_samples, raft::distance::is_min_close(index.metric()), neighbors + offset_q * k, distances + offset_q * k, diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh index aaf48ae830..3897b83aa6 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -153,7 +154,7 @@ auto deserialize(raft::resources const& handle, std::istream& is) -> index -#include - -#include -#include -#include - #include #include #include #include #include +#include +#include #include #include #include @@ -38,7 +33,12 @@ #include #include #include +#include +#include +#include +#include #include +#include #include #include #include @@ -46,16 +46,14 @@ #include #include -#include #include #include #include +#include #include #include -#include - #include #include @@ -1363,59 +1361,6 @@ void process_and_fill_codes(raft::resources const& handle, RAFT_CUDA_TRY(cudaPeekAtLastError()); } -/** Update the state of the dependent index members. */ -template -void recompute_internal_state(const raft::resources& res, index& index) -{ - auto stream = resource::get_cuda_stream(res); - auto tmp_res = resource::get_workspace_resource(res); - rmm::device_uvector sorted_sizes(index.n_lists(), stream, tmp_res); - - // Actualize the list pointers - auto data_ptrs = index.data_ptrs(); - auto inds_ptrs = index.inds_ptrs(); - for (uint32_t label = 0; label < index.n_lists(); label++) { - auto& list = index.lists()[label]; - const auto data_ptr = list ? list->data.data_handle() : nullptr; - const auto inds_ptr = list ? list->indices.data_handle() : nullptr; - copy(&data_ptrs(label), &data_ptr, 1, stream); - copy(&inds_ptrs(label), &inds_ptr, 1, stream); - } - - // Sort the cluster sizes in the descending order. - int begin_bit = 0; - int end_bit = sizeof(uint32_t) * 8; - size_t cub_workspace_size = 0; - cub::DeviceRadixSort::SortKeysDescending(nullptr, - cub_workspace_size, - index.list_sizes().data_handle(), - sorted_sizes.data(), - index.n_lists(), - begin_bit, - end_bit, - stream); - rmm::device_buffer cub_workspace(cub_workspace_size, stream, tmp_res); - cub::DeviceRadixSort::SortKeysDescending(cub_workspace.data(), - cub_workspace_size, - index.list_sizes().data_handle(), - sorted_sizes.data(), - index.n_lists(), - begin_bit, - end_bit, - stream); - // copy the results to CPU - std::vector sorted_sizes_host(index.n_lists()); - copy(sorted_sizes_host.data(), sorted_sizes.data(), index.n_lists(), stream); - resource::sync_stream(res); - - // accumulate the sorted cluster sizes - auto accum_sorted_sizes = index.accum_sorted_sizes(); - accum_sorted_sizes(0) = 0; - for (uint32_t label = 0; label < sorted_sizes_host.size(); label++) { - accum_sorted_sizes(label + 1) = accum_sorted_sizes(label) + sorted_sizes_host[label]; - } -} - /** * Helper function: allocate enough space in the list, compute the offset, at which to start * writing, and fill-in indices. @@ -1463,7 +1408,7 @@ void extend_list_with_codes(raft::resources const& res, // Pack the data pack_list_data(res, index, new_codes, label, offset); // Update the pointers and the sizes - recompute_internal_state(res, *index); + ivf::detail::recompute_internal_state(res, *index); } /** @@ -1482,7 +1427,7 @@ void extend_list(raft::resources const& res, // Encode the data encode_list_data(res, index, new_vectors, label, offset); // Update the pointers and the sizes - recompute_internal_state(res, *index); + ivf::detail::recompute_internal_state(res, *index); } /** @@ -1495,7 +1440,7 @@ void erase_list(raft::resources const& res, index* index, uint32_t label) uint32_t zero = 0; copy(index->list_sizes().data_handle() + label, &zero, 1, resource::get_cuda_stream(res)); index->lists()[label].reset(); - recompute_internal_state(res, *index); + ivf::detail::recompute_internal_state(res, *index); } /** Copy the state of an index into a new index, but share the list data among the two. */ @@ -1539,7 +1484,7 @@ auto clone(const raft::resources& res, const index& source) -> index target.lists() = source.lists(); // Make sure the device pointers point to the new lists - recompute_internal_state(res, target); + ivf::detail::recompute_internal_state(res, target); return target; } @@ -1688,7 +1633,7 @@ void extend(raft::resources const& handle, } // Update the pointers and the sizes - recompute_internal_state(handle, *index); + ivf::detail::recompute_internal_state(handle, *index); // Recover old cluster sizes: they are used as counters in the fill-codes kernel copy(list_sizes, orig_list_sizes.data(), n_clusters, stream); diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_codepacking.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_codepacking.cuh index 52969dd176..bd03409f66 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_codepacking.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_codepacking.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,9 @@ #pragma once +#include #include #include - -#include #include #include #include diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh index 37b6efc1eb..5e1a9b46d6 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,16 @@ #pragma once -#include // __half #include // RAFT_WEAK_FUNCTION #include // raft::distance::DistanceType #include // raft::neighbors::ivf_pq::detail::fp_8bit #include // raft::neighbors::ivf_pq::codebook_gen #include // none_ivf_sample_filter #include // RAFT_EXPLICIT -#include // rmm::cuda_stream_view + +#include // rmm::cuda_stream_view + +#include // __half #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-inl.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-inl.cuh index bd88c029e1..462d134b8e 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-inl.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-inl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,14 +18,15 @@ #include // raft::distance::DistanceType #include // matrix::detail::select::warpsort::warp_sort_distributed -#include // dummy_block_sort_t -#include // codebook_gen -#include // none_ivf_sample_filter -#include // RAFT_CUDA_TRY -#include // raft::atomicMin -#include // raft::Pow2 -#include // raft::TxN_t -#include // rmm::cuda_stream_view +#include // dummy_block_sort_t +#include // codebook_gen +#include // none_ivf_sample_filter +#include // RAFT_CUDA_TRY +#include // raft::atomicMin +#include // raft::Pow2 +#include // raft::TxN_t + +#include // rmm::cuda_stream_view namespace raft::neighbors::ivf_pq::detail { @@ -72,8 +73,8 @@ struct pq_block_sort { }; template -struct pq_block_sort<0, T, IdxT> : dummy_block_sort_t { - using type = dummy_block_sort_t; +struct pq_block_sort<0, T, IdxT> : ivf::detail::dummy_block_sort_t { + using type = ivf::detail::dummy_block_sort_t; static auto mem_required(uint32_t) -> size_t { return 0; } static auto get_mem_required(uint32_t) { return mem_required; } }; diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_dummy_block_sort.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_dummy_block_sort.cuh deleted file mode 100644 index a00b6a50ff..0000000000 --- a/cpp/include/raft/neighbors/detail/ivf_pq_dummy_block_sort.cuh +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // matrix::detail::select::warpsort::warp_sort_distributed - -/* - * This header file is a bit of an ugly duckling. The type dummy_block_sort is - * needed by both ivf_pq_search.cuh and ivf_pq_compute_similarity.cuh. - * - * I have decided to move it to it's own header file, which is overkill. Perhaps - * there is a nicer solution. - * - */ - -namespace raft::neighbors::ivf_pq::detail { - -template -struct dummy_block_sort_t { - using queue_t = matrix::detail::select::warpsort::warp_sort_distributed; - template - __device__ dummy_block_sort_t(int k, Args...){}; -}; - -} // namespace raft::neighbors::ivf_pq::detail diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_fp_8bit.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_fp_8bit.cuh index 68c8a513f6..1dea998f9b 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_fp_8bit.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_fp_8bit.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,6 @@ #pragma once -#include - -#include - #include #include #include @@ -32,6 +28,8 @@ #include #include #include +#include +#include #include #include #include @@ -42,7 +40,6 @@ #include #include - #include #include diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_search.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_search.cuh index d000a1a4d3..d445f909e5 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_search.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_search.cuh @@ -16,23 +16,15 @@ #pragma once -#include -#include -#include - -#include -#include -#include -#include -#include - #include #include #include #include #include +#include #include #include +#include #include #include #include @@ -40,6 +32,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -51,7 +49,6 @@ #include #include - #include #include @@ -171,217 +168,6 @@ void select_clusters(raft::resources const& handle, true); } -/** - * For each query, we calculate a cumulative sum of the cluster sizes that we probe, and return that - * in chunk_indices. Essentially this is a segmented inclusive scan of the cluster sizes. The total - * number of samples per query (sum of the cluster sizes that we probe) is returned in n_samples. - */ -template -__launch_bounds__(BlockDim) RAFT_KERNEL - calc_chunk_indices_kernel(uint32_t n_probes, - const uint32_t* cluster_sizes, // [n_clusters] - const uint32_t* clusters_to_probe, // [n_queries, n_probes] - uint32_t* chunk_indices, // [n_queries, n_probes] - uint32_t* n_samples // [n_queries] - ) -{ - using block_scan = cub::BlockScan; - __shared__ typename block_scan::TempStorage shm; - - // locate the query data - clusters_to_probe += n_probes * blockIdx.x; - chunk_indices += n_probes * blockIdx.x; - - // block scan - const uint32_t n_probes_aligned = Pow2::roundUp(n_probes); - uint32_t total = 0; - for (uint32_t probe_ix = threadIdx.x; probe_ix < n_probes_aligned; probe_ix += BlockDim) { - auto label = probe_ix < n_probes ? clusters_to_probe[probe_ix] : 0u; - auto chunk = probe_ix < n_probes ? cluster_sizes[label] : 0u; - if (threadIdx.x == 0) { chunk += total; } - block_scan(shm).InclusiveSum(chunk, chunk, total); - __syncthreads(); - if (probe_ix < n_probes) { chunk_indices[probe_ix] = chunk; } - } - // save the total size - if (threadIdx.x == 0) { n_samples[blockIdx.x] = total; } -} - -struct calc_chunk_indices { - public: - struct configured { - void* kernel; - dim3 block_dim; - dim3 grid_dim; - uint32_t n_probes; - - inline void operator()(const uint32_t* cluster_sizes, - const uint32_t* clusters_to_probe, - uint32_t* chunk_indices, - uint32_t* n_samples, - rmm::cuda_stream_view stream) - { - void* args[] = // NOLINT - {&n_probes, &cluster_sizes, &clusters_to_probe, &chunk_indices, &n_samples}; - RAFT_CUDA_TRY(cudaLaunchKernel(kernel, grid_dim, block_dim, args, 0, stream)); - } - }; - - static inline auto configure(uint32_t n_probes, uint32_t n_queries) -> configured - { - return try_block_dim<1024>(n_probes, n_queries); - } - - private: - template - static auto try_block_dim(uint32_t n_probes, uint32_t n_queries) -> configured - { - if constexpr (BlockDim >= WarpSize * 2) { - if (BlockDim >= n_probes * 2) { return try_block_dim<(BlockDim / 2)>(n_probes, n_queries); } - } - return {reinterpret_cast(calc_chunk_indices_kernel), - dim3(BlockDim, 1, 1), - dim3(n_queries, 1, 1), - n_probes}; - } -}; - -/** - * Look up the chunk id corresponding to the sample index. - * - * Each query vector was compared to all the vectors from n_probes clusters, and sample_ix is an - * ordered number of one of such vectors. This function looks up to which chunk it belongs, - * and returns the index within the chunk (which is also an index within a cluster). - * - * @param[inout] sample_ix - * input: the offset of the sample in the batch; - * output: the offset inside the chunk (probe) / selected cluster. - * @param[in] n_probes number of probes - * @param[in] chunk_indices offsets of the chunks within the batch [n_probes] - * @return chunk index (== n_probes when the input index is not in the valid range, - * which can happen if there is not enough data to output in the selected clusters). - */ -__device__ inline auto find_chunk_ix(uint32_t& sample_ix, // NOLINT - uint32_t n_probes, - const uint32_t* chunk_indices) -> uint32_t -{ - uint32_t ix_min = 0; - uint32_t ix_max = n_probes; - do { - uint32_t i = (ix_min + ix_max) / 2; - if (chunk_indices[i] <= sample_ix) { - ix_min = i + 1; - } else { - ix_max = i; - } - } while (ix_min < ix_max); - if (ix_min > 0) { sample_ix -= chunk_indices[ix_min - 1]; } - return ix_min; -} - -template -__launch_bounds__(BlockDim) RAFT_KERNEL - postprocess_neighbors_kernel(IdxT* neighbors_out, // [n_queries, topk] - const uint32_t* neighbors_in, // [n_queries, topk] - const IdxT* const* db_indices, // [n_clusters][..] - const uint32_t* clusters_to_probe, // [n_queries, n_probes] - const uint32_t* chunk_indices, // [n_queries, n_probes] - uint32_t n_queries, - uint32_t n_probes, - uint32_t topk) -{ - const uint64_t i = threadIdx.x + BlockDim * uint64_t(blockIdx.x); - const uint32_t query_ix = i / uint64_t(topk); - if (query_ix >= n_queries) { return; } - const uint32_t k = i % uint64_t(topk); - neighbors_in += query_ix * topk; - neighbors_out += query_ix * topk; - chunk_indices += query_ix * n_probes; - clusters_to_probe += query_ix * n_probes; - uint32_t data_ix = neighbors_in[k]; - const uint32_t chunk_ix = find_chunk_ix(data_ix, n_probes, chunk_indices); - const bool valid = chunk_ix < n_probes; - neighbors_out[k] = - valid ? db_indices[clusters_to_probe[chunk_ix]][data_ix] : ivf_pq::kOutOfBoundsRecord; -} - -/** - * Transform found sample indices into the corresponding database indices - * (as stored in index.indices()). - * The sample indices are the record indices as they appear in the database view formed by the - * probed clusters / defined by the `chunk_indices`. - * We assume the searched sample sizes (for a single query) fit into `uint32_t`. - */ -template -void postprocess_neighbors(IdxT* neighbors_out, // [n_queries, topk] - const uint32_t* neighbors_in, // [n_queries, topk] - const IdxT* const* db_indices, // [n_clusters][..] - const uint32_t* clusters_to_probe, // [n_queries, n_probes] - const uint32_t* chunk_indices, // [n_queries, n_probes] - uint32_t n_queries, - uint32_t n_probes, - uint32_t topk, - rmm::cuda_stream_view stream) -{ - constexpr int kPNThreads = 256; - const int pn_blocks = raft::div_rounding_up_unsafe(n_queries * topk, kPNThreads); - postprocess_neighbors_kernel - <<>>(neighbors_out, - neighbors_in, - db_indices, - clusters_to_probe, - chunk_indices, - n_queries, - n_probes, - topk); -} - -/** - * Post-process the scores depending on the metric type; - * translate the element type if necessary. - */ -template -void postprocess_distances(float* out, // [n_queries, topk] - const ScoreT* in, // [n_queries, topk] - distance::DistanceType metric, - uint32_t n_queries, - uint32_t topk, - float scaling_factor, - rmm::cuda_stream_view stream) -{ - size_t len = size_t(n_queries) * size_t(topk); - switch (metric) { - case distance::DistanceType::L2Unexpanded: - case distance::DistanceType::L2Expanded: { - linalg::unaryOp(out, - in, - len, - raft::compose_op(raft::mul_const_op{scaling_factor * scaling_factor}, - raft::cast_op{}), - stream); - } break; - case distance::DistanceType::L2SqrtUnexpanded: - case distance::DistanceType::L2SqrtExpanded: { - linalg::unaryOp( - out, - in, - len, - raft::compose_op{ - raft::mul_const_op{scaling_factor}, raft::sqrt_op{}, raft::cast_op{}}, - stream); - } break; - case distance::DistanceType::InnerProduct: { - linalg::unaryOp(out, - in, - len, - raft::compose_op(raft::mul_const_op{-scaling_factor * scaling_factor}, - raft::cast_op{}), - stream); - } break; - default: RAFT_FAIL("Unexpected metric."); - } -} - /** * An approximation to the number of times each cluster appears in a batched sample. * @@ -520,11 +306,11 @@ void ivfpq_search_worker(raft::resources const& handle, neighbors_uint32 = neighbors_uint32_buf.data(); } - calc_chunk_indices::configure(n_probes, n_queries)(index.list_sizes().data_handle(), - clusters_to_probe, - chunk_index.data(), - num_samples.data(), - stream); + ivf::detail::calc_chunk_indices::configure(n_probes, n_queries)(index.list_sizes().data_handle(), + clusters_to_probe, + chunk_index.data(), + num_samples.data(), + stream); auto coresidency = expected_probe_coresidency(index.n_lists(), n_probes, n_queries); @@ -621,9 +407,10 @@ void ivfpq_search_worker(raft::resources const& handle, if (manage_local_topk) { query_kths_buf.emplace( make_device_mdarray(handle, mr, make_extents(n_queries))); - linalg::map(handle, - query_kths_buf->view(), - raft::const_op{dummy_block_sort_t::queue_t::kDummy}); + linalg::map( + handle, + query_kths_buf->view(), + raft::const_op{ivf::detail::dummy_block_sort_t::queue_t::kDummy}); query_kths = query_kths_buf->data_handle(); } compute_similarity_run(search_instance, @@ -663,17 +450,17 @@ void ivfpq_search_worker(raft::resources const& handle, true); // Postprocessing - postprocess_distances( - distances, topk_dists.data(), index.metric(), n_queries, topK, scaling_factor, stream); - postprocess_neighbors(neighbors, - neighbors_uint32, - index.inds_ptrs().data_handle(), - clusters_to_probe, - chunk_index.data(), - n_queries, - n_probes, - topK, - stream); + ivf::detail::postprocess_distances( + distances, topk_dists.data(), index.metric(), n_queries, topK, scaling_factor, true, stream); + ivf::detail::postprocess_neighbors(neighbors, + neighbors_uint32, + index.inds_ptrs().data_handle(), + clusters_to_probe, + chunk_index.data(), + n_queries, + n_probes, + topK, + stream); } /** diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh index 038bf8d7cc..4428fa370b 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,14 @@ #pragma once -#include -#include -#include -#include - #include #include +#include #include #include +#include +#include +#include #include #include @@ -160,7 +159,7 @@ auto deserialize(raft::resources const& handle_, std::istream& is) -> index +#include #include #include #include -#include -#include namespace raft::neighbors::detail { diff --git a/cpp/include/raft/neighbors/detail/nn_descent.cuh b/cpp/include/raft/neighbors/detail/nn_descent.cuh index f624a6015b..cd3c6f3947 100644 --- a/cpp/include/raft/neighbors/detail/nn_descent.cuh +++ b/cpp/include/raft/neighbors/detail/nn_descent.cuh @@ -16,23 +16,6 @@ #pragma once -#include -#include -#include - -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include - #include "../nn_descent_types.hpp" #include @@ -48,6 +31,23 @@ #include #include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + namespace raft::neighbors::experimental::nn_descent::detail { using pinned_memory_resource = thrust::universal_host_pinned_memory_resource; diff --git a/cpp/include/raft/neighbors/detail/refine_device.cuh b/cpp/include/raft/neighbors/detail/refine_device.cuh index 5c9f1459e7..e76e52657b 100644 --- a/cpp/include/raft/neighbors/detail/refine_device.cuh +++ b/cpp/include/raft/neighbors/detail/refine_device.cuh @@ -50,6 +50,7 @@ void refine_device(raft::resources const& handle, matrix_idx dim = dataset.extent(1); uint32_t k = static_cast(indices.extent(1)); + // TODO: this restriction could be lifted with some effort RAFT_EXPECTS(k <= raft::matrix::detail::select::warpsort::kMaxCapacity, "k must be less than topk::kMaxCapacity (%d).", raft::matrix::detail::select::warpsort::kMaxCapacity); @@ -98,6 +99,8 @@ void refine_device(raft::resources const& handle, refinement_index.metric(), 1, k, + 0, + nullptr, raft::distance::is_min_close(metric), raft::neighbors::filtering::none_ivf_sample_filter(), indices.data_handle(), diff --git a/cpp/include/raft/neighbors/detail/refine_host-ext.hpp b/cpp/include/raft/neighbors/detail/refine_host-ext.hpp index a2f75d0f8b..69d2bd29b2 100644 --- a/cpp/include/raft/neighbors/detail/refine_host-ext.hpp +++ b/cpp/include/raft/neighbors/detail/refine_host-ext.hpp @@ -16,13 +16,13 @@ #pragma once -#include // int64_t - #include // _RAFT_HAS_CUDA #include // raft::host_matrix_view #include // raft::distance::DistanceType #include // RAFT_EXPLICIT +#include // int64_t + #if defined(_RAFT_HAS_CUDA) #include #endif diff --git a/cpp/include/raft/neighbors/detail/refine_host-inl.hpp b/cpp/include/raft/neighbors/detail/refine_host-inl.hpp index a54525f3e6..9aff451dfc 100644 --- a/cpp/include/raft/neighbors/detail/refine_host-inl.hpp +++ b/cpp/include/raft/neighbors/detail/refine_host-inl.hpp @@ -21,9 +21,10 @@ #include #include -#include #include +#include + namespace raft::neighbors::detail { template diff --git a/cpp/include/raft/neighbors/hnsw.hpp b/cpp/include/raft/neighbors/hnsw.hpp index dceb98c5aa..964c3ffacd 100644 --- a/cpp/include/raft/neighbors/hnsw.hpp +++ b/cpp/include/raft/neighbors/hnsw.hpp @@ -19,13 +19,13 @@ #include "detail/hnsw.hpp" #include "hnsw.hpp" -#include - -#include #include #include #include +#include +#include + namespace raft::neighbors::hnsw { /** diff --git a/cpp/include/raft/neighbors/hnsw_serialize.hpp b/cpp/include/raft/neighbors/hnsw_serialize.hpp index 45819c8fb5..f7450bb3d6 100644 --- a/cpp/include/raft/neighbors/hnsw_serialize.hpp +++ b/cpp/include/raft/neighbors/hnsw_serialize.hpp @@ -18,9 +18,9 @@ #include "detail/hnsw_serialize.hpp" #include "hnsw_types.hpp" -#include #include +#include namespace raft::neighbors::hnsw { diff --git a/cpp/include/raft/neighbors/hnsw_types.hpp b/cpp/include/raft/neighbors/hnsw_types.hpp index aa4cefbc30..645a0903b7 100644 --- a/cpp/include/raft/neighbors/hnsw_types.hpp +++ b/cpp/include/raft/neighbors/hnsw_types.hpp @@ -17,11 +17,13 @@ #pragma once #include "ann_types.hpp" -#include + #include -#include #include + +#include +#include #include namespace raft::neighbors::hnsw { diff --git a/cpp/include/raft/neighbors/ivf_flat-ext.cuh b/cpp/include/raft/neighbors/ivf_flat-ext.cuh index 063105cf46..a1783dfcfd 100644 --- a/cpp/include/raft/neighbors/ivf_flat-ext.cuh +++ b/cpp/include/raft/neighbors/ivf_flat-ext.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,16 @@ #pragma once -#include // int64_t - #include // raft::device_matrix_view #include // raft::resources #include -#include // raft::neighbors::ivf_flat::index -#include // RAFT_EXPLICIT +#include // raft::neighbors::ivf_flat::index +#include // RAFT_EXPLICIT + #include // rmm::mr::device_memory_resource +#include // int64_t + #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY namespace raft::neighbors::ivf_flat { diff --git a/cpp/include/raft/neighbors/ivf_flat-inl.cuh b/cpp/include/raft/neighbors/ivf_flat-inl.cuh index 13c8e3d86d..ed1d320795 100644 --- a/cpp/include/raft/neighbors/ivf_flat-inl.cuh +++ b/cpp/include/raft/neighbors/ivf_flat-inl.cuh @@ -16,14 +16,13 @@ #pragma once +#include +#include #include #include #include #include -#include -#include - #include #include diff --git a/cpp/include/raft/neighbors/ivf_flat_helpers.cuh b/cpp/include/raft/neighbors/ivf_flat_helpers.cuh index 7a05c9991c..883e72c839 100644 --- a/cpp/include/raft/neighbors/ivf_flat_helpers.cuh +++ b/cpp/include/raft/neighbors/ivf_flat_helpers.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,10 @@ #pragma once -#include -#include - #include #include - +#include +#include #include namespace raft::neighbors::ivf_flat::helpers { @@ -137,9 +135,42 @@ void reset_index(const raft::resources& res, index* index) { auto stream = resource::get_cuda_stream(res); + utils::memzero( + index->accum_sorted_sizes().data_handle(), index->accum_sorted_sizes().size(), stream); utils::memzero(index->list_sizes().data_handle(), index->list_sizes().size(), stream); utils::memzero(index->data_ptrs().data_handle(), index->data_ptrs().size(), stream); utils::memzero(index->inds_ptrs().data_handle(), index->inds_ptrs().size(), stream); } + +/** + * @brief Helper exposing the re-computation of list sizes and related arrays if IVF lists have been + * modified. + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * raft::resources res; + * // use default index parameters + * ivf_flat::index_params index_params; + * // initialize an empty index + * ivf_flat::index index(res, index_params, D); + * ivf_flat::helpers::reset_index(res, &index); + * // recompute the internal state of the index + * ivf_flat::helpers::recompute_internal_state(res, &index); + * @endcode + * + * @tparam T + * @tparam IdxT + * + * @param[in] res raft resource + * @param[inout] index pointer to IVF-FLAT index + */ +template +void recompute_internal_state(const raft::resources& res, index* index) +{ + auto& list = index->lists()[0]; + ivf::detail::recompute_internal_state(res, *index); +} + /** @} */ } // namespace raft::neighbors::ivf_flat::helpers diff --git a/cpp/include/raft/neighbors/ivf_flat_types.hpp b/cpp/include/raft/neighbors/ivf_flat_types.hpp index 180fe2e21b..0908e3b0b0 100644 --- a/cpp/include/raft/neighbors/ivf_flat_types.hpp +++ b/cpp/include/raft/neighbors/ivf_flat_types.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,21 +17,19 @@ #pragma once #include "ann_types.hpp" -#include #include #include #include #include #include +#include #include #include #include #include #include -#include - #include // std::max #include #include @@ -222,8 +220,31 @@ struct index : ann::index { } } + /** + * Accumulated list sizes, sorted in descending order [n_lists + 1]. + * The last value contains the total length of the index. + * The value at index zero is always zero. + * + * That is, the content of this span is as if the `list_sizes` was sorted and then accumulated. + * + * This span is used during search to estimate the maximum size of the workspace. + */ + inline auto accum_sorted_sizes() noexcept -> host_vector_view + { + return accum_sorted_sizes_.view(); + } + [[nodiscard]] inline auto accum_sorted_sizes() const noexcept + -> host_vector_view + { + return accum_sorted_sizes_.view(); + } + /** Total length of the index. */ - [[nodiscard]] constexpr inline auto size() const noexcept -> IdxT { return total_size_; } + [[nodiscard]] constexpr inline auto size() const noexcept -> IdxT + { + return accum_sorted_sizes()(n_lists()); + } + /** Dimensionality of the data. */ [[nodiscard]] constexpr inline auto dim() const noexcept -> uint32_t { @@ -257,9 +278,10 @@ struct index : ann::index { list_sizes_{make_device_vector(res, n_lists)}, data_ptrs_{make_device_vector(res, n_lists)}, inds_ptrs_{make_device_vector(res, n_lists)}, - total_size_{0} + accum_sorted_sizes_{make_host_vector(n_lists + 1)} { check_consistency(); + accum_sorted_sizes_(n_lists) = 0; } /** Construct an empty index. It needs to be trained and then populated. */ @@ -298,33 +320,6 @@ struct index : ann::index { return conservative_memory_allocation_; } - /** - * Update the state of the dependent index members. - */ - void recompute_internal_state(raft::resources const& res) - { - auto stream = resource::get_cuda_stream(res); - - // Actualize the list pointers - auto this_lists = lists(); - auto this_data_ptrs = data_ptrs(); - auto this_inds_ptrs = inds_ptrs(); - for (uint32_t label = 0; label < this_lists.size(); label++) { - auto& list = this_lists[label]; - const auto data_ptr = list ? list->data.data_handle() : nullptr; - const auto inds_ptr = list ? list->indices.data_handle() : nullptr; - copy(&this_data_ptrs(label), &data_ptr, 1, stream); - copy(&this_inds_ptrs(label), &inds_ptr, 1, stream); - } - auto this_list_sizes = list_sizes().data_handle(); - total_size_ = thrust::reduce(resource::get_thrust_policy(res), - this_list_sizes, - this_list_sizes + this_lists.size(), - 0, - raft::add_op{}); - check_consistency(); - } - void allocate_center_norms(raft::resources const& res) { switch (metric_) { @@ -349,6 +344,20 @@ struct index : ann::index { return lists_; } + /** Throw an error if the index content is inconsistent. */ + void check_consistency() + { + auto n_lists = lists_.size(); + RAFT_EXPECTS(dim() % veclen_ == 0, "dimensionality is not a multiple of the veclen"); + RAFT_EXPECTS(list_sizes_.extent(0) == n_lists, "inconsistent list size"); + RAFT_EXPECTS(data_ptrs_.extent(0) == n_lists, "inconsistent list size"); + RAFT_EXPECTS(inds_ptrs_.extent(0) == n_lists, "inconsistent list size"); + RAFT_EXPECTS( // + (centers_.extent(0) == list_sizes_.extent(0)) && // + (!center_norms_.has_value() || centers_.extent(0) == center_norms_->extent(0)), + "inconsistent number of lists (clusters)"); + } + private: /** * TODO: in theory, we can lift this to the template parameter and keep it at hardware maximum @@ -366,21 +375,7 @@ struct index : ann::index { // Computed members device_vector data_ptrs_; device_vector inds_ptrs_; - IdxT total_size_; - - /** Throw an error if the index content is inconsistent. */ - void check_consistency() - { - auto n_lists = lists_.size(); - RAFT_EXPECTS(dim() % veclen_ == 0, "dimensionality is not a multiple of the veclen"); - RAFT_EXPECTS(list_sizes_.extent(0) == n_lists, "inconsistent list size"); - RAFT_EXPECTS(data_ptrs_.extent(0) == n_lists, "inconsistent list size"); - RAFT_EXPECTS(inds_ptrs_.extent(0) == n_lists, "inconsistent list size"); - RAFT_EXPECTS( // - (centers_.extent(0) == list_sizes_.extent(0)) && // - (!center_norms_.has_value() || centers_.extent(0) == center_norms_->extent(0)), - "inconsistent number of lists (clusters)"); - } + host_vector accum_sorted_sizes_; static auto calculate_veclen(uint32_t dim) -> uint32_t { diff --git a/cpp/include/raft/neighbors/ivf_list.hpp b/cpp/include/raft/neighbors/ivf_list.hpp index ad06a3ee71..08879ed059 100644 --- a/cpp/include/raft/neighbors/ivf_list.hpp +++ b/cpp/include/raft/neighbors/ivf_list.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,16 +16,15 @@ #pragma once -#include -#include -#include - #include #include #include #include +#include +#include #include #include +#include #include #include diff --git a/cpp/include/raft/neighbors/ivf_pq-ext.cuh b/cpp/include/raft/neighbors/ivf_pq-ext.cuh index 188219dfa2..160a2753a5 100644 --- a/cpp/include/raft/neighbors/ivf_pq-ext.cuh +++ b/cpp/include/raft/neighbors/ivf_pq-ext.cuh @@ -16,14 +16,15 @@ #pragma once -#include // int64_t +#include // raft::device_matrix_view +#include // raft::resources +#include // raft::neighbors::ivf_pq::index +#include // RAFT_EXPLICIT -#include // raft::device_matrix_view -#include // raft::resources -#include // raft::neighbors::ivf_pq::index -#include // RAFT_EXPLICIT #include // rmm::mr::device_memory_resource +#include // int64_t + #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY namespace raft::neighbors::ivf_pq { diff --git a/cpp/include/raft/neighbors/ivf_pq-inl.cuh b/cpp/include/raft/neighbors/ivf_pq-inl.cuh index d14456d6f6..a893153e1a 100644 --- a/cpp/include/raft/neighbors/ivf_pq-inl.cuh +++ b/cpp/include/raft/neighbors/ivf_pq-inl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,14 @@ #pragma once +#include +#include +#include #include #include #include #include -#include -#include -#include - #include #include // shared_ptr diff --git a/cpp/include/raft/neighbors/ivf_pq_helpers.cuh b/cpp/include/raft/neighbors/ivf_pq_helpers.cuh index fec31f1c61..8bd3ab2591 100644 --- a/cpp/include/raft/neighbors/ivf_pq_helpers.cuh +++ b/cpp/include/raft/neighbors/ivf_pq_helpers.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,16 +16,15 @@ #pragma once -#include +#include #include +#include #include #include - -#include -#include - #include +#include + namespace raft::neighbors::ivf_pq::helpers { using namespace raft::spatial::knn::detail; // NOLINT /** @@ -733,7 +732,7 @@ void set_centers(raft::resources const& res, * ivf::resize_list(res, list, spec, new_size, 0); * raft::update_device(index.list_sizes(), &new_size, 1, stream); * // recompute the internal state of the index - * ivf_pq::recompute_internal_state(res, &index); + * ivf_pq::helpers::recompute_internal_state(res, &index); * @endcode * * @tparam IdxT @@ -745,7 +744,7 @@ template void recompute_internal_state(const raft::resources& res, index* index) { auto& list = index->lists()[0]; - ivf_pq::detail::recompute_internal_state(res, *index); + ivf::detail::recompute_internal_state(res, *index); } /** diff --git a/cpp/include/raft/neighbors/ivf_pq_types.hpp b/cpp/include/raft/neighbors/ivf_pq_types.hpp index 3e6cc443b3..81e2886b18 100644 --- a/cpp/include/raft/neighbors/ivf_pq_types.hpp +++ b/cpp/include/raft/neighbors/ivf_pq_types.hpp @@ -16,15 +16,14 @@ #pragma once -#include -#include - #include #include #include #include #include #include +#include +#include #include #include diff --git a/cpp/include/raft/neighbors/nn_descent_types.hpp b/cpp/include/raft/neighbors/nn_descent_types.hpp index fd1df2965e..e1fc96878a 100644 --- a/cpp/include/raft/neighbors/nn_descent_types.hpp +++ b/cpp/include/raft/neighbors/nn_descent_types.hpp @@ -17,11 +17,11 @@ #pragma once #include "ann_types.hpp" -#include #include #include #include +#include #include #include diff --git a/cpp/include/raft/neighbors/refine-ext.cuh b/cpp/include/raft/neighbors/refine-ext.cuh index 3ccd3891b7..fc57494b22 100644 --- a/cpp/include/raft/neighbors/refine-ext.cuh +++ b/cpp/include/raft/neighbors/refine-ext.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,14 @@ #pragma once -#include // int64_t - #include // raft::device_matrix_view #include // // raft::host_matrix_view #include // raft::resources #include // raft::distance::DistanceType #include // RAFT_EXPLICIT +#include // int64_t + #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY namespace raft::neighbors { diff --git a/cpp/include/raft/neighbors/sample_filter.cuh b/cpp/include/raft/neighbors/sample_filter.cuh index 5864590034..5e9790cf24 100644 --- a/cpp/include/raft/neighbors/sample_filter.cuh +++ b/cpp/include/raft/neighbors/sample_filter.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,11 @@ #pragma once +#include + #include #include -#include - namespace raft::neighbors::filtering { /** * @brief Filter an index with a bitset diff --git a/cpp/include/raft/neighbors/sample_filter_types.hpp b/cpp/include/raft/neighbors/sample_filter_types.hpp index 25030f48c8..b0c482d914 100644 --- a/cpp/include/raft/neighbors/sample_filter_types.hpp +++ b/cpp/include/raft/neighbors/sample_filter_types.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,11 @@ #pragma once +#include + #include #include -#include - namespace raft::neighbors::filtering { /* A filter that filters nothing. This is the default behavior. */ diff --git a/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp b/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp index fa2689e960..fbe38fd261 100644 --- a/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp +++ b/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,11 @@ * limitations under the License. */ -#include #include #include +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/include/raft/random/detail/make_blobs.cuh b/cpp/include/raft/random/detail/make_blobs.cuh index 1140b3484f..a9c7fc0b40 100644 --- a/cpp/include/raft/random/detail/make_blobs.cuh +++ b/cpp/include/raft/random/detail/make_blobs.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,16 @@ #pragma once #include "permute.cuh" + #include #include #include #include #include #include + #include + #include namespace raft { diff --git a/cpp/include/raft/random/detail/make_regression.cuh b/cpp/include/raft/random/detail/make_regression.cuh index ecae8832fa..f79534c75a 100644 --- a/cpp/include/raft/random/detail/make_regression.cuh +++ b/cpp/include/raft/random/detail/make_regression.cuh @@ -20,8 +20,6 @@ #pragma once -#include - #include #include #include @@ -32,8 +30,11 @@ #include #include #include + #include +#include + namespace raft::random { namespace detail { diff --git a/cpp/include/raft/random/detail/multi_variable_gaussian.cuh b/cpp/include/raft/random/detail/multi_variable_gaussian.cuh index b88374d5b4..e88cbbdeea 100644 --- a/cpp/include/raft/random/detail/multi_variable_gaussian.cuh +++ b/cpp/include/raft/random/detail/multi_variable_gaussian.cuh @@ -16,9 +16,7 @@ #pragma once #include "curand_wrappers.hpp" -#include -#include -#include + #include #include #include @@ -31,8 +29,14 @@ #include #include #include + #include + #include + +#include +#include +#include #include // mvg.cuh takes in matrices that are column major (as in fortran) diff --git a/cpp/include/raft/random/detail/permute.cuh b/cpp/include/raft/random/detail/permute.cuh index cf68a746a9..37caa51ad3 100644 --- a/cpp/include/raft/random/detail/permute.cuh +++ b/cpp/include/raft/random/detail/permute.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,14 @@ #pragma once -#include -#include #include #include #include +#include + +#include + namespace raft::random { namespace detail { diff --git a/cpp/include/raft/random/detail/rng_device.cuh b/cpp/include/raft/random/detail/rng_device.cuh index 26bf93c260..12c67679ba 100644 --- a/cpp/include/raft/random/detail/rng_device.cuh +++ b/cpp/include/raft/random/detail/rng_device.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,10 @@ #include #include -#include - #include +#include + #include namespace raft { diff --git a/cpp/include/raft/random/detail/rng_impl_deprecated.cuh b/cpp/include/raft/random/detail/rng_impl_deprecated.cuh index 8895d22cf0..dd2de15b87 100644 --- a/cpp/include/raft/random/detail/rng_impl_deprecated.cuh +++ b/cpp/include/raft/random/detail/rng_impl_deprecated.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,15 +22,18 @@ #include "rng_device.cuh" -#include #include #include #include #include #include -#include + #include +#include + +#include + namespace raft { namespace random { namespace detail { diff --git a/cpp/include/raft/random/device/sample.cuh b/cpp/include/raft/random/device/sample.cuh index 24279f82e4..d0e5200185 100644 --- a/cpp/include/raft/random/device/sample.cuh +++ b/cpp/include/raft/random/device/sample.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,13 @@ #pragma once -#include - #include #include #include #include +#include + namespace raft::random::device { /** diff --git a/cpp/include/raft/random/make_blobs.cuh b/cpp/include/raft/random/make_blobs.cuh index 079ab43b74..4fd1f44f64 100644 --- a/cpp/include/raft/random/make_blobs.cuh +++ b/cpp/include/raft/random/make_blobs.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,13 @@ #pragma once #include "detail/make_blobs.cuh" -#include + #include #include #include +#include + namespace raft::random { /** diff --git a/cpp/include/raft/random/make_regression.cuh b/cpp/include/raft/random/make_regression.cuh index 0aa9cc4daa..94d10257f9 100644 --- a/cpp/include/raft/random/make_regression.cuh +++ b/cpp/include/raft/random/make_regression.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,13 +23,14 @@ #pragma once -#include -#include +#include "detail/make_regression.cuh" + #include #include #include -#include "detail/make_regression.cuh" +#include +#include namespace raft::random { diff --git a/cpp/include/raft/random/multi_variable_gaussian.cuh b/cpp/include/raft/random/multi_variable_gaussian.cuh index eada1c9521..ab3f433422 100644 --- a/cpp/include/raft/random/multi_variable_gaussian.cuh +++ b/cpp/include/raft/random/multi_variable_gaussian.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #pragma once #include "detail/multi_variable_gaussian.cuh" + #include #include diff --git a/cpp/include/raft/random/permute.cuh b/cpp/include/raft/random/permute.cuh index d349b68add..7c3a4cc6c8 100644 --- a/cpp/include/raft/random/permute.cuh +++ b/cpp/include/raft/random/permute.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,10 +21,11 @@ #include "detail/permute.cuh" -#include #include #include #include + +#include #include namespace raft::random { diff --git a/cpp/include/raft/random/rmat_rectangular_generator.cuh b/cpp/include/raft/random/rmat_rectangular_generator.cuh index 90cd9baf81..5598b25c8e 100644 --- a/cpp/include/raft/random/rmat_rectangular_generator.cuh +++ b/cpp/include/raft/random/rmat_rectangular_generator.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include "detail/rmat_rectangular_generator.cuh" + #include namespace raft::random { diff --git a/cpp/include/raft/random/rng.cuh b/cpp/include/raft/random/rng.cuh index a946471312..4e63669f98 100644 --- a/cpp/include/raft/random/rng.cuh +++ b/cpp/include/raft/random/rng.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,11 +19,13 @@ #include "detail/rng_impl.cuh" #include "detail/rng_impl_deprecated.cuh" // necessary for now (to be removed) #include "rng_state.hpp" -#include -#include + #include #include #include + +#include +#include #include #include diff --git a/cpp/include/raft/random/sample_without_replacement.cuh b/cpp/include/raft/random/sample_without_replacement.cuh index b074f68af6..fad1d4adfa 100644 --- a/cpp/include/raft/random/sample_without_replacement.cuh +++ b/cpp/include/raft/random/sample_without_replacement.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ #include "detail/rng_impl.cuh" #include "rng_state.hpp" -#include -#include + #include #include #include + +#include +#include #include #include diff --git a/cpp/include/raft/solver/detail/lap_functions.cuh b/cpp/include/raft/solver/detail/lap_functions.cuh index 30a4961abf..dc39082f63 100644 --- a/cpp/include/raft/solver/detail/lap_functions.cuh +++ b/cpp/include/raft/solver/detail/lap_functions.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * Copyright 2020 KETAN DATE & RAKESH NAGI * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,11 +25,11 @@ #pragma once #include -#include - #include #include +#include #include + #include #include diff --git a/cpp/include/raft/solver/linear_assignment.cuh b/cpp/include/raft/solver/linear_assignment.cuh index a88356a42e..7ee0f5fbc3 100644 --- a/cpp/include/raft/solver/linear_assignment.cuh +++ b/cpp/include/raft/solver/linear_assignment.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * Copyright 2020 KETAN DATE & RAKESH NAGI * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,14 +30,14 @@ #include #include +#include +#include + #include #include #include -#include -#include - namespace raft::solver { /** diff --git a/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh index 4a4453417d..69949a71ed 100644 --- a/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,16 @@ #pragma once -#include #include - #include #include #include #include + #include +#include + namespace raft { namespace sparse { namespace convert { diff --git a/cpp/include/raft/sparse/convert/detail/coo.cuh b/cpp/include/raft/sparse/convert/detail/coo.cuh index 1302ed1220..0a498bb1ca 100644 --- a/cpp/include/raft/sparse/convert/detail/coo.cuh +++ b/cpp/include/raft/sparse/convert/detail/coo.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,22 +16,21 @@ #pragma once -#include +#include #include +#include #include #include +#include #include #include -#include +#include #include #include -#include -#include - namespace raft { namespace sparse { namespace convert { diff --git a/cpp/include/raft/sparse/convert/detail/csr.cuh b/cpp/include/raft/sparse/convert/detail/csr.cuh index 11e745f680..a5d7de9a07 100644 --- a/cpp/include/raft/sparse/convert/detail/csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/csr.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,30 +16,29 @@ #pragma once -#include #include #include - #include +#include #include +#include +#include +#include #include #include + #include +#include #include #include -#include +#include #include #include #include -#include -#include -#include -#include - namespace raft { namespace sparse { namespace convert { diff --git a/cpp/include/raft/sparse/convert/detail/dense.cuh b/cpp/include/raft/sparse/convert/detail/dense.cuh index f3db8a36c1..e60e494d34 100644 --- a/cpp/include/raft/sparse/convert/detail/dense.cuh +++ b/cpp/include/raft/sparse/convert/detail/dense.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,23 +16,23 @@ #pragma once -#include #include +#include #include #include +#include + +#include #include #include -#include +#include #include #include #include -#include -#include - namespace raft { namespace sparse { namespace convert { diff --git a/cpp/include/raft/sparse/detail/coo.cuh b/cpp/include/raft/sparse/detail/coo.cuh index cbcbee0139..91ba363168 100644 --- a/cpp/include/raft/sparse/detail/coo.cuh +++ b/cpp/include/raft/sparse/detail/coo.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,12 @@ * limitations under the License. */ -#include #include + #include +#include + #pragma once namespace raft { diff --git a/cpp/include/raft/sparse/detail/csr.cuh b/cpp/include/raft/sparse/detail/csr.cuh index 4f8496a026..9901e89b8c 100644 --- a/cpp/include/raft/sparse/detail/csr.cuh +++ b/cpp/include/raft/sparse/detail/csr.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,24 +16,24 @@ #pragma once -#include #include +#include #include #include + #include #include +#include #include #include -#include +#include #include #include #include -#include - namespace raft { namespace sparse { namespace detail { diff --git a/cpp/include/raft/sparse/detail/cusparse_wrappers.h b/cpp/include/raft/sparse/detail/cusparse_wrappers.h index cc3ae3ab87..08efbb7106 100644 --- a/cpp/include/raft/sparse/detail/cusparse_wrappers.h +++ b/cpp/include/raft/sparse/detail/cusparse_wrappers.h @@ -16,12 +16,14 @@ #pragma once -#include #include #include #include + #include +#include + namespace raft { namespace sparse { namespace detail { diff --git a/cpp/include/raft/sparse/distance/detail/bin_distance.cuh b/cpp/include/raft/sparse/distance/detail/bin_distance.cuh index ae16b1adcc..7a2396c2cd 100644 --- a/cpp/include/raft/sparse/distance/detail/bin_distance.cuh +++ b/cpp/include/raft/sparse/distance/detail/bin_distance.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,17 +16,19 @@ #pragma once -#include -#include - #include "common.hpp" + +#include #include #include #include #include #include + #include +#include + #include namespace raft { diff --git a/cpp/include/raft/sparse/distance/detail/coo_spmv.cuh b/cpp/include/raft/sparse/distance/detail/coo_spmv.cuh index c0d5fbc365..b0469f559a 100644 --- a/cpp/include/raft/sparse/distance/detail/coo_spmv.cuh +++ b/cpp/include/raft/sparse/distance/detail/coo_spmv.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,24 +16,22 @@ #pragma once +#include "../../csr.hpp" +#include "../../detail/utils.h" +#include "common.hpp" #include "coo_spmv_strategies/dense_smem_strategy.cuh" #include "coo_spmv_strategies/hash_strategy.cuh" -#include +#include #include #include #include -#include "../../csr.hpp" -#include "../../detail/utils.h" -#include "common.hpp" - +#include #include #include -#include - namespace raft { namespace sparse { namespace distance { diff --git a/cpp/include/raft/sparse/distance/detail/coo_spmv_kernel.cuh b/cpp/include/raft/sparse/distance/detail/coo_spmv_kernel.cuh index 43e8dea3ee..d8f5b2eca5 100644 --- a/cpp/include/raft/sparse/distance/detail/coo_spmv_kernel.cuh +++ b/cpp/include/raft/sparse/distance/detail/coo_spmv_kernel.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,10 @@ #pragma once -#include - #include #include #include +#include namespace raft { namespace sparse { diff --git a/cpp/include/raft/sparse/distance/detail/coo_spmv_strategies/base_strategy.cuh b/cpp/include/raft/sparse/distance/detail/coo_spmv_strategies/base_strategy.cuh index 1c2f83c69b..fc5881e2d4 100644 --- a/cpp/include/raft/sparse/distance/detail/coo_spmv_strategies/base_strategy.cuh +++ b/cpp/include/raft/sparse/distance/detail/coo_spmv_strategies/base_strategy.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include "../coo_spmv_kernel.cuh" #include "../utils.cuh" #include "coo_mask_row_iterators.cuh" + #include #include diff --git a/cpp/include/raft/sparse/distance/detail/coo_spmv_strategies/hash_strategy.cuh b/cpp/include/raft/sparse/distance/detail/coo_spmv_strategies/hash_strategy.cuh index 20c722f3e0..e271f2cdbe 100644 --- a/cpp/include/raft/sparse/distance/detail/coo_spmv_strategies/hash_strategy.cuh +++ b/cpp/include/raft/sparse/distance/detail/coo_spmv_strategies/hash_strategy.cuh @@ -17,11 +17,11 @@ #pragma once #include "base_strategy.cuh" + #include #include #include - #include #include diff --git a/cpp/include/raft/sparse/distance/detail/ip_distance.cuh b/cpp/include/raft/sparse/distance/detail/ip_distance.cuh index 39e67acdea..84229120ea 100644 --- a/cpp/include/raft/sparse/distance/detail/ip_distance.cuh +++ b/cpp/include/raft/sparse/distance/detail/ip_distance.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,22 +16,24 @@ #pragma once -#include -#include -#include -#include -#include -#include - #include "common.hpp" + #include #include +#include +#include #include +#include #include #include #include +#include +#include + #include +#include + #include namespace raft { diff --git a/cpp/include/raft/sparse/distance/detail/l2_distance.cuh b/cpp/include/raft/sparse/distance/detail/l2_distance.cuh index c198f91a31..93d1f048ec 100644 --- a/cpp/include/raft/sparse/distance/detail/l2_distance.cuh +++ b/cpp/include/raft/sparse/distance/detail/l2_distance.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,26 +16,26 @@ #pragma once -#include -#include - #include "common.hpp" + +#include #include #include #include #include #include #include +#include #include #include -#include -#include +#include #include #include #include +#include namespace raft { namespace sparse { diff --git a/cpp/include/raft/sparse/distance/detail/lp_distance.cuh b/cpp/include/raft/sparse/distance/detail/lp_distance.cuh index ff9534a157..b178e02c34 100644 --- a/cpp/include/raft/sparse/distance/detail/lp_distance.cuh +++ b/cpp/include/raft/sparse/distance/detail/lp_distance.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,25 +16,24 @@ #pragma once -#include -#include +#include "common.hpp" #include #include +#include #include -#include -#include -#include - +#include #include #include +#include +#include -#include "common.hpp" -#include +#include -#include +#include #include +#include namespace raft { namespace sparse { diff --git a/cpp/include/raft/sparse/distance/distance.cuh b/cpp/include/raft/sparse/distance/distance.cuh index 702846f586..ead44f0c51 100644 --- a/cpp/include/raft/sparse/distance/distance.cuh +++ b/cpp/include/raft/sparse/distance/distance.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,17 +20,16 @@ #pragma once #include "detail/common.hpp" -#include #include - #include - #include #include #include #include +#include + namespace raft { namespace sparse { namespace distance { diff --git a/cpp/include/raft/sparse/linalg/detail/add.cuh b/cpp/include/raft/sparse/linalg/detail/add.cuh index 72dd4a60f9..4be15693ef 100644 --- a/cpp/include/raft/sparse/linalg/detail/add.cuh +++ b/cpp/include/raft/sparse/linalg/detail/add.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,25 +16,24 @@ #pragma once -#include - #include +#include #include #include + #include #include +#include #include #include -#include +#include #include #include #include -#include - namespace raft { namespace sparse { namespace linalg { diff --git a/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp b/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp index b15614905b..c10c0de426 100644 --- a/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp +++ b/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp @@ -16,14 +16,14 @@ #pragma once -#include - #include #include #include #include #include +#include + namespace raft { namespace sparse { namespace linalg { diff --git a/cpp/include/raft/sparse/linalg/detail/degree.cuh b/cpp/include/raft/sparse/linalg/detail/degree.cuh index a68968ea97..df31192cf7 100644 --- a/cpp/include/raft/sparse/linalg/detail/degree.cuh +++ b/cpp/include/raft/sparse/linalg/detail/degree.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,14 @@ #pragma once +#include +#include #include #include #include -#include -#include -#include +#include namespace raft { namespace sparse { diff --git a/cpp/include/raft/sparse/linalg/detail/norm.cuh b/cpp/include/raft/sparse/linalg/detail/norm.cuh index d68e1e992f..3702111f83 100644 --- a/cpp/include/raft/sparse/linalg/detail/norm.cuh +++ b/cpp/include/raft/sparse/linalg/detail/norm.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,27 +16,25 @@ #pragma once -#include #include #include #include #include +#include +#include #include #include -#include - +#include #include #include -#include +#include #include #include #include -#include - namespace raft { namespace sparse { namespace linalg { diff --git a/cpp/include/raft/sparse/linalg/detail/spectral.cuh b/cpp/include/raft/sparse/linalg/detail/spectral.cuh index 51836ca9aa..a1642d1455 100644 --- a/cpp/include/raft/sparse/linalg/detail/spectral.cuh +++ b/cpp/include/raft/sparse/linalg/detail/spectral.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,15 @@ */ #include -#include - +#include +#include #include #include #include #include -#include +#include -#include -#include +#include namespace raft { namespace sparse { diff --git a/cpp/include/raft/sparse/linalg/detail/symmetrize.cuh b/cpp/include/raft/sparse/linalg/detail/symmetrize.cuh index 6febf0aadc..d343bcbf66 100644 --- a/cpp/include/raft/sparse/linalg/detail/symmetrize.cuh +++ b/cpp/include/raft/sparse/linalg/detail/symmetrize.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,31 +16,30 @@ #pragma once -#include #include - +#include +#include #include +#include +#include +#include #include #include +#include + #include #include -#include -#include +#include #include #include -#include +#include #include #include #include -#include -#include -#include -#include - namespace raft { namespace sparse { namespace linalg { diff --git a/cpp/include/raft/sparse/linalg/detail/transpose.h b/cpp/include/raft/sparse/linalg/detail/transpose.h index 1484804348..3a646b9a6e 100644 --- a/cpp/include/raft/sparse/linalg/detail/transpose.h +++ b/cpp/include/raft/sparse/linalg/detail/transpose.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,24 +16,23 @@ #pragma once -#include - #include +#include #include #include + #include +#include #include #include -#include +#include #include #include #include -#include - namespace raft { namespace sparse { namespace linalg { diff --git a/cpp/include/raft/sparse/neighbors/detail/cross_component_nn.cuh b/cpp/include/raft/sparse/neighbors/detail/cross_component_nn.cuh index 749152ee07..a47d5a6f34 100644 --- a/cpp/include/raft/sparse/neighbors/detail/cross_component_nn.cuh +++ b/cpp/include/raft/sparse/neighbors/detail/cross_component_nn.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,6 @@ */ #pragma once -#include -#include #include #include #include @@ -33,25 +31,24 @@ #include #include #include - #include #include + #include +#include #include #include +#include #include #include #include +#include #include #include #include -#include -#include - -#include - +#include #include namespace raft::sparse::neighbors::detail { diff --git a/cpp/include/raft/sparse/neighbors/detail/knn.cuh b/cpp/include/raft/sparse/neighbors/detail/knn.cuh index f9d019931a..68bba31360 100644 --- a/cpp/include/raft/sparse/neighbors/detail/knn.cuh +++ b/cpp/include/raft/sparse/neighbors/detail/knn.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,19 +17,18 @@ #pragma once #include -#include - #include #include -#include -#include - #include #include #include #include #include #include +#include +#include + +#include #include diff --git a/cpp/include/raft/sparse/neighbors/detail/knn_graph.cuh b/cpp/include/raft/sparse/neighbors/detail/knn_graph.cuh index 4344025065..4e46904c83 100644 --- a/cpp/include/raft/sparse/neighbors/detail/knn_graph.cuh +++ b/cpp/include/raft/sparse/neighbors/detail/knn_graph.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,15 +17,13 @@ #pragma once #include -#include -#include - +#include #include #include - #include +#include +#include -#include #include #include diff --git a/cpp/include/raft/sparse/op/detail/filter.cuh b/cpp/include/raft/sparse/op/detail/filter.cuh index aa375443f5..3df85e6871 100644 --- a/cpp/include/raft/sparse/op/detail/filter.cuh +++ b/cpp/include/raft/sparse/op/detail/filter.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,28 +16,27 @@ #pragma once -#include - +#include #include +#include +#include #include #include + #include #include +#include #include #include #include -#include +#include #include #include #include -#include -#include -#include - namespace raft { namespace sparse { namespace op { diff --git a/cpp/include/raft/sparse/op/detail/reduce.cuh b/cpp/include/raft/sparse/op/detail/reduce.cuh index 8a1a63e424..1e5dd87958 100644 --- a/cpp/include/raft/sparse/op/detail/reduce.cuh +++ b/cpp/include/raft/sparse/op/detail/reduce.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,30 +16,29 @@ #pragma once -#include #include #include - +#include +#include #include +#include +#include #include #include - -#include #include + +#include + +#include #include #include -#include -#include +#include #include #include #include -#include -#include -#include - namespace raft { namespace sparse { namespace op { diff --git a/cpp/include/raft/sparse/op/detail/row_op.cuh b/cpp/include/raft/sparse/op/detail/row_op.cuh index 256ae28f8c..73dbec6389 100644 --- a/cpp/include/raft/sparse/op/detail/row_op.cuh +++ b/cpp/include/raft/sparse/op/detail/row_op.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,23 +16,21 @@ #pragma once -#include - #include +#include #include #include +#include #include #include -#include +#include #include #include #include -#include - namespace raft { namespace sparse { namespace op { diff --git a/cpp/include/raft/sparse/op/detail/slice.cuh b/cpp/include/raft/sparse/op/detail/slice.cuh index 4d2f1a4195..c2d786c9fe 100644 --- a/cpp/include/raft/sparse/op/detail/slice.cuh +++ b/cpp/include/raft/sparse/op/detail/slice.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,25 +16,23 @@ #pragma once -#include - #include #include #include +#include #include #include +#include #include #include -#include +#include #include #include #include -#include - namespace raft { namespace sparse { namespace op { diff --git a/cpp/include/raft/sparse/op/detail/sort.h b/cpp/include/raft/sparse/op/detail/sort.h index 2f73671132..85ae825035 100644 --- a/cpp/include/raft/sparse/op/detail/sort.h +++ b/cpp/include/raft/sparse/op/detail/sort.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,8 +21,10 @@ #include #include #include + #include +#include #include #include #include @@ -31,8 +33,6 @@ #include -#include - #include namespace raft { diff --git a/cpp/include/raft/sparse/solver/detail/lanczos.cuh b/cpp/include/raft/sparse/solver/detail/lanczos.cuh index 74f8931a17..9ecb4b729f 100644 --- a/cpp/include/raft/sparse/solver/detail/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/detail/lanczos.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,14 +19,8 @@ // for cmath: #define _USE_MATH_DEFINES -#include #include #include -#include - -#include -#include - #include #include #include @@ -34,6 +28,13 @@ #include #include +#include + +#include + +#include +#include + namespace raft::sparse::solver::detail { // curandGeneratorNormalX diff --git a/cpp/include/raft/sparse/solver/detail/mst_kernels.cuh b/cpp/include/raft/sparse/solver/detail/mst_kernels.cuh index 5b56ea45f0..968eee4002 100644 --- a/cpp/include/raft/sparse/solver/detail/mst_kernels.cuh +++ b/cpp/include/raft/sparse/solver/detail/mst_kernels.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,11 +18,10 @@ #pragma once #include +#include #include -#include - namespace raft::sparse::solver::detail { template diff --git a/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh b/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh index 0182668117..7ce8871f06 100644 --- a/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh +++ b/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,10 @@ #pragma once -#include #include #include - #include #include - #include #include @@ -43,6 +40,8 @@ #include #include +#include + #include namespace raft::sparse::solver { diff --git a/cpp/include/raft/sparse/solver/detail/mst_utils.cuh b/cpp/include/raft/sparse/solver/detail/mst_utils.cuh index a33141192b..f375623791 100644 --- a/cpp/include/raft/sparse/solver/detail/mst_utils.cuh +++ b/cpp/include/raft/sparse/solver/detail/mst_utils.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,10 @@ #pragma once -#include #include +#include + namespace raft::sparse::solver::detail { template diff --git a/cpp/include/raft/sparse/solver/mst_solver.cuh b/cpp/include/raft/sparse/solver/mst_solver.cuh index 193431251f..8317e66d4a 100644 --- a/cpp/include/raft/sparse/solver/mst_solver.cuh +++ b/cpp/include/raft/sparse/solver/mst_solver.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ #pragma once #include + #include #include diff --git a/cpp/include/raft/spatial/knn/ann_common.h b/cpp/include/raft/spatial/knn/ann_common.h index 73a70cbefe..722c01d561 100644 --- a/cpp/include/raft/spatial/knn/ann_common.h +++ b/cpp/include/raft/spatial/knn/ann_common.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,9 +23,9 @@ #include "detail/processing.hpp" #include "ivf_flat_types.hpp" -#include #include +#include namespace raft { namespace spatial { diff --git a/cpp/include/raft/spatial/knn/detail/ann_quantized.cuh b/cpp/include/raft/spatial/knn/detail/ann_quantized.cuh index 964292f6cb..041ab225f9 100644 --- a/cpp/include/raft/spatial/knn/detail/ann_quantized.cuh +++ b/cpp/include/raft/spatial/knn/detail/ann_quantized.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,19 +18,17 @@ #include "../ann_common.h" #include "../ivf_flat.cuh" -#include - #include "processing.cuh" -#include -#include -#include +#include +#include +#include #include #include #include #include - -#include +#include +#include #include diff --git a/cpp/include/raft/spatial/knn/detail/ann_utils.cuh b/cpp/include/raft/spatial/knn/detail/ann_utils.cuh index d75c7bf531..d862e586e3 100644 --- a/cpp/include/raft/spatial/knn/detail/ann_utils.cuh +++ b/cpp/include/raft/spatial/knn/detail/ann_utils.cuh @@ -26,11 +26,11 @@ #include #include +#include + #include #include -#include - namespace raft::spatial::knn::detail::utils { /** Whether pointers are accessible on the device or on the host. */ diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover.cuh index 879f54fd81..5c91b1ad6d 100644 --- a/cpp/include/raft/spatial/knn/detail/ball_cover.cuh +++ b/cpp/include/raft/spatial/knn/detail/ball_cover.cuh @@ -16,27 +16,21 @@ #pragma once -#include -#include -#include - #include "../ball_cover_types.hpp" #include "ball_cover/common.cuh" #include "ball_cover/registers.cuh" #include "haversine_distance.cuh" -#include -#include - -#include - -#include - +#include +#include +#include #include #include #include +#include #include #include +#include #include #include @@ -51,6 +45,10 @@ #include #include +#include + +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover/common.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover/common.cuh index ce72b2648f..00c34b851d 100644 --- a/cpp/include/raft/spatial/knn/detail/ball_cover/common.cuh +++ b/cpp/include/raft/spatial/knn/detail/ball_cover/common.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,10 +18,12 @@ #include "../haversine_distance.cuh" #include "registers_types.cuh" -#include + #include #include +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh index 2ed6ee3284..81500a0eae 100644 --- a/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh +++ b/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh @@ -16,11 +16,13 @@ #pragma once -#include "../../ball_cover_types.hpp" // BallCoverIndex -#include "registers_types.cuh" // DistFunc -#include // uint32_t +#include "../../ball_cover_types.hpp" // BallCoverIndex +#include "registers_types.cuh" // DistFunc + #include //RAFT_EXPLICIT +#include // uint32_t + #if defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) namespace raft::spatial::knn::detail { diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover/registers-inl.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover/registers-inl.cuh index 8b4e8f287e..e4acd1dc4c 100644 --- a/cpp/include/raft/spatial/knn/detail/ball_cover/registers-inl.cuh +++ b/cpp/include/raft/spatial/knn/detail/ball_cover/registers-inl.cuh @@ -16,17 +16,13 @@ #pragma once -#include "common.cuh" -#include -#include - #include "../../ball_cover_types.hpp" #include "../haversine_distance.cuh" +#include "common.cuh" #include "registers_types.cuh" // DistFunc -#include -#include - +#include +#include #include #include @@ -34,6 +30,10 @@ #include #include +#include + +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover/registers_types.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover/registers_types.cuh index 7f4268d2dc..f38a3eeec9 100644 --- a/cpp/include/raft/spatial/knn/detail/ball_cover/registers_types.cuh +++ b/cpp/include/raft/spatial/knn/detail/ball_cover/registers_types.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,8 @@ #pragma once #include "../haversine_distance.cuh" // compute_haversine -#include // uint32_t + +#include // uint32_t namespace raft { namespace spatial { diff --git a/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh b/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh index 0eca119450..1968bdb205 100644 --- a/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh +++ b/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh @@ -15,11 +15,12 @@ */ #pragma once -#include // size_t -#include // uint32_t #include // DistanceType #include // RAFT_EXPLICIT +#include // size_t +#include // uint32_t + #if defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) namespace raft::spatial::knn::detail { diff --git a/cpp/include/raft/spatial/knn/detail/fused_l2_knn-inl.cuh b/cpp/include/raft/spatial/knn/detail/fused_l2_knn-inl.cuh index 0c9f0059f9..ab64614e68 100644 --- a/cpp/include/raft/spatial/knn/detail/fused_l2_knn-inl.cuh +++ b/cpp/include/raft/spatial/knn/detail/fused_l2_knn-inl.cuh @@ -14,12 +14,15 @@ * limitations under the License. */ #pragma once -#include -#include #include #include + +#include + +#include // TODO: Need to hide the PairwiseDistance class impl and expose to public API #include "processing.cuh" + #include #include #include diff --git a/cpp/include/raft/spatial/knn/detail/haversine_distance.cuh b/cpp/include/raft/spatial/knn/detail/haversine_distance.cuh index 5fb912843d..98d0919906 100644 --- a/cpp/include/raft/spatial/knn/detail/haversine_distance.cuh +++ b/cpp/include/raft/spatial/knn/detail/haversine_distance.cuh @@ -16,13 +16,12 @@ #pragma once -#include -#include -#include - #include #include #include +#include +#include +#include namespace raft { namespace spatial { diff --git a/cpp/include/raft/spatial/knn/detail/processing.cuh b/cpp/include/raft/spatial/knn/detail/processing.cuh index b4b1cb2c14..072e4ebd90 100644 --- a/cpp/include/raft/spatial/knn/detail/processing.cuh +++ b/cpp/include/raft/spatial/knn/detail/processing.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ #include #include #include + #include namespace raft { diff --git a/cpp/include/raft/spectral/cluster_solvers.cuh b/cpp/include/raft/spectral/cluster_solvers.cuh index ecd14ebf42..b693ac4af3 100644 --- a/cpp/include/raft/spectral/cluster_solvers.cuh +++ b/cpp/include/raft/spectral/cluster_solvers.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ #include #include + #include // for std::pair namespace raft { diff --git a/cpp/include/raft/spectral/cluster_solvers_deprecated.cuh b/cpp/include/raft/spectral/cluster_solvers_deprecated.cuh index 008f1e5d13..40b0324548 100644 --- a/cpp/include/raft/spectral/cluster_solvers_deprecated.cuh +++ b/cpp/include/raft/spectral/cluster_solvers_deprecated.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ #pragma once #include + #include // for std::pair namespace raft { diff --git a/cpp/include/raft/spectral/detail/lapack.hpp b/cpp/include/raft/spectral/detail/lapack.hpp index 2d9e5ae9a4..203e6e50f7 100644 --- a/cpp/include/raft/spectral/detail/lapack.hpp +++ b/cpp/include/raft/spectral/detail/lapack.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,12 @@ */ #pragma once -#include - #include #include #include +#include + // for now; TODO: check if/where this `define` should be; // #define USE_LAPACK diff --git a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp index 6de0dced8a..30dd6e5e69 100644 --- a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp +++ b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,15 +23,15 @@ #include #include #include + #include +#include #include #include #include #include -#include - #include // ========================================================= diff --git a/cpp/include/raft/spectral/detail/modularity_maximization.hpp b/cpp/include/raft/spectral/detail/modularity_maximization.hpp index f60fbf863e..2a3b5cf36c 100644 --- a/cpp/include/raft/spectral/detail/modularity_maximization.hpp +++ b/cpp/include/raft/spectral/detail/modularity_maximization.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,23 +16,23 @@ #pragma once -#include #include #include -#include +#include +#include +#include +#include +#include #include #include #include #include -#include +#include +#include -#include -#include -#include -#include -#include +#include namespace raft { namespace spectral { diff --git a/cpp/include/raft/spectral/detail/partition.hpp b/cpp/include/raft/spectral/detail/partition.hpp index ed59ad3149..f5fc40aad6 100644 --- a/cpp/include/raft/spectral/detail/partition.hpp +++ b/cpp/include/raft/spectral/detail/partition.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,23 +15,23 @@ */ #pragma once -#include #include #include -#include +#include +#include +#include +#include +#include #include #include #include #include -#include +#include +#include -#include -#include -#include -#include -#include +#include namespace raft { namespace spectral { diff --git a/cpp/include/raft/spectral/detail/warn_dbg.hpp b/cpp/include/raft/spectral/detail/warn_dbg.hpp index d714e1dc86..26453e4398 100644 --- a/cpp/include/raft/spectral/detail/warn_dbg.hpp +++ b/cpp/include/raft/spectral/detail/warn_dbg.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,11 @@ */ #pragma once +#include + #include #include -#include - #ifdef DEBUG #define COUT() (std::cout) #define CERR() (std::cerr) diff --git a/cpp/include/raft/spectral/modularity_maximization.cuh b/cpp/include/raft/spectral/modularity_maximization.cuh index 7d4f42f952..ab1398a2a1 100644 --- a/cpp/include/raft/spectral/modularity_maximization.cuh +++ b/cpp/include/raft/spectral/modularity_maximization.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,10 +18,10 @@ #pragma once -#include - #include +#include + namespace raft { namespace spectral { diff --git a/cpp/include/raft/spectral/partition.cuh b/cpp/include/raft/spectral/partition.cuh index b7289aae3d..f7ea456ac5 100644 --- a/cpp/include/raft/spectral/partition.cuh +++ b/cpp/include/raft/spectral/partition.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ #pragma once -#include - #include +#include + namespace raft { namespace spectral { diff --git a/cpp/include/raft/stats/detail/adjusted_rand_index.cuh b/cpp/include/raft/stats/detail/adjusted_rand_index.cuh index 120d4d1686..b108697e51 100644 --- a/cpp/include/raft/stats/detail/adjusted_rand_index.cuh +++ b/cpp/include/raft/stats/detail/adjusted_rand_index.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,20 +23,23 @@ #pragma once #include "contingencyMatrix.cuh" -#include -#include + #include #include #include #include #include + #include #include +#include #include #include #include +#include + namespace raft { namespace stats { namespace detail { diff --git a/cpp/include/raft/stats/detail/batched/silhouette_score.cuh b/cpp/include/raft/stats/detail/batched/silhouette_score.cuh index 559532b0de..643ef77500 100644 --- a/cpp/include/raft/stats/detail/batched/silhouette_score.cuh +++ b/cpp/include/raft/stats/detail/batched/silhouette_score.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,16 @@ #pragma once #include "../silhouette_score.cuh" + #include #include #include #include #include + #include #include + #include #include #include diff --git a/cpp/include/raft/stats/detail/contingencyMatrix.cuh b/cpp/include/raft/stats/detail/contingencyMatrix.cuh index 9f19b798ea..afc4f25a66 100644 --- a/cpp/include/raft/stats/detail/contingencyMatrix.cuh +++ b/cpp/include/raft/stats/detail/contingencyMatrix.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,13 +19,12 @@ #include #include +#include #include #include #include #include -#include - #include namespace raft { diff --git a/cpp/include/raft/stats/detail/dispersion.cuh b/cpp/include/raft/stats/detail/dispersion.cuh index d873fcf79a..07b7daf678 100644 --- a/cpp/include/raft/stats/detail/dispersion.cuh +++ b/cpp/include/raft/stats/detail/dispersion.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,17 @@ #pragma once -#include -#include #include #include #include #include + #include +#include + +#include + namespace raft { namespace stats { namespace detail { diff --git a/cpp/include/raft/stats/detail/entropy.cuh b/cpp/include/raft/stats/detail/entropy.cuh index fc4fc5fb6b..4a73cd22a8 100644 --- a/cpp/include/raft/stats/detail/entropy.cuh +++ b/cpp/include/raft/stats/detail/entropy.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,15 +20,18 @@ */ #pragma once -#include -#include #include #include #include #include + #include #include +#include + +#include + namespace raft { namespace stats { namespace detail { diff --git a/cpp/include/raft/stats/detail/histogram.cuh b/cpp/include/raft/stats/detail/histogram.cuh index dee61cf1eb..68c3b03dcc 100644 --- a/cpp/include/raft/stats/detail/histogram.cuh +++ b/cpp/include/raft/stats/detail/histogram.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ #include #include #include + #include // This file is a shameless amalgamation of independent works done by diff --git a/cpp/include/raft/stats/detail/kl_divergence.cuh b/cpp/include/raft/stats/detail/kl_divergence.cuh index 6d9cc41790..14ff9f7459 100644 --- a/cpp/include/raft/stats/detail/kl_divergence.cuh +++ b/cpp/include/raft/stats/detail/kl_divergence.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,12 +21,14 @@ #pragma once -#include #include #include #include + #include +#include + namespace raft { namespace stats { namespace detail { diff --git a/cpp/include/raft/stats/detail/mutual_info_score.cuh b/cpp/include/raft/stats/detail/mutual_info_score.cuh index cc80d6c752..c67c5b0c6e 100644 --- a/cpp/include/raft/stats/detail/mutual_info_score.cuh +++ b/cpp/include/raft/stats/detail/mutual_info_score.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,16 +25,19 @@ */ #pragma once -#include -#include #include #include #include #include #include + #include #include +#include + +#include + namespace raft { namespace stats { namespace detail { diff --git a/cpp/include/raft/stats/detail/neighborhood_recall.cuh b/cpp/include/raft/stats/detail/neighborhood_recall.cuh index 3f05225e74..fe3a3f6ece 100644 --- a/cpp/include/raft/stats/detail/neighborhood_recall.cuh +++ b/cpp/include/raft/stats/detail/neighborhood_recall.cuh @@ -16,7 +16,6 @@ #pragma once -#include #include #include #include @@ -27,9 +26,9 @@ #include #include - #include +#include #include namespace raft::stats::detail { diff --git a/cpp/include/raft/stats/detail/rand_index.cuh b/cpp/include/raft/stats/detail/rand_index.cuh index 744e8dd8e8..4747547e35 100644 --- a/cpp/include/raft/stats/detail/rand_index.cuh +++ b/cpp/include/raft/stats/detail/rand_index.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,13 +52,16 @@ #pragma once -#include -#include #include #include #include + #include +#include + +#include + namespace raft { namespace stats { namespace detail { diff --git a/cpp/include/raft/stats/detail/scores.cuh b/cpp/include/raft/stats/detail/scores.cuh index ae592289ac..947df6848a 100644 --- a/cpp/include/raft/stats/detail/scores.cuh +++ b/cpp/include/raft/stats/detail/scores.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ #pragma once -#include #include #include #include @@ -24,13 +23,17 @@ #include #include #include + #include #include + #include #include #include #include +#include + #define N_THREADS 512 namespace raft { diff --git a/cpp/include/raft/stats/detail/silhouette_score.cuh b/cpp/include/raft/stats/detail/silhouette_score.cuh index 889a3f5931..4285f84fcc 100644 --- a/cpp/include/raft/stats/detail/silhouette_score.cuh +++ b/cpp/include/raft/stats/detail/silhouette_score.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,6 @@ #pragma once -#include -#include -#include -#include -#include #include #include #include @@ -32,8 +27,17 @@ #include #include #include + #include +#include + +#include + +#include +#include +#include + namespace raft { namespace stats { namespace detail { diff --git a/cpp/include/raft/stats/detail/trustworthiness_score.cuh b/cpp/include/raft/stats/detail/trustworthiness_score.cuh index ef37b2a034..8b9e5c2cc8 100644 --- a/cpp/include/raft/stats/detail/trustworthiness_score.cuh +++ b/cpp/include/raft/stats/detail/trustworthiness_score.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ #include #include #include + #include #include diff --git a/cpp/include/raft/stats/dispersion.cuh b/cpp/include/raft/stats/dispersion.cuh index 216f637c94..ded7c8178b 100644 --- a/cpp/include/raft/stats/dispersion.cuh +++ b/cpp/include/raft/stats/dispersion.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,11 +19,12 @@ #pragma once -#include #include #include #include +#include + namespace raft { namespace stats { diff --git a/cpp/include/raft/stats/minmax.cuh b/cpp/include/raft/stats/minmax.cuh index 5c5ff346a4..d2c410dab1 100644 --- a/cpp/include/raft/stats/minmax.cuh +++ b/cpp/include/raft/stats/minmax.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ #pragma once -#include #include #include #include @@ -26,6 +25,7 @@ #include #include +#include namespace raft { namespace stats { diff --git a/cpp/include/raft/util/cache.cuh b/cpp/include/raft/util/cache.cuh index 255dea16bb..902a1f9ae4 100644 --- a/cpp/include/raft/util/cache.cuh +++ b/cpp/include/raft/util/cache.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,16 +16,17 @@ #pragma once -#include - #include #include #include #include #include + #include #include +#include + #include namespace raft::cache { diff --git a/cpp/include/raft/util/cache_util.cuh b/cpp/include/raft/util/cache_util.cuh index 9f081bb992..df79f8e78b 100644 --- a/cpp/include/raft/util/cache_util.cuh +++ b/cpp/include/raft/util/cache_util.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,10 @@ #pragma once -#include #include +#include + namespace raft { namespace cache { diff --git a/cpp/include/raft/util/cuda_data_type.hpp b/cpp/include/raft/util/cuda_data_type.hpp index acf49730bd..5d7d36405b 100644 --- a/cpp/include/raft/util/cuda_data_type.hpp +++ b/cpp/include/raft/util/cuda_data_type.hpp @@ -15,10 +15,12 @@ */ #pragma once -#include #include + #include +#include + namespace raft { template diff --git a/cpp/include/raft/util/cuda_rt_essentials.hpp b/cpp/include/raft/util/cuda_rt_essentials.hpp index 77612f97bc..6b221d6700 100644 --- a/cpp/include/raft/util/cuda_rt_essentials.hpp +++ b/cpp/include/raft/util/cuda_rt_essentials.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,9 +20,10 @@ // The scope is necessarily limited to ensure that compilation times are // minimized. Please make sure not to include large / expensive files from here. -#include #include +#include + #include namespace raft { diff --git a/cpp/include/raft/util/cuda_utils.cuh b/cpp/include/raft/util/cuda_utils.cuh index bf46e069e4..08df368bdf 100644 --- a/cpp/include/raft/util/cuda_utils.cuh +++ b/cpp/include/raft/util/cuda_utils.cuh @@ -16,15 +16,17 @@ #pragma once +#include +#include +#include + #include #include #include + #include -#include -#include -#include -#include +#include // For backward compatibility, we include the follow headers. They contain // functionality that were previously contained in cuda_utils.cuh #include diff --git a/cpp/include/raft/util/cudart_utils.hpp b/cpp/include/raft/util/cudart_utils.hpp index 743ffd743c..e5ce15e8a3 100644 --- a/cpp/include/raft/util/cudart_utils.hpp +++ b/cpp/include/raft/util/cudart_utils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,9 +25,10 @@ #include #include +#include + #include #include -#include #include #include #include diff --git a/cpp/include/raft/util/cutlass_utils.cuh b/cpp/include/raft/util/cutlass_utils.cuh index da402c9427..36b9c5de8e 100644 --- a/cpp/include/raft/util/cutlass_utils.cuh +++ b/cpp/include/raft/util/cutlass_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,10 @@ #pragma once -#include #include +#include + namespace raft { /** diff --git a/cpp/include/raft/util/detail/cub_wrappers.cuh b/cpp/include/raft/util/detail/cub_wrappers.cuh index 0ce749d9c8..0327d3f9ab 100644 --- a/cpp/include/raft/util/detail/cub_wrappers.cuh +++ b/cpp/include/raft/util/detail/cub_wrappers.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,10 @@ #pragma once -#include #include +#include + namespace raft { /** diff --git a/cpp/include/raft/util/device_atomics.cuh b/cpp/include/raft/util/device_atomics.cuh index 14856bed8e..6fc6f42ed3 100644 --- a/cpp/include/raft/util/device_atomics.cuh +++ b/cpp/include/raft/util/device_atomics.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ */ #include + #include namespace raft { diff --git a/cpp/include/raft/util/device_loads_stores.cuh b/cpp/include/raft/util/device_loads_stores.cuh index 780939ee85..2c954ec99a 100644 --- a/cpp/include/raft/util/device_loads_stores.cuh +++ b/cpp/include/raft/util/device_loads_stores.cuh @@ -16,11 +16,13 @@ #pragma once -#include // uintX_t -#include #include #include // DI +#include + +#include // uintX_t + namespace raft { /** diff --git a/cpp/include/raft/util/device_utils.cuh b/cpp/include/raft/util/device_utils.cuh index 757c60a731..e524325a52 100644 --- a/cpp/include/raft/util/device_utils.cuh +++ b/cpp/include/raft/util/device_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include + #include // pair namespace raft { diff --git a/cpp/include/raft/util/fast_int_div.cuh b/cpp/include/raft/util/fast_int_div.cuh index a0cb8f0f53..8ff850b24b 100644 --- a/cpp/include/raft/util/fast_int_div.cuh +++ b/cpp/include/raft/util/fast_int_div.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include + #include namespace raft::util { diff --git a/cpp/include/raft/util/integer_utils.hpp b/cpp/include/raft/util/integer_utils.hpp index e9a386c426..5224d5ac4c 100644 --- a/cpp/include/raft/util/integer_utils.hpp +++ b/cpp/include/raft/util/integer_utils.hpp @@ -1,7 +1,7 @@ /* * Copyright 2019 BlazingDB, Inc. * Copyright 2019 Eyal Rozenberg - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +23,9 @@ * */ -#include #include + +#include #include #include diff --git a/cpp/include/raft/util/memory_type_dispatcher.cuh b/cpp/include/raft/util/memory_type_dispatcher.cuh index 94d838415a..919e4b3484 100644 --- a/cpp/include/raft/util/memory_type_dispatcher.cuh +++ b/cpp/include/raft/util/memory_type_dispatcher.cuh @@ -18,6 +18,7 @@ #include #include #include + #include #include diff --git a/cpp/include/raft/util/reduction.cuh b/cpp/include/raft/util/reduction.cuh index 362396f9b8..7e897e67f5 100644 --- a/cpp/include/raft/util/reduction.cuh +++ b/cpp/include/raft/util/reduction.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,13 @@ #pragma once -#include - #include #include #include #include +#include + namespace raft { /** diff --git a/cpp/include/raft/util/seive.hpp b/cpp/include/raft/util/seive.hpp index ab7c77ac85..509a04c4e6 100644 --- a/cpp/include/raft/util/seive.hpp +++ b/cpp/include/raft/util/seive.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #pragma once #include + #include // Taken from: diff --git a/cpp/include/raft/util/vectorized.cuh b/cpp/include/raft/util/vectorized.cuh index 8af2e498e9..8c56554a13 100644 --- a/cpp/include/raft/util/vectorized.cuh +++ b/cpp/include/raft/util/vectorized.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,10 @@ #pragma once -#include #include +#include + namespace raft { template diff --git a/cpp/include/raft/util/warp_primitives.cuh b/cpp/include/raft/util/warp_primitives.cuh index 94fddbe0f3..953c137cdf 100644 --- a/cpp/include/raft/util/warp_primitives.cuh +++ b/cpp/include/raft/util/warp_primitives.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,12 @@ #pragma once -#include - #include #include #include +#include + namespace raft { /** True CUDA alignment of a type (adapted from CUB) */ diff --git a/cpp/include/raft_runtime/cluster/kmeans.hpp b/cpp/include/raft_runtime/cluster/kmeans.hpp index d47faf1510..dcc42fe9d3 100644 --- a/cpp/include/raft_runtime/cluster/kmeans.hpp +++ b/cpp/include/raft_runtime/cluster/kmeans.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,12 @@ * limitations under the License. */ +#include #include #include #include #include -#include - namespace raft::runtime::cluster::kmeans { /** diff --git a/cpp/include/raft_runtime/neighbors/cagra.hpp b/cpp/include/raft_runtime/neighbors/cagra.hpp index 8389929b15..57cb157fbd 100644 --- a/cpp/include/raft_runtime/neighbors/cagra.hpp +++ b/cpp/include/raft_runtime/neighbors/cagra.hpp @@ -16,13 +16,13 @@ #pragma once -#include -#include -#include - #include #include #include +#include +#include + +#include namespace raft::runtime::neighbors::cagra { diff --git a/cpp/include/raft_runtime/neighbors/ivf_flat.hpp b/cpp/include/raft_runtime/neighbors/ivf_flat.hpp index 5b8918ec7f..4f65bd6fef 100644 --- a/cpp/include/raft_runtime/neighbors/ivf_flat.hpp +++ b/cpp/include/raft_runtime/neighbors/ivf_flat.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include + #include namespace raft::runtime::neighbors::ivf_flat { diff --git a/cpp/include/raft_runtime/random/rmat_rectangular_generator.hpp b/cpp/include/raft_runtime/random/rmat_rectangular_generator.hpp index bf5259aeba..dc9ba9ba7b 100644 --- a/cpp/include/raft_runtime/random/rmat_rectangular_generator.hpp +++ b/cpp/include/raft_runtime/random/rmat_rectangular_generator.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,11 @@ * limitations under the License. */ -#include - #include #include +#include + namespace raft::runtime::random { /** diff --git a/cpp/internal/raft_internal/neighbors/naive_knn.cuh b/cpp/internal/raft_internal/neighbors/naive_knn.cuh index 35d533316f..79206c7a43 100644 --- a/cpp/internal/raft_internal/neighbors/naive_knn.cuh +++ b/cpp/internal/raft_internal/neighbors/naive_knn.cuh @@ -23,7 +23,6 @@ #include #include -#include #include #include #include diff --git a/cpp/internal/raft_internal/neighbors/refine_helper.cuh b/cpp/internal/raft_internal/neighbors/refine_helper.cuh index 4a06116877..665ec23d8e 100644 --- a/cpp/internal/raft_internal/neighbors/refine_helper.cuh +++ b/cpp/internal/raft_internal/neighbors/refine_helper.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,17 +15,17 @@ */ #pragma once -#include -#include - #include #include #include #include +#include #include #include #include +#include + #include #include diff --git a/cpp/src/distance/fused_l2_nn.cu b/cpp/src/distance/fused_l2_nn.cu index 6011aaec29..f29ab08dc1 100644 --- a/cpp/src/distance/fused_l2_nn.cu +++ b/cpp/src/distance/fused_l2_nn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,11 @@ * limitations under the License. */ -#include // int64_t #include // raft::KeyValuePair #include +#include // int64_t + #define instantiate_raft_distance_fusedL2NNMinReduce(DataT, OutT, IdxT) \ template void raft::distance::fusedL2NNMinReduce(OutT * min, \ const DataT* x, \ diff --git a/cpp/src/matrix/detail/select_k_double_uint32_t.cu b/cpp/src/matrix/detail/select_k_double_uint32_t.cu index 9aa4e957af..21c954ca46 100644 --- a/cpp/src/matrix/detail/select_k_double_uint32_t.cu +++ b/cpp/src/matrix/detail/select_k_double_uint32_t.cu @@ -14,9 +14,10 @@ * limitations under the License. */ -#include // uint32_t #include +#include // uint32_t + #define instantiate_raft_matrix_detail_select_k(T, IdxT) \ template void raft::matrix::detail::select_k(raft::resources const& handle, \ const T* in_val, \ diff --git a/cpp/src/neighbors/ball_cover.cu b/cpp/src/neighbors/ball_cover.cu index 0a59060c8e..e9c78f8e7c 100644 --- a/cpp/src/neighbors/ball_cover.cu +++ b/cpp/src/neighbors/ball_cover.cu @@ -14,9 +14,10 @@ * limitations under the License. */ -#include #include +#include + #define instantiate_raft_neighbors_ball_cover(idx_t, value_t, int_t, matrix_idx_t) \ template void raft::neighbors::ball_cover::build_index( \ raft::resources const& handle, \ diff --git a/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu b/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu index d482d24772..4269d27225 100644 --- a/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu +++ b/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu @@ -1,6 +1,6 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,10 @@ * */ -#include #include +#include + #define instantiate_raft_neighbors_brute_force_fused_l2_knn( \ value_t, idx_t, idx_layout, query_layout) \ template void raft::neighbors::brute_force::fused_l2_knn( \ diff --git a/cpp/src/neighbors/brute_force_knn_index_float.cu b/cpp/src/neighbors/brute_force_knn_index_float.cu index faf99ceb3c..de94be4c09 100644 --- a/cpp/src/neighbors/brute_force_knn_index_float.cu +++ b/cpp/src/neighbors/brute_force_knn_index_float.cu @@ -1,6 +1,6 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,13 @@ * limitations under the License. */ -#include #include #include #include #include +#include + template void raft::neighbors::brute_force::search( raft::resources const& res, const raft::neighbors::brute_force::index& idx, diff --git a/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu b/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu index a05b387dae..1c08cf8e82 100644 --- a/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu +++ b/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu @@ -1,6 +1,6 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,10 @@ * */ -#include #include +#include + #define instantiate_raft_neighbors_brute_force_knn( \ idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ template void raft::neighbors::brute_force:: \ diff --git a/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu b/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu index d1b9a4bb43..809cf6eec0 100644 --- a/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu +++ b/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu @@ -1,6 +1,6 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,10 @@ * */ -#include #include +#include + #define instantiate_raft_neighbors_brute_force_knn( \ idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ template void raft::neighbors::brute_force:: \ diff --git a/cpp/src/neighbors/brute_force_knn_int_float_int.cu b/cpp/src/neighbors/brute_force_knn_int_float_int.cu index 354f72e3f5..2ffa864dea 100644 --- a/cpp/src/neighbors/brute_force_knn_int_float_int.cu +++ b/cpp/src/neighbors/brute_force_knn_int_float_int.cu @@ -1,6 +1,6 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,10 @@ * */ -#include #include +#include + #define instantiate_raft_neighbors_brute_force_knn( \ idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ template void raft::neighbors::brute_force:: \ diff --git a/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu b/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu index 2f6bd9b888..dde92765b5 100644 --- a/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu +++ b/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu @@ -1,6 +1,6 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,10 @@ * */ -#include #include +#include + #define instantiate_raft_neighbors_brute_force_knn( \ idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ template void raft::neighbors::brute_force:: \ diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu index a1d6cca7d5..def33e493e 100644 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu +++ b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,8 @@ const raft::distance::DistanceType metric, \ const uint32_t n_probes, \ const uint32_t k, \ + const uint32_t max_samples, \ + const uint32_t* chunk_indices, \ const bool select_min, \ IvfSampleFilterT sample_filter, \ IdxT* neighbors, \ diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu index 3c467a12d8..e96600ee02 100644 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu +++ b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu @@ -31,6 +31,8 @@ const raft::distance::DistanceType metric, \ const uint32_t n_probes, \ const uint32_t k, \ + const uint32_t max_samples, \ + const uint32_t* chunk_indices, \ const bool select_min, \ IvfSampleFilterT sample_filter, \ IdxT* neighbors, \ diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu index 514301562d..13c9d2e283 100644 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu +++ b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,8 @@ const raft::distance::DistanceType metric, \ const uint32_t n_probes, \ const uint32_t k, \ + const uint32_t max_samples, \ + const uint32_t* chunk_indices, \ const bool select_min, \ IvfSampleFilterT sample_filter, \ IdxT* neighbors, \ diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu index 32698a8e80..51f02343fc 100644 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu +++ b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,8 @@ const raft::distance::DistanceType metric, \ const uint32_t n_probes, \ const uint32_t k, \ + const uint32_t max_samples, \ + const uint32_t* chunk_indices, \ const bool select_min, \ IvfSampleFilterT sample_filter, \ IdxT* neighbors, \ diff --git a/cpp/src/raft_runtime/cluster/cluster_cost_double.cu b/cpp/src/raft_runtime/cluster/cluster_cost_double.cu index ab0299c5d0..abe07c50e7 100644 --- a/cpp/src/raft_runtime/cluster/cluster_cost_double.cu +++ b/cpp/src/raft_runtime/cluster/cluster_cost_double.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "cluster_cost.cuh" + #include #include diff --git a/cpp/src/raft_runtime/cluster/cluster_cost_float.cu b/cpp/src/raft_runtime/cluster/cluster_cost_float.cu index 22cfa1b0ff..1634788b62 100644 --- a/cpp/src/raft_runtime/cluster/cluster_cost_float.cu +++ b/cpp/src/raft_runtime/cluster/cluster_cost_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "cluster_cost.cuh" + #include #include diff --git a/cpp/src/raft_runtime/cluster/update_centroids_double.cu b/cpp/src/raft_runtime/cluster/update_centroids_double.cu index 18f618eed0..0bc6e33273 100644 --- a/cpp/src/raft_runtime/cluster/update_centroids_double.cu +++ b/cpp/src/raft_runtime/cluster/update_centroids_double.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "update_centroids.cuh" + #include #include diff --git a/cpp/src/raft_runtime/cluster/update_centroids_float.cu b/cpp/src/raft_runtime/cluster/update_centroids_float.cu index 31bb42a661..3f98e3fb38 100644 --- a/cpp/src/raft_runtime/cluster/update_centroids_float.cu +++ b/cpp/src/raft_runtime/cluster/update_centroids_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "update_centroids.cuh" + #include #include diff --git a/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu b/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu index 82a225aca4..f421b2fe39 100644 --- a/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu +++ b/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu @@ -23,6 +23,7 @@ #include #include #include + #include #include diff --git a/cpp/src/raft_runtime/neighbors/cagra_build.cu b/cpp/src/raft_runtime/neighbors/cagra_build.cu index 5fc472c952..9268551b8c 100644 --- a/cpp/src/raft_runtime/neighbors/cagra_build.cu +++ b/cpp/src/raft_runtime/neighbors/cagra_build.cu @@ -17,6 +17,7 @@ #include #include #include + #include #include diff --git a/cpp/src/raft_runtime/neighbors/cagra_search.cu b/cpp/src/raft_runtime/neighbors/cagra_search.cu index 93244077f4..b550d2e521 100644 --- a/cpp/src/raft_runtime/neighbors/cagra_search.cu +++ b/cpp/src/raft_runtime/neighbors/cagra_search.cu @@ -15,6 +15,7 @@ */ #include + #include #include diff --git a/cpp/src/raft_runtime/neighbors/cagra_serialize.cu b/cpp/src/raft_runtime/neighbors/cagra_serialize.cu index f386bcce8e..0677575e23 100644 --- a/cpp/src/raft_runtime/neighbors/cagra_serialize.cu +++ b/cpp/src/raft_runtime/neighbors/cagra_serialize.cu @@ -14,16 +14,17 @@ * limitations under the License. */ -#include -#include - #include #include #include + #include #include +#include +#include + namespace raft::runtime::neighbors::cagra { #define RAFT_INST_CAGRA_SERIALIZE(DTYPE) \ diff --git a/cpp/src/raft_runtime/neighbors/hnsw.cpp b/cpp/src/raft_runtime/neighbors/hnsw.cpp index 1f9e6b0a0b..6eb770abd6 100644 --- a/cpp/src/raft_runtime/neighbors/hnsw.cpp +++ b/cpp/src/raft_runtime/neighbors/hnsw.cpp @@ -14,13 +14,14 @@ * limitations under the License. */ -#include #include #include #include #include +#include + namespace raft::neighbors::hnsw { #define RAFT_INST_HNSW(T) \ template <> \ diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu index 7fccb95411..b0c4ec3a49 100644 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu +++ b/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include namespace raft::runtime::neighbors::ivf_flat { diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu index 7a156a4e02..6f26e8fc5f 100644 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu +++ b/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include namespace raft::runtime::neighbors::ivf_flat { diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu index 049b8b00da..576112f0a4 100644 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu +++ b/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ * limitations under the License. */ -#include -#include - #include #include #include + #include +#include +#include + namespace raft::runtime::neighbors::ivf_flat { #define RAFT_IVF_FLAT_SERIALIZE_INST(DTYPE) \ diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_build.cu b/cpp/src/raft_runtime/neighbors/ivfpq_build.cu index 65a29429bc..d5c6cd4d28 100644 --- a/cpp/src/raft_runtime/neighbors/ivfpq_build.cu +++ b/cpp/src/raft_runtime/neighbors/ivfpq_build.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include + #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/src/raft_runtime/random/common.cuh b/cpp/src/raft_runtime/random/common.cuh index f5f8a7c0ae..06309d2037 100644 --- a/cpp/src/raft_runtime/random/common.cuh +++ b/cpp/src/raft_runtime/random/common.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #include #include + #include #define FUNC_DEF(IdxT, ProbT) \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu index 0d09f88b65..bbc1b55eb2 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdist_func) \ template void \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu index 3681acf245..8c46888e05 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu index 3fa20779b7..c5a6970a94 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu index 7abc89cc11..19eb4b681b 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu index 6251a86867..6f878e1296 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu index 07b97ac718..e8fbcd0528 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu index 4fc18184b0..57b8c790ef 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu index 882496c7d9..6e1cc1e35e 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu index 0a736d7e13..d1123aff7d 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu index 23aff93966..3e118e2f8f 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu index d3ec2b4c65..674f532e6c 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu index dd9f0e4658..dd3a422baa 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu index 849bbf0f96..0b05ca91a9 100644 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu +++ b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu @@ -23,9 +23,10 @@ * */ -#include // int64_t #include +#include // int64_t + #define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ template void raft::spatial::knn::detail:: \ diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu index b73cf31c58..aba67b8a3d 100644 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu +++ b/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,12 @@ * limitations under the License. */ -#include // size_t -#include // int_Xt #include // DistanceType #include +#include // size_t +#include // int_Xt + #define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ template void raft::spatial::knn::detail::fusedL2Knn( \ size_t D, \ diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu index 35ef37c984..c9d0c49727 100644 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu +++ b/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,12 @@ * limitations under the License. */ -#include // size_t -#include // int_Xt #include // DistanceType #include +#include // size_t +#include // int_Xt + #define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ template void raft::spatial::knn::detail::fusedL2Knn( \ size_t D, \ diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu index ff23d9c41b..e6e9765151 100644 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu +++ b/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,12 @@ * limitations under the License. */ -#include // size_t -#include // int_Xt #include // DistanceType #include +#include // size_t +#include // int_Xt + #define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ template void raft::spatial::knn::detail::fusedL2Knn( \ size_t D, \ diff --git a/cpp/template/src/ivf_flat_example.cu b/cpp/template/src/ivf_flat_example.cu index 2b3980f696..60694aea0f 100644 --- a/cpp/template/src/ivf_flat_example.cu +++ b/cpp/template/src/ivf_flat_example.cu @@ -15,6 +15,7 @@ */ #include "common.cuh" + #include #include #include diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index f23ba2ed1a..9ab9604c09 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -320,6 +320,7 @@ if(BUILD_TESTS) test/sparse/sddmm.cu test/sparse/sort.cu test/sparse/spgemmi.cu + test/sparse/spmm.cu test/sparse/symmetrize.cu ) diff --git a/cpp/test/cluster/cluster_solvers.cu b/cpp/test/cluster/cluster_solvers.cu index e283f6c3c5..cc0a381bbf 100644 --- a/cpp/test/cluster/cluster_solvers.cu +++ b/cpp/test/cluster/cluster_solvers.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,17 @@ * limitations under the License. */ -#include -#include -#include #include #include #include - #include #include +#include + +#include +#include + namespace raft { namespace spectral { diff --git a/cpp/test/cluster/cluster_solvers_deprecated.cu b/cpp/test/cluster/cluster_solvers_deprecated.cu index a95b013852..954e3e5bb6 100644 --- a/cpp/test/cluster/cluster_solvers_deprecated.cu +++ b/cpp/test/cluster/cluster_solvers_deprecated.cu @@ -14,14 +14,15 @@ * limitations under the License. */ -#include -#include -#include #include #include - #include +#include + +#include +#include + namespace raft { namespace spectral { diff --git a/cpp/test/cluster/kmeans.cu b/cpp/test/cluster/kmeans.cu index ebbe1c46be..33c17c527d 100644 --- a/cpp/test/cluster/kmeans.cu +++ b/cpp/test/cluster/kmeans.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,21 +15,25 @@ */ #include "../test_utils.cuh" -#include -#include -#include -#include #include #include #include +#include #include #include #include #include + #include + #include +#include + +#include +#include + namespace raft { template diff --git a/cpp/test/cluster/kmeans_balanced.cu b/cpp/test/cluster/kmeans_balanced.cu index 5b1d764b28..5009eaf122 100644 --- a/cpp/test/cluster/kmeans_balanced.cu +++ b/cpp/test/cluster/kmeans_balanced.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,22 +15,26 @@ */ #include "../test_utils.h" -#include -#include -#include -#include #include #include #include #include +#include #include #include #include #include + #include + #include +#include + +#include +#include + /* This test takes advantage of the fact that make_blobs generates balanced clusters. * It doesn't currently test whether the algorithm can make balanced clusters with an imbalanced * dataset. diff --git a/cpp/test/cluster/kmeans_find_k.cu b/cpp/test/cluster/kmeans_find_k.cu index 8f017a0231..8e05ad3695 100644 --- a/cpp/test/cluster/kmeans_find_k.cu +++ b/cpp/test/cluster/kmeans_find_k.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,17 +15,19 @@ */ #include "../test_utils.h" -#include -#include -#include -#include #include #include +#include #include #include #include +#include + +#include +#include + namespace raft { template diff --git a/cpp/test/cluster/linkage.cu b/cpp/test/cluster/linkage.cu index f8ae46cbf9..ba7ed4254e 100644 --- a/cpp/test/cluster/linkage.cu +++ b/cpp/test/cluster/linkage.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,13 +24,12 @@ #undef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "../test_utils.cuh" -#include +#include +#include #include #include #include - -#include #include #include diff --git a/cpp/test/cluster/spectral.cu b/cpp/test/cluster/spectral.cu index 580564ccbc..b8ee611f29 100644 --- a/cpp/test/cluster/spectral.cu +++ b/cpp/test/cluster/spectral.cu @@ -16,13 +16,15 @@ #include "../test_utils.cuh" -#include -#include -#include #include #include #include +#include + +#include +#include + namespace raft { namespace cluster { diff --git a/cpp/test/core/device_resources_manager.cpp b/cpp/test/core/device_resources_manager.cpp index 335e1cf578..c7c9e175ea 100644 --- a/cpp/test/core/device_resources_manager.cpp +++ b/cpp/test/core/device_resources_manager.cpp @@ -25,7 +25,6 @@ #include #include - #include #include diff --git a/cpp/test/core/device_setter.cpp b/cpp/test/core/device_setter.cpp index 5a4ff01346..283cf65980 100644 --- a/cpp/test/core/device_setter.cpp +++ b/cpp/test/core/device_setter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include #include #include #include +#include + +#include + namespace raft { TEST(DeviceSetter, ScopedDevice) { diff --git a/cpp/test/core/interruptible.cu b/cpp/test/core/interruptible.cu index aa8f16aa3c..e883e74e6f 100644 --- a/cpp/test/core/interruptible.cu +++ b/cpp/test/core/interruptible.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,18 @@ * limitations under the License. */ -#include -#include -#include -#include -#include #include #include #include + #include + +#include +#include + +#include +#include +#include #include #include diff --git a/cpp/test/core/logger.cpp b/cpp/test/core/logger.cpp index d7f001a700..7f31beed71 100644 --- a/cpp/test/core/logger.cpp +++ b/cpp/test/core/logger.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,8 +19,10 @@ #undef RAFT_ACTIVE_LEVEL #define RAFT_ACTIVE_LEVEL 6 -#include #include + +#include + #include namespace raft { diff --git a/cpp/test/core/math_device.cu b/cpp/test/core/math_device.cu index a11a95fff3..fd6cf1b1c6 100644 --- a/cpp/test/core/math_device.cu +++ b/cpp/test/core/math_device.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,17 @@ * limitations under the License. */ -#include - #include "../test_utils.h" + #include + #include #include #include +#include + #ifdef _RAFT_HAS_CUDA #include #include diff --git a/cpp/test/core/math_host.cpp b/cpp/test/core/math_host.cpp index 5808905713..f7f016adb6 100644 --- a/cpp/test/core/math_host.cpp +++ b/cpp/test/core/math_host.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,12 @@ * limitations under the License. */ -#include - #include "../test_utils.h" + #include +#include + TEST(MathHost, Abs) { // Integer abs diff --git a/cpp/test/core/mdarray.cu b/cpp/test/core/mdarray.cu index b0ab36c6e3..6a5cadd0ed 100644 --- a/cpp/test/core/mdarray.cu +++ b/cpp/test/core/mdarray.cu @@ -15,28 +15,31 @@ */ #include "../test_utils.cuh" -#include -#include #include #include #include #include #include #include +#include #include #include #include + #include #include #include #include + #include #include #include #include #include +#include + namespace { namespace stdex = std::experimental; void check_status(int32_t* d_status, rmm::cuda_stream_view stream) diff --git a/cpp/test/core/mdbuffer.cu b/cpp/test/core/mdbuffer.cu index d93d532938..7d5cc6c210 100644 --- a/cpp/test/core/mdbuffer.cu +++ b/cpp/test/core/mdbuffer.cu @@ -15,14 +15,17 @@ */ #include "../test_utils.h" -#include -#include + #include #include #include #include #include #include + +#include + +#include #include #include diff --git a/cpp/test/core/mdspan_copy.cpp b/cpp/test/core/mdspan_copy.cpp index 2f938e3035..b8aee7f8f5 100644 --- a/cpp/test/core/mdspan_copy.cpp +++ b/cpp/test/core/mdspan_copy.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,12 @@ */ #include "../test_utils.h" -#include -#include + #include + +#include + +#include #ifndef RAFT_DISABLE_CUDA #include #include diff --git a/cpp/test/core/mdspan_copy.cu b/cpp/test/core/mdspan_copy.cu index 95d7d3befd..b68ba38914 100644 --- a/cpp/test/core/mdspan_copy.cu +++ b/cpp/test/core/mdspan_copy.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,16 @@ */ #include "../test_utils.h" -#include -#include + #include #include #include #include +#include + +#include + namespace raft { TEST(MDSpanCopy, Mdspan3DDeviceDeviceCuda) { diff --git a/cpp/test/core/mdspan_utils.cu b/cpp/test/core/mdspan_utils.cu index 90ecba1a10..42e5fa605e 100644 --- a/cpp/test/core/mdspan_utils.cu +++ b/cpp/test/core/mdspan_utils.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ -#include #include #include #include #include #include +#include + namespace raft { namespace stdex = std::experimental; diff --git a/cpp/test/core/memory_type.cpp b/cpp/test/core/memory_type.cpp index cd8aa6bd9e..07ca66ccc1 100644 --- a/cpp/test/core/memory_type.cpp +++ b/cpp/test/core/memory_type.cpp @@ -13,9 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include + #include + #include -#include namespace raft { TEST(MemoryType, IsDeviceAccessible) diff --git a/cpp/test/core/numpy_serializer.cu b/cpp/test/core/numpy_serializer.cu index 5c562d68f7..e8f12ad706 100644 --- a/cpp/test/core/numpy_serializer.cu +++ b/cpp/test/core/numpy_serializer.cu @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #include #include #include @@ -25,6 +23,8 @@ #include #include +#include + #include #include #include diff --git a/cpp/test/core/nvtx.cpp b/cpp/test/core/nvtx.cpp index e6c29fa3d8..e726f4e1e4 100644 --- a/cpp/test/core/nvtx.cpp +++ b/cpp/test/core/nvtx.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ * limitations under the License. */ #ifdef NVTX_ENABLED -#include #include + +#include /** * tests for the functionality of generating next color based on string * entered in the NVTX Range marker wrappers diff --git a/cpp/test/core/operators_device.cu b/cpp/test/core/operators_device.cu index fa4cd25a15..8642ae76a9 100644 --- a/cpp/test/core/operators_device.cu +++ b/cpp/test/core/operators_device.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,19 @@ * limitations under the License. */ -#include -#include - -#include - #include "../test_utils.cuh" + #include #include + #include #include +#include + +#include +#include + template RAFT_KERNEL eval_op_on_device_kernel(OutT* out, OpT op, Args... args) { diff --git a/cpp/test/core/operators_host.cpp b/cpp/test/core/operators_host.cpp index de66fda919..cbb7f060d9 100644 --- a/cpp/test/core/operators_host.cpp +++ b/cpp/test/core/operators_host.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,16 @@ * limitations under the License. */ -#include -#include - -#include - #include "../test_utils.h" + #include #include +#include + +#include +#include + TEST(OperatorsHost, IdentityOp) { raft::identity_op op; diff --git a/cpp/test/core/seive.cu b/cpp/test/core/seive.cu index 8634abf3be..8b6c202b6a 100644 --- a/cpp/test/core/seive.cu +++ b/cpp/test/core/seive.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ * limitations under the License. */ -#include #include +#include + namespace raft { namespace common { TEST(Seive, Test) diff --git a/cpp/test/core/span.cpp b/cpp/test/core/span.cpp index 1a21b5ff47..78f4ef7277 100644 --- a/cpp/test/core/span.cpp +++ b/cpp/test/core/span.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,12 @@ * limitations under the License. */ #include "test_span.hpp" + +#include + #include + #include // iota -#include namespace raft { TEST(Span, DlfConstructors) diff --git a/cpp/test/core/span.cu b/cpp/test/core/span.cu index 667c7b19f7..03d8f61f37 100644 --- a/cpp/test/core/span.cu +++ b/cpp/test/core/span.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,7 @@ * limitations under the License. */ #include "test_span.hpp" -#include -#include // iota + #include #include #include @@ -28,6 +27,10 @@ #include #include +#include + +#include // iota + namespace raft { struct TestStatus { private: diff --git a/cpp/test/core/sparse_matrix.cpp b/cpp/test/core/sparse_matrix.cpp index a5f26a3e53..2b13bbf1ca 100644 --- a/cpp/test/core/sparse_matrix.cpp +++ b/cpp/test/core/sparse_matrix.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include #include #include #include + +#include + #include namespace raft { diff --git a/cpp/test/core/sparse_matrix.cu b/cpp/test/core/sparse_matrix.cu index 3cf0bc98d8..4a1188bd15 100644 --- a/cpp/test/core/sparse_matrix.cu +++ b/cpp/test/core/sparse_matrix.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include #include #include #include + +#include + #include namespace raft { diff --git a/cpp/test/core/stream_view.cpp b/cpp/test/core/stream_view.cpp index 715c53fe21..4148307979 100644 --- a/cpp/test/core/stream_view.cpp +++ b/cpp/test/core/stream_view.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ * limitations under the License. */ -#include #include #include + +#include #ifndef RAFT_DISABLE_CUDA #include #endif diff --git a/cpp/test/core/temporary_device_buffer.cu b/cpp/test/core/temporary_device_buffer.cu index f61fa826a9..dbcb1ecd44 100644 --- a/cpp/test/core/temporary_device_buffer.cu +++ b/cpp/test/core/temporary_device_buffer.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,9 @@ */ #include "../test_utils.cuh" -#include #include +#include #include #include diff --git a/cpp/test/distance/dist_adj.cu b/cpp/test/distance/dist_adj.cu index 7c3ec08cf5..a22fb7b1f9 100644 --- a/cpp/test/distance/dist_adj.cu +++ b/cpp/test/distance/dist_adj.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,18 @@ */ #include "../test_utils.cuh" -#include +#include "dist_adj.cuh" + #include #include #include #include #include #include + #include -#include "dist_adj.cuh" +#include namespace raft { namespace distance { diff --git a/cpp/test/distance/dist_adj.cuh b/cpp/test/distance/dist_adj.cuh index ee4554ff29..2861cb33de 100644 --- a/cpp/test/distance/dist_adj.cuh +++ b/cpp/test/distance/dist_adj.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "dist_adj_threshold.cuh" + #include #define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ diff --git a/cpp/test/distance/dist_adj_distance_instance.cu b/cpp/test/distance/dist_adj_distance_instance.cu index d4685d8095..158a5986c2 100644 --- a/cpp/test/distance/dist_adj_distance_instance.cu +++ b/cpp/test/distance/dist_adj_distance_instance.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,11 @@ #undef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "dist_adj_threshold.cuh" -#include + #include +#include + #define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ template void raft::distance::distance( \ raft::resources const& handle, \ diff --git a/cpp/test/distance/distance_base.cuh b/cpp/test/distance/distance_base.cuh index 40e5013d1b..938cd219d0 100644 --- a/cpp/test/distance/distance_base.cuh +++ b/cpp/test/distance/distance_base.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,18 +15,20 @@ */ #include "../test_utils.cuh" -#include -#include // common::nvtx::range -#include +#include // common::nvtx::range #include // make_device_matrix_view #include // raft::sqrt -#include // raft::resources +#include +#include // raft::resources #include #include // raft::distance::DistanceType #include + #include // rmm::device_uvector +#include + namespace raft { namespace distance { diff --git a/cpp/test/distance/fused_l2_nn.cu b/cpp/test/distance/fused_l2_nn.cu index 2a99acd0e4..6fd8f15808 100644 --- a/cpp/test/distance/fused_l2_nn.cu +++ b/cpp/test/distance/fused_l2_nn.cu @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -24,6 +24,8 @@ #include #include +#include + namespace raft { namespace distance { diff --git a/cpp/test/distance/gram.cu b/cpp/test/distance/gram.cu index a9dbd8328f..e911a25ff1 100644 --- a/cpp/test/distance/gram.cu +++ b/cpp/test/distance/gram.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,16 +16,20 @@ #include "../test_utils.cuh" #include "gram_base.cuh" -#include -#include -#include + #include #include #include #include #include + #include +#include + +#include +#include + namespace raft::distance::kernels { struct GramMatrixInputs { diff --git a/cpp/test/distance/gram_base.cuh b/cpp/test/distance/gram_base.cuh index 1862d1a540..170bcb76c1 100644 --- a/cpp/test/distance/gram_base.cuh +++ b/cpp/test/distance/gram_base.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,17 @@ * limitations under the License. */ -#include -#include #include #include #include #include #include + #include +#include +#include + namespace raft { namespace distance { namespace kernels { diff --git a/cpp/test/distance/masked_nn.cu b/cpp/test/distance/masked_nn.cu index 67ef1952ca..34fa07c45b 100644 --- a/cpp/test/distance/masked_nn.cu +++ b/cpp/test/distance/masked_nn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,7 @@ */ #include "../test_utils.h" -#include -#include + #include #include #include @@ -29,6 +28,10 @@ #include #include +#include + +#include + namespace raft::distance::masked_nn { // The adjacency pattern determines what distances get computed. diff --git a/cpp/test/distance/masked_nn_compress_to_bits.cu b/cpp/test/distance/masked_nn_compress_to_bits.cu index 1563974e56..2512af5e4f 100644 --- a/cpp/test/distance/masked_nn_compress_to_bits.cu +++ b/cpp/test/distance/masked_nn_compress_to_bits.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,7 @@ #include "../test_utils.cuh" #include "../test_utils.h" -#include -#include -#include + #include #include #include @@ -31,6 +29,11 @@ #include #include +#include + +#include +#include + namespace raft::distance::masked_nn::compress_to_bits { /** diff --git a/cpp/test/label/label.cu b/cpp/test/label/label.cu index bda87d423c..4c3479182f 100644 --- a/cpp/test/label/label.cu +++ b/cpp/test/label/label.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include +#include "../test_utils.cuh" #include - -#include "../test_utils.cuh" #include #include +#include + #include #include diff --git a/cpp/test/label/merge_labels.cu b/cpp/test/label/merge_labels.cu index 3e12f9171e..e6cb235b0f 100644 --- a/cpp/test/label/merge_labels.cu +++ b/cpp/test/label/merge_labels.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,20 @@ * limitations under the License. */ -#include -#include -#include - #include "../test_utils.cuh" + +#include #include +#include #include + #include #include + #include +#include + #include namespace raft { diff --git a/cpp/test/lap/lap.cu b/cpp/test/lap/lap.cu index 7576465633..47407f3ea7 100644 --- a/cpp/test/lap/lap.cu +++ b/cpp/test/lap/lap.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * Copyright 2020 KETAN DATE & RAKESH NAGI * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,14 +22,15 @@ * for the Linear Assignment Problem." Parallel Computing 57 (2016): 52-72. * */ -#include #include +#include #include -#include +#include #include -#include + +#include #include #define PROBLEMSIZE 1000 // Number of rows/columns diff --git a/cpp/test/linalg/add.cu b/cpp/test/linalg/add.cu index 668f4c0691..981db432ee 100644 --- a/cpp/test/linalg/add.cu +++ b/cpp/test/linalg/add.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,14 @@ #include "../test_utils.cuh" #include "add.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/axpy.cu b/cpp/test/linalg/axpy.cu index 44ec4c1385..f40dae9ff1 100644 --- a/cpp/test/linalg/axpy.cu +++ b/cpp/test/linalg/axpy.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "../test_utils.cuh" + #include #include - -#include "../test_utils.cuh" -#include #include #include #include +#include + namespace raft { namespace linalg { // Reference axpy implementation. diff --git a/cpp/test/linalg/binary_op.cu b/cpp/test/linalg/binary_op.cu index df8a38bf4e..fd00e72fe8 100644 --- a/cpp/test/linalg/binary_op.cu +++ b/cpp/test/linalg/binary_op.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,17 @@ #include "../test_utils.cuh" #include "binary_op.cuh" -#include + #include #include #include #include #include + #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/binary_op.cuh b/cpp/test/linalg/binary_op.cuh index 6b502ecf7c..c76b7b606e 100644 --- a/cpp/test/linalg/binary_op.cuh +++ b/cpp/test/linalg/binary_op.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include "../test_utils.cuh" + #include #include diff --git a/cpp/test/linalg/cholesky_r1.cu b/cpp/test/linalg/cholesky_r1.cu index 961ab8a7a2..f87e07402f 100644 --- a/cpp/test/linalg/cholesky_r1.cu +++ b/cpp/test/linalg/cholesky_r1.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,20 @@ * limitations under the License. */ -#include +#include "../test_utils.cuh" + #include #include #include #include #include #include + #include #include -#include "../test_utils.cuh" +#include + #include #include namespace raft { diff --git a/cpp/test/linalg/coalesced_reduction.cu b/cpp/test/linalg/coalesced_reduction.cu index 6d78788aef..2061f28d36 100644 --- a/cpp/test/linalg/coalesced_reduction.cu +++ b/cpp/test/linalg/coalesced_reduction.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #include "../test_utils.cuh" #include "reduce.cuh" -#include + #include #include #include @@ -24,6 +24,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/divide.cu b/cpp/test/linalg/divide.cu index 06d98acb0e..14f3ce4ea6 100644 --- a/cpp/test/linalg/divide.cu +++ b/cpp/test/linalg/divide.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,14 @@ #include "../test_utils.cuh" #include "unary_op.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/dot.cu b/cpp/test/linalg/dot.cu index 34a2e7e7e2..120ee45202 100644 --- a/cpp/test/linalg/dot.cu +++ b/cpp/test/linalg/dot.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,15 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "../test_utils.cuh" + #include #include - -#include "../test_utils.cuh" -#include #include #include + #include +#include + namespace raft { namespace linalg { // Reference dot implementation. diff --git a/cpp/test/linalg/eig.cu b/cpp/test/linalg/eig.cu index 49990ce300..460b99aaa0 100644 --- a/cpp/test/linalg/eig.cu +++ b/cpp/test/linalg/eig.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,15 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/eig_sel.cu b/cpp/test/linalg/eig_sel.cu index a067cd140e..38e5b5f85e 100644 --- a/cpp/test/linalg/eig_sel.cu +++ b/cpp/test/linalg/eig_sel.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,14 @@ #if CUDART_VERSION >= 10010 #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/eltwise.cu b/cpp/test/linalg/eltwise.cu index 531af71395..b03db48630 100644 --- a/cpp/test/linalg/eltwise.cu +++ b/cpp/test/linalg/eltwise.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/gemm_layout.cu b/cpp/test/linalg/gemm_layout.cu index fa509aaa29..7a764b77d3 100644 --- a/cpp/test/linalg/gemm_layout.cu +++ b/cpp/test/linalg/gemm_layout.cu @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/gemv.cu b/cpp/test/linalg/gemv.cu index e90f88e0e1..a842035229 100644 --- a/cpp/test/linalg/gemv.cu +++ b/cpp/test/linalg/gemv.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/map.cu b/cpp/test/linalg/map.cu index 9a88460640..1dcc1732a0 100644 --- a/cpp/test/linalg/map.cu +++ b/cpp/test/linalg/map.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #include "../test_utils.cuh" #include "unary_op.cuh" -#include + #include #include #include @@ -27,6 +27,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/map_then_reduce.cu b/cpp/test/linalg/map_then_reduce.cu index 8e13babec6..9972285450 100644 --- a/cpp/test/linalg/map_then_reduce.cu +++ b/cpp/test/linalg/map_then_reduce.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,7 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include @@ -24,9 +23,14 @@ #include #include #include + #include #include +#include + +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/matrix_vector.cu b/cpp/test/linalg/matrix_vector.cu index f56f51b48b..498ef118bb 100644 --- a/cpp/test/linalg/matrix_vector.cu +++ b/cpp/test/linalg/matrix_vector.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #include "../test_utils.cuh" #include "matrix_vector_op.cuh" -#include + #include #include #include @@ -25,6 +25,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/matrix_vector_op.cu b/cpp/test/linalg/matrix_vector_op.cu index c39e9dd164..cd812327ae 100644 --- a/cpp/test/linalg/matrix_vector_op.cu +++ b/cpp/test/linalg/matrix_vector_op.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,15 @@ #include "../test_utils.cuh" #include "matrix_vector_op.cuh" -#include + #include #include #include #include #include + +#include + #include namespace raft { diff --git a/cpp/test/linalg/matrix_vector_op.cuh b/cpp/test/linalg/matrix_vector_op.cuh index d28e7cc5b5..25d979aa57 100644 --- a/cpp/test/linalg/matrix_vector_op.cuh +++ b/cpp/test/linalg/matrix_vector_op.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "../test_utils.cuh" + #include #include diff --git a/cpp/test/linalg/mean_squared_error.cu b/cpp/test/linalg/mean_squared_error.cu index 5d0694f812..7409ee3357 100644 --- a/cpp/test/linalg/mean_squared_error.cu +++ b/cpp/test/linalg/mean_squared_error.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,15 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "../test_utils.cuh" + #include #include - -#include "../test_utils.cuh" -#include #include #include + #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/multiply.cu b/cpp/test/linalg/multiply.cu index 0cd890b575..f90832e828 100644 --- a/cpp/test/linalg/multiply.cu +++ b/cpp/test/linalg/multiply.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,14 @@ #include "../test_utils.cuh" #include "unary_op.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/norm.cu b/cpp/test/linalg/norm.cu index 888ebe28d7..e4f064949c 100644 --- a/cpp/test/linalg/norm.cu +++ b/cpp/test/linalg/norm.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -24,6 +24,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/normalize.cu b/cpp/test/linalg/normalize.cu index 63dbd9a19b..252c3906d6 100644 --- a/cpp/test/linalg/normalize.cu +++ b/cpp/test/linalg/normalize.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -26,6 +26,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/power.cu b/cpp/test/linalg/power.cu index b8e8355fa0..b804c29334 100644 --- a/cpp/test/linalg/power.cu +++ b/cpp/test/linalg/power.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/randomized_svd.cu b/cpp/test/linalg/randomized_svd.cu index 9e1d3df6dc..01ddba6214 100644 --- a/cpp/test/linalg/randomized_svd.cu +++ b/cpp/test/linalg/randomized_svd.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -24,6 +24,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/reduce.cu b/cpp/test/linalg/reduce.cu index 8578fe9637..3ce8e33efd 100644 --- a/cpp/test/linalg/reduce.cu +++ b/cpp/test/linalg/reduce.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #include "../test_utils.cuh" #include "reduce.cuh" -#include + #include #include #include @@ -25,6 +25,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/reduce.cuh b/cpp/test/linalg/reduce.cuh index b191fba6c6..73fad14245 100644 --- a/cpp/test/linalg/reduce.cuh +++ b/cpp/test/linalg/reduce.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,16 +16,18 @@ #pragma once -#include #include #include #include #include + #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/reduce_cols_by_key.cu b/cpp/test/linalg/reduce_cols_by_key.cu index dbd54a350f..80fd06cad4 100644 --- a/cpp/test/linalg/reduce_cols_by_key.cu +++ b/cpp/test/linalg/reduce_cols_by_key.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -23,6 +23,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/reduce_rows_by_key.cu b/cpp/test/linalg/reduce_rows_by_key.cu index 203dbdd5d1..d8cc3f0295 100644 --- a/cpp/test/linalg/reduce_rows_by_key.cu +++ b/cpp/test/linalg/reduce_rows_by_key.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,16 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include #include +#include + +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/rsvd.cu b/cpp/test/linalg/rsvd.cu index b5d10d215c..8a2abe5f0d 100644 --- a/cpp/test/linalg/rsvd.cu +++ b/cpp/test/linalg/rsvd.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,15 +15,18 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include #include #include + #include +#include + #include namespace raft { diff --git a/cpp/test/linalg/sqrt.cu b/cpp/test/linalg/sqrt.cu index 06d1b76b78..6717914fdc 100644 --- a/cpp/test/linalg/sqrt.cu +++ b/cpp/test/linalg/sqrt.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/strided_reduction.cu b/cpp/test/linalg/strided_reduction.cu index 06096a74a7..e964769ab3 100644 --- a/cpp/test/linalg/strided_reduction.cu +++ b/cpp/test/linalg/strided_reduction.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #include "../test_utils.cuh" #include "reduce.cuh" -#include + #include #include #include @@ -24,6 +24,8 @@ #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/subtract.cu b/cpp/test/linalg/subtract.cu index 292abdc07f..29714f82e6 100644 --- a/cpp/test/linalg/subtract.cu +++ b/cpp/test/linalg/subtract.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/svd.cu b/cpp/test/linalg/svd.cu index a092b16e22..400eceded2 100644 --- a/cpp/test/linalg/svd.cu +++ b/cpp/test/linalg/svd.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,15 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/ternary_op.cu b/cpp/test/linalg/ternary_op.cu index 7182f52fcd..1ac2725271 100644 --- a/cpp/test/linalg/ternary_op.cu +++ b/cpp/test/linalg/ternary_op.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/transpose.cu b/cpp/test/linalg/transpose.cu index 6579011856..f6857d3ffa 100644 --- a/cpp/test/linalg/transpose.cu +++ b/cpp/test/linalg/transpose.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,18 +15,18 @@ */ #include "../test_utils.cuh" -#include +#include +#include #include #include #include #include -#include -#include - #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/unary_op.cu b/cpp/test/linalg/unary_op.cu index ba1bdad2f1..d3eb8cc9d3 100644 --- a/cpp/test/linalg/unary_op.cu +++ b/cpp/test/linalg/unary_op.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,15 @@ #include "../test_utils.cuh" #include "unary_op.cuh" -#include + #include #include #include #include #include +#include + namespace raft { namespace linalg { diff --git a/cpp/test/linalg/unary_op.cuh b/cpp/test/linalg/unary_op.cuh index a276024e14..5c1e049f0f 100644 --- a/cpp/test/linalg/unary_op.cuh +++ b/cpp/test/linalg/unary_op.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include "../test_utils.cuh" + #include #include diff --git a/cpp/test/matrix/argmax.cu b/cpp/test/matrix/argmax.cu index 40ca358fe1..cb3fd4a3fb 100644 --- a/cpp/test/matrix/argmax.cu +++ b/cpp/test/matrix/argmax.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,17 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include #include #include +#include + +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/argmin.cu b/cpp/test/matrix/argmin.cu index 29013bbdd8..060b4a78db 100644 --- a/cpp/test/matrix/argmin.cu +++ b/cpp/test/matrix/argmin.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,17 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include #include #include +#include + +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/columnSort.cu b/cpp/test/matrix/columnSort.cu index 1ea9b4ae67..246ed7179d 100644 --- a/cpp/test/matrix/columnSort.cu +++ b/cpp/test/matrix/columnSort.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,20 @@ */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include #include + #include +#include + +#include +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/diagonal.cu b/cpp/test/matrix/diagonal.cu index a75e5290ae..c6e1f1a0d2 100644 --- a/cpp/test/matrix/diagonal.cu +++ b/cpp/test/matrix/diagonal.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -23,6 +23,8 @@ #include #include +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/eye.cu b/cpp/test/matrix/eye.cu index 33ed8a00ba..9617d68eca 100644 --- a/cpp/test/matrix/eye.cu +++ b/cpp/test/matrix/eye.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include - #include #include +#include + namespace raft::matrix { template diff --git a/cpp/test/matrix/gather.cu b/cpp/test/matrix/gather.cu index b1228f05ca..4c13d0c1e9 100644 --- a/cpp/test/matrix/gather.cu +++ b/cpp/test/matrix/gather.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -24,8 +24,11 @@ #include #include #include + #include +#include + namespace raft { template -#include + #include #include #include @@ -26,8 +25,13 @@ #include #include #include + #include +#include + +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/math.cu b/cpp/test/matrix/math.cu index 3596e63603..aefe5727c5 100644 --- a/cpp/test/matrix/math.cu +++ b/cpp/test/matrix/math.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,20 +15,20 @@ */ #include "../test_utils.cuh" -#include -#include #include +#include #include #include #include #include #include #include - #include #include +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/matrix.cu b/cpp/test/matrix/matrix.cu index a10280a30c..ccae4266e9 100644 --- a/cpp/test/matrix/matrix.cu +++ b/cpp/test/matrix/matrix.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,20 +15,22 @@ */ #include "../test_utils.cuh" -#include + #include #include #include - #include #include #include + #include #include #include #include +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/norm.cu b/cpp/test/matrix/norm.cu index e370b7ce0c..2b635c5e28 100644 --- a/cpp/test/matrix/norm.cu +++ b/cpp/test/matrix/norm.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,16 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include + #include +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/reverse.cu b/cpp/test/matrix/reverse.cu index 191fc50198..7aa913b059 100644 --- a/cpp/test/matrix/reverse.cu +++ b/cpp/test/matrix/reverse.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/scatter.cu b/cpp/test/matrix/scatter.cu index 3a1a40086e..7f478c7b93 100644 --- a/cpp/test/matrix/scatter.cu +++ b/cpp/test/matrix/scatter.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -23,12 +23,15 @@ #include #include #include + #include #include #include #include +#include + namespace raft { template diff --git a/cpp/test/matrix/select_k.cuh b/cpp/test/matrix/select_k.cuh index 412a9ae5a2..7f9b7b3fc3 100644 --- a/cpp/test/matrix/select_k.cuh +++ b/cpp/test/matrix/select_k.cuh @@ -15,20 +15,20 @@ */ #include "../test_utils.cuh" -#include - -#include +#include #include #include #include #include -#include +#include #include #include +#include + #include #include diff --git a/cpp/test/matrix/slice.cu b/cpp/test/matrix/slice.cu index fbf735aaf7..1a6377580e 100644 --- a/cpp/test/matrix/slice.cu +++ b/cpp/test/matrix/slice.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,16 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include + #include +#include + namespace raft { namespace matrix { diff --git a/cpp/test/matrix/triangular.cu b/cpp/test/matrix/triangular.cu index 00b8a89b02..ce6dfdfef5 100644 --- a/cpp/test/matrix/triangular.cu +++ b/cpp/test/matrix/triangular.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,15 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include #include +#include + namespace raft { namespace matrix { diff --git a/cpp/test/mr/device/buffer.cpp b/cpp/test/mr/device/buffer.cpp index 447ab77b35..d14aa09b7a 100644 --- a/cpp/test/mr/device/buffer.cpp +++ b/cpp/test/mr/device/buffer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,16 @@ * limitations under the License. */ -#include -#include -#include #include + #include #include +#include + +#include +#include + namespace raft { namespace mr { namespace device { diff --git a/cpp/test/mr/host/buffer.cpp b/cpp/test/mr/host/buffer.cpp index 89b185547a..5688ff6376 100644 --- a/cpp/test/mr/host/buffer.cpp +++ b/cpp/test/mr/host/buffer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,12 @@ * limitations under the License. */ +#include + #include + #include #include -#include namespace raft { namespace mr { diff --git a/cpp/test/neighbors/ann_brute_force.cuh b/cpp/test/neighbors/ann_brute_force.cuh index 1cba6bfb39..6370c5ee83 100644 --- a/cpp/test/neighbors/ann_brute_force.cuh +++ b/cpp/test/neighbors/ann_brute_force.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,40 +18,40 @@ #include "../test_utils.cuh" #include "ann_utils.cuh" #include "knn_utils.cuh" + #include +#include #include +#include #include #include #include #include +#include #include +#include +#include +#include +#include #include #include #include +#include +#include #include #include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include #include - -#include - #include + +#include #include +#include + #include #include #include diff --git a/cpp/test/neighbors/ann_brute_force/test_float.cu b/cpp/test/neighbors/ann_brute_force/test_float.cu index f618f44b61..f157b5f65c 100644 --- a/cpp/test/neighbors/ann_brute_force/test_float.cu +++ b/cpp/test/neighbors/ann_brute_force/test_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_brute_force.cuh" +#include + namespace raft::neighbors::brute_force { using AnnBruteForceTest_float = AnnBruteForceTest; diff --git a/cpp/test/neighbors/ann_cagra.cuh b/cpp/test/neighbors/ann_cagra.cuh index 296a5f07fc..a111de0762 100644 --- a/cpp/test/neighbors/ann_cagra.cuh +++ b/cpp/test/neighbors/ann_cagra.cuh @@ -19,13 +19,11 @@ #include "../test_utils.cuh" #include "ann_utils.cuh" -#include - -#include #include #include #include +#include #include #include #include @@ -34,13 +32,14 @@ #include #include -#include +#include -#include +#include +#include #include -#include +#include #include #include diff --git a/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu b/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu index 6f9e8dbd43..ff7e839abf 100644 --- a/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu +++ b/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,11 @@ * limitations under the License. */ -#include - #include "../ann_cagra.cuh" #include "search_kernel_uint64_t.cuh" +#include + namespace raft::neighbors::cagra { typedef AnnCagraTest AnnCagraTestF_I64; diff --git a/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu index 944c2cbc89..7d29ce4f99 100644 --- a/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu +++ b/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_cagra.cuh" +#include + namespace raft::neighbors::cagra { typedef AnnCagraTest AnnCagraTestF_U32; diff --git a/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu b/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu index fdd510bc5d..bcdd95bece 100644 --- a/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu +++ b/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu @@ -14,11 +14,11 @@ * limitations under the License. */ -#include - #include "../ann_cagra.cuh" #include "search_kernel_uint64_t.cuh" +#include + namespace raft::neighbors::cagra { typedef AnnCagraTest AnnCagraTestH_I64; diff --git a/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu index dea6ae0c23..ec7144f8d0 100644 --- a/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu +++ b/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_cagra.cuh" +#include + namespace raft::neighbors::cagra { typedef AnnCagraTest AnnCagraTestH_U32; diff --git a/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu index 3d9dc76953..b2242d89b1 100644 --- a/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu +++ b/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_cagra.cuh" +#include + namespace raft::neighbors::cagra { typedef AnnCagraTest AnnCagraTestI8_U32; diff --git a/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu index c5b1b1704b..302b2bec18 100644 --- a/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu +++ b/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_cagra.cuh" +#include + namespace raft::neighbors::cagra { typedef AnnCagraTest AnnCagraTestU8_U32; diff --git a/cpp/test/neighbors/ann_ivf_flat.cuh b/cpp/test/neighbors/ann_ivf_flat.cuh index 39439d392d..de6af589fa 100644 --- a/cpp/test/neighbors/ann_ivf_flat.cuh +++ b/cpp/test/neighbors/ann_ivf_flat.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,41 +17,41 @@ #include "../test_utils.cuh" #include "ann_utils.cuh" + #include +#include #include +#include #include #include #include #include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include #include +#include #include #include #include +#include +#include +#include #include #include #include #include +#include +#include + +#include #include #include - -#include - #include + +#include #include +#include + #include #include #include @@ -354,7 +354,7 @@ class AnnIVFFlatTest : public ::testing::TestWithParam> { ivf::resize_list(handle_, lists[label], list_device_spec, list_size, 0); } - idx.recompute_internal_state(handle_); + helpers::recompute_internal_state(handle_, &idx); using interleaved_group = Pow2; @@ -608,6 +608,23 @@ const std::vector> inputs = { {1000, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, true}, {10000, 131072, 8, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false}, + // various combinations with k>raft::matrix::detail::select::warpsort::kMaxCapacity + {1000, 10000, 16, 1024, 40, 1024, raft::distance::DistanceType::L2SqrtExpanded, true}, + {1000, 10000, 2053, 512, 50, 1024, raft::distance::DistanceType::L2SqrtExpanded, false}, + {1000, 10000, 2049, 2048, 70, 1024, raft::distance::DistanceType::L2SqrtExpanded, false}, + {1000, 10000, 16, 4000, 100, 2048, raft::distance::DistanceType::L2SqrtExpanded, false}, + {10, 10000, 16, 4000, 100, 2048, raft::distance::DistanceType::L2SqrtExpanded, false}, + {10, 10000, 16, 4000, 120, 2048, raft::distance::DistanceType::L2SqrtExpanded, true}, + {20, 100000, 16, 257, 20, 1024, raft::distance::DistanceType::L2SqrtExpanded, true}, + {1000, 100000, 16, 259, 20, 1024, raft::distance::DistanceType::L2Expanded, true, true}, + {10000, 131072, 8, 280, 20, 1024, raft::distance::DistanceType::InnerProduct, false}, + {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::L2Expanded, false}, + {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::L2SqrtExpanded, false}, + {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::InnerProduct, false}, + {100000, 1024, 16, 300, 20, 60, raft::distance::DistanceType::L2Expanded, false}, + {100000, 1024, 16, 500, 20, 60, raft::distance::DistanceType::L2SqrtExpanded, false}, + {100000, 1024, 16, 700, 20, 60, raft::distance::DistanceType::InnerProduct, false}, + // host input data {1000, 10000, 16, 10, 40, 1024, raft::distance::DistanceType::L2Expanded, false, true}, {1000, 10000, 16, 10, 50, 1024, raft::distance::DistanceType::L2Expanded, false, true}, diff --git a/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu index 3bfea283e5..2ff17b8536 100644 --- a/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu +++ b/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_ivf_flat.cuh" +#include + namespace raft::neighbors::ivf_flat { typedef AnnIVFFlatTest AnnIVFFlatTestF; diff --git a/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu index 2f542bd6ec..6fe12506aa 100644 --- a/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu +++ b/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_ivf_flat.cuh" +#include + namespace raft::neighbors::ivf_flat { typedef AnnIVFFlatTest AnnIVFFlatTestF_int8; diff --git a/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu index 7659707089..ab6001c71b 100644 --- a/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu +++ b/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_ivf_flat.cuh" +#include + namespace raft::neighbors::ivf_flat { typedef AnnIVFFlatTest AnnIVFFlatTestF_uint8; diff --git a/cpp/test/neighbors/ann_ivf_pq.cuh b/cpp/test/neighbors/ann_ivf_pq.cuh index e028ef061f..4ebe02027f 100644 --- a/cpp/test/neighbors/ann_ivf_pq.cuh +++ b/cpp/test/neighbors/ann_ivf_pq.cuh @@ -17,11 +17,9 @@ #include "../test_utils.cuh" #include "ann_utils.cuh" -#include - -#include #include +#include #include #include #include @@ -32,16 +30,18 @@ #include #include +#include + #include #include #include #include -#include - #include #include +#include + #include #include #include diff --git a/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu index 5405ddc4a3..a6cfab1f19 100644 --- a/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu +++ b/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ // // TODO: consider removing this test or consider adding an instantiation to the // library. + #undef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "../ann_ivf_pq.cuh" diff --git a/cpp/test/neighbors/ann_nn_descent.cuh b/cpp/test/neighbors/ann_nn_descent.cuh index 85574442ad..495af081f1 100644 --- a/cpp/test/neighbors/ann_nn_descent.cuh +++ b/cpp/test/neighbors/ann_nn_descent.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,12 +18,12 @@ #include "../test_utils.cuh" #include "ann_utils.cuh" -#include - #include #include #include +#include + #include #include diff --git a/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu index 6aa503ca4d..ec6d04ad12 100644 --- a/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu +++ b/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_nn_descent.cuh" +#include + namespace raft::neighbors::experimental::nn_descent { typedef AnnNNDescentTest AnnNNDescentTestF_U32; diff --git a/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu index 863f7edcc0..27fa42d636 100644 --- a/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu +++ b/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_nn_descent.cuh" +#include + namespace raft::neighbors::experimental::nn_descent { typedef AnnNNDescentTest AnnNNDescentTestI8_U32; diff --git a/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu index 1a1b38fc19..3afe79dcc4 100644 --- a/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu +++ b/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include "../ann_nn_descent.cuh" +#include + namespace raft::neighbors::experimental::nn_descent { typedef AnnNNDescentTest AnnNNDescentTestUI8_U32; diff --git a/cpp/test/neighbors/ann_utils.cuh b/cpp/test/neighbors/ann_utils.cuh index 18860100f3..afd083d512 100644 --- a/cpp/test/neighbors/ann_utils.cuh +++ b/cpp/test/neighbors/ann_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ #pragma once +#include "../test_utils.cuh" + #include // raft::make_device_matrix #include #include @@ -24,14 +26,14 @@ #include #include +#include + #include #include #include -#include - -#include "../test_utils.cuh" #include + #include namespace raft::neighbors { diff --git a/cpp/test/neighbors/ball_cover.cu b/cpp/test/neighbors/ball_cover.cu index 62fbdd6edb..ffd17d6d74 100644 --- a/cpp/test/neighbors/ball_cover.cu +++ b/cpp/test/neighbors/ball_cover.cu @@ -16,6 +16,7 @@ #include "../test_utils.cuh" #include "spatial_data.h" + #include #include #include @@ -32,8 +33,9 @@ #include #include -#include #include + +#include #include #include diff --git a/cpp/test/neighbors/epsilon_neighborhood.cu b/cpp/test/neighbors/epsilon_neighborhood.cu index 8b35e3ca70..7bf56b18d6 100644 --- a/cpp/test/neighbors/epsilon_neighborhood.cu +++ b/cpp/test/neighbors/epsilon_neighborhood.cu @@ -15,8 +15,7 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include @@ -25,8 +24,13 @@ #include #include #include + #include +#include + +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/test/neighbors/fused_l2_knn.cu b/cpp/test/neighbors/fused_l2_knn.cu index 80a01ce568..e4d018aff0 100644 --- a/cpp/test/neighbors/fused_l2_knn.cu +++ b/cpp/test/neighbors/fused_l2_knn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,16 +16,15 @@ #include "../test_utils.cuh" #include "./knn_utils.cuh" -#include #include +#include +#include #include #include #include #include -#include - #include #include diff --git a/cpp/test/neighbors/haversine.cu b/cpp/test/neighbors/haversine.cu index f1b03a0ab5..55d9fe209e 100644 --- a/cpp/test/neighbors/haversine.cu +++ b/cpp/test/neighbors/haversine.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,16 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include + #include + +#include + +#include #include namespace raft { diff --git a/cpp/test/neighbors/knn.cu b/cpp/test/neighbors/knn.cu index 5fe88ee89e..a98de85bda 100644 --- a/cpp/test/neighbors/knn.cu +++ b/cpp/test/neighbors/knn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,10 @@ */ #include "../test_utils.cuh" -#include #include #include +#include #include #include diff --git a/cpp/test/neighbors/knn_utils.cuh b/cpp/test/neighbors/knn_utils.cuh index b79e7cffe8..3b68ec417e 100644 --- a/cpp/test/neighbors/knn_utils.cuh +++ b/cpp/test/neighbors/knn_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,14 @@ #pragma once -#include - #include "../test_utils.cuh" -#include #include +#include + +#include + namespace raft::spatial::knn { template struct idx_dist_pair { diff --git a/cpp/test/neighbors/refine.cu b/cpp/test/neighbors/refine.cu index c5865a7618..05e6048e56 100644 --- a/cpp/test/neighbors/refine.cu +++ b/cpp/test/neighbors/refine.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,9 @@ #include "../test_utils.cuh" #include "ann_utils.cuh" -#include - -#include #include +#include #include #include #include @@ -28,6 +26,8 @@ #include #include +#include + #include #include diff --git a/cpp/test/neighbors/tiled_knn.cu b/cpp/test/neighbors/tiled_knn.cu index f41a36dde3..d378100711 100644 --- a/cpp/test/neighbors/tiled_knn.cu +++ b/cpp/test/neighbors/tiled_knn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,10 @@ #include "../test_utils.cuh" #include "./ann_utils.cuh" #include "./knn_utils.cuh" -#include #include #include +#include #include // raft::distance::pairwise_distance #include #include diff --git a/cpp/test/random/make_blobs.cu b/cpp/test/random/make_blobs.cu index d78fbaf15b..f4b9278bb0 100644 --- a/cpp/test/random/make_blobs.cu +++ b/cpp/test/random/make_blobs.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,18 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include - #include #include #include +#include + +#include + namespace raft { namespace random { diff --git a/cpp/test/random/multi_variable_gaussian.cu b/cpp/test/random/multi_variable_gaussian.cu index 0a1969b5c3..62bad8e543 100644 --- a/cpp/test/random/multi_variable_gaussian.cu +++ b/cpp/test/random/multi_variable_gaussian.cu @@ -15,9 +15,7 @@ */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include @@ -26,9 +24,14 @@ #include #include -#include #include +#include + +#include +#include +#include + // mvg.h takes in column-major matrices (as in Fortran) #define IDX2C(i, j, ld) (j * ld + i) diff --git a/cpp/test/random/permute.cu b/cpp/test/random/permute.cu index f95c8c71c0..b8311c6175 100644 --- a/cpp/test/random/permute.cu +++ b/cpp/test/random/permute.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,15 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include - #include #include + +#include #include namespace raft { diff --git a/cpp/test/random/rmat_rectangular_generator.cu b/cpp/test/random/rmat_rectangular_generator.cu index 5a314d0f62..8d668f7a8a 100644 --- a/cpp/test/random/rmat_rectangular_generator.cu +++ b/cpp/test/random/rmat_rectangular_generator.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,21 +14,22 @@ * limitations under the License. */ -#include -#include -#include -#include -#include - #include "../test_utils.cuh" +#include #include #include #include - #include #include +#include + +#include +#include + +#include + namespace raft { namespace random { diff --git a/cpp/test/random/rng.cu b/cpp/test/random/rng.cu index 8ec35de112..a37f150d4c 100644 --- a/cpp/test/random/rng.cu +++ b/cpp/test/random/rng.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,21 +14,23 @@ * limitations under the License. */ -#include -#include -#include - #include "../test_utils.cuh" -#include -#include + +#include #include #include #include #include - #include #include +#include + +#include +#include + +#include + namespace raft { namespace random { diff --git a/cpp/test/random/rng_discrete.cu b/cpp/test/random/rng_discrete.cu index d1293f34ea..0a407a457f 100644 --- a/cpp/test/random/rng_discrete.cu +++ b/cpp/test/random/rng_discrete.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,17 +15,19 @@ */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include #include - #include #include + +#include + +#include +#include #include namespace raft { diff --git a/cpp/test/random/rng_int.cu b/cpp/test/random/rng_int.cu index 88f1a282b4..28d28f8321 100644 --- a/cpp/test/random/rng_int.cu +++ b/cpp/test/random/rng_int.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,17 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include #include #include +#include + +#include + namespace raft { namespace random { diff --git a/cpp/test/random/sample_without_replacement.cu b/cpp/test/random/sample_without_replacement.cu index 0d02567ea6..81de89921f 100644 --- a/cpp/test/random/sample_without_replacement.cu +++ b/cpp/test/random/sample_without_replacement.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,16 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include #include #include + +#include + #include #include diff --git a/cpp/test/sparse/add.cu b/cpp/test/sparse/add.cu index 9f0ddd551d..c8f9f546ac 100644 --- a/cpp/test/sparse/add.cu +++ b/cpp/test/sparse/add.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include +#include "../test_utils.cuh" +#include #include #include #include - -#include "../test_utils.cuh" #include +#include + #include #include diff --git a/cpp/test/sparse/convert_coo.cu b/cpp/test/sparse/convert_coo.cu index f05b6dfb08..a33bec1c0b 100644 --- a/cpp/test/sparse/convert_coo.cu +++ b/cpp/test/sparse/convert_coo.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,15 @@ * limitations under the License. */ -#include -#include +#include "../test_utils.cuh" +#include #include #include #include - #include -#include "../test_utils.cuh" +#include #include #include diff --git a/cpp/test/sparse/convert_csr.cu b/cpp/test/sparse/convert_csr.cu index 8be107ef3e..4af792a9ea 100644 --- a/cpp/test/sparse/convert_csr.cu +++ b/cpp/test/sparse/convert_csr.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,13 @@ */ #include "../test_utils.cuh" -#include -#include -#include +#include #include #include +#include + +#include #include diff --git a/cpp/test/sparse/csr_row_slice.cu b/cpp/test/sparse/csr_row_slice.cu index 1438498628..f25287d45a 100644 --- a/cpp/test/sparse/csr_row_slice.cu +++ b/cpp/test/sparse/csr_row_slice.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,18 @@ * limitations under the License. */ -#include +#include "../test_utils.cuh" + #include #include -#include - -#include #include #include +#include #include -#include "../test_utils.cuh" +#include +#include namespace raft { namespace sparse { diff --git a/cpp/test/sparse/csr_to_dense.cu b/cpp/test/sparse/csr_to_dense.cu index 47168c91ef..a093e64b73 100644 --- a/cpp/test/sparse/csr_to_dense.cu +++ b/cpp/test/sparse/csr_to_dense.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,18 @@ * limitations under the License. */ -#include +#include "../test_utils.cuh" + #include #include -#include - -#include #include #include +#include #include -#include "../test_utils.cuh" +#include +#include namespace raft { namespace sparse { diff --git a/cpp/test/sparse/csr_transpose.cu b/cpp/test/sparse/csr_transpose.cu index 2379fa2de9..f4d010f62e 100644 --- a/cpp/test/sparse/csr_transpose.cu +++ b/cpp/test/sparse/csr_transpose.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,16 @@ * limitations under the License. */ -#include -#include - -#include +#include "../test_utils.cuh" +#include #include #include #include #include -#include "../test_utils.cuh" +#include +#include namespace raft { namespace sparse { diff --git a/cpp/test/sparse/degree.cu b/cpp/test/sparse/degree.cu index b5b22ff15c..3449104929 100644 --- a/cpp/test/sparse/degree.cu +++ b/cpp/test/sparse/degree.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,11 @@ */ #include "../test_utils.cuh" -#include -#include #include +#include + +#include #include diff --git a/cpp/test/sparse/dist_coo_spmv.cu b/cpp/test/sparse/dist_coo_spmv.cu index c729334d00..591cd31e55 100644 --- a/cpp/test/sparse/dist_coo_spmv.cu +++ b/cpp/test/sparse/dist_coo_spmv.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,21 +14,21 @@ * limitations under the License. */ -#include -#include +#include "../test_utils.cuh" #include #include +#include #include #include +#include #include +#include #include -#include -#include -#include +#include -#include "../test_utils.cuh" +#include #include diff --git a/cpp/test/sparse/distance.cu b/cpp/test/sparse/distance.cu index 6b4e5c7cfa..a61d212720 100644 --- a/cpp/test/sparse/distance.cu +++ b/cpp/test/sparse/distance.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,16 @@ * limitations under the License. */ -#include -#include - -#include +#include "../test_utils.cuh" +#include #include #include -#include - #include +#include -#include "../test_utils.cuh" +#include +#include namespace raft { namespace sparse { diff --git a/cpp/test/sparse/filter.cu b/cpp/test/sparse/filter.cu index 956bb9c351..5816942d7c 100644 --- a/cpp/test/sparse/filter.cu +++ b/cpp/test/sparse/filter.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,15 @@ */ #include "../test_utils.cuh" -#include -#include -#include +#include #include #include #include #include +#include + +#include #include diff --git a/cpp/test/sparse/gram.cu b/cpp/test/sparse/gram.cu index ca43aa83b9..3505a3ddf5 100644 --- a/cpp/test/sparse/gram.cu +++ b/cpp/test/sparse/gram.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,9 +21,7 @@ #include "../distance/gram_base.cuh" #include "../test_utils.cuh" -#include -#include -#include + #include #include #include @@ -31,8 +29,14 @@ #include #include #include + #include +#include + +#include +#include + namespace raft::distance::kernels { /** diff --git a/cpp/test/sparse/mst.cu b/cpp/test/sparse/mst.cu index de0694ca10..6efd22967b 100644 --- a/cpp/test/sparse/mst.cu +++ b/cpp/test/sparse/mst.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,30 +14,25 @@ * limitations under the License. */ -#include -#include - #include "../test_utils.cuh" -#include -#include -#include -#include +#include #include #include #include #include - -#include - -#include +#include #include #include #include +#include +#include + #include +#include #include template diff --git a/cpp/test/sparse/neighbors/brute_force.cu b/cpp/test/sparse/neighbors/brute_force.cu index ed5b92afc2..ed2705a253 100644 --- a/cpp/test/sparse/neighbors/brute_force.cu +++ b/cpp/test/sparse/neighbors/brute_force.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include -#include - #include "../../test_utils.cuh" + +#include #include #include - #include +#include +#include + namespace raft { namespace sparse { namespace selection { diff --git a/cpp/test/sparse/neighbors/cross_component_nn.cu b/cpp/test/sparse/neighbors/cross_component_nn.cu index 7cadf25e88..13f7ae9de4 100644 --- a/cpp/test/sparse/neighbors/cross_component_nn.cu +++ b/cpp/test/sparse/neighbors/cross_component_nn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,28 +24,28 @@ // // TODO: edge case testing. Reference: https://github.com/rapidsai/raft/issues/1669 -#include -#include - -#include - -#include -#include -#include - -#include -#include -#include -#include +#include "../../test_utils.cuh" +#include #include #include #include #include #include +#include +#include +#include +#include +#include +#include + #include -#include "../../test_utils.cuh" +#include + +#include + +#include namespace raft { namespace sparse { diff --git a/cpp/test/sparse/neighbors/knn_graph.cu b/cpp/test/sparse/neighbors/knn_graph.cu index 232bd76da1..db610099e9 100644 --- a/cpp/test/sparse/neighbors/knn_graph.cu +++ b/cpp/test/sparse/neighbors/knn_graph.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,16 @@ */ #include "../../test_utils.cuh" -#include + #include +#include +#include #include + #include #include -#include -#include +#include #include diff --git a/cpp/test/sparse/norm.cu b/cpp/test/sparse/norm.cu index 10003b1fb6..32ccacd463 100644 --- a/cpp/test/sparse/norm.cu +++ b/cpp/test/sparse/norm.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include - #include "../test_utils.cuh" +#include #include #include #include #include +#include + #include #include diff --git a/cpp/test/sparse/normalize.cu b/cpp/test/sparse/normalize.cu index 77ddda225a..1e198d00c4 100644 --- a/cpp/test/sparse/normalize.cu +++ b/cpp/test/sparse/normalize.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include - #include "../test_utils.cuh" +#include #include #include #include #include +#include + #include #include diff --git a/cpp/test/sparse/reduce.cu b/cpp/test/sparse/reduce.cu index 7713d96821..f777f4781d 100644 --- a/cpp/test/sparse/reduce.cu +++ b/cpp/test/sparse/reduce.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,21 @@ * limitations under the License. */ -#include -#include - #include "../test_utils.cuh" -#include -#include + +#include #include #include #include #include + #include +#include + +#include +#include + namespace raft { namespace sparse { diff --git a/cpp/test/sparse/row_op.cu b/cpp/test/sparse/row_op.cu index 4d15a26b9d..702751f290 100644 --- a/cpp/test/sparse/row_op.cu +++ b/cpp/test/sparse/row_op.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * limitations under the License. */ -#include -#include +#include "../test_utils.cuh" +#include #include #include - -#include "../test_utils.cuh" #include +#include + #include #include diff --git a/cpp/test/sparse/sddmm.cu b/cpp/test/sparse/sddmm.cu index 9323ee8c2b..8ff20581c9 100644 --- a/cpp/test/sparse/sddmm.cu +++ b/cpp/test/sparse/sddmm.cu @@ -14,19 +14,20 @@ * limitations under the License. */ -#include - -#include -#include +#include "../test_utils.cuh" #include #include #include #include #include + #include -#include "../test_utils.cuh" +#include + +#include +#include namespace raft { namespace sparse { diff --git a/cpp/test/sparse/sort.cu b/cpp/test/sparse/sort.cu index 2e26225e78..2e3464f19f 100644 --- a/cpp/test/sparse/sort.cu +++ b/cpp/test/sparse/sort.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,13 @@ */ #include "../test_utils.cuh" -#include + #include #include +#include #include -#include +#include #include #include diff --git a/cpp/test/sparse/spectral_matrix.cu b/cpp/test/sparse/spectral_matrix.cu index 0bed73a722..52f7eff10e 100644 --- a/cpp/test/sparse/spectral_matrix.cu +++ b/cpp/test/sparse/spectral_matrix.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,16 @@ * limitations under the License. */ -#include -#include -#include #include #include #include - #include +#include + +#include +#include + namespace raft { namespace spectral { namespace matrix { diff --git a/cpp/test/sparse/spgemmi.cu b/cpp/test/sparse/spgemmi.cu index 71c18dd9ac..1811f3cef4 100644 --- a/cpp/test/sparse/spgemmi.cu +++ b/cpp/test/sparse/spgemmi.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,19 @@ * limitations under the License. */ -#include -#include -#include - #include "../test_utils.cuh" +#include +#include #include #include #include #include + #include +#include + #include #include diff --git a/cpp/test/sparse/spmm.cu b/cpp/test/sparse/spmm.cu new file mode 100644 index 0000000000..f53e857cd8 --- /dev/null +++ b/cpp/test/sparse/spmm.cu @@ -0,0 +1,327 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../test_utils.cuh" + +#include +#include +#include +#include +#include + +#include + +#include + +namespace raft { +namespace sparse { +namespace linalg { + +template +struct SpmmInputs { + bool trans_x; + bool trans_y; + int M; + int N; + int K; + int ldy; + int ldz; + bool row_major; + T alpha; + T beta; + unsigned long long int seed; +}; + +template +::std::ostream& operator<<(::std::ostream& os, const SpmmInputs& params) +{ + os << " trans_x: " << params.trans_x << " trans_y: " << params.trans_y << " M: " << params.M + << ", N: " << params.N << ", K: " << params.K << ", ldy: " << params.ldy + << ", ldz: " << params.ldz << ", row_major: " << params.row_major + << ", alpha: " << params.alpha << ", beta: " << params.beta; + return os; +} + +// Reference GEMM implementation. +template +RAFT_KERNEL naiveGemm(bool trans_x, + bool trans_y, + int M, + int N, + int K, + T alpha, + T* X, + int ldx, + bool x_row_major, + T* Y, + int ldy, + bool y_row_major, + T beta, + T* Z, + int ldz, + bool z_row_major) +{ + int tidx = blockIdx.x * blockDim.x + threadIdx.x; + int tidy = blockIdx.y * blockDim.y + threadIdx.y; + + for (int m = tidy; m < M; m += (blockDim.y * gridDim.y)) { + for (int n = tidx; n < N; n += (blockDim.x * gridDim.x)) { + T temp = T(0.0); + for (int k = 0; k < K; k++) { + int xIndex = x_row_major != trans_x ? m * ldx + k : m + k * ldx; + int yIndex = y_row_major != trans_y ? k * ldy + n : k + n * ldy; + temp += X[xIndex] * Y[yIndex]; + } + int zIndex = z_row_major ? m * ldz + n : m + n * ldz; + Z[zIndex] = beta * Z[zIndex] + alpha * temp; + } + } +} + +template +class SpmmTest : public ::testing::TestWithParam> { + protected: + void SetUp() override + { + params = ::testing::TestWithParam>::GetParam(); + + cudaStream_t stream = resource::get_cuda_stream(handle); + + // We compute Z = X * Y and compare against reference result + // Dimensions of X : M x K + // Dimensions of Y : K x N + // Dimensions of Z : M x N + + auto [ldx, ldy, ldz, x_size, y_size, z_size] = getXYZStrides(); + + RAFT_CUDA_TRY(cudaMalloc(&X, x_size * sizeof(T))); + RAFT_CUDA_TRY(cudaMalloc(&Y, y_size * sizeof(T))); + RAFT_CUDA_TRY(cudaMalloc(&Z_ref, z_size * sizeof(T))); + RAFT_CUDA_TRY(cudaMalloc(&Z, z_size * sizeof(T))); + + raft::random::RngState r(params.seed); + raft::random::uniform(handle, r, X, x_size, T(-10.0), T(10.0)); + raft::random::uniform(handle, r, Y, y_size, T(-10.0), T(10.0)); + + RAFT_CUDA_TRY( + cudaMalloc(&X_indptr, ((params.trans_x ? params.K : params.M) + 1) * sizeof(int))); + RAFT_CUDA_TRY(cudaMalloc(&X_indices, z_size * sizeof(int))); + RAFT_CUDA_TRY(cudaMalloc(&X_data, z_size * sizeof(T))); + + // this will erase all entries below eps and update X accordingly + X_nnz = generateCsrFromDense(X, + params.trans_x ? params.K : params.M, + params.trans_x ? params.M : params.K, + X_indptr, + X_indices, + X_data); + + resource::sync_stream(handle); + } + + std::tuple getXYZStrides() + { + // X is always row major + int ldx = params.trans_x ? params.M : params.K; + int ldy = + params.ldy > 0 ? params.ldy : (params.row_major != params.trans_y ? params.N : params.K); + int ldz = params.ldz > 0 ? params.ldz : (params.row_major ? params.N : params.M); + size_t x_size = params.M * params.K; + size_t y_size = params.row_major != params.trans_y ? params.K * ldy : params.N * ldy; + size_t z_size = params.row_major ? params.M * ldz : params.N * ldz; + + return {ldx, ldy, ldz, x_size, y_size, z_size}; + } + + void runTest() + { + auto stream = resource::get_cuda_stream(handle); + + auto [ldx, ldy, ldz, x_size, y_size, z_size] = getXYZStrides(); + + thrust::fill(resource::get_thrust_policy(handle), Z_ref, Z_ref + z_size, T(0.0)); + thrust::fill(resource::get_thrust_policy(handle), Z, Z + z_size, T(0.0)); + + // create csr matrix view + auto X_csr_structure = raft::make_device_compressed_structure_view( + X_indptr, + X_indices, + params.trans_x ? params.K : params.M, + params.trans_x ? params.M : params.K, + X_nnz); + auto X_csr = raft::device_csr_matrix_view( + raft::device_span(X_data, X_csr_structure.get_nnz()), X_csr_structure); + + auto y_stride_view = + params.row_major + ? raft::make_device_strided_matrix_view( + Y, params.trans_y ? params.N : params.K, params.trans_y ? params.K : params.N, ldy) + : raft::make_device_strided_matrix_view( + Y, params.trans_y ? params.N : params.K, params.trans_y ? params.K : params.N, ldy); + + auto z_stride_view = params.row_major + ? raft::make_device_strided_matrix_view( + Z, params.M, params.N, ldz) + : raft::make_device_strided_matrix_view( + Z, params.M, params.N, ldz); + + T alpha = params.alpha; + T beta = params.beta; + + dim3 blocks(raft::ceildiv(params.M, 128), raft::ceildiv(params.N, 4), 1); + dim3 threads(128, 4, 1); + naiveGemm<<>>(params.trans_x, + params.trans_y, + params.M, + params.N, + params.K, + alpha, + X, + ldx, + true, + Y, + ldy, + params.row_major, + beta, + Z_ref, + ldz, + params.row_major); + + spmm( + handle, params.trans_x, params.trans_y, &alpha, X_csr, y_stride_view, &beta, z_stride_view); + + resource::sync_stream(handle, stream); + + ASSERT_TRUE(raft::devArrMatch(Z_ref, Z, z_size, raft::CompareApprox(1e-3f), stream)); + } + + void TearDown() override + { + RAFT_CUDA_TRY(cudaFree(Z_ref)); + RAFT_CUDA_TRY(cudaFree(Z)); + RAFT_CUDA_TRY(cudaFree(X)); + RAFT_CUDA_TRY(cudaFree(Y)); + RAFT_CUDA_TRY(cudaFree(X_indptr)); + RAFT_CUDA_TRY(cudaFree(X_indices)); + RAFT_CUDA_TRY(cudaFree(X_data)); + } + + int generateCsrFromDense(T* dense, int n_rows, int n_cols, int* indptr, int* indices, T* data) + { + double eps = 1e-4; + + cudaStream_t stream = resource::get_cuda_stream(handle); + + size_t dense_size = n_rows * n_cols; + std::vector dense_host(dense_size); + raft::update_host(dense_host.data(), dense, dense_size, stream); + resource::sync_stream(handle, stream); + + std::vector indptr_host(n_rows + 1); + std::vector indices_host(dense_size); + std::vector data_host(dense_size); + + // create csr matrix from dense (with threshold) + int nnz = 0; + for (int i = 0; i < n_rows; ++i) { + indptr_host[i] = nnz; + for (int j = 0; j < n_cols; ++j) { + T value = dense_host[i * n_cols + j]; + if (value > eps) { + indices_host[nnz] = j; + data_host[nnz] = value; + nnz++; + } else { + dense_host[i * n_cols + j] = T(0.0); + } + } + } + indptr_host[n_rows] = nnz; + + raft::update_device(dense, dense_host.data(), dense_size, stream); + raft::update_device(indptr, indptr_host.data(), n_rows + 1, stream); + raft::update_device(indices, indices_host.data(), nnz, stream); + raft::update_device(data, data_host.data(), nnz, stream); + resource::sync_stream(handle, stream); + return nnz; + } + + protected: + raft::resources handle; + SpmmInputs params; + + T* X = NULL; + T* Y = NULL; + T* Z_ref = NULL; + T* Z = NULL; + + // CSR + int* X_indptr = NULL; + int* X_indices = NULL; + T* X_data = NULL; + int X_nnz = 0; +}; + +// M / N / K / ldy / ldz / y_row_major / z_row_major / seed +const std::vector> inputsf = { + {true, false, 80, 70, 80, 0, 0, true, 1.0, 2.0, 76430ULL}, + {false, false, 80, 100, 40, 0, 0, true, 3.0, 0.0, 426646ULL}, + {false, false, 20, 100, 20, 0, 0, false, 2.0, 2.0, 237703ULL}, + {false, true, 100, 60, 30, 0, 0, false, 1.0, 0.0, 538004ULL}, + {false, false, 80, 70, 80, 106, 0, true, 1.0, 2.0, 76430ULL}, + {false, false, 80, 100, 40, 0, 110, true, 3.0, 0.0, 426646ULL}, + {true, false, 20, 100, 20, 106, 0, false, 2.0, 2.0, 237703ULL}, + {false, false, 100, 60, 30, 0, 110, false, 1.0, 0.0, 538004ULL}, + {false, false, 50, 10, 60, 106, 110, true, 1.0, 1.0, 73012ULL}, + {true, false, 90, 90, 30, 106, 110, true, 1.0, 0.0, 538147ULL}, + {false, false, 30, 100, 10, 106, 110, false, 1.0, 1.0, 412352ULL}, + {false, false, 40, 80, 100, 106, 110, false, 1.0, 0.0, 2979410ULL}, + {true, false, 50, 10, 60, 106, 110, true, 1.0, 1.0, 73012ULL}, + {true, true, 90, 90, 30, 106, 110, true, 1.0, 0.0, 538147ULL}, + {false, true, 30, 100, 10, 106, 110, false, 1.0, 1.0, 412352ULL}, + {true, true, 40, 80, 100, 106, 110, false, 1.0, 0.0, 2979410ULL}}; + +const std::vector> inputsd = { + {false, false, 10, 70, 40, 0, 0, true, 1.0, 2.0, 535648ULL}, + {false, false, 30, 30, 30, 0, 0, true, 3.0, 0.0, 956681ULL}, + {true, false, 70, 80, 50, 0, 0, false, 2.0, 2.0, 875083ULL}, + {false, false, 80, 90, 70, 0, 0, false, 1.0, 0.0, 50744ULL}, + {false, false, 10, 70, 40, 106, 0, true, 1.0, 2.0, 535648ULL}, + {true, false, 30, 30, 30, 0, 110, true, 3.0, 0.0, 956681ULL}, + {false, false, 70, 80, 50, 106, 0, false, 2.0, 2.0, 875083ULL}, + {false, false, 80, 90, 70, 0, 110, false, 1.0, 0.0, 50744ULL}, + {false, true, 90, 90, 30, 106, 110, true, 1.0, 1.0, 506321ULL}, + {false, false, 40, 100, 70, 106, 110, true, 1.0, 0.0, 638418ULL}, + {false, true, 80, 50, 30, 106, 110, false, 1.0, 1.0, 701529ULL}, + {false, false, 50, 80, 60, 106, 110, false, 1.0, 0.0, 893038ULL}, + {true, false, 90, 90, 30, 106, 110, true, 1.0, 1.0, 506321ULL}, + {true, true, 40, 100, 70, 106, 110, true, 1.0, 0.0, 638418ULL}, + {false, true, 80, 50, 30, 106, 110, false, 1.0, 1.0, 701529ULL}, + {true, true, 50, 80, 60, 106, 110, false, 1.0, 0.0, 893038ULL}}; + +typedef SpmmTest SpmmTestF; +TEST_P(SpmmTestF, Result) { runTest(); } + +typedef SpmmTest SpmmTestD; +TEST_P(SpmmTestD, Result) { runTest(); } + +INSTANTIATE_TEST_SUITE_P(SpmmTests, SpmmTestF, ::testing::ValuesIn(inputsf)); + +INSTANTIATE_TEST_SUITE_P(SpmmTests, SpmmTestD, ::testing::ValuesIn(inputsd)); + +} // end namespace linalg +} // end namespace sparse +} // end namespace raft diff --git a/cpp/test/sparse/symmetrize.cu b/cpp/test/sparse/symmetrize.cu index 666396eb5b..e1a74dc40b 100644 --- a/cpp/test/sparse/symmetrize.cu +++ b/cpp/test/sparse/symmetrize.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,18 @@ * limitations under the License. */ -#include +#include "../test_utils.cuh" + #include #include #include #include #include + #include #include -#include "../test_utils.cuh" +#include #include diff --git a/cpp/test/stats/accuracy.cu b/cpp/test/stats/accuracy.cu index 5bc0506e7f..5fa2b3b6de 100644 --- a/cpp/test/stats/accuracy.cu +++ b/cpp/test/stats/accuracy.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,20 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include #include #include + #include + +#include #include #include + +#include #include namespace raft { diff --git a/cpp/test/stats/adjusted_rand_index.cu b/cpp/test/stats/adjusted_rand_index.cu index fb7b3825fc..f2207dfd21 100644 --- a/cpp/test/stats/adjusted_rand_index.cu +++ b/cpp/test/stats/adjusted_rand_index.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,16 @@ */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include + +#include + +#include +#include #include namespace raft { diff --git a/cpp/test/stats/completeness_score.cu b/cpp/test/stats/completeness_score.cu index c5c134418c..b985799b0f 100644 --- a/cpp/test/stats/completeness_score.cu +++ b/cpp/test/stats/completeness_score.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,17 @@ * limitations under the License. */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include #include + +#include + +#include +#include #include namespace raft { diff --git a/cpp/test/stats/contingencyMatrix.cu b/cpp/test/stats/contingencyMatrix.cu index acfd1aecfe..221c4b4613 100644 --- a/cpp/test/stats/contingencyMatrix.cu +++ b/cpp/test/stats/contingencyMatrix.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,20 @@ */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include -#include + #include +#include + +#include +#include +#include + namespace raft { namespace stats { diff --git a/cpp/test/stats/cov.cu b/cpp/test/stats/cov.cu index ca9d437717..41812979b6 100644 --- a/cpp/test/stats/cov.cu +++ b/cpp/test/stats/cov.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,17 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include #include + #include +#include + namespace raft { namespace stats { diff --git a/cpp/test/stats/dispersion.cu b/cpp/test/stats/dispersion.cu index 9ef678050d..54b1cee4dd 100644 --- a/cpp/test/stats/dispersion.cu +++ b/cpp/test/stats/dispersion.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,20 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include #include #include + #include + +#include #include #include + +#include #include namespace raft { diff --git a/cpp/test/stats/entropy.cu b/cpp/test/stats/entropy.cu index dea8828b26..54a1e88dcd 100644 --- a/cpp/test/stats/entropy.cu +++ b/cpp/test/stats/entropy.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,20 @@ * limitations under the License. */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include -#include + #include +#include + +#include +#include +#include + namespace raft { namespace stats { diff --git a/cpp/test/stats/histogram.cu b/cpp/test/stats/histogram.cu index 86f708db62..18b8436a73 100644 --- a/cpp/test/stats/histogram.cu +++ b/cpp/test/stats/histogram.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -24,6 +24,8 @@ #include #include +#include + namespace raft { namespace stats { diff --git a/cpp/test/stats/homogeneity_score.cu b/cpp/test/stats/homogeneity_score.cu index 88247f5b50..bcce94fca0 100644 --- a/cpp/test/stats/homogeneity_score.cu +++ b/cpp/test/stats/homogeneity_score.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,16 @@ * limitations under the License. */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include + +#include + +#include +#include #include namespace raft { diff --git a/cpp/test/stats/information_criterion.cu b/cpp/test/stats/information_criterion.cu index 9e57f2c84f..ac3dff1d7a 100644 --- a/cpp/test/stats/information_criterion.cu +++ b/cpp/test/stats/information_criterion.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,12 @@ */ #include "../test_utils.cuh" -#include - -#include +#include #include +#include #include + #include #include diff --git a/cpp/test/stats/kl_divergence.cu b/cpp/test/stats/kl_divergence.cu index 5714583675..477fd6d892 100644 --- a/cpp/test/stats/kl_divergence.cu +++ b/cpp/test/stats/kl_divergence.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,15 @@ * limitations under the License. */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include + +#include + +#include +#include #include namespace raft { diff --git a/cpp/test/stats/mean.cu b/cpp/test/stats/mean.cu index 0cb90b6d46..67931e2eed 100644 --- a/cpp/test/stats/mean.cu +++ b/cpp/test/stats/mean.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include #include + +#include #include #include diff --git a/cpp/test/stats/mean_center.cu b/cpp/test/stats/mean_center.cu index 4d797e60fe..b44d87d1bd 100644 --- a/cpp/test/stats/mean_center.cu +++ b/cpp/test/stats/mean_center.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,15 @@ #include "../linalg/matrix_vector_op.cuh" #include "../test_utils.cuh" -#include + #include #include #include #include #include +#include + namespace raft { namespace stats { diff --git a/cpp/test/stats/meanvar.cu b/cpp/test/stats/meanvar.cu index df3d9d9c00..725e773cb0 100644 --- a/cpp/test/stats/meanvar.cu +++ b/cpp/test/stats/meanvar.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,15 @@ */ #include "../test_utils.cuh" -#include + #include #include #include #include #include +#include + #include namespace raft { diff --git a/cpp/test/stats/minmax.cu b/cpp/test/stats/minmax.cu index 3715bc5bd3..7563cb12be 100644 --- a/cpp/test/stats/minmax.cu +++ b/cpp/test/stats/minmax.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,7 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include @@ -24,9 +23,13 @@ #include #include #include + +#include #include #include +#include + namespace raft { namespace stats { diff --git a/cpp/test/stats/mutual_info_score.cu b/cpp/test/stats/mutual_info_score.cu index 9f31350844..811bc793cf 100644 --- a/cpp/test/stats/mutual_info_score.cu +++ b/cpp/test/stats/mutual_info_score.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,16 @@ * limitations under the License. */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include + +#include + +#include +#include #include namespace raft { diff --git a/cpp/test/stats/neighborhood_recall.cu b/cpp/test/stats/neighborhood_recall.cu index 43ae7059bd..1a76154e2e 100644 --- a/cpp/test/stats/neighborhood_recall.cu +++ b/cpp/test/stats/neighborhood_recall.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,12 +21,11 @@ #include #include #include - -#include - #include #include +#include + #include namespace raft::stats { diff --git a/cpp/test/stats/r2_score.cu b/cpp/test/stats/r2_score.cu index aa4f069f09..a4a3ed4de6 100644 --- a/cpp/test/stats/r2_score.cu +++ b/cpp/test/stats/r2_score.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,20 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include #include #include + #include + +#include #include #include + +#include #include namespace raft { diff --git a/cpp/test/stats/rand_index.cu b/cpp/test/stats/rand_index.cu index 41b0823e76..678bc455f0 100644 --- a/cpp/test/stats/rand_index.cu +++ b/cpp/test/stats/rand_index.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,16 @@ */ #include "../test_utils.cuh" -#include +#include +#include +#include #include #include #include #include -#include -#include #include namespace raft { diff --git a/cpp/test/stats/regression_metrics.cu b/cpp/test/stats/regression_metrics.cu index b0c4cca530..d03784f834 100644 --- a/cpp/test/stats/regression_metrics.cu +++ b/cpp/test/stats/regression_metrics.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,17 +15,21 @@ */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include #include + #include + +#include #include #include + +#include +#include #include namespace raft { diff --git a/cpp/test/stats/silhouette_score.cu b/cpp/test/stats/silhouette_score.cu index ed69b2f3ac..ad080f5894 100644 --- a/cpp/test/stats/silhouette_score.cu +++ b/cpp/test/stats/silhouette_score.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,20 @@ * limitations under the License. */ #include "../test_utils.cuh" -#include -#include -#include + #include #include +#include #include -#include -#include #include +#include + +#include +#include +#include + namespace raft { namespace stats { diff --git a/cpp/test/stats/stddev.cu b/cpp/test/stats/stddev.cu index 998f7a88e7..cf57d3a923 100644 --- a/cpp/test/stats/stddev.cu +++ b/cpp/test/stats/stddev.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "../test_utils.cuh" -#include + #include #include #include @@ -23,6 +23,8 @@ #include #include +#include + namespace raft { namespace stats { diff --git a/cpp/test/stats/sum.cu b/cpp/test/stats/sum.cu index 290c53a563..5a549f8ba4 100644 --- a/cpp/test/stats/sum.cu +++ b/cpp/test/stats/sum.cu @@ -15,8 +15,8 @@ */ #include "../test_utils.cuh" -#include +#include #include #include #include diff --git a/cpp/test/stats/trustworthiness.cu b/cpp/test/stats/trustworthiness.cu index 502a59f5c2..846c192022 100644 --- a/cpp/test/stats/trustworthiness.cu +++ b/cpp/test/stats/trustworthiness.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,15 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include +#include #include -#include +#include + +#include #include namespace raft { diff --git a/cpp/test/stats/v_measure.cu b/cpp/test/stats/v_measure.cu index 0cc164f277..c44df84ada 100644 --- a/cpp/test/stats/v_measure.cu +++ b/cpp/test/stats/v_measure.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,16 @@ * limitations under the License. */ #include "../test_utils.cuh" -#include -#include -#include + #include #include #include #include + +#include + +#include +#include #include namespace raft { diff --git a/cpp/test/stats/weighted_mean.cu b/cpp/test/stats/weighted_mean.cu index da1a825da1..407f3f14ea 100644 --- a/cpp/test/stats/weighted_mean.cu +++ b/cpp/test/stats/weighted_mean.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,20 @@ */ #include "../test_utils.cuh" -#include -#include + #include #include #include #include #include + #include #include +#include + +#include + namespace raft { namespace stats { diff --git a/cpp/test/test.cpp b/cpp/test/test.cpp index 7477d7d0b5..b4727103f4 100644 --- a/cpp/test/test.cpp +++ b/cpp/test/test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,11 @@ * limitations under the License. */ +#include + #include + #include -#include namespace raft { diff --git a/cpp/test/test_utils.cuh b/cpp/test/test_utils.cuh index 1afa7acc83..810a0d7985 100644 --- a/cpp/test/test_utils.cuh +++ b/cpp/test/test_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,16 +18,19 @@ #include "test_utils.h" -#include -#include -#include #include #include #include + #include + #include +#include + #include +#include +#include #include #include #include diff --git a/cpp/test/test_utils.h b/cpp/test/test_utils.h index 75590463b0..cf9a885cfb 100644 --- a/cpp/test/test_utils.h +++ b/cpp/test/test_utils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,12 @@ #pragma once -#include -#include +#include #include -#include +#include +#include namespace raft { diff --git a/cpp/test/util/cudart_utils.cpp b/cpp/test/util/cudart_utils.cpp index 3fde494823..0ca05d5006 100644 --- a/cpp/test/util/cudart_utils.cpp +++ b/cpp/test/util/cudart_utils.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ #include #include #include -#include #include @@ -25,6 +24,7 @@ #include #include +#include namespace raft { diff --git a/cpp/test/util/device_atomics.cu b/cpp/test/util/device_atomics.cu index f1a97794ed..c5bb0ad3b6 100644 --- a/cpp/test/util/device_atomics.cu +++ b/cpp/test/util/device_atomics.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,21 @@ * limitations under the License. */ +#include +#include + +#include +#include +#include + +#include + #include #include #include -#include #include #include #include -#include -#include -#include -#include -#include namespace raft { diff --git a/cpp/test/util/integer_utils.cpp b/cpp/test/util/integer_utils.cpp index ed5dddf72d..7101fd2740 100644 --- a/cpp/test/util/integer_utils.cpp +++ b/cpp/test/util/integer_utils.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,11 @@ * limitations under the License. */ +#include + #include + #include -#include namespace raft { diff --git a/cpp/test/util/integer_utils.cu b/cpp/test/util/integer_utils.cu index 1682bc9dc7..83f94d7fad 100644 --- a/cpp/test/util/integer_utils.cu +++ b/cpp/test/util/integer_utils.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,14 @@ */ #include "../test_utils.cuh" -#include + #include #include + #include +#include + namespace raft { namespace util { diff --git a/cpp/test/util/memory_type_dispatcher.cu b/cpp/test/util/memory_type_dispatcher.cu index 5e24ff5719..5b8d36af73 100644 --- a/cpp/test/util/memory_type_dispatcher.cu +++ b/cpp/test/util/memory_type_dispatcher.cu @@ -15,8 +15,7 @@ */ #include "../test_utils.h" -#include -#include + #include #include #include @@ -28,6 +27,10 @@ #include #include #include + +#include + +#include #include #include diff --git a/cpp/test/util/pow2_utils.cu b/cpp/test/util/pow2_utils.cu index e29e4eeb9c..f618852596 100644 --- a/cpp/test/util/pow2_utils.cu +++ b/cpp/test/util/pow2_utils.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ * limitations under the License. */ -#include #include +#include + namespace raft { template diff --git a/dependencies.yaml b/dependencies.yaml index a451b6836f..72aa3427d1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -392,13 +392,17 @@ dependencies: packages: - python=3.10 - matrix: + py: "3.11" packages: - - python>=3.9,<3.11 + - python=3.11 + - matrix: + packages: + - python>=3.9,<3.12 run_pylibraft: common: - output_types: [conda, pyproject] packages: - - &numpy numpy>=1.21 + - &numpy numpy>=1.23 - output_types: [conda] packages: - *rmm_conda diff --git a/python/pylibraft/pylibraft/test/test_ivf_flat.py b/python/pylibraft/pylibraft/test/test_ivf_flat.py index 23140073f1..2e38dab7bc 100644 --- a/python/pylibraft/pylibraft/test/test_ivf_flat.py +++ b/python/pylibraft/pylibraft/test/test_ivf_flat.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -297,6 +297,14 @@ def test_ivf_flat_params(params): "k": 129, "n_probes": 100, }, + { + "k": 257, + "n_probes": 100, + }, + { + "k": 4096, + "n_probes": 100, + }, ], ) def test_ivf_pq_search_params(params): diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 5e9701a47a..6468220330 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -36,7 +36,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", - "numpy>=1.21", + "numpy>=1.23", "rmm==24.4.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -44,6 +44,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] [project.optional-dependencies] diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index bd256e8e92..b869290d5c 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "dask-cuda==24.4.*", "joblib>=0.11", "numba>=0.57", - "numpy>=1.21", + "numpy>=1.23", "pylibraft==24.4.*", "rapids-dask-dependency==24.4.*", "ucx-py==0.37.*", @@ -46,6 +46,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] [project.optional-dependencies]