Use template instantiations in benchmarks as well.

- move common -ext.cuh headers to raft_internal - move .cu files back to src
rapidsai · Mar 11, 2024 · 713951c · 713951c
1 parent 1df5de7
commit 713951c
Show file tree

Hide file tree

Showing 32 changed files with 50 additions and 37 deletions.
diff --git a/cpp/bench/prims/CMakeLists.txt b/cpp/bench/prims/CMakeLists.txt
@@ -156,6 +156,14 @@ if(BUILD_PRIMS_BENCH)
     bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu
     bench/prims/neighbors/knn/ivf_pq_int8_t_int64_t.cu
     bench/prims/neighbors/knn/ivf_pq_uint8_t_int64_t.cu
+    src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu
     bench/prims/neighbors/refine_float_int64_t.cu
     bench/prims/neighbors/refine_uint8_t_int64_t.cu
     bench/prims/main.cpp

diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-#undef RAFT_EXPLICIT_INSTANTIATE_ONLY  // Enable instantiation of search with filter
 #include "../knn.cuh"
 
+#include <raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh>
+#include <raft_internal/neighbors/test/neighborsivf_pq_search_test-ext.cuh>
 namespace raft::bench::spatial {
 
 KNN_REGISTER(float, int64_t, ivf_pq_filter_knn, kInputsFilter, kNoCopyOnly, kScopeFull);

diff --git a/...q_compute_similarity_filters_test-ext.cuh → ...q_compute_similarity_filters_test-ext.cuh b/...q_compute_similarity_filters_test-ext.cuh → ...q_compute_similarity_filters_test-ext.cuh
diff --git a/...ors/ann_ivf_pq/ivf_pq_search_test-ext.cuh → ...rnal/neighbors/ivf_pq_search_test-ext.cuh b/...ors/ann_ivf_pq/ivf_pq_search_test-ext.cuh → ...rnal/neighbors/ivf_pq_search_test-ext.cuh
@@ -16,15 +16,15 @@
 
 #pragma once
 
-#include "ivf_pq_compute_similarity_filters_test-ext.cuh"
-
 #include <raft/core/device_mdspan.hpp>  // raft::device_matrix_view
 #include <raft/core/resources.hpp>      // raft::resources
 #include <raft/neighbors/ivf_pq.cuh>
 #include <raft/neighbors/ivf_pq_types.hpp>  // raft::neighbors::ivf_pq::index
 #include <raft/neighbors/sample_filter.cuh>
 #include <raft/neighbors/sample_filter_types.hpp>
 
+#include <raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh>
+
 #include <cstdint>  // int64_t
 
 #define instantiate_raft_neighbors_ivf_pq_search(T, IdxT)            \

diff --git a/...ompute_similarity_float_float_bitset32.cu → ...ompute_similarity_float_float_bitset32.cu b/...ompute_similarity_float_float_bitset32.cu → ...ompute_similarity_float_float_bitset32.cu
diff --git a/...ompute_similarity_float_float_bitset64.cu → ...ompute_similarity_float_float_bitset64.cu b/...ompute_similarity_float_float_bitset64.cu → ...ompute_similarity_float_float_bitset64.cu
diff --git a/..._compute_similarity_float_float_filt32.cu → ..._compute_similarity_float_float_filt32.cu b/..._compute_similarity_float_float_filt32.cu → ..._compute_similarity_float_float_filt32.cu
diff --git a/...te_similarity_float_fp8_false_bitset32.cu → ...te_similarity_float_fp8_false_bitset32.cu b/...te_similarity_float_fp8_false_bitset32.cu → ...te_similarity_float_fp8_false_bitset32.cu
diff --git a/...te_similarity_float_fp8_false_bitset64.cu → ...te_similarity_float_fp8_false_bitset64.cu b/...te_similarity_float_fp8_false_bitset64.cu → ...te_similarity_float_fp8_false_bitset64.cu
diff --git a/...pute_similarity_float_fp8_false_filt32.cu → ...pute_similarity_float_fp8_false_filt32.cu b/...pute_similarity_float_fp8_false_filt32.cu → ...pute_similarity_float_fp8_false_filt32.cu
diff --git a/...ute_similarity_float_fp8_true_bitset32.cu → ...ute_similarity_float_fp8_true_bitset32.cu b/...ute_similarity_float_fp8_true_bitset32.cu → ...ute_similarity_float_fp8_true_bitset32.cu
diff --git a/...ute_similarity_float_fp8_true_bitset64.cu → ...ute_similarity_float_fp8_true_bitset64.cu b/...ute_similarity_float_fp8_true_bitset64.cu → ...ute_similarity_float_fp8_true_bitset64.cu
diff --git a/...mpute_similarity_float_fp8_true_filt32.cu → ...mpute_similarity_float_fp8_true_filt32.cu b/...mpute_similarity_float_fp8_true_filt32.cu → ...mpute_similarity_float_fp8_true_filt32.cu
diff --git a/...compute_similarity_float_half_bitset32.cu → ...compute_similarity_float_half_bitset32.cu b/...compute_similarity_float_half_bitset32.cu → ...compute_similarity_float_half_bitset32.cu
diff --git a/...compute_similarity_float_half_bitset64.cu → ...compute_similarity_float_half_bitset64.cu b/...compute_similarity_float_half_bitset64.cu → ...compute_similarity_float_half_bitset64.cu
diff --git a/...q_compute_similarity_float_half_filt32.cu → ...q_compute_similarity_float_half_filt32.cu b/...q_compute_similarity_float_half_filt32.cu → ...q_compute_similarity_float_half_filt32.cu
diff --git a/...ute_similarity_half_fp8_false_bitset32.cu → ...ute_similarity_half_fp8_false_bitset32.cu b/...ute_similarity_half_fp8_false_bitset32.cu → ...ute_similarity_half_fp8_false_bitset32.cu
diff --git a/...ute_similarity_half_fp8_false_bitset64.cu → ...ute_similarity_half_fp8_false_bitset64.cu b/...ute_similarity_half_fp8_false_bitset64.cu → ...ute_similarity_half_fp8_false_bitset64.cu
diff --git a/...mpute_similarity_half_fp8_false_filt32.cu → ...mpute_similarity_half_fp8_false_filt32.cu b/...mpute_similarity_half_fp8_false_filt32.cu → ...mpute_similarity_half_fp8_false_filt32.cu
diff --git a/...pute_similarity_half_fp8_true_bitset32.cu → ...pute_similarity_half_fp8_true_bitset32.cu b/...pute_similarity_half_fp8_true_bitset32.cu → ...pute_similarity_half_fp8_true_bitset32.cu
diff --git a/...pute_similarity_half_fp8_true_bitset64.cu → ...pute_similarity_half_fp8_true_bitset64.cu b/...pute_similarity_half_fp8_true_bitset64.cu → ...pute_similarity_half_fp8_true_bitset64.cu
diff --git a/...ompute_similarity_half_fp8_true_filt32.cu → ...ompute_similarity_half_fp8_true_filt32.cu b/...ompute_similarity_half_fp8_true_filt32.cu → ...ompute_similarity_half_fp8_true_filt32.cu
diff --git a/..._compute_similarity_half_half_bitset32.cu → ..._compute_similarity_half_half_bitset32.cu b/..._compute_similarity_half_half_bitset32.cu → ..._compute_similarity_half_half_bitset32.cu
diff --git a/..._compute_similarity_half_half_bitset64.cu → ..._compute_similarity_half_half_bitset64.cu b/..._compute_similarity_half_half_bitset64.cu → ..._compute_similarity_half_half_bitset64.cu
diff --git a/...pq_compute_similarity_half_half_filt32.cu → ...pq_compute_similarity_half_half_filt32.cu b/...pq_compute_similarity_half_half_filt32.cu → ...pq_compute_similarity_half_half_filt32.cu
diff --git a/.../ivf_pq_search_filtering_float_int64_t.cu → .../ivf_pq_search_filtering_float_int64_t.cu b/.../ivf_pq_search_filtering_float_int64_t.cu → .../ivf_pq_search_filtering_float_int64_t.cu
@@ -14,15 +14,15 @@
  * limitations under the License.
  */
 
-#include "ivf_pq_compute_similarity_filters_test-ext.cuh"
-
 #include <raft/core/device_mdspan.hpp>  // raft::device_matrix_view
 #include <raft/core/resources.hpp>      // raft::resources
 #include <raft/neighbors/ivf_pq-inl.cuh>
 #include <raft/neighbors/ivf_pq_types.hpp>  // raft::neighbors::ivf_pq::index
 #include <raft/neighbors/sample_filter.cuh>
 #include <raft/neighbors/sample_filter_types.hpp>
 
+#include <raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh>
+
 #include <cstdint>  // int64_t
 
 #define instantiate_raft_neighbors_ivf_pq_search_with_filtering(T, IdxT, FilterT) \

diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt
@@ -403,28 +403,28 @@ if(BUILD_TESTS)
     test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu
     test/neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu
     test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_search_filtering_float_int64_t.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_float_filt32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_fp8_false_filt32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_fp8_true_filt32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_half_filt32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_fp8_false_filt32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_fp8_true_filt32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_half_filt32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_float_bitset32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_half_bitset32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_half_bitset32.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_float_bitset64.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_float_half_bitset64.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu
-    test/neighbors/ann_ivf_pq/ivf_pq_compute_similarity_half_half_bitset64.cu
+    src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu
+    src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu
     test/neighbors/ann_ivf_pq/test_float_uint32_t.cu
     test/neighbors/ann_ivf_pq/test_float_int64_t.cu
     test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu

diff --git a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-#include "ivf_pq_compute_similarity_filters_test-ext.cuh"
-
 #include <raft/neighbors/ivf_pq-inl.cuh>
 #include <raft/neighbors/ivf_pq_types.hpp>  // raft::neighbors::ivf_pq::index
 #include <raft/neighbors/sample_filter.cuh>
 
+#include <raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh>
+
 #define instantiate_raft_neighbors_ivf_pq_search(T, IdxT)            \
   template void raft::neighbors::ivf_pq::search<T, IdxT>(            \
     raft::resources const& handle,                                   \

diff --git a/cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu
@@ -15,8 +15,9 @@
  */
 
 #include "../ann_ivf_pq.cuh"
-#include "ivf_pq_compute_similarity_filters_test-ext.cuh"
-#include "ivf_pq_search_test-ext.cuh"
+
+#include <raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh>
+#include <raft_internal/neighbors/ivf_pq_search_test-ext.cuh>
 
 namespace raft::neighbors::ivf_pq {
 

diff --git a/cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu
@@ -15,8 +15,9 @@
  */
 
 #include "../ann_ivf_pq.cuh"
-#include "ivf_pq_compute_similarity_filters_test-ext.cuh"
-#include "ivf_pq_search_test-ext.cuh"
+
+#include <raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh>
+#include <raft_internal/neighbors/ivf_pq_search_test-ext.cuh>
 
 namespace raft::neighbors::ivf_pq {
 

diff --git a/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu
@@ -16,8 +16,9 @@
 
 #include "../ann_ivf_pq.cuh"
 #include "ivf_pq_build_test-ext.cuh"
-#include "ivf_pq_compute_similarity_filters_test-ext.cuh"
-#include "ivf_pq_search_test-ext.cuh"
+
+#include <raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh>
+#include <raft_internal/neighbors/ivf_pq_search_test-ext.cuh>
 
 namespace raft::neighbors::ivf_pq {
 

diff --git a/cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu
@@ -15,7 +15,8 @@
  */
 
 #include "../ann_ivf_pq.cuh"
-#include "ivf_pq_search_test-ext.cuh"
+
+#include <raft_internal/neighbors/ivf_pq_search_test-ext.cuh>
 namespace raft::neighbors::ivf_pq {
 
 using f32_i08_i64 = ivf_pq_test<float, int8_t, int64_t>;