From 4a13077bf10e4f8abbb266202eceb2b3b4b75c75 Mon Sep 17 00:00:00 2001 From: achirkin Date: Wed, 13 Mar 2024 10:06:39 +0100 Subject: [PATCH] Fix incorrect signature in cagra-q single-cta instances --- .../cagra/q_search_single_cta_00_generate.py | 59 ++++++++++--------- ...float_uint32_dim1024_t32_8pq_2subd_half.cu | 20 ++++--- ...float_uint32_dim1024_t32_8pq_4subd_half.cu | 20 ++++--- ...a_float_uint32_dim128_t8_8pq_2subd_half.cu | 20 ++++--- ...a_float_uint32_dim128_t8_8pq_4subd_half.cu | 20 ++++--- ..._float_uint32_dim256_t16_8pq_2subd_half.cu | 20 ++++--- ..._float_uint32_dim256_t16_8pq_4subd_half.cu | 20 ++++--- ..._float_uint32_dim512_t32_8pq_2subd_half.cu | 20 ++++--- ..._float_uint32_dim512_t32_8pq_4subd_half.cu | 20 ++++--- ...float_uint64_dim1024_t32_8pq_2subd_half.cu | 20 ++++--- ...float_uint64_dim1024_t32_8pq_4subd_half.cu | 20 ++++--- ...a_float_uint64_dim128_t8_8pq_2subd_half.cu | 20 ++++--- ...a_float_uint64_dim128_t8_8pq_4subd_half.cu | 20 ++++--- ..._float_uint64_dim256_t16_8pq_2subd_half.cu | 20 ++++--- ..._float_uint64_dim256_t16_8pq_4subd_half.cu | 20 ++++--- ..._float_uint64_dim512_t32_8pq_2subd_half.cu | 20 ++++--- ..._float_uint64_dim512_t32_8pq_4subd_half.cu | 20 ++++--- ..._half_uint32_dim1024_t32_8pq_2subd_half.cu | 20 ++++--- ..._half_uint32_dim1024_t32_8pq_4subd_half.cu | 20 ++++--- ...ta_half_uint32_dim128_t8_8pq_2subd_half.cu | 20 ++++--- ...ta_half_uint32_dim128_t8_8pq_4subd_half.cu | 20 ++++--- ...a_half_uint32_dim256_t16_8pq_2subd_half.cu | 20 ++++--- ...a_half_uint32_dim256_t16_8pq_4subd_half.cu | 20 ++++--- ...a_half_uint32_dim512_t32_8pq_2subd_half.cu | 20 ++++--- ...a_half_uint32_dim512_t32_8pq_4subd_half.cu | 20 ++++--- ..._half_uint64_dim1024_t32_8pq_2subd_half.cu | 20 ++++--- ..._half_uint64_dim1024_t32_8pq_4subd_half.cu | 20 ++++--- ...ta_half_uint64_dim128_t8_8pq_2subd_half.cu | 20 ++++--- ...ta_half_uint64_dim128_t8_8pq_4subd_half.cu | 20 ++++--- ...a_half_uint64_dim256_t16_8pq_2subd_half.cu | 20 ++++--- ...a_half_uint64_dim256_t16_8pq_4subd_half.cu | 20 ++++--- ...a_half_uint64_dim512_t32_8pq_2subd_half.cu | 20 ++++--- ...a_half_uint64_dim512_t32_8pq_4subd_half.cu | 20 ++++--- ..._int8_uint32_dim1024_t32_8pq_2subd_half.cu | 20 ++++--- ..._int8_uint32_dim1024_t32_8pq_4subd_half.cu | 20 ++++--- ...ta_int8_uint32_dim128_t8_8pq_2subd_half.cu | 20 ++++--- ...ta_int8_uint32_dim128_t8_8pq_4subd_half.cu | 20 ++++--- ...a_int8_uint32_dim256_t16_8pq_2subd_half.cu | 20 ++++--- ...a_int8_uint32_dim256_t16_8pq_4subd_half.cu | 20 ++++--- ...a_int8_uint32_dim512_t32_8pq_2subd_half.cu | 20 ++++--- ...a_int8_uint32_dim512_t32_8pq_4subd_half.cu | 20 ++++--- ...uint8_uint32_dim1024_t32_8pq_2subd_half.cu | 20 ++++--- ...uint8_uint32_dim1024_t32_8pq_4subd_half.cu | 20 ++++--- ...a_uint8_uint32_dim128_t8_8pq_2subd_half.cu | 20 ++++--- ...a_uint8_uint32_dim128_t8_8pq_4subd_half.cu | 20 ++++--- ..._uint8_uint32_dim256_t16_8pq_2subd_half.cu | 20 ++++--- ..._uint8_uint32_dim256_t16_8pq_4subd_half.cu | 20 ++++--- ..._uint8_uint32_dim512_t32_8pq_2subd_half.cu | 20 ++++--- ..._uint8_uint32_dim512_t32_8pq_4subd_half.cu | 20 ++++--- 49 files changed, 558 insertions(+), 461 deletions(-) diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py index 0bdc898445..39fe16ec0d 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py @@ -39,40 +39,41 @@ */ #include +#include #include #define COMMA , namespace raft::neighbors::cagra::detail::single_cta_search { -#define instantiate_single_cta_select_and_run( \\ - TEAM_SIZE, MAX_DATASET_DIM, DATASET_DESC_T, SAMPLE_FILTER_T) \\ - template void \\ - select_and_run( \\ - DATASET_DESC_T dataset_desc, \\ - raft::device_matrix_view graph, \\ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \\ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \\ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \\ - const uint32_t num_queries, \\ - const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \\ - uint32_t* const num_executed_iterations, \\ - uint32_t topk, \\ - uint32_t num_itopk_candidates, \\ - uint32_t block_size, \\ - uint32_t smem_size, \\ - int64_t hash_bitlen, \\ - typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \\ - size_t small_hash_bitlen, \\ - size_t small_hash_reset_interval, \\ - uint32_t num_random_samplings, \\ - uint64_t rand_xor_mask, \\ - uint32_t num_seeds, \\ - size_t itopk_size, \\ - size_t search_width, \\ - size_t min_iterations, \\ - size_t max_iterations, \\ - SAMPLE_FILTER_T sample_filter, \\ +#define instantiate_single_cta_select_and_run( \\ + TEAM_SIZE, MAX_DATASET_DIM, DATASET_DESC_T, SAMPLE_FILTER_T) \\ + template void \\ + select_and_run( \\ + DATASET_DESC_T dataset_desc, \\ + raft::device_matrix_view graph, \\ + typename DATASET_DESC_T::INDEX_T* topk_indices_ptr, \\ + typename DATASET_DESC_T::DISTANCE_T* topk_distances_ptr, \\ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \\ + const uint32_t num_queries, \\ + const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \\ + uint32_t* num_executed_iterations, \\ + uint32_t topk, \\ + uint32_t num_itopk_candidates, \\ + uint32_t block_size, \\ + uint32_t smem_size, \\ + int64_t hash_bitlen, \\ + typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \\ + size_t small_hash_bitlen, \\ + size_t small_hash_reset_interval, \\ + uint32_t num_random_samplings, \\ + uint64_t rand_xor_mask, \\ + uint32_t num_seeds, \\ + size_t itopk_size, \\ + size_t search_width, \\ + size_t min_iterations, \\ + size_t max_iterations, \\ + SAMPLE_FILTER_T sample_filter, \\ cudaStream_t stream); """ @@ -116,7 +117,7 @@ with open(path, "w") as f: f.write(header) f.write( - f"instantiate_single_cta_select_and_run(\n {team}, {mxdim}, raft::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA 0 COMMA 0 COMMA {distance_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" + f"instantiate_single_cta_select_and_run(\n {team}, {mxdim}, raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA 0 COMMA {distance_t} COMMA {idx_t} COMMA 0>, raft::neighbors::filtering::none_cagra_sample_filter);\n" ) f.write(trailer) diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu index 850ba9f6d2..82c04ae6b4 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu index 850ba9f6d2..06f7e89139 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu index 2b7d2932d3..c166f2a886 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu index 2b7d2932d3..deb3ca2b7c 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu index 9a6c0a5180..ef7ba2d4ee 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu index 9a6c0a5180..5417c76a1f 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu index 29f957e7de..73979c3df7 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu index 29f957e7de..51262ab840 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu index 603b4de90c..dbc24d1925 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu index 603b4de90c..14e708cc43 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu index 167a9f223b..010853be21 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu index 167a9f223b..9c48c472d3 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu index 7d29697ecd..a336cdbfe4 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu index 7d29697ecd..6bf48e52c3 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu index a60997ca62..6694947446 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu index a60997ca62..dffa0e02fc 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - float COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + float COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu index 496c071a74..0da0e84ba2 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu index 496c071a74..e1d660bde1 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu index d576714289..0910109a4b 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu index d576714289..65bc1d31a7 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu index e2ddb41c98..da3953653f 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu index e2ddb41c98..91c5cd8d07 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu index 9b5747b010..1dec16d7d9 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu index 9b5747b010..9945212a3c 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu index 7ce49bd8a9..32e5e896b3 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu index 7ce49bd8a9..c1fb41bb24 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu index 962f625445..d8b269e369 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu index 962f625445..ffc5942e9c 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu index 9bedc22d73..8f3d50cb9c 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu index 9bedc22d73..67786c18ef 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu index 16ff41a67d..7acb50b587 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu index 16ff41a67d..b2a125a9c2 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - half COMMA uint64_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + half COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint64_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu index 801b206fb4..848456639d 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - int8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + int8_t COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu index 801b206fb4..b34506f97e 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - int8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + int8_t COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu index 21783a7419..f2f30aca62 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - int8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + int8_t COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu index 21783a7419..1ed91a066f 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - int8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + int8_t COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu index fdd1997c03..9809c318ac 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - int8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + int8_t COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu index fdd1997c03..5a6c09447d 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - int8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + int8_t COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu index 813e2aa8bd..4b1e2ba605 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - int8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + int8_t COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu index 813e2aa8bd..73f2e92eb8 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - int8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + int8_t COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu index 8ffb39b839..d300ae1669 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - uint8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + uint8_t COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu index 8ffb39b839..71f3b1d118 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - uint8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 1024, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + uint8_t COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu index 62d2bc0a26..e45e904f9c 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - uint8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + uint8_t COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu index 62d2bc0a26..5a65f9e97f 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - uint8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 8, + 128, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + uint8_t COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu index 39775db752..a17a254ea4 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - uint8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + uint8_t COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu index 39775db752..5ea7dbf09c 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - uint8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 16, + 256, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + uint8_t COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu index 16a66d3afb..330ee5e6fc 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - uint8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + uint8_t COMMA half COMMA 8 COMMA 2 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu index 16a66d3afb..2dda81f379 100644 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu +++ b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu @@ -24,6 +24,7 @@ * */ +#include #include #include @@ -36,12 +37,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { template void select_and_run( \ DATASET_DESC_T dataset_desc, \ raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ + typename DATASET_DESC_T::INDEX_T * topk_indices_ptr, \ + typename DATASET_DESC_T::DISTANCE_T * topk_distances_ptr, \ + const typename DATASET_DESC_T::DATA_T* queries_ptr, \ const uint32_t num_queries, \ const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ + uint32_t* num_executed_iterations, \ uint32_t topk, \ uint32_t num_itopk_candidates, \ uint32_t block_size, \ @@ -60,11 +61,12 @@ namespace raft::neighbors::cagra::detail::single_cta_search { SAMPLE_FILTER_T sample_filter, \ cudaStream_t stream); -instantiate_single_cta_select_and_run(32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t< - uint8_t COMMA uint32_t COMMA 0 COMMA 0 COMMA float>, - raft::neighbors::filtering::none_cagra_sample_filter); +instantiate_single_cta_select_and_run( + 32, + 512, + raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< + uint8_t COMMA half COMMA 8 COMMA 4 COMMA 0 COMMA float COMMA uint32_t COMMA 0>, + raft::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_search_kernel