extract field_chunk_tokens

TabbyML · May 23, 2024 · 894e319 · 894e319
1 parent 56116b0
commit 894e319
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 17 deletions.
diff --git a/crates/tabby-common/src/index/code/mod.rs b/crates/tabby-common/src/index/code/mod.rs
@@ -35,11 +35,7 @@ impl CodeSearchSchema {
         let subqueries: Vec<Box<dyn Query>> = tokens
             .iter()
             .map(|text| {
-                let mut term = Term::from_field_text(
-                    schema.field_chunk_tokens,
-                    &text
-                );
-                term.append_type_and_str(text.as_ref());
+                let term = Term::from_field_text(schema.field_chunk_tokens, text);
                 let term_query: Box<dyn Query> =
                     Box::new(TermQuery::new(term, IndexRecordOption::Basic));
 

diff --git a/crates/tabby-common/src/index/doc.rs b/crates/tabby-common/src/index/doc.rs
@@ -2,7 +2,9 @@ use std::borrow::Cow;
 
 use lazy_static::lazy_static;
 use tantivy::{
-    query::{BooleanQuery, ExistsQuery, Occur, TermQuery}, schema::{Field, JsonObjectOptions, Schema, TextFieldIndexing, FAST, INDEXED, STORED, STRING}, tokenizer::TokenizerManager, Term
+    query::{BooleanQuery, ExistsQuery, Occur, TermQuery},
+    schema::{Field, JsonObjectOptions, Schema, TextFieldIndexing, FAST, INDEXED, STORED, STRING},
+    Term,
 };
 
 use super::new_multiterms_const_query;
@@ -104,11 +106,7 @@ impl DocSearchSchema {
     ) -> BooleanQuery {
         let iter = DocSearchSchema::binarize_embedding(embedding).map(Cow::Owned);
 
-        new_multiterms_const_query(
-            self.field_chunk_tokens,
-            embedding_dims,
-            iter,
-        )
+        new_multiterms_const_query(self.field_chunk_tokens, embedding_dims, iter)
     }
 
     /// Build a query to find the document with the given `doc_id`.

diff --git a/crates/tabby-common/src/index/mod.rs b/crates/tabby-common/src/index/mod.rs
@@ -18,7 +18,7 @@ fn new_multiterms_const_query<'a>(
 ) -> BooleanQuery {
     let subqueries: Vec<Box<dyn Query>> = terms
         .map(|text| {
-            let mut term = Term::from_field_text(field, text.as_ref());
+            let term = Term::from_field_text(field, text.as_ref());
             let term_query: Box<dyn Query> =
                 Box::new(TermQuery::new(term, IndexRecordOption::Basic));
 
@@ -34,10 +34,9 @@ fn new_multiterms_const_query<'a>(
 
 #[cfg(test)]
 mod tests {
-    use serde_json::json;
+
     use tantivy::{
         collector::TopDocs,
-        doc,
         query::Query,
         schema::{Schema, STRING},
         Index, IndexWriter, TantivyDocument,

diff --git a/crates/tabby-scheduler/src/index.rs b/crates/tabby-scheduler/src/index.rs
@@ -9,7 +9,10 @@ pub trait DocumentBuilder<T>: Send + Sync {
     fn format_id(&self, id: &str) -> String;
     async fn build_id(&self, document: &T) -> String;
     async fn build_attributes(&self, document: &T) -> serde_json::Value;
-    async fn build_chunk_attributes(&self, document: &T) -> BoxStream<(Vec<String>, serde_json::Value)>;
+    async fn build_chunk_attributes(
+        &self,
+        document: &T,
+    ) -> BoxStream<(Vec<String>, serde_json::Value)>;
 }
 
 pub struct DocIndex<T> {

diff --git a/crates/tabby-scheduler/src/web/mod.rs b/crates/tabby-scheduler/src/web/mod.rs
@@ -74,10 +74,9 @@ impl DocumentBuilder<SourceDocument> for WebBuilder {
                 }
 
                 let chunk = json!({
-                        // FIXME: tokenize chunk text
                         webdoc::fields::CHUNK_TEXT: chunk_text,
                 });
-                
+
                 yield (chunk_embedding_tokens, chunk)
             }
         };