Update Lindera to 0.8 and Tantivy to 0.15. (#22)

* Update Lindera to 0.8 and Tantivy to 0.15.

CHANGES.md

@@ -2,6 +2,9 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+- Update Lindera to 0.8 and Tantivy to 0.15. #22 @mosuka
+
 ## 0.7.2 (2021-02-08)
 - Upgrade Tantivy to 0.14.0 #19 @mosuka
 - Bump up version to 0.7.2 #21 @mosuka

Cargo.toml

@@ -17,7 +17,7 @@ categories = ["text-processing"]
 license = "MIT"
 
 [dependencies]
-tantivy = "0.14"
+tantivy = "0.15"
 
-lindera = "0.7"
-lindera-core = "0.7"
+lindera = "0.8.0"
+lindera-core = "0.8.0"

examples/basic_example.rs

@@ -1,9 +1,12 @@
-use lindera_tantivy::tokenizer::LinderaTokenizer;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions};
 use tantivy::{doc, Index};
 
+use lindera::tokenizer::TokenizerConfig;
+use lindera_core::viterbi::{Mode, Penalty};
+use lindera_tantivy::tokenizer::LinderaTokenizer;
+
 fn main() -> tantivy::Result<()> {
     // create schema builder
     let mut schema_builder = Schema::builder();
@@ -50,10 +53,16 @@ fn main() -> tantivy::Result<()> {
     // create index on memory
     let index = Index::create_in_ram(schema.clone());
 
+    let config = TokenizerConfig {
+        dict_path: None,
+        user_dict_path: None,
+        mode: Mode::Decompose(Penalty::default()),
+    };
+
     // register Lindera tokenizer
     index
         .tokenizers()
-        .register("lang_ja", LinderaTokenizer::new("decompose", ""));
+        .register("lang_ja", LinderaTokenizer::with_config(config).unwrap());
 
     // create index writer
     let mut index_writer = index.writer(50_000_000)?;

src/tokenizer.rs

@@ -1,8 +1,10 @@
-use crate::stream::LinderaTokenStream;
-use lindera::tokenizer::Tokenizer as LTokenizer;
-use lindera_core::core::viterbi::{Mode, Penalty};
 use tantivy::tokenizer::{BoxTokenStream, Tokenizer};
 
+use lindera::tokenizer::{Tokenizer as LTokenizer, TokenizerConfig};
+use lindera_core::LinderaResult;
+
+use crate::stream::LinderaTokenStream;
+
 /// Tokenize text with the specified mode and dictionary.
 ///
 /// Example: `すもももももももものうち` would be tokenized as (mode: "normal", dict: "")
@@ -18,7 +20,7 @@ use tantivy::tokenizer::{BoxTokenStream, Tokenizer};
 /// use lindera_tantivy::tokenizer::*;
 /// use tantivy::tokenizer::Tokenizer;
 ///
-/// let tokenizer = LinderaTokenizer::new("normal", "");
+/// let tokenizer = LinderaTokenizer::new().unwrap();
 /// let mut stream = tokenizer.token_stream("すもももももももものうち");
 /// {
 ///     let token = stream.next().unwrap();
@@ -70,26 +72,26 @@ pub struct LinderaTokenizer {
 }
 
 impl LinderaTokenizer {
-    pub fn new(mode: &str, dict: &str) -> LinderaTokenizer {
-        let mode = match mode {
-            "normal" => Mode::Normal,
-            "decompose" => Mode::Decompose(Penalty::default()),
-            _ => {
-                // show error message
-                println!("unsupported mode: {}", mode);
-                Mode::Normal
-            }
-        };
-        LinderaTokenizer {
-            tokenizer: LTokenizer::new(mode, dict),
-        }
+    pub fn new() -> LinderaResult<LinderaTokenizer> {
+        Ok(LinderaTokenizer {
+            tokenizer: LTokenizer::new()?,
+        })
+    }
+
+    pub fn with_config(config: TokenizerConfig) -> LinderaResult<LinderaTokenizer> {
+        Ok(LinderaTokenizer {
+            tokenizer: LTokenizer::with_config(config)?,
+        })
     }
 }
 
 impl Tokenizer for LinderaTokenizer {
     fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
         let mut tokenizer = self.tokenizer.clone();
-        let result = tokenizer.tokenize(text);
+        let result = match tokenizer.tokenize(text) {
+            Ok(result) => result,
+            Err(_err) => Vec::new(),
+        };
 
         BoxTokenStream::from(LinderaTokenStream {
             result,
@@ -114,7 +116,9 @@ mod tests {
     #[test]
     fn test_tokenizer_equal() {
         let tokens = test_helper(
-            LinderaTokenizer::new("normal", "").token_stream("すもももももももものうち"),
+            LinderaTokenizer::new()
+                .unwrap()
+                .token_stream("すもももももももものうち"),
         );
         assert_eq!(tokens.len(), 7);
         {