Enhance Kanji Recognition in Japanese Language Detection

accuracy-reports/aggregated-accuracy-values.csv

@@ -11,7 +11,7 @@ Bokmal,NaN,NaN,NaN,NaN,49,24,44,80,NaN,NaN,NaN,NaN,49,27,47,74,58,38,58,76
 Bosnian,18,4,15,36,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,29,22,28,36,34,29,34,40
 Bulgarian,65,31,72,92,69,44,67,96,NaN,NaN,NaN,NaN,77,56,80,96,86,70,91,99
 Catalan,37,4,29,79,51,29,45,80,NaN,NaN,NaN,NaN,58,33,60,81,70,50,73,86
-Chinese,33,NaN,2,98,100,100,100,100,97,93,98,100,100,100,100,100,100,100,100,100
+Chinese,33,NaN,2,98,100,100,100,100,97,93,98,100,100,100,100,100,95,89,96,100
 Croatian,51,33,46,72,61,34,54,94,NaN,NaN,NaN,NaN,59,36,57,85,72,53,74,90
 Czech,73,50,79,90,63,42,66,82,NaN,NaN,NaN,NaN,70,54,71,87,80,65,84,91
 Danish,59,26,56,94,53,31,45,84,NaN,NaN,NaN,NaN,70,45,70,95,81,61,83,97

accuracy-reports/lingua-high-accuracy/Chinese.txt

@@ -1,14 +1,14 @@
 ##### Chinese #####
 
->>> Accuracy on average: 100%
+>>> Accuracy on average: 95.16%
 
 >> Detection of 1000 single words (average length: 1 chars)
-Accuracy: 100%
-Erroneously classified as 
+Accuracy: 89.1%
+Erroneously classified as Japanese: 10.9%
 
 >> Detection of 1000 word pairs (average length: 2 chars)
-Accuracy: 100%
-Erroneously classified as 
+Accuracy: 96.4%
+Erroneously classified as Japanese: 3.6%
 
 >> Detection of 729 sentences (average length: 48 chars)
 Accuracy: 100%

src/constant.rs

@@ -24,7 +24,7 @@ use crate::alphabet::CharSet;
 use crate::language::Language;
 
 pub(crate) static JAPANESE_CHARACTER_SET: Lazy<CharSet> =
-    Lazy::new(|| CharSet::from_char_classes(&["Hiragana", "Katakana", "Han"]));
+    Lazy::new(|| CharSet::from_char_classes(&["Hiragana", "Katakana", "Japanese_Han"]));
 pub(crate) static MULTIPLE_WHITESPACE: Lazy<Regex> = Lazy::new(|| Regex::new("\\s+").unwrap());
 pub(crate) static NUMBERS: Lazy<Regex> = Lazy::new(|| Regex::new("\\p{N}").unwrap());
 pub(crate) static PUNCTUATION: Lazy<Regex> = Lazy::new(|| Regex::new("\\p{P}").unwrap());

src/detector.rs

@@ -795,6 +795,8 @@ impl LanguageDetector {
     ) -> Option<Language> {
         let mut total_language_counts = HashMap::<Option<Language>, u32>::new();
         let half_word_count = (words.len() as f64) * 0.5;
+        let mut cjk_lang_uncertainty:usize=0;
+        let cjk_lang_uncertainty_max_ratio =0.9999999999;
 
         for word in words {
             let mut word_language_counts = HashMap::<Language, u32>::new();
@@ -811,18 +813,19 @@ impl LanguageDetector {
                 }
 
                 if !is_match {
-                    if cfg!(feature = "chinese") && Alphabet::Han.matches_char(character) {
+
+                    if cfg!(feature = "japanese") //we need to test for both and later guess at which one it is
+                        && JAPANESE_CHARACTER_SET.is_char_match(character)
+                    {
                         self.increment_counter(
                             &mut word_language_counts,
-                            Language::from_str("Chinese").unwrap(),
+                            Language::from_str("Japanese").unwrap(),
                             1,
                         );
-                    } else if cfg!(feature = "japanese")
-                        && JAPANESE_CHARACTER_SET.is_char_match(character)
-                    {
+                    }                    if cfg!(feature = "chinese") && Alphabet::Han.matches_char(character) {
                         self.increment_counter(
                             &mut word_language_counts,
-                            Language::from_str("Japanese").unwrap(),
+                            Language::from_str("Chinese").unwrap(),
                             1,
                         );
                     } else if Alphabet::Latin.matches_char(character)
@@ -854,11 +857,17 @@ impl LanguageDetector {
                 && word_language_counts.contains_key(&Language::from_str("Chinese").unwrap())
                 && word_language_counts.contains_key(&Language::from_str("Japanese").unwrap())
             {
+                self.increment_counter(
+                    &mut total_language_counts,
+                    Some(Language::from_str("Chinese").unwrap()),
+                    1,
+                );
                 self.increment_counter(
                     &mut total_language_counts,
                     Some(Language::from_str("Japanese").unwrap()),
                     1,
                 );
+                cjk_lang_uncertainty +=1;
             } else {
                 let sorted_word_language_counts = word_language_counts
                     .into_iter()
@@ -898,10 +907,26 @@ impl LanguageDetector {
             && cfg!(feature = "japanese")
             && total_language_counts.contains_key(&Some(Language::from_str("Chinese").unwrap()))
             && total_language_counts.contains_key(&Some(Language::from_str("Japanese").unwrap()))
+            && (cjk_lang_uncertainty as f32 / words.len() as f32) >= cjk_lang_uncertainty_max_ratio
+            && self.is_low_accuracy_mode_enabled
         {
-            return Some(Language::from_str("Japanese").unwrap());
+            // Retrieve the counts for Chinese and Japanese languages
+            let chinese_count = *total_language_counts
+                .get(&Some(Language::Chinese))
+                .unwrap_or(&0);
+            let japanese_count = *total_language_counts
+                .get(&Some(Language::Japanese))
+                .unwrap_or(&0);
+            // Compare the counts and return the language with the higher count
+            if chinese_count >= japanese_count {
+                return Some(Language::Chinese);
+            } else {
+                return Some(Language::Japanese);
+            }
         }
 
+
+
         let sorted_total_language_counts = total_language_counts
             .into_iter()
             .sorted_by(|(_, first_count), (_, second_count)| second_count.cmp(first_count))

src/language.rs

@@ -1095,7 +1095,6 @@ impl Language {
 
 #[cfg(test)]
 mod tests {
-    use std::str::FromStr;
 
     use crate::language::Language::*;
 

src/model.rs

@@ -206,7 +206,6 @@ fn get_utf8_slice(string: &str, start: usize, end: usize) -> &str {
 
 #[cfg(test)]
 mod tests {
-    use itertools::Itertools;
     use rstest::*;
 
     use super::*;