diff --git a/.eslintrc.json b/.eslintrc.json
index 64e852589..1eb1278c1 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -560,6 +560,7 @@
"ext/js/dom/css-style-applier.js",
"ext/js/language/CJK-util.js",
"ext/js/language/ja/japanese.js",
+ "ext/js/language/text-processors.js",
"ext/js/language/text-utilities.js",
"ext/js/templates/anki-template-renderer-content-manager.js",
"ext/js/templates/anki-template-renderer.js",
diff --git a/ext/js/language/CJK-util.js b/ext/js/language/CJK-util.js
index 5c59afb58..5c5646826 100644
--- a/ext/js/language/CJK-util.js
+++ b/ext/js/language/CJK-util.js
@@ -15,6 +15,8 @@
* along with this program. If not, see .
*/
+import {basicTextProcessorOptions} from './text-processors.js';
+
/** @type {import('CJK-util').CodepointRange} */
const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff];
/** @type {import('CJK-util').CodepointRange} */
@@ -94,3 +96,40 @@ export function isCodePointInRanges(codePoint, ranges) {
}
return false;
}
+
+/** @type {import('CJK-util').CodepointRange} */
+export const KANGXI_RADICALS_RANGE = [0x2f00, 0x2fdf];
+
+/** @type {import('CJK-util').CodepointRange} */
+export const CJK_RADICALS_SUPPLEMENT_RANGE = [0x2e80, 0x2eff];
+
+/** @type {import('CJK-util').CodepointRange} */
+export const CJK_STROKES_RANGE = [0x31c0, 0x31ef];
+
+/** @type {import('CJK-util').CodepointRange[]} */
+export const CJK_RADICALS_RANGES = [
+ KANGXI_RADICALS_RANGE,
+ CJK_RADICALS_SUPPLEMENT_RANGE,
+ CJK_STROKES_RANGE,
+];
+
+/**
+ * @param {string} text
+ * @returns {string}
+ */
+export function normalizeRadicals(text) {
+ let result = '';
+ for (let i = 0; i < text.length; i++) {
+ const codePoint = text[i].codePointAt(0);
+ result += codePoint && (isCodePointInRanges(codePoint, CJK_RADICALS_RANGES)) ? text[i].normalize('NFKD') : text[i];
+ }
+ return result;
+}
+
+/** @type {import('language').TextProcessor} */
+export const normalizeRadicalCharacters = {
+ name: 'Normalize radical characters',
+ description: '⼀ → 一 (U+2F00 → U+4E00)',
+ options: basicTextProcessorOptions,
+ process: (str, setting) => (setting ? normalizeRadicals(str) : str),
+};
diff --git a/ext/js/language/language-descriptors.js b/ext/js/language/language-descriptors.js
index a07437b71..02e2f35e1 100644
--- a/ext/js/language/language-descriptors.js
+++ b/ext/js/language/language-descriptors.js
@@ -16,6 +16,7 @@
*/
import {removeArabicScriptDiacritics} from './ar/arabic-text-preprocessors.js';
+import {normalizeRadicalCharacters} from './CJK-util.js';
import {eszettPreprocessor} from './de/german-text-preprocessors.js';
import {germanTransforms} from './de/german-transforms.js';
import {englishTransforms} from './en/english-transforms.js';
@@ -212,6 +213,7 @@ const languageDescriptors = [
convertHalfWidthCharacters,
alphabeticToHiragana,
normalizeCombiningCharacters,
+ normalizeRadicalCharacters,
alphanumericWidthVariants,
convertHiraganaToKatakana,
collapseEmphaticSequences,
@@ -374,6 +376,9 @@ const languageDescriptors = [
iso639_3: 'yue',
name: 'Cantonese',
exampleText: '讀',
+ textPreprocessors: {
+ normalizeRadicalCharacters,
+ },
},
{
iso: 'zh',
@@ -382,6 +387,9 @@ const languageDescriptors = [
exampleText: '读',
isTextLookupWorthy: isStringPartiallyChinese,
readingNormalizer: normalizePinyin,
+ textPreprocessors: {
+ normalizeRadicalCharacters,
+ },
},
];
diff --git a/types/ext/language-descriptors.d.ts b/types/ext/language-descriptors.d.ts
index 62643e0c7..199ec4b58 100644
--- a/types/ext/language-descriptors.d.ts
+++ b/types/ext/language-descriptors.d.ts
@@ -135,6 +135,7 @@ type AllTextProcessors = {
convertHalfWidthCharacters: TextProcessor;
alphabeticToHiragana: TextProcessor;
normalizeCombiningCharacters: TextProcessor;
+ normalizeRadicalCharacters: TextProcessor;
alphanumericWidthVariants: BidirectionalConversionPreprocessor;
convertHiraganaToKatakana: BidirectionalConversionPreprocessor;
collapseEmphaticSequences: TextProcessor<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>;
@@ -200,6 +201,14 @@ type AllTextProcessors = {
normalizeDiacritics: TextProcessor<'old' | 'new' | 'off'>;
};
};
- yue: Record;
- zh: Record;
+ yue: {
+ pre: {
+ normalizeRadicalCharacters: TextProcessor;
+ };
+ };
+ zh: {
+ pre: {
+ normalizeRadicalCharacters: TextProcessor;
+ };
+ };
};