add language select, abstract text transformations

.eslintrc.json

@@ -614,8 +614,8 @@
                 "ext/js/core.js",
                 "ext/js/core/extension-error.js",
                 "ext/js/**/sandbox/**/*.js",
-                "ext/js/language/japanese.js",
-                "ext/js/language/japanese-wanakana.js"
+                "ext/js/language/languages/ja/japanese.js",
+                "ext/js/language/languages/ja/japanese-wanakana.js"
             ],
             "env": {
                 "webextensions": false

dev/jsconfig.json

@@ -29,6 +29,7 @@
             "error": ["../types/ext/error"],
             "event-listener-collection": ["../types/ext/event-listener-collection"],
             "japanese-util": ["../types/ext/japanese-util"],
+            "language": ["../types/ext/language"],
             "ext/json-schema": ["../types/ext/json-schema"],
             "log": ["../types/ext/log"],
             "settings": ["../types/ext/settings"],

ext/data/schemas/options-schema.json

@@ -74,13 +74,15 @@
                             "sentenceParsing",
                             "inputs",
                             "clipboard",
-                            "accessibility"
+                            "accessibility",
+                            "languages"
                         ],
                         "properties": {
                             "general": {
                                 "type": "object",
                                 "required": [
                                     "enable",
+                                    "language",
                                     "resultOutputMode",
                                     "debugInfo",
                                     "maxResults",
@@ -126,6 +128,10 @@
                                         "type": "boolean",
                                         "default": true
                                     },
+                                    "language": {
+                                        "type": "string",
+                                        "default": "ja"
+                                    },
                                     "resultOutputMode": {
                                         "type": "string",
                                         "enum": ["group", "merge", "split"],
@@ -722,12 +728,6 @@
                             "translation": {
                                 "type": "object",
                                 "required": [
-                                    "convertHalfWidthCharacters",
-                                    "convertNumericCharacters",
-                                    "convertAlphabeticCharacters",
-                                    "convertHiraganaToKatakana",
-                                    "convertKatakanaToHiragana",
-                                    "collapseEmphaticSequences",
                                     "textReplacements",
                                     "searchResolution"
                                 ],
@@ -740,36 +740,6 @@
                                         ],
                                         "default": "letter"
                                     },
-                                    "convertHalfWidthCharacters": {
-                                        "type": "string",
-                                        "enum": ["false", "true", "variant"],
-                                        "default": "false"
-                                    },
-                                    "convertNumericCharacters": {
-                                        "type": "string",
-                                        "enum": ["false", "true", "variant"],
-                                        "default": "false"
-                                    },
-                                    "convertAlphabeticCharacters": {
-                                        "type": "string",
-                                        "enum": ["false", "true", "variant"],
-                                        "default": "false"
-                                    },
-                                    "convertHiraganaToKatakana": {
-                                        "type": "string",
-                                        "enum": ["false", "true", "variant"],
-                                        "default": "false"
-                                    },
-                                    "convertKatakanaToHiragana": {
-                                        "type": "string",
-                                        "enum": ["false", "true", "variant"],
-                                        "default": "variant"
-                                    },
-                                    "collapseEmphaticSequences": {
-                                        "type": "string",
-                                        "enum": ["false", "true", "full"],
-                                        "default": "false"
-                                    },
                                     "textReplacements": {
                                         "type": "object",
                                         "required": [
@@ -813,6 +783,39 @@
                                     }
                                 }
                             },
+                            "languages": {
+                                "type": "object",
+                                "propertyNames": {
+                                    "type": "string"
+                                },
+                                "additionalProperties": {
+                                    "type": "object",
+                                    "properties": {
+                                        "textTransformations": {
+                                            "type": "object",
+                                            "propertyNames": {
+                                                "type": "string"
+                                            },
+                                            "additionalProperties": {
+                                                "type": "string",
+                                                "default": "false"
+                                            }
+                                        }
+                                    }
+                                },
+                                "default": {
+                                    "ja": {
+                                        "textTransformations": {
+                                            "convertHalfWidthCharacters": "false",
+                                            "convertNumericCharacters": "false",
+                                            "convertAlphabeticCharacters": "false",
+                                            "convertHiraganaToKatakana": "false",
+                                            "convertKatakanaToHiragana": "variant",
+                                            "collapseEmphaticSequences": "false"
+                                        }
+                                    }
+                                }
+                            },
                             "dictionaries": {
                                 "type": "array",
                                 "items": {

ext/js/background/backend.js

@@ -23,21 +23,21 @@ import {ClipboardReader} from '../comm/clipboard-reader.js';
 import {Mecab} from '../comm/mecab.js';
 import {createApiMap, invokeApiMapHandler} from '../core/api-map.js';
 import {ExtensionError} from '../core/extension-error.js';
-import {readResponseJson} from '../core/json.js';
 import {log} from '../core/logger.js';
-import {clone, deferPromise, isObject, promiseTimeout} from '../core/utilities.js';
+import {clone, deferPromise, fetchJson, fetchText, isObject, promiseTimeout} from '../core/utilities.js';
 import {AnkiUtil} from '../data/anki-util.js';
 import {OptionsUtil} from '../data/options-util.js';
 import {PermissionsUtil} from '../data/permissions-util.js';
 import {ArrayBufferUtil} from '../data/sandbox/array-buffer-util.js';
 import {DictionaryDatabase} from '../dictionary/dictionary-database.js';
 import {Environment} from '../extension/environment.js';
 import {ObjectPropertyAccessor} from '../general/object-property-accessor.js';
-import {distributeFuriganaInflected, isCodePointJapanese, isStringPartiallyJapanese, convertKatakanaToHiragana as jpConvertKatakanaToHiragana} from '../language/japanese.js';
+import {LanguageUtil} from '../language/language-util.js';
+import {distributeFuriganaInflected, isCodePointJapanese, isStringPartiallyJapanese, convertKatakanaToHiragana as jpConvertKatakanaToHiragana} from '../language/languages/ja/japanese.js';
 import {Translator} from '../language/translator.js';
 import {AudioDownloader} from '../media/audio-downloader.js';
 import {MediaUtil} from '../media/media-util.js';
-import {ClipboardReaderProxy, DictionaryDatabaseProxy, OffscreenProxy, TranslatorProxy} from './offscreen-proxy.js';
+import {ClipboardReaderProxy, DictionaryDatabaseProxy, LanguageUtilProxy, OffscreenProxy, TranslatorProxy} from './offscreen-proxy.js';
 import {ProfileConditionsUtil} from './profile-conditions-util.js';
 import {RequestBuilder} from './request-builder.js';
 import {injectStylesheet} from './script-manager.js';
@@ -65,8 +65,11 @@ export class Backend {
             this._offscreen = null;
             /** @type {DictionaryDatabase|DictionaryDatabaseProxy} */
             this._dictionaryDatabase = new DictionaryDatabase();
+            /** @type {LanguageUtil|LanguageUtilProxy} */
+            this._languageUtil = new LanguageUtil();
             /** @type {Translator|TranslatorProxy} */
             this._translator = new Translator({
+                languageUtil: this._languageUtil,
                 database: this._dictionaryDatabase
             });
             /** @type {ClipboardReader|ClipboardReaderProxy} */
@@ -81,6 +84,8 @@ export class Backend {
             this._offscreen = new OffscreenProxy(webExtension);
             /** @type {DictionaryDatabase|DictionaryDatabaseProxy} */
             this._dictionaryDatabase = new DictionaryDatabaseProxy(this._offscreen);
+            /** @type {LanguageUtil|LanguageUtilProxy} */
+            this._languageUtil = new LanguageUtilProxy(this._offscreen);
             /** @type {Translator|TranslatorProxy} */
             this._translator = new TranslatorProxy(this._offscreen);
             /** @type {ClipboardReader|ClipboardReaderProxy} */
@@ -187,7 +192,9 @@ export class Backend {
             ['textHasJapaneseCharacters',    this._onApiTextHasJapaneseCharacters.bind(this)],
             ['getTermFrequencies',           this._onApiGetTermFrequencies.bind(this)],
             ['findAnkiNotes',                this._onApiFindAnkiNotes.bind(this)],
-            ['openCrossFramePort',           this._onApiOpenCrossFramePort.bind(this)]
+            ['openCrossFramePort',           this._onApiOpenCrossFramePort.bind(this)],
+            ['getLanguages',                 this._onApiGetLanguages.bind(this)],
+            ['getTextTransformations',       this._onApiGetTextTransformations.bind(this)]
         ]);
         /* eslint-enable no-multi-spaces */
 
@@ -282,14 +289,16 @@ export class Backend {
                 log.error(e);
             }
 
-            /** @type {import('deinflector').ReasonsRaw} */
-            const deinflectionReasons = await this._fetchJson('/data/deinflect.json');
-            this._translator.prepare(deinflectionReasons);
-
             await this._optionsUtil.prepare();
-            this._defaultAnkiFieldTemplates = (await this._fetchText('/data/templates/default-anki-field-templates.handlebars')).trim();
+            this._defaultAnkiFieldTemplates = (await fetchText('/data/templates/default-anki-field-templates.handlebars')).trim();
             this._options = await this._optionsUtil.load();
 
+            await this._languageUtil.prepare();
+
+            /** @type {import('deinflector').ReasonsRaw} */
+            const deinflectionReasons = await fetchJson('/data/deinflect.json');
+            this._translator.prepare(deinflectionReasons);
+
             this._applyOptions('background');
 
             const options = this._getProfileOptions({current: true}, false);
@@ -672,7 +681,7 @@ export class Backend {
         if (!url.startsWith('/') || url.startsWith('//') || !url.endsWith('.css')) {
             throw new Error('Invalid URL');
         }
-        return await this._fetchText(url);
+        return await fetchText(url);
     }
 
     /** @type {import('api').ApiHandler<'getEnvironmentInfo'>} */
@@ -687,7 +696,7 @@ export class Backend {
 
     /** @type {import('api').ApiHandler<'getDisplayTemplatesHtml'>} */
     async _onApiGetDisplayTemplatesHtml() {
-        return await this._fetchText('/display-templates.html');
+        return await fetchText('/display-templates.html');
     }
 
     /** @type {import('api').ApiHandler<'getZoom'>} */
@@ -911,6 +920,16 @@ export class Backend {
         return {targetTabId, targetFrameId};
     }
 
+    /** @type {import('api').ApiHandler<'getLanguages'>} */
+    _onApiGetLanguages() {
+        return this._languageUtil.getLanguages();
+    }
+
+    /** @type {import('api').ApiHandler<'getTextTransformations'>} */
+    _onApiGetTextTransformations({language}) {
+        return this._languageUtil.getTextTransformations(language);
+    }
+
     // Command handlers
 
     /**
@@ -1852,44 +1871,6 @@ export class Backend {
         });
     }
 
-    /**
-     * @param {string} url
-     * @returns {Promise<Response>}
-     */
-    async _fetchAsset(url) {
-        const response = await fetch(chrome.runtime.getURL(url), {
-            method: 'GET',
-            mode: 'no-cors',
-            cache: 'default',
-            credentials: 'omit',
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer'
-        });
-        if (!response.ok) {
-            throw new Error(`Failed to fetch ${url}: ${response.status}`);
-        }
-        return response;
-    }
-
-    /**
-     * @param {string} url
-     * @returns {Promise<string>}
-     */
-    async _fetchText(url) {
-        const response = await this._fetchAsset(url);
-        return await response.text();
-    }
-
-    /**
-     * @template [T=unknown]
-     * @param {string} url
-     * @returns {Promise<T>}
-     */
-    async _fetchJson(url) {
-        const response = await this._fetchAsset(url);
-        return await readResponseJson(response);
-    }
-
     /**
      * @template {import('application').ApiNames} TName
      * @param {import('application').ApiMessage<TName>} message
@@ -2406,7 +2387,7 @@ export class Backend {
         if (typeof deinflect !== 'boolean') { deinflect = true; }
         const enabledDictionaryMap = this._getTranslatorEnabledDictionaryMap(options);
         const {
-            general: {mainDictionary, sortFrequencyDictionary, sortFrequencyDictionaryOrder},
+            general: {mainDictionary, sortFrequencyDictionary, sortFrequencyDictionaryOrder, language},
             scanning: {alphanumeric},
             translation: {
                 convertHalfWidthCharacters,
@@ -2419,6 +2400,7 @@ export class Backend {
                 searchResolution
             }
         } = options;
+        const textTransformationsOptions = options.languages[language].textTransformations || {};
         const textReplacements = this._getTranslatorTextReplacements(textReplacementsOptions);
         let excludeDictionaryDefinitions = null;
         if (mode === 'merge' && !enabledDictionaryMap.has(mainDictionary)) {
@@ -2448,7 +2430,9 @@ export class Backend {
             searchResolution,
             textReplacements,
             enabledDictionaryMap,
-            excludeDictionaryDefinitions
+            excludeDictionaryDefinitions,
+            language,
+            textTransformationsOptions
         };
     }
 

ext/js/background/offscreen-proxy.js

@@ -149,6 +149,38 @@ export class DictionaryDatabaseProxy {
     }
 }
 
+export class LanguageUtilProxy {
+    /**
+     * @param {OffscreenProxy} offscreen
+     */
+    constructor(offscreen) {
+        /** @type {OffscreenProxy} */
+        this._offscreen = offscreen;
+    }
+
+    /**
+     * @returns {Promise<void>}
+     */
+    async prepare() {
+        await this._offscreen.sendMessagePromise({action: 'languageUtilPrepareOffscreen'});
+    }
+
+    /**
+     * @param {string} iso
+     * @returns {Promise<import('language').TextTransformation[]>}
+     */
+    async getTextTransformations(iso) {
+        return this._offscreen.sendMessagePromise({action: 'getTextTransformationsOffscreen', params: {iso}});
+    }
+
+    /**
+     * @returns {Promise<import('language').Language[]>}
+     */
+    async getLanguages() {
+        return this._offscreen.sendMessagePromise({action: 'getLanguagesOffscreen'});
+    }
+}
+
 export class TranslatorProxy {
     /**
      * @param {OffscreenProxy} offscreen