improve dictionary validation (#520)

* improve dictionary validation * fix * improve performance * add undefined check * rename types * rename types * rename
yomidevs · Jan 20, 2024 · 0011afe · 0011afe
1 parent be267e1
commit 0011afe
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 43 deletions.
diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js
@@ -40,35 +40,33 @@ function readSchema(relativeFileName) {
 /**
  * @param {import('dev/schema-validate').ValidateMode} mode
  * @param {import('jszip')} zip
- * @param {string} fileNameFormat
- * @param {import('dev/dictionary-validate').Schema} schema
+ * @param {import('dev/dictionary-validate').SchemasDetails} schemasDetails
  */
-async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) {
-    let jsonSchema;
-    try {
-        jsonSchema = createJsonSchema(mode, schema);
-    } catch (e) {
-        const e2 = toError(e);
-        e2.message += `\n(in file ${fileNameFormat})}`;
-        throw e2;
-    }
-    let index = 1;
-    while (true) {
-        const fileName = fileNameFormat.replace(/\?/, `${index}`);
+async function validateDictionaryBanks(mode, zip, schemasDetails) {
+    for (const [fileName, file] of Object.entries(zip.files)) {
+        for (const [fileNameFormat, schema] of schemasDetails) {
+            if (!fileNameFormat.test(fileName)) { continue; }
 
-        const file = zip.files[fileName];
-        if (!file) { break; }
+            let jsonSchema;
+            try {
+                jsonSchema = createJsonSchema(mode, schema);
+            } catch (e) {
+                const e2 = toError(e);
+                e2.message += `\n(in file ${fileName})}`;
+                throw e2;
+            }
 
-        const data = parseJson(await file.async('string'));
-        try {
-            jsonSchema.validate(data);
-        } catch (e) {
-            const e2 = toError(e);
-            e2.message += `\n(in file ${fileName})}`;
-            throw e2;
-        }
+            const data = parseJson(await file.async('string'));
 
-        ++index;
+            try {
+                jsonSchema.validate(data);
+            } catch (e) {
+                const e2 = toError(e);
+                e2.message += `\n(in file ${fileName})}`;
+                throw e2;
+            }
+            break;
+        }
     }
 }
 
@@ -98,11 +96,16 @@ export async function validateDictionary(mode, archive, schemas) {
         throw e2;
     }
 
-    await validateDictionaryBanks(mode, archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
-    await validateDictionaryBanks(mode, archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);
-    await validateDictionaryBanks(mode, archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3);
-    await validateDictionaryBanks(mode, archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3);
-    await validateDictionaryBanks(mode, archive, 'tag_bank_?.json', schemas.tagBankV3);
+    /** @type {import('dev/dictionary-validate').SchemasDetails} */
+    const schemasDetails = [
+        [/^term_bank_(\d+)\.json$/, version === 1 ? schemas.termBankV1 : schemas.termBankV3],
+        [/^term_meta_bank_(\d+)\.json$/, schemas.termMetaBankV3],
+        [/^kanji_bank_(\d+)\.json$/, version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3],
+        [/^kanji_meta_bank_(\d+)\.json$/, schemas.kanjiMetaBankV3],
+        [/^tag_bank_(\d+)\.json$/, schemas.tagBankV3]
+    ];
+
+    await validateDictionaryBanks(mode, archive, schemasDetails);
 }
 
 /**

diff --git a/ext/js/dictionary/dictionary-importer.js b/ext/js/dictionary/dictionary-importer.js
@@ -115,13 +115,13 @@ export class DictionaryImporter {
 
         // Files
         /** @type {import('dictionary-importer').QueryDetails} */
-        const queryDetails = new Map([
+        const queryDetails = [
             ['termFiles', /^term_bank_(\d+)\.json$/],
             ['termMetaFiles', /^term_meta_bank_(\d+)\.json$/],
             ['kanjiFiles', /^kanji_bank_(\d+)\.json$/],
             ['kanjiMetaFiles', /^kanji_meta_bank_(\d+)\.json$/],
             ['tagFiles', /^tag_bank_(\d+)\.json$/]
-        ]);
+        ];
         const {termFiles, termMetaFiles, kanjiFiles, kanjiMetaFiles, tagFiles} = Object.fromEntries(this._getArchiveFiles(fileMap, queryDetails));
 
         // Load data
@@ -692,16 +692,18 @@ export class DictionaryImporter {
     _getArchiveFiles(fileMap, queryDetails) {
         /** @type {import('dictionary-importer').QueryResult} */
         const results = new Map();
-        for (const [name, value] of fileMap.entries()) {
-            for (const [fileType, fileNameFormat] of queryDetails.entries()) {
-                let entries = results.get(fileType);
-                if (typeof entries === 'undefined') {
-                    entries = [];
-                    results.set(fileType, entries);
-                }
 
-                if (fileNameFormat.test(name)) {
-                    entries.push(value);
+        for (const [fileType] of queryDetails) {
+            results.set(fileType, []);
+        }
+
+        for (const [fileName, fileEntry] of fileMap.entries()) {
+            for (const [fileType, fileNameFormat] of queryDetails) {
+                if (!fileNameFormat.test(fileName)) { continue; }
+                const entries = results.get(fileType);
+
+                if (typeof entries !== 'undefined') {
+                    entries.push(fileEntry);
                     break;
                 }
             }

diff --git a/types/dev/dictionary-validate.d.ts b/types/dev/dictionary-validate.d.ts
@@ -29,3 +29,8 @@ export type Schemas = {
     termBankV3: Schema;
     termMetaBankV3: Schema;
 };
+
+/**
+ * An array of tuples of a regular expression for file types inside a dictionary and its corresponding schema.
+ */
+export type SchemasDetails = [fileNameFormat: RegExp, schema: unknown][];
diff --git a/types/ext/dictionary-importer.d.ts b/types/ext/dictionary-importer.d.ts
@@ -113,9 +113,9 @@ export type ImportRequirementContext = {
 export type ArchiveFileMap = Map<string, ZipJS.Entry>;
 
 /**
- * A map of file types inside a dictionary and its corresponding regular expressions.
+ * An array of tuples of a file type inside a dictionary and its corresponding regular expression.
  */
-export type QueryDetails = Map<string, RegExp>;
+export type QueryDetails = [fileType: string, fileNameFormat: RegExp][];
 
 /**
  * A map of file types inside a dictionary and its matching entries.