diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..759c6342e --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +inference/**/*.gz filter=lfs diff=lfs merge=lfs -text diff --git a/Taskfile.yml b/Taskfile.yml index dfd520353..09cc5c7b2 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -101,16 +101,19 @@ tasks: inference-test-wasm: desc: Run inference build-wasm JS tests. - deps: - - task: inference-build-wasm - vars: - # When the host system is macOS, the WASM build fails when - # building with multiple threads in the Docker container. - # If the host system is macOS, pass -j 1. - CLI_ARGS: '{{if eq (env "HOST_OS") "Darwin"}}-j 1{{end}}' cmds: - >- - cd inference/wasm/tests && npm install && npm run test + ./inference/scripts/test-wasm.py {{.CLI_ARGS}} + + lint-eslint: + desc: Checks the styling of the JS code with eslint. + cmds: + - cd ./inference/wasm/tests && npm install && npm run lint + + lint-eslint-fix: + desc: Fixes the styling of the JS code with eslint. + cmds: + - cd ./inference/wasm/tests && npm install && npm run lint:fix lint-black: desc: Checks the styling of the Python code with Black. @@ -141,12 +144,14 @@ tasks: lint-fix: desc: Fix all automatically fixable errors. This is useful to run before pushing. cmds: + - task: lint-eslint-fix - task: lint-black-fix - task: lint-ruff-fix lint: desc: Run all available linting tools. cmds: + - task: lint-eslint - task: lint-black - task: lint-ruff diff --git a/inference/.gitignore b/inference/.gitignore index 354d1f09f..e90e06a19 100644 --- a/inference/.gitignore +++ b/inference/.gitignore @@ -15,15 +15,18 @@ compile_commands.json CTestTestfile.cmake _deps +# Build paths +build +build-local +build-native +build-wasm -/build -/build-local -/build-native -/build-wasm -models -wasm/test_page/node_modules -wasm/module/worker/bergamot-translator-worker.* -wasm/module/browsermt-bergamot-translator-*.tgz +# WASM +wasm/tests/generated +wasm/tests/models/**/*.bin +wasm/tests/models/**/*.spm +wasm/tests/node_modules +wasm/tests/.vitest-reports # VSCode .vscode diff --git a/inference/CMakeLists.txt b/inference/CMakeLists.txt index dc9762689..5bb7addd3 100644 --- a/inference/CMakeLists.txt +++ b/inference/CMakeLists.txt @@ -162,7 +162,7 @@ if(COMPILE_WASM) -sEXPORTED_FUNCTIONS=[_int8PrepareAFallback,_int8PrepareBFallback,_int8PrepareBFromTransposedFallback,_int8PrepareBFromQuantizedTransposedFallback,_int8PrepareBiasFallback,_int8MultiplyAndAddBiasFallback,_int8SelectColumnsOfBFallback] # Necessary for mozintgemm linking. This prepares the `wasmMemory` variable ahead of time as # opposed to delegating that task to the wasm binary itself. This way we can link MozIntGEMM - # module to the same memory as the main bergamot-translator module. + # module to the same memory as the main bergamot-translator-source module. -sIMPORTED_MEMORY=1 # Dynamic execution is either frowned upon or blocked inside browser extensions -sDYNAMIC_EXECUTION=0 diff --git a/inference/scripts/build-wasm.py b/inference/scripts/build-wasm.py index 92f6c1193..0ecf5268c 100755 --- a/inference/scripts/build-wasm.py +++ b/inference/scripts/build-wasm.py @@ -18,11 +18,12 @@ MARIAN_PATH = os.path.join(THIRD_PARTY_PATH, "browsermt-marian-dev") EMSDK_PATH = os.path.join(THIRD_PARTY_PATH, "emsdk") EMSDK_ENV_PATH = os.path.join(EMSDK_PATH, "emsdk_env.sh") -WASM_PATH = os.path.join(BUILD_PATH, "bergamot-translator-worker.wasm") -JS_PATH = os.path.join(BUILD_PATH, "bergamot-translator-worker.js") +WASM_ARTIFACT = os.path.join(BUILD_PATH, "bergamot-translator.wasm") +JS_ARTIFACT = os.path.join(BUILD_PATH, "bergamot-translator.js") PATCHES_PATH = os.path.join(INFERENCE_PATH, "patches") BUILD_DIRECTORY = os.path.join(INFERENCE_PATH, "build-wasm") -GEMM_SCRIPT = os.path.join(INFERENCE_PATH, "wasm", "patch-artifacts-import-gemm-module.sh") +WASM_PATH = os.path.join(INFERENCE_PATH, "wasm") +GEMM_SCRIPT = os.path.join(WASM_PATH, "patch-artifacts-import-gemm-module.sh") DETECT_DOCKER_SCRIPT = os.path.join(SCRIPTS_PATH, "detect-docker.sh") patches = [ @@ -95,6 +96,56 @@ def revert_git_patch(repo_path, patch_path): subprocess.check_call(["git", "apply", "-R", "--reject", patch_path], cwd=PROJECT_ROOT_PATH) +def prepare_js_artifact(): + """ + Prepares the Bergamot JS artifact for use in Gecko by adding the proper license header + to the file, including helpful memory-growth logging, and wrapping the generated code + in a single function that both takes and returns the Bergamot WASM module. + """ + # Start with the license header and function wrapper + source = ( + "\n".join( + [ + "/* This Source Code Form is subject to the terms of the Mozilla Public", + " * License, v. 2.0. If a copy of the MPL was not distributed with this", + " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */", + "", + "function loadBergamot(Module) {", + "", + ] + ) + + "\n" + ) + + # Read the original JS file and indent its content + with open(JS_ARTIFACT, "r", encoding="utf8") as file: + for line in file: + source += " " + line + + # Close the function wrapper + source += "\n return Module;\n}" + + # Use the Module's printing + source = source.replace("console.log(", "Module.print(") + + # Add instrumentation to log memory size information + source = source.replace( + "function updateGlobalBufferAndViews(buf) {", + """ + function updateGlobalBufferAndViews(buf) { + const mb = (buf.byteLength / 1_000_000).toFixed(); + Module.print( + `Growing wasm buffer to ${mb}MB (${buf.byteLength} bytes).` + ); + """, + ) + + print(f"\nšŸ“„ Updating {JS_ARTIFACT} in place") + # Write the modified content back to the original file + with open(JS_ARTIFACT, "w", encoding="utf8") as file: + file.write(source) + + def build_bergamot(args: Optional[list[str]]): if args.clobber and os.path.exists(BUILD_PATH): shutil.rmtree(BUILD_PATH) @@ -127,7 +178,18 @@ def run_shell(command): print("\nšŸƒ Running CMake for Bergamot\n") run_shell(f"emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off {flags} {INFERENCE_PATH}") - cores = args.j if args.j else multiprocessing.cpu_count() + if args.j: + # If -j is specified explicitly, use it. + cores = args.j + elif os.getenv("HOST_OS") == "Darwin": + # There is an issue building with multiple cores when the Linux Docker container is + # running on a macOS host system. If the Docker container was created with HOST_OS + # set to Darwin, we should use only 1 core to build. + cores = 1 + else: + # Otherwise, build with as many cores as we have. + cores = multiprocessing.cpu_count() + print(f"\nšŸƒ Building Bergamot with emmake using {cores} cores\n") try: @@ -142,14 +204,14 @@ def run_shell(command): subprocess.check_call(["bash", GEMM_SCRIPT, BUILD_PATH]) print("\nāœ… Build complete\n") - print(" " + JS_PATH) - print(" " + WASM_PATH) + print(" " + JS_ARTIFACT) + print(" " + WASM_ARTIFACT) # Get the sizes of the build artifacts. - wasm_size = os.path.getsize(WASM_PATH) + wasm_size = os.path.getsize(WASM_ARTIFACT) gzip_size = int( subprocess.run( - f"gzip -c {WASM_PATH} | wc -c", + f"gzip -c {WASM_ARTIFACT} | wc -c", check=True, shell=True, capture_output=True, @@ -158,6 +220,8 @@ def run_shell(command): print(f" Uncompressed wasm size: {to_human_readable(wasm_size)}") print(f" Compressed wasm size: {to_human_readable(gzip_size)}") + prepare_js_artifact() + finally: print("\nšŸ–Œļø Reverting the source code patches\n") for repo_path, patch_path in patches[::-1]: @@ -167,6 +231,16 @@ def run_shell(command): def main(): args = parser.parse_args() + if ( + os.path.exists(BUILD_PATH) + and os.path.isdir(BUILD_PATH) + and os.listdir(BUILD_PATH) + and not args.clobber + ): + print(f"\nšŸ—ļø Build directory {BUILD_PATH} already exists and is non-empty.\n") + print(" Pass the --clobber flag to rebuild if desired.") + return + if not os.path.exists(THIRD_PARTY_PATH): os.mkdir(THIRD_PARTY_PATH) diff --git a/inference/scripts/clean.sh b/inference/scripts/clean.sh index 73f5ae5eb..d184c9cbe 100755 --- a/inference/scripts/clean.sh +++ b/inference/scripts/clean.sh @@ -10,20 +10,21 @@ cd "$(dirname $0)/.." # List of directories to clean dirs=("build-local" "build-wasm" "emsdk") -# Flag to track if any directories were cleaned -cleaned=false - # Check and remove directories for dir in "${dirs[@]}"; do if [ -d "$dir" ]; then echo "Removing $dir..." rm -rf "$dir" - cleaned=true fi done -# If no directories were cleaned, print a message -if [ "$cleaned" = false ]; then - echo "Nothing to clean" -fi +echo "Removing generated wasm artifacts..." +rm -rf wasm/tests/generated/*.js +rm -rf wasm/tests/generated/*.wasm +rm -rf wasm/tests/generated/*.sha256 + +echo "Removing extracted model files..." +rm -rf wasm/tests/models/**/*.bin +rm -rf wasm/tests/models/**/*.spm +echo diff --git a/inference/scripts/test-wasm.py b/inference/scripts/test-wasm.py new file mode 100755 index 000000000..2a571b889 --- /dev/null +++ b/inference/scripts/test-wasm.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import os +import shutil +import subprocess +import sys + +SCRIPTS_PATH = os.path.realpath(os.path.dirname(__file__)) +INFERENCE_PATH = os.path.dirname(SCRIPTS_PATH) +BUILD_PATH = os.path.join(INFERENCE_PATH, "build-wasm") +WASM_PATH = os.path.join(INFERENCE_PATH, "wasm") +WASM_TESTS_PATH = os.path.join(WASM_PATH, "tests") +GENERATED_PATH = os.path.join(WASM_TESTS_PATH, "generated") +MODELS_PATH = os.path.join(WASM_TESTS_PATH, "models") +WASM_ARTIFACT = os.path.join(BUILD_PATH, "bergamot-translator.wasm") +JS_ARTIFACT = os.path.join(BUILD_PATH, "bergamot-translator.js") +JS_ARTIFACT_HASH = os.path.join(GENERATED_PATH, "bergamot-translator.js.sha256") + + +def calculate_sha256(file_path): + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + + +def main(): + parser = argparse.ArgumentParser( + description="Test WASM by building and handling artifacts.", + formatter_class=argparse.RawTextHelpFormatter, + ) + + parser.add_argument("--clobber", action="store_true", help="Clobber the build artifacts") + parser.add_argument( + "--debug", + action="store_true", + help="Build with debug symbols, useful for profiling", + ) + parser.add_argument( + "-j", + type=int, + help="Number of cores to use for building (default: all available cores)", + ) + args = parser.parse_args() + + build_wasm_script = os.path.join(SCRIPTS_PATH, "build-wasm.py") + build_command = [sys.executable, build_wasm_script] + if args.clobber: + build_command.append("--clobber") + if args.debug: + build_command.append("--debug") + if args.j: + build_command.extend(["-j", str(args.j)]) + + print("\nšŸš€ Starting build-wasm.py") + subprocess.run(build_command, check=True) + + print("\nšŸ“„ Pulling translations model files with git lfs\n") + subprocess.run(["git", "lfs", "pull"], cwd=MODELS_PATH, check=True) + print(f" Pulled all files in {MODELS_PATH}") + + print("\nšŸ“ Copying generated build artifacts to the WASM test directory\n") + + os.makedirs(GENERATED_PATH, exist_ok=True) + shutil.copy2(WASM_ARTIFACT, GENERATED_PATH) + shutil.copy2(JS_ARTIFACT, GENERATED_PATH) + + print(f" Copied the following artifacts to {GENERATED_PATH}:") + print(f" - {JS_ARTIFACT}") + print(f" - {WASM_ARTIFACT}") + + print(f"\nšŸ”‘ Calculating SHA-256 hash of {JS_ARTIFACT}\n") + hash_value = calculate_sha256(JS_ARTIFACT) + with open(JS_ARTIFACT_HASH, "w") as hash_file: + hash_file.write(f"{hash_value} {os.path.basename(JS_ARTIFACT)}\n") + print(f" Hash of {JS_ARTIFACT} written to") + print(f" {JS_ARTIFACT_HASH}") + + print("\nšŸ“‚ Decompressing model files required for WASM testing\n") + subprocess.run(["gzip", "-dkrf", MODELS_PATH], check=True) + print(f" Decompressed models in {MODELS_PATH}\n") + + print("\nšŸ”§ Installing npm dependencies for WASM JS tests\n") + subprocess.run(["npm", "install"], cwd=WASM_TESTS_PATH, check=True) + + print("\nšŸ“Š Running Translations WASM JS tests\n") + subprocess.run(["npm", "run", "test"], cwd=WASM_TESTS_PATH, check=True) + + print("\nāœ… test-wasm.py completed successfully.\n") + + +if __name__ == "__main__": + main() diff --git a/inference/src/tests/CMakeLists.txt b/inference/src/tests/CMakeLists.txt index cd0e4c777..d1941e68d 100644 --- a/inference/src/tests/CMakeLists.txt +++ b/inference/src/tests/CMakeLists.txt @@ -16,7 +16,7 @@ if(NOT MSVC) set(TEST_BINARIES async blocking intgemm-resolve wasm) foreach(binary ${TEST_BINARIES}) add_executable("${binary}" "${binary}.cpp") - target_link_libraries("${binary}" bergamot-translator) + target_link_libraries("${binary}" bergamot-translator-source) set_target_properties("${binary}" PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests/") endforeach(binary) diff --git a/inference/src/tests/units/CMakeLists.txt b/inference/src/tests/units/CMakeLists.txt index 9cfb50006..8474b5db5 100644 --- a/inference/src/tests/units/CMakeLists.txt +++ b/inference/src/tests/units/CMakeLists.txt @@ -11,9 +11,9 @@ foreach(test ${UNIT_TESTS}) target_include_directories("run_${test}" PRIVATE ${CATCH_INCLUDE_DIR} "${CMAKE_SOURCE_DIR}/src") if(CUDA_FOUND) - target_link_libraries("run_${test}" ${EXT_LIBS} marian ${EXT_LIBS} marian_cuda ${EXT_LIBS} Catch bergamot-translator) + target_link_libraries("run_${test}" ${EXT_LIBS} marian ${EXT_LIBS} marian_cuda ${EXT_LIBS} Catch bergamot-translator-source) else(CUDA_FOUND) - target_link_libraries("run_${test}" marian ${EXT_LIBS} Catch bergamot-translator) + target_link_libraries("run_${test}" marian ${EXT_LIBS} Catch bergamot-translator-source) endif(CUDA_FOUND) if(msvc) diff --git a/inference/src/translator/CMakeLists.txt b/inference/src/translator/CMakeLists.txt index 1d773b46b..c03cc74f5 100644 --- a/inference/src/translator/CMakeLists.txt +++ b/inference/src/translator/CMakeLists.txt @@ -2,7 +2,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/project_version.h.in ${CMAKE_CURRENT_BINARY_DIR}/project_version.h @ONLY) -add_library(bergamot-translator STATIC +add_library(bergamot-translator-source STATIC byte_array_util.cpp text_processor.cpp translation_model.cpp @@ -23,23 +23,23 @@ if (USE_WASM_COMPATIBLE_SOURCE) # Using wasm compatible sources should include this compile definition; # Has to be done here because we are including marian headers + some sources # in local repository use these definitions - target_compile_definitions(bergamot-translator PUBLIC USE_SSE2 WASM_COMPATIBLE_SOURCE) + target_compile_definitions(bergamot-translator-source PUBLIC USE_SSE2 WASM_COMPATIBLE_SOURCE) endif() if(COMPILE_WASM) - target_compile_definitions(bergamot-translator PUBLIC WASM) + target_compile_definitions(bergamot-translator-source PUBLIC WASM) # Enable code that is required for generating JS bindings - target_compile_definitions(bergamot-translator PRIVATE WASM_BINDINGS) - target_compile_options(bergamot-translator PRIVATE ${WASM_COMPILE_FLAGS}) - target_link_options(bergamot-translator PRIVATE ${WASM_LINK_FLAGS}) + target_compile_definitions(bergamot-translator-source PRIVATE WASM_BINDINGS) + target_compile_options(bergamot-translator-source PRIVATE ${WASM_COMPILE_FLAGS}) + target_link_options(bergamot-translator-source PRIVATE ${WASM_LINK_FLAGS}) endif(COMPILE_WASM) if(ENABLE_CACHE_STATS) - target_compile_definitions(bergamot-translator PUBLIC ENABLE_CACHE_STATS) + target_compile_definitions(bergamot-translator-source PUBLIC ENABLE_CACHE_STATS) endif(ENABLE_CACHE_STATS) -target_link_libraries(bergamot-translator marian ssplit) +target_link_libraries(bergamot-translator-source marian ssplit) -target_include_directories(bergamot-translator +target_include_directories(bergamot-translator-source PUBLIC ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/src) diff --git a/inference/wasm/CMakeLists.txt b/inference/wasm/CMakeLists.txt index ef8fd988a..10f447ec1 100644 --- a/inference/wasm/CMakeLists.txt +++ b/inference/wasm/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(bergamot-translator-worker +add_executable(bergamot-translator bindings/service_bindings.cpp bindings/response_options_bindings.cpp bindings/response_bindings.cpp @@ -9,21 +9,21 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/project_version.js.in ${CMAKE_CURRENT_BINARY_DIR}/project_version.js @ONLY) # This header inclusion needs to go away later as path to public headers of bergamot -# translator should be directly available from "bergamot-translator" target -target_include_directories(bergamot-translator-worker +# translator should be directly available from "bergamot-translator-source" target +target_include_directories(bergamot-translator PRIVATE ${CMAKE_SOURCE_DIR}/src/translator PRIVATE ${CMAKE_SOURCE_DIR} ) # This compile definition is required for generating binding code properly -target_compile_definitions(bergamot-translator-worker PRIVATE WASM_BINDINGS) -target_compile_options(bergamot-translator-worker PRIVATE ${WASM_COMPILE_FLAGS}) -target_link_options(bergamot-translator-worker PRIVATE ${WASM_LINK_FLAGS}) -target_link_options(bergamot-translator-worker PRIVATE --extern-pre-js=${CMAKE_CURRENT_BINARY_DIR}/project_version.js) +target_compile_definitions(bergamot-translator PRIVATE WASM_BINDINGS) +target_compile_options(bergamot-translator PRIVATE ${WASM_COMPILE_FLAGS}) +target_link_options(bergamot-translator PRIVATE ${WASM_LINK_FLAGS}) +target_link_options(bergamot-translator PRIVATE --extern-pre-js=${CMAKE_CURRENT_BINARY_DIR}/project_version.js) -set_target_properties(bergamot-translator-worker PROPERTIES +set_target_properties(bergamot-translator PROPERTIES SUFFIX ".js" RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} ) -target_link_libraries(bergamot-translator-worker bergamot-translator) +target_link_libraries(bergamot-translator bergamot-translator-source) diff --git a/inference/wasm/bindings/bergamot-translator.d.ts b/inference/wasm/bindings/bergamot-translator.d.ts new file mode 100644 index 000000000..07005ef40 --- /dev/null +++ b/inference/wasm/bindings/bergamot-translator.d.ts @@ -0,0 +1,146 @@ + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +export namespace Bergamot { + /** + * The main module that is returned from bergamot-translator.js. + */ + export interface BergamotModule { + BlockingService: typeof BlockingService; + AlignedMemoryList: typeof AlignedMemoryList; + TranslationModel: typeof TranslationModel; + AlignedMemory: typeof AlignedMemory; + VectorResponseOptions: typeof VectorResponseOptions; + VectorString: typeof VectorString; + } + + /** + * This class represents a C++ std::vector. The implementations will extend from it. + */ + export class Vector { + size(): number; + get(index: number): T; + push_back(item: T); + } + + export class VectorResponse extends Vector {} + export class VectorString extends Vector {} + export class VectorResponseOptions extends Vector {} + export class AlignedMemoryList extends Vector {} + + /** + * A blocking (e.g. non-threaded) translation service, via Bergamot. + */ + export class BlockingService { + /** + * Translate multiple messages in a single synchronous API call using a single model. + */ + translate( + translationModel, + vectorSourceText: VectorString, + vectorResponseOptions: VectorResponseOptions + ): VectorResponse; + + /** + * Translate by pivoting between two models + * + * For example to translate "fr" to "es", pivot using "en": + * "fr" to "en" + * "en" to "es" + */ + translateViaPivoting( + first: TranslationModel, + second: TranslationModel, + vectorSourceText: VectorString, + vectorResponseOptions: VectorResponseOptions + ): VectorResponse; + } + + /** + * The actual translation model, which is passed into the `BlockingService` methods. + */ + export class TranslationModel {} + + /** + * The models need to be placed in the wasm memory space. This object represents + * aligned memory that was allocated on the wasm side of things. The memory contents + * can be set via the getByteArrayView method and the Uint8Array.prototype.set method. + */ + export class AlignedMemory { + constructor(size: number, alignment: number); + size(): number; + getByteArrayView(): Uint8Array/** + * The following are the types that are provided by the Bergamot wasm library. + */; + } + + /** + * The response from the translation. This definition isn't complete, but just + * contains a subset of the available methods. + */ + export class Response { + getOriginalText(): string; + getTranslatedText(): string; + } + + /** + * The options to configure a translation response. + */ + export class ResponseOptions { + // Include the quality estimations. + qualityScores: boolean; + // Include the alignments. + alignment: boolean; + // Remove HTML tags from text and insert it back into the output. + html: boolean; + // Whether to include sentenceMappings or not. Alignments require + // sentenceMappings and are available irrespective of this option if + // `alignment=true`. + sentenceMappings: boolean + } +} + +/** + * A single language model file. + */ +interface LanguageTranslationModelFile { + buffer: ArrayBuffer, +} + +/** + * The files necessary to run the translations, these will be sent to the Bergamot + * translation engine. + */ +interface LanguageTranslationModelFiles { + // The machine learning language model. + model: LanguageTranslationModelFile, + // The lexical shortlist that limits possible output of the decoder and makes + // inference faster. + lex: LanguageTranslationModelFile, + // A model that can generate a translation quality estimation. + qualityModel?: LanguageTranslationModelFile, + + // There is either a single vocab file: + vocab?: LanguageTranslationModelFile, + + // Or there are two: + srcvocab?: LanguageTranslationModelFile, + trgvocab?: LanguageTranslationModelFile, +}; + +/** + * This is the type that is generated when the models are loaded into wasm aligned memory. + */ +type LanguageTranslationModelFilesAligned = { + [K in keyof LanguageTranslationModelFiles]: AlignedMemory +}; + +/** + * These are the files that are that are necessary to start the translations engine. + */ +interface TranslationsEnginePayload { + bergamotWasmArrayBuffer: ArrayBuffer, + languageModelFiles: LanguageTranslationModelFiles[] +} diff --git a/inference/wasm/patch-artifacts-import-gemm-module.sh b/inference/wasm/patch-artifacts-import-gemm-module.sh index d9fa648fe..aaf367f81 100644 --- a/inference/wasm/patch-artifacts-import-gemm-module.sh +++ b/inference/wasm/patch-artifacts-import-gemm-module.sh @@ -23,7 +23,7 @@ if [ "$#" -eq 1 ]; then ARTIFACTS_FOLDER="$1" fi -ARTIFACT="$ARTIFACTS_FOLDER/bergamot-translator-worker.js" +ARTIFACT="$ARTIFACTS_FOLDER/bergamot-translator.js" if [ ! -e "$ARTIFACT" ]; then echo "Error: Artifact \"$ARTIFACT\" doesn't exist" exit diff --git a/inference/wasm/tests/.gitignore b/inference/wasm/tests/.gitignore deleted file mode 100644 index c2658d7d1..000000000 --- a/inference/wasm/tests/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules/ diff --git a/inference/wasm/tests/engine/translations-engine.mjs b/inference/wasm/tests/engine/translations-engine.mjs new file mode 100644 index 000000000..3015b923b --- /dev/null +++ b/inference/wasm/tests/engine/translations-engine.mjs @@ -0,0 +1,254 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file is a pared-down version of `translations-engine.sys.mjs` from the Firefox source code. + * https://searchfox.org/mozilla-central/rev/450aacd753c98b3200f120ed4340e1ed53b7ff47/toolkit/components/translations/content/translations-engine.sys.mjs + * + * This version excludes the Firefox-specific complexity and mechanisms that are required for integration into + * the Firefox Translations ecosystem. This allows us to test the WASM bindings directly within development + * environment in which they are generated, before they are vendored into Firefox. + * + * The worker used within this file runs in a Node.js worker_threads environment, but is designed to simulate + * the same code paths of communicating with Web Worker in a browser environment. A subset of the Web Worker + * functionality is simulated to provide the required API surface. + * + * @see {WebWorkerSimulator} + */ + +/** + * @typedef {import('./../../bindings/bergamot-translator.d.ts').LanguageTranslationModelFiles} LanguageTranslationModelFiles + */ + +import path from "path"; +import fs from "fs/promises"; +import { Worker } from "node:worker_threads"; + +const MODELS_PATH = "./models"; +const WORKER_PATH = "./engine/translations-engine.worker.mjs"; +const WASM_PATH = "./generated/bergamot-translator.wasm"; +const PIVOT = "en"; + +export class TranslationsEngine { + #worker; + #isReady; + #isReadyResolve; + #isReadyReject; + #currentTranslationResolve = null; + #currentTranslationReject = null; + + /** + * Constructs a new Translator instance. + * + * @param {string} sourceLanguage - The source language code (e.g., 'es'). + * @param {string} targetLanguage - The target language code (e.g., 'fr'). + */ + constructor(sourceLanguage, targetLanguage) { + this.#worker = new Worker(WORKER_PATH); + + this.#worker.on("message", (data) => this.#handleMessage({ data })); + this.#worker.on("error", (error) => this.#handleError({ error })); + + this.#isReady = this.#initWorker(sourceLanguage, targetLanguage); + } + + /** + * Private method to initialize the worker by reading necessary files and sending the initialization message. + * + * @returns {Promise} + */ + async #initWorker(sourceLanguage, targetLanguage) { + try { + const wasmBuffer = await fs.readFile(WASM_PATH); + const languageModelFiles = await this.#prepareLanguageModelFiles( + sourceLanguage, + targetLanguage, + ); + + // Return a promise that resolves or rejects based on worker messages + const isReadyPromise = new Promise((resolve, reject) => { + this.#isReadyResolve = resolve; + this.#isReadyReject = reject; + }); + + this.#worker.postMessage({ + type: "initialize", + enginePayload: { + bergamotWasmArrayBuffer: wasmBuffer.buffer, + languageModelFiles, + }, + }); + + return isReadyPromise; + } catch (error) { + throw new Error(` + šŸšØ Failed to read one or more files required for translation šŸšØ + + ${error} + + ā© NEXT STEPS ā© + + To ensure that test dependencies are properly setup, please run the following command: + + āÆ task inference-test-wasm + `); + } + } + + /** + * Private helper method to prepare the language model files. + * + * @param {string} sourceLanguage - The source language code. + * @param {string} targetLanguage - The target language code. + * @returns {Promise>} - An array of language model files. + */ + async #prepareLanguageModelFiles(sourceLanguage, targetLanguage) { + let languageModelFilePromises; + + if (sourceLanguage === PIVOT || targetLanguage === PIVOT) { + languageModelFilePromises = [ + this.#loadLanguageModelFiles(sourceLanguage, targetLanguage), + ]; + } else { + languageModelFilePromises = [ + this.#loadLanguageModelFiles(sourceLanguage, PIVOT), + this.#loadLanguageModelFiles(PIVOT, targetLanguage), + ]; + } + + return Promise.all(languageModelFilePromises); + } + + /** + * Private helper method to load language model files. + * + * @param {string} sourceLanguage - The source language code. + * @param {string} targetLanguage - The target language code. + * @returns {Promise} - An object containing the model, lexicon, and vocabulary buffers. + */ + async #loadLanguageModelFiles(sourceLanguage, targetLanguage) { + const langPairDirectory = `${MODELS_PATH}/${sourceLanguage}${targetLanguage}`; + + const lexPath = path.join( + langPairDirectory, + `lex.50.50.${sourceLanguage}${targetLanguage}.s2t.bin`, + ); + const modelPath = path.join( + langPairDirectory, + `model.${sourceLanguage}${targetLanguage}.intgemm.alphas.bin`, + ); + const vocabPath = path.join( + langPairDirectory, + `vocab.${sourceLanguage}${targetLanguage}.spm`, + ); + + const [lexBuffer, modelBuffer, vocabBuffer] = await Promise.all([ + fs.readFile(lexPath), + fs.readFile(modelPath), + fs.readFile(vocabPath), + ]); + + return { + model: { buffer: modelBuffer }, + lex: { buffer: lexBuffer }, + vocab: { buffer: vocabBuffer }, + }; + } + + /** + * Private method to handle incoming messages from the worker. + * + * @param {MessageEvent} event - The message event from the worker. + */ + #handleMessage(event) { + const { data } = event; + + switch (data.type) { + case "initialization-success": { + this.#isReadyResolve && this.#isReadyResolve(); + break; + } + case "initialization-error": { + this.#isReadyReject && this.#isReadyReject(new Error(data.error)); + break; + } + case "translation-response": { + if (this.#currentTranslationResolve) { + this.#currentTranslationResolve(data.targetText); + this.#clearCurrentTranslation(); + } + break; + } + case "translation-error": { + if (this.#currentTranslationReject) { + this.#currentTranslationReject(new Error(data.error.message)); + this.#clearCurrentTranslation(); + } + break; + } + default: { + console.warn(`Unknown message type: ${data.type}`); + } + } + } + + /** + * Private method to handle errors from the worker. + * + * @param {ErrorEvent} error - The error event from the worker. + */ + #handleError(error) { + if (this.#isReadyReject) { + this.#isReadyReject(error); + } + if (this.#currentTranslationReject) { + this.#currentTranslationReject(error); + this.#clearCurrentTranslation(); + } + } + + /** + * Translates the given source text. + * + * @param {string} sourceText - The text to translate. + * @param {boolean} [isHTML=false] - Indicates if the source text is HTML. + * @returns {Promise} - The translated text. + */ + async translate(sourceText, isHTML = false) { + await this.#isReady; + + return new Promise((resolve, reject) => { + this.#currentTranslationResolve = resolve; + this.#currentTranslationReject = reject; + + // Send translation request + this.#worker.postMessage({ + type: "translation-request", + sourceText, + isHTML, + }); + }); + } + + /** + * Clears the current translation promise handlers. + */ + #clearCurrentTranslation() { + this.#currentTranslationResolve = null; + this.#currentTranslationReject = null; + } + + /** + * Terminates the worker and cleans up resources. + */ + terminate() { + if (this.#worker) { + this.#clearCurrentTranslation(); + this.#worker.terminate(); + this.#worker.onmessage = null; + this.#worker.onerror = null; + this.#worker = null; + } + } +} diff --git a/inference/wasm/tests/engine/translations-engine.worker.mjs b/inference/wasm/tests/engine/translations-engine.worker.mjs new file mode 100644 index 000000000..08089193b --- /dev/null +++ b/inference/wasm/tests/engine/translations-engine.worker.mjs @@ -0,0 +1,432 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file is a pared-down version of `translations-engine.worker.js` within the Firefox source code: + * https://searchfox.org/mozilla-central/rev/450aacd753c98b3200f120ed4340e1ed53b7ff47/toolkit/components/translations/content/translations-engine.worker.js + * + * This version excludes the Firefox-specific complexity and mechanisms that are required for integration into + * the Firefox Translations ecosystem. This allows us to test the WASM bindings directly within development + * environment in which they are generated, before they are vendored into Firefox. + * + * This file runs within a Node.js worker_threads environment, but is designed to simulate the same code paths + * of loading and running our generated code in a Web Workers environment. A subset of the WorkerGlobalScope + * functionality is simulated to provide the required API surface. + * + * @see {WorkerGlobalScopeSimulator} + */ + +/** + * Importing types from the TypeScript declaration file using JSDoc. + * This allows us to use the types in our JavaScript code for better documentation and tooling support. + * + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.BergamotModule} BergamotModule + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.TranslationModel} TranslationModel + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.BlockingService} BlockingService + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.VectorString} VectorString + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.VectorResponseOptions} VectorResponseOptions + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.VectorResponse} VectorResponse + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.ResponseOptions} ResponseOptions + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.Response} Response + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.AlignedMemory} AlignedMemory + * @typedef {import('./../../bindings/bergamot-translator.d.ts').Bergamot.AlignedMemoryList} AlignedMemoryList + * @typedef {import('./../../bindings/bergamot-translator.d.ts').LanguageTranslationModelFiles} LanguageTranslationModelFiles + * @typedef {import('./../../bindings/bergamot-translator.d.ts').LanguageTranslationModelFilesAligned} LanguageTranslationModelFilesAligned + * @typedef {import('./../../bindings/bergamot-translator.d.ts').TranslationsEnginePayload} TranslationsEnginePayload + */ + +import WorkerGlobalScopeSimulator from "./worker-global-scope-simulator.mjs"; + +/** + * Simulate the WorkerGlobalScope from a browser environment within Node.js + * https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope + */ +const self = new WorkerGlobalScopeSimulator(); + +try { + /* global loadBergamot */ + self.importScripts("./generated/bergamot-translator.js"); +} catch (error) { + self.postMessage({ type: "initialization-error", error: error.message }); +} + +/** + * Constants defining alignment requirements for different model files. + * + * @type {Record} + */ +const MODEL_FILE_ALIGNMENTS = { + model: 256, + lex: 64, + vocab: 64, + qualityModel: 64, + srcvocab: 64, + trgvocab: 64, +}; + +/** + * Event listener for handling initialization messages. + */ +self.addEventListener("message", handleInitializationMessage); + +/** + * Handles the initialization message to set up the translation engine. + * + * @param {MessageEvent} event - The message event containing initialization data. + */ +async function handleInitializationMessage(event) { + const { data } = event; + if (data.type !== "initialize") { + return; + } + + try { + /** @type {TranslationsEnginePayload} */ + const enginePayload = data.enginePayload; + const { bergamotWasmArrayBuffer, languageModelFiles } = enginePayload; + + /** @type {BergamotModule} */ + const bergamot = await BergamotUtils.initializeWasm( + bergamotWasmArrayBuffer, + ); + + const engine = new Engine(bergamot, languageModelFiles); + + // Handle translation requests + self.addEventListener("message", async (messageEvent) => { + const messageData = messageEvent.data; + + if (messageData.type === "translation-request") { + const { sourceText, isHTML } = messageData; + + try { + const { whitespaceBefore, whitespaceAfter, cleanedSourceText } = + cleanText(sourceText); + + const targetText = + whitespaceBefore + + engine.translate(cleanedSourceText, isHTML) + + whitespaceAfter; + + self.postMessage({ + type: "translation-response", + targetText, + }); + } catch (error) { + self.postMessage({ + type: "translation-error", + error: { message: error.message, stack: error.stack }, + }); + } + } + }); + + self.postMessage({ type: "initialization-success" }); + } catch (error) { + self.postMessage({ + type: "initialization-error", + error: error.message, + }); + } + + self.removeEventListener("message", handleInitializationMessage); +} + +/** + * The Engine class handles translation using the Bergamot WASM module. + */ +class Engine { + /** + * The initialized Bergamot WASM module. + * + * @type {BergamotModule} + */ + #bergamot; + + /** + * An array of translation models. + * + * @type {TranslationModel[]} + */ + #languageTranslationModels; + + /** + * The translation service used to perform translations. + * + * @type {BlockingService} + */ + #translationService; + + /** + * Constructs the Engine instance. + * + * @param {BergamotModule} bergamot - Initialized Bergamot module. + * @param {LanguageTranslationModelFiles[]} languageTranslationModelFiles - Model files for translation. + */ + constructor(bergamot, languageTranslationModelFiles) { + this.#bergamot = bergamot; + + this.#languageTranslationModels = languageTranslationModelFiles.map( + (modelFiles) => { + return BergamotUtils.constructSingleTranslationModel( + bergamot, + modelFiles, + ); + }, + ); + + this.#translationService = new bergamot.BlockingService({ cacheSize: 0 }); + } + + /** + * Translates the given source text. + * + * @param {string} sourceText - Text to translate. + * @param {boolean} isHTML - Indicates if the text contains HTML. + * @returns {string} Translated text. + */ + translate(sourceText, isHTML) { + return this.#syncTranslate(sourceText, isHTML); + } + + /** + * Performs synchronous translation. + * + * @param {string} sourceText - Text to translate. + * @param {boolean} isHTML - Indicates if the text contains HTML. + * @returns {string} Translated text. + */ + #syncTranslate(sourceText, isHTML) { + /** @type {VectorResponse} */ + let responses; + const { messages, options } = BergamotUtils.getTranslationArgs( + this.#bergamot, + sourceText, + isHTML, + ); + + try { + if (messages.size() === 0) { + return ""; + } + + if (this.#languageTranslationModels.length === 1) { + responses = this.#translationService.translate( + this.#languageTranslationModels[0], + messages, + options, + ); + } else if (this.#languageTranslationModels.length === 2) { + responses = this.#translationService.translateViaPivoting( + this.#languageTranslationModels[0], + this.#languageTranslationModels[1], + messages, + options, + ); + } else { + throw new Error( + "Too many models were provided to the translation worker.", + ); + } + + const targetText = responses.get(0).getTranslatedText(); + return targetText; + } finally { + messages.delete(); + options.delete(); + responses?.delete(); + } + } +} + +/** + * Utility class for Bergamot WASM operations. + */ +class BergamotUtils { + /** + * Constructs a single translation model. + * + * @param {BergamotModule} bergamot - Initialized Bergamot module. + * @param {LanguageTranslationModelFiles} languageTranslationModelFiles - Model files for translation. + * @returns {TranslationModel} Constructed translation model. + */ + static constructSingleTranslationModel( + bergamot, + languageTranslationModelFiles, + ) { + const { model, lex, vocab, qualityModel, srcvocab, trgvocab } = + BergamotUtils.allocateModelMemory( + bergamot, + languageTranslationModelFiles, + ); + + /** @type {AlignedMemoryList} */ + const vocabList = new bergamot.AlignedMemoryList(); + + if (vocab) { + vocabList.push_back(vocab); + } else if (srcvocab && trgvocab) { + vocabList.push_back(srcvocab); + vocabList.push_back(trgvocab); + } else { + throw new Error("Vocabulary key is not found."); + } + + const config = BergamotUtils.generateTextConfig({ + "beam-size": "1", + normalize: "1.0", + "word-penalty": "0", + "max-length-break": "128", + "mini-batch-words": "1024", + workspace: "128", + "max-length-factor": "2.0", + "skip-cost": "true", + "cpu-threads": "0", + quiet: "true", + "quiet-translation": "true", + "gemm-precision": "int8shiftAlphaAll", + alignment: "soft", + }); + + return new bergamot.TranslationModel( + config, + model, + lex, + vocabList, + qualityModel ?? null, + ); + } + + /** + * Allocates aligned memory for the model files. + * + * @param {BergamotModule} bergamot - Initialized Bergamot module. + * @param {LanguageTranslationModelFiles} languageTranslationModelFiles - Model files for translation. + * @returns {LanguageTranslationModelFilesAligned} Allocated memory for each file type. + */ + static allocateModelMemory(bergamot, languageTranslationModelFiles) { + /** @type {LanguageTranslationModelFilesAligned} */ + const results = {}; + + for (const [fileType, file] of Object.entries( + languageTranslationModelFiles, + )) { + const alignment = MODEL_FILE_ALIGNMENTS[fileType]; + if (!alignment) { + throw new Error(`Unknown file type: "${fileType}"`); + } + + /** @type {AlignedMemory} */ + const alignedMemory = new bergamot.AlignedMemory( + file.buffer.byteLength, + alignment, + ); + alignedMemory.getByteArrayView().set(new Uint8Array(file.buffer)); + + results[fileType] = alignedMemory; + } + + return results; + } + + /** + * Initializes the Bergamot WASM module. + * + * @param {ArrayBuffer} wasmBinary - The WASM binary data. + * @returns {Promise} Resolves with the initialized Bergamot module. + */ + static initializeWasm(wasmBinary) { + return new Promise((resolve, reject) => { + /** @type {BergamotModule} */ + const bergamot = loadBergamot({ + INITIAL_MEMORY: 234_291_200, + print: () => {}, + // Uncomment this line to display logs in tests. + // To show logs, run with the --runner=basic flag. + // print: (...args) => console.log(...args), + onAbort() { + reject(new Error("Error loading Bergamot WASM module.")); + }, + onRuntimeInitialized: () => { + try { + resolve(bergamot); + } catch (e) { + reject(e); + } + }, + wasmBinary, + }); + }); + } + + /** + * Generates a configuration string for the Marian translation service. + * + * @param {Record} config - Configuration key-value pairs. + * @returns {string} Formatted configuration string. + */ + static generateTextConfig(config) { + const indent = " "; + let result = "\n"; + + for (const [key, value] of Object.entries(config)) { + result += `${indent}${key}: ${value}\n`; + } + + return result + indent; + } + + /** + * Prepares translation arguments for the Bergamot module. + * + * @param {BergamotModule} bergamot - Initialized Bergamot module. + * @param {string} sourceText - Text to translate. + * @param {boolean} isHTML - Indicates if the text contains HTML. + * @returns {{messages: VectorString, options: VectorResponseOptions}} Prepared messages and options. + */ + static getTranslationArgs(bergamot, sourceText, isHTML) { + /** @type {VectorString} */ + const messages = new bergamot.VectorString(); + /** @type {VectorResponseOptions} */ + const options = new bergamot.VectorResponseOptions(); + + if (sourceText) { + messages.push_back(sourceText); + options.push_back({ + qualityScores: false, + alignment: true, + html: isHTML, + }); + } + + return { messages, options }; + } +} + +/** + * Regular expression to match whitespace before and after the text. + * + * @type {RegExp} + */ +const whitespaceRegex = /^(\s*)(.*?)(\s*)$/s; + +/** + * Cleans the text before translation by preserving surrounding whitespace and removing soft hyphens. + * + * @param {string} sourceText - The original text to clean. + * @returns {{whitespaceBefore: string, whitespaceAfter: string, cleanedSourceText: string}} Contains whitespace before, after, and the cleaned text. + */ +function cleanText(sourceText) { + const result = whitespaceRegex.exec(sourceText); + if (!result) { + throw new Error("Failed to match whitespace in the source text."); + } + const whitespaceBefore = result[1]; + const whitespaceAfter = result[3]; + let cleanedSourceText = result[2]; + + cleanedSourceText = cleanedSourceText.replace(/\u00AD/g, ""); + + return { whitespaceBefore, whitespaceAfter, cleanedSourceText }; +} diff --git a/inference/wasm/tests/engine/worker-global-scope-simulator.mjs b/inference/wasm/tests/engine/worker-global-scope-simulator.mjs new file mode 100644 index 000000000..2c7ac32ff --- /dev/null +++ b/inference/wasm/tests/engine/worker-global-scope-simulator.mjs @@ -0,0 +1,157 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { isMainThread, parentPort } from "node:worker_threads"; +import EventEmitter from "events"; +import { readFileSync } from "fs"; +import crypto from "crypto"; + +const BERGAMOT_HASH_PATH = "./generated/bergamot-translator.js.sha256"; + +/** + * WorkerGlobalScopeSimulator simulates the WorkerGlobalScope in a Node.js worker_threads environment. + * + * It provides a minimal implementation of the Web Workers API by mapping required functions to their + * Node.js `worker_threads` equivalents. This class allows us to test our code, intended for Web Workers + * to be tested in a Node.js environment without modification. + * + * Note: Only the functionality required to rest the WASM translation bindings is implemented here. + * This is not a full implementation, nor is this intended for general-purpose use. + * + * https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API + * https://nodejs.org/api/worker_threads.html + * + * @extends EventEmitter + */ +export default class WorkerGlobalScopeSimulator extends EventEmitter { + /** + * Constructs a new WorkerGlobalScopeSimulator instance. + * + * Initializes event handling to receive messages from the parent thread and emits + * them as 'message' events to simulate the Web Workers messaging API. + * + * @throws {Error} If instantiated from the main thread instead of a worker thread. + */ + constructor() { + super(); + + WorkerGlobalScopeSimulator.#ensureThreadIsWorker(); + + parentPort.on("message", (data) => { + this.emit("message", { data }); + }); + } + + /** + * Ensures that the code is running inside a worker thread. + * + * @throws {Error} If called from the main thread. + */ + static #ensureThreadIsWorker() { + if (isMainThread || !parentPort) { + throw new Error(` + Attempt to call ${this.name} from the main thread. + ${this.name} should only be used within a worker thread. + `); + } + } + + /** + * Reads and verifies the script by comparing its hash with the expected hash. + * + * @param {string} scriptPath - Path to the script to read and verify. + * @returns {string} The content of the verified script. + * @throws {Error} If the hash does not match or files cannot be read. + */ + static #readAndVerifyScript(scriptPath) { + const hashFileContent = readFileSync(BERGAMOT_HASH_PATH, { + encoding: "utf-8", + }); + const [expectedHash] = hashFileContent.trim().split(/\s+/); + if (!expectedHash) { + throw new Error(`Unable to extract hash from ${BERGAMOT_HASH_PATH}`); + } + + const scriptContent = readFileSync(scriptPath, { encoding: "utf-8" }); + const scriptContentHash = crypto + .createHash("sha256") + .update(scriptContent, "utf8") + .digest("hex"); + + if (scriptContentHash !== expectedHash) { + throw new Error(`Hash mismatch for script ${scriptPath} + Expected: ${expectedHash} + Received: ${scriptContentHash} + `); + } + + return scriptContent; + } + + /** + * Imports and executes a script, simulating the importScripts() function + * available in Web Workers. + * + * This function executes eval.call() and is not intended for general-purpose + * use. This is why it only takes a single script argument and validates that + * the script matches the expected hash before evaluating. + * + * @param {string} scriptPath - Path to the script to import and execute. + * @throws {Error} If the script fails to import or execute. + */ + importScripts(scriptPath) { + WorkerGlobalScopeSimulator.#ensureThreadIsWorker(); + + try { + const scriptContent = + WorkerGlobalScopeSimulator.#readAndVerifyScript(scriptPath); + eval.call(globalThis, scriptContent); + } catch (error) { + throw new Error(` + šŸšØ Failed to read or import the required script for translation šŸšØ + + ${error} + + ā© NEXT STEPS ā© + + To ensure that test dependencies are properly set up, please run the following command: + + āÆ task inference-test-wasm + `); + } + } + + /** + * Adds an event listener for the specified event type. + * + * @param {string} event - The event type to listen for. + * @param {Function} listener - The function to call when the event occurs. + */ + addEventListener(event, listener) { + WorkerGlobalScopeSimulator.#ensureThreadIsWorker(); + this.on(event, listener); + } + + /** + * Removes an event listener for the specified event type. + * + * @param {string} event - The event type to stop listening for. + * @param {Function} listener - The function to remove. + */ + removeEventListener(event, listener) { + WorkerGlobalScopeSimulator.#ensureThreadIsWorker(); + this.off(event, listener); + } + + /** + * Posts a message to the parent thread, simulating the `postMessage` function + * available in Web Workers. + * + * @param {any} message - The message to send to the parent thread. + */ + postMessage(message) { + WorkerGlobalScopeSimulator.#ensureThreadIsWorker(); + parentPort.postMessage(message); + } +} diff --git a/inference/wasm/tests/eslint.config.mjs b/inference/wasm/tests/eslint.config.mjs new file mode 100644 index 000000000..24946ac0f --- /dev/null +++ b/inference/wasm/tests/eslint.config.mjs @@ -0,0 +1,35 @@ +import js from "@eslint/js"; +import prettierPlugin from "eslint-plugin-prettier"; +import prettierConfig from "eslint-config-prettier"; + +export default [ + { + ignores: ["generated/**/*"], + }, + js.configs.recommended, + prettierConfig, + { + plugins: { + prettier: prettierPlugin, + }, + languageOptions: { + globals: { + console: "readonly", + }, + }, + rules: { + curly: ["error", "all"], + indent: [ + "error", + 2, + { + SwitchCase: 1, + }, + ], + "prefer-const": "error", + semi: "error", + quotes: ["error", "double"], + "prettier/prettier": "error", + }, + }, +]; diff --git a/inference/wasm/tests/models/enes/lex.50.50.enes.s2t.bin.gz b/inference/wasm/tests/models/enes/lex.50.50.enes.s2t.bin.gz new file mode 100644 index 000000000..119657856 --- /dev/null +++ b/inference/wasm/tests/models/enes/lex.50.50.enes.s2t.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ada43ef1592f2f8f2ab26125358a814f7704f28ab77829a859846d21630f28fb +size 1720700 diff --git a/inference/wasm/tests/models/enes/model.enes.intgemm.alphas.bin.gz b/inference/wasm/tests/models/enes/model.enes.intgemm.alphas.bin.gz new file mode 100644 index 000000000..c90d00efd --- /dev/null +++ b/inference/wasm/tests/models/enes/model.enes.intgemm.alphas.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f2cdfb25d855c7c4dd874c2d5d97154339ca9e93c5336c2ec0236415ffcb1ae +size 12602853 diff --git a/inference/wasm/tests/models/enes/vocab.enes.spm.gz b/inference/wasm/tests/models/enes/vocab.enes.spm.gz new file mode 100644 index 000000000..8274e8bb6 --- /dev/null +++ b/inference/wasm/tests/models/enes/vocab.enes.spm.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0148a29b7145e61871846bfda323a9cf70d1e559f14e161d5410423febdeda74 +size 414566 diff --git a/inference/wasm/tests/models/enfr/lex.50.50.enfr.s2t.bin.gz b/inference/wasm/tests/models/enfr/lex.50.50.enfr.s2t.bin.gz new file mode 100644 index 000000000..32ba76e82 --- /dev/null +++ b/inference/wasm/tests/models/enfr/lex.50.50.enfr.s2t.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5eaebf198b787b718b81948ddb184780a559e6eefdb7416fdf405ef3e50576 +size 4177155 diff --git a/inference/wasm/tests/models/enfr/model.enfr.intgemm.alphas.bin.gz b/inference/wasm/tests/models/enfr/model.enfr.intgemm.alphas.bin.gz new file mode 100644 index 000000000..b6b88a15b --- /dev/null +++ b/inference/wasm/tests/models/enfr/model.enfr.intgemm.alphas.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39efbb6ab154967e2762f7c4baa0ff2f2fa08f32bfe5f6d29b787726476e828 +size 12293754 diff --git a/inference/wasm/tests/models/enfr/vocab.enfr.spm.gz b/inference/wasm/tests/models/enfr/vocab.enfr.spm.gz new file mode 100644 index 000000000..d2ba36311 --- /dev/null +++ b/inference/wasm/tests/models/enfr/vocab.enfr.spm.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6822a172454fcf6acba0c926cde59022e4d525b0052dd8c89fb7bc76a1542e0 +size 419721 diff --git a/inference/wasm/tests/models/esen/lex.50.50.esen.s2t.bin.gz b/inference/wasm/tests/models/esen/lex.50.50.esen.s2t.bin.gz new file mode 100644 index 000000000..23fa1beaa --- /dev/null +++ b/inference/wasm/tests/models/esen/lex.50.50.esen.s2t.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa6c3a148854089e9f2b51d8a49fef896fad658dc1b4d8760a0b293254c03fde +size 1978538 diff --git a/inference/wasm/tests/models/esen/model.esen.intgemm.alphas.bin.gz b/inference/wasm/tests/models/esen/model.esen.intgemm.alphas.bin.gz new file mode 100644 index 000000000..b8e1306d8 --- /dev/null +++ b/inference/wasm/tests/models/esen/model.esen.intgemm.alphas.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03ef9234003b450b0b94f84d683bd2e215bc0a72db2bdac6ebe503356dfe73c6 +size 13215960 diff --git a/inference/wasm/tests/models/esen/vocab.esen.spm.gz b/inference/wasm/tests/models/esen/vocab.esen.spm.gz new file mode 100644 index 000000000..8274e8bb6 --- /dev/null +++ b/inference/wasm/tests/models/esen/vocab.esen.spm.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0148a29b7145e61871846bfda323a9cf70d1e559f14e161d5410423febdeda74 +size 414566 diff --git a/inference/wasm/tests/models/fren/lex.50.50.fren.s2t.bin.gz b/inference/wasm/tests/models/fren/lex.50.50.fren.s2t.bin.gz new file mode 100644 index 000000000..c469fe2bd --- /dev/null +++ b/inference/wasm/tests/models/fren/lex.50.50.fren.s2t.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b933f2b42516ed0d325dd862ee7c50acc74ffc5d2a66059b357b931f788df7 +size 4761904 diff --git a/inference/wasm/tests/models/fren/model.fren.intgemm.alphas.bin.gz b/inference/wasm/tests/models/fren/model.fren.intgemm.alphas.bin.gz new file mode 100644 index 000000000..ff5af0a65 --- /dev/null +++ b/inference/wasm/tests/models/fren/model.fren.intgemm.alphas.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f5618b93d08bd82bdafc096b5cdfe459e24b263f0a5a23dcf641a070ebb60b5 +size 12641501 diff --git a/inference/wasm/tests/models/fren/vocab.fren.spm.gz b/inference/wasm/tests/models/fren/vocab.fren.spm.gz new file mode 100644 index 000000000..a7a99fc5c --- /dev/null +++ b/inference/wasm/tests/models/fren/vocab.fren.spm.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e90e0e635234445df4defafdea8aff23f8a0d68c73744462c021b5dbff36e55f +size 419721 diff --git a/inference/wasm/tests/package-lock.json b/inference/wasm/tests/package-lock.json index 07d7dab19..90e0ef9c8 100644 --- a/inference/wasm/tests/package-lock.json +++ b/inference/wasm/tests/package-lock.json @@ -9,6 +9,9 @@ "version": "1.0.0", "license": "MPL-2.0", "devDependencies": { + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.2.1", + "prettier": "^3.3.3", "vitest": "^2.1.4" } }, @@ -403,6 +406,210 @@ "node": ">=12" } }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", + "integrity": "sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", + "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.18.0.tgz", + "integrity": "sha512-fTxvnS1sRMu3+JjXwJG0j/i4RT9u4qJ+lqS/yCGap4lH4zZGzQ7tu+xZqQmcMZq5OBZDL4QRxQzRjkWcGt8IVw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@eslint/object-schema": "^2.1.4", + "debug": "^4.3.1", + "minimatch": "^3.1.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.7.0.tgz", + "integrity": "sha512-xp5Jirz5DyPYlPiKat8jaq0EmYvDXKKpzTbxXMpT9eqlRJkRKIz9AGMdlvYjih+im+QlhWrpvVjl8IPC/lHlUw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.1.0.tgz", + "integrity": "sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "9.14.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.14.0.tgz", + "integrity": "sha512-pFoEtFWCPyDOl+C6Ift+wC7Ro89otjigCf5vcuWqWgqNSQbRrpjSvdeE6ofLz4dHmyxD5f7gIdGT4+p36L6Twg==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.4.tgz", + "integrity": "sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.2.2.tgz", + "integrity": "sha512-CXtq5nR4Su+2I47WPOlWud98Y5Lv8Kyxp2ukhgFx/eW6Blm18VXJO5WuQylPugRo8nbluoi6GvvxBLqHcvqUUw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.6", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.6.tgz", + "integrity": "sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.3.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node/node_modules/@humanwhocodes/retry": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.3.1.tgz", + "integrity": "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.1.tgz", + "integrity": "sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", @@ -410,6 +617,19 @@ "dev": true, "license": "MIT" }, + "node_modules/@pkgr/core": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", + "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.24.4", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.24.4.tgz", @@ -669,6 +889,14 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/@vitest/expect": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.4.tgz", @@ -782,6 +1010,74 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/acorn": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0", + "peer": true + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -792,6 +1088,26 @@ "node": ">=12" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/cac": { "version": "6.7.14", "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", @@ -802,6 +1118,17 @@ "node": ">=8" } }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/chai": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-5.1.2.tgz", @@ -819,6 +1146,24 @@ "node": ">=12" } }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/check-error": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.1.tgz", @@ -829,6 +1174,52 @@ "node": ">= 16" } }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/cross-spawn": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.5.tgz", + "integrity": "sha512-ZVJrKKYunU38/76t0RMOulHOnUcbU9GbpWKAOZ0mhjr7CX6FVrH+4FrAapSOekrgFQ3f/8gwMEuIft0aKq6Hug==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -857,6 +1248,14 @@ "node": ">=6" } }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/esbuild": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", @@ -896,62 +1295,584 @@ "@esbuild/win32-x64": "0.21.5" } }, - "node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true, "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "node_modules/expect-type": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.1.0.tgz", - "integrity": "sha512-bFi65yM+xZgk+u/KRIpekdSYkTB5W1pEf0Lt8Q8Msh7b+eQ7LXVtIB1Bkm4fvclDEL1b2CZkMhv2mOeF8tMdkA==", - "dev": true, - "license": "Apache-2.0", + "peer": true, "engines": { - "node": ">=12.0.0" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "node_modules/eslint": { + "version": "9.14.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.14.0.tgz", + "integrity": "sha512-c2FHsVBr87lnUtjP4Yhvk4yEhKrQavGafRA/Se1ouse8PfbfC/Qh9Mxa00yWsZRlqeUB9raXip0aiiUZkgnr9g==", "dev": true, - "hasInstallScript": true, "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.18.0", + "@eslint/core": "^0.7.0", + "@eslint/eslintrc": "^3.1.0", + "@eslint/js": "9.14.0", + "@eslint/plugin-kit": "^0.2.0", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.0", + "@types/estree": "^1.0.6", + "@types/json-schema": "^7.0.15", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.2.0", + "eslint-visitor-keys": "^4.2.0", + "espree": "^10.3.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } } }, - "node_modules/loupe": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.2.tgz", - "integrity": "sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==", + "node_modules/eslint-config-prettier": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz", + "integrity": "sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==", "dev": true, - "license": "MIT" + "license": "MIT", + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } }, - "node_modules/magic-string": { - "version": "0.30.12", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz", - "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==", + "node_modules/eslint-plugin-prettier": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.2.1.tgz", + "integrity": "sha512-gH3iR3g4JfF+yYPaJYkN7jEl9QbweL/YfkoRlNnuIEHEz1vHVlCmWOS+eGGiRuzHQXdJFCOTxRgvju9b8VUmrw==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0" + "prettier-linter-helpers": "^1.0.0", + "synckit": "^0.9.1" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-prettier" + }, + "peerDependencies": { + "@types/eslint": ">=8.0.0", + "eslint": ">=8.0.0", + "eslint-config-prettier": "*", + "prettier": ">=3.0.0" + }, + "peerDependenciesMeta": { + "@types/eslint": { + "optional": true + }, + "eslint-config-prettier": { + "optional": true + } } }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "node_modules/eslint-scope": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.2.0.tgz", + "integrity": "sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", + "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.3.0.tgz", + "integrity": "sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "acorn": "^8.14.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "dev": true, + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expect-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.1.0.tgz", + "integrity": "sha512-bFi65yM+xZgk+u/KRIpekdSYkTB5W1pEf0Lt8Q8Msh7b+eQ7LXVtIB1Bkm4fvclDEL1b2CZkMhv2mOeF8tMdkA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/fast-diff": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", + "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", + "dev": true, + "license": "ISC", + "peer": true + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC", + "peer": true + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/loupe": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.2.tgz", + "integrity": "sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==", + "dev": true, + "license": "MIT" + }, + "node_modules/magic-string": { + "version": "0.30.12", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz", + "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true, "license": "MIT" }, @@ -974,6 +1895,103 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/pathe": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", @@ -1027,6 +2045,68 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prettier": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=4" + } + }, "node_modules/rollup": { "version": "4.24.4", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.24.4.tgz", @@ -1065,6 +2145,31 @@ "fsevents": "~2.3.2" } }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -1096,6 +2201,59 @@ "dev": true, "license": "MIT" }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/synckit": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.2.tgz", + "integrity": "sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@pkgr/core": "^0.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -1140,6 +2298,38 @@ "node": ">=14.0.0" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, "node_modules/vite": { "version": "5.4.10", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.10.tgz", @@ -1288,6 +2478,23 @@ } } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/why-is-node-running": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", @@ -1304,6 +2511,31 @@ "engines": { "node": ">=8" } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/inference/wasm/tests/package.json b/inference/wasm/tests/package.json index 0dff8073a..280517641 100644 --- a/inference/wasm/tests/package.json +++ b/inference/wasm/tests/package.json @@ -5,6 +5,8 @@ "test": "tests" }, "scripts": { + "lint": "eslint .", + "lint:fix": "eslint --fix .", "test": "vitest --run", "test:watch": "vitest" }, @@ -12,6 +14,9 @@ "license": "MPL-2.0", "description": "WASM tests for the inference engine.", "devDependencies": { + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.2.1", + "prettier": "^3.3.3", "vitest": "^2.1.4" } } diff --git a/inference/wasm/tests/stub.test.js b/inference/wasm/tests/stub.test.js deleted file mode 100644 index 3958490d9..000000000 --- a/inference/wasm/tests/stub.test.js +++ /dev/null @@ -1,7 +0,0 @@ -import { describe, it, expect } from 'vitest'; - -describe('Basic Test Suite', () => { - it('should pass a basic test', () => { - expect(1 + 1).toBe(2); - }); -}); diff --git a/inference/wasm/tests/test-cases/shared.mjs b/inference/wasm/tests/test-cases/shared.mjs new file mode 100644 index 000000000..6f5c4a51b --- /dev/null +++ b/inference/wasm/tests/test-cases/shared.mjs @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { it, expect } from "vitest"; +import { TranslationsEngine } from "./engine/translations-engine.mjs"; + +/** + * Runs a translation test, constructing a Translations Engine for the given + * sourceLanguage and targetLanguage, then asserting that the translation of + * the sourceText matches the expectedText. + * + * @param {Object} params - The parameters for the test. + * @param {string} params.sourceLanguage - The source language code. + * @param {string} params.targetLanguage - The target language code. + * @param {string} params.sourceText - The text to translate. + * @param {string} params.expectedText - The expected translated text. + * @param {boolean} params.isHTML - Whether the text to translate contains HTML tags. + */ +export function runTranslationTest({ + sourceLanguage, + targetLanguage, + sourceText, + expectedText, + isHTML = false, +}) { + it(`(${sourceLanguage} -> ${targetLanguage}): Translate "${sourceText}"`, async () => { + const translator = new TranslationsEngine(sourceLanguage, targetLanguage); + + const translatedText = await translator.translate(sourceText, isHTML); + + expect(translatedText).toBe(expectedText); + + translator.terminate(); + }); +} diff --git a/inference/wasm/tests/test-cases/translate-html-no-pivot.test.mjs b/inference/wasm/tests/test-cases/translate-html-no-pivot.test.mjs new file mode 100644 index 000000000..434642987 --- /dev/null +++ b/inference/wasm/tests/test-cases/translate-html-no-pivot.test.mjs @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe } from "vitest"; +import { runTranslationTest } from "./test-cases/shared.mjs"; + +/** + * This file tests the WASM bindings for non-pivot translation requests + * that contain HTML tags within the source text. + */ + +const testCases = [ + { + sourceLanguage: "es", + targetLanguage: "en", + sourceText: "El perro azul.", + expectedText: "The blue dog.", + isHTML: true, + }, + { + sourceLanguage: "en", + targetLanguage: "es", + sourceText: "The blue dog.", + expectedText: "El perro azul.", + isHTML: true, + }, + { + sourceLanguage: "fr", + targetLanguage: "en", + sourceText: "Le chien bleu.", + expectedText: "The blue dog.", + isHTML: true, + }, + { + sourceLanguage: "en", + targetLanguage: "fr", + sourceText: "The blue dog.", + expectedText: "Le chien bleu.", + isHTML: true, + }, +]; + +describe("HTML Non-Pivot Translations", () => { + testCases.forEach(runTranslationTest); +}); diff --git a/inference/wasm/tests/test-cases/translate-html-with-pivot.test.mjs b/inference/wasm/tests/test-cases/translate-html-with-pivot.test.mjs new file mode 100644 index 000000000..66462bd2c --- /dev/null +++ b/inference/wasm/tests/test-cases/translate-html-with-pivot.test.mjs @@ -0,0 +1,32 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe } from "vitest"; +import { runTranslationTest } from "./test-cases/shared.mjs"; + +/** + * This file tests the WASM bindings for pivot translation requests + * that contain HTML tags within the source text. + */ + +const testCases = [ + { + sourceLanguage: "es", + targetLanguage: "fr", + sourceText: "El perro azul.", + expectedText: "Le chien bleu.", + isHTML: true, + }, + { + sourceLanguage: "fr", + targetLanguage: "es", + sourceText: "Le chien bleu.", + expectedText: "El perro azul.", + isHTML: true, + }, +]; + +describe("HTML Pivot Translations", () => { + testCases.forEach(runTranslationTest); +}); diff --git a/inference/wasm/tests/test-cases/translate-plain-text-no-pivot.test.mjs b/inference/wasm/tests/test-cases/translate-plain-text-no-pivot.test.mjs new file mode 100644 index 000000000..cc39a228d --- /dev/null +++ b/inference/wasm/tests/test-cases/translate-plain-text-no-pivot.test.mjs @@ -0,0 +1,42 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe } from "vitest"; +import { runTranslationTest } from "./test-cases/shared.mjs"; + +/** + * This file tests the WASM bindings for non-pivot translation requests + * that contain only plain text without HTML tags. + */ + +const testCases = [ + { + sourceLanguage: "es", + targetLanguage: "en", + sourceText: "Hola mundo", + expectedText: "Hello world", + }, + { + sourceLanguage: "en", + targetLanguage: "es", + sourceText: "Hello world", + expectedText: "Hola mundo", + }, + { + sourceLanguage: "fr", + targetLanguage: "en", + sourceText: "Bonjour le monde", + expectedText: "Hello world", + }, + { + sourceLanguage: "en", + targetLanguage: "fr", + sourceText: "Hello world", + expectedText: "Bonjour le monde", + }, +]; + +describe("Plain-Text Non-Pivot Translations", () => { + testCases.forEach(runTranslationTest); +}); diff --git a/inference/wasm/tests/test-cases/translate-plain-text-with-pivot.test.mjs b/inference/wasm/tests/test-cases/translate-plain-text-with-pivot.test.mjs new file mode 100644 index 000000000..8ff148952 --- /dev/null +++ b/inference/wasm/tests/test-cases/translate-plain-text-with-pivot.test.mjs @@ -0,0 +1,32 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe } from "vitest"; +import { runTranslationTest } from "./test-cases/shared.mjs"; + +/** + * This file tests the WASM bindings for pivot translation requests + * that contain only plain text without HTML tags. + */ + +const testCases = [ + { + sourceLanguage: "es", + targetLanguage: "fr", + sourceText: "El perro azul.", + expectedText: "Le chien bleu.", + isHTML: true, + }, + { + sourceLanguage: "fr", + targetLanguage: "es", + sourceText: "Le chien bleu.", + expectedText: "El perro azul.", + isHTML: true, + }, +]; + +describe("Plain-Text Pivot Translations", () => { + testCases.forEach(runTranslationTest); +}); diff --git a/inference/wasm/tests/vitest.config.mjs b/inference/wasm/tests/vitest.config.mjs new file mode 100644 index 000000000..4142755f8 --- /dev/null +++ b/inference/wasm/tests/vitest.config.mjs @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + reporters: ["default"], + }, +}); diff --git a/taskcluster/docker/base/Dockerfile b/taskcluster/docker/base/Dockerfile index 4a333efe3..e16718b19 100644 --- a/taskcluster/docker/base/Dockerfile +++ b/taskcluster/docker/base/Dockerfile @@ -9,6 +9,8 @@ RUN mkdir -p /builds && \ mkdir /builds/worker/artifacts && \ chown worker:worker /builds/worker/artifacts +COPY known_hosts /etc/ssh/ssh_known_hosts + WORKDIR /builds/worker/ #---------------------------------------------------------------------------------------------------------------------- @@ -28,6 +30,7 @@ RUN apt-get update -qq \ python3-yaml \ locales \ git \ + git-lfs \ tmux \ htop \ vim \ @@ -46,7 +49,7 @@ RUN pip install zstandard # %include-run-task # Allow scripts to detect if they are running in docker -ENV IS_DOCKER 1 +ENV IS_DOCKER=1 ENV SHELL=/bin/bash \ HOME=/builds/worker \ diff --git a/taskcluster/docker/base/known_hosts b/taskcluster/docker/base/known_hosts new file mode 100644 index 000000000..10ef85a09 --- /dev/null +++ b/taskcluster/docker/base/known_hosts @@ -0,0 +1,3 @@ +github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= +github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= +github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl diff --git a/taskcluster/kinds/tests/kind.yml b/taskcluster/kinds/tests/kind.yml index f55ddbad0..77ccb4fcf 100644 --- a/taskcluster/kinds/tests/kind.yml +++ b/taskcluster/kinds/tests/kind.yml @@ -63,7 +63,21 @@ tasks: # make dry-run && # make test-dry-run - black: + lint-eslint: + # Runs the eslint linter, which lints JavaScript files. + worker-type: b-cpu + worker: + max-run-time: 3600 + docker-image: {in-tree: inference} + run: + command: + - bash + - -c + - >- + task lint-eslint + run-on-tasks-for: ["github-push", "github-pull-request"] + + lint-black: # Run python's black formatter, which formats python files. worker-type: b-cpu worker: @@ -77,7 +91,7 @@ tasks: task lint-black run-on-tasks-for: ["github-push", "github-pull-request"] - lint: + lint-ruff: # Run ruff, a python linter. worker-type: b-cpu worker: diff --git a/utils/tasks/docker-build.sh b/utils/tasks/docker-build.sh index c71857434..06e7fc3bc 100755 --- a/utils/tasks/docker-build.sh +++ b/utils/tasks/docker-build.sh @@ -5,16 +5,19 @@ set -x source utils/tasks/docker-setup.sh +DOCKER_BASE_PATH=taskcluster/docker/base +DOCKER_TEST_PATH=taskcluster/docker/test + docker build \ - --file taskcluster/docker/base/Dockerfile \ - --tag translations-base . + --file "$DOCKER_BASE_PATH/Dockerfile" \ + --tag translations-base $DOCKER_BASE_PATH docker build \ --build-arg DOCKER_IMAGE_PARENT=translations-base \ - --file taskcluster/docker/test/Dockerfile \ - --tag translations-test . + --file "$DOCKER_TEST_PATH/Dockerfile" \ + --tag translations-test $DOCKER_TEST_PATH docker build \ --build-arg DOCKER_IMAGE_PARENT=translations-test \ - --file docker/Dockerfile \ + --file "docker/Dockerfile" \ --tag translations-local .