From 154cac8bd5817c0e95fcb45f1e6e5784776eae90 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Dec 2023 14:42:00 +0700 Subject: [PATCH 01/10] chore(web): drops unused index.js artifact from lm-worker --- common/web/lm-worker/build-bundler.js | 6 +++--- common/web/lm-worker/build-polyfill-concatenator.js | 2 +- common/web/lm-worker/build.sh | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/common/web/lm-worker/build-bundler.js b/common/web/lm-worker/build-bundler.js index 0066c05c296..8ce9108a9f8 100644 --- a/common/web/lm-worker/build-bundler.js +++ b/common/web/lm-worker/build-bundler.js @@ -6,7 +6,7 @@ */ import esbuild from 'esbuild'; -import { bundleObjEntryPoints, esmConfiguration, prepareTslibTreeshaking } from '../es-bundling/build/index.mjs'; +import { bundleObjEntryPoints, iifeConfiguration, prepareTslibTreeshaking } from '../es-bundling/build/index.mjs'; import fs from 'fs'; @@ -31,8 +31,8 @@ if(process.argv.length > 2) { } const embeddedWorkerBuildOptions = await prepareTslibTreeshaking({ - ...esmConfiguration, - ...bundleObjEntryPoints('lib', 'build/obj/index.js', 'build/obj/worker-main.js'), + ...iifeConfiguration, + ...bundleObjEntryPoints('lib', 'build/obj/worker-main.js'), // To be explicit, in comparison to the other build below. Even if our other // configs change their default, we should not minify for THIS build; we'll // have a separate minify pass later, after concatenating our polyfills. diff --git a/common/web/lm-worker/build-polyfill-concatenator.js b/common/web/lm-worker/build-polyfill-concatenator.js index 3b350eb55ea..ed0b39d8f01 100644 --- a/common/web/lm-worker/build-polyfill-concatenator.js +++ b/common/web/lm-worker/build-polyfill-concatenator.js @@ -91,7 +91,7 @@ let sourceFileSet = [ // Needed to support Symbol.iterator, as used by the correction algorithm. // Needed for Android / Chromium browser pre-43. loadPolyfill('src/polyfills/symbol-es6.min.js', 'src/polyfills/symbol-es6.min.js'), - loadCompiledModuleFilePair('build/lib/worker-main.mjs', 'build/lib/worker-main.mjs') + loadCompiledModuleFilePair('build/lib/worker-main.js', 'build/lib/worker-main.js') ]; let fullWorkerConcatenation = concatScriptsAndSourcemaps(sourceFileSet, "worker-main.polyfilled.js", separatorFile); diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index fdb4db75f45..949bc122f0e 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -50,7 +50,6 @@ function do_build() { fi # Declaration bundling. - tsc --emitDeclarationOnly --outFile ./build/lib/index.d.ts tsc --emitDeclarationOnly --outFile ./build/lib/worker-main.d.ts echo "Preparing the polyfills + worker for script-embedding" From 0653398ef2303d796e9778bac6727fdb593e9391 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Dec 2023 15:04:52 +0700 Subject: [PATCH 02/10] chore(web): mild lm-worker bundling simplification --- common/web/lm-worker/build-wrap-and-minify.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/common/web/lm-worker/build-wrap-and-minify.js b/common/web/lm-worker/build-wrap-and-minify.js index 07032e539b2..916d33b29b6 100644 --- a/common/web/lm-worker/build-wrap-and-minify.js +++ b/common/web/lm-worker/build-wrap-and-minify.js @@ -44,10 +44,7 @@ if(MINIFY) { sourcemapJSON = convertSourcemap.fromJSON(fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js.map`)).toObject(); -const workerConcatenation = { - script: fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js`), - sourcemapJSON: sourcemapJSON -} +const script = fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js`); // While it IS possible to do partial sourcemaps (without the sources, but with everything else) within the worker... // the resulting sourcemaps are -surprisingly- large - larger than the code itself! @@ -60,7 +57,7 @@ console.log(`Wrapping + generating final output: ${MINIFY ? 'minified' : 'unmini // Now, to build the wrapper... // First, let's build the encoded sourcemap. -const encodedSrcMap = convertSourcemap.fromObject(workerConcatenation.sourcemapJSON).toBase64(); +const encodedSrcMap = convertSourcemap.fromObject(sourcemapJSON).toBase64(); const srcMapString = `//# sourceMappingURL=data:application/json;charset=utf-8;base64,${encodedSrcMap}`; /* @@ -68,7 +65,7 @@ const srcMapString = `//# sourceMappingURL=data:application/json;charset=utf-8;b * but my attempts to do so end up triggering errors when loading. */ -let rawScript = workerConcatenation.script.toString(); +let rawScript = script.toString(); // Two layers of encoding: one for the raw source (parsed by the JS engine), // one to 'unwrap' it from a string _within_ that source. let jsonEncoded = JSON.stringify(rawScript); From ad56b9ea53ae121f96117178370e31b6592bd31b Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Dec 2023 15:32:21 +0700 Subject: [PATCH 03/10] chore(web): centralizes lm-worker build polyfill handling --- .../lm-worker/build-polyfill-concatenator.js | 16 ++++++++++++- common/web/lm-worker/build-wrap-and-minify.js | 23 ++++--------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/common/web/lm-worker/build-polyfill-concatenator.js b/common/web/lm-worker/build-polyfill-concatenator.js index ed0b39d8f01..8cb4c99ca8e 100644 --- a/common/web/lm-worker/build-polyfill-concatenator.js +++ b/common/web/lm-worker/build-polyfill-concatenator.js @@ -4,6 +4,8 @@ import SourceMapCombiner from 'combine-source-map'; import convertSourceMap from 'convert-source-map'; // Transforms sourcemaps among various common formats. // Base64, stringified-JSON, end-of-file comment... +import esbuild from 'esbuild'; + let loadPolyfill = function(scriptFile, sourceMapFile) { // May want to retool the pathing somewhat! return { @@ -114,4 +116,16 @@ fullWorkerConcatenation.script += `//# sourceMappingURL=${fullWorkerConcatenatio fs.writeFileSync(`build/lib/${fullWorkerConcatenation.scriptFilename}`, fullWorkerConcatenation.script); if(fullWorkerConcatenation.sourcemapJSON) { fs.writeFileSync(`build/lib/${fullWorkerConcatenation.scriptFilename}.map`, convertSourceMap.fromObject(fullWorkerConcatenation.sourcemapJSON).toJSON()); -} \ No newline at end of file +} + +await esbuild.build({ + entryPoints: [`build/lib/worker-main.polyfilled.js`], + sourcemap: 'external', + sourcesContent: true, + minify: true, + // Do NOT enable - will break under Android 5.0 / Chrome 35 environments, likely through Chrome 42. + // https://caniuse.com/mdn-javascript_builtins_function_name_configurable_true + keepNames: false, + target: 'es5', + outfile: `build/lib/worker-main.polyfilled.min.js` +}); \ No newline at end of file diff --git a/common/web/lm-worker/build-wrap-and-minify.js b/common/web/lm-worker/build-wrap-and-minify.js index 916d33b29b6..25128098588 100644 --- a/common/web/lm-worker/build-wrap-and-minify.js +++ b/common/web/lm-worker/build-wrap-and-minify.js @@ -1,5 +1,4 @@ import fs from 'fs'; -import esbuild from 'esbuild'; import convertSourcemap from 'convert-source-map'; // Transforms sourcemaps among various common formats. // Base64, stringified-JSON, end-of-file comment... @@ -26,23 +25,11 @@ if(process.argv.length > 2) { } } -let sourcemapJSON = ''; - -if(MINIFY) { - await esbuild.build({ - entryPoints: [`build/lib/worker-main.polyfilled.js`], - sourcemap: 'external', - sourcesContent: DEBUG, - minify: true, - // Do NOT enable - will break under Android 5.0 / Chrome 35 environments, likely through Chrome 42. - // https://caniuse.com/mdn-javascript_builtins_function_name_configurable_true - keepNames: false, - target: 'es5', - outfile: `build/lib/worker-main.polyfilled.min.js` - }); -} +let sourcemapJSON = convertSourcemap.fromJSON(fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js.map`)).toObject(); -sourcemapJSON = convertSourcemap.fromJSON(fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js.map`)).toObject(); +if(!DEBUG) { + sourcemapJSON.sourcesContent = []; +} const script = fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js`); @@ -76,7 +63,7 @@ let wrapper = ` export var LMLayerWorkerCode = ${jsonEncoded}; -${MINIFY && "// Sourcemaps have been omitted for this release build."} +${MINIFY && "// Sourcemaps have been omitted for this release build." || ''} export var LMLayerWorkerSourcemapComment = "${DEBUG ? srcMapString : ''}"; // --END:LMLayerWorkerCode From 2abf2c0142a9cc611cc838ac51b0a60571baea47 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Dec 2023 15:46:48 +0700 Subject: [PATCH 04/10] change(web): lm-worker wrapper generalization prep --- ...ld-wrap-and-minify.js => build-wrapper.js} | 24 +++++++++++-------- common/web/lm-worker/build.sh | 4 ++-- 2 files changed, 16 insertions(+), 12 deletions(-) rename common/web/lm-worker/{build-wrap-and-minify.js => build-wrapper.js} (71%) diff --git a/common/web/lm-worker/build-wrap-and-minify.js b/common/web/lm-worker/build-wrapper.js similarity index 71% rename from common/web/lm-worker/build-wrap-and-minify.js rename to common/web/lm-worker/build-wrapper.js index 25128098588..0a6dbe151d5 100644 --- a/common/web/lm-worker/build-wrap-and-minify.js +++ b/common/web/lm-worker/build-wrapper.js @@ -25,13 +25,10 @@ if(process.argv.length > 2) { } } -let sourcemapJSON = convertSourcemap.fromJSON(fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js.map`)).toObject(); +const sourceFile = `build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js`; +const destFile = `build/lib/worker-main.wrapped${MINIFY ? '.min' : ''}.js`; -if(!DEBUG) { - sourcemapJSON.sourcesContent = []; -} - -const script = fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js`); +const script = fs.readFileSync(sourceFile); // While it IS possible to do partial sourcemaps (without the sources, but with everything else) within the worker... // the resulting sourcemaps are -surprisingly- large - larger than the code itself! @@ -44,8 +41,15 @@ console.log(`Wrapping + generating final output: ${MINIFY ? 'minified' : 'unmini // Now, to build the wrapper... // First, let's build the encoded sourcemap. -const encodedSrcMap = convertSourcemap.fromObject(sourcemapJSON).toBase64(); -const srcMapString = `//# sourceMappingURL=data:application/json;charset=utf-8;base64,${encodedSrcMap}`; + +// Wrapped in a function so we can leverage `const` with the result. +function buildSrcMapString() { + const sourcemapJSON = convertSourcemap.fromJSON(fs.readFileSync(`${sourceFile}.map`)).toObject(); + const encodedSrcMap = convertSourcemap.fromObject(sourcemapJSON).toBase64(); + return `//# sourceMappingURL=data:application/json;charset=utf-8;base64,${encodedSrcMap}`; +} + +const srcMapString = DEBUG ? buildSrcMapString() : ""; /* * It'd be nice to do a 'partial' encodeURIComponent that only gets the important bits... @@ -64,9 +68,9 @@ let wrapper = ` export var LMLayerWorkerCode = ${jsonEncoded}; ${MINIFY && "// Sourcemaps have been omitted for this release build." || ''} -export var LMLayerWorkerSourcemapComment = "${DEBUG ? srcMapString : ''}"; +export var LMLayerWorkerSourcemapComment = "${srcMapString}"; // --END:LMLayerWorkerCode `; -fs.writeFileSync(`build/lib/worker-main.wrapped${MINIFY ? '.min' : ''}.js`, wrapper); \ No newline at end of file +fs.writeFileSync(destFile, wrapper); \ No newline at end of file diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 949bc122f0e..51206d9e1b9 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -55,8 +55,8 @@ function do_build() { echo "Preparing the polyfills + worker for script-embedding" node build-polyfill-concatenator.js - node build-wrap-and-minify.js --debug - node build-wrap-and-minify.js --minify + node build-wrapper.js --debug + node build-wrapper.js --minify } function do_test() { From 0e20cc3552ec9d5372e44096061f1ef36fd237a5 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Dec 2023 15:49:57 +0700 Subject: [PATCH 05/10] chore(web): a bit more lm-worker wrapping polish --- common/web/lm-worker/build-wrapper.js | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/common/web/lm-worker/build-wrapper.js b/common/web/lm-worker/build-wrapper.js index 0a6dbe151d5..56c539a83c4 100644 --- a/common/web/lm-worker/build-wrapper.js +++ b/common/web/lm-worker/build-wrapper.js @@ -30,14 +30,6 @@ const destFile = `build/lib/worker-main.wrapped${MINIFY ? '.min' : ''}.js`; const script = fs.readFileSync(sourceFile); -// While it IS possible to do partial sourcemaps (without the sources, but with everything else) within the worker... -// the resulting sourcemaps are -surprisingly- large - larger than the code itself! -// -// So... we don't use that strategy here. - -console.log(); -console.log(`Wrapping + generating final output: ${MINIFY ? 'minified' : 'unminified'} + ${DEBUG ? 'full sourcemaps' : 'reduced sourcemaps'}`); - // Now, to build the wrapper... // First, let's build the encoded sourcemap. @@ -49,6 +41,8 @@ function buildSrcMapString() { return `//# sourceMappingURL=data:application/json;charset=utf-8;base64,${encodedSrcMap}`; } +// While it IS possible to do partial sourcemaps (without the sources, but with everything else) within the worker... +// the resulting sourcemaps are -surprisingly- large - larger than the code itself! const srcMapString = DEBUG ? buildSrcMapString() : ""; /* @@ -67,7 +61,7 @@ let wrapper = ` export var LMLayerWorkerCode = ${jsonEncoded}; -${MINIFY && "// Sourcemaps have been omitted for this release build." || ''} +${!DEBUG && "// Sourcemaps have been omitted for this release build." || ''} export var LMLayerWorkerSourcemapComment = "${srcMapString}"; // --END:LMLayerWorkerCode From 9f2816ca9989c9533acf651503d2e1357dd0d290 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 15 Dec 2023 09:47:50 +0700 Subject: [PATCH 06/10] change(web): lm-worker wrapper script generalization --- common/web/lm-worker/build-wrapper.js | 65 ++++++++++++++++++++------- common/web/lm-worker/build.sh | 4 +- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/common/web/lm-worker/build-wrapper.js b/common/web/lm-worker/build-wrapper.js index 56c539a83c4..ebfe4a11970 100644 --- a/common/web/lm-worker/build-wrapper.js +++ b/common/web/lm-worker/build-wrapper.js @@ -3,36 +3,71 @@ import fs from 'fs'; import convertSourcemap from 'convert-source-map'; // Transforms sourcemaps among various common formats. // Base64, stringified-JSON, end-of-file comment... -let DEBUG = false; -let MINIFY = false; +let INCLUDE_SRCMAPS = false; + +let sourceFromArgs; +let destFromArgs; + +function doHelp(errCode) { + console.log(` +Summary: + Creates a "wrapped" version of the lm-worker for compilation into and inclusion as part + of a different JS bundle for later use as the core of a predictive-text WebWorker. + +Usage: + node build-wrapper.js [options...] + +Parameters: + : Fully-bundled and compiled JS file to be wrapped. + +Options: + --help Shows this script's documentation + --out= Specifies the destination path for the wrapped output. + + If missing, the output will be placed next to the source and given + the same path, but with '.wrapped.js' replacing the original '.js' + extension. + --sourceMap Includes the script's original sourcemaps within the wrapped output +` ); + process.exit(errCode || 0); +} if(process.argv.length > 2) { for(let i = 2; i < process.argv.length; i++) { const arg = process.argv[i]; switch(arg) { - case '--debug': - DEBUG = true; + case '--help': + doHelp(); + break; + case '--sourceMap': // bc TS uses this exact flag. esbuild... uses sourcemap (in the JS config) + INCLUDE_SRCMAPS = true; break; - case '--minify': - MINIFY = true; + case '--out': + destFromArgs = process.argv[++i]; break; - // May add other options if desired in the future. default: - console.error("Invalid command-line option set for script; only --debug and --minify are permitted."); - process.exit(1); + if(!sourceFromArgs) { + sourceFromArgs = arg; + } else { + doHelp(1); + } } } } -const sourceFile = `build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js`; -const destFile = `build/lib/worker-main.wrapped${MINIFY ? '.min' : ''}.js`; +if(!sourceFromArgs || sourceFromArgs.substring(sourceFromArgs.length - 3) != '.js') { + console.error("No input file has been specified; aborting."); + console.log(); + doHelp(1); +} -const script = fs.readFileSync(sourceFile); +const sourceFile = sourceFromArgs; +const destFile = destFromArgs || sourceFromArgs.substring(0, sourceFromArgs.length - 3) + '.wrapped.js'; // Now, to build the wrapper... -// First, let's build the encoded sourcemap. +const script = fs.readFileSync(sourceFile); // Wrapped in a function so we can leverage `const` with the result. function buildSrcMapString() { @@ -43,7 +78,7 @@ function buildSrcMapString() { // While it IS possible to do partial sourcemaps (without the sources, but with everything else) within the worker... // the resulting sourcemaps are -surprisingly- large - larger than the code itself! -const srcMapString = DEBUG ? buildSrcMapString() : ""; +const srcMapString = INCLUDE_SRCMAPS ? buildSrcMapString() : ""; /* * It'd be nice to do a 'partial' encodeURIComponent that only gets the important bits... @@ -61,7 +96,7 @@ let wrapper = ` export var LMLayerWorkerCode = ${jsonEncoded}; -${!DEBUG && "// Sourcemaps have been omitted for this release build." || ''} +${!INCLUDE_SRCMAPS && "// Sourcemaps have been omitted for this release build." || ''} export var LMLayerWorkerSourcemapComment = "${srcMapString}"; // --END:LMLayerWorkerCode diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 51206d9e1b9..865b8bc1223 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -55,8 +55,8 @@ function do_build() { echo "Preparing the polyfills + worker for script-embedding" node build-polyfill-concatenator.js - node build-wrapper.js --debug - node build-wrapper.js --minify + node build-wrapper.js build/lib/worker-main.polyfilled.js --out build/lib/worker-main.wrapped.js --sourceMap + node build-wrapper.js build/lib/worker-main.polyfilled.min.js --out build/lib/worker-main.wrapped.min.js } function do_test() { From 7392fdd572cdfcc0a5697b5080c00adbdbb87598 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 15 Dec 2023 11:04:45 +0700 Subject: [PATCH 07/10] refactor(web): lm-worker polyfiller rework --- ...ll-concatenator.js => build-polyfiller.js} | 72 +++++++++++++++++-- common/web/lm-worker/build.sh | 2 +- 2 files changed, 67 insertions(+), 7 deletions(-) rename common/web/lm-worker/{build-polyfill-concatenator.js => build-polyfiller.js} (68%) diff --git a/common/web/lm-worker/build-polyfill-concatenator.js b/common/web/lm-worker/build-polyfiller.js similarity index 68% rename from common/web/lm-worker/build-polyfill-concatenator.js rename to common/web/lm-worker/build-polyfiller.js index 8cb4c99ca8e..676c412cfe6 100644 --- a/common/web/lm-worker/build-polyfill-concatenator.js +++ b/common/web/lm-worker/build-polyfiller.js @@ -6,6 +6,66 @@ import convertSourceMap from 'convert-source-map'; // Transforms sourcemaps amon import esbuild from 'esbuild'; +let sourceFromArgs; +let destFromArgs; + +function doHelp(errCode) { + console.log(` +Summary: + Creates a polyfilled version of the lm-worker to ensure compatibility with the needs of + Keyman for Android when run on Android 5.0 / API 21 without an updated Chrome webview. + +Usage: + node build-polyfiller.js [options...] + +Parameters: + : Fully-bundled and compiled JS file to be polyfilled. + +Options: + --help Shows this script's documentation + --out= Specifies the destination path for the polyfilled output. + + If missing, the output will be placed next to the source and given + the same path, but with '.polyfilled.js' replacing the original '.js' + extension. + + Either way, a minified version will also be output, using the same + path but with the final '.js' replaced by '.min.js'. +` ); + process.exit(errCode || 0); +} + +if(process.argv.length > 2) { + for(let i = 2; i < process.argv.length; i++) { + const arg = process.argv[i]; + + switch(arg) { + case '--help': + doHelp(); + break; + case '--out': + destFromArgs = process.argv[++i]; + break; + default: + if(!sourceFromArgs) { + sourceFromArgs = arg; + } else { + doHelp(1); + } + } + } +} + +if(!sourceFromArgs || sourceFromArgs.substring(sourceFromArgs.length - 3) != '.js') { + console.error("No input file has been specified; aborting."); + console.log(); + doHelp(1); +} + +const sourceFile = sourceFromArgs; +const destFile = destFromArgs || sourceFromArgs.substring(0, sourceFromArgs.length - 3) + '.polyfilled.js'; +const minDestFile = destFile.substring(0, destFile.length - 3) + '.min.js'; + let loadPolyfill = function(scriptFile, sourceMapFile) { // May want to retool the pathing somewhat! return { @@ -93,10 +153,10 @@ let sourceFileSet = [ // Needed to support Symbol.iterator, as used by the correction algorithm. // Needed for Android / Chromium browser pre-43. loadPolyfill('src/polyfills/symbol-es6.min.js', 'src/polyfills/symbol-es6.min.js'), - loadCompiledModuleFilePair('build/lib/worker-main.js', 'build/lib/worker-main.js') + loadCompiledModuleFilePair(sourceFile, sourceFile) ]; -let fullWorkerConcatenation = concatScriptsAndSourcemaps(sourceFileSet, "worker-main.polyfilled.js", separatorFile); +let fullWorkerConcatenation = concatScriptsAndSourcemaps(sourceFileSet, destFile, separatorFile); // New stage: cleaning the sourcemaps @@ -113,13 +173,13 @@ console.log("Pass 2: Output intermediate state and perform minification"); fullWorkerConcatenation.script = fullWorkerConcatenation.script.substring(0, fullWorkerConcatenation.script.lastIndexOf('//# sourceMappingURL')); fullWorkerConcatenation.script += `//# sourceMappingURL=${fullWorkerConcatenation.scriptFilename}.map`; -fs.writeFileSync(`build/lib/${fullWorkerConcatenation.scriptFilename}`, fullWorkerConcatenation.script); +fs.writeFileSync(`${fullWorkerConcatenation.scriptFilename}`, fullWorkerConcatenation.script); if(fullWorkerConcatenation.sourcemapJSON) { - fs.writeFileSync(`build/lib/${fullWorkerConcatenation.scriptFilename}.map`, convertSourceMap.fromObject(fullWorkerConcatenation.sourcemapJSON).toJSON()); + fs.writeFileSync(`${fullWorkerConcatenation.scriptFilename}.map`, convertSourceMap.fromObject(fullWorkerConcatenation.sourcemapJSON).toJSON()); } await esbuild.build({ - entryPoints: [`build/lib/worker-main.polyfilled.js`], + entryPoints: [destFile], sourcemap: 'external', sourcesContent: true, minify: true, @@ -127,5 +187,5 @@ await esbuild.build({ // https://caniuse.com/mdn-javascript_builtins_function_name_configurable_true keepNames: false, target: 'es5', - outfile: `build/lib/worker-main.polyfilled.min.js` + outfile: minDestFile }); \ No newline at end of file diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 865b8bc1223..98070113f03 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -53,7 +53,7 @@ function do_build() { tsc --emitDeclarationOnly --outFile ./build/lib/worker-main.d.ts echo "Preparing the polyfills + worker for script-embedding" - node build-polyfill-concatenator.js + node build-polyfiller.js build/lib/worker-main.js --out build/lib/worker-main.polyfilled.js node build-wrapper.js build/lib/worker-main.polyfilled.js --out build/lib/worker-main.wrapped.js --sourceMap node build-wrapper.js build/lib/worker-main.polyfilled.min.js --out build/lib/worker-main.wrapped.min.js From 17605a405ec5bff0e76dd48c5e1f92ae36600089 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 15 Dec 2023 11:34:20 +0700 Subject: [PATCH 08/10] change(web): lm-worker build dir constants --- common/web/lm-worker/build-bundler.js | 2 +- common/web/lm-worker/build.sh | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/common/web/lm-worker/build-bundler.js b/common/web/lm-worker/build-bundler.js index 8ce9108a9f8..3319aaa2d06 100644 --- a/common/web/lm-worker/build-bundler.js +++ b/common/web/lm-worker/build-bundler.js @@ -32,7 +32,7 @@ if(process.argv.length > 2) { const embeddedWorkerBuildOptions = await prepareTslibTreeshaking({ ...iifeConfiguration, - ...bundleObjEntryPoints('lib', 'build/obj/worker-main.js'), + ...bundleObjEntryPoints('intermediate', 'build/obj/worker-main.js'), // To be explicit, in comparison to the other build below. Even if our other // configs change their default, we should not minify for THIS build; we'll // have a separate minify pass later, after concatenating our polyfills. diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 98070113f03..48ba1f908d0 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -23,6 +23,9 @@ cd "$THIS_SCRIPT_PATH" WORKER_OUTPUT=build/obj WORKER_OUTPUT_FILENAME=build/lib/worker-main.js +INTERMEDIATE=./build/intermediate +LIB=./build/lib + ################################ Main script ################################ builder_describe \ @@ -50,13 +53,17 @@ function do_build() { fi # Declaration bundling. - tsc --emitDeclarationOnly --outFile ./build/lib/worker-main.d.ts + tsc --emitDeclarationOnly --outFile $INTERMEDIATE/worker-main.d.ts echo "Preparing the polyfills + worker for script-embedding" - node build-polyfiller.js build/lib/worker-main.js --out build/lib/worker-main.polyfilled.js - - node build-wrapper.js build/lib/worker-main.polyfilled.js --out build/lib/worker-main.wrapped.js --sourceMap - node build-wrapper.js build/lib/worker-main.polyfilled.min.js --out build/lib/worker-main.wrapped.min.js + node build-polyfiller.js $INTERMEDIATE/worker-main.js \ + --out $INTERMEDIATE/worker-main.polyfilled.js + + mkdir -p $LIB + node build-wrapper.js $INTERMEDIATE/worker-main.polyfilled.js \ + --out $LIB/worker-main.wrapped.js --sourceMap + node build-wrapper.js $INTERMEDIATE/worker-main.polyfilled.min.js \ + --out $LIB/worker-main.wrapped.min.js } function do_test() { From 1e6f9765a9171afacece208c50640b241dd6b2ec Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 20 Dec 2023 09:45:23 +0700 Subject: [PATCH 09/10] chore(web): adds low-arg warning to scripts --- common/web/lm-worker/build-polyfiller.js | 3 +++ common/web/lm-worker/build-wrapper.js | 3 +++ 2 files changed, 6 insertions(+) diff --git a/common/web/lm-worker/build-polyfiller.js b/common/web/lm-worker/build-polyfiller.js index 676c412cfe6..9e8ed17fb25 100644 --- a/common/web/lm-worker/build-polyfiller.js +++ b/common/web/lm-worker/build-polyfiller.js @@ -54,6 +54,9 @@ if(process.argv.length > 2) { } } } +} else { + // Display help + abort. + doHelp(1); } if(!sourceFromArgs || sourceFromArgs.substring(sourceFromArgs.length - 3) != '.js') { diff --git a/common/web/lm-worker/build-wrapper.js b/common/web/lm-worker/build-wrapper.js index ebfe4a11970..3565a84332a 100644 --- a/common/web/lm-worker/build-wrapper.js +++ b/common/web/lm-worker/build-wrapper.js @@ -54,6 +54,9 @@ if(process.argv.length > 2) { } } } +} else { + // Display help + abort. + doHelp(1); } if(!sourceFromArgs || sourceFromArgs.substring(sourceFromArgs.length - 3) != '.js') { From 6c38e45e9118590bee45ae6f3292e2c3de0bdb85 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 8 Jan 2024 11:57:52 +0700 Subject: [PATCH 10/10] chore(web): Apply suggestions from code review Co-authored-by: Marc Durdin --- common/web/lm-worker/build-polyfiller.js | 19 ++++++++++--------- common/web/lm-worker/build-wrapper.js | 20 ++++++++++++-------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/common/web/lm-worker/build-polyfiller.js b/common/web/lm-worker/build-polyfiller.js index 9e8ed17fb25..59cf558e24d 100644 --- a/common/web/lm-worker/build-polyfiller.js +++ b/common/web/lm-worker/build-polyfiller.js @@ -9,7 +9,10 @@ import esbuild from 'esbuild'; let sourceFromArgs; let destFromArgs; -function doHelp(errCode) { +function doHelp(errMessage) { + if(errMessage) { + console.error(errMessage + '\n'); + } console.log(` Summary: Creates a polyfilled version of the lm-worker to ensure compatibility with the needs of @@ -23,7 +26,7 @@ Parameters: Options: --help Shows this script's documentation - --out= Specifies the destination path for the polyfilled output. + --out Specifies the destination path for the polyfilled output. If missing, the output will be placed next to the source and given the same path, but with '.polyfilled.js' replacing the original '.js' @@ -32,7 +35,7 @@ Options: Either way, a minified version will also be output, using the same path but with the final '.js' replaced by '.min.js'. ` ); - process.exit(errCode || 0); + process.exit(errMessage ? 1 : 0); } if(process.argv.length > 2) { @@ -50,19 +53,17 @@ if(process.argv.length > 2) { if(!sourceFromArgs) { sourceFromArgs = arg; } else { - doHelp(1); + doHelp("Input file can only be specified once; aborting"); } } } } else { // Display help + abort. - doHelp(1); + doHelp("Required parameters missing"); } if(!sourceFromArgs || sourceFromArgs.substring(sourceFromArgs.length - 3) != '.js') { - console.error("No input file has been specified; aborting."); - console.log(); - doHelp(1); + doHelp("No input file has been specified; aborting."); } const sourceFile = sourceFromArgs; @@ -176,7 +177,7 @@ console.log("Pass 2: Output intermediate state and perform minification"); fullWorkerConcatenation.script = fullWorkerConcatenation.script.substring(0, fullWorkerConcatenation.script.lastIndexOf('//# sourceMappingURL')); fullWorkerConcatenation.script += `//# sourceMappingURL=${fullWorkerConcatenation.scriptFilename}.map`; -fs.writeFileSync(`${fullWorkerConcatenation.scriptFilename}`, fullWorkerConcatenation.script); +fs.writeFileSync(fullWorkerConcatenation.scriptFilename, fullWorkerConcatenation.script); if(fullWorkerConcatenation.sourcemapJSON) { fs.writeFileSync(`${fullWorkerConcatenation.scriptFilename}.map`, convertSourceMap.fromObject(fullWorkerConcatenation.sourcemapJSON).toJSON()); } diff --git a/common/web/lm-worker/build-wrapper.js b/common/web/lm-worker/build-wrapper.js index 3565a84332a..368d5c28159 100644 --- a/common/web/lm-worker/build-wrapper.js +++ b/common/web/lm-worker/build-wrapper.js @@ -8,7 +8,11 @@ let INCLUDE_SRCMAPS = false; let sourceFromArgs; let destFromArgs; -function doHelp(errCode) { +function doHelp(errMessage) { + if(errMessage) { + console.error(errMessage + '\n'); + } + console.log(` Summary: Creates a "wrapped" version of the lm-worker for compilation into and inclusion as part @@ -22,14 +26,14 @@ Parameters: Options: --help Shows this script's documentation - --out= Specifies the destination path for the wrapped output. + --out Specifies the destination path for the wrapped output. If missing, the output will be placed next to the source and given the same path, but with '.wrapped.js' replacing the original '.js' extension. --sourceMap Includes the script's original sourcemaps within the wrapped output ` ); - process.exit(errCode || 0); + process.exit(errMessage ? 1 : 0); } if(process.argv.length > 2) { @@ -41,6 +45,8 @@ if(process.argv.length > 2) { doHelp(); break; case '--sourceMap': // bc TS uses this exact flag. esbuild... uses sourcemap (in the JS config) + case '--sourcemap': + case '--source-map': INCLUDE_SRCMAPS = true; break; case '--out': @@ -50,19 +56,17 @@ if(process.argv.length > 2) { if(!sourceFromArgs) { sourceFromArgs = arg; } else { - doHelp(1); + doHelp("Input file can only be specified once; aborting"); } } } } else { // Display help + abort. - doHelp(1); + doHelp("Required parameters missing"); } if(!sourceFromArgs || sourceFromArgs.substring(sourceFromArgs.length - 3) != '.js') { - console.error("No input file has been specified; aborting."); - console.log(); - doHelp(1); + doHelp("No input file has been specified; aborting."); } const sourceFile = sourceFromArgs;