change(web): lm-worker bundling + wrapper script rework 📜

common/web/lm-worker/build-bundler.js

@@ -6,7 +6,7 @@
  */
 
 import esbuild from 'esbuild';
-import { bundleObjEntryPoints, esmConfiguration, prepareTslibTreeshaking } from '../es-bundling/build/index.mjs';
+import { bundleObjEntryPoints, iifeConfiguration, prepareTslibTreeshaking } from '../es-bundling/build/index.mjs';
 
 import fs from 'fs';
 
@@ -31,8 +31,8 @@ if(process.argv.length > 2) {
 }
 
 const embeddedWorkerBuildOptions = await prepareTslibTreeshaking({
-  ...esmConfiguration,
-  ...bundleObjEntryPoints('lib', 'build/obj/index.js', 'build/obj/worker-main.js'),
+  ...iifeConfiguration,
+  ...bundleObjEntryPoints('intermediate', 'build/obj/worker-main.js'),
   // To be explicit, in comparison to the other build below.  Even if our other
   // configs change their default, we should not minify for THIS build; we'll
   // have a separate minify pass later, after concatenating our polyfills.

common/web/lm-worker/build-polyfill-concatenator.js → common/web/lm-worker/build-polyfiller.js

@@ -4,6 +4,72 @@ import SourceMapCombiner from 'combine-source-map';
 import convertSourceMap from 'convert-source-map'; // Transforms sourcemaps among various common formats.
                                                    // Base64, stringified-JSON, end-of-file comment...
 
+import esbuild from 'esbuild';
+
+let sourceFromArgs;
+let destFromArgs;
+
+function doHelp(errMessage) {
+  if(errMessage) {
+    console.error(errMessage + '\n');
+  }
+  console.log(`
+Summary:
+  Creates a polyfilled version of the lm-worker to ensure compatibility with the needs of
+  Keyman for Android when run on Android 5.0 / API 21 without an updated Chrome webview.
+
+Usage:
+  node build-polyfiller.js <input-file> [options...]
+
+Parameters:
+  <input-file>:  Fully-bundled and compiled JS file to be polyfilled.
+
+Options:
+  --help            Shows this script's documentation
+  --out <out-file>  Specifies the destination path for the polyfilled output.
+
+                    If missing, the output will be placed next to the source and given
+                    the same path, but with '.polyfilled.js' replacing the original '.js'
+                    extension.
+
+                    Either way, a minified version will also be output, using the same
+                    path but with the final '.js' replaced by '.min.js'.
+` );
+  process.exit(errMessage ? 1 : 0);
+}
+
+if(process.argv.length > 2) {
+  for(let i = 2; i < process.argv.length; i++) {
+    const arg = process.argv[i];
+
+    switch(arg) {
+      case '--help':
+        doHelp();
+        break;
+      case '--out':
+        destFromArgs = process.argv[++i];
+        break;
+      default:
+        if(!sourceFromArgs) {
+          sourceFromArgs = arg;
+        } else {
+          doHelp("Input file can only be specified once; aborting");
+        }
+    }
+  }
+} else {
+  // Display help + abort.
+  doHelp("Required parameters missing");
+}
+
+if(!sourceFromArgs || sourceFromArgs.substring(sourceFromArgs.length - 3) != '.js') {
+  doHelp("No input file has been specified; aborting.");
+}
+
+const sourceFile = sourceFromArgs;
+const destFile = destFromArgs || sourceFromArgs.substring(0, sourceFromArgs.length - 3) + '.polyfilled.js';
+const minDestFile = destFile.substring(0, destFile.length - 3) + '.min.js';
+
 let loadPolyfill = function(scriptFile, sourceMapFile) {
   // May want to retool the pathing somewhat!
   return {
@@ -89,10 +155,10 @@ let sourceFileSet = [
   // Needed to support Symbol.iterator, as used by the correction algorithm.
   // Needed for Android / Chromium browser pre-43.
   loadPolyfill('src/polyfills/symbol-es6.min.js', 'src/polyfills/symbol-es6.min.js'),
-  loadCompiledModuleFilePair('build/lib/worker-main.mjs', 'build/lib/worker-main.mjs')
+  loadCompiledModuleFilePair(sourceFile, sourceFile)
 ];
 
-let fullWorkerConcatenation = concatScriptsAndSourcemaps(sourceFileSet, "worker-main.polyfilled.js", separatorFile);
+let fullWorkerConcatenation = concatScriptsAndSourcemaps(sourceFileSet, destFile, separatorFile);
 
 // New stage:  cleaning the sourcemaps
 
@@ -109,7 +175,19 @@ console.log("Pass 2:  Output intermediate state and perform minification");
 fullWorkerConcatenation.script = fullWorkerConcatenation.script.substring(0, fullWorkerConcatenation.script.lastIndexOf('//# sourceMappingURL'));
 fullWorkerConcatenation.script += `//# sourceMappingURL=${fullWorkerConcatenation.scriptFilename}.map`;
 
-fs.writeFileSync(`build/lib/${fullWorkerConcatenation.scriptFilename}`, fullWorkerConcatenation.script);
+fs.writeFileSync(fullWorkerConcatenation.scriptFilename, fullWorkerConcatenation.script);
 if(fullWorkerConcatenation.sourcemapJSON) {
-  fs.writeFileSync(`build/lib/${fullWorkerConcatenation.scriptFilename}.map`, convertSourceMap.fromObject(fullWorkerConcatenation.sourcemapJSON).toJSON());
-}
+  fs.writeFileSync(`${fullWorkerConcatenation.scriptFilename}.map`, convertSourceMap.fromObject(fullWorkerConcatenation.sourcemapJSON).toJSON());
+}
+
+await esbuild.build({
+  entryPoints: [destFile],
+  sourcemap: 'external',
+  sourcesContent: true,
+  minify: true,
+  // Do NOT enable - will break under Android 5.0 / Chrome 35 environments, likely through Chrome 42.
+  // https://caniuse.com/mdn-javascript_builtins_function_name_configurable_true
+  keepNames: false,
+  target: 'es5',
+  outfile: minDestFile
+});

common/web/lm-worker/build-wrapper.js

@@ -0,0 +1,112 @@
+import fs from 'fs';
+
+import convertSourcemap from 'convert-source-map'; // Transforms sourcemaps among various common formats.
+                                                   // Base64, stringified-JSON, end-of-file comment...
+
+let INCLUDE_SRCMAPS = false;
+
+let sourceFromArgs;
+let destFromArgs;
+
+function doHelp(errMessage) {
+  if(errMessage) {
+    console.error(errMessage + '\n');
+  }
+
+  console.log(`
+Summary:
+  Creates a "wrapped" version of the lm-worker for compilation into and inclusion as part
+  of a different JS bundle for later use as the core of a predictive-text WebWorker.
+
+Usage:
+  node build-wrapper.js <input-file> [options...]
+
+Parameters:
+  <input-file>:  Fully-bundled and compiled JS file to be wrapped.
+
+Options:
+  --help            Shows this script's documentation
+  --out <out-file>  Specifies the destination path for the wrapped output.
+
+                    If missing, the output will be placed next to the source and given
+                    the same path, but with '.wrapped.js' replacing the original '.js'
+                    extension.
+  --sourceMap       Includes the script's original sourcemaps within the wrapped output
+` );
+  process.exit(errMessage ? 1 : 0);
+}
+
+if(process.argv.length > 2) {
+  for(let i = 2; i < process.argv.length; i++) {
+    const arg = process.argv[i];
+
+    switch(arg) {
+      case '--help':
+        doHelp();
+        break;
+      case '--sourceMap':  // bc TS uses this exact flag.  esbuild... uses sourcemap (in the JS config)
+      case '--sourcemap':
+      case '--source-map':
+        INCLUDE_SRCMAPS = true;
+        break;
+      case '--out':
+        destFromArgs = process.argv[++i];
+        break;
+      default:
+        if(!sourceFromArgs) {
+          sourceFromArgs = arg;
+        } else {
+          doHelp("Input file can only be specified once; aborting");
+        }
+    }
+  }
+} else {
+  // Display help + abort.
+  doHelp("Required parameters missing");
+}
+
+if(!sourceFromArgs || sourceFromArgs.substring(sourceFromArgs.length - 3) != '.js') {
+  doHelp("No input file has been specified; aborting.");
+}
+
+const sourceFile = sourceFromArgs;
+const destFile = destFromArgs || sourceFromArgs.substring(0, sourceFromArgs.length - 3) + '.wrapped.js';
+
+// Now, to build the wrapper...
+
+const script = fs.readFileSync(sourceFile);
+
+// Wrapped in a function so we can leverage `const` with the result.
+function buildSrcMapString() {
+  const sourcemapJSON = convertSourcemap.fromJSON(fs.readFileSync(`${sourceFile}.map`)).toObject();
+  const encodedSrcMap = convertSourcemap.fromObject(sourcemapJSON).toBase64();
+  return `//# sourceMappingURL=data:application/json;charset=utf-8;base64,${encodedSrcMap}`;
+}
+
+// While it IS possible to do partial sourcemaps (without the sources, but with everything else) within the worker...
+// the resulting sourcemaps are -surprisingly- large - larger than the code itself!
+const srcMapString = INCLUDE_SRCMAPS ? buildSrcMapString() : "";
+
+/*
+ * It'd be nice to do a 'partial' encodeURIComponent that only gets the important bits...
+ * but my attempts to do so end up triggering errors when loading.
+ */
+
+let rawScript = script.toString();
+// Two layers of encoding:  one for the raw source (parsed by the JS engine),
+// one to 'unwrap' it from a string _within_ that source.
+let jsonEncoded = JSON.stringify(rawScript);
+
+let wrapper = `
+// Autogenerated code.  Do not modify!
+// --START:LMLayerWorkerCode--
+
+export var LMLayerWorkerCode = ${jsonEncoded};
+
+${!INCLUDE_SRCMAPS && "// Sourcemaps have been omitted for this release build." || ''}
+export var LMLayerWorkerSourcemapComment = "${srcMapString}";
+
+// --END:LMLayerWorkerCode
+`;
+
+fs.writeFileSync(destFile, wrapper);

common/web/lm-worker/build.sh

@@ -23,6 +23,9 @@ cd "$THIS_SCRIPT_PATH"
 WORKER_OUTPUT=build/obj
 WORKER_OUTPUT_FILENAME=build/lib/worker-main.js
 
+INTERMEDIATE=./build/intermediate
+LIB=./build/lib
+
 ################################ Main script ################################
 
 builder_describe \
@@ -50,14 +53,17 @@ function do_build() {
   fi
 
   # Declaration bundling.
-  tsc --emitDeclarationOnly --outFile ./build/lib/index.d.ts
-  tsc --emitDeclarationOnly --outFile ./build/lib/worker-main.d.ts
+  tsc --emitDeclarationOnly --outFile $INTERMEDIATE/worker-main.d.ts
 
   echo "Preparing the polyfills + worker for script-embedding"
-  node build-polyfill-concatenator.js
-
-  node build-wrap-and-minify.js --debug
-  node build-wrap-and-minify.js --minify
+  node build-polyfiller.js $INTERMEDIATE/worker-main.js \
+    --out $INTERMEDIATE/worker-main.polyfilled.js
+
+  mkdir -p $LIB
+  node build-wrapper.js $INTERMEDIATE/worker-main.polyfilled.js \
+    --out $LIB/worker-main.wrapped.js     --sourceMap
+  node build-wrapper.js $INTERMEDIATE/worker-main.polyfilled.min.js \
+    --out $LIB/worker-main.wrapped.min.js
 }
 
 function do_test() {