feat(developer): Windows package installer compiler

Replaces kmcomp's package installer build infrastructure. While this is
mostly legacy, package installers are still used in adhoc distribution
scenarios, so we need to continue to support with the new kmc.

The package installer is a Windows self-extracting zip archive. The user
is expected to find setup-redist.exe and keymandesktop.msi themselves,
from the Keyman Developer release files. They are not included with
Keyman Developer, but are available as standalone downloads from
https://downloads.keyman.com/windows/.

`kmc build` has been updated to include two subcommands:
* `kmc build ldml-test-data`
* `kmc build windows-package-installer`

Both of these subcommands are a little long, but the use cases for them
are fairly narrow, so I believe this is okay. `kmc build-test-data` has
been removed, as `kmc build ldml-test-data` replaces it.

common/web/types/src/package/kps-file.ts

@@ -160,8 +160,14 @@ export interface KpsFileStartMenuItems {
 }
 
 export interface KpsFileStrings {
-  //TODO: validate this structure
-  string: string[] | string;
+  string: KpsFileString[] | KpsFileString;
+}
+
+export interface KpsFileString {
+  $: {
+    name: string;
+    value: string;
+  }
 }
 
 export interface KpsFileLanguageExamples {

developer/src/kmc-package/src/compiler/kmp-compiler.ts

@@ -22,8 +22,13 @@ export class KmpCompiler {
   }
 
   public transformKpsToKmpObject(kpsFilename: string): KmpJsonFile.KmpJsonFile {
+    const kps = this.loadKpsFile(kpsFilename);
+    return this.transformKpsFileToKmpObject(kpsFilename, kps);
+  }
+
+  public loadKpsFile(kpsFilename: string): KpsFile.KpsFile {
     // Load the KPS data from XML as JS structured data.
-    const buffer = this.callbacks.loadFile(kpsFilename);
+    const buffer = this.callbacks.loadFile(kpsFilename, false);
     if(!buffer) {
       this.callbacks.reportMessage(CompilerMessages.Error_FileDoesNotExist({filename: kpsFilename}));
       return null;
@@ -41,7 +46,11 @@ export class KmpCompiler {
         return a;
     })();
 
-    let kps: KpsFile.KpsFile = kpsPackage.package;
+    const kps: KpsFile.KpsFile = kpsPackage.package;
+    return kps;
+  }
+
+  public transformKpsFileToKmpObject(kpsFilename: string, kps: KpsFile.KpsFile): KmpJsonFile.KmpJsonFile {
 
     //
     // To convert to kmp.json, we need to:

developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts

@@ -0,0 +1,145 @@
+/**
+ * Create a .exe installer that bundles one or more .kmp files, together with
+ * setup.exe, keymandesktop.msi, and generates and includes a setup.inf also.
+ *
+ * This module is effectively deprecated, but is present to keep parity with the
+ * legacy kmcomp compiler. Thus, it is included as part of the package compiler,
+ * and will be removed in a future version.
+ *
+ * This tool assumes that the installer .msi is the same version as the
+ * compiler, unlike the legacy compiler which read this metadata from the .msi.
+ */
+
+import JSZip from 'jszip';
+import { CompilerCallbacks, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types";
+import KEYMAN_VERSION from "@keymanapp/keyman-version";
+import { KmpCompiler } from "./kmp-compiler.js";
+import { CompilerMessages } from "./messages.js";
+
+const SETUP_INF_FILENAME = 'setup.inf';
+const PRODUCT_NAME = 'Keyman';
+
+export interface WindowsPackageInstallerSources {
+  msiFilename: string;
+  setupExeFilename: string;
+  licenseFilename: string; // MIT license
+  titleImageFilename?: string;
+
+  appName?: string;
+  startDisabled: boolean;
+  startWithConfiguration: boolean;
+};
+
+export class WindowsPackageInstallerCompiler {
+  private kmpCompiler: KmpCompiler;
+
+  constructor(private callbacks: CompilerCallbacks) {
+    this.kmpCompiler = new KmpCompiler(this.callbacks);
+  }
+
+  public async compile(kpsFilename: string, sources: WindowsPackageInstallerSources): Promise<Uint8Array> {
+    const kps = this.kmpCompiler.loadKpsFile(kpsFilename);
+
+    // Check existence of required files
+    for(const filename of [sources.licenseFilename, sources.msiFilename, sources.setupExeFilename]) {
+      if(!this.callbacks.fs.existsSync(filename)) {
+        this.callbacks.reportMessage(CompilerMessages.Error_FileDoesNotExist({filename}));
+        return null;
+      }
+    }
+
+    // Check existence of optional files
+    for(const filename of [sources.titleImageFilename]) {
+      if(filename && !this.callbacks.fs.existsSync(filename)) {
+        this.callbacks.reportMessage(CompilerMessages.Error_FileDoesNotExist({filename}));
+        return null;
+      }
+    }
+
+    // Note: we never use the MSIFileName field from the .kps any more
+    // Nor do we use the MSIOptions field.
+
+    // Build the zip
+    const zipBuffer = await this.buildZip(kps, kpsFilename, sources);
+    if(!zipBuffer) {
+      // Error messages already reported by buildZip
+      return null;
+    }
+
+    // Build the sfx
+    const sfxBuffer = this.buildSfx(zipBuffer, sources);
+    return sfxBuffer;
+  }
+
+  private async buildZip(kps: KpsFile.KpsFile, kpsFilename: string, sources: WindowsPackageInstallerSources): Promise<Uint8Array> {
+    const kmpJson: KmpJsonFile.KmpJsonFile = this.kmpCompiler.transformKpsFileToKmpObject(kpsFilename, kps);
+    if(!kmpJson.info?.name?.description) {
+      this.callbacks.reportMessage(CompilerMessages.Error_PackageNameCannotBeBlank());
+      return null;
+    }
+
+    const kmpFilename = this.callbacks.path.basename(kpsFilename, KeymanFileTypes.Source.Package) + KeymanFileTypes.Binary.Package;
+    const setupInfBuffer = this.buildSetupInf(sources, kmpJson, kmpFilename, kps);
+    const kmpBuffer = await this.kmpCompiler.buildKmpFile(kpsFilename, kmpJson);
+
+    // Note that this does not technically generate a "valid" sfx according to
+    // the zip spec, because the offsets in the .zip are relative to the start
+    // of the zip, rather than to the start of the sfx. However, as Keyman's
+    // installer chops out the zip from the sfx and loads it in a new stream, it
+    // works as expected.
+    const zip = JSZip();
+    zip.file(SETUP_INF_FILENAME, setupInfBuffer);
+    zip.file(kmpFilename, kmpBuffer);
+    zip.file(this.callbacks.path.basename(sources.msiFilename), this.callbacks.loadFile(sources.msiFilename));
+    zip.file(this.callbacks.path.basename(sources.licenseFilename), this.callbacks.loadFile(sources.licenseFilename));
+    if(sources.titleImageFilename) {
+      zip.file(this.callbacks.path.basename(sources.titleImageFilename), this.callbacks.loadFile(sources.titleImageFilename));
+    }
+
+    return zip.generateAsync({type: 'uint8array', compression:'DEFLATE'});
+  }
+
+  private buildSetupInf(sources: WindowsPackageInstallerSources, kmpJson: KmpJsonFile.KmpJsonFile, kmpFilename: string, kps: KpsFile.KpsFile) {
+    let setupInf = `[Setup]
+Version=${KEYMAN_VERSION.VERSION}
+MSIFileName=${this.callbacks.path.basename(sources.msiFilename)}
+MSIOptions=
+AppName=${sources.appName ?? PRODUCT_NAME}
+License=${this.callbacks.path.basename(sources.licenseFilename)}
+`;
+    if (sources.titleImageFilename) {
+      setupInf += `TitleImage=${this.callbacks.path.basename(sources.titleImageFilename)}\n`;
+    }
+    if (kmpJson.options.graphicFile) {
+      setupInf += `BitmapFileName=${this.callbacks.path.basename(kmpJson.options.graphicFile)}\n`;
+    }
+    if (sources.startDisabled) {
+      setupInf += `StartDisabled=True\n`;
+    }
+    if (sources.startWithConfiguration) {
+      setupInf += `StartWithConfiguration=True\n`;
+    }
+
+    setupInf += `\n[Packages]\n`;
+    setupInf += kmpFilename + '\n';
+    // TODO: multiple packages?
+    const strings = !kps.strings?.string ? [] : (Array.isArray(kps.strings.string) ? kps.strings.string : [kps.strings.string]);
+    if (strings.length) {
+      setupInf += `\n[Strings]\n`;
+      for (const str of strings) {
+        setupInf += `${str.$?.name}=${str.$?.value}\n`;
+      }
+    }
+
+    const setupInfBuffer = new TextEncoder().encode(setupInf);
+    return setupInfBuffer;
+  }
+
+  private buildSfx(zipBuffer: Uint8Array, sources: WindowsPackageInstallerSources): Uint8Array {
+    const setupRedistBuffer = this.callbacks.loadFile(sources.setupExeFilename);
+    const buffer = new Uint8Array(setupRedistBuffer.length + zipBuffer.length);
+    buffer.set(setupRedistBuffer, 0);
+    buffer.set(zipBuffer, setupRedistBuffer.length);
+    return buffer;
+  }
+}

developer/src/kmc-package/src/main.ts

@@ -1,2 +1,3 @@
 export { KmpCompiler } from "./compiler/kmp-compiler.js";
-export { PackageValidation } from "./compiler/package-validation.js";
+export { PackageValidation } from "./compiler/package-validation.js";
+export { WindowsPackageInstallerSources, WindowsPackageInstallerCompiler } from "./compiler/windows-package-installer-compiler.js";

developer/src/kmc-package/test/fixtures/windows-installer/keymandesktop.txt

@@ -0,0 +1,3 @@
+This placeholder file represents keymandesktop.msi, the Windows Installer package for Keyman for Windows.
+
+(Not including the real file to reduce file size in repo, as it is not necessary in order to complete the unit test)

developer/src/kmc-package/test/fixtures/windows-installer/license.txt

@@ -0,0 +1,3 @@
+This placeholder file represents license.txt.
+
+(Not including the real file to reduce file size in repo, as it is not necessary in order to complete the unit test)

developer/src/kmc-package/test/fixtures/windows-installer/setup.txt

@@ -0,0 +1,3 @@
+This placeholder file represents setup-redist.exe, the sfx loader.
+
+(Not including the real file to reduce file size in repo, as it is not necessary in order to complete the unit test)

developer/src/kmc-package/test/test-windows-package-installer-compiler.ts

@@ -0,0 +1,79 @@
+import 'mocha';
+import * as fs from 'fs';
+import * as path from 'path';
+import { assert } from 'chai';
+import JSZip from 'jszip';
+import KEYMAN_VERSION from "@keymanapp/keyman-version";
+import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
+import { makePathToFixture } from './helpers/index.js';
+import { WindowsPackageInstallerCompiler, WindowsPackageInstallerSources } from '../src/compiler/windows-package-installer-compiler.js';
+
+describe('WindowsPackageInstallerCompiler', function () {
+  it(`should build an SFX archive`, async function () {
+    this.timeout(10000); // this test can take a little while to run
+
+    const callbacks = new TestCompilerCallbacks();
+    let compiler = new WindowsPackageInstallerCompiler(callbacks);
+
+    const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps');
+    const sources: WindowsPackageInstallerSources = {
+      licenseFilename: makePathToFixture('windows-installer', 'license.txt'),
+      msiFilename: makePathToFixture('windows-installer', 'keymandesktop.txt'),
+      setupExeFilename: makePathToFixture('windows-installer', 'setup.txt'),
+      startDisabled: false,
+      startWithConfiguration: true,
+      appName: 'Testing',
+    };
+
+    const sfxBuffer = await compiler.compile(kpsPath, sources);
+
+    // This returns a buffer with a SFX loader and a zip suffix. For the sake of repository size
+    // we actually provide a stub SFX loader and a stub MSI file, which is enough to verify that
+    // the compiler is generating what it thinks is a valid file.
+
+    const zip = JSZip();
+
+    // Check that file.kmp contains just 3 files - setup.inf, keymandesktop.msi, and khmer_angkor.kmp,
+    // and that they match exactly what we expect
+    const setupExeSize = fs.statSync(sources.setupExeFilename).size;
+    const setupExe = sfxBuffer.slice(0, setupExeSize);
+    const zipBuffer = sfxBuffer.slice(setupExeSize);
+
+    // Verify setup.exe sfx loader
+    const setupExeFixture = fs.readFileSync(sources.setupExeFilename);
+    assert.deepEqual(setupExe, setupExeFixture);
+
+    // Load the zip from the buffer
+    const zipFile = await zip.loadAsync(zipBuffer, {checkCRC32: true});
+
+    // Verify setup.inf; note that BitmapFileName splash.gif comes from the .kmp
+    const setupInfFixture = `[Setup]
+Version=${KEYMAN_VERSION.VERSION}
+MSIFileName=${path.basename(sources.msiFilename)}
+MSIOptions=
+AppName=${sources.appName}
+License=${path.basename(sources.licenseFilename)}
+BitmapFileName=splash.gif
+StartWithConfiguration=True
+
+[Packages]
+khmer_angkor.kmp
+`;
+
+    const setupInf = new TextDecoder().decode(await zipFile.file('setup.inf').async('uint8array'));
+    assert.equal(setupInf.trim(), setupInfFixture.trim());
+
+    const verifyFile = async (filename: string) => {
+      const fixture = fs.readFileSync(filename);
+      const file = await zipFile.file(path.basename(filename)).async('uint8array');
+      assert.deepEqual(file, fixture, `File in zip '${filename}' did not match fixture`);
+    };
+
+    await verifyFile(sources.msiFilename);
+    await verifyFile(sources.licenseFilename);
+
+    // We only test for existence of the file in the zip for now
+    const kmp = await zipFile.file('khmer_angkor.kmp').async('uint8array');
+    assert.isNotEmpty(kmp);
+  });
+});

developer/src/kmc/README.md

@@ -38,17 +38,29 @@ To see more command line options by using the `--help` option:
 
     kmc --help
 
-kmlmc Usage
------------
+---
 
-To compile a lexical model from its `.model.ts` source, use `kmlmc`:
+To compile a lexical model from its `.model.ts` source, use `kmc`:
 
-    kmlmc my-lexical-model.model.ts --outFile my-lexical-model.js
+    kmc build my-lexical-model.model.ts --outFile my-lexical-model.model.js
 
 To see more command line options by using the `--help` option:
 
-    kmlmc --help
-    kmlmp --help
+    kmc --help
+
+---
+
+kmc can now build package installers for Windows. Example usage (Bash on
+Windows, using 'node .' instead of 'kmc' to run the local build):
+
+```
+node . build windows-package-installer \
+  $KEYMAN_ROOT/developer/src/kmc-package/test/fixtures/khmer_angkor/source/khmer_angkor.kps \
+  --msi /c/Program\ Files\ \(x86\)/Common\ Files/Keyman/Cached\ Installer\ Files/keymandesktop.msi \
+  --exe $KEYMAN_ROOT/windows/bin/desktop/setup-redist.exe \
+  --license $KEYMAN_ROOT/LICENSE.md \
+  --out-file ./khmer.exe
+```
 
 How to build from source
 ------------------------

developer/src/kmc/src/commands/build.ts

@@ -9,6 +9,8 @@ import { CompilerFileCallbacks, CompilerOptions, KeymanFileTypes } from '@keyman
 import { BaseOptions } from '../util/baseOptions.js';
 import { expandFileLists } from '../util/fileLists.js';
 import { isProject } from '../util/projectLoader.js';
+import { buildTestData } from './buildTestData/index.js';
+import { buildWindowsPackageInstaller } from './buildWindowsPackageInstaller/index.js';
 
 
 function commandOptionsToCompilerOptions(options: any): CompilerOptions {
@@ -30,7 +32,7 @@ function commandOptionsToCompilerOptions(options: any): CompilerOptions {
 }
 
 export function declareBuild(program: Command) {
-  BaseOptions.addAll(program
+  const command = BaseOptions.addAll(program
     .command('build [infile...]')
     .description(`Compile one or more source files or projects.`)
     .addHelpText('after', `
@@ -74,6 +76,23 @@ If no input file is supplied, kmc will build the current folder.`)
         }
       }
     });
+
+  command
+    .command('ldml-test-data')
+    .description('Convert LDML keyboard test .xml to .json')
+    .action(buildTestData);
+
+  command
+    .command('windows-package-installer <infile>')
+    .description('Build an executable installer for Windows for a Keyman package')
+    .option('--msi <msiFilename>', 'Location of keymandesktop.msi')
+    .option('--exe <exeFilename>', 'Location of setup.exe')
+    .option('--license <licenseFilename>', 'Location of license.txt')
+    .option('--title-image [titleImageFilename]', 'Location of title image')
+    .option('--app-name [applicationName]', 'Installer property: name of the application to be installed', 'Keyman')
+    .option('--start-disabled', 'Installer property: do not enable keyboards after installation completes')
+    .option('--start-with-configuration', 'Installer property: start Keyman Configuration after installation completes')
+    .action(buildWindowsPackageInstaller);
 }
 
 async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise<boolean> {