CLDR-16836 kbd: ebnf: move tests to js out of shell, check XML files

.github/workflows/keyboard.yml

@@ -38,6 +38,6 @@ jobs:
       - name: Compile Keyboards
         run: kmc --error-reporting build keyboards/3.0/*.xml
       - name: Check ABNF
-        run: bash tools/scripts/keyboard-abnf-tests/check-keyboard-abnf.sh
+        run: 'cd tools/scripts/keyboard-abnf-tests && npm ci && npm t'
       - name: Run Kbd Charts
         run: 'cd docs/charts/keyboards && npm ci && npm run build'

tools/scripts/keyboard-abnf-tests/.gitignore

@@ -0,0 +1 @@
+/node_modules

tools/scripts/keyboard-abnf-tests/README.md

@@ -0,0 +1,23 @@
+# Keyboard-abnf-tests
+
+Tests for and against the ABNF files, written in Node.js
+
+## To use
+
+- `npm ci`
+- `npm t`
+
+## To update
+
+Note there are four files. There's a `.d` directory for each ABNF file in keyboards/abnf/.  The "pass" files are expected to pass the ABNF and the "fail" to fail it.  Lines beginning with # are comments and skipped.
+
+- transform-from-required.d/from-match.pass.txt
+- transform-from-required.d/from-match.fail.txt
+- transform-to-required.d/to-replacement.pass.txt
+- transform-to-required.d/to-replacement.fail.txt
+
+## Copyright
+
+Copyright © 1991-2025 Unicode, Inc.
+All rights reserved.
+[Terms of use](https://www.unicode.org/copyright.html)

tools/scripts/keyboard-abnf-tests/lib/index.mjs

@@ -0,0 +1,123 @@
+// Copyright (c) 2025 Unicode, Inc.
+// For terms of use, see http://www.unicode.org/copyright.html
+// SPDX-License-Identifier: Unicode-3.0
+
+import { XMLParser } from "fast-xml-parser";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import * as abnf from "abnf";
+import peggy from "peggy";
+
+/** relative path to ABNF */
+export const ABNF_DIR = "../../../keyboards/abnf";
+
+/**
+ * @param {string} abnfPath path to .abnf file
+ * @returns the raw parser
+ */
+export async function getAbnfParser(abnfPath) {
+  const parsed = await abnf.parseFile(abnfPath);
+  const opts = {
+    grammarSource: abnfPath,
+    trace: false,
+  };
+  const text = parsed.toFormat({ format: "peggy" });
+  const parser = peggy.generate(text, opts);
+
+  return parser;
+}
+
+/**
+ * @param {string} abnfPath path to .abnf file
+ * @param {Object} parser parser from getAbnfParser
+ * @returns function taking a string and returning results (or throwing)
+ */
+export async function getParseFunction(abnfPath) {
+  const parser = await getAbnfParser(abnfPath);
+  const opts = {
+    grammarSource: abnfPath,
+    trace: false,
+  };
+  const fn = (str) => parser.parse(str, opts);
+  return fn;
+}
+
+/**
+ * Check XML file for valid transform from= and to=
+ * @param {string} path path to keyboard XML
+ * @returns true if OK,otherwise throws
+ */
+export async function checkXml(path) {
+  // load the ABNF files. This creates two functions, parseFrom and parseTo
+  // that can take any text and match against the grammar.
+
+  const parseFrom = await getParseFunction(
+    join(ABNF_DIR, "transform-from-required.abnf")
+  );
+  const parseTo = await getParseFunction(
+    join(ABNF_DIR, "transform-to-required.abnf")
+  );
+
+  // read the XML and parse it
+  const text = readFileSync(path);
+  const parser = new XMLParser({
+    ignoreAttributes: false,
+    trimValues: false,
+    htmlEntities: true,
+  });
+  const r = parser.parse(text, false);
+
+  // pull out the transforms
+  let transforms = r?.keyboard3?.transforms;
+
+  if (!transforms) return true; // no transforms
+
+  // If there's only one element, it will be element: {} instead of element: [{ … }]
+  // this is how the XML parser works. We convert it to an array for processing.
+  if (!Array.isArray(transforms)) {
+    transforms = [transforms];
+  }
+
+  for (const transformSet of transforms) {
+    let transformGroups = transformSet?.transformGroup;
+
+    if (!transformGroups) continue; // no transforms
+
+    // If there's only one element, it will be element: {} instead of element: [{ … }]
+    // this is how the XML parser works. We convert it to an array for processing.
+    if (!Array.isArray(transformGroups)) {
+      // there was only one transformGroup
+      transformGroups = [transformGroups];
+    }
+
+    for (const transformGroup of transformGroups) {
+      let transforms = transformGroup?.transform;
+      if (!transforms) continue;
+
+      // If there's only one element, it will be element: {} instead of element: [{ … }]
+      // this is how the XML parser works. We convert it to an array for processing.
+      if (!Array.isArray(transforms)) {
+        transforms = [transforms];
+      }
+      for (const transform of transforms) {
+        // Check the from= string against the from ABNF
+        const fromStr = transform["@_from"];
+        try {
+          parseFrom(fromStr);
+        } catch (e) {
+          throw Error(`Bad from="${fromStr}"`, { cause: e });
+        }
+        // Check the to= string against the to ABNF
+        const toStr = transform["@_to"];
+        if (toStr) { // it's legal to have a missing to=
+          try {
+            parseTo(toStr);
+          } catch (e) {
+            throw Error(`Bad to="${toStr}"`, { cause: e });
+          }
+        }
+      }
+    }
+  }
+  return true;
+}

tools/scripts/keyboard-abnf-tests/lib/util.mjs

@@ -0,0 +1,22 @@
+// Copyright (c) 2025 Unicode, Inc.
+// For terms of use, see http://www.unicode.org/copyright.html
+// SPDX-License-Identifier: Unicode-3.0
+
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { ABNF_DIR } from "./index.mjs";
+
+/**
+ *
+ * @param {function} callback  given abnfFile, abnfPath, abnfText
+ */
+export async function forEachAbnf(callback) {
+  return await Promise.all(
+    readdirSync(ABNF_DIR).map((abnfFile) => {
+      if (!/^.*\.abnf$/.test(abnfFile)) return;
+      const abnfPath = join(ABNF_DIR, abnfFile);
+      const abnfText = readFileSync(abnfPath);
+      return callback({ abnfFile, abnfPath, abnfText });
+    })
+  );
+}

tools/scripts/keyboard-abnf-tests/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@unicode-org/keyboard-abnf-tests",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "node --test"
+  },
+  "keywords": [],
+  "author": "Steven R. Loomis <[email protected]>",
+  "license": "Unicode-3.0",
+  "description": "Tests for the keyboard ABNF",
+  "private": true,
+  "dependencies": {
+    "abnf": "^4.3.1",
+    "fast-xml-parser": "^4.5.1",
+    "peggy": "^4.2.0"
+  }
+}