diff --git a/.github/workflows/keyboard.yml b/.github/workflows/keyboard.yml index a8e527b9810..42bc2f52854 100644 --- a/.github/workflows/keyboard.yml +++ b/.github/workflows/keyboard.yml @@ -38,6 +38,6 @@ jobs: - name: Compile Keyboards run: kmc --error-reporting build keyboards/3.0/*.xml - name: Check ABNF - run: bash tools/scripts/keyboard-abnf-tests/check-keyboard-abnf.sh + run: 'cd tools/scripts/keyboard-abnf-tests && npm ci && npm t' - name: Run Kbd Charts run: 'cd docs/charts/keyboards && npm ci && npm run build' diff --git a/tools/scripts/keyboard-abnf-tests/.gitignore b/tools/scripts/keyboard-abnf-tests/.gitignore new file mode 100644 index 00000000000..07e6e472cc7 --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/.gitignore @@ -0,0 +1 @@ +/node_modules diff --git a/tools/scripts/keyboard-abnf-tests/README.md b/tools/scripts/keyboard-abnf-tests/README.md new file mode 100644 index 00000000000..a7c724143ea --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/README.md @@ -0,0 +1,23 @@ +# Keyboard-abnf-tests + +Tests for and against the ABNF files, written in Node.js + +## To use + +- `npm ci` +- `npm t` + +## To update + +Note there are four files. There's a `.d` directory for each ABNF file in keyboards/abnf/. The "pass" files are expected to pass the ABNF and the "fail" to fail it. Lines beginning with # are comments and skipped. + +- transform-from-required.d/from-match.pass.txt +- transform-from-required.d/from-match.fail.txt +- transform-to-required.d/to-replacement.pass.txt +- transform-to-required.d/to-replacement.fail.txt + +## Copyright + +Copyright © 1991-2025 Unicode, Inc. +All rights reserved. +[Terms of use](https://www.unicode.org/copyright.html) diff --git a/tools/scripts/keyboard-abnf-tests/check-keyboard-abnf.sh b/tools/scripts/keyboard-abnf-tests/check-keyboard-abnf.sh deleted file mode 100755 index da348c9a46d..00000000000 --- a/tools/scripts/keyboard-abnf-tests/check-keyboard-abnf.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -ABNF_DIR=keyboards/abnf -TEST_DIR=tools/scripts/keyboard-abnf-tests -abnf_check="npx --package=abnf abnf_check" -abnf_test="npx --package=abnf abnf_test" -TEMP=$(mktemp -d) -echo "-- checking ABNF --" - -for abnf in ${ABNF_DIR}/*.abnf; do - echo Validating ${abnf} - ${abnf_check} ${abnf} || exit 1 -done - -echo "-- running test suites --" - -for abnf in ${ABNF_DIR}/*.abnf; do - echo Testing ${abnf} - base=$(basename ${abnf} .abnf) - # fix for node-abnf issue - fgrep -v SKIP-NODE-ABNF < ${abnf} > ${TEMP}/${base}.abnf - abnf=${TEMP}/${base}.abnf - SUITEDIR=${TEST_DIR}/${base}.d - if [[ -d ${SUITEDIR} ]]; - then - echo " Test suite ${SUITEDIR}" - for testf in ${SUITEDIR}/*.pass.txt; do - start=$(basename ${testf} .pass.txt) - echo " Testing PASS ${testf} for ${start}" - while IFS="" read -r str || [ -n "$str" ] - do - if echo "${str}" | grep -v -q '^#'; then - echo "# '${str}'" - (${abnf_test} ${abnf} -t "${str}") 2>&1 >/dev/null || exit 1 - fi - done <${testf} - done - for testf in ${SUITEDIR}/*.fail.txt; do - start=$(basename ${testf} .fail.txt) - echo " Testing FAIL ${testf} for ${start}" - while IFS="" read -r str || [ -n "$str" ] - do - if echo "${str}" | grep -v -q '^#'; then - echo "# '${str}'" - (${abnf_test} ${abnf} -t "${str}") 2>&1 > /dev/null && (echo ERROR should have failed ; exit 1) - fi - done <${testf} - done - else - echo " Warning: ${SUITEDIR} did not exist" - fi - # npx --package=abnf abnf_check ${abnf} || exit 1 -done - -echo "All OK" -exit 0 - diff --git a/tools/scripts/keyboard-abnf-tests/lib/index.mjs b/tools/scripts/keyboard-abnf-tests/lib/index.mjs new file mode 100644 index 00000000000..9add5715d65 --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/lib/index.mjs @@ -0,0 +1,123 @@ +// Copyright (c) 2025 Unicode, Inc. +// For terms of use, see http://www.unicode.org/copyright.html +// SPDX-License-Identifier: Unicode-3.0 + +import { XMLParser } from "fast-xml-parser"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import * as abnf from "abnf"; +import peggy from "peggy"; + +/** relative path to ABNF */ +export const ABNF_DIR = "../../../keyboards/abnf"; + +/** + * @param {string} abnfPath path to .abnf file + * @returns the raw parser + */ +export async function getAbnfParser(abnfPath) { + const parsed = await abnf.parseFile(abnfPath); + const opts = { + grammarSource: abnfPath, + trace: false, + }; + const text = parsed.toFormat({ format: "peggy" }); + const parser = peggy.generate(text, opts); + + return parser; +} + +/** + * @param {string} abnfPath path to .abnf file + * @param {Object} parser parser from getAbnfParser + * @returns function taking a string and returning results (or throwing) + */ +export async function getParseFunction(abnfPath) { + const parser = await getAbnfParser(abnfPath); + const opts = { + grammarSource: abnfPath, + trace: false, + }; + const fn = (str) => parser.parse(str, opts); + return fn; +} + +/** + * Check XML file for valid transform from= and to= + * @param {string} path path to keyboard XML + * @returns true if OK,otherwise throws + */ +export async function checkXml(path) { + // load the ABNF files. This creates two functions, parseFrom and parseTo + // that can take any text and match against the grammar. + + const parseFrom = await getParseFunction( + join(ABNF_DIR, "transform-from-required.abnf") + ); + const parseTo = await getParseFunction( + join(ABNF_DIR, "transform-to-required.abnf") + ); + + // read the XML and parse it + const text = readFileSync(path); + const parser = new XMLParser({ + ignoreAttributes: false, + trimValues: false, + htmlEntities: true, + }); + const r = parser.parse(text, false); + + // pull out the transforms + let transforms = r?.keyboard3?.transforms; + + if (!transforms) return true; // no transforms + + // If there's only one element, it will be element: {} instead of element: [{ … }] + // this is how the XML parser works. We convert it to an array for processing. + if (!Array.isArray(transforms)) { + transforms = [transforms]; + } + + for (const transformSet of transforms) { + let transformGroups = transformSet?.transformGroup; + + if (!transformGroups) continue; // no transforms + + // If there's only one element, it will be element: {} instead of element: [{ … }] + // this is how the XML parser works. We convert it to an array for processing. + if (!Array.isArray(transformGroups)) { + // there was only one transformGroup + transformGroups = [transformGroups]; + } + + for (const transformGroup of transformGroups) { + let transforms = transformGroup?.transform; + if (!transforms) continue; + + // If there's only one element, it will be element: {} instead of element: [{ … }] + // this is how the XML parser works. We convert it to an array for processing. + if (!Array.isArray(transforms)) { + transforms = [transforms]; + } + for (const transform of transforms) { + // Check the from= string against the from ABNF + const fromStr = transform["@_from"]; + try { + parseFrom(fromStr); + } catch (e) { + throw Error(`Bad from="${fromStr}"`, { cause: e }); + } + // Check the to= string against the to ABNF + const toStr = transform["@_to"]; + if (toStr) { // it's legal to have a missing to= + try { + parseTo(toStr); + } catch (e) { + throw Error(`Bad to="${toStr}"`, { cause: e }); + } + } + } + } + } + return true; +} diff --git a/tools/scripts/keyboard-abnf-tests/lib/util.mjs b/tools/scripts/keyboard-abnf-tests/lib/util.mjs new file mode 100644 index 00000000000..dffabd4c463 --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/lib/util.mjs @@ -0,0 +1,22 @@ +// Copyright (c) 2025 Unicode, Inc. +// For terms of use, see http://www.unicode.org/copyright.html +// SPDX-License-Identifier: Unicode-3.0 + +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { ABNF_DIR } from "./index.mjs"; + +/** + * + * @param {function} callback given abnfFile, abnfPath, abnfText + */ +export async function forEachAbnf(callback) { + return await Promise.all( + readdirSync(ABNF_DIR).map((abnfFile) => { + if (!/^.*\.abnf$/.test(abnfFile)) return; + const abnfPath = join(ABNF_DIR, abnfFile); + const abnfText = readFileSync(abnfPath); + return callback({ abnfFile, abnfPath, abnfText }); + }) + ); +} diff --git a/tools/scripts/keyboard-abnf-tests/package-lock.json b/tools/scripts/keyboard-abnf-tests/package-lock.json new file mode 100644 index 00000000000..37e8089e870 --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/package-lock.json @@ -0,0 +1,124 @@ +{ + "name": "@unicode-org/keyboard-abnf-tests", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@unicode-org/keyboard-abnf-tests", + "version": "1.0.0", + "license": "Unicode-3.0", + "dependencies": { + "abnf": "^4.3.1", + "fast-xml-parser": "^4.5.1", + "peggy": "^4.2.0" + } + }, + "node_modules/@peggyjs/from-mem": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@peggyjs/from-mem/-/from-mem-1.3.5.tgz", + "integrity": "sha512-oRyzXE7nirAn+5yYjCdWQHg3EG2XXcYRoYNOK8Quqnmm+9FyK/2YWVunwudlYl++M3xY+gIAdf0vAYS+p0nKfQ==", + "dependencies": { + "semver": "7.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/abnf": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/abnf/-/abnf-4.3.1.tgz", + "integrity": "sha512-j4A8wWqKqkcSjx5xFESo9GtW2EUvlUZutcWB1knhxSP9kaXJ/YwL0g6dvMhHRjCPCNsIWwNGoKMHzPwemSpCvw==", + "dependencies": { + "commander": "^13.0.0", + "peggy": "^4.2.0" + }, + "bin": { + "abnf_ast": "bin/abnf_ast.js", + "abnf_check": "bin/abnf_check.js", + "abnf_gen": "bin/abnf_gen.js", + "abnf_test": "bin/abnf_test.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/commander": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-13.0.0.tgz", + "integrity": "sha512-oPYleIY8wmTVzkvQq10AEok6YcTC4sRUBl8F9gVuwchGVUCTbl/vhLTaQqutuuySYOsu8YTgV+OxKc/8Yvx+mQ==", + "engines": { + "node": ">=18" + } + }, + "node_modules/fast-xml-parser": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.1.tgz", + "integrity": "sha512-y655CeyUQ+jj7KBbYMc4FG01V8ZQqjN+gDYGJ50RtfsUB8iG9AmwmwoAgeKLJdmueKKMrH1RJ7yXHTSoczdv5w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/peggy": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/peggy/-/peggy-4.2.0.tgz", + "integrity": "sha512-ZjzyJYY8NqW8JOZr2PbS/J0UH/hnfGALxSDsBUVQg5Y/I+ZaPuGeBJ7EclUX2RvWjhlsi4pnuL1C/K/3u+cDeg==", + "dependencies": { + "@peggyjs/from-mem": "1.3.5", + "commander": "^12.1.0", + "source-map-generator": "0.8.0" + }, + "bin": { + "peggy": "bin/peggy.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/peggy/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "engines": { + "node": ">=18" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/source-map-generator": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/source-map-generator/-/source-map-generator-0.8.0.tgz", + "integrity": "sha512-psgxdGMwl5MZM9S3FWee4EgsEaIjahYV5AzGnwUvPhWeITz/j6rKpysQHlQ4USdxvINlb8lKfWGIXwfkrgtqkA==", + "engines": { + "node": ">= 10" + } + }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + } + } +} diff --git a/tools/scripts/keyboard-abnf-tests/package.json b/tools/scripts/keyboard-abnf-tests/package.json new file mode 100644 index 00000000000..1bcfdb96f4a --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/package.json @@ -0,0 +1,18 @@ +{ + "name": "@unicode-org/keyboard-abnf-tests", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "test": "node --test" + }, + "keywords": [], + "author": "Steven R. Loomis ", + "license": "Unicode-3.0", + "description": "Tests for the keyboard ABNF", + "private": true, + "dependencies": { + "abnf": "^4.3.1", + "fast-xml-parser": "^4.5.1", + "peggy": "^4.2.0" + } +} diff --git a/tools/scripts/keyboard-abnf-tests/test/abnf-valid.test.mjs b/tools/scripts/keyboard-abnf-tests/test/abnf-valid.test.mjs new file mode 100644 index 00000000000..91b8e005446 --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/test/abnf-valid.test.mjs @@ -0,0 +1,24 @@ +// Copyright (c) 2025 Unicode, Inc. +// For terms of use, see http://www.unicode.org/copyright.html +// SPDX-License-Identifier: Unicode-3.0 + +import * as abnf from "abnf"; +import { test } from "node:test"; +import * as assert from "node:assert"; +import { forEachAbnf } from "../lib/util.mjs"; + +function check_refs(parsed) { + const errs = abnf.checkRefs(parsed); + if (!errs) return 0; + for (const err of errs) { + console.error(err); + } + return 3; +} + +await forEachAbnf(async ({ abnfFile, abnfText, abnfPath }) => { + await test(`Test validity: ${abnfFile}`, async (t) => { + const parsed = await abnf.parseFile(abnfPath); + assert.equal(check_refs(parsed), 0); + }); +}); diff --git a/tools/scripts/keyboard-abnf-tests/test/datadriven.test.mjs b/tools/scripts/keyboard-abnf-tests/test/datadriven.test.mjs new file mode 100644 index 00000000000..b655e80caff --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/test/datadriven.test.mjs @@ -0,0 +1,63 @@ +// Copyright (c) 2025 Unicode, Inc. +// For terms of use, see http://www.unicode.org/copyright.html +// SPDX-License-Identifier: Unicode-3.0 + +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { test } from "node:test"; +import { basename, join } from "node:path"; +import * as assert from "node:assert"; +import { forEachAbnf } from "../lib/util.mjs"; +import { getParseFunction } from "../lib/index.mjs"; + +async function assertTest({ t, abnfPath, testText, expect }) { + const parser = await getParseFunction(abnfPath); + for (const str of testText + .trim() + .split("\n") + .filter((l) => !/^#/.test(l))) { + await t.test(`"${str}"`, async (t) => { + if (!expect) { + assert.throws( + () => parser(str), + `Expected this expression to fail parsing` + ); + } else { + const results = parser(str); + assert.ok(results); + } + }); + } +} + +await forEachAbnf(async ({ abnfFile, abnfText, abnfPath }) => { + await test(`Test Data: ${abnfFile}`, async (t) => { + const stub = basename(abnfFile, ".abnf"); + const testDir = `./${stub}.d`; + assert.ok(existsSync(testDir), `No test dir: ${testDir}`); + const tests = readdirSync(testDir)?.filter((f) => + /^.*\.(pass|fail)\.txt/.test(f) + ); + assert.ok(tests && tests.length, `No tests in ${testDir}`); + for (const testFile of tests) { + if (testFile.endsWith(".pass.txt")) { + await t.test(`${stub}/${testFile}`, async (t) => { + await assertTest({ + t, + abnfPath, + testText: readFileSync(join(testDir, testFile), "utf-8"), + expect: true, + }); + }); + } else if (testFile.endsWith(".fail.txt")) { + await t.test(`${stub}/${testFile}`, async (t) => { + await assertTest({ + t, + abnfPath, + testText: readFileSync(join(testDir, testFile), "utf-8"), + expect: false, + }); + }); + } else throw Error(`Unknown testFile ${testFile}`); + } + }); +}); diff --git a/tools/scripts/keyboard-abnf-tests/test/xml.test.mjs b/tools/scripts/keyboard-abnf-tests/test/xml.test.mjs new file mode 100644 index 00000000000..49d423f02a1 --- /dev/null +++ b/tools/scripts/keyboard-abnf-tests/test/xml.test.mjs @@ -0,0 +1,22 @@ +// Copyright (c) 2025 Unicode, Inc. +// For terms of use, see http://www.unicode.org/copyright.html +// SPDX-License-Identifier: Unicode-3.0 + +import { join } from "node:path"; +import { readdirSync } from "node:fs"; +import { test } from "node:test"; +import { checkXml } from "../lib/index.mjs"; + +const KBD_DIR = "../../../keyboards/3.0"; + +await test("Testing Keyboard XML files for valid transform from/to attributes", async (t) => { + // keyboards, excluding -test.xml files + const kbds = readdirSync(KBD_DIR).filter((f) => + /^.*(? { + await checkXml(join(KBD_DIR, kbd)); + }); + } +});