chore(web): Merge branch 'master' into fix/web/legacy-kbd-modifier-tr…

…acking

HISTORY.md

@@ -1,5 +1,30 @@
 # Keyman Version History
 
+## 17.0.248 alpha 2024-01-19
+
+* chore(android): Update targetSdkVersion to 34 (#10393)
+*  (#10352)
+* fix(web): keyboard-documentation rendering mode (#10417)
+* fix(ios): multiple keyboard slide-in animations on app start (#10362)
+* fix(web): U_ key id -> text for all subkeys; is now preprocessed (#10434)
+* chore(android/app): Update whatsnew for 17.0 (#10395)
+* fix(web): bulk renderer interface for recent-version targeting keyboards (#10427)
+* feat(core): cross-segment markers (#10394)
+
+## 17.0.247 alpha 2024-01-18
+
+* fix(ios): banner image management (#10337)
+* feat(core): unescape u (#10356)
+* chore(developer,core): change sample and test files to use \u{…} (#10391)
+
+## 17.0.246 alpha 2024-01-17
+
+* fix(web): Add null check for changing the keyboard during typing (#10346)
+* chore(common): Update crowdin strings for Khmer (#10411)
+* docs(common): Update website README.md (#10399)
+* docs(linux): Add documentation for Core API verification (#10409)
+* fix(web): right-flick gesture-preview positioning (#10406)
+
 ## 17.0.245 alpha 2024-01-16
 
 * chore(core): Ignore C++ symbols (#10386)

VERSION.md

@@ -1 +1 @@
-17.0.246
+17.0.249

android/KMAPro/kMAPro/build.gradle

@@ -10,7 +10,7 @@ ext.rootPath = '../../'
 apply from: "$rootPath/version.gradle"
 
 android {
-    compileSdkVersion 33
+    compileSdk 34
     namespace="com.tavultesoft.kmapro"
 
     // Don't compress kmp files so they can be copied via AssetManager
@@ -21,7 +21,7 @@ android {
     defaultConfig {
         applicationId "com.tavultesoft.kmapro"
         minSdkVersion 21
-        targetSdkVersion 33
+        targetSdkVersion 34
 
         //println "===DUMPING PROPERTIES==="
         //dumpProperties(project) // Use this to dump all external properties for debugging TeamCity integration

android/KMEA/app/build.gradle

@@ -7,12 +7,12 @@ ext.rootPath = '../../'
 apply from: "$rootPath/version.gradle"
 
 android {
-    compileSdkVersion 33
+    compileSdk 34
     namespace "com.keyman.engine"
 
     defaultConfig {
         minSdkVersion 21
-        targetSdkVersion 33
+        targetSdkVersion 34
 
         // VERSION_CODE and VERSION_NAME from version.gradle but Gradle removes them for libraries
         buildConfigField "String", "KEYMAN_ENGINE_VERSION_NAME", "\""+VERSION_NAME+"\""

android/Samples/KMSample1/app/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 android {
-    compileSdkVersion 33
+    compileSdk 34
     namespace="com.keyman.kmsample1"
 
     // Don't compress kmp files so they can be copied via AssetManager
@@ -14,7 +14,7 @@ android {
     defaultConfig {
         applicationId "com.keyman.kmsample1"
         minSdkVersion 21
-        targetSdkVersion 33
+        targetSdkVersion 34
         versionCode 1
         versionName "1.0"
     }

android/Samples/KMSample2/app/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 android {
-    compileSdkVersion 33
+    compileSdk 34
     namespace="com.keyman.kmsample2"
 
     // Don't compress kmp files so they can be copied via AssetManager
@@ -14,7 +14,7 @@ android {
     defaultConfig {
         applicationId "com.keyman.kmsample2"
         minSdkVersion 21
-        targetSdkVersion 33
+        targetSdkVersion 34
         versionCode 1
         versionName "1.0"
     }

android/Tests/KeyboardHarness/app/build.gradle

@@ -6,7 +6,7 @@ ext.rootPath = '../../../'
 apply from: "$rootPath/version.gradle"
 
 android {
-    compileSdkVersion 33
+    compileSdk 34
     namespace="com.keyman.android.tests.keyboardHarness"
 
     // Don't compress kmp files so they can be copied via AssetManager
@@ -21,7 +21,7 @@ android {
     defaultConfig {
         applicationId "com.keyman.android.tests.keyboardHarness"
         minSdkVersion 21
-        targetSdkVersion 33
+        targetSdkVersion 34
 
         // VERSION_CODE and VERSION_NAME from version.gradle
         versionCode VERSION_CODE as Integer

android/Tests/keycode/app/build.gradle

@@ -4,13 +4,12 @@ ext.rootPath = '../../../'
 apply from: "$rootPath/version.gradle"
 
 android {
-    compileSdkVersion 33
     namespace="com.keyman.android.tests.keycode"
 
     defaultConfig {
         applicationId "com.keyman.android.tests.keycode"
         minSdkVersion 21
-        targetSdkVersion 33
+        targetSdkVersion 34
 
         // VERSION_CODE and VERSION_NAME from version.gradle
         versionCode VERSION_CODE as Integer

android/help/about/whatsnew.md

@@ -2,3 +2,13 @@
 title: What's New
 ---
 Here are some of the new features we have added to Keyman 17.0 for Android:
+
+* New gesture support (#5029)
+* When suggestions aren't enabled, display a themed banner (#9696)
+* Smoother keyboard initialization (#10022)
+
+Additional changes:
+
+* Remove built-in browser (#8428)
+* Use web-based popup key longpresses (#9591)
+* Performance improvements

common/web/gesture-recognizer/src/engine/gestureRecognizer.ts

@@ -33,6 +33,11 @@ export class GestureRecognizer<HoveredItemType, StateToken = any> extends Touchp
   }
 
   public destroy() {
+    // When shutting down the gesture engine, we should go ahead and clear out all related
+    // gesture-source tracking.
+    this.activeGestures.forEach((sequence) => sequence.cancel());
+    this.activeSources.forEach((source) => source.terminate(true));
+
     this.mouseEngine.unregisterEventHandlers();
     this.touchEngine.unregisterEventHandlers();
 

common/web/keyboard-processor/src/keyboards/activeLayout.ts

@@ -331,6 +331,10 @@ export class ActiveKeyBase {
       }
     }
 
+    if(!key.text && typeof key.id == 'string') {
+      key.text = ActiveKey.unicodeIDToText(key.id);
+    }
+
     // Ensure subkeys are also properly extended.
     if(key.sk) {
       analysisFlagObj.hasLongpresses = true;

common/web/types/src/ldml-keyboard/pattern-parser.ts

@@ -3,7 +3,7 @@
  */
 
 import { constants } from "@keymanapp/ldml-keyboard-constants";
-import { MATCH_QUAD_ESCAPE, isOneChar, unescapeOneQuadString, unescapeString } from "../util/util.js";
+import { MATCH_QUAD_ESCAPE, isOneChar, unescapeOneQuadString, unescapeString, hexQuad } from "../util/util.js";
 
 
 /**
@@ -51,10 +51,12 @@ export class MarkerParser {
    * Marker sentinel as a string - U+FFFF
    */
   public static readonly SENTINEL = String.fromCodePoint(constants.uc_sentinel);
+  static readonly SENTINEL_MATCH = '\\u' + hexQuad(constants.uc_sentinel);
   /**
    * Marker code as a string - U+0008
    */
   public static readonly MARKER_CODE = String.fromCodePoint(constants.marker_code);
+  static readonly MARKER_CODE_MATCH = '\\u' + hexQuad(constants.marker_code);
 
   /** Minimum ID (trailing code unit) */
   public static readonly MIN_MARKER_INDEX = constants.marker_min_index;
@@ -65,18 +67,10 @@ export class MarkerParser {
   /** Max count of markers */
   public static readonly MAX_MARKER_COUNT = constants.marker_max_count;
 
-  /** 0000 … FFFF */
-  private static hexQuad(n: number): string {
-    if (n < 0x000 || n > 0xFFFF) {
-      throw RangeError(`${n} not in [0x0000,0xFFFF]`);
-    }
-    return n.toString(16).padStart(4, '0');
-  }
-
   private static anyMarkerMatch() : string {
-    const start = MarkerParser.hexQuad(this.MIN_MARKER_INDEX);
-    const end   = MarkerParser.hexQuad(this.MAX_MARKER_INDEX);
-    return `${this.SENTINEL}${this.MARKER_CODE}[\\u${start}-\\u${end}]`; // TODO-LDML: #9121 wrong escape format
+    const start = hexQuad(this.MIN_MARKER_INDEX);
+    const end   = hexQuad(this.MAX_MARKER_INDEX);
+    return `${this.SENTINEL_MATCH}${this.MARKER_CODE_MATCH}[\\u${start}-\\u${end}]`; // TODO-LDML: #9121 wrong escape format
   }
 
   /** Expression that matches any marker */
@@ -103,7 +97,7 @@ export class MarkerParser {
     if (!forMatch) {
       return String.fromCharCode(n);
     } else {
-      return `\\u${MarkerParser.hexQuad(n)}`; // TODO-LDML: #9121 wrong escape format
+      return `\\u${hexQuad(n)}`; // TODO-LDML: #9121 wrong escape format
     }
   }
 
@@ -112,7 +106,11 @@ export class MarkerParser {
     if (n < MarkerParser.MIN_MARKER_INDEX || n > MarkerParser.ANY_MARKER_INDEX) {
       throw RangeError(`Internal Error: marker index out of range ${n}`);
     }
-    return this.SENTINEL + this.MARKER_CODE + this.markerCodeToString(n, forMatch);
+    if (forMatch) {
+      return this.SENTINEL_MATCH + this.MARKER_CODE_MATCH + this.markerCodeToString(n, forMatch);
+    } else {
+      return this.SENTINEL + this.MARKER_CODE + this.markerCodeToString(n, forMatch);
+    }
   }
 
   /** @returns all marker strings as sentinel values */

common/web/types/src/util/util.ts

@@ -22,8 +22,11 @@ export function boxXmlArray(o: any, x: string): void {
 export const MATCH_HEX_ESCAPE = /\\u{([0-9a-fA-F ]{1,})}/g;
 // const MATCH_HEX_ESCAPE = /\\u{((?:(?:[0-9a-fA-F]{1,5})|(?:10[0-9a-fA-F]{4})(?: (?!}))?)+)}/g;
 
+/** regex for single quad escape such as \u0127 or \U00000000 */
+export const CONTAINS_QUAD_ESCAPE = /(?:\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{8}))/;
+
 /** regex for single quad escape such as \u0127 */
-export const MATCH_QUAD_ESCAPE = /\\u([0-9a-fA-F]{4})/g;
+export const MATCH_QUAD_ESCAPE = new RegExp(CONTAINS_QUAD_ESCAPE, 'g');
 
 export class UnescapeError extends Error {
 }
@@ -39,22 +42,31 @@ function unescapeOne(hex: string): string {
 }
 
 /**
- * Unescape one single quad string such as \u0127.
+ * Unescape one single quad string such as \u0127 / \U00000000
  * Throws exception if the string doesn't match MATCH_QUAD_ESCAPE
+ * Note this does not attempt to handle or reject surrogates.
+ * So, `\\uD838\\uDD09` will work but other combinations may not.
  * @param s input string
  * @returns output
  */
 export function unescapeOneQuadString(s: string): string {
   if (!s || !s.match(MATCH_QUAD_ESCAPE)) {
     throw new UnescapeError(`Not a quad escape: ${s}`);
   }
-  function processMatch(str: string, matched: string): string {
-    return unescapeOne(matched);
+  function processMatch(str: string, m16: string, m32: string): string {
+    return unescapeOne(m16 || m32); // either \u or \U
   }
   s = s.replace(MATCH_QUAD_ESCAPE, processMatch);
   return s;
 }
 
+/** unscape multiple occurences of \u0127 style strings */
+export function unescapeQuadString(s: string): string {
+  s = s.replaceAll(MATCH_QUAD_ESCAPE, (quad) => unescapeOneQuadString(quad));
+  return s;
+}
+
+
 /**
  * Unescapes a string according to UTS#18§1.1, see <https://www.unicode.org/reports/tr18/#Hex_notation>
  * @param s escaped string
@@ -88,6 +100,87 @@ export function unescapeString(s: string): string {
   return s;
 }
 
+/** 0000 … FFFF */
+export function hexQuad(n: number): string {
+  if (n < 0x0000 || n > 0xFFFF) {
+    throw RangeError(`${n} not in [0x0000,0xFFFF]`);
+  }
+  return n.toString(16).padStart(4, '0');
+}
+
+/** 00000000 … FFFFFFFF */
+export function hexOcts(n: number): string {
+  if (n < 0x0000 || n > 0xFFFFFFFF) {
+    throw RangeError(`${n} not in [0x00000000,0xFFFFFFFF]`);
+  }
+  return n.toString(16).padStart(8, '0');
+}
+
+/** escape one char for regex in \uXXXX form */
+function escapeRegexChar(ch: string) {
+  const code = ch.codePointAt(0);
+  if (code <= 0xFFFF) {
+    return '\\u' + hexQuad(code);
+  } else {
+    return '\\U' + hexOcts(code);
+  }
+}
+
+/** chars that must be escaped: syntax, C0 + C1 controls */
+const REGEX_SYNTAX_CHAR = /^[\u0000-\u001F\u007F-\u009F{}\[\]\\?.^$*()/-]$/;
+
+function escapeRegexCharIfSyntax(ch: string) {
+  // escape if syntax or not valid
+  if (REGEX_SYNTAX_CHAR.test(ch) || !isValidUnicode(ch.codePointAt(0))) {
+    return escapeRegexChar(ch);
+  } else {
+    return ch; // leave unescaped
+  }
+}
+
+/**
+ * Unescape one codepoint to \u or \U format
+ * @param hex one codepoint in hex, such as '0127'
+ * @returns the unescaped codepoint
+ */
+function regexOne(hex: string): string {
+  const unescaped = unescapeOne(hex);
+  // re-escape as 16 or 32 bit code units
+  return Array.from(unescaped).map(ch => escapeRegexCharIfSyntax(ch)).join('');
+}
+/**
+ * Unescapes a string according to UTS#18§1.1, see <https://www.unicode.org/reports/tr18/#Hex_notation>
+ * @param s escaped string
+ * @returns
+ */
+export function unescapeStringToRegex(s: string): string {
+  if(!s) {
+    return s;
+  }
+  try {
+    /**
+     * process one regex match
+     * @param str ignored
+     * @param matched the entire match such as '0127' or '22 22'
+     * @returns the unescaped match
+     */
+    function processMatch(str: string, matched: string) : string {
+      const codepoints = matched.split(' ');
+      const unescaped = codepoints.map(regexOne);
+      return unescaped.join('');
+    }
+    s = s.replaceAll(MATCH_HEX_ESCAPE, processMatch);
+  } catch(e) {
+    if (e instanceof RangeError) {
+      throw new UnescapeError(`Out of range while unescaping '${s}': ${e.message}`, { cause: e });
+      /* c8 ignore next 3 */
+    } else {
+      throw e; // pass through some other error
+    }
+  }
+  return s;
+}
+
 /** True if this string *could* be a UTF-32 single char */
 export function
 isOneChar(value: string) : boolean {

common/web/types/test/ldml-keyboard/test-pattern-parser.ts

@@ -92,6 +92,10 @@ describe('Test of Pattern Parsers', () => {
         `Give me \\m{a} and \\m{c}, or \\m{.}.`, markers),
         `Give me \uFFFF\u0008\u0001 and \uFFFF\u0008\u0003, or \uFFFF\u0008\uD7FF.`
       );
+      assert.equal(MarkerParser.toSentinelString(
+        `Give me \\m{a} and \\m{c}, or \\m{.}.`, markers, true),
+        `Give me \\uffff\\u0008\\u0001 and \\uffff\\u0008\\u0003, or ${MarkerParser.ANY_MARKER_MATCH}.`
+      );
       assert.throws(() =>
         MarkerParser.toSentinelString(
           `Want to see something funny? \\m{zzz}`, // out of range