chore(common/web): Merge branch 'master' into fix/common/web/types/12…

…809-improve-null-handling-and-initialisation-of-KvkFileWriter

HISTORY.md

@@ -1,5 +1,15 @@
 # Keyman Version History
 
+## 18.0.156 alpha 2024-12-09
+
+* fix(developer): remove platforms from kmc-generate LM readme (#12803)
+* test(developer): add test for ERROR_DescriptionIsMissing to kmc-keyboard-info (#12804)
+* chore(developer): verify bundled node version when building installer (#12806)
+* fix(developer): support hint property in displaymap (#12807)
+* fix(common/web): delete replaceExtension in types/src/util/file-types.ts (#12762)
+* chore(developer): add some docs for language examples in kmp.json (#12805)
+* fix(core): implement ldml_processor::get_key_list() (#12644)
+
 ## 18.0.155 alpha 2024-12-07
 
 * chore(android,windows): Update crowdin for Czech (#12792)

VERSION.md

@@ -1 +1 @@
-18.0.156
+18.0.157

common/web/types/tests/kvk/kvk-file-writer.tests.ts

@@ -269,10 +269,10 @@ function initVisualKeyboardKey(
   bitmap: number[] = null,
 ): VisualKeyboardKey {
   const vkk: VisualKeyboardKey = {
-    vkey: vkey,
-    flags: flags,
-    shift: shift,
-    text: text,
+    vkey,
+    flags,
+    shift,
+    text,
     bitmap: bitmap ? new Uint8Array(bitmap) : null,
   };
   return vkk;

core/include/keyman/keyman_core_api_vkeys.h

@@ -16,6 +16,7 @@
 #pragma once
 
 enum km_core_modifier_state {
+  KM_CORE_MODIFIER_NONE        = 0,
   KM_CORE_MODIFIER_LCTRL       = 1 << 0,
   KM_CORE_MODIFIER_RCTRL       = 1 << 1,
   KM_CORE_MODIFIER_LALT        = 1 << 2,

core/src/kmx/kmx_plus.h

@@ -455,7 +455,13 @@ class COMP_KMXPLUS_LAYR_Helper {
   bool setLayr(const COMP_KMXPLUS_LAYR *newLayr);
   bool valid() const;
 
+  /**
+   * @param list index from 0 to layr->listCount
+   */
   const COMP_KMXPLUS_LAYR_LIST  *getList(KMX_DWORD list) const;
+  /**
+   * @param entry index value: COMP_KMXPLUS_LAYR_LIST.layer but less than COMP_KMXPLUS_LAYR_LIST.layer+COMP_KMXPLUS_LAYR_LIST.count
+   */
   const COMP_KMXPLUS_LAYR_ENTRY *getEntry(KMX_DWORD entry) const;
   const COMP_KMXPLUS_LAYR_ROW   *getRow(KMX_DWORD row) const;
   const COMP_KMXPLUS_LAYR_KEY   *getKey(KMX_DWORD key) const;

core/src/ldml/ldml_processor.cpp

@@ -328,8 +328,7 @@ km_core_attr const & ldml_processor::attributes() const {
 }
 
 km_core_keyboard_key  * ldml_processor::get_key_list() const {
-  km_core_keyboard_key* key_list = new km_core_keyboard_key(KM_CORE_KEYBOARD_KEY_LIST_END);
-  return key_list;
+  return keys.get_key_list();
 }
 
 km_core_keyboard_imx  * ldml_processor::get_imx_list() const {

core/src/ldml/ldml_processor.hpp

@@ -34,10 +34,6 @@ class ldml_processor : public abstract_processor {
       const std::vector<uint8_t> & data
     );
 
-    static bool is_kmxplus_file(
-      const std::vector<uint8_t> & data
-    );
-
     km_core_status
     process_event(
       km_core_state *state,

core/src/ldml/ldml_vkeys.cpp

@@ -8,6 +8,8 @@
 #include "ldml_vkeys.hpp"
 #include "kmx_file.h"
 #include <ldml/keyman_core_ldml.h>
+#include <set>
+#include <assert.h>
 
 namespace km {
 namespace core {
@@ -22,6 +24,114 @@ vkeys::add(km_core_virtual_key vk, km_core_ldml_modifier_state modifier_state, s
   const vkey_id id(vk, modifier_state);
   // assign the string
   vkey_to_string[id] = output;
+  // includes all keys - including gaps.
+  all_vkeys.insert(id);
+}
+
+km_core_keyboard_key  *
+vkeys::get_key_list() const {
+  // prescan to find out which modifier flags are used
+
+  std::size_t other_key_count = 0;        // number of 'OTHER' keys, which will need to expand to all of the other_state set ( so other_state.size())
+
+  std::set<km::core::ldml::km_core_ldml_modifier_state> all_modifiers;
+  for (const auto &k : all_vkeys) {
+    const auto mod = k.second;
+    if (mod == LDML_KEYS_MOD_OTHER) {
+      other_key_count++;
+    }
+    all_modifiers.insert(mod);
+  }
+  std::set<km_core_modifier_state> other_state;
+
+  // Alt
+  if (all_modifiers.count(LDML_KEYS_MOD_ALT) == 0 && all_modifiers.count(LDML_KEYS_MOD_ALTL) == 0 && all_modifiers.count(LDML_KEYS_MOD_ALTR) == 0) {
+    // no ALT keys were seen, so OTHER includes ALT
+    other_state.insert(KM_CORE_MODIFIER_ALT);
+  } else if(all_modifiers.count(LDML_KEYS_MOD_ALTL) == 0) {
+    other_state.insert(KM_CORE_MODIFIER_RALT);
+  } else if(all_modifiers.count(LDML_KEYS_MOD_ALTR) == 0) {
+    other_state.insert(KM_CORE_MODIFIER_LALT);
+  }
+
+  // ctrl
+  if (all_modifiers.count(LDML_KEYS_MOD_CTRL) == 0 && all_modifiers.count(LDML_KEYS_MOD_CTRLL) == 0 && all_modifiers.count(LDML_KEYS_MOD_CTRLR) == 0) {
+    // no CTRL keys were seen, so OTHER includes CTRL
+    other_state.insert(KM_CORE_MODIFIER_CTRL);
+  } else if(all_modifiers.count(LDML_KEYS_MOD_CTRLL) == 0) {
+    other_state.insert(KM_CORE_MODIFIER_RCTRL);
+  } else if(all_modifiers.count(LDML_KEYS_MOD_CTRLR) == 0) {
+    other_state.insert(KM_CORE_MODIFIER_LCTRL);
+  }
+
+  // shift
+  if (all_modifiers.count(LDML_KEYS_MOD_SHIFT) == 0) {
+    other_state.insert(KM_CORE_MODIFIER_SHIFT);
+  }
+
+  // caps
+  if (all_modifiers.count(LDML_KEYS_MOD_CAPS) == 0) {
+    other_state.insert(KM_CORE_MODIFIER_CAPS);
+  }
+
+  // none- it's possible there is no 'none' layer
+  if (all_modifiers.count(LDML_KEYS_MOD_NONE) == 0) {
+    other_state.insert(KM_CORE_MODIFIER_NONE);
+  }
+
+  // We need ALL combinations of the other_state, except for 'all off'.
+  // The number of additions will be (2**(other_state.size())-1
+  // Also, since the LDML_KEYS_MOD_OTHER modifier is excluded, we
+  // will need to subtract 1 when calculating new_list_size
+  const std::size_t other_expanded_count = (1 << other_state.size()) - 1;
+
+  std::vector<uint32_t> other_expanded_mods(other_expanded_count);
+
+  // populate the expanded list.
+  // we start at 1 because 0 is "all bits off" (00000b)
+  for (std::size_t expansion = 1; expansion <= other_expanded_count; expansion++) {
+    uint32_t &expanded_mod = other_expanded_mods.at(expansion-1) = KM_CORE_MODIFIER_NONE;
+    std::size_t bit_mask = 1;
+    for (const auto mod : other_state) {
+      // Check if this modifier is on in this iteration of the expansion
+      // bit_mask will be 2^0 … 2^(other_state().size()-1)
+      if (bit_mask & expansion) {
+        // do we include this entry? check if this bit is on
+        expanded_mod |= mod;
+      }
+      // shift the bitmask over
+      assert(expanded_mod <= KM_CORE_MODIFIER_MASK_CAPS);
+      bit_mask <<= 1;
+    }
+  }
+
+  const std::size_t new_list_size = all_vkeys.size()                                // original size
+                                    + (other_key_count * (other_expanded_count - 1))// number of additional entries needed
+                                    + 1;                                            // terminator
+  km_core_keyboard_key *list = new km_core_keyboard_key[new_list_size];
+  std::size_t n = 0;
+  for (const auto &k : all_vkeys) {
+    const auto vkey = k.first;
+    const auto mod  = k.second;
+
+    if (mod == LDML_KEYS_MOD_OTHER) {
+      // expand to all of other_state
+      for (const auto expanded_mod : other_expanded_mods) {
+        list[n].key             = vkey;
+        list[n++].modifier_flag = expanded_mod;
+        assert(n <= new_list_size);
+      }
+    } else {
+      assert(mod <= KM_CORE_MODIFIER_MASK_CAPS); // that no LDMLisms escape
+      list[n].key             = vkey;
+      list[n++].modifier_flag = mod;
+      assert(n <= new_list_size);
+    }
+  }
+  // add the list terminator
+  list[n++] = KM_CORE_KEYBOARD_KEY_LIST_END;
+  assert(n == new_list_size);
+  return list;
 }
 
 static const uint16_t BOTH_ALT  = LALTFLAG  | RALTFLAG;

core/src/ldml/ldml_vkeys.hpp

@@ -12,6 +12,7 @@
 #include <unordered_map>
 #include <utility>
 #include <vector>
+#include <set>
 
 #include "keyman_core.h"
 
@@ -37,6 +38,7 @@ typedef std::pair<km_core_virtual_key, km_core_ldml_modifier_state> vkey_id;
 class vkeys {
 private:
   std::map<vkey_id, std::u16string> vkey_to_string;
+  std::set<vkey_id> all_vkeys;
 
 public:
   vkeys();
@@ -53,6 +55,11 @@ class vkeys {
   std::u16string
   lookup(km_core_virtual_key vk, uint16_t modifier_state, bool &found) const;
 
+  /**
+   * For implementing ldml_processor::get_key_list()
+   */
+  km_core_keyboard_key* get_key_list() const;
+
 private:
   /**
    * Non-recursive internal lookup of a specific ID

core/tests/unit/ldml/keyboards/k_004_tinyshift.xml

@@ -2,8 +2,9 @@
 
 <!--
 
-@@keys: [SHIFT K_BKQUOTE][K_1][K_BKQUOTE]
-@@expected: \u0037\u1790\u17B6\u0127
+@@keys: [SHIFT K_BKQUOTE][K_1][K_BKQUOTE][LCTRL K_BKQUOTE][RALT K_BKQUOTE]
+@@expected: \u0037\u1790\u17B6\u0127\u1790\u17B6\u0065
+@@keylist: [SHIFT K_BKQUOTE][SHIFT K_1][K_BKQUOTE][K_1][CTRL-do-not-use K_1][ALT-do-not-use K_BKQUOTE][CAPS K_BKQUOTE][ALT-do-not-use CAPS K_BKQUOTE]
 
 -->
 <keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
@@ -23,5 +24,11 @@
     <layer id="shift" modifiers="shift">
       <row keys="seven eee" /> <!-- number row -->
     </layer>
+    <layer id="control" modifiers="ctrl">
+      <row keys="that gap" /> <!-- number row -->
+    </layer>
+    <layer id="catchall" modifiers="other">
+      <row keys="eee gap" /> <!-- number row -->
+    </layer>
   </layers>
 </keyboard3>

core/tests/unit/ldml/ldml.cpp

@@ -16,6 +16,7 @@
 #include <sstream>
 #include <string>
 #include <type_traits>
+#include <set>
 
 #include "path.hpp"
 #include "state.hpp"
@@ -268,6 +269,63 @@ verify_context(std::u16string &text_store, km_core_state *&test_state, std::vect
   delete[] buf;
 }
 
+
+/**
+ * @param actual the list from get_key_list()
+ * @param expected optional list with keys to check, can be empty - not exhaistive
+ * @returns true if passing
+ */
+bool
+verify_key_list(std::set<km::tests::key_event> &actual, std::set<km::tests::key_event> &expected) {
+  bool equals = true;
+  // error if any bad modifier keys
+  for(const auto &akey : actual) {
+    if (akey.modifier_state > KM_CORE_MODIFIER_MASK_CAPS) {
+      equals = false;
+      std::u16string dump = convert<char, char16_t>(akey.dump());  // akey.dump()
+      std::wcout << console_color::fg(console_color::BRIGHT_RED) << "- FAIL - key_map had key with bad modifier " << akey.modifier_state << ": " << dump << console_color::reset() << std::endl;
+    }
+  }
+  // error if any expected keys missing (note expected may be empty)
+  for(const auto &ekey : expected) {
+    if (actual.count(ekey) == 0) {
+      equals = false;
+      std::u16string dump = convert<char, char16_t>(ekey.dump());  // akey.dump()
+      std::wcout << console_color::fg(console_color::BRIGHT_RED) << "- FAIL - key_map had missing key " << dump << console_color::reset() << std::endl;
+    }
+  }
+  if (equals) {
+      std::wcout << console_color::fg(console_color::GREEN) << " " << actual.size() << " vkeys OK, verified  " << expected.size() << console_color::reset() << std::endl;
+  }
+  return equals;
+}
+
+/**
+ * @param actual_list the list from get_key_list()
+ * @param keylist optional string with keys to check, can be empty
+ * @param test the LDML test source, for additional data
+ * @returns true if passing
+ */
+bool
+verify_key_list(const km_core_keyboard_key *actual_list, const std::u16string &expected_list, const km::tests::LdmlTestSource &test) {
+  std::set<km::tests::key_event> actual, expected;
+  // convert actual list
+  while (actual_list != nullptr && !(actual_list->key == 0 && actual_list->modifier_flag == 0)) {
+    km::tests::key_event k(actual_list->key, (uint16_t)actual_list->modifier_flag);
+    actual.insert(k);
+    actual_list++; // advance pointer
+  }
+  // parse the expected list
+  std::string keylist = convert<char16_t, char>(expected_list);
+  while (!keylist.empty() && keylist[0] == '[') {
+    const km::tests::key_event k = km::tests::LdmlEmbeddedTestSource::parse_next_key(keylist);
+    if (!k.empty()) {
+      expected.emplace(k);
+    }
+  }
+  return verify_key_list(actual, expected);
+}
+
 int
 run_test(const km::core::path &source, const km::core::path &compiled, km::tests::LdmlTestSource& test_source) {
   km_core_keyboard * test_kb = nullptr;
@@ -374,6 +432,17 @@ run_test(const km::core::path &source, const km::core::path &compiled, km::tests
         errorLine = __LINE__;
       }
     } break;
+    case km::tests::LDML_ACTION_CHECK_KEYLIST: {
+      std::cout << "- checking keylist" << std::endl;
+      // get keylist from kbd
+      const km_core_keyboard_key* actual_list = test_kb->get_key_list();
+      if (!verify_key_list(actual_list, action.string, test_source)) {
+        errorLine = __LINE__;
+      } else {
+        std::cout << " .. passes." << std::endl;
+      }
+      delete [] actual_list;
+    } break;
     case km::tests::LDML_ACTION_FAIL: {
       // test requested failure
       std::wcout << console_color::fg(console_color::BRIGHT_RED) << "- FAIL: " << action.string << console_color::reset()
@@ -424,7 +493,7 @@ int run_all_tests(const km::core::path &source, const km::core::path &compiled,
 
   std::vector<std::string> failures; // track failures for summary
 
-  int embedded_result = embedded_test_source.load_source(source);
+  int embedded_result = embedded_test_source.load_source(source, compiled);
 
   if (!filter.empty()) {
     // Always skip the embedded test if there's a filter.