Merge pull request #12822 from keymanapp/feat/core/exportapiwasmcore

feat(core): expose `km_core_keyboard_load_from_blob` to WASM 🎼

core/include/keyman/keyman_core_api_bits.h

@@ -33,7 +33,7 @@
     #undef _kmn_export_flag
     #undef _kmn_import_flag
     #undef _kmn_static_flag
-  #else  // How MSVC sepcifies function level attributes adn deprecation
+  #else  // How MSVC sepcifies function level attributes and deprecation
     #define _kmn_and
     #define _kmn_tag_fn(a)     __declspec(a)
     #define _kmn_deprecated_flag deprecated

core/meson.build

@@ -37,7 +37,10 @@ endif
 
 # shared with a number of subdirs
 if cpp_compiler.get_id() == 'emscripten'
-  wasm_exported_runtime_methods = '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\']'
+  wasm_exported_runtime_methods = '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']'
+
+  # For Google Test
+  add_global_arguments('-pthread', language: [ 'cpp', 'c' ] )
 endif
 
 subdir('docs/internal')

core/src/km_core_keyboard_api.cpp

@@ -40,18 +40,16 @@ namespace
 }  // namespace
 
 km_core_status
-km_core_keyboard_load_from_blob(
+keyboard_load_from_blob_internal(
   const km_core_path_name kb_name,
-  const void* blob,
-  const size_t blob_size,
+  const std::vector<uint8_t>& buf,
   km_core_keyboard** keyboard
 ) {
   assert(keyboard);
-  if (!keyboard || !blob) {
+  if (!keyboard) {
     return KM_CORE_STATUS_INVALID_ARGUMENT;
   }
 
-  std::vector<uint8_t> buf((uint8_t*)blob, (uint8_t*)blob + blob_size);
   *keyboard = nullptr;
   try {
     abstract_processor* kp = processor_factory(kb_name, buf);
@@ -67,6 +65,21 @@ km_core_keyboard_load_from_blob(
   return KM_CORE_STATUS_OK;
 }
 
+km_core_status
+km_core_keyboard_load_from_blob(
+  const km_core_path_name kb_name,
+  const void* blob,
+  const size_t blob_size,
+  km_core_keyboard** keyboard
+) {
+  if (!blob) {
+    return KM_CORE_STATUS_INVALID_ARGUMENT;
+  }
+
+  std::vector<uint8_t> buf((uint8_t*)blob, (uint8_t*)blob + blob_size);
+  return keyboard_load_from_blob_internal(kb_name, buf, keyboard);
+}
+
 void
 km_core_keyboard_dispose(km_core_keyboard *keyboard)
 {

core/src/meson.build

@@ -32,8 +32,6 @@ else
 endif
 
 if cpp_compiler.get_id() == 'emscripten'
-  # TODO: why do we need this defn here?
-  defns += ['-DKM_CORE_LIBRARY']
   icu_if_not_on_wasm = []
 else
   # only include this if NOT on wasm.
@@ -54,19 +52,21 @@ if cpp_compiler.get_id() == 'emscripten'
     '-g', '-Wlimited-postlink-optimizations',
     '-lembind'
   ]
+  util_normalize_table_generator = executable(
+    'util_normalize_table_generator',
+    ['util_normalize_table_generator.cpp'],
+    cpp_args: defns + warns,
+    include_directories: [inc],
+    link_args: links,
+    dependencies: [icu_uc, icu_i18n],
+  )
 
-  util_normalize_table_generator = executable('util_normalize_table_generator',
-      ['util_normalize_table_generator.cpp'],
-      cpp_args: defns + warns,
-      include_directories: [inc],
-      link_args: links,
-      dependencies: [icu_uc, icu_i18n],
-      )
-
-  util_normalize_table_h = custom_target('util_normalize_table.h',
-                      output: 'util_normalize_table.h',
-                      command: [util_normalize_table_generator],
-                      capture:true)
+  util_normalize_table_h = custom_target(
+    'util_normalize_table.h',
+    output: 'util_normalize_table.h',
+    command: [util_normalize_table_generator],
+    capture:true
+  )
 
   generated_headers += util_normalize_table_h
 endif
@@ -138,14 +138,24 @@ mock_files = files(
   'mock/mock_processor.cpp',
 )
 
+extra_defns = []
+extra_links = []
 if cpp_compiler.get_id() == 'emscripten'
-  host_links = ['--whole-archive', '-sALLOW_MEMORY_GROWTH=1', '-sMODULARIZE=1',
-    '-sEXPORT_ES6', '-sENVIRONMENT=web,webview',
-    '--emit-tsd', 'km-core-interface.d.ts', '-sERROR_ON_UNDEFINED_SYMBOLS=0']
-
-  links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']',
-      # Forcing inclusion of debug symbols
-      '-g', '-Wlimited-postlink-optimizations', '--bind']
+  extra_defns += ['-g']
+  extra_links += ['-sSAFE_HEAP=1', wasm_exported_runtime_methods,
+    # Forcing inclusion of debug symbols
+    '-g', '-Wlimited-postlink-optimizations',
+    '--emit-tsd', 'keymancore.d.ts',
+    '-lembind'
+  ]
+  host_links = [
+    '--whole-archive',
+    '-sALLOW_MEMORY_GROWTH=1',
+    '-sMODULARIZE=1', '-sEXPORT_NAME=createCoreProcessor',
+    '-sEXPORT_ES6',
+    '-sENVIRONMENT=web,webview',
+    # '-sERROR_ON_UNDEFINED_SYMBOLS=0'
+  ]
 endif
 
 lib = library('keymancore',
@@ -155,8 +165,8 @@ lib = library('keymancore',
   mock_files,
   version_res,
   generated_headers,
-  cpp_args: defns + warns + flags,
-  link_args: links,
+  cpp_args: defns + extra_defns + warns + flags,
+  link_args: links + extra_links,
   version: lib_version,
   include_directories: inc,
   pic: true,
@@ -179,9 +189,9 @@ pkg.generate(
 if cpp_compiler.get_id() == 'emscripten'
   # Build an executable
   host = executable('km-core',
-    cpp_args: defns,
+    cpp_args: defns + extra_defns,
     include_directories: inc,
-    link_args: links + host_links,
+    link_args: links + host_links + extra_links,
     objects: lib.extract_all_objects(recursive: false))
 
   if get_option('buildtype') == 'release'

core/src/wasm.cpp

@@ -1,20 +1,19 @@
 #ifdef __EMSCRIPTEN__
-#ifdef __EMSCRIPTEN__
 #include <emscripten/emscripten.h>
 #include <emscripten/bind.h>
-
-#else
-#define EMSCRIPTEN_KEEPALIVE
-#endif
+#include <vector>
 
 #ifdef __cplusplus
 #define EXTERN extern "C" EMSCRIPTEN_KEEPALIVE
 #else
 #define EXTERN EMSCRIPTEN_KEEPALIVE
 #endif
 
+#include "processor.hpp"
 #include <keyman_core.h>
 
+namespace em = emscripten;
+
 constexpr km_core_attr const engine_attrs = {
   256,
   KM_CORE_LIB_CURRENT,
@@ -24,21 +23,83 @@ constexpr km_core_attr const engine_attrs = {
   "SIL International"
 };
 
-EMSCRIPTEN_KEEPALIVE km_core_attr const & tmp_wasm_attributes() {
+EMSCRIPTEN_KEEPALIVE km_core_attr const& tmp_wasm_attributes() {
   return engine_attrs;
 }
 
+template <typename T> class CoreReturn {
+public:
+  CoreReturn(int status = 0, const T* obj = nullptr) : status(status), object(obj) {
+  }
+  void setStatus(int status) {
+    this->status = status;
+  }
+  int getStatus() const {
+    return status;
+  }
+  void setObject(const T* obj) {
+    object = obj;
+  }
+  const T* getObject() const {
+    return object;
+  }
+
+private:
+  int status;
+  const T* object;
+};
+
+km_core_status
+keyboard_load_from_blob_internal(const km_core_path_name kb_name, const std::vector<uint8_t>& buf, km_core_keyboard** keyboard);
+
+EMSCRIPTEN_KEEPALIVE const CoreReturn<km_core_keyboard>*
+km_core_keyboard_load_from_blob_wasm(
+  std::string kb_name,
+  const emscripten::val& blob_val
+) {
+  std::vector<uint8_t> blob = emscripten::convertJSArrayToNumberVector<uint8_t>(blob_val);
+  km_core_keyboard* keyboard_ptr = nullptr;
+
+  km_core_status retVal = ::keyboard_load_from_blob_internal(kb_name.c_str(), blob, &keyboard_ptr);
+  return new CoreReturn<km_core_keyboard>(retVal, keyboard_ptr);
+}
+
 EMSCRIPTEN_BINDINGS(core_interface) {
 
-  emscripten::value_object<km_core_attr>("km_core_attr")
+  em::value_object<km_core_attr>("km_core_attr")
     .field("max_context", &km_core_attr::max_context)
     .field("current", &km_core_attr::current)
     .field("revision", &km_core_attr::revision)
     .field("age", &km_core_attr::age)
     .field("technology", &km_core_attr::technology)
-    //.field("vendor", &km_core_attr::vendor, emscripten::allow_raw_pointers())
+    //.field("vendor", &km_core_attr::vendor, em::allow_raw_pointers())
     ;
 
-  emscripten::function("tmp_wasm_attributes", &tmp_wasm_attributes);
+  em::function("tmp_wasm_attributes", &tmp_wasm_attributes);
+
+
+  // Unfortunately embind has an open issue with enums and typescript where it
+  // only generates a type for the enum, but not the values in a usable way.
+  // Therefore it's not much use to expose the enum here.
+  // See https://github.com/emscripten-core/emscripten/issues/18585
+
+  // em::enum_<km_core_status_codes>("km_core_status_codes")
+  //   .value("OK", KM_CORE_STATUS_OK)
+  //   .value("NO_MEM", KM_CORE_STATUS_NO_MEM)
+  //   .value("IO_ERROR", KM_CORE_STATUS_IO_ERROR)
+  //   .value("INVALID_ARGUMENT", KM_CORE_STATUS_INVALID_ARGUMENT)
+  //   .value("KEY_ERROR", KM_CORE_STATUS_KEY_ERROR)
+  //   .value("INSUFFICENT_BUFFER", KM_CORE_STATUS_INSUFFICENT_BUFFER)
+  //   .value("INVALID_UTF", KM_CORE_STATUS_INVALID_UTF)
+  //   .value("INVALID_KEYBOARD", KM_CORE_STATUS_INVALID_KEYBOARD)
+  //   .value("NOT_IMPLEMENTED", KM_CORE_STATUS_NOT_IMPLEMENTED)
+  //   .value("OS_ERROR", KM_CORE_STATUS_OS_ERROR);
+
+  em::class_<km_core_keyboard>("km_core_keyboard");
+  em::class_<CoreReturn<km_core_keyboard>>("CoreKeyboardReturn")
+      .property("status", &CoreReturn<km_core_keyboard>::getStatus)
+      .property("object", &CoreReturn<km_core_keyboard>::getObject, em::allow_raw_pointers());
+
+  em::function("keyboard_load_from_blob", &km_core_keyboard_load_from_blob_wasm, em::allow_raw_pointers());
 }
 #endif

web/src/engine/core-processor/build.sh

@@ -23,7 +23,7 @@ builder_describe "Keyman Core WASM integration" \
   "--ci+                     Set to utilize CI-based test configurations & reporting."
 
 builder_describe_outputs \
-  configure    "/web/src/engine/core-processor/src/import/core/km-core-interface.d.ts" \
+  configure    "/web/src/engine/core-processor/src/import/core/keymancore.d.ts" \
   build        "/web/build/${SUBPROJECT_NAME}/lib/index.mjs"
 
 builder_parse "$@"
@@ -41,12 +41,12 @@ do_configure() {
   mkdir -p "src/import/core/"
   # we don't need this file for release builds, but it's nice to have
   # for reference and auto-completion
-  cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/km-core-interface.d.ts" "src/import/core/"
+  cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/keymancore.d.ts" "src/import/core/"
 }
 
 copy_deps() {
   mkdir -p "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/"
-  cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/"km-core-interface.d.ts "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/"
+  cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/keymancore.d.ts" "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/"
   cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/"km-core{.js,.wasm} "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/"
 }
 

web/src/engine/core-processor/src/core-processor.ts

@@ -1,4 +1,4 @@
-type km_core_attr = import('./import/core/km-core-interface.js').km_core_attr;
+type km_core_attr = import('./import/core/keymancore.js').km_core_attr;
 
 export class CoreProcessor {
   private instance: any;