From 1bedc62b5722d0fbca27af306eb1286837116961 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 11 Oct 2024 18:25:46 +0200 Subject: [PATCH 1/5] feat(core): expose `km_core_keyboard_load_from_blob` to WASM --- core/include/keyman/keyman_core_api_bits.h | 2 +- core/src/km_core_keyboard_api.cpp | 23 ++++-- core/src/meson.build | 50 +++++++------ core/src/wasm.cpp | 84 +++++++++++++++++++--- 4 files changed, 123 insertions(+), 36 deletions(-) diff --git a/core/include/keyman/keyman_core_api_bits.h b/core/include/keyman/keyman_core_api_bits.h index bd1f519bffe..2fcc832f01e 100644 --- a/core/include/keyman/keyman_core_api_bits.h +++ b/core/include/keyman/keyman_core_api_bits.h @@ -33,7 +33,7 @@ #undef _kmn_export_flag #undef _kmn_import_flag #undef _kmn_static_flag - #else // How MSVC sepcifies function level attributes adn deprecation + #else // How MSVC sepcifies function level attributes and deprecation #define _kmn_and #define _kmn_tag_fn(a) __declspec(a) #define _kmn_deprecated_flag deprecated diff --git a/core/src/km_core_keyboard_api.cpp b/core/src/km_core_keyboard_api.cpp index 70d37fa63b0..cef9f99162b 100644 --- a/core/src/km_core_keyboard_api.cpp +++ b/core/src/km_core_keyboard_api.cpp @@ -40,18 +40,16 @@ namespace } // namespace km_core_status -km_core_keyboard_load_from_blob( +keyboard_load_from_blob_internal( const km_core_path_name kb_name, - const void* blob, - const size_t blob_size, + const std::vector& buf, km_core_keyboard** keyboard ) { assert(keyboard); - if (!keyboard || !blob) { + if (!keyboard) { return KM_CORE_STATUS_INVALID_ARGUMENT; } - std::vector buf((uint8_t*)blob, (uint8_t*)blob + blob_size); *keyboard = nullptr; try { abstract_processor* kp = processor_factory(kb_name, buf); @@ -67,6 +65,21 @@ km_core_keyboard_load_from_blob( return KM_CORE_STATUS_OK; } +km_core_status +km_core_keyboard_load_from_blob( + const km_core_path_name kb_name, + const void* blob, + const size_t blob_size, + km_core_keyboard** keyboard +) { + if (!blob) { + return KM_CORE_STATUS_INVALID_ARGUMENT; + } + + std::vector buf((uint8_t*)blob, (uint8_t*)blob + blob_size); + return keyboard_load_from_blob_internal(kb_name, buf, keyboard); +} + void km_core_keyboard_dispose(km_core_keyboard *keyboard) { diff --git a/core/src/meson.build b/core/src/meson.build index 030ab0a5144..d80aae84bdd 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -32,8 +32,6 @@ else endif if cpp_compiler.get_id() == 'emscripten' - # TODO: why do we need this defn here? - defns += ['-DKM_CORE_LIBRARY'] icu_if_not_on_wasm = [] else # only include this if NOT on wasm. @@ -54,19 +52,21 @@ if cpp_compiler.get_id() == 'emscripten' '-g', '-Wlimited-postlink-optimizations', '-lembind' ] + util_normalize_table_generator = executable( + 'util_normalize_table_generator', + ['util_normalize_table_generator.cpp'], + cpp_args: defns + warns, + include_directories: [inc], + link_args: links, + dependencies: [icu_uc, icu_i18n], + ) - util_normalize_table_generator = executable('util_normalize_table_generator', - ['util_normalize_table_generator.cpp'], - cpp_args: defns + warns, - include_directories: [inc], - link_args: links, - dependencies: [icu_uc, icu_i18n], - ) - - util_normalize_table_h = custom_target('util_normalize_table.h', - output: 'util_normalize_table.h', - command: [util_normalize_table_generator], - capture:true) + util_normalize_table_h = custom_target( + 'util_normalize_table.h', + output: 'util_normalize_table.h', + command: [util_normalize_table_generator], + capture:true + ) generated_headers += util_normalize_table_h endif @@ -139,13 +139,21 @@ mock_files = files( ) if cpp_compiler.get_id() == 'emscripten' - host_links = ['--whole-archive', '-sALLOW_MEMORY_GROWTH=1', '-sMODULARIZE=1', - '-sEXPORT_ES6', '-sENVIRONMENT=web,webview', - '--emit-tsd', 'km-core-interface.d.ts', '-sERROR_ON_UNDEFINED_SYMBOLS=0'] - - links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']', - # Forcing inclusion of debug symbols - '-g', '-Wlimited-postlink-optimizations', '--bind'] + links += [ + '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']', + # Forcing inclusion of debug symbols + '-g', '-Wlimited-postlink-optimizations', + '-lembind' + ] + host_links = [ + '--whole-archive', + '-sALLOW_MEMORY_GROWTH=1', + '-sMODULARIZE=1', '-sEXPORT_NAME=createCoreProcessor', + '-sEXPORT_ES6', + '-sENVIRONMENT=web,webview', + '--emit-tsd', 'km-core-interface.d.ts', + # '-sERROR_ON_UNDEFINED_SYMBOLS=0' + ] endif lib = library('keymancore', diff --git a/core/src/wasm.cpp b/core/src/wasm.cpp index 90fc85cda88..f11158470e2 100644 --- a/core/src/wasm.cpp +++ b/core/src/wasm.cpp @@ -1,11 +1,7 @@ #ifdef __EMSCRIPTEN__ -#ifdef __EMSCRIPTEN__ #include #include - -#else -#define EMSCRIPTEN_KEEPALIVE -#endif +#include #ifdef __cplusplus #define EXTERN extern "C" EMSCRIPTEN_KEEPALIVE @@ -13,8 +9,11 @@ #define EXTERN EMSCRIPTEN_KEEPALIVE #endif +#include "processor.hpp" #include +namespace em = emscripten; + constexpr km_core_attr const engine_attrs = { 256, KM_CORE_LIB_CURRENT, @@ -24,21 +23,88 @@ constexpr km_core_attr const engine_attrs = { "SIL International" }; -EMSCRIPTEN_KEEPALIVE km_core_attr const & tmp_wasm_attributes() { +EMSCRIPTEN_KEEPALIVE km_core_attr const& tmp_wasm_attributes() { return engine_attrs; } +template class CoreReturn { +public: + CoreReturn(int status = 0, const T* obj = nullptr) : status(status), object(obj) { + } + void setStatus(int status) { + this->status = status; + } + int getStatus() const { + return status; + } + void setObject(const T* obj) { + object = obj; + } + const T* getObject() const { + return object; + } + +private: + int status; + const T* object; +}; + +km_core_status +keyboard_load_from_blob_internal(const km_core_path_name kb_name, const std::vector& buf, km_core_keyboard** keyboard); + +EMSCRIPTEN_KEEPALIVE const CoreReturn* +km_core_keyboard_load_from_blob_wasm( + std::string kb_name, + const emscripten::val& blob_val +) { + std::vector blob; + km_core_keyboard* keyboard_ptr = nullptr; + + const auto length = blob_val["length"].as(); + blob.resize(length); + + emscripten::val memoryView{emscripten::typed_memory_view(length, blob.data())}; + memoryView.call("set", blob_val); + km_core_status retVal = ::keyboard_load_from_blob_internal(kb_name.c_str(), blob, &keyboard_ptr); + return new CoreReturn(retVal, keyboard_ptr); +} + EMSCRIPTEN_BINDINGS(core_interface) { - emscripten::value_object("km_core_attr") + em::value_object("km_core_attr") .field("max_context", &km_core_attr::max_context) .field("current", &km_core_attr::current) .field("revision", &km_core_attr::revision) .field("age", &km_core_attr::age) .field("technology", &km_core_attr::technology) - //.field("vendor", &km_core_attr::vendor, emscripten::allow_raw_pointers()) + //.field("vendor", &km_core_attr::vendor, em::allow_raw_pointers()) ; - emscripten::function("tmp_wasm_attributes", &tmp_wasm_attributes); + em::function("tmp_wasm_attributes", &tmp_wasm_attributes); + + + // Unfortunately embind has an open issue with enums and typescript where it + // only generates a type for the enum, but not the values in a usable way. + // Therefore it's not much use to expose the enum here. + // See https://github.com/emscripten-core/emscripten/issues/18585 + + // em::enum_("km_core_status_codes") + // .value("OK", KM_CORE_STATUS_OK) + // .value("NO_MEM", KM_CORE_STATUS_NO_MEM) + // .value("IO_ERROR", KM_CORE_STATUS_IO_ERROR) + // .value("INVALID_ARGUMENT", KM_CORE_STATUS_INVALID_ARGUMENT) + // .value("KEY_ERROR", KM_CORE_STATUS_KEY_ERROR) + // .value("INSUFFICENT_BUFFER", KM_CORE_STATUS_INSUFFICENT_BUFFER) + // .value("INVALID_UTF", KM_CORE_STATUS_INVALID_UTF) + // .value("INVALID_KEYBOARD", KM_CORE_STATUS_INVALID_KEYBOARD) + // .value("NOT_IMPLEMENTED", KM_CORE_STATUS_NOT_IMPLEMENTED) + // .value("OS_ERROR", KM_CORE_STATUS_OS_ERROR); + + em::class_("km_core_keyboard"); + em::class_>("CoreKeyboardReturn") + .property("status", &CoreReturn::getStatus) + .property("object", &CoreReturn::getObject, em::allow_raw_pointers()); + + em::function("keyboard_load_from_blob", &km_core_keyboard_load_from_blob_wasm, em::allow_raw_pointers()); } #endif From 067c8a0a5b47f47e6eaf1e035b5bdff956eb7346 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 11 Dec 2024 19:47:38 +0100 Subject: [PATCH 2/5] chore(core): cleanup `meson.build` --- core/meson.build | 2 +- core/src/meson.build | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/meson.build b/core/meson.build index 8c6cbfdc61d..6f0ad87db50 100644 --- a/core/meson.build +++ b/core/meson.build @@ -37,7 +37,7 @@ endif # shared with a number of subdirs if cpp_compiler.get_id() == 'emscripten' - wasm_exported_runtime_methods = '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\']' + wasm_exported_runtime_methods = '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']' endif subdir('docs/internal') diff --git a/core/src/meson.build b/core/src/meson.build index d80aae84bdd..a9771e4747e 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -138,11 +138,14 @@ mock_files = files( 'mock/mock_processor.cpp', ) +extra_defns = [] +extra_links = [] if cpp_compiler.get_id() == 'emscripten' - links += [ - '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']', + extra_defns += ['-g'] + extra_links += ['-sSAFE_HEAP=1', wasm_exported_runtime_methods, # Forcing inclusion of debug symbols '-g', '-Wlimited-postlink-optimizations', + '--emit-tsd', 'keymancore.d.ts', '-lembind' ] host_links = [ @@ -163,8 +166,8 @@ lib = library('keymancore', mock_files, version_res, generated_headers, - cpp_args: defns + warns + flags, - link_args: links, + cpp_args: defns + extra_defns + warns + flags, + link_args: links + extra_links, version: lib_version, include_directories: inc, pic: true, @@ -187,9 +190,9 @@ pkg.generate( if cpp_compiler.get_id() == 'emscripten' # Build an executable host = executable('km-core', - cpp_args: defns, + cpp_args: defns + extra_defns, include_directories: inc, - link_args: links + host_links, + link_args: links + host_links + extra_links, objects: lib.extract_all_objects(recursive: false)) if get_option('buildtype') == 'release' From 193dd77ffbe1a12fa961e9365c369f45cb3cd87a Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 16 Dec 2024 17:19:52 +0100 Subject: [PATCH 3/5] fix(core): fix build of unit tests Without this change building unit tests fail with an error that `--shared-memory` is disallowed. --- core/meson.build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/meson.build b/core/meson.build index 6f0ad87db50..a481567b850 100644 --- a/core/meson.build +++ b/core/meson.build @@ -38,6 +38,9 @@ endif # shared with a number of subdirs if cpp_compiler.get_id() == 'emscripten' wasm_exported_runtime_methods = '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']' + + # For Google Test + add_global_arguments('-pthread', language: [ 'cpp', 'c' ] ) endif subdir('docs/internal') From 06a8c7a867aea88df187f2407d4828f6e0771509 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 16 Dec 2024 17:22:11 +0100 Subject: [PATCH 4/5] feat(core): properly implement converting blob --- core/src/wasm.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/src/wasm.cpp b/core/src/wasm.cpp index f11158470e2..a0dc5c1f790 100644 --- a/core/src/wasm.cpp +++ b/core/src/wasm.cpp @@ -57,14 +57,9 @@ km_core_keyboard_load_from_blob_wasm( std::string kb_name, const emscripten::val& blob_val ) { - std::vector blob; + std::vector blob = emscripten::convertJSArrayToNumberVector(blob_val); km_core_keyboard* keyboard_ptr = nullptr; - const auto length = blob_val["length"].as(); - blob.resize(length); - - emscripten::val memoryView{emscripten::typed_memory_view(length, blob.data())}; - memoryView.call("set", blob_val); km_core_status retVal = ::keyboard_load_from_blob_internal(kb_name.c_str(), blob, &keyboard_ptr); return new CoreReturn(retVal, keyboard_ptr); } From d48acd320476ff31c7234e98b6cb6b1fc0d159af Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 16 Dec 2024 18:01:09 +0100 Subject: [PATCH 5/5] chore(web): rename type library to match target name This change renames the type library from `km-core=-interface.d.ts` to `keymancore.d.ts` to match the name of the target (`keymancore`) to make it more obvious where this comes from and where/how it gets build. --- core/src/meson.build | 1 - web/src/engine/core-processor/build.sh | 6 +++--- web/src/engine/core-processor/src/core-processor.ts | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/meson.build b/core/src/meson.build index a9771e4747e..ec76b843d86 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -154,7 +154,6 @@ if cpp_compiler.get_id() == 'emscripten' '-sMODULARIZE=1', '-sEXPORT_NAME=createCoreProcessor', '-sEXPORT_ES6', '-sENVIRONMENT=web,webview', - '--emit-tsd', 'km-core-interface.d.ts', # '-sERROR_ON_UNDEFINED_SYMBOLS=0' ] endif diff --git a/web/src/engine/core-processor/build.sh b/web/src/engine/core-processor/build.sh index 072e5a89f35..09568da5c20 100755 --- a/web/src/engine/core-processor/build.sh +++ b/web/src/engine/core-processor/build.sh @@ -23,7 +23,7 @@ builder_describe "Keyman Core WASM integration" \ "--ci+ Set to utilize CI-based test configurations & reporting." builder_describe_outputs \ - configure "/web/src/engine/core-processor/src/import/core/km-core-interface.d.ts" \ + configure "/web/src/engine/core-processor/src/import/core/keymancore.d.ts" \ build "/web/build/${SUBPROJECT_NAME}/lib/index.mjs" builder_parse "$@" @@ -41,12 +41,12 @@ do_configure() { mkdir -p "src/import/core/" # we don't need this file for release builds, but it's nice to have # for reference and auto-completion - cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/km-core-interface.d.ts" "src/import/core/" + cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/keymancore.d.ts" "src/import/core/" } copy_deps() { mkdir -p "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/" - cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/"km-core-interface.d.ts "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/" + cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/keymancore.d.ts" "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/" cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/"km-core{.js,.wasm} "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/" } diff --git a/web/src/engine/core-processor/src/core-processor.ts b/web/src/engine/core-processor/src/core-processor.ts index e1584ad1ea8..7e54fcc1aa6 100644 --- a/web/src/engine/core-processor/src/core-processor.ts +++ b/web/src/engine/core-processor/src/core-processor.ts @@ -1,4 +1,4 @@ -type km_core_attr = import('./import/core/km-core-interface.js').km_core_attr; +type km_core_attr = import('./import/core/keymancore.js').km_core_attr; export class CoreProcessor { private instance: any;