WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

WIP

HPCC-29914 Add embedded wasm support

Signed-off-by: Gordon Smith <[email protected]>

WIP rebase on 9.4.x

CMakeLists.txt

@@ -169,6 +169,7 @@ if ( PLUGIN )
     HPCC_ADD_SUBDIRECTORY (dali/base)
     HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED")
+    HPCC_ADD_SUBDIRECTORY (plugins/wasmembed "WASMEMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/memcached "MEMCACHED")
     HPCC_ADD_SUBDIRECTORY (plugins/redis "REDIS")
     HPCC_ADD_SUBDIRECTORY (plugins/javaembed "JAVAEMBED")

cmake_modules/plugins.cmake

@@ -41,6 +41,7 @@ set(PLUGINS_LIST
     SQLITE3EMBED
     SQS
     V8EMBED
+    WASMEMBED
     EXAMPLEPLUGIN
 )
 

plugins/CMakeLists.txt

@@ -30,6 +30,7 @@ add_subdirectory (proxies)
 add_subdirectory (sqlite3)
 add_subdirectory (mysql)
 add_subdirectory (v8embed)
+add_subdirectory (wasmembed)
 HPCC_ADD_SUBDIRECTORY (py3embed "USE_PYTHON3")
 HPCC_ADD_SUBDIRECTORY (pyembed "USE_PYTHON2")
 add_subdirectory (javaembed)

plugins/wasmembed/CMakeLists.txt

@@ -0,0 +1,48 @@
+project(wasmembed)
+
+if(WASMEMBED)
+    ADD_PLUGIN(wasmembed)
+    if(MAKE_WASMEMBED)
+
+        add_subdirectory(secure-enclave)
+
+        include_directories(
+            ./../../common/thorhelper
+            ./../../dali/base
+            ./../../rtl/eclrtl
+            ./../../rtl/include
+            ./../../rtl/nbcd
+            ./../../system/include
+            ./../../system/jlib
+            ./../../system/mp
+        )
+
+        add_definitions(-D_USRDLL -DWASMEMBED_EXPORTS)
+
+        add_library(wasmembed SHARED 
+            wasmembed.cpp
+        )
+
+        target_link_libraries(wasmembed
+            roxiemem
+            eclrtl
+            jlib
+            secure-enclave
+        )
+
+        install(
+            TARGETS wasmembed 
+            DESTINATION plugins
+        )
+
+    else()
+        message(WARNING "Cannot build wasmembed plugin")
+    endif()
+endif()
+
+if(PLATFORM OR CLIENTTOOLS_ONLY)
+    install(
+        FILES ${CMAKE_CURRENT_SOURCE_DIR}/wasm.ecllib
+        DESTINATION plugins
+        COMPONENT Runtime)
+endif()

plugins/wasmembed/secure-enclave/CMakeLists.txt

@@ -0,0 +1,41 @@
+project(secure-enclave)
+
+set(CMAKE_CXX_STANDARD 20)
+
+find_path(WASMTIME_CPP_API_INCLUDE_DIRS "wasmtime-cpp-api/wasmtime.hh"
+    PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+)
+if (WIN32)
+    find_library(WASMTIME_LIB NAMES wasmtime.dll
+        PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+    )
+else()
+    find_library(WASMTIME_LIB NAMES wasmtime
+        PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+    )
+endif()
+
+include_directories(
+    ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-c-api
+    ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-cpp-api
+    ./../../../system/include
+    ./../../../rtl/eclrtl
+    ./../../../system/jlib
+)
+
+add_definitions(-D_USRDLL -DSECUREENCLAVE_EXPORTS)
+
+add_library(secure-enclave SHARED
+    secure-enclave.cpp
+)
+
+target_link_libraries(secure-enclave PRIVATE
+    ${WASMTIME_LIB}
+)
+
+install(
+    TARGETS secure-enclave 
+    DESTINATION plugins
+    CALC_DEPS
+)
+

plugins/wasmembed/secure-enclave/abi.hpp

@@ -0,0 +1,157 @@
+/*
+  See:  https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md
+        https://github.com/WebAssembly/component-model/blob/main/design/mvp/canonical-abi/definitions.py
+*/
+
+#include <wasmtime.hh>
+#include <cmath>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+
+auto UTF16_TAG = 1 << 31;
+
+int align_to(int ptr, int alignment)
+{
+    return std::ceil(ptr / alignment) * alignment;
+}
+
+//  loading ---
+
+int load_int(const wasmtime::Span<uint8_t> &data, int32_t ptr, int32_t nbytes, bool is_signed = false)
+{
+    int result = 0;
+    for (int i = 0; i < nbytes; i++)
+    {
+        int b = data[ptr + i];
+        if (i == 3 && is_signed && b >= 0x80)
+        {
+            b -= 0x100;
+        }
+        result += b << (i * 8);
+    }
+    return result;
+}
+
+std::string global_encoding = "utf8";
+std::string load_string_from_range(const wasmtime::Span<uint8_t> &data, uint32_t ptr, uint32_t tagged_code_units)
+{
+    std::string encoding = "utf-8";
+    uint32_t byte_length = tagged_code_units;
+    uint32_t alignment = 1;
+    if (global_encoding.compare("utf8") == 0)
+    {
+        alignment = 1;
+        byte_length = tagged_code_units;
+        encoding = "utf-8";
+    }
+    else if (global_encoding.compare("utf16") == 0)
+    {
+        alignment = 2;
+        byte_length = 2 * tagged_code_units;
+        encoding = "utf-16-le";
+    }
+    else if (global_encoding.compare("latin1+utf16") == 0)
+    {
+        alignment = 2;
+        if (tagged_code_units & UTF16_TAG)
+        {
+            byte_length = 2 * (tagged_code_units ^ UTF16_TAG);
+            encoding = "utf-16-le";
+        }
+        else
+        {
+            byte_length = tagged_code_units;
+            encoding = "latin-1";
+        }
+    }
+
+    if (ptr != align_to(ptr, alignment))
+    {
+        throw std::runtime_error("Invalid alignment");
+    }
+    if (ptr + byte_length > data.size())
+    {
+        throw std::runtime_error("Out of bounds");
+    }
+
+    std::string s;
+    s.resize(byte_length);
+    memcpy(&s[0], &data[ptr], byte_length);
+    return s;
+}
+
+std::string load_string(const wasmtime::Span<uint8_t> &data, uint32_t ptr)
+{
+    uint32_t begin = load_int(data, ptr, 4);
+    uint32_t tagged_code_units = load_int(data, ptr + 4, 4);
+    return load_string_from_range(data, begin, tagged_code_units);
+}
+
+//  Storing  ---
+void store_int(const wasmtime::Span<uint8_t> &data, int64_t v, size_t ptr, size_t nbytes, bool _signed = false)
+{
+    // convert v to little-endian byte array
+    std::vector<uint8_t> bytes(nbytes);
+    for (size_t i = 0; i < nbytes; i++)
+    {
+        bytes[i] = (v >> (i * 8)) & 0xFF;
+    }
+    // copy bytes to memory
+    memcpy(&data[ptr], bytes.data(), nbytes);
+}
+
+//  Other  ---
+std::vector<uint8_t> read_wasm_binary_to_buffer(const std::string &filename)
+{
+    std::ifstream file(filename, std::ios::binary | std::ios::ate);
+    if (!file)
+    {
+        throw std::runtime_error("Failed to open file");
+    }
+
+    std::streamsize size = file.tellg();
+    file.seekg(0, std::ios::beg);
+
+    std::vector<uint8_t> buffer(size);
+    if (!file.read(reinterpret_cast<char *>(buffer.data()), size))
+    {
+        throw std::runtime_error("Failed to read file");
+    }
+
+    return buffer;
+}
+
+std::string extractContentInDoubleQuotes(const std::string &input)
+{
+
+    int firstQuote = input.find_first_of('"');
+    int secondQuote = input.find('"', firstQuote + 1);
+    if (firstQuote == std::string::npos || secondQuote == std::string::npos)
+    {
+        return "";
+    }
+    return input.substr(firstQuote + 1, secondQuote - firstQuote - 1);
+}
+
+std::pair<std::string, std::string> splitQualifiedID(const std::string &qualifiedName)
+{
+    std::istringstream iss(qualifiedName);
+    std::vector<std::string> tokens;
+    std::string token;
+
+    while (std::getline(iss, token, '.'))
+    {
+        tokens.push_back(token);
+    }
+    if (tokens.size() != 2)
+    {
+        throw std::runtime_error("Invalid import function " + qualifiedName + ", expected format: <module>.<function>");
+    }
+    return std::make_pair(tokens[0], tokens[1]);
+}
+
+std::string createQualifiedID(const std::string &wasmName, const std::string &funcName)
+{
+    return wasmName + "." + funcName;
+}

plugins/wasmembed/secure-enclave/hpcc-platform.wit

@@ -0,0 +1,11 @@
+package hpcc-systems:hpcc-platform
+
+world wasmembed {
+  //  guests dispose all params as needed
+  //  guests should dispose "results" as needed
+  import dbglog: func(msg: string)
+
+  //  guests dispose all params as needed
+  //  hosts call cabi_post_XXX to dispose "results" as needed
+  //  export myfunc(params: string) -> string
+}