Evaluate file implemented by reading external jsonnet file at runtime…

… through cli (#337)

* Rename crate arakoo to arakoo-jsonnet

* Use arakoo-jsonnet in wasm binding

* Implemented Evaluate File - By reading jsonnet file at runtime

* Improve file reading functionality

JS/jsonnet/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "arakoo"
+name = "arakoo-jsonnet"
 edition.workspace = true
 version.workspace = true
 

JS/jsonnet/src/jsonnet.js

@@ -67,6 +67,10 @@ if (!isArakoo) {
             let vars = JSON.stringify(this.vars);
             return __jsonnet_evaluate_snippet(vars, snippet);
         }
+        evaluateFile(filename){
+            let vars = JSON.stringify(this.vars);
+            return __jsonnet_evaluate_file(vars, filename);
+        }
 
         destroy() {}
     };

JS/jsonnet/src/jsonnet_wasm_bg.js

@@ -1,4 +1,4 @@
-import { read_file } from "./snippets/arakoo-07ad5af4ed8e3fe0/read-file.js";
+import { read_file } from './snippets/arakoo-jsonnet-17c737407ebd2d3c/read-file.js';
 
 let wasm;
 export function __wbg_set_wasm(val) {
@@ -334,15 +334,13 @@ export function __wbindgen_string_new(arg0, arg1) {
     return addHeapObject(ret);
 }
 
-export function __wbg_readfile_3df9f1d22ad880df() {
-    return handleError(function (arg0, arg1, arg2) {
-        const ret = read_file(getStringFromWasm0(arg1, arg2));
-        const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len1 = WASM_VECTOR_LEN;
-        getInt32Memory0()[arg0 / 4 + 1] = len1;
-        getInt32Memory0()[arg0 / 4 + 0] = ptr1;
-    }, arguments);
-}
+export function __wbg_readfile_5b48d0f7e3518df2() { return handleError(function (arg0, arg1, arg2) {
+    const ret = read_file(getStringFromWasm0(arg1, arg2));
+    const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len1 = WASM_VECTOR_LEN;
+    getInt32Memory0()[arg0 / 4 + 1] = len1;
+    getInt32Memory0()[arg0 / 4 + 0] = ptr1;
+}, arguments) };
 
 export function __wbindgen_object_clone_ref(arg0) {
     const ret = getObject(arg0);

JS/jsonnet/src/lib.rs

@@ -130,16 +130,8 @@ pub fn ext_string(vm: *mut VM, key: &str, value: &str) {
     // let context_initializer_ref = vm.state.context_initializer();
 
     // Dereference the Ref to access the trait object
-    let context_initializer = &*vm.state.context_initializer();
-    println!("{:?}", context_initializer.as_any().type_id());
-
     let context_initializer = vm.state.context_initializer();
 
-    println!(
-        "Type of context initializer: {:?}",
-        std::any::type_name_of_val(&*context_initializer)
-    );
-
     context_initializer
         .as_any()
         .downcast_ref::<context::ArakooContextInitializer>()

JS/wasm/crates/apis/src/jsonnet/mod.rs

@@ -1,6 +1,8 @@
 use javy::quickjs::{JSContextRef, JSValue, JSValueRef};
 
-use crate::{jsonnet_evaluate, jsonnet_output, jsonnet_output_len, JSApiSet};
+use crate::{
+    jsonnet_evaluate, jsonnet_evaluate_file, jsonnet_output, jsonnet_output_len, JSApiSet,
+};
 
 pub(super) struct Jsonnet;
 
@@ -12,11 +14,55 @@ impl JSApiSet for Jsonnet {
             "__jsonnet_evaluate_snippet",
             context.wrap_callback(jsonnet_evaluate_snippet_callback())?,
         )?;
+        global.set_property(
+            "__jsonnet_evaluate_file",
+            context.wrap_callback(jsonnet_evaluate_file_callback())?,
+        )?;
 
         Ok(())
     }
 }
 
+fn jsonnet_evaluate_file_callback(
+) -> impl FnMut(&JSContextRef, JSValueRef, &[JSValueRef]) -> anyhow::Result<JSValue> {
+    move |_ctx, _this, args| {
+        // check the number of arguments
+        if args.len() != 2 {
+            return Err(anyhow::anyhow!("Expected 2 arguments, got {}", args.len()));
+        }
+        let var = args.get(0).unwrap().to_string();
+        let path = args.get(1).unwrap().to_string();
+        let var_len = var.len() as i32;
+        let path_len = path.len() as i32;
+        let var_ptr = var.as_ptr();
+        let path_ptr = path.as_ptr();
+
+        unsafe { jsonnet_evaluate_file(var_ptr, var_len, path_ptr, path_len) }
+        let out_len = unsafe { jsonnet_output_len() };
+
+        let mut out_buffer = Vec::with_capacity(out_len as usize);
+        let out_ptr = out_buffer.as_mut_ptr();
+        let out_buffer = unsafe {
+            jsonnet_output(out_ptr);
+            Vec::from_raw_parts(out_ptr, out_len as usize, out_len as usize)
+        };
+        // println!("out_buffer: {}", String::from_utf8(out_buffer.clone()).expect("unable to convert to string"));
+
+        let jsonnet_output: serde_json::Value = match serde_json::from_slice(&out_buffer) {
+            Ok(output) => output,
+            Err(e) => {
+                eprintln!("Failed to parse jsonnet output: {}", e);
+                return Err(anyhow::anyhow!(
+                    "Failed to parse jsonnet output: {}",
+                    e.to_string()
+                ));
+            }
+        };
+        let jsonnet_output = jsonnet_output.to_string();
+        Ok(jsonnet_output.into())
+    }
+}
+
 fn jsonnet_evaluate_snippet_callback(
 ) -> impl FnMut(&JSContextRef, JSValueRef, &[JSValueRef]) -> anyhow::Result<JSValue> {
     move |_ctx, _this, args| {

JS/wasm/crates/apis/src/lib.rs

@@ -67,6 +67,7 @@ mod jsonnet;
 #[link(wasm_import_module = "arakoo")]
 extern "C" {
     fn jsonnet_evaluate(var_ptr: *const u8, var_len: i32, code_ptr: *const u8, code_len: i32);
+    fn jsonnet_evaluate_file(var_ptr: *const u8, var_len: i32, path_ptr: *const u8, path_len: i32);
     fn jsonnet_output_len() -> i32;
     fn jsonnet_output(ptr: *mut u8);
     fn fetch(request_pointer: *const u8, request_len: i32);

JS/wasm/crates/serve/Cargo.toml

@@ -24,3 +24,4 @@ jrsonnet-evaluator = { version = "0.5.0-pre95" }
 jrsonnet-parser = { version = "0.5.0-pre95" }
 jrsonnet-stdlib = { version = "0.5.0-pre95" }
 reqwest = { version = "0.11", features = ["json"] }
+arakoo-jsonnet = {path = "../../../jsonnet"}

JS/wasm/crates/serve/src/binding.rs

@@ -1,30 +1,27 @@
-use std::sync::{Arc, Mutex};
+use std::{
+    env,
+    sync::{Arc, Mutex},
+};
 
-use jrsonnet_evaluator::{
-    apply_tla,
-    function::TlaArg,
-    gc::GcHashMap,
-    manifest::{JsonFormat, ManifestFormat},
-    tb,
-    trace::{CompactFormat, PathResolver, TraceFormat},
-    FileImportResolver, State, Val,
+use arakoo_jsonnet::{
+    ext_string, jsonnet_destroy, jsonnet_evaluate_file, jsonnet_evaluate_snippet, jsonnet_make,
 };
-use jrsonnet_parser::IStr;
-use jrsonnet_stdlib::ContextInitializer;
 
+use std::{fs, io};
 use tokio::runtime::Builder;
 use tracing::error;
 use wasi_common::WasiCtx;
 use wasmtime::*;
 
 use crate::io::{WasmInput, WasmOutput};
 
-pub struct VM {
-    state: State,
-    manifest_format: Box<dyn ManifestFormat>,
-    trace_format: Box<dyn TraceFormat>,
-    tla_args: GcHashMap<IStr, TlaArg>,
-}
+// pub struct VM {
+//     state: State,
+//     manifest_format: Box<dyn ManifestFormat>,
+//     trace_format: Box<dyn TraceFormat>,
+//     tla_args: GcHashMap<IStr, TlaArg>,
+// }
+
 /// Adds exported functions to the Wasm linker.
 ///
 /// This function wraps the `jsonnet_evaluate`, `jsonnet_output_len`, and `jsonnet_output`
@@ -34,7 +31,7 @@ pub struct VM {
 pub fn add_jsonnet_to_linker(linker: &mut Linker<WasiCtx>) -> anyhow::Result<()> {
     // Create a shared output buffer that will be used to store the result of the Jsonnet evaluation.
     let output: Arc<Mutex<String>> = Arc::new(Mutex::new(String::new()));
-    let output_clone = output.clone();
+    let mut output_clone = output.clone();
 
     // Wrap the `jsonnet_evaluate` function to be called from WebAssembly.
     linker.func_wrap(
@@ -85,47 +82,91 @@ pub fn add_jsonnet_to_linker(linker: &mut Linker<WasiCtx>) -> anyhow::Result<()>
             };
 
             // Initialize the Jsonnet VM state with default settings.
-            let state = State::default();
-            state.settings_mut().import_resolver = tb!(FileImportResolver::default());
-            state.settings_mut().context_initializer = tb!(ContextInitializer::new(
-                state.clone(),
-                PathResolver::new_cwd_fallback(),
-            ));
-            // Create the Jsonnet VM with the default settings.
-            let vm = VM {
-                state,
-                manifest_format: Box::new(JsonFormat::default()),
-                trace_format: Box::new(CompactFormat::default()),
-                tla_args: GcHashMap::default(),
-            };
+            let vm = jsonnet_make();
 
             // Evaluate the Jsonnet code snippet using the provided path and variables.
             let code = path;
-            let any_initializer = vm.state.context_initializer();
-            let context = any_initializer
-                .as_any()
-                .downcast_ref::<ContextInitializer>()
-                .unwrap();
             for (key, value) in var_json.as_object().unwrap() {
-                context.add_ext_var(key.into(), Val::Str(value.as_str().unwrap().into()));
+                // context.add_ext_var(key.into(), Val::Str(value.as_str().unwrap().into()));
+                ext_string(
+                    vm,
+                    key,
+                    value.as_str().expect("ext_string value is not a string"),
+                );
             }
-            let out = match vm
-                .state
-                .evaluate_snippet("snippet", code)
-                .and_then(|val| apply_tla(vm.state.clone(), &vm.tla_args, val))
-                .and_then(|val| val.manifest(&vm.manifest_format))
-            {
+            let out = jsonnet_evaluate_snippet(vm, "deleteme", code);
+            // Store the output of the Jsonnet evaluation in the shared output buffer.
+            let mut output = output.lock().unwrap();
+            *output = out;
+            jsonnet_destroy(vm);
+            Ok(())
+        },
+    )?;
+
+    output_clone = output.clone();
+    // Wrap the `jsonnet_evaluate_file` function to be called from WebAssembly.
+    linker.func_wrap(
+        "arakoo",
+        "jsonnet_evaluate_file",
+        move |mut caller: Caller<'_, WasiCtx>,
+              var_ptr: i32,
+              var_len: i32,
+              path_ptr: i32,
+              code_len: i32| {
+            // Clone the output buffer for use within the closure.
+            let output = output_clone.clone();
+            // Get the WebAssembly memory instance.
+            let mem = match caller.get_export("memory") {
+                Some(Extern::Memory(mem)) => mem,
+                _ => return Err(Trap::NullReference.into()),
+            };
+            // Calculate the offsets for the variable and path buffers in WebAssembly memory.
+            let var_offset = var_ptr as u32 as usize;
+            let path_offset = path_ptr as u32 as usize;
+            // Create buffers to read the variable and path data from WebAssembly memory.
+            let mut var_buffer = vec![0; var_len as usize];
+            let mut path_buffer = vec![0; code_len as usize];
+
+            // Read the path data from WebAssembly memory and convert it to a string.
+            let path = match mem.read(&caller, path_offset, &mut path_buffer) {
+                Ok(_) => match std::str::from_utf8(&path_buffer) {
+                    Ok(s) => s,
+                    Err(_) => return Err(Trap::BadSignature.into()),
+                },
+                _ => return Err(Trap::MemoryOutOfBounds.into()),
+            };
+            // Read the variable data from WebAssembly memory and convert it to a string.
+            let var = match mem.read(&caller, var_offset, &mut var_buffer) {
+                Ok(_) => match std::str::from_utf8(&var_buffer) {
+                    Ok(s) => s,
+                    Err(_) => return Err(Trap::BadSignature.into()),
+                },
+                _ => return Err(Trap::MemoryOutOfBounds.into()),
+            };
+            // Parse the variable data as JSON.
+            let var_json: serde_json::Value = match serde_json::from_str(var) {
                 Ok(v) => v,
                 Err(e) => {
-                    error!("Error evaluating snippet: {}", e);
-                    let mut out = String::new();
-                    vm.trace_format.write_trace(&mut out, &e).unwrap();
-                    out
+                    error!("Error parsing var: {}", e);
+                    return Err(Trap::BadSignature.into());
                 }
             };
-            // Store the output of the Jsonnet evaluation in the shared output buffer.
-            let mut output = output.lock().unwrap();
+            // println!("var_json: {:?}", var_json);
+
+            let vm = jsonnet_make();
+
+            for (key, value) in var_json.as_object().unwrap() {
+                ext_string(
+                    vm,
+                    key,
+                    value.as_str().expect("ext_string value is not a string"),
+                );
+            }
+            let code = fs::read_to_string(path).expect("File not found");
+            let out = jsonnet_evaluate_snippet(vm, "deleteme", &code);
+            let mut output: std::sync::MutexGuard<'_, String> = output.lock().unwrap();
             *output = out;
+
             Ok(())
         },
     )?;

JS/wasm/examples/ec-wasmjs-hono/build.js

@@ -1,16 +1,35 @@
 import { build } from "esbuild";
-
+import fs from "fs";
 let runtime = process.argv[2];
 
+// let jsonnetLoadPlugin = {
+//     name: "jsonnet-load",
+//     setup(build) {
+//         build.onLoad({ filter: /\.jsonnet$/ }, async (args) => {
+//             let code = await fs.promises.readFile(args.path, "utf-8");
+//             return {
+//                 contents: code,
+//                 loader: "text",
+//             };
+//         })
+//     }
+// }
+
 build({
-    entryPoints: ["src/index.js"],
+    entryPoints: ["src/index.js", "src/example.jsonnet"],
+    // entryPoints: ["src/index.js"],
     bundle: true,
-    minify: true,
-    outfile: "bin/app.js",
+    // minify: true,
+    minifySyntax: true,
+    // outfile: "bin/app.js",
+    outdir: "bin",
     format: "esm",
     target: "esnext",
     platform: "node",
     // external: ["arakoo"],
+    loader:{
+        ".jsonnet":"copy"
+    },
     define: {
         "process.env.arakoo": JSON.stringify(runtime === "arakoo"),
     },

JS/wasm/examples/ec-wasmjs-hono/package.json

@@ -14,7 +14,7 @@
         "@hono/node-server": "^1.3.1",
         "axios": "^1.6.2",
         "crypto": "^1.0.1",
-        "@arakoodev/jsonnet": "0.1.2",
+        "@arakoodev/jsonnet": "file:../../../jsonnet/",
         "http": "^0.0.1-security",
         "stream": "^0.0.2"
     }

JS/wasm/examples/ec-wasmjs-hono/src/example.jsonnet

@@ -0,0 +1,11 @@
+local name = 'bob';
+{
+  p1: {
+    name: std.extVar('extName'),
+    age: 20,
+  },
+  p2: {
+    name: name,
+    age: 30,
+  },
+}