Add a test to keep server_api json.go in sync with parse_inputs.rs

arbitrator/prover/src/lib.rs

@@ -33,9 +33,12 @@ use machine::{
     PreimageResolver,
 };
 use once_cell::sync::OnceCell;
+use parse_input::FileData;
 use static_assertions::const_assert_eq;
 use std::{
     ffi::CStr,
+    fs::File,
+    io::BufReader,
     num::NonZeroUsize,
     os::raw::{c_char, c_int},
     path::Path,
@@ -84,6 +87,39 @@ pub unsafe extern "C" fn arbitrator_load_machine(
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn arbitrator_deserialize_and_serialize_file_data(
+    read_path: *const c_char,
+    write_path: *const c_char,
+) -> c_int {
+    let read_path = cstr_to_string(read_path);
+    let write_path = cstr_to_string(write_path);
+
+    let file = File::open(read_path);
+    let reader = match file {
+        Ok(file) => BufReader::new(file),
+        Err(err) => {
+            eprintln!("Failed to open read_path of FileData: {}", err);
+            return 1;
+        }
+    };
+    let data = match FileData::from_reader(reader) {
+        Ok(data) => data,
+        Err(err) => {
+            eprintln!("Failed to deserialize FileData: {}", err);
+            return 2;
+        }
+    };
+
+    match data.write_to_file(&write_path) {
+        Ok(()) => 0,
+        Err(err) => {
+            eprintln!("Failed to serialize FileData: {}", err);
+            3
+        }
+    }
+}
+
 unsafe fn arbitrator_load_machine_impl(
     binary_path: *const c_char,
     library_paths: *const *const c_char,

arbitrator/prover/src/parse_input.rs

@@ -1,12 +1,14 @@
 use arbutil::Bytes32;
 use serde::Deserialize;
+use serde::Serialize;
 use serde_json;
 use serde_with::base64::Base64;
 use serde_with::As;
 use serde_with::DisplayFromStr;
 use std::{
     collections::HashMap,
-    io::{self, BufRead},
+    fs::File,
+    io::{self, BufRead, BufWriter},
 };
 
 /// prefixed_hex deserializes hex strings which are prefixed with `0x`
@@ -16,7 +18,7 @@ use std::{
 /// It is an error to use this deserializer on a string that does not
 /// begin with `0x`.
 mod prefixed_hex {
-    use serde::{self, Deserialize, Deserializer};
+    use serde::{self, Deserialize, Deserializer, Serializer};
 
     pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
     where
@@ -29,6 +31,14 @@ mod prefixed_hex {
             Err(serde::de::Error::custom("missing 0x prefix"))
         }
     }
+
+    pub fn serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let hex_string = format!("0x{}", hex::encode(bytes));
+        serializer.serialize_str(&hex_string)
+    }
 }
 
 #[derive(Debug)]
@@ -61,15 +71,15 @@ impl From<Vec<u8>> for UserWasm {
     }
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(rename_all = "PascalCase")]
 pub struct BatchInfo {
     pub number: u64,
     #[serde(with = "As::<Base64>")]
     pub data_b64: Vec<u8>,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Serialize)]
 #[serde(rename_all = "PascalCase")]
 pub struct StartState {
     #[serde(with = "prefixed_hex")]
@@ -88,7 +98,7 @@ pub struct StartState {
 ///
 /// Note: It is important to change this file whenever the go JSON
 /// serialization changes.
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Serialize)]
 #[serde(rename_all = "PascalCase")]
 pub struct FileData {
     pub id: u64,
@@ -109,4 +119,11 @@ impl FileData {
         let data = serde_json::from_reader(&mut reader)?;
         Ok(data)
     }
+
+    pub fn write_to_file(&self, file_path: &str) -> io::Result<()> {
+        let file = File::create(file_path)?;
+        let writer = BufWriter::new(file);
+        serde_json::to_writer_pretty(writer, &self)?;
+        Ok(())
+    }
 }

system_tests/validationinputjson_rustfiledata_test.go

@@ -0,0 +1,120 @@
+package arbtest
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+
+	"github.com/offchainlabs/nitro/arbutil"
+	"github.com/offchainlabs/nitro/validator"
+	"github.com/offchainlabs/nitro/validator/server_api"
+	"github.com/offchainlabs/nitro/validator/server_arb"
+)
+
+func getWriteDataFromRustSide(t *testing.T, inputJSON *server_api.InputJSON) []byte {
+	t.Helper()
+	dir := t.TempDir()
+
+	readData, err := inputJSON.Marshal()
+	Require(t, err)
+	readPath := filepath.Join(dir, fmt.Sprintf("block_inputs_%d_read.json", inputJSON.Id))
+	Require(t, os.WriteFile(readPath, readData, 0600))
+
+	writePath := filepath.Join(dir, fmt.Sprintf("block_inputs_%d_write.json", inputJSON.Id))
+	Require(t, server_arb.DeserializeAndSerializeFileData(readPath, writePath))
+	writeData, err := os.ReadFile(writePath)
+	Require(t, err)
+
+	return writeData
+}
+
+func TestGoInputJSONRustFileDataRoundtripWithoutUserWasms(t *testing.T) {
+	preimages := make(map[arbutil.PreimageType]map[common.Hash][]byte)
+	preimages[arbutil.Keccak256PreimageType] = make(map[common.Hash][]byte)
+	preimages[arbutil.Keccak256PreimageType][common.MaxHash] = []byte{1}
+
+	// Don't include DebugChain as it isn't used on rust side
+	sampleValidationInput := &validator.ValidationInput{
+		Id:            1,
+		HasDelayedMsg: true,
+		DelayedMsgNr:  2,
+		Preimages:     preimages,
+		BatchInfo:     []validator.BatchInfo{{Number: 3}},
+		DelayedMsg:    []byte{4},
+		StartState: validator.GoGlobalState{
+			BlockHash:  common.MaxHash,
+			SendRoot:   common.MaxHash,
+			Batch:      5,
+			PosInBatch: 6,
+		},
+	}
+	sampleValidationInputJSON := server_api.ValidationInputToJson(sampleValidationInput)
+	writeData := getWriteDataFromRustSide(t, sampleValidationInputJSON)
+
+	var resWithoutUserWasms server_api.InputJSON
+	Require(t, json.Unmarshal(writeData, &resWithoutUserWasms))
+	if !reflect.DeepEqual(*sampleValidationInputJSON, resWithoutUserWasms) {
+		t.Fatal("ValidationInputJSON without UserWasms, mismatch on rust and go side")
+	}
+
+}
+
+type inputJSONWithUserWasmsOnly struct {
+	UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte
+}
+
+// UnmarshalJSON is a custom function defined to encapsulate how UserWasms are handled on the rust side.
+// When ValidationInputToJson is called on ValidationInput, it compresses the wasm data byte array and
+// then encodes this to a base64 string, this when deserialized on the rust side through FileData- the
+// compressed data is first uncompressed and also the module hash (Bytes32) is read without the 0x prefix,
+// so we define a custom UnmarshalJSON to extract UserWasms map from the data written by rust side.
+func (u *inputJSONWithUserWasmsOnly) UnmarshalJSON(data []byte) error {
+	type rawUserWasms struct {
+		UserWasms map[ethdb.WasmTarget]map[string]string
+	}
+	var rawUWs rawUserWasms
+	if err := json.Unmarshal(data, &rawUWs); err != nil {
+		return err
+	}
+	tmp := make(map[ethdb.WasmTarget]map[common.Hash][]byte)
+	for wasmTarget, innerMap := range rawUWs.UserWasms {
+		tmp[wasmTarget] = make(map[common.Hash][]byte)
+		for hashKey, value := range innerMap {
+			valBytes, err := base64.StdEncoding.DecodeString(value)
+			if err != nil {
+				return err
+			}
+			tmp[wasmTarget][common.HexToHash("0x"+hashKey)] = valBytes
+		}
+	}
+	u.UserWasms = tmp
+	return nil
+}
+
+func TestGoInputJSONRustFileDataRoundtripWithUserWasms(t *testing.T) {
+	userWasms := make(map[ethdb.WasmTarget]map[common.Hash][]byte)
+	userWasms["arch1"] = make(map[common.Hash][]byte)
+	userWasms["arch1"][common.MaxHash] = []byte{2}
+
+	// Don't include DebugChain as it isn't used on rust side
+	sampleValidationInput := &validator.ValidationInput{
+		Id:        1,
+		UserWasms: userWasms,
+		BatchInfo: []validator.BatchInfo{{Number: 1}}, // This needs to be set for FileData to successfully deserialize, else it errors for invalid type null
+	}
+	sampleValidationInputJSON := server_api.ValidationInputToJson(sampleValidationInput)
+	writeData := getWriteDataFromRustSide(t, sampleValidationInputJSON)
+
+	var resUserWasmsOnly inputJSONWithUserWasmsOnly
+	Require(t, json.Unmarshal(writeData, &resUserWasmsOnly))
+	if !reflect.DeepEqual(userWasms, resUserWasmsOnly.UserWasms) {
+		t.Fatal("ValidationInputJSON with UserWasms only, mismatch on rust and go side")
+	}
+}

validator/server_arb/machine.go

@@ -312,6 +312,20 @@ func (m *ArbitratorMachine) DeserializeAndReplaceState(path string) error {
 	}
 }
 
+func DeserializeAndSerializeFileData(readPath, writePath string) error {
+	cReadPath := C.CString(readPath)
+	cWritePath := C.CString(writePath)
+	status := C.arbitrator_deserialize_and_serialize_file_data(cReadPath, cWritePath)
+	C.free(unsafe.Pointer(cReadPath))
+	C.free(unsafe.Pointer(cWritePath))
+
+	if status != 0 {
+		return fmt.Errorf("failed to call arbitrator_deserialize_and_serialize_file_data. Error code: %d", status)
+	} else {
+		return nil
+	}
+}
+
 func (m *ArbitratorMachine) AddSequencerInboxMessage(index uint64, data []byte) error {
 	defer runtime.KeepAlive(m)
 	m.mutex.Lock()