[move] Initial trace format and implementation (#18729) (#19858)

This PR adds the initial trace format, and the implementation of this
trace format in the VM. I've gone through the tracing format with most
of you synchronously so won't write out all the details here (but I will
take it on to write a spec for the trace format once this is done so
other folks can use it when consuming this format). Happy to go through
it at any point in real time.

Other TODO: right now the `MoveValue`s only support serialize and not
deserialize back into Rust so we can only push a trace out from Rust but
not consume it. The next thing I'm working on right now is adding
support for that, and it may be either an update to this PR, or an
additional PR depending on how involved that is...

Tests and integration of this into the Move CLI (not the Sui CLI yet) is
in the PR above this one.

This keeps the tracing largely feature-gated in the VM, and the only
additional overhead/change at runtime with `gas-profiling` turned off is
the additional argument, but this argument is unused and should be
optimized away (and at worst only add essentially no overhead).

I kept the `gas-profiling` feature flag and gated the new tracing under
it. The plan being to eventually rename that flag at the same time that
we update test coverage and gas profiling to use this new tracing format
as well (but that's for a couple future PRs!).


#19452 is stacked on top of this
and cleans up the insertion points of the tracer into the VM.

---

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol:
- [ ] Nodes (Validators and Full nodes):
- [ ] Indexer:
- [ ] JSON-RPC:
- [ ] GraphQL:
- [ ] CLI:
- [ ] Rust SDK:
- [ ] REST API:

## Description 

Describe the changes or additions included in this PR.

## Test plan 

How did you test the new or updated feature?

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] Indexer: 
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:
- [ ] REST API:

crates/sui-types/Cargo.toml

@@ -103,7 +103,6 @@ default = []
 test-utils = []
 gas-profiler = [
     "move-vm-profiler/gas-profiler",
-    "move-vm-types/gas-profiler",
     "move-vm-test-utils/gas-profiler",
 ]
 fuzzing = ["move-core-types/fuzzing"]

external-crates/move/Cargo.toml

@@ -140,6 +140,7 @@ module-generation = { path = "crates/module-generation" }
 move-abstract-interpreter = { path = "crates/move-abstract-interpreter" }
 move-abstract-stack = { path = "crates/move-abstract-stack" }
 move-binary-format = { path = "crates/move-binary-format" }
+move-trace-format = { path = "crates/move-trace-format" }
 move-borrow-graph = { path = "crates/move-borrow-graph" }
 move-bytecode-source-map = { path = "crates/move-bytecode-source-map" }
 move-bytecode-utils = { path = "crates/move-bytecode-utils" }

external-crates/move/crates/move-binary-format/src/file_format_common.rs

@@ -529,11 +529,10 @@ pub const VERSION_MAX: u32 = VERSION_7;
 // TODO(#145): finish v4 compatibility; as of now, only metadata is implemented
 pub const VERSION_MIN: u32 = VERSION_5;
 
-/// The encoding of the instruction is the serialized form of it, but disregarding the
-/// serialization of the instruction's argument(s).
-pub fn instruction_key(instruction: &Bytecode) -> u8 {
+/// The corresponding opcode for each bytecode (disregards the argument).
+pub fn instruction_opcode(instruction: &Bytecode) -> Opcodes {
     use Bytecode::*;
-    let opcode = match instruction {
+    match instruction {
         Pop => Opcodes::POP,
         Ret => Opcodes::RET,
         BrTrue(_) => Opcodes::BR_TRUE,
@@ -621,6 +620,11 @@ pub fn instruction_key(instruction: &Bytecode) -> u8 {
         MutBorrowGlobalGenericDeprecated(_) => Opcodes::MUT_BORROW_GLOBAL_GENERIC_DEPRECATED,
         ImmBorrowGlobalDeprecated(_) => Opcodes::IMM_BORROW_GLOBAL_DEPRECATED,
         ImmBorrowGlobalGenericDeprecated(_) => Opcodes::IMM_BORROW_GLOBAL_GENERIC_DEPRECATED,
-    };
-    opcode as u8
+    }
+}
+
+/// The encoding of the instruction is the serialized form of it, but disregarding the
+/// serialization of the instruction's argument(s).
+pub fn instruction_key(instruction: &Bytecode) -> u8 {
+    instruction_opcode(instruction) as u8
 }

external-crates/move/crates/move-bytecode-source-map/Cargo.toml

@@ -17,6 +17,7 @@ move-command-line-common.workspace = true
 bcs.workspace = true
 
 serde.workspace = true
+serde_json.workspace = true
 
 [features]
 default = []

external-crates/move/crates/move-bytecode-source-map/src/utils.rs

@@ -5,12 +5,19 @@
 use crate::source_map::SourceMap;
 use anyhow::{format_err, Result};
 use move_ir_types::location::Loc;
-use std::{fs::File, io::Read, path::Path};
+use std::{
+    fs::File,
+    io::{Read, Write},
+    path::Path,
+};
 
 pub type Error = (Loc, String);
 pub type Errors = Vec<Error>;
 
 pub fn source_map_from_file(file_path: &Path) -> Result<SourceMap> {
+    if file_path.extension().is_some_and(|ext| ext == "json") {
+        return deserialize_from_json(file_path);
+    }
     let mut bytes = Vec::new();
     File::open(file_path)
         .ok()
@@ -19,3 +26,31 @@ pub fn source_map_from_file(file_path: &Path) -> Result<SourceMap> {
     bcs::from_bytes::<SourceMap>(&bytes)
         .map_err(|_| format_err!("Error deserializing into source map"))
 }
+
+pub fn serialize_to_json(map: &SourceMap) -> Result<Vec<u8>> {
+    serde_json::to_vec(map).map_err(|e| format_err!("Error serializing to json: {}", e))
+}
+
+pub fn serialize_to_json_file(map: &SourceMap, file_path: &Path) -> Result<()> {
+    let json = serde_json::to_string_pretty(map)
+        .map_err(|e| format_err!("Error serializing to json: {}", e))?;
+    let mut f =
+        std::fs::File::create(file_path).map_err(|e| format_err!("Error creating file: {}", e))?;
+    f.write_all(json.as_bytes())
+        .map_err(|e| format_err!("Error writing to file: {}", e))?;
+    Ok(())
+}
+
+pub fn deserialize_from_json(file_path: &Path) -> Result<SourceMap> {
+    let mut file = File::open(file_path).map_err(|e| format_err!("Error opening file: {}", e))?;
+    let mut json = String::new();
+    file.read_to_string(&mut json)
+        .map_err(|e| format_err!("Error reading file: {}", e))?;
+    serde_json::from_str(&json).map_err(|e| format_err!("Error deserializing from json: {}", e))
+}
+
+pub fn convert_to_json(file_path: &Path) -> Result<()> {
+    let map = source_map_from_file(file_path)?;
+    let json_file_path = file_path.with_extension("json");
+    serialize_to_json_file(&map, &json_file_path)
+}

external-crates/move/crates/move-cli/Cargo.toml

@@ -58,5 +58,10 @@ harness = false
 name = "build_testsuite"
 harness = false
 
+[[test]]
+name = "tracing_testsuite"
+harness = false
+
 [features]
 tiered-gas = ["move-vm-test-utils/tiered-gas"]
+gas-profiler = ["move-vm-runtime/gas-profiler"]

external-crates/move/crates/move-cli/src/base/test.rs

@@ -68,6 +68,10 @@ pub struct Test {
     /// The number of iterations to run each test that uses generated values (only used with #[random_test]).
     #[clap(name = "rand-num-iters", long = "rand-num-iters")]
     pub rand_num_iters: Option<u64>,
+
+    // Enable tracing for tests
+    #[clap(long = "trace-execution", value_name = "PATH")]
+    pub trace_execution: Option<Option<String>>,
 }
 
 impl Test {
@@ -108,6 +112,7 @@ impl Test {
             compute_coverage: _,
             seed,
             rand_num_iters,
+            trace_execution,
         } = self;
         UnitTestingConfig {
             gas_limit,
@@ -118,6 +123,7 @@ impl Test {
             verbose: verbose_mode,
             seed,
             rand_num_iters,
+            trace_execution,
             ..UnitTestingConfig::default_with_bound(None)
         }
     }

external-crates/move/crates/move-cli/tests/tracing_tests/tracing-unit-tests/Move.toml

@@ -0,0 +1,9 @@
+  [package]
+  name = "tracing_unit_tests"
+  edition = "2024.beta" # edition = "legacy" to use legacy (pre-2024) Move
+
+  [dependencies]
+  MoveStdlib = { local = "../../../../move-stdlib" }
+
+  [addresses]
+  std = "0x1"

external-crates/move/crates/move-cli/tests/tracing_tests/tracing-unit-tests/args.exp

@@ -0,0 +1,28 @@
+Command `test -t 1 --trace-execution new_traces`:
+INCLUDING DEPENDENCY MoveStdlib
+BUILDING tracing_unit_tests
+Running Move unit tests
+[ PASS    ] 0x1::calls::test_call_order
+[ PASS    ] 0x1::calls::test_call_return_order
+[ PASS    ] 0x1::calls::test_complex_nested_calls
+[ PASS    ] 0x1::calls::test_return_order
+[ PASS    ] 0x1::errors::aborter
+[ PASS    ] 0x1::errors::bad_cast
+[ PASS    ] 0x1::errors::div_0
+[ PASS    ] 0x1::errors::fail_during_abort
+[ PASS    ] 0x1::errors::overshift_l
+[ PASS    ] 0x1::errors::overshift_r
+[ PASS    ] 0x1::errors::underflow
+[ PASS    ] 0x1::natives::get_orig_type_name_test
+[ PASS    ] 0x1::natives::get_type_name_test
+[ PASS    ] 0x1::packs::test_gen_pack_order
+[ PASS    ] 0x1::packs::test_gen_unpack_order
+[ PASS    ] 0x1::packs::test_pack_order
+[ PASS    ] 0x1::packs::test_unpack_order
+[ PASS    ] 0x1::references::nested_struct_reference_mutation
+[ PASS    ] 0x1::references::pass_mut_assign_in_other_fn
+[ PASS    ] 0x1::references::test_struct_borrow
+[ PASS    ] 0x1::references::test_vector_mut_borrow
+[ PASS    ] 0x1::references::test_vector_mut_borrow_pop
+Test result: OK. Total tests: 22; passed: 22; failed: 0
+External Command `diff -qr new_traces saved_traces`:

external-crates/move/crates/move-cli/tests/tracing_tests/tracing-unit-tests/args.txt

@@ -0,0 +1,2 @@
+test -t 1 --trace-execution new_traces
+> diff -qr new_traces saved_traces

external-crates/move/crates/move-cli/tests/tracing_tests/tracing-unit-tests/new_traces/0x1__calls__test_call_order.json

@@ -0,0 +1 @@
+{"events":[{"OpenFrame":{"frame":{"binary_member_index":0,"frame_id":0,"function_name":"test_call_order","is_native":false,"locals_types":[],"module":{"address":"0000000000000000000000000000000000000000000000000000000000000001","name":"calls"},"parameters":[],"return_types":[],"type_instantiation":[]},"gas_left":1000000000}},{"Instruction":{"gas_left":999999998,"instruction":"LD_U64","pc":0,"type_parameters":[]}},{"Effect":{"Push":{"RuntimeValue":{"value":1}}}},{"Instruction":{"gas_left":999999996,"instruction":"LD_TRUE","pc":1,"type_parameters":[]}},{"Effect":{"Push":{"RuntimeValue":{"value":true}}}},{"Instruction":{"gas_left":999999994,"instruction":"LD_U8","pc":2,"type_parameters":[]}},{"Effect":{"Push":{"RuntimeValue":{"value":1}}}},{"Instruction":{"gas_left":999999994,"instruction":"CALL","pc":3,"type_parameters":[]}},{"OpenFrame":{"frame":{"binary_member_index":1,"frame_id":8,"function_name":"f_test_call_order","is_native":false,"locals_types":[{"ref_type":null,"type_":"u64"},{"ref_type":null,"type_":"bool"},{"ref_type":null,"type_":"u8"}],"module":{"address":"0000000000000000000000000000000000000000000000000000000000000001","name":"calls"},"parameters":[{"RuntimeValue":{"value":1}},{"RuntimeValue":{"value":true}},{"RuntimeValue":{"value":1}}],"return_types":[],"type_instantiation":[]},"gas_left":999999994}},{"Instruction":{"gas_left":999994823,"instruction":"RET","pc":0,"type_parameters":[]}},{"CloseFrame":{"frame_id":8,"gas_left":999994823,"return_":[]}},{"Instruction":{"gas_left":999994184,"instruction":"RET","pc":4,"type_parameters":[]}},{"CloseFrame":{"frame_id":0,"gas_left":999994184,"return_":[]}}],"version":1}

external-crates/move/crates/move-cli/tests/tracing_tests/tracing-unit-tests/new_traces/0x1__calls__test_call_return_order.json

@@ -0,0 +1 @@
+{"events":[{"OpenFrame":{"frame":{"binary_member_index":4,"frame_id":0,"function_name":"test_call_return_order","is_native":false,"locals_types":[],"module":{"address":"0000000000000000000000000000000000000000000000000000000000000001","name":"calls"},"parameters":[],"return_types":[],"type_instantiation":[]},"gas_left":1000000000}},{"Instruction":{"gas_left":1000000000,"instruction":"CALL","pc":0,"type_parameters":[]}},{"OpenFrame":{"frame":{"binary_member_index":3,"frame_id":2,"function_name":"f_test_return_order","is_native":false,"locals_types":[],"module":{"address":"0000000000000000000000000000000000000000000000000000000000000001","name":"calls"},"parameters":[],"return_types":[{"ref_type":null,"type_":"u64"},{"ref_type":null,"type_":"bool"},{"ref_type":null,"type_":"u8"}],"type_instantiation":[]},"gas_left":1000000000}},{"Instruction":{"gas_left":999998865,"instruction":"LD_U64","pc":0,"type_parameters":[]}},{"Effect":{"Push":{"RuntimeValue":{"value":1}}}},{"Instruction":{"gas_left":999998863,"instruction":"LD_TRUE","pc":1,"type_parameters":[]}},{"Effect":{"Push":{"RuntimeValue":{"value":true}}}},{"Instruction":{"gas_left":999998861,"instruction":"LD_U8","pc":2,"type_parameters":[]}},{"Effect":{"Push":{"RuntimeValue":{"value":0}}}},{"Instruction":{"gas_left":999998222,"instruction":"RET","pc":3,"type_parameters":[]}},{"CloseFrame":{"frame_id":2,"gas_left":999998222,"return_":[{"RuntimeValue":{"value":1}},{"RuntimeValue":{"value":true}},{"RuntimeValue":{"value":0}}]}},{"Instruction":{"gas_left":999998222,"instruction":"CALL","pc":1,"type_parameters":[]}},{"OpenFrame":{"frame":{"binary_member_index":1,"frame_id":12,"function_name":"f_test_call_order","is_native":false,"locals_types":[{"ref_type":null,"type_":"u64"},{"ref_type":null,"type_":"bool"},{"ref_type":null,"type_":"u8"}],"module":{"address":"0000000000000000000000000000000000000000000000000000000000000001","name":"calls"},"parameters":[{"RuntimeValue":{"value":1}},{"RuntimeValue":{"value":true}},{"RuntimeValue":{"value":0}}],"return_types":[],"type_instantiation":[]},"gas_left":999998222}},{"Instruction":{"gas_left":999993051,"instruction":"RET","pc":0,"type_parameters":[]}},{"CloseFrame":{"frame_id":12,"gas_left":999993051,"return_":[]}},{"Instruction":{"gas_left":999992412,"instruction":"RET","pc":2,"type_parameters":[]}},{"CloseFrame":{"frame_id":0,"gas_left":999992412,"return_":[]}}],"version":1}