Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[short][move e2e bench] Add gas metrics and more existing transaction types #15302

Merged
merged 2 commits into from
Nov 19, 2024
Merged

[short][move e2e bench] Add gas metrics and more existing transaction types #15302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
89444f9
[short][move e2e bench] Add gas metrics and more existing transaction…
igor-aptos Nov 18, 2024
830e5e8
addressing comments
igor-aptos Nov 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions aptos-move/aptos-gas-calibration/src/measurements.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,15 +162,17 @@ fn compile_and_run_samples_ir(
.equation_names
.push(format!("{}::{}", &identifier, func_identifier.0));

let elapsed = executor.exec_func_record_running_time(
&module_id,
&func_identifier.0,
vec![],
func_identifier.1.clone(),
iterations,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
);
let elapsed = executor
.exec_func_record_running_time(
&module_id,
&func_identifier.0,
vec![],
func_identifier.1.clone(),
iterations,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
)
.elapsed_micros();
gas_measurement.regular_meter.push(elapsed);

// record with abstract gas meter
Expand Down
20 changes: 11 additions & 9 deletions aptos-move/aptos-gas-calibration/src/measurements_helpers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,15 +94,17 @@ pub fn execute_user_txn(
iterations: u64,
args: Vec<Vec<u8>>,
) -> u128 {
let elapsed = executor.exec_func_record_running_time(
module_name,
function_name,
vec![],
args,
iterations,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
);
let elapsed = executor
.exec_func_record_running_time(
module_name,
function_name,
vec![],
args,
iterations,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
)
.elapsed_micros();
println!("running time (microseconds): {}", elapsed);
elapsed
}
Expand Down
20 changes: 11 additions & 9 deletions aptos-move/aptos-vm-benchmarks/src/helper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,17 @@ pub fn execute_module_txn(

// sign user transaction and only records the body of the transaction
pub fn execute_user_txn(executor: &mut FakeExecutor, module_name: &ModuleId, function_name: &str) {
let elapsed = executor.exec_func_record_running_time(
module_name,
function_name,
vec![],
vec![],
10,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
);
let elapsed = executor
.exec_func_record_running_time(
module_name,
function_name,
vec![],
vec![],
10,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
)
.elapsed_micros();
println!("running time (microseconds): {}", elapsed);
}

Expand Down
96 changes: 56 additions & 40 deletions aptos-move/e2e-benchmark/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

use aptos_language_e2e_tests::{
account::Account,
executor::{ExecFuncTimerDynamicArgs, FakeExecutor, GasMeterType},
executor::{ExecFuncTimerDynamicArgs, FakeExecutor, GasMeterType, Measurement},
};
use aptos_transaction_generator_lib::{
publishing::{
Expand Down Expand Up @@ -46,7 +46,7 @@ fn execute_and_time_entry_point(
publisher_address: &AccountAddress,
executor: &mut FakeExecutor,
iterations: u64,
) -> u128 {
) -> Measurement {
let mut rng = StdRng::seed_from_u64(14);
let entry_fun = entry_point
.create_payload(
Expand Down Expand Up @@ -77,9 +77,9 @@ fn execute_and_time_entry_point(
)
}

const ALLOWED_REGRESSION: f32 = 0.15;
const ALLOWED_IMPROVEMENT: f32 = 0.15;
const ABSOLUTE_BUFFER_US: f32 = 2.0;
const ALLOWED_REGRESSION: f64 = 0.15;
const ALLOWED_IMPROVEMENT: f64 = 0.15;
const ABSOLUTE_BUFFER_US: f64 = 2.0;

const CALIBRATION_VALUES: &str = "
Loop { loop_count: Some(100000), loop_type: NoOp } 6 0.988 1.039 41212.4
Expand All @@ -88,9 +88,9 @@ CreateObjects { num_objects: 10, object_payload_size: 0 } 6 0.940 1.026 152.1
CreateObjects { num_objects: 10, object_payload_size: 10240 } 6 0.934 1.051 9731.3
CreateObjects { num_objects: 100, object_payload_size: 0 } 6 0.966 1.051 1458.3
CreateObjects { num_objects: 100, object_payload_size: 10240 } 6 0.969 1.077 11196.4
InitializeVectorPicture { length: 40 } 6 0.973 1.066 75.0
VectorPicture { length: 40 } 6 0.955 1.092 22.0
VectorPictureRead { length: 40 } 6 0.952 1.047 21.0
InitializeVectorPicture { length: 128 } 6 0.973 1.066 170.3
VectorPicture { length: 128 } 6 0.955 1.092 46.2
VectorPictureRead { length: 128 } 6 0.952 1.047 45.1
InitializeVectorPicture { length: 30720 } 6 0.969 1.071 27295.8
VectorPicture { length: 30720 } 6 0.957 1.066 6560.2
VectorPictureRead { length: 30720 } 6 0.948 1.053 6642.8
Expand All @@ -103,11 +103,16 @@ TokenV1MintAndTransferNFTSequential 6 0.991 1.067 543.7
TokenV2AmbassadorMint { numbered: true } 6 0.987 1.052 474.4
LiquidityPoolSwap { is_stable: true } 6 0.970 1.042 555.4
LiquidityPoolSwap { is_stable: false } 6 0.925 1.001 535.3
CoinInitAndMint 6 0.925 1.001 197.1
FungibleAssetMint 6 0.925 1.001 231.6
IncGlobalMilestoneAggV2 { milestone_every: 1 } 6 0.925 1.001 33.3
IncGlobalMilestoneAggV2 { milestone_every: 2 } 6 0.925 1.001 19.1
EmitEvents { count: 1000 } 6 0.925 1.001 8493.7
";

struct CalibrationInfo {
// count: usize,
expected_time: f32,
expected_time_micros: f64,
}

fn get_parsed_calibration_values() -> HashMap<String, CalibrationInfo> {
Expand All @@ -118,7 +123,7 @@ fn get_parsed_calibration_values() -> HashMap<String, CalibrationInfo> {
let parts = line.split('\t').collect::<Vec<_>>();
(parts[0].to_string(), CalibrationInfo {
// count: parts[1].parse().unwrap(),
expected_time: parts[parts.len() - 1].parse().unwrap(),
expected_time_micros: parts[parts.len() - 1].parse().unwrap(),
})
})
.collect()
Expand Down Expand Up @@ -146,7 +151,7 @@ fn main() {
loop_type: LoopType::Arithmetic,
},
// This is a cheap bcs (serializing vec<u8>), so not representative of what BCS native call should cost.
// (, EntryPoints::Loop { loop_count: Some(1000), loop_type: LoopType::BCS { len: 1024 }}),
// (, EntryPoints::Loop { loop_count: Some(1000), loop_type: LoopType::BcsToBytes { len: 1024 }}),
EntryPoints::CreateObjects {
num_objects: 10,
object_payload_size: 0,
Expand All @@ -163,9 +168,9 @@ fn main() {
num_objects: 100,
object_payload_size: 10 * 1024,
},
EntryPoints::InitializeVectorPicture { length: 40 },
EntryPoints::VectorPicture { length: 40 },
EntryPoints::VectorPictureRead { length: 40 },
EntryPoints::InitializeVectorPicture { length: 128 },
EntryPoints::VectorPicture { length: 128 },
EntryPoints::VectorPictureRead { length: 128 },
EntryPoints::InitializeVectorPicture { length: 30 * 1024 },
EntryPoints::VectorPicture { length: 30 * 1024 },
EntryPoints::VectorPictureRead { length: 30 * 1024 },
Expand All @@ -188,22 +193,27 @@ fn main() {
EntryPoints::TokenV2AmbassadorMint { numbered: true },
EntryPoints::LiquidityPoolSwap { is_stable: true },
EntryPoints::LiquidityPoolSwap { is_stable: false },
EntryPoints::CoinInitAndMint,
EntryPoints::FungibleAssetMint,
EntryPoints::IncGlobalMilestoneAggV2 { milestone_every: 1 },
EntryPoints::IncGlobalMilestoneAggV2 { milestone_every: 2 },
EntryPoints::EmitEvents { count: 1000 },
];

let mut failures = Vec::new();
let mut json_lines = Vec::new();

println!(
"{:>15} {:>15} {:>15} entry point",
"wall time (us)", "expected (us)", "diff(- is impr)"
"{:>13} {:>13} {:>13}{:>13} {:>13} {:>13} entry point",
"walltime(us)", "expected(us)", "dif(- is impr)", "gas/s", "exe gas", "io gas",
);

for (index, entry_point) in entry_points.into_iter().enumerate() {
let entry_point_name = format!("{:?}", entry_point);
let expected_time = calibration_values
let expected_time_micros = calibration_values
.get(&entry_point_name)
.unwrap()
.expected_time;
.expect(&entry_point_name)
.expected_time_micros;
let publisher = executor.new_account_at(AccountAddress::random());

let mut package_handler = PackageHandler::new(entry_point.package_name());
Expand All @@ -215,7 +225,6 @@ fn main() {
0,
package.publish_transaction_payload(),
);
println!("Published package: {:?}", entry_point.package_name());
if let Some(init_entry_point) = entry_point.initialize_entry_point() {
execute_txn(
&mut executor,
Expand All @@ -228,52 +237,59 @@ fn main() {
Some(publisher.address()),
),
);
println!(
"Executed init entry point: {:?}",
entry_point.initialize_entry_point()
);
}

let elapsed_micros = execute_and_time_entry_point(
let measurement = execute_and_time_entry_point(
&entry_point,
&package,
publisher.address(),
&mut executor,
if expected_time > 10000.0 {
if expected_time_micros > 10000.0 {
6
} else if expected_time > 1000.0 {
} else if expected_time_micros > 1000.0 {
10
} else {
100
},
);
let diff = (elapsed_micros as f32 - expected_time) / expected_time * 100.0;
let elapsed_micros = measurement.elapsed_micros_f64();
let diff = (elapsed_micros - expected_time_micros) / expected_time_micros * 100.0;
let execution_gas_units = measurement.execution_gas_units();
let io_gas_units = measurement.io_gas_units();
let gps = (execution_gas_units + io_gas_units) / measurement.elapsed_secs_f64();
println!(
"{:15} {:15.1} {:14.1}% {:?}",
elapsed_micros, expected_time, diff, entry_point
"{:13.1} {:13.1} {:12.1}% {:13.0} {:13.2} {:13.2} {:?}",
elapsed_micros,
expected_time_micros,
diff,
gps,
execution_gas_units,
io_gas_units,
entry_point
);

json_lines.push(json!({
"grep": "grep_json_aptos_move_vm_perf",
"transaction_type": entry_point_name,
"wall_time_us": elapsed_micros,
"expected_wall_time_us": expected_time,
"gas_units_per_second": gps,
"execution_gas_units": execution_gas_units,
"io_gas_units": io_gas_units,
"expected_wall_time_us": expected_time_micros,
"test_index": index,
}));

if elapsed_micros as f32
> expected_time as f32 * (1.0 + ALLOWED_REGRESSION) + ABSOLUTE_BUFFER_US
{
if elapsed_micros > expected_time_micros * (1.0 + ALLOWED_REGRESSION) + ABSOLUTE_BUFFER_US {
failures.push(format!(
"Performance regression detected: {}us, expected: {}us, diff: {}%, for {:?}",
elapsed_micros, expected_time, diff, entry_point
"Performance regression detected: {:.1}us, expected: {:.1}us, diff: {}%, for {:?}",
elapsed_micros, expected_time_micros, diff, entry_point
));
} else if elapsed_micros as f32 + ABSOLUTE_BUFFER_US
< expected_time as f32 * (1.0 - ALLOWED_IMPROVEMENT)
} else if elapsed_micros + ABSOLUTE_BUFFER_US
< expected_time_micros * (1.0 - ALLOWED_IMPROVEMENT)
{
failures.push(format!(
"Performance improvement detected: {}us, expected {}us, diff: {}%, for {:?}. You need to adjust expected time!",
elapsed_micros, expected_time, diff, entry_point
"Performance improvement detected: {:.1}us, expected {:.1}us, diff: {}%, for {:?}. You need to adjust expected time!",
elapsed_micros, expected_time_micros, diff, entry_point
));
}
}
Expand Down
Loading
Loading