-
Notifications
You must be signed in to change notification settings - Fork 3.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Calibration script for move e2e and single node perf benchmarks
- Loading branch information
1 parent
5d87d94
commit f3ef251
Showing
5 changed files
with
249 additions
and
77 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| Loop { loop_count: Some(100000), loop_type: NoOp } 60 0.955 1.074 41893.7 | ||
| Loop { loop_count: Some(10000), loop_type: Arithmetic } 60 0.965 1.078 25915.0 | ||
| CreateObjects { num_objects: 10, object_payload_size: 0 } 60 0.924 1.082 158.1 | ||
| CreateObjects { num_objects: 10, object_payload_size: 10240 } 60 0.951 1.118 9356.2 | ||
| CreateObjects { num_objects: 100, object_payload_size: 0 } 60 0.926 1.082 1574.2 | ||
| CreateObjects { num_objects: 100, object_payload_size: 10240 } 60 0.952 1.092 11541.9 | ||
| InitializeVectorPicture { length: 128 } 10 0.965 1.038 163.3 | ||
| VectorPicture { length: 128 } 10 0.938 1.060 48.8 | ||
| VectorPictureRead { length: 128 } 10 0.977 1.077 46.4 | ||
| InitializeVectorPicture { length: 30720 } 60 0.948 1.123 27893.4 | ||
| VectorPicture { length: 30720 } 60 0.931 1.125 6923.1 | ||
| VectorPictureRead { length: 30720 } 60 0.934 1.102 6923.1 | ||
| SmartTablePicture { length: 30720, num_points_per_txn: 200 } 60 0.952 1.109 43594.7 | ||
| SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 60 0.957 1.120 73865.4 | ||
| ResourceGroupsSenderWriteTag { string_length: 1024 } 60 0.934 1.134 15.0 | ||
| ResourceGroupsSenderMultiChange { string_length: 1024 } 60 0.929 1.122 32.3 | ||
| TokenV1MintAndTransferFT 60 0.958 1.093 385.2 | ||
| TokenV1MintAndTransferNFTSequential 60 0.973 1.139 588.1 | ||
| TokenV2AmbassadorMint { numbered: true } 60 0.960 1.141 512.5 | ||
| LiquidityPoolSwap { is_stable: true } 60 0.961 1.103 590.3 | ||
| LiquidityPoolSwap { is_stable: false } 60 0.954 1.134 552.2 | ||
| CoinInitAndMint 10 0.975 1.043 199.6 | ||
| FungibleAssetMint 10 0.954 1.038 236.3 | ||
| IncGlobalMilestoneAggV2 { milestone_every: 1 } 10 0.960 1.047 32.9 | ||
| IncGlobalMilestoneAggV2 { milestone_every: 2 } 10 0.971 1.066 18.1 | ||
| EmitEvents { count: 1000 } 10 0.969 1.052 8615.5 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,173 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| import argparse | ||
| import requests | ||
|
|
||
| def humio_secret(): | ||
| import subprocess | ||
| return subprocess.run(["gcloud", "secrets", "versions", "access", "--secret=ci_humio_read_token", "--project=aptos-shared-secrets", "latest"], capture_output=True).stdout.decode("utf-8") | ||
|
|
||
|
|
||
| def humio_url(): | ||
| return "https://cloud.us.humio.com/api/v1/repositories/github/query" | ||
|
|
||
| def parse_args(): | ||
| parser = argparse.ArgumentParser(description='Benchmark calibration tools') | ||
|
|
||
| parser.add_argument( | ||
| '--branch', | ||
| type=str, | ||
| help='Optional branch, if passed - only looks at results run on that branch through adhoc runs', | ||
| ) | ||
|
|
||
| parser.add_argument( | ||
| '--move-e2e', | ||
| action='store_true', | ||
| help='Calibrate move e2e test', | ||
| ) | ||
|
|
||
| return parser.parse_args() | ||
|
|
||
| def query_humio(query_string, time_interval="5d"): | ||
| query = { | ||
| "queryString": query_string, | ||
| "start": time_interval, | ||
| } | ||
|
|
||
| secret = humio_secret() | ||
|
|
||
| resp = requests.post( | ||
| url=humio_url(), | ||
| json=query, | ||
| headers={ | ||
| "Authorization": f"Bearer {secret}", | ||
| "Content-Type": "application/json", | ||
| }, | ||
| ) | ||
|
|
||
| return resp.text.strip() | ||
|
|
||
| def main(): | ||
| args = parse_args() | ||
|
|
||
| if args.move_e2e: | ||
| prefix = ( | ||
| """ | ||
| github.job.name = "single-node-performance" | ||
| | github.workflow.head_branch = "{branch}" | ||
| | "grep_json_aptos_move_vm_perf" | ||
| | parseJson(message) | ||
| """.format( | ||
| branch=args.branch | ||
| ) | ||
| if args.branch is not None | ||
| else """ | ||
| github.job.name = "execution-performance / single-node-performance" | ||
| | github.workflow.head_branch = "main" | ||
| | "grep_json_aptos_move_vm_perf" | ||
| | parseJson(message) | ||
| """ | ||
| ) | ||
|
|
||
| query_string = ( | ||
| prefix | ||
| + """ | ||
| | groupBy([test_index, transaction_type], function=[count(as="count"), avg(expected_wall_time_us, as="expected"), avg(wall_time_us, as="avg_wall_time_us"), min(wall_time_us, as="min_wall_time_us"), max(wall_time_us, as="max_wall_time_us"), percentile(field=wall_time_us, accuracy=0.001, percentiles=[50])]) | ||
| | min_ratio := min_wall_time_us / _50 | ||
| | avg_ratio := avg_wall_time_us / _50 | ||
| | max_ratio := max_wall_time_us / _50 | ||
| | offset_avg_from_expected := _50 / expected | ||
| | format("%.1f", field=_50, as="median") | ||
| | format("%.1f", field=avg_wall_time_us, as="avg_wall_time_us") | ||
| | format("%.1f", field=min_wall_time_us, as="min_wall_time_us") | ||
| | format("%.1f", field=max_wall_time_us, as="max_wall_time_us") | ||
| | format("%.3f", field=min_ratio, as="min_ratio") | ||
| | format("%.3f", field=max_ratio, as="max_ratio") | ||
| | format("%.3f", field=offset_avg_from_expected, as="offset_median_from_expected") | ||
| | table([transaction_type, count, min_ratio, max_ratio, median, expected], sortby=test_index, reverse=false) | ||
| """ | ||
| ) | ||
|
|
||
| columns = ["transaction_type", "count", "min_ratio", "max_ratio", "median"] | ||
|
|
||
| def split_line(line): | ||
| line = line.strip() | ||
| if "}" in line: | ||
| parts = line.split("}") | ||
| res = [parts[0] + "}"] + parts[1].split(", ")[1:] | ||
| return res | ||
| else: | ||
| return line.split(", ") | ||
| output_file_name = "aptos-move/e2e-benchmark/data/calibration_values.tsv" | ||
|
|
||
| else: | ||
| prefix = ( | ||
| """ | ||
| github.job.name = "single-node-performance" | ||
| | github.workflow.head_branch = "{branch}" | ||
| | "grep_json_single_node_perf" | ||
| | parseJson(message) | ||
| | source = "ADHOC" | ||
| | code_perf_version = "v8" | ||
| """.format( | ||
| branch=args.branch | ||
| ) | ||
| if args.branch is not None | ||
| else """ | ||
| github.job.name = "execution-performance / single-node-performance" | ||
| | github.workflow.head_branch = "main" | ||
| | "grep_json_single_node_perf" | ||
| | parseJson(message) | ||
| """ | ||
| ) | ||
|
|
||
| query_string = ( | ||
| prefix | ||
| + """ | ||
| | groupBy([test_index, transaction_type, module_working_set_size, executor_type, code_perf_version], function=[count(as="count"), avg(expected_tps, as="expected"), avg(tps, as="avg_tps"), min(tps, as="min_tps"), max(tps, as="max_tps"), percentile(field=tps, accuracy=0.001, percentiles=[50])]) | ||
| | min_ratio := min_tps / _50 | ||
| | avg_ratio := avg_tps / _50 | ||
| | max_ratio := max_tps / _50 | ||
| | offset_avg_from_expected := _50 / expected | ||
| | format("%.1f", field=_50, as="median") | ||
| | format("%.1f", field=avg_tps, as="avg_tps") | ||
| | format("%.1f", field=min_tps, as="min_tps") | ||
| | format("%.1f", field=max_tps, as="max_tps") | ||
| | format("%.3f", field=min_ratio, as="min_ratio") | ||
| | format("%.3f", field=max_ratio, as="max_ratio") | ||
| | format("%.3f", field=offset_avg_from_expected, as="offset_median_from_expected") | ||
| | table([transaction_type, module_working_set_size, executor_type, count, min_ratio, max_ratio, median], sortby=test_index, reverse=false) | ||
| """ | ||
| ) | ||
|
|
||
| columns = [ | ||
| "transaction_type", | ||
| "module_working_set_size", | ||
| "executor_type", | ||
| "count", | ||
| "min_ratio", | ||
| "max_ratio", | ||
| "median", | ||
| ] | ||
|
|
||
| def split_line(line): | ||
| return line.strip().split(", ") | ||
|
|
||
| output_file_name = "testsuite/single_node_performance_values.tsv" | ||
|
|
||
| response_text = query_humio(query_string) | ||
|
|
||
| parsed = [ | ||
| {(parts := key_value.split("->"))[0]: parts[1] for key_value in split_line(line)} | ||
| for line in response_text.split("\n") | ||
| ] | ||
|
|
||
| with open(output_file_name, "w") as f: | ||
| for line in parsed: | ||
| f.write("\t".join([line[column] for column in columns])) | ||
| f.write("\n") | ||
|
|
||
| print(f"Written to {output_file_name}") | ||
|
|
||
| if __name__ == '__main__': | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| no-op 1 VM 6 0.938 1.019 38925.3 | ||
| no-op 1000 VM 6 0.943 1.019 36444.6 | ||
| apt-fa-transfer 1 VM 6 0.927 1.018 26954.7 | ||
| apt-fa-transfer 1 NativeVM 6 0.927 1.018 35259.7 | ||
| account-generation 1 VM 6 0.96 1.02 20606.2 | ||
| account-generation 1 NativeVM 6 0.96 1.02 28216.2 | ||
| account-resource32-b 1 VM 6 0.94 1.026 34260.4 | ||
| modify-global-resource 1 VM 6 0.993 1.021 2260.5 | ||
| modify-global-resource 100 VM 6 0.982 1.02 33129.7 | ||
| publish-package 1 VM 6 0.983 1.012 1672.6 | ||
| mix_publish_transfer 1 VM 6 0.972 1.044 20832.8 | ||
| batch100-transfer 1 VM 6 0.953 1.024 645.1 | ||
| batch100-transfer 1 NativeVM 6 0.953 1.024 1437.0 | ||
| vector-picture30k 1 VM 6 0.992 1.039 103.6 | ||
| vector-picture30k 100 VM 6 0.913 1.015 1831.5 | ||
| smart-table-picture30-k-with200-change 1 VM 6 0.976 1.034 16.1 | ||
| smart-table-picture30-k-with200-change 100 VM 6 0.985 1.018 212.9 | ||
| modify-global-resource-agg-v2 1 VM 6 0.976 1.035 33992.5 | ||
| modify-global-flag-agg-v2 1 VM 6 0.986 1.016 4224 | ||
| modify-global-bounded-agg-v2 1 VM 6 0.964 1.047 7661.6 | ||
| modify-global-milestone-agg-v2 1 VM 6 0.973 1.017 25187.1 | ||
| resource-groups-global-write-tag1-kb 1 VM 6 0.989 1.03 9215.7 | ||
| resource-groups-global-write-and-read-tag1-kb 1 VM 6 0.982 1.018 5538.3 | ||
| resource-groups-sender-write-tag1-kb 1 VM 6 0.985 1.059 20084.2 | ||
| resource-groups-sender-multi-change1-kb 1 VM 6 0.968 1.034 16400.4 | ||
| token-v1ft-mint-and-transfer 1 VM 6 0.987 1.022 1156.3 | ||
| token-v1ft-mint-and-transfer 100 VM 6 0.964 1.024 17842.6 | ||
| token-v1nft-mint-and-transfer-sequential 1 VM 6 0.984 1.017 735.7 | ||
| token-v1nft-mint-and-transfer-sequential 100 VM 6 0.966 1.017 12819.7 | ||
| coin-init-and-mint 1 VM 6 0.95 1.024 26906.4 | ||
| coin-init-and-mint 100 VM 6 0.985 1.022 22312.6 | ||
| fungible-asset-mint 1 VM 6 0.955 1.013 23001.6 | ||
| fungible-asset-mint 100 VM 6 0.955 1.015 19973.5 | ||
| no-op5-signers 1 VM 6 0.934 1.016 38708.6 | ||
| token-v2-ambassador-mint 1 VM 6 0.975 1.008 15179.3 | ||
| token-v2-ambassador-mint 100 VM 6 0.985 1.007 15150.8 | ||
| liquidity-pool-swap 1 VM 6 0.987 1.018 805.5 | ||
| liquidity-pool-swap 100 VM 6 0.993 1.02 11156.3 | ||
| liquidity-pool-swap-stable 1 VM 6 0.985 1.017 778.7 | ||
| liquidity-pool-swap-stable 100 VM 6 0.982 1.009 11056.6 | ||
| deserialize-u256 1 VM 6 0.968 1.026 36444.6 | ||
| no-op-fee-payer 1 VM 6 0.994 1.026 2046 | ||
| no-op-fee-payer 100 VM 6 0.96 1.014 32866.5 | ||
| simple-script 1 VM 6 0.941 1.012 38206.1 |