Skip to content

style: Use Arc::clone() instead of .clone() #5796

style: Use Arc::clone() instead of .clone()

style: Use Arc::clone() instead of .clone() #5796

Workflow file for this run

name: PR Benchmarks
on: pull_request
env:
CARGO_INCREMENTAL: "0"
RUST_BACKTRACE: 1
CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
NODE_DATA_PATH: /home/runner/.local/share/safe/node
jobs:
benchmark-cli:
name: Compare sn_cli benchmarks to main
# right now only ubuntu, running on multiple systems would require many pushes...\
# perhaps this can be done with one consolidation action in the future, pulling down all results and pushing
# once to the branch..
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
continue-on-error: true
########################
### Setup ###
########################
- run: cargo install cargo-criterion
- name: install ripgrep
run: sudo apt-get -y install ripgrep
- name: Download 95mb file to be uploaded with the safe client
shell: bash
run: wget https://sn-node.s3.eu-west-2.amazonaws.com/the-test-data.zip
# As normal user won't care much about initial client startup,
# but be more alerted on communication speed during transmission.
# Meanwhile the criterion testing code includes the client startup as well,
# it will be better to execute bench test with `local-discovery`,
# to make the measurement results reflect speed improvement or regression more accurately.
- name: Build sn bins
run: cargo build --release --bin safe --bin safenode --features local-discovery
timeout-minutes: 30
- name: Build faucet bin
run: cargo build --release --bin faucet --features local-discovery --features gifting --no-default-features
timeout-minutes: 30
- name: Start a local network
uses: maidsafe/sn-local-testnet-action@main
env:
SN_LOG: "all"
with:
action: start
interval: 2000
node-path: target/release/safenode
faucet-path: target/release/faucet
platform: ubuntu-latest
build: true
- name: Check SAFE_PEERS was set
shell: bash
run: echo "The SAFE_PEERS variable has been set to $SAFE_PEERS"
#########################
### Upload large file ###
#########################
- name: Fund cli wallet
shell: bash
run: target/release/safe --log-output-dest=data-dir wallet get-faucet 127.0.0.1:8000
env:
SN_LOG: "all"
- name: Start a client instance to compare memory usage
shell: bash
run: target/release/safe --log-output-dest=data-dir files upload the-test-data.zip --retry-strategy quick
env:
SN_LOG: "all"
- name: Cleanup uploaded_files folder to avoid pollute download benchmark
shell: bash
run: rm -rf $CLIENT_DATA_PATH/uploaded_files
###########################
### Client Mem Analysis ###
###########################
- name: Check client memory usage
shell: bash
run: |
client_peak_mem_limit_mb="1024" # mb
client_avg_mem_limit_mb="512" # mb
peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs --glob safe.* -o --no-line-number --no-filename |
awk -F':' '/"memory_used_mb":/{print $2}' |
sort -n |
tail -n 1
)
echo "Peak memory usage: $peak_mem_usage MB"
if (( $(echo "$peak_mem_usage > $client_peak_mem_limit_mb" | bc -l) )); then
echo "Client peak memory usage exceeded threshold: $client_peak_mem_limit_mb MB"
exit 1
fi
total_mem=$(
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs --glob safe.* -o --no-line-number --no-filename |
awk -F':' '/"memory_used_mb":/ {sum += $2} END {printf "%.0f\n", sum}'
)
num_of_times=$(
rg "\"memory_used_mb\"" $CLIENT_DATA_PATH/logs --glob safe.* -c --stats |
rg "(\d+) matches" |
rg "\d+" -o
)
echo "num_of_times: $num_of_times"
echo "Total memory is: $total_mem"
average_mem=$(($total_mem/$(($num_of_times))))
echo "Average memory is: $average_mem"
if (( $(echo "$average_mem > $client_avg_mem_limit_mb" | bc -l) )); then
echo "Client average memory usage exceeded threshold: $client_avg_mem_limit_mb MB"
exit 1
fi
# Write the client memory usage to a file
echo '[
{
"name": "client-peak-memory-usage-during-upload",
"value": '$peak_mem_usage',
"unit": "MB"
},
{
"name": "client-average-memory-usage-during-upload",
"value": '$average_mem',
"unit": "MB"
}
]' > client_memory_usage.json
- name: check client_memory_usage.json
shell: bash
run: cat client_memory_usage.json
- name: Alert for client memory usage
uses: benchmark-action/github-action-benchmark@v1
with:
name: "Memory Usage of Client during uploading large file"
tool: "customSmallerIsBetter"
output-file-path: client_memory_usage.json
# Where the previous data file is stored
external-data-json-path: ./cache/client-mem-usage.json
# Workflow will fail when an alert happens
fail-on-alert: true
# GitHub API token to make a commit comment
github-token: ${{ secrets.GITHUB_TOKEN }}
# Enable alert commit comment
comment-on-alert: true
# 200% regression will result in alert
alert-threshold: "200%"
# Enable Job Summary for PRs
summary-always: true
########################
### Benchmark ###
########################
- name: Bench `safe` cli
shell: bash
# Criterion outputs the actual bench results to stderr "2>&1 tee output.txt" takes stderr,
# passes to tee which displays it in the terminal and writes to output.txt
run: |
cargo criterion --features=local-discovery --message-format=json 2>&1 -p sn_cli | tee -a output.txt
cat output.txt | rg benchmark-complete | jq -s 'map({
name: (.id | split("/"))[-1],
unit: "MiB/s",
value: ((if .throughput[0].unit == "KiB/s" then (.throughput[0].per_iteration / (1024*1024*1024)) else (.throughput[0].per_iteration / (1024*1024)) end) / (.mean.estimate / 1e9))
})' > files-benchmark.json
timeout-minutes: 15
- name: Confirming the number of files uploaded and downloaded during the benchmark test
shell: bash
run: |
ls -l $CLIENT_DATA_PATH
ls -l $CLIENT_DATA_PATH/uploaded_files
ls -l $CLIENT_DATA_PATH/safe_files
- name: Store benchmark result
uses: benchmark-action/github-action-benchmark@v1
with:
# What benchmark tool the output.txt came from
tool: "customBiggerIsBetter"
output-file-path: files-benchmark.json
# Where the previous data file is stored
external-data-json-path: ./cache/benchmark-data.json
# Workflow will fail when an alert happens
fail-on-alert: true
# GitHub API token to make a commit comment
github-token: ${{ secrets.GITHUB_TOKEN }}
# Enable alert commit comment
comment-on-alert: true
# 200% regression will result in alert
alert-threshold: "200%"
# Enable Job Summary for PRs
summary-always: true
- name: Start a client to carry out download to output the logs
shell: bash
run: target/release/safe --log-output-dest=data-dir files download --retry-strategy quick
- name: Start a client to simulate criterion upload
shell: bash
run: |
ls -l target/release
target/release/safe --log-output-dest=data-dir files upload target/release/faucet --retry-strategy quick
#########################
### Stop Network ###
#########################
- name: Stop the local network
if: always()
uses: maidsafe/sn-local-testnet-action@main
with:
action: stop
log_file_prefix: safe_test_logs_benchmark
platform: ubuntu-latest
build: true
- name: Upload Faucet folder
uses: actions/upload-artifact@main
with:
name: faucet_folder
path: /home/runner/.local/share/safe/test_faucet
continue-on-error: true
if: always()
#########################
### Node Mem Analysis ###
#########################
# The large file uploaded will increase node's peak mem usage a lot
- name: Check node memory usage
shell: bash
run: |
node_peak_mem_limit_mb="250" # mb
peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $NODE_DATA_PATH/*/logs/* -o --no-line-number --no-filename |
awk -F':' '/"memory_used_mb":/{print $2}' |
sort -n |
tail -n 1
)
echo "Memory usage: $peak_mem_usage MB"
if (( $(echo "$peak_mem_usage > $node_peak_mem_limit_mb" | bc -l) )); then
echo "Node memory usage exceeded threshold: $peak_mem_usage MB"
exit 1
fi
# Write the node memory usage to a file
echo '[
{
"name": "node-memory-usage-through-safe-benchmark",
"value": '$peak_mem_usage',
"unit": "MB"
}
]' > node_memory_usage.json
- name: check node_memory_usage.json
shell: bash
run: cat node_memory_usage.json
- name: Alert for node memory usage
uses: benchmark-action/github-action-benchmark@v1
with:
tool: "customSmallerIsBetter"
output-file-path: node_memory_usage.json
# Where the previous data file is stored
external-data-json-path: ./cache/node-mem-usage.json
# Workflow will fail when an alert happens
fail-on-alert: true
# GitHub API token to make a commit comment
github-token: ${{ secrets.GITHUB_TOKEN }}
# Enable alert commit comment
comment-on-alert: true
# Comment on the PR
comment-always: true
# 200% regression will result in alert
alert-threshold: "200%"
# Enable Job Summary for PRs
summary-always: true
###########################################
### Swarm_driver handling time Analysis ###
###########################################
- name: Check swarm_driver handling time
shell: bash
run: |
num_of_times=$(
rg "SwarmCmd handled in [0-9.]+ms:" $NODE_DATA_PATH/*/logs/* --glob safe.* -c --stats |
rg "(\d+) matches" |
rg "\d+" -o
)
echo "Number of long cmd handling times: $num_of_times"
total_long_handling_ms=$(
rg "SwarmCmd handled in [0-9.]+ms:" $NODE_DATA_PATH/*/logs/* --glob safe.* -o --no-line-number --no-filename |
awk -F' |ms:' '{sum += $4} END {printf "%.0f\n", sum}'
)
echo "Total cmd long handling time is: $total_long_handling_ms ms"
average_handling_ms=$(($total_long_handling_ms/$(($num_of_times))))
echo "Average cmd long handling time is: $average_handling_ms ms"
total_long_handling=$(($total_long_handling_ms))
total_num_of_times=$(($num_of_times))
num_of_times=$(
rg "SwarmEvent handled in [0-9.]+ms:" $NODE_DATA_PATH/*/logs/* --glob safe.* -c --stats |
rg "(\d+) matches" |
rg "\d+" -o
)
echo "Number of long event handling times: $num_of_times"
total_long_handling_ms=$(
rg "SwarmEvent handled in [0-9.]+ms:" $NODE_DATA_PATH/*/logs/* --glob safe.* -o --no-line-number --no-filename |
awk -F' |ms:' '{sum += $4} END {printf "%.0f\n", sum}'
)
echo "Total event long handling time is: $total_long_handling_ms ms"
average_handling_ms=$(($total_long_handling_ms/$(($num_of_times))))
echo "Average event long handling time is: $average_handling_ms ms"
total_long_handling=$(($total_long_handling_ms+$total_long_handling))
total_num_of_times=$(($num_of_times+$total_num_of_times))
average_handling_ms=$(($total_long_handling/$(($total_num_of_times))))
echo "Total swarm_driver long handling times is: $total_num_of_times"
echo "Total swarm_driver long handling duration is: $total_long_handling ms"
echo "Total average swarm_driver long handling duration is: $average_handling_ms ms"
total_num_of_times_limit_hits="30000" # hits
total_long_handling_limit_ms="400000" # ms
average_handling_limit_ms="20" # ms
if (( $(echo "$total_num_of_times > $total_num_of_times_limit_hits" | bc -l) )); then
echo "Swarm_driver long handling times exceeded threshold: $total_num_of_times hits"
exit 1
fi
if (( $(echo "$total_long_handling > $total_long_handling_limit_ms" | bc -l) )); then
echo "Swarm_driver total long handling duration exceeded threshold: $total_long_handling ms"
exit 1
fi
if (( $(echo "$average_handling_ms > $average_handling_limit_ms" | bc -l) )); then
echo "Swarm_driver average long handling time exceeded threshold: $average_handling_ms ms"
exit 1
fi
# Write the node memory usage to a file
echo '[
{
"name": "swarm_driver long handling times",
"value": '$total_num_of_times',
"unit": "hits"
},
{
"name": "swarm_driver long handling total_time",
"value": '$total_long_handling',
"unit": "ms"
},
{
"name": "swarm_driver average long handling time",
"value": '$average_handling_ms',
"unit": "ms"
}
]' > swarm_driver_long_handlings.json
- name: check swarm_driver_long_handlings.json
shell: bash
run: cat swarm_driver_long_handlings.json
- name: Alert for swarm_driver long handlings
uses: benchmark-action/github-action-benchmark@v1
with:
tool: "customSmallerIsBetter"
output-file-path: swarm_driver_long_handlings.json
# Where the previous data file is stored
external-data-json-path: ./cache/swarm_driver_long_handlings.json
# Workflow will fail when an alert happens
fail-on-alert: true
# GitHub API token to make a commit comment
github-token: ${{ secrets.GITHUB_TOKEN }}
# Enable alert commit comment
comment-on-alert: true
# Comment on the PR
comment-always: true
# 200% regression will result in alert
alert-threshold: "200%"
# Enable Job Summary for PRs
summary-always: true
benchmark-cash:
name: Compare sn_transfer benchmarks to main
# right now only ubuntu, running on multiple systems would require many pushes...\
# perhaps this can be done with one consolidation action in the future, pulling down all results and pushing
# once to the branch..
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
continue-on-error: true
########################
### Setup ###
########################
- run: cargo install cargo-criterion
- name: install ripgrep
run: sudo apt-get -y install ripgrep
########################
### Benchmark ###
########################
- name: Bench `sn_transfers`
shell: bash
# Criterion outputs the actual bench results to stderr "2>&1 tee output.txt" takes stderr,
# passes to tee which displays it in the terminal and writes to output.txt
run: |
cargo criterion --message-format=json 2>&1 -p sn_transfers | tee -a output.txt
cat output.txt