generate-benchmark-charts.yml

name: Benchmark Chart Generation

# Do not run this workflow on pull request since this workflow has permission to modify contents.
on:
  push:
    branches:
      - main

permissions:
  # deployments permission to deploy GitHub pages website
  deployments: write
  # contents permission to update benchmark contents in gh-pages branch
  contents: write

env:
  CARGO_INCREMENTAL: '0'
  RUST_BACKTRACE: 1
  CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
  NODE_DATA_PATH: /home/runner/.local/share/safe/node

jobs:
  benchmark-cli:
    if: "!startsWith(github.event.head_commit.message, 'chore(release):')"
    name: Run and log benchmark criterion results on gh-pages
    # right now only ubuntu, running on multiple systems would require many pushes...\
    # perhaps this can be done with one consolidation action in the future, pulling down all results and pushing
    # once to the branch..
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
          
      - uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          override: true
          components: rustfmt, clippy

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - run: cargo install cargo-criterion

      - name: ubuntu install ripgrep
        run: sudo apt-get -y install ripgrep

      - name: Download 95mb file to be uploaded with the safe client
        shell: bash
        run: wget https://sn-node.s3.eu-west-2.amazonaws.com/the-test-data.zip
     
      - name: Build node and client
        run: cargo build --release --features local-discovery --bin safenode --bin safe --bin faucet
        timeout-minutes: 30

      - name: Start a local network
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: start
          interval: 2000
          node-path: target/release/safenode
          faucet-path: target/release/faucet
          platform: ubuntu-latest

      - name: Create and fund a wallet to pay for files storage
        run: |
          cargo run --bin faucet --release -- --log-output-dest=data-dir send 1000000 $(cargo run --bin safe --release -- wallet address | tail -n 1) | tail -n 1 > transfer_hex
          cargo run --bin safe --release -- wallet receive --file transfer_hex
        env:
          SN_LOG: "all"
        timeout-minutes: 10

      ########################
      ### Benchmark        ###
      ########################
      - name: Bench `safe` cli
        shell: bash
        # Criterion outputs the actual bench results to stderr "2>&1 tee output.txt" takes stderr,
        # passes to tee which displays it in the terminal and writes to output.txt
        run: |
          cargo criterion --features=local-discovery --message-format=json 2>&1 -p sn_cli | tee -a output.txt
          cat output.txt | rg benchmark-complete | jq -s 'map({
            name: (.id | split("/"))[-1],
            unit: "MiB/s",
            value: ((if .throughput[0].unit == "KiB/s" then (.throughput[0].per_iteration / (1024*1024*1024)) else (.throughput[0].per_iteration / (1024*1024)) end) / (.mean.estimate / 1e9))
          })' > files-benchmark.json

      - name: Remove git hooks so gh-pages git commits will work
        shell: bash
        run: rm -rf .git/hooks/pre-commit

      - name: check files-benchmark.json
        shell: bash
        run: cat files-benchmark.json
        
      # gh-pages branch is updated and pushed automatically with extracted benchmark data
      - name: Store cli files benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: "`safe files` benchmarks"
          tool: 'customBiggerIsBetter'
          output-file-path: files-benchmark.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          max-items-in-chart: 300

      - name: Start a client instance to compare memory usage
        shell: bash
        run: cargo run --bin safe --release -- --log-output-dest=data-dir files upload the-test-data.zip
        env:
          SN_LOG: "all"
      
      #########################
      ### Stop Network      ###
      #########################

      - name: Stop the local network
        if: always()
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: stop
          platform: ubuntu-latest

      #########################
      ### Mem Analysis      ###
      #########################

      - name: Check node memory usage
        shell: bash
        run: |
          peak_mem_usage=$(
            rg '"memory_used_mb":[^,]*' $NODE_DATA_PATH/*/logs/* -o --no-line-number --no-filename | 
            awk -F':' '/"memory_used_mb":/{print $2}' | 
            sort -n | 
            tail -n 1
          )
         
          # Write the node memory usage to a file
          echo '[
              {
                  "name": "Peak memory w/ `safe` benchmarks",
                  "value": '$peak_mem_usage',
                  "unit": "MB"
              }
          ]' > node_memory_usage.json

      - name: check node_memory_usage.json
        shell: bash
        run: cat node_memory_usage.json

      - name: Upload Node Memory Usage
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: 'Node memory'
          tool: 'customSmallerIsBetter'
          output-file-path: node_memory_usage.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          max-items-in-chart: 300

      - name: Check client memory usage
        shell: bash
        run: |
          peak_mem_usage=$(
            rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs --glob safe.* -o --no-line-number --no-filename | 
            awk -F':' '/"memory_used_mb":/{print $2}' | 
            sort -n | 
            tail -n 1
          )
          total_mem=$(
            rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs --glob safe.* -o --no-line-number --no-filename | 
            awk -F':' '/"memory_used_mb":/ {sum += $2} END {printf "%.0f\n", sum}'
          )
          num_of_times=$(
            rg "\"memory_used_mb\"" $CLIENT_DATA_PATH/logs --glob safe.* -c --stats |
            rg "(\d+) matches" |
            rg "\d+" -o
          )
          echo "num_of_times: $num_of_times"
          echo "Total memory is: $total_mem"
          average_mem=$(($total_mem/$(($num_of_times))))
          echo "Average memory is: $average_mem"


          # Write the client memory usage to a file
          echo '[
              {
                  "name": "Peak memory usage w/ upload",
                  "value": '$peak_mem_usage',
                  "unit": "MB"
              },
              {
                  "name": "Average memory usage w/ upload",
                  "value": '$average_mem',
                  "unit": "MB"
              }
          ]' > client_memory_usage.json

      - name: check client_memory_usage.json
        shell: bash
        run: cat client_memory_usage.json

      - name: Upload Client Memory Usage
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: 'Client memory'
          tool: 'customSmallerIsBetter'
          output-file-path: client_memory_usage.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          max-items-in-chart: 300

      ###########################################
      ### Swarm_driver handling time Analysis ###
      ###########################################

      - name: Check swarm_driver handling time
        shell: bash
        run: |
          num_of_times=$(
            rg "SwarmCmd handled in [^,]*ms:" $NODE_DATA_PATH/*/logs/* --glob safe.* -c --stats |
            rg "(\d+) matches" |
            rg "\d+" -o
          )
          echo "Number of long cmd handling times: $num_of_times"
          total_long_handling_ms=$(
            rg "SwarmCmd handled in [^,]*ms:" $NODE_DATA_PATH/*/logs/* --glob safe.* -o --no-line-number --no-filename |
            awk -F' |ms:' '{sum += $4} END {printf "%.0f\n", sum}'
          )
          echo "Total cmd long handling time is: $total_long_handling_ms ms"
          average_handling_ms=$(($total_long_handling_ms/$(($num_of_times))))
          echo "Average cmd long handling time is: $average_handling_ms ms"
          total_long_handling=$(($total_long_handling_ms))
          total_num_of_times=$(($num_of_times))
          num_of_times=$(
            rg "SwarmEvent handled in [^,]*ms:" $NODE_DATA_PATH/*/logs/* --glob safe.* -c --stats |
            rg "(\d+) matches" |
            rg "\d+" -o
          )
          echo "Number of long event handling times: $num_of_times"
          total_long_handling_ms=$(
            rg "SwarmEvent handled in [^,]*ms:" $NODE_DATA_PATH/*/logs/* --glob safe.* -o --no-line-number --no-filename |
            awk -F' |ms:' '{sum += $4} END {printf "%.0f\n", sum}'
          )
          echo "Total event long handling time is: $total_long_handling_ms ms"
          average_handling_ms=$(($total_long_handling_ms/$(($num_of_times))))
          echo "Average event long handling time is: $average_handling_ms ms"
          total_long_handling=$(($total_long_handling_ms+$total_long_handling))
          total_num_of_times=$(($num_of_times+$total_num_of_times))
          average_handling_ms=$(($total_long_handling/$(($total_num_of_times))))
          echo "Total swarm_driver long handling times is: $total_num_of_times"
          echo "Total swarm_driver long handling duration is: $total_long_handling ms"
          echo "Total average swarm_driver long handling duration is: $average_handling_ms ms"
         
          # Write the node memory usage to a file
          echo '[
              {
                  "name": "swarm_driver long handling times",
                  "value": '$total_num_of_times',
                  "unit": "hits"
              },
              {
                  "name": "swarm_driver long handling total_time",
                  "value": '$total_long_handling',
                  "unit": "ms"
              },
              {
                  "name": "swarm_driver average long handling time",
                  "value": '$average_handling_ms',
                  "unit": "ms"
              }
          ]' > swarm_driver_long_handlings.json

      - name: check swarm_driver_long_handlings.json
        shell: bash
        run: cat swarm_driver_long_handlings.json

      - name: Upload swarm_driver Long Handlings
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: 'swarm_driver long handlings'
          tool: 'customSmallerIsBetter'
          output-file-path: swarm_driver_long_handlings.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          max-items-in-chart: 300