Benchmark #289
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| schedule: | |
| # Run at 03:00 clock UTC on Monday and Wednesday | |
| - cron: "0 03 * * 1,3" | |
| pull_request: | |
| paths: | |
| - '.github/workflows/benchmark.yml' | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| contrib-build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.work' | |
| cache-dependency-path: "**/*.sum" | |
| - name: Build Contrib | |
| run: | | |
| make -e DOCKER=false nydusify-release | |
| - name: Upload Nydusify | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nydusify-artifact | |
| path: contrib/nydusify/cmd/nydusify | |
| nydus-build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Rust Cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| shared-key: Linux-cargo-amd64 | |
| - uses: dsherret/rust-toolchain-file@v1 | |
| - name: Build Nydus | |
| run: | | |
| make release | |
| - name: Upload Nydus Binaries | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nydus-artifact | |
| path: | | |
| target/release/nydus-image | |
| target/release/nydusd | |
| benchmark-description: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Description | |
| run: | | |
| echo "## Benchmark Environment" > $GITHUB_STEP_SUMMARY | |
| echo "| operating system | cpu | memory " >> $GITHUB_STEP_SUMMARY | |
| echo "|:----------------:|:---:|:------ " >> $GITHUB_STEP_SUMMARY | |
| echo "| ubuntu-22.04 | 2-core CPU (x86_64) | 7GB |" >> $GITHUB_STEP_SUMMARY | |
| benchmark-oci: | |
| runs-on: ubuntu-latest | |
| needs: [contrib-build, nydus-build] | |
| strategy: | |
| matrix: | |
| include: | |
| - image: wordpress | |
| tag: 6.1.1 | |
| - image: node | |
| tag: 19.8 | |
| - image: python | |
| tag: 3.10.7 | |
| - image: golang | |
| tag: 1.19.3 | |
| - image: ruby | |
| tag: 3.1.3 | |
| - image: amazoncorretto | |
| tag: 8-al2022-jdk | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download Nydus | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nydus-artifact | |
| path: target/release | |
| - name: Download Nydusify | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nydusify-artifact | |
| path: contrib/nydusify/cmd | |
| - name: Prepare Environment | |
| run: | | |
| sudo bash misc/prepare.sh | |
| - name: BenchMark Test | |
| run: | | |
| export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }} | |
| export BENCHMARK_MODE=oci | |
| export BENCHMARK_METRIC_FILE=${{ matrix.image }}-oci.json | |
| export SNAPSHOTTER=overlayfs | |
| sudo -E make smoke-benchmark | |
| - name: Save BenchMark Result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-oci-${{ matrix.image }} | |
| path: smoke/${{ matrix.image }}-oci.json | |
| benchmark-fsversion-v5: | |
| runs-on: ubuntu-latest | |
| needs: [contrib-build, nydus-build] | |
| strategy: | |
| matrix: | |
| include: | |
| - image: wordpress | |
| tag: 6.1.1 | |
| - image: node | |
| tag: 19.8 | |
| - image: python | |
| tag: 3.10.7 | |
| - image: golang | |
| tag: 1.19.3 | |
| - image: ruby | |
| tag: 3.1.3 | |
| - image: amazoncorretto | |
| tag: 8-al2022-jdk | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download Nydus | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nydus-artifact | |
| path: target/release | |
| - name: Download Nydusify | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nydusify-artifact | |
| path: contrib/nydusify/cmd | |
| - name: Prepare Environment | |
| run: | | |
| sudo bash misc/prepare.sh | |
| - name: BenchMark Test | |
| run: | | |
| export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }} | |
| export BENCHMARK_MODE=fs-version-5 | |
| export BENCHMARK_METRIC_FILE=${{ matrix.image }}-fsversion-v5.json | |
| sudo -E make smoke-benchmark | |
| - name: Save BenchMark Result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-fsversion-v5-${{ matrix.image }} | |
| path: smoke/${{ matrix.image }}-fsversion-v5.json | |
| benchmark-fsversion-v6: | |
| runs-on: ubuntu-latest | |
| needs: [contrib-build, nydus-build] | |
| strategy: | |
| matrix: | |
| include: | |
| - image: wordpress | |
| tag: 6.1.1 | |
| - image: node | |
| tag: 19.8 | |
| - image: python | |
| tag: 3.10.7 | |
| - image: golang | |
| tag: 1.19.3 | |
| - image: ruby | |
| tag: 3.1.3 | |
| - image: amazoncorretto | |
| tag: 8-al2022-jdk | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download Nydus | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nydus-artifact | |
| path: target/release | |
| - name: Download Nydusify | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nydusify-artifact | |
| path: contrib/nydusify/cmd | |
| - name: Prepare Environment | |
| run: | | |
| sudo bash misc/prepare.sh | |
| - name: BenchMark Test | |
| run: | | |
| export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }} | |
| export BENCHMARK_MODE=fs-version-6 | |
| export BENCHMARK_METRIC_FILE=${{ matrix.image }}-fsversion-v6.json | |
| sudo -E make smoke-benchmark | |
| - name: Save BenchMark Result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-fsversion-v6-${{ matrix.image }} | |
| path: smoke/${{ matrix.image }}-fsversion-v6.json | |
| benchmark-zran: | |
| runs-on: ubuntu-latest | |
| needs: [contrib-build, nydus-build] | |
| strategy: | |
| matrix: | |
| include: | |
| - image: wordpress | |
| tag: 6.1.1 | |
| - image: node | |
| tag: 19.8 | |
| - image: python | |
| tag: 3.10.7 | |
| - image: golang | |
| tag: 1.19.3 | |
| - image: ruby | |
| tag: 3.1.3 | |
| - image: amazoncorretto | |
| tag: 8-al2022-jdk | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download Nydus | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nydus-artifact | |
| path: target/release | |
| - name: Download Nydusify | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nydusify-artifact | |
| path: contrib/nydusify/cmd | |
| - name: Prepare Environment | |
| run: | | |
| sudo bash misc/prepare.sh | |
| - name: BenchMark Test | |
| run: | | |
| export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }} | |
| export BENCHMARK_MODE=zran | |
| export BENCHMARK_METRIC_FILE=${{ matrix.image }}-zran.json | |
| sudo -E make smoke-benchmark | |
| - name: Save BenchMark Result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-zran-${{ matrix.image }} | |
| path: smoke/${{ matrix.image }}-zran.json | |
| benchmark-result: | |
| runs-on: ubuntu-latest | |
| needs: [benchmark-oci, benchmark-fsversion-v5, benchmark-fsversion-v6, benchmark-zran] | |
| strategy: | |
| matrix: | |
| include: | |
| - image: wordpress | |
| tag: 6.1.1 | |
| - image: node | |
| tag: 19.8 | |
| - image: python | |
| tag: 3.10.7 | |
| - image: golang | |
| tag: 1.19.3 | |
| - image: ruby | |
| tag: 3.1.3 | |
| - image: amazoncorretto | |
| tag: 8-al2022-jdk | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download benchmark-oci | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-oci-${{ matrix.image }} | |
| path: benchmark-result | |
| - name: Download benchmark-fsversion-v5 | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-fsversion-v5-${{ matrix.image }} | |
| path: benchmark-result | |
| - name: Download benchmark-fsversion-v6 | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-fsversion-v6-${{ matrix.image }} | |
| path: benchmark-result | |
| - name: Download benchmark-zran | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-zran-${{ matrix.image }} | |
| path: benchmark-result | |
| - name: Benchmark Summary | |
| run: | | |
| case ${{matrix.image}} in | |
| "wordpress") | |
| echo "### workload: wait the 80 port response" > $GITHUB_STEP_SUMMARY | |
| ;; | |
| "node") | |
| echo "### workload: node index.js; wait the 80 port response" > $GITHUB_STEP_SUMMARY | |
| ;; | |
| "python") | |
| echo "### workload: python -c 'print("hello")'" > $GITHUB_STEP_SUMMARY | |
| ;; | |
| "golang") | |
| echo "### workload: go run main.go" > $GITHUB_STEP_SUMMARY | |
| ;; | |
| "ruby") | |
| echo "### workload: ruby -e "puts \"hello\""" > $GITHUB_STEP_SUMMARY | |
| ;; | |
| "amazoncorretto") | |
| echo "### workload: javac Main.java; java Main" > $GITHUB_STEP_SUMMARY | |
| ;; | |
| esac | |
| cd benchmark-result | |
| metric_files=( | |
| "${{ matrix.image }}-oci.json" | |
| "${{ matrix.image }}-fsversion-v5.json" | |
| "${{ matrix.image }}-fsversion-v6.json" | |
| "${{ matrix.image }}-zran.json" | |
| ) | |
| echo "| bench-result | e2e-time(s) | read-count | read-amount(MB) | image-size(MB) |convert-time(s)|" >> $GITHUB_STEP_SUMMARY | |
| echo "|:-------------|:-----------:|:----------:|:---------------:|:--------------:|:-------------:|" >> $GITHUB_STEP_SUMMARY | |
| for file in "${metric_files[@]}"; do | |
| name=$(basename "$file" .json | sed 's/^[^-]*-\(.*\)$/\1/') | |
| data=$(jq -r '. | "\(.e2e_time / 1e9) \(.read_count) \(.read_amount_total / (1024 * 1024)) \(.image_size / (1024 * 1024)) \(.conversion_elapsed / 1e9)"' "$file" | \ | |
| awk '{ printf "%.2f | %.0f | %.2f | %.2f | %.2f", $1, $2, $3, $4, $5 }') | |
| echo "| $name | $data |" >> $GITHUB_STEP_SUMMARY | |
| done |