Skip to content

Commit

Permalink
Add CI perf summary (#786)
Browse files Browse the repository at this point in the history
  • Loading branch information
mtfriesen authored Jan 16, 2025
1 parent 6b96c83 commit 5ab695b
Show file tree
Hide file tree
Showing 5 changed files with 126 additions and 18 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: 'Perf Tests'
description: 'Run Perf Tests'
name: 'XSK Perf Tests'
description: 'Run XSK Perf Tests'
inputs:
ref:
required: false
Expand Down Expand Up @@ -58,22 +58,27 @@ runs:
path: ./${{ inputs.id }}/artifacts/bin
- name: Run xskperfsuite (Generic, Native)
shell: PowerShell
run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }}
run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }}
- name: Run xskperfsuite (TX-inspect, RX-inject)
shell: PowerShell
run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Generic" -TxInspect -RxInject -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }}
run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Generic" -TxInspect -RxInject -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }}
- name: Run xskperfsuite (ZerocopySimulation)
shell: PowerShell
run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }}
run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }}
- name: Run xskperfsuite (Winsock, RIO)
shell: PowerShell
run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }}
run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }}
- name: Upload Logs
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
if: ${{ always() }}
with:
name: logs_perf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}
name: logs_xskperf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}
path: ./${{ inputs.id }}/artifacts/logs
- name: Upload Perf Data
uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b
with:
name: perfdata_xskperf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}
path: ./${{ inputs.id }}/artifacts/perf
- name: Check Drivers
if: ${{ always() }}
shell: PowerShell
Expand Down
41 changes: 33 additions & 8 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -323,8 +323,8 @@ jobs:
path: artifacts/logs
if-no-files-found: ignore

perf_tests:
name: Perf Tests
xskperf_tests:
name: XSK Perf Tests
needs: [build, build_base, resolve_base]
env:
ITERATIONS: ${{ (github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' ) && '8' || '3' }}
Expand All @@ -351,12 +351,12 @@ jobs:
sparse-checkout: |
.github
tools
- uses: ./.github/actions/perf
- uses: ./.github/actions/xskperf
with:
config: ${{ matrix.configuration }}
os: ${{ matrix.windows }}
platform: ${{ matrix.platform }}
- uses: ./.github/actions/perf
- uses: ./.github/actions/xskperf
with:
config: ${{ matrix.configuration }}
os: ${{ matrix.windows }}
Expand All @@ -365,7 +365,7 @@ jobs:
id: "baseref"
- name: Compare Results
shell: PowerShell
run: tools\perfcompare.ps1 -DataFile1 baseref/artifacts/logs/xskperfsuite.json -DataFile2 artifacts/logs/xskperfsuite.json -Metric pps
run: tools\perfcompare.ps1 -DataFile1 baseref/artifacts/perf/xskperfsuite*.json -DataFile2 artifacts/perf/xskperfsuite*.json -Metric pps

ring_perf_tests:
name: Ring Perf Tests
Expand Down Expand Up @@ -434,21 +434,46 @@ jobs:
run: tools/prepare-machine.ps1 -ForPerfTest -Platform ${{ matrix.platform }} -RequireNoReboot -Verbose
- name: Run rxfilter (drop, generic)
shell: PowerShell
run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Generic
run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Generic -RawResultsFile artifacts/perf/rxfilterperf_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }}.json -CommitHash ${{ github.sha }}
- name: Run rxfilter (drop, native)
shell: PowerShell
run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Native
run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Native -RawResultsFile artifacts/perf/rxfilterperf_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }}.json -CommitHash ${{ github.sha }}
- name: Upload Logs
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08
if: ${{ always() }}
with:
name: logs_rxfilter_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }}
path: artifacts/logs
if-no-files-found: ignore
- name: Upload Perf Data
uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b
with:
name: perfdata_rxfilter_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }}
path: artifacts/perf
if-no-files-found: ignore
- name: Check Drivers
shell: PowerShell
run: tools/check-drivers.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose

perf_test_summary:
name: Perf Test Summary
needs: [xskperf_tests, rxfilter_perf_tests]
runs-on: windows-2022
steps:
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
- name: Download Artifacts
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
with:
path: artifacts/perf
pattern: perfdata_*
merge-multiple: true
- name: Summarize Perf Data
shell: PowerShell
run: |
Install-Module -Name FormatMarkdownTable -Force -AllowClobber -Scope CurrentUser -Repository PSGallery -Verbose
tools/summarize-perf.ps1 -Files artifacts/perf/*.json | Format-MarkdownTableTableStyle -Property ScenarioName, Platform, CommitHash, MetricName, Average, Minimum, Maximum -DoNotCopyToClipboard -ShowMarkdown -HideStandardOutput >> $env:GITHUB_STEP_SUMMARY
downlevel_functional_tests:
name: Downlevel Functional Tests
needs: build
Expand Down Expand Up @@ -607,7 +632,7 @@ jobs:
Complete:
name: Complete
if: always()
needs: [build, build_allpackage, onebranch_build_validation, functional_tests, stress_tests, pktfuzz_tests, perf_tests, ring_perf_tests, rxfilter_perf_tests, xskfwdkm_test, downlevel_functional_tests, create_artifacts]
needs: [build, build_allpackage, onebranch_build_validation, functional_tests, stress_tests, pktfuzz_tests, xskperf_tests, ring_perf_tests, rxfilter_perf_tests, xskfwdkm_test, downlevel_functional_tests, create_artifacts]
runs-on: ubuntu-latest
permissions: {} # No need for any permissions.
steps:
Expand Down
2 changes: 1 addition & 1 deletion tools/common.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ function Write-PerfDataSet {
)

New-Item -ItemType Directory -Force -Path (Split-Path $File) | Out-Null
$DataSet | ConvertTo-Json -Depth 100 | Out-File -FilePath $File -Encoding utf8
ConvertTo-Json -InputObject $DataSet -Depth 100 | Out-File -FilePath $File -Encoding utf8
}

function New-PerfData {
Expand Down
29 changes: 27 additions & 2 deletions tools/rxfilterperf.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,13 @@ param (
[switch]$Fndis = $false,

[Parameter(Mandatory = $false)]
[string]$Action = "Drop"
[string]$Action = "Drop",

[Parameter(Mandatory=$false)]
[string]$RawResultsFile = "",

[Parameter(Mandatory=$false)]
[string]$CommitHash = ""
)

Set-StrictMode -Version 'Latest'
Expand Down Expand Up @@ -84,7 +90,26 @@ try {

& $RootDir\tools\xdpmpratesim.ps1 -AdapterName XDPMP -RxFramesPerInterval 1000

Write-Output "Filtered $(($EndPackets - $StartPackets)) packets in $Timeout seconds ($(($EndPackets - $StartPackets) / 1000 / $Timeout) Kpps)."
$Kpps = ($EndPackets - $StartPackets) / 1000 / $Timeout
Write-Output "Filtered $(($EndPackets - $StartPackets)) packets in $Timeout seconds ($Kpps Kpps)."

if (![string]::IsNullOrEmpty($RawResultsFile)) {
$Results = New-PerfDataSet -Files $RawResultsFile

[void]$Results.Add($(
New-PerfData -ScenarioName "RXFILTER-$XdpMode-$Action-$($QueueCount)Q" `
-Platform $Platform -CommitHash $CommitHash -Metrics @(
@{
Name = "pps"
Value = $Kpps
Scale = 1000
}
)
)
)

Write-PerfDataSet -DataSet $Results -File $RawResultsFile
}
} finally {
if ($RxFilterProcess) {
Stop-Process -InputObject $RxFilterProcess
Expand Down
53 changes: 53 additions & 0 deletions tools/summarize-perf.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
param (
[Parameter(Mandatory=$true)]
[string[]]$Files
)

Set-StrictMode -Version 'Latest'
$ErrorActionPreference = 'Stop'

# Important paths.
$RootDir = Split-Path $PSScriptRoot -Parent
. $RootDir\tools\common.ps1

$Summary = @{}

# First, merge metrics from all runs, indexed by scenario name, platform, and commit hash.
foreach ($File in Get-Item -Path $Files) {
$Data = New-PerfDataSet -Files $File

foreach ($Run in $Data) {
$Key = [ordered]@{
ScenarioName = $Run.ScenarioName
Platform = $Run.Platform
CommitHash = $Run.CommitHash
}
$KeyString = $Key | Out-String

if (!$Summary.ContainsKey($KeyString)) {
$Summary[$KeyString] = @{
Key = $Key
Metrics = @{}
}
}

foreach ($Metric in $Run.Metrics) {
[array]$Summary[$KeyString].Metrics[$Metric.Name] += $Metric.Value
}
}
}

# Second, flatten the merged metrics and summarize the results across runs.
$Summary.Values | ForEach-Object {
foreach ($Metric in $_.Metrics.GetEnumerator()) {
[pscustomobject]@{
ScenarioName = $_.Key.ScenarioName
Platform = $_.Key.Platform
CommitHash = $_.Key.CommitHash
MetricName = $Metric.Key
Average = $Metric.Value | Measure-Object -Average | Select-Object -ExpandProperty Average
Minimum = $Metric.Value | Measure-Object -Minimum | Select-Object -ExpandProperty Minimum
Maximum = $Metric.Value | Measure-Object -Maximum | Select-Object -ExpandProperty Maximum
}
}
} | Sort-Object -Property ScenarioName, Platform, CommitHash, MetricName

0 comments on commit 5ab695b

Please sign in to comment.