Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#9714: Create separate unstable pipeline for stable diffusion device perf #9812

Closed
wants to merge 3 commits into from
Closed

#9714: Create separate unstable pipeline for stable diffusion device perf #9812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70836bc
#9714: Move WH device perf to a virtual machine with a higher timeout…
tt-rkim Jun 28, 2024
982d64d
#9714: Separate stable diffusion into an unstable single-card pipeline
tt-rkim Jun 28, 2024
d13d15a
#9714: Add handling for new pipelines in run_tests.sh
tt-rkim Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions .github/workflows/perf-device-models.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: "Device perf regressions and output report"
name: "(Single-card) Device perf tests"

on:
workflow_dispatch:
Expand All @@ -14,8 +14,10 @@ jobs:
fail-fast: false
matrix:
test-info: [
{name: "GS", arch: grayskull, runs-on: ["perf-no-reset-grayskull", "self-reset"], machine-type: "bare_metal", timeout: 40},
{name: "N300 WH B0", arch: wormhole_b0, runs-on: ["perf-wormhole_b0", "self-reset"], machine-type: "bare_metal", timeout: 30},
{name: "GS", arch: grayskull, runs-on: ["perf-no-reset-grayskull", "self-reset"], machine-type: "bare_metal", model-type: "other", timeout: 40},
# Runs on virtual machine now
{name: "N300", arch: wormhole_b0, runs-on: ["model-runner-wormhole_b0"], machine-type: "bare_metal", model-type: "other", timeout: 30},
{name: "Unstable N300", arch: wormhole_b0, runs-on: ["model-runner-wormhole_b0"], machine-type: "bare_metal", model-type: "unstable_models", timeout: 25},
]
name: "${{ matrix.test-info.name }} device perf"
env:
Expand Down Expand Up @@ -43,7 +45,7 @@ jobs:
timeout-minutes: ${{ matrix.test-info.timeout }}
run: |
source python_env/bin/activate
./tests/scripts/run_tests.sh --tt-arch $ARCH_NAME --pipeline-type models_device_performance_${{ matrix.test-info.machine-type }}
./tests/scripts/run_tests.sh --tt-arch $ARCH_NAME --pipeline-type ${{ matrix.test-info.model-type }}_device_performance_${{ matrix.test-info.machine-type }}
- name: Check device perf report exists
id: check-device-perf-report
if: ${{ !cancelled() }}
Expand All @@ -56,5 +58,5 @@ jobs:
if: ${{ !cancelled() && steps.check-device-perf-report.conclusion == 'success' }}
uses: actions/upload-artifact@v4
with:
name: device-perf-report-csv-${{ matrix.test-info.arch }}-${{ matrix.test-info.machine-type }}
name: device-perf-report-csv-${{ matrix.test-info.arch }}-${{ matrix.test-info.machine-type }}-${{ matrix.test-info.model-type }}
path: "${{ steps.check-device-perf-report.outputs.device_perf_report_filename }}"
14 changes: 11 additions & 3 deletions tests/scripts/run_performance.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,6 @@ run_perf_models_cnn_javelin() {
run_device_perf_models() {
local test_marker=$1

env pytest tests/device_perf_tests/stable_diffusion -m $test_marker --timeout=600

if [ "$tt_arch" == "grayskull" ]; then
#TODO(MO): Until #6560 is fixed, GS device profiler test are grouped with
#Model Device perf regression tests to make sure thy run on no-soft-reset BMs
Expand Down Expand Up @@ -93,6 +91,14 @@ run_device_perf_models() {
env python models/perf/merge_device_perf_results.py
}

run_unstable_device_perf_models() {
local test_marker=$1

env pytest tests/device_perf_tests/stable_diffusion -m $test_marker --timeout=600

## Merge all the generated reports
env python models/perf/merge_device_perf_results.py
}
run_device_perf_ops() {
local test_marker=$1

Expand Down Expand Up @@ -140,7 +146,9 @@ main() {
exit 1
fi

if [[ "$pipeline_type" == *"device_performance"* ]]; then
if [[ "$pipeline_type" == "unstable_models_device_performance"* ]]; then
run_unstable_device_perf_models "$test_marker"
elif [[ "$pipeline_type" == "other_device_performance"* ]]; then
run_device_perf_models "$test_marker"
run_device_perf_ops "$test_marker"
elif [[ "$pipeline_type" == "llm_javelin_models_performance"* ]]; then
Expand Down
2 changes: 1 addition & 1 deletion tests/scripts/run_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ run_pipeline_tests() {
run_post_commit_pipeline_tests "$tt_arch" "$pipeline_type" "$dispatch_mode"
elif [[ $pipeline_type == "frequent_api" ]]; then
run_frequent_api_pipeline_tests "$tt_arch" "$pipeline_type" "$dispatch_mode"
elif [[ $pipeline_type == *"models_performance_bare_metal" || $pipeline_type == "models_device_performance_bare_metal" ]]; then
elif [[ $pipeline_type == *"models_performance_bare_metal" || $pipeline_type == *"device_performance_bare_metal" ]]; then
run_models_performance_bare_metal_pipeline_tests "$tt_arch" "$pipeline_type" "$dispatch_mode"
elif [[ $pipeline_type == "models_performance_virtual_machine" ]]; then
run_models_performance_virtual_machine_pipeline_tests "$tt_arch" "$pipeline_type"
Expand Down
Loading