From bd3f615ad7e4445ba50392574b711c4278fe7a6c Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 1 Nov 2024 01:49:56 +0000 Subject: [PATCH] [Bash][3/3] Remove benchmack tests from rerun disbled test (#139422) Fixes [#5774](https://github.com/pytorch/test-infra/issues/5774) # Overview Remove benchmark tests from rerun-disabled-tests, this is considered non-unittest. See one page doc: [[Bootcamp Task] Remove non-unittest test during rerun-disabled-tests](https://docs.google.com/document/d/1xffkt_LNC5ZLsoVQDmuKbNqYnMUW_xYYStv66Pr-qac/edit?tab=t.0) # Steps to fix the issue - [ ] Create inductor-unittest.yml to handle unit test and daily rerun for inductor - [ ] Create Inductor-cu124-unittest.yml to handle unit tests and daily rerun for inductor-cu124 - [x] Disable benchmark test in mixed test such as CPP_Wrapper which includes both unittest and benchmark test Pull Request resolved: https://github.com/pytorch/pytorch/pull/139422 Approved by: https://github.com/huydhn --- .ci/pytorch/test.sh | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.ci/pytorch/test.sh b/.ci/pytorch/test.sh index 43014b2b10fecc..b1c0e9b09bed05 100755 --- a/.ci/pytorch/test.sh +++ b/.ci/pytorch/test.sh @@ -384,22 +384,30 @@ test_inductor_cpp_wrapper() { # unit tests with cpp wrapper. python test/run_test.py --include inductor/test_torchinductor.py --verbose - python benchmarks/dynamo/timm_models.py --device cuda --accuracy --amp \ + + # Run inductor benchmark tests with cpp wrapper. + # Skip benchmark tests if it's in rerun-disabled-mode. + if [[ "${PYTORCH_TEST_RERUN_DISABLED_TESTS}" == "1" ]]; then + echo "skip dynamo benchmark tests for rerun-disabled-test" + else + echo "run dynamo benchmark tests with cpp wrapper" + python benchmarks/dynamo/timm_models.py --device cuda --accuracy --amp \ --training --inductor --disable-cudagraphs --only vit_base_patch16_224 \ --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_training.csv" - python benchmarks/dynamo/check_accuracy.py \ - --actual "$TEST_REPORTS_DIR/inductor_cpp_wrapper_training.csv" \ - --expected "benchmarks/dynamo/ci_expected_accuracy/inductor_timm_training.csv" - - python benchmarks/dynamo/torchbench.py --device cuda --accuracy \ - --bfloat16 --inference --inductor --only hf_T5 --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" - python benchmarks/dynamo/torchbench.py --device cuda --accuracy \ - --bfloat16 --inference --inductor --only llama --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" - python benchmarks/dynamo/torchbench.py --device cuda --accuracy \ - --bfloat16 --inference --inductor --only moco --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" - python benchmarks/dynamo/check_accuracy.py \ - --actual "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" \ - --expected "benchmarks/dynamo/ci_expected_accuracy/inductor_torchbench_inference.csv" + python benchmarks/dynamo/check_accuracy.py \ + --actual "$TEST_REPORTS_DIR/inductor_cpp_wrapper_training.csv" \ + --expected "benchmarks/dynamo/ci_expected_accuracy/inductor_timm_training.csv" + + python benchmarks/dynamo/torchbench.py --device cuda --accuracy \ + --bfloat16 --inference --inductor --only hf_T5 --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" + python benchmarks/dynamo/torchbench.py --device cuda --accuracy \ + --bfloat16 --inference --inductor --only llama --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" + python benchmarks/dynamo/torchbench.py --device cuda --accuracy \ + --bfloat16 --inference --inductor --only moco --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" + python benchmarks/dynamo/check_accuracy.py \ + --actual "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" \ + --expected "benchmarks/dynamo/ci_expected_accuracy/inductor_torchbench_inference.csv" + fi } # "Global" flags for inductor benchmarking controlled by TEST_CONFIG