From 4453723e930f08bd18d5be68216b4e35db37aa0f Mon Sep 17 00:00:00 2001 From: "Zhong, Ruijie" Date: Tue, 2 Apr 2024 13:43:49 +0000 Subject: [PATCH] modify the E2E test sciripts --- .github/scripts/inductor-xpu-e2e.sh | 39 ++++++++++++----------- .github/scripts/results-check.sh | 18 +++++------ .github/workflows/inductor_xpu_e2e_ci.yml | 2 +- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/scripts/inductor-xpu-e2e.sh b/.github/scripts/inductor-xpu-e2e.sh index 3eec1f8cb..91248f7b8 100644 --- a/.github/scripts/inductor-xpu-e2e.sh +++ b/.github/scripts/inductor-xpu-e2e.sh @@ -1,7 +1,7 @@ -suite=${1:-huggingface} -dt=${2:-amp_bf16} -mode=${3:-inference} -scenario=${4:-accuracy} +SUITE=${1:-huggingface} +DT=${2:-amp_bf16} +MODE=${3:-inference} +SCENARIO=${4:-accuracy} expected_pass_num=${5:-46} echo -e "=========================================================================" @@ -10,9 +10,9 @@ echo -e "======================================================================= source activate e2e_ci cp .github/scripts/inductor_xpu_test.sh ../pytorch cd ../pytorch -if [[ ${suite} == "timm_models" ]]; then - pip install --no-deps "git+https://github.com/rwightman/pytorch-image-models@b9d43c7dcac1fe05e851dd7be7187b108af593d2" -elif [[ ${suite} == "torchbench" ]]; then +if [[ ${SUITE} == "timm_MODEls" ]]; then + pip install --no-deps "git+https://github.com/rwightman/pytorch-image-MODEls@b9d43c7dcac1fe05e851dd7be7187b108af593d2" +elif [[ ${SUITE} == "torchbench" ]]; then pip install transformers==4.38.1 --no-deps pip install timm==0.9.7 --no-deps apt-get update -y @@ -38,23 +38,24 @@ echo -e "======================================================================= echo -e "E2E Test" echo -e "=========================================================================" source /opt/intel/oneapi/setvars.sh +export HUGGING_FACE_HUB_TOKEN=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 #export PYTORCH_ENABLE_XPU_FALLBACK=1 rm -rf inductor_log -bash inductor_xpu_test.sh ${suite} ${dt} ${mode} ${scenario} xpu 3 +bash inductor_xpu_test.sh ${SUITE} ${DT} ${MODE} ${SCENARIO} xpu 3 echo -e "=========================================================================" echo -e "Test Results Summary" echo -e "=========================================================================" -cd ../pytorch/inductor_log/${suite} -cd ${dt} -echo -e "============ Summary for ${suite} ${dt} ${mode} ${scenario} ============" | tee -a ./${suite}_${dt}_${mode}_${scenario}_e2e_summary.log -awk -i inplace '!seen[$0]++' inductor_${suite}_${dt}_${mode}_xpu_${scenario}.csv -csv_lines=$(cat inductor_${suite}_${dt}_${mode}_xpu_${scenario}.csv | wc -l) +cd ../pytorch/inductor_log/${SUITE} +cd ${DT} +echo -e "============ Summary for ${SUITE} ${DT} ${MODE} ${SCENARIO} ============" | tee -a ./${SUITE}_${DT}_${MODE}_${SCENARIO}_e2e_summary.log +awk -i inplace '!seen[$0]++' inductor_${SUITE}_${DT}_${MODE}_xpu_${SCENARIO}.csv +csv_lines=$(cat inductor_${SUITE}_${DT}_${MODE}_xpu_${SCENARIO}.csv | wc -l) let num_total=csv_lines-1 -num_passed=$(grep -c "pass" inductor_${suite}_${dt}_${mode}_xpu_${scenario}.csv) +num_passed=$(grep -c "pass" inductor_${SUITE}_${DT}_${MODE}_xpu_${SCENARIO}.csv) let num_failed=num_total-num_passed -pass_rate=`awk 'BEGIN{printf "%.2f%%\n",('$num_passed'/'$num_total')*100}'` -echo "num_total: $num_total" | tee -a ./${suite}_${dt}_${mode}_${scenario}_e2e_summary.log -echo "num_passed: $num_passed" | tee -a ./${suite}_${dt}_${mode}_${scenario}_e2e_summary.log -echo "num_failed: $num_failed" | tee -a ./${suite}_${dt}_${mode}_${scenario}_e2e_summary.log -echo "pass_rate: $pass_rate" | tee -a ./${suite}_${dt}_${mode}_${scenario}_e2e_summary.log +#pass_rate=`awk 'BEGIN{printf "%.2f%%\n",('$num_passed'/'$num_total')*100}'` +echo "num_total: $num_total" | tee -a ./${SUITE}_${DT}_${MODE}_${SCENARIO}_e2e_summary.log +echo "num_passed: $num_passed" | tee -a ./${SUITE}_${DT}_${MODE}_${SCENARIO}_e2e_summary.log +echo "num_failed: $num_failed" | tee -a ./${SUITE}_${DT}_${MODE}_${SCENARIO}_e2e_summary.log +#echo "pass_rate: $pass_rate" | tee -a ./${SUITE}_${DT}_${MODE}_${SCENARIO}_e2e_summary.log diff --git a/.github/scripts/results-check.sh b/.github/scripts/results-check.sh index 0758a63b4..08353e689 100644 --- a/.github/scripts/results-check.sh +++ b/.github/scripts/results-check.sh @@ -1,17 +1,17 @@ -suite=${1:-huggingface} -dt=${2:-amp_bf16} -mode=${3:-inference} -scenario=${4:-accuracy} +SUITE=${1:-huggingface} +DT=${2:-amp_bf16} +MODE=${3:-inference} +SCENARIO=${4:-accuracy} expected_pass_num=${5:-46} echo -e "=========================================================================" echo -e "Results Check" echo -e "=========================================================================" -cd ../pytorch/inductor_log/${suite} -cd ${dt} -num_passed=$(grep "num_passed:" ${suite}_${dt}_${mode}_${scenario}_e2e_summary.log | sed -e 's/.*://;s/[^0-9.]//') -if [ $num_passed -lt ${{ inputs.expected_pass_num }} ]; then - echo -e "[ERROR] Inductor E2E CI test for ${suite} ${dt} ${mode} passed_num < ${{ inputs.expected_pass_num }}" +cd ../pytorch/inductor_log/${SUITE} +cd ${DT} +num_passed=$(grep "num_passed:" ${SUITE}_${DT}_${MODE}_${SCENARIO}_e2e_summary.log | sed -e 's/.*://;s/[^0-9.]//') +if [ $num_passed -lt ${expected_pass_num} ]; then + echo -e "[ERROR] Inductor E2E CI test for ${SUITE} ${DT} ${MODE} passed_num < ${expected_pass_num}" exit 1 fi diff --git a/.github/workflows/inductor_xpu_e2e_ci.yml b/.github/workflows/inductor_xpu_e2e_ci.yml index 0b0f9bb56..9025eb345 100644 --- a/.github/workflows/inductor_xpu_e2e_ci.yml +++ b/.github/workflows/inductor_xpu_e2e_ci.yml @@ -68,7 +68,7 @@ jobs: - name: Huggingface AMP_BF16 Inference Accuracy Test run: | - bash .github/scripts/inductor-xpu-e2e.sh --suite huggingface --dt amp_bf16 --mode inference --scenario accuracy --expected_pass_num 46 + bash .github/scripts/inductor-xpu-e2e.sh - name: Upload Triton Inductor E2E CI Data uses: actions/upload-artifact@v3