[feat: Benchmarking Workflow] add stuff for a benchmarking workflow #801
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fast tests for PRs - Test Fetcher | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - ci-* | |
| env: | |
| DIFFUSERS_IS_CI: yes | |
| OMP_NUM_THREADS: 4 | |
| MKL_NUM_THREADS: 4 | |
| PYTEST_TIMEOUT: 60 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| setup_pr_tests: | |
| name: Setup PR Tests | |
| runs-on: docker-cpu | |
| container: | |
| image: diffusers/diffusers-pytorch-cpu | |
| options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ | |
| defaults: | |
| run: | |
| shell: bash | |
| outputs: | |
| matrix: ${{ steps.set_matrix.outputs.matrix }} | |
| test_map: ${{ steps.set_matrix.outputs.test_map }} | |
| steps: | |
| - name: Checkout diffusers | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install dependencies | |
| run: | | |
| apt-get update && apt-get install libsndfile1-dev libgl1 -y | |
| python -m pip install -e .[quality,test] | |
| - name: Environment | |
| run: | | |
| python utils/print_env.py | |
| echo $(git --version) | |
| - name: Fetch Tests | |
| run: | | |
| python utils/tests_fetcher.py | tee test_preparation.txt | |
| - name: Report fetched tests | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test_fetched | |
| path: test_preparation.txt | |
| - id: set_matrix | |
| name: Create Test Matrix | |
| # The `keys` is used as GitHub actions matrix for jobs, i.e. `models`, `pipelines`, etc. | |
| # The `test_map` is used to get the actual identified test files under each key. | |
| # If no test to run (so no `test_map.json` file), create a dummy map (empty matrix will fail) | |
| run: | | |
| if [ -f test_map.json ]; then | |
| keys=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); d = list(test_map.keys()); print(json.dumps(d))') | |
| test_map=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); print(json.dumps(test_map))') | |
| else | |
| keys=$(python3 -c 'keys = ["dummy"]; print(keys)') | |
| test_map=$(python3 -c 'test_map = {"dummy": []}; print(test_map)') | |
| fi | |
| echo $keys | |
| echo $test_map | |
| echo "matrix=$keys" >> $GITHUB_OUTPUT | |
| echo "test_map=$test_map" >> $GITHUB_OUTPUT | |
| run_pr_tests: | |
| name: Run PR Tests | |
| needs: setup_pr_tests | |
| if: contains(fromJson(needs.setup_pr_tests.outputs.matrix), 'dummy') != true | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| modules: ${{ fromJson(needs.setup_pr_tests.outputs.matrix) }} | |
| runs-on: docker-cpu | |
| container: | |
| image: diffusers/diffusers-pytorch-cpu | |
| options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout diffusers | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install dependencies | |
| run: | | |
| apt-get update && apt-get install libsndfile1-dev libgl1 -y | |
| python -m pip install -e .[quality,test] | |
| python -m pip install accelerate | |
| - name: Environment | |
| run: | | |
| python utils/print_env.py | |
| - name: Run all selected tests on CPU | |
| run: | | |
| python -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }} | |
| - name: Failure short reports | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| run: | | |
| cat reports/${{ matrix.modules }}_tests_cpu_stats.txt | |
| cat reports/${{ matrix.modules }}_tests_cpu_failures_short.txt | |
| - name: Test suite reports artifacts | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.modules }}_test_reports | |
| path: reports | |
| run_staging_tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - name: Hub tests for models, schedulers, and pipelines | |
| framework: hub_tests_pytorch | |
| runner: docker-cpu | |
| image: diffusers/diffusers-pytorch-cpu | |
| report: torch_hub | |
| name: ${{ matrix.config.name }} | |
| runs-on: ${{ matrix.config.runner }} | |
| container: | |
| image: ${{ matrix.config.image }} | |
| options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout diffusers | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install dependencies | |
| run: | | |
| apt-get update && apt-get install libsndfile1-dev libgl1 -y | |
| python -m pip install -e .[quality,test] | |
| - name: Environment | |
| run: | | |
| python utils/print_env.py | |
| - name: Run Hub tests for models, schedulers, and pipelines on a staging env | |
| if: ${{ matrix.config.framework == 'hub_tests_pytorch' }} | |
| run: | | |
| HUGGINGFACE_CO_STAGING=true python -m pytest \ | |
| -m "is_staging_test" \ | |
| --make-reports=tests_${{ matrix.config.report }} \ | |
| tests | |
| - name: Failure short reports | |
| if: ${{ failure() }} | |
| run: cat reports/tests_${{ matrix.config.report }}_failures_short.txt | |
| - name: Test suite reports artifacts | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: pr_${{ matrix.config.report }}_test_reports | |
| path: reports |