Merge branch 'main' into enable-ort-gpu-tests

huggingface · Dec 10, 2024 · ab62319 · ab62319
2 parents 3eecee6 + 4a7cb29
commit ab62319
Show file tree

Hide file tree

Showing 119 changed files with 4,195 additions and 4,631 deletions.
diff --git a/.github/workflows/build_main_documentation.yml b/.github/workflows/build_main_documentation.yml
@@ -10,20 +10,14 @@ on:
 
 jobs:
   build_documentation:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v2
         with:
           repository: 'huggingface/doc-builder'
           path: doc-builder
 
-      - uses: actions/checkout@v2
-        with:
-          repository: 'huggingface/doc-build'
-          path: doc-build
-          token: ${{ secrets.HUGGINGFACE_PUSH }}
-
       - uses: actions/checkout@v2
         with:
           repository: 'huggingface/optimum'
@@ -57,6 +51,7 @@ jobs:
       - name: Free disk space
         run: |
           df -h
+          sudo apt-get update
           sudo apt-get purge -y '^apache.*'
           sudo apt-get purge -y '^imagemagick.*'
           sudo apt-get purge -y '^dotnet.*'
@@ -66,7 +61,7 @@ jobs:
           sudo apt-get purge -y '^mysql.*'
           sudo apt-get purge -y '^java.*'
           sudo apt-get purge -y '^openjdk.*'
-          sudo apt-get purge -y microsoft-edge-stable google-cloud-cli azure-cli google-chrome-stable firefox powershell mono-devel
+          sudo apt-get purge -y microsoft-edge-stable azure-cli google-chrome-stable firefox mono-devel
           df -h
           sudo apt-get autoremove -y >/dev/null 2>&1
           sudo apt-get clean
@@ -110,6 +105,8 @@ jobs:
 
       - name: Setup environment
         run: |
+          python -m venv venv-doc
+          source venv-doc/bin/activate
           pip uninstall -y doc-builder
           cd doc-builder
           git pull origin main
@@ -135,6 +132,7 @@ jobs:
 
       - name: Make Furiosa documentation
         run: |
+          source venv-doc/bin/activate
           cd optimum-furiosa
           pip install .
           sudo apt install software-properties-common
@@ -159,6 +157,7 @@ jobs:
       - name: Make TPU documentation
         run: |
           sudo docker system prune -a -f
+          source venv-doc/bin/activate
           cd optimum-tpu
           pip install -U pip
           pip install . -f https://storage.googleapis.com/libtpu-releases/index.html
@@ -192,6 +191,7 @@ jobs:
 
       - name: Push to repositories
         run: |
+          source venv-doc/bin/activate
           cd optimum/optimum-doc-build
           sudo chmod -R ugo+rwx optimum
           doc-builder push optimum --doc_build_repo_id "hf-doc-build/doc-build" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit ${{ github.sha }} See: https://github.com/huggingface/optimum/commit/${{ github.sha }}" --n_retries 5 --upload_version_yml

diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml
@@ -8,14 +8,15 @@ on:
       - "optimum/**.py"
       - "docs/**.mdx"
       - "docs/**.yml"
+      - ".github/workflows/build_pr_documentation.yml"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   build_documentation:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
       PR_NUMBER: ${{ github.event.number }}
@@ -60,6 +61,8 @@ jobs:
 
       - name: Setup environment
         run: |
+          python -m venv venv-doc
+          source venv-doc/bin/activate
           pip uninstall -y doc-builder
           cd doc-builder
           git pull origin main
@@ -99,6 +102,7 @@ jobs:
       - name: Make TPU documentation
         run: |
           sudo docker system prune -a -f
+          source venv-doc/bin/activate
           cd optimum-tpu
           pip install -U pip
           pip install . -f https://storage.googleapis.com/libtpu-releases/index.html

diff --git a/.github/workflows/check_code_quality.yml b/.github/workflows/check_code_quality.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8]
+        python-version: [3.9]
         os: [ubuntu-20.04]
 
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/dev_test_benckmark.yml b/.github/workflows/dev_test_benckmark.yml
@@ -12,12 +12,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-        - 3.8
-        - 3.9
-        os:
-        - ubuntu-20.04
-    runs-on: ${{ matrix.os }}
+        python-version: ['3.9', '3.11']
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python ${{ matrix.python-version }}
@@ -27,7 +23,7 @@ jobs:
     - name: Install dependencies
       run: |
         pip install wheel
-        pip install .[tests,onnxruntime,benchmark]
+        pip install .[tests,onnxruntime,benchmark] datasets
         pip install -U git+https://github.com/huggingface/evaluate
         pip install -U git+https://github.com/huggingface/diffusers
         pip install -U git+https://github.com/huggingface/transformers

diff --git a/.github/workflows/dev_test_bettertransformer.yml b/.github/workflows/dev_test_bettertransformer.yml
@@ -12,18 +12,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-        - 3.8
         os:
         - ubuntu-20.04
         - macos-13
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-    - name: Setup Python ${{ matrix.python-version }}
+    - name: Setup Python
       uses: actions/setup-python@v2
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: '3.9'
     - name: Install dependencies
       run: |
         pip install .[tests]

diff --git a/.github/workflows/dev_test_dummy_inputs.yml b/.github/workflows/dev_test_dummy_inputs.yml
@@ -12,9 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-        - 3.8
-        - 3.9
+        python-version: ['3.9', '3.11']
         os:
         - ubuntu-20.04
         - macos-13

diff --git a/.github/workflows/dev_test_exporters.yml b/.github/workflows/dev_test_exporters.yml
@@ -12,12 +12,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-        - 3.8
-        - 3.9
-        os:
-        - ubuntu-20.04
-    runs-on: ${{ matrix.os }}
+        python-version: ['3.9', '3.11']
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python ${{ matrix.python-version }}

diff --git a/.github/workflows/dev_test_fx.yml b/.github/workflows/dev_test_fx.yml
@@ -12,9 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-        - 3.8
-        - 3.9
+        python-version: ['3.9', '3.11']
         os:
         - ubuntu-20.04
         - macos-13

diff --git a/.github/workflows/dev_test_onnx.yml b/.github/workflows/dev_test_onnx.yml
@@ -12,9 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-        - 3.8
-        - 3.9
+        python-version: ['3.9', '3.11']
         os:
         - ubuntu-20.04
         - macos-13

diff --git a/.github/workflows/dev_test_onnxruntime.yml b/.github/workflows/dev_test_onnxruntime.yml
@@ -12,9 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-        - 3.8
-        - 3.9
+        python-version: ['3.9', '3.11']
         os:
         - ubuntu-20.04
         - windows-2019

diff --git a/.github/workflows/dev_test_optimum_common.yml b/.github/workflows/dev_test_optimum_common.yml
@@ -12,10 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-        - 3.7
-        - 3.8
-        - 3.9
+        python-version: ['3.9', '3.11']
         os:
         - ubuntu-20.04
         - windows-2019

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
@@ -0,0 +1,23 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v8
+        with:
+          stale-issue-message: 'This issue has been marked as stale because it has been open for 30 days with no activity. This thread will be automatically closed in 5 days if no further activity occurs.'
+          stale-pr-message: 'This PR has been marked as stale because it has been open for 90 days with no activity. This thread will be automatically closed in 30 days if no further activity occurs.'
+          exempt-issue-labels: 'bug,exporters,good first issue,onnx,onnxruntime,quantization'
+          days-before-issue-stale: 30
+          days-before-issue-close: 5
+          days-before-pr-stale: 90
+          days-before-pr-close: 30
+          exempt-all-pr-assignees: true
diff --git a/.github/workflows/test_benckmark.yml b/.github/workflows/test_benckmark.yml
@@ -4,9 +4,9 @@ name: Benchmark suite / Python - Test
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -17,20 +17,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: [3.9]
         os: [ubuntu-20.04]
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        pip install wheel
-        pip install .[tests,onnxruntime,benchmark]
-    - name: Test with unittest
-      run: |
-        python -m unittest discover --start-directory tests/benchmark --pattern 'test_*.py'
+      - uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install wheel
+          pip install .[tests,onnxruntime,benchmark] datasets
+      - name: Test with unittest
+        run: |
+          python -m unittest discover --start-directory tests/benchmark --pattern 'test_*.py'
diff --git a/.github/workflows/test_bettertransformer.yml b/.github/workflows/test_bettertransformer.yml
@@ -15,9 +15,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
-        os: [ubuntu-20.04, macos-13]
-        exclude: [{ python-version: 3.8, os: macos-13 }]
+        python-version: [3.9]
+        os: [ubuntu-20.04, macos-14]
 
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/.github/workflows/test_cli.yml b/.github/workflows/test_cli.yml
@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: [3.9]
         os: [ubuntu-20.04, macos-13]
 
     runs-on: ${{ matrix.os }}
@@ -34,7 +34,7 @@ jobs:
         run: |
           pip install --upgrade pip
           pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip install .[tests,exporters,exporters-tf]
+          pip install .[tests,exporters-tf]
 
       - name: Test with pytest
         run: |

diff --git a/.github/workflows/test_export_onnx.yml b/.github/workflows/test_export_onnx.yml
@@ -2,9 +2,9 @@ name: Exporters ONNX / Python - Test
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -15,27 +15,27 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ['3.9']
         os: [ubuntu-20.04]
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies for pytorch export
-      run: |
-        pip install .[tests,exporters]
-    - name: Test with unittest
-      working-directory: tests
-      run: |
-        pytest exporters/onnx/test_onnx_*.py -s -n auto -m "not tensorflow_test and not timm_test" --durations=0
-    - name: Install dependencies for tensorflow export
-      run: |
-        pip install .[tests,exporters-tf]
-    - name: Test with unittest
-      working-directory: tests
-      run: |
-        pytest exporters/onnx/test_onnx_*.py -n auto -m "tensorflow_test" -s --durations=0
+      - uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies for pytorch export
+        run: |
+          pip install .[tests,exporters]
+      - name: Test with unittest
+        working-directory: tests
+        run: |
+          pytest exporters/onnx/test_onnx_*.py -s -n auto -m "not tensorflow_test and not timm_test" --durations=0
+      - name: Install dependencies for tensorflow export
+        run: |
+          pip install .[tests,exporters-tf]
+      - name: Test with unittest
+        working-directory: tests
+        run: |
+          pytest exporters/onnx/test_onnx_*.py -n auto -m "tensorflow_test" -s --durations=0