installation: Switch to UV for dependency management (#3294)

## Summary Adopt `uv` for dependency management, replacing the existing `pip` approach. ## Motivation `uv` is developed by [Astral](https://astral.sh/), who also wrote `ruff` which is already in use in xDSL CI pipelines. It offers the benefits of much faster dependency resolution and installation, along with leveraging the more modern `pyproject.toml` configuration format defined in PEPs [518](https://peps.python.org/pep-0518/) and [618](https://peps.python.org/pep-0621/). Requested by @superlopuh ## Changes This replacement requires the following changes: - [x] Modification to the `venv` target of the Makefile to create the virtual environment - [x] Update CI flows to install with `uv` as opposed to `pip`, including testing - [x] `ci-core.yaml` - [x] `ci-mlir.yaml` - [x] `ci-notebooks.yaml` - [x] `ci-pyright-fails.yaml` - [x] `code-formatting.yaml` - [ ] `jupyterlite.yaml` (cannot easily be spawned due to workflow triggers) - [ ] `pythonpublish.yaml` (cannot easily be spawned due to required secrets) - [x] `release-notes.yaml` (requires no changes) - [x] Update documentation to reflect these changes - [x] Test across operating systems - [x] Windows - [x] MacOS - [x] NixOS - [x] Resolve changes required to Dependabot ## Post-merge checklist Once merged, we should check the following things: - [ ] The release flow worked correctly (including versioneer) - [ ] Users are able to switch to `uv` and instructions are clear - [ ] The `update-bot` workflow creates helpful and correct PRs in a timely manner --------- Co-authored-by: Sasha Lopoukhine <[email protected]> Co-authored-by: Alex Rice <[email protected]>
xdslproject · Dec 11, 2024 · 056f76e · 056f76e
1 parent 1baf7d8
commit 056f76e
Show file tree

Hide file tree

Showing 15 changed files with 2,765 additions and 136 deletions.
diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
@@ -15,28 +15,26 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
+    env:
+      VENV_DIR: .venv
     strategy:
       matrix:
         python-version: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up Python
-      uses: actions/setup-python@v5
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
       with:
-        python-version: ${{ matrix.python-version }}
-        cache: 'pip' # caching pip dependencies
-        cache-dependency-path: |
-          setup.py
-          requirements.txt
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
 
-    - name: Upgrade pip
-      run: |
-        pip install --upgrade pip
+    - name: Set up Python ${{ matrix.python-version }}
+      run: uv python install ${{ matrix.python-version }}
 
-    - name: Install the package and dev dependencies locally
-      run: pip install -e ".[dev]"
+    - name: Install the project
+      run: VENV_EXTRAS="--extra dev" make venv
 
     - name: Run all tests
       run: |

diff --git a/.github/workflows/ci-mlir.yml b/.github/workflows/ci-mlir.yml
@@ -19,6 +19,7 @@ jobs:
         python-version: ['3.10', '3.11', '3.12']
 
     env:
+      VENV_DIR: .venv
       LLVM_SYMBOLIZER_PATH: /usr/lib/llvm-11/bin/llvm-symbolizer
       MLIR-Version: d401987fe349a87c53fe25829215b080b70c0c1a
     steps:
@@ -33,24 +34,23 @@ jobs:
         packages: mesa-vulkan-drivers
         version: 1.0
 
-    - name: Python Setup
-      uses: actions/setup-python@v5
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
       with:
-        python-version: ${{ matrix.python-version }}
-        cache: 'pip' # caching pip dependencies
-        cache-dependency-path: |
-          setup.py
-          requirements.txt
+        enable-cache: true
+        cache-dependency-glob: "xdsl/uv.lock"
 
-    - name: Upgrade pip
+    - name: Set up Python ${{ matrix.python-version }}
       run: |
-        pip install --upgrade pip
+        # Change directory so that xdsl-opt can be found during installation.
+        cd xdsl
+        uv python install ${{ matrix.python-version }}
 
     - name: Install the package locally and nbval
       run: |
         # Change directory so that xdsl-opt can be found during installation.
         cd xdsl
-        pip install -r requirements.txt
+        make venv
 
     - name: Cache binaries
       id: cache-binary
@@ -99,36 +99,36 @@ jobs:
     - name: Test with pytest and generate code coverage
       run: |
         cd xdsl
-        pytest -W error --cov --cov-config=.coveragerc .
+        uv run pytest -W error --cov --cov-config=.coveragerc .
 
     - name: Execute lit tests
       run: |
         cd xdsl
         # Add mlir-opt to the path
         export PATH=$PATH:${GITHUB_WORKSPACE}/llvm-project/build/bin/
-        lit -v tests/filecheck/ -DCOVERAGE
-        lit -v docs/Toy/examples/ -DCOVERAGE
+        uv run lit -v tests/filecheck/ -DCOVERAGE
+        uv run lit -v docs/Toy/examples/ -DCOVERAGE
 
     - name: Test MLIR dependent examples/tutorials
       run: |
         cd xdsl
         # Add mlir-opt to the path
         export PATH=$PATH:${GITHUB_WORKSPACE}/llvm-project/build/bin/
-        pytest --nbval docs/mlir_interoperation.ipynb --maxfail 1 -vv
+        uv run pytest --nbval docs/mlir_interoperation.ipynb --maxfail 1 -vv
 
     - name: Test ONNX-dependent marimo notebooks
       run: |
         cd xdsl
         # Add mlir-opt to the path
         export PATH=$PATH:${GITHUB_WORKSPACE}/llvm-project/build/bin/
-        make tests-marimo-onnx
+        uv run make tests-marimo-onnx
 
     - name: Combine coverage data
       run: |
         cd xdsl
-        coverage combine --append
-        coverage report
-        coverage xml
+        uv run coverage combine --append
+        uv run coverage report
+        uv run coverage xml
 
     - name: Upload coverage to Codecov
       if: matrix.python-version == '3.10'

diff --git a/.github/workflows/ci-notebooks.yml b/.github/workflows/ci-notebooks.yml
@@ -15,29 +15,32 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
+    env:
+      VENV_DIR: .venv
     strategy:
       matrix:
         python-version: ['3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
       with:
-        python-version: ${{ matrix.python-version }}
-        cache: 'pip' # caching pip dependencies
-        cache-dependency-path: |
-          setup.py
-          requirements.txt
-    - name: Upgrade pip
-      run: |
-        pip install --upgrade pip
-    - name: Install the package locally
-      run: pip install -r requirements.txt
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+
+    - name: Set up Python ${{ matrix.python-version }}
+      run: uv python install ${{ matrix.python-version }}
+
+    - name: Install the project
+      run: make venv
+
     - name: Run examples/tutorials
       run: |
         # mlir_interoperation.ipynb is dependent on MLIR, and is tested in the MLIR-enabled workflow.
-        pytest -W error --nbval -vv docs --ignore=docs/mlir_interoperation.ipynb
+        uv run pytest -W error --nbval -vv docs --ignore=docs/mlir_interoperation.ipynb
+
     - name: Test marimo notebooks
       run: |
         make tests-marimo
diff --git a/.github/workflows/ci-pyright-fails.yml b/.github/workflows/ci-pyright-fails.yml
@@ -19,24 +19,31 @@ jobs:
         python-version: ["3.10", "3.11", "3.12"]
 
     env:
+      VENV_DIR: .venv
       PYRIGHT_VERSION: 1.0
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip' # caching pip dependencies
-          cache-dependency-path: |
-            setup.py
-            requirements.txt
-      - name: Install the package locally
-        run:
-          pip install -r requirements.txt
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install the project and activate its virtual environment
+        run: |
+          make venv
+          echo "$PWD/.venv/bin" >> $GITHUB_PATH
+
       - name: Generate IRDL stubs
         run: xdsl-stubgen
+
       - name: Pyright
         uses: jakebailey/pyright-action@v2
         with:
           version: PATH
+          venv-path: .venv
diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml
@@ -21,18 +21,14 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up Python
-      uses: actions/setup-python@v5
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
       with:
-        python-version: ${{ matrix.python-version }}
-        cache: 'pip' # caching pip dependencies
-        cache-dependency-path: |
-          setup.py
-          requirements.txt
-
-    - name: Upgrade pip
-      run: |
-        pip install --upgrade pip
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+
+    - name: Set up Python ${{ matrix.python-version }}
+      run: uv python install ${{ matrix.python-version }}
 
     - name: Run code formatting checks with pre-commit
       uses: pre-commit/[email protected]
diff --git a/.github/workflows/jupyterlite.yml b/.github/workflows/jupyterlite.yml
@@ -1,8 +1,7 @@
 name: Deploy JupyterLite Page
 
 on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
+  # Trigger the workflow every day at 4:15am
   schedule:
     - cron: '15 4 * * *'
 
@@ -15,18 +14,15 @@ jobs:
         with:
           path: xdsl
 
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-          cache: 'pip' # caching pip dependencies
-          cache-dependency-path: |
-            setup.py
-            requirements.txt
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Set up Python
+        run: uv python install 3.11
 
       - name: Install dependencies
         run: |
-          python -m pip install jupyterlite-core==0.2.3 jupyterlite-pyodide-kernel==0.2.1 libarchive-c build pyodide-build==0.24.1 jupyter-server
+          uv tool install jupyterlite-core==0.2.3 jupyterlite-pyodide-kernel==0.2.1 libarchive-c build pyodide-build==0.24.1 jupyter-server
 
       - name: Build xDSL source distribution
         run: |

diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml
@@ -9,18 +9,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+
     - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-        cache: 'pip' # caching pip dependencies
-        cache-dependency-path: |
-          setup.py
-          requirements.txt
+      run: uv python install 3.11
+
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        uv tool install setuptools wheel twine
+
     - name: Build and publish
       run: |
         python setup.py sdist bdist_wheel

diff --git a/.github/workflows/update-bot.yml b/.github/workflows/update-bot.yml
@@ -0,0 +1,38 @@
+name: Update Bot
+
+on:
+  workflow_dispatch:
+  # Set the schedule, every week at 8:00am on Monday
+  schedule:
+    - cron: 0 8 * * 1
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  lock:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v3
+
+      - run: |
+          echo "\`\`\`" > uv_output.md
+          uv lock &>> uv_output.md
+          echo "\`\`\`" >> uv_output.md
+
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: Update uv lockfile
+          title: Update uv lockfile
+          body-path: uv_output.md
+          branch: update-uv
+          base: main
+          labels: install
+          delete-branch: true
+          add-paths: uv.lock
+          assignees: math-fehr, georgebisbas, superlopuh