From 8f925a713bc23fb8a421b3ab1bdd0e023cd58f09 Mon Sep 17 00:00:00 2001 From: Sasha Lopoukhine Date: Mon, 6 Nov 2023 09:07:51 +0000 Subject: [PATCH] misc: move dependencies out of requirements-optional (#1744) After reading up a bit on the recommended uses of `requirements.txt`, `pyproject.toml`, and `setup.py`, I have come to this understanding: `requirements.txt` is supposed to be used only for local development. The understanding is that it is a document that is pinned to the local venv of the developers of the project. It is supposed to be more strict and specific than the packaging for pypi publishing, which is supposed to be more permissive. `pyproject.toml`/`setup.py` are supposed to specify the ranges of version of dependencies for clients of xdsl that `pip install xdsl` from their own projects. It is not recommended to couple the two concepts to each other. This PR shuffles the same set of dependencies around between files. `requirements.txt` now holds all the developer dependencies for testing and linting. People who `pip install xdsl` should not care that their version of pytest is not the same as ours. `pyproject.toml` now holds the info of the required dependencies (immutabledict, typing-extensions, ordered-set), and "extras" dependencies (riscemu, wgpu). The new command to install xdsl for local development is `pip install -r requirements.txt` (or `make venv`). I've tested that `pip install -e .` produces exactly the same output from `pip list` before and after, and the command above installs exactly the same packages as `pip install -e ".[extras]"` before. Now the same command skips the testing projects, which I think is good. Here are some links for future reference: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ https://peps.python.org/pep-0508/ https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html --- .github/workflows/ci-core.yml | 2 +- .github/workflows/ci-mlir.yml | 2 +- .github/workflows/ci-notebooks.yml | 2 +- .github/workflows/ci-pyright-fails.yml | 4 +- Makefile | 5 +-- pyproject.toml | 10 ++++- requirements-optional.txt | 19 --------- requirements.txt | 21 ++++++++-- setup.py | 54 -------------------------- 9 files changed, 34 insertions(+), 85 deletions(-) delete mode 100644 requirements-optional.txt diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index b05aac2ab2..5fb1e511d6 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -38,7 +38,7 @@ jobs: pip install --upgrade pip - name: Install the package locally - run: pip install -e ".[extras]" + run: pip install -r requirements.txt - name: Test with pytest run: | diff --git a/.github/workflows/ci-mlir.yml b/.github/workflows/ci-mlir.yml index d8b9f1dd4f..fac2dbca42 100644 --- a/.github/workflows/ci-mlir.yml +++ b/.github/workflows/ci-mlir.yml @@ -45,7 +45,7 @@ jobs: run: | # Change directory so that xdsl-opt can be found during installation. cd xdsl - pip install -e ".[extras]" + pip install -r requirements.txt - name: Cache binaries id: cache-binary diff --git a/.github/workflows/ci-notebooks.yml b/.github/workflows/ci-notebooks.yml index 75a0e65b1c..b421119bc7 100644 --- a/.github/workflows/ci-notebooks.yml +++ b/.github/workflows/ci-notebooks.yml @@ -29,7 +29,7 @@ jobs: run: | pip install --upgrade pip - name: Install the package locally - run: pip install -e ".[extras]" + run: pip install -r requirements.txt - name: Run examples/tutorials run: | # mlir_interoperation.ipynb is dependent on MLIR, and is tested in the MLIR-enabled workflow. diff --git a/.github/workflows/ci-pyright-fails.yml b/.github/workflows/ci-pyright-fails.yml index 3ce643ef21..0674b925ae 100644 --- a/.github/workflows/ci-pyright-fails.yml +++ b/.github/workflows/ci-pyright-fails.yml @@ -31,10 +31,10 @@ jobs: run: | pip install --upgrade pip - name: Install the package locally - run: pip install -e ".[extras]" + run: pip install -r requirements.txt - name: retrieve pyright version run: | # get pyright version from requirements optional file: - VER=$(sed -n 's/^.*pyright==\s*\(\S*\).*$/\1/p' requirements-optional.txt) + VER=$(sed -n 's/^.*pyright==\s*\(\S*\).*$/\1/p' requirements.txt) echo "PYRIGHT_VERSION=$VER" >> $GITHUB_ENV - name: Pyright uses: jakebailey/pyright-action@v1 diff --git a/Makefile b/Makefile index 3e910bb0ae..b509794267 100644 --- a/Makefile +++ b/Makefile @@ -18,11 +18,10 @@ TESTS_COVERAGE_FILE = ${COVERAGE_FILE}.tests .PHONY: coverage coverage-tests coverage-filecheck-tests coverage-report-html coverage-report-md # set up the venv with all dependencies for development -venv: requirements-optional.txt requirements.txt +venv: requirements.txt python3 -m venv ${VENV_DIR} . ${VENV_DIR}/bin/activate - python3 -m pip --require-virtualenv install -r requirements-optional.txt -r requirements.txt - python3 -m pip --require-virtualenv install -e ".[extras]" + python3 -m pip --require-virtualenv install -r requirements.txt # remove all caches and the venv clean: diff --git a/pyproject.toml b/pyproject.toml index 63e5ffc259..a9f01494df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,15 @@ requires-python = ">=3.10" license = { text = "MIT License" } authors = [{ name = "Mathieu Fehr", email = "mathieu.fehr@ed.ac.uk" }] classifiers = ["Programming Language :: Python :: 3"] -dynamic = ["version", "dependencies"] +dependencies = [ + "immutabledict<3.0.1", + "typing-extensions>=4.7,<5", + "ordered-set==4.1.0", +] +dynamic = ["version"] + +[project.optional-dependencies] +extras = ["riscemu==2.2.5", "wgpu==0.11.0"] [project.urls] Homepage = "https://xdsl.dev/" diff --git a/requirements-optional.txt b/requirements-optional.txt deleted file mode 100644 index a55b2de0ac..0000000000 --- a/requirements-optional.txt +++ /dev/null @@ -1,19 +0,0 @@ -black[jupyter] -toml<0.11 -pytest-cov -coverage<8.0.0 -ipykernel -pytest<8.0 -nbval<0.11 -filecheck<0.0.24 -lit<18.0.0 -pre-commit==3.5.0 -ruff==0.1.4 -riscemu==2.2.5 -asv<0.7 -isort==5.12.0 -nbconvert>=7.7.2,<8.0.0 -# pyright version has to be fixed with `==`. The CI parses this file -# and installs the according version for typechecking. -pyright==1.1.334 -wgpu==0.11.0 diff --git a/requirements.txt b/requirements.txt index be203ac98b..525f90a05a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,19 @@ pip<24.0 -immutabledict<3.0.1 -typing-extensions>=4.7,<5 -ordered-set==4.1.0 +black[jupyter] +toml<0.11 +pytest-cov +coverage<8.0.0 +ipykernel +pytest<8.0 +nbval<0.11 +filecheck<0.0.24 +lit<18.0.0 +pre-commit==3.5.0 +ruff==0.1.4 +asv<0.7 +isort==5.12.0 +nbconvert>=7.7.2,<8.0.0 +# pyright version has to be fixed with `==`. The CI parses this file +# and installs the according version for typechecking. +pyright==1.1.334 +-e .[extras] diff --git a/setup.py b/setup.py index 1c44206368..cb875c28ff 100644 --- a/setup.py +++ b/setup.py @@ -1,66 +1,12 @@ -import re from collections.abc import Mapping -from pathlib import Path from typing import cast from setuptools import Command, find_packages, setup import versioneer -# Add README.md as long description -this_directory = Path(__file__).parent -long_description = (this_directory / "README.md").read_text() - -git_regex = r"git\+(?Phttps:\/\/github.com\/[\w]+\/[\w]+\.git)(@(?P[\w]+))?(#egg=(?P[\w]+))?" - -with open("requirements.txt") as f: - required = f.read().splitlines() - -with open("requirements-optional.txt") as f: - optionals = f.read().splitlines() - -reqs: list[str] = [] -for ir in required: - if ir[0:3] == "git": - name = ir.split("/")[-1] - reqs += [f"{name} @ {ir}@main"] - else: - reqs += [ir] - -extras_require = {} -for mreqs, mode in zip( - [ - optionals, - ], - [ - "extras", - ], -): - opt_reqs: list[str] = [] - for ir in mreqs: - # For conditionals like pytest=2.1; python == 3.6 - if ";" in ir: - entries = ir.split(";") - extras_require[entries[1]] = entries[0] - elif ir[0:3] == "git": - m = re.match(git_regex, ir) - assert m is not None - items = m.groupdict() - name = items["name"] - url = items["url"] - version = items.get("version") - if version is None: - version = "main" - - opt_reqs += [f"{name} @ git+{url}@{version}"] - else: - opt_reqs += [ir] - extras_require[mode] = opt_reqs - setup( version=versioneer.get_version(), cmdclass=cast(Mapping[str, type[Command]], versioneer.get_cmdclass()), packages=find_packages(), - install_requires=reqs, - extras_require=extras_require, )