From d25ee693a387ce32fdb37c898a0d028267726a6e Mon Sep 17 00:00:00 2001 From: Ioannis Filippidis Date: Wed, 15 Sep 2021 02:43:02 +0200 Subject: [PATCH] API: configure Cython build via environment variables Previously configuration via `pip` used `--install-option`, but this parameter is removed in `pip >= 23.1`. - REL: use `pip`, instead of `setup.py install`, which is deprecated in `setuptools == 58.3.0`. - REL: require `setuptools >= 65.6.0` in `install_requires`, to use `pip install . --use-pep517` - CI: use `pip` to install `dd` - API: warn about `extern/` license files when running `bdist_wheel`, to enable `pip install .` for local builds. --- .github/workflows/main.yml | 10 +- .github/workflows/setup_build_env.sh | 3 +- Makefile | 80 ++++++++------ README.md | 154 +++++--------------------- doc.md | 159 +++++++++++++++++++++++++++ download.py | 6 +- examples/README.md | 17 +-- examples/install_dd_buddy.sh | 6 +- examples/install_dd_cudd.sh | 6 +- examples/install_dd_sylvan.sh | 6 +- setup.py | 24 +++- 11 files changed, 299 insertions(+), 172 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ae0b788a..c598663a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,7 +29,15 @@ jobs: set -o posix echo "Exported environment variables:" export -p - python setup.py install --fetch --cudd --cudd_zdd --sylvan + export \ + DD_FETCH=1 \ + DD_CUDD=1 \ + DD_CUDD_ZDD=1 \ + DD_SYLVAN=1 + pip install . \ + --verbose \ + --use-pep517 \ + --no-build-isolation - name: Install test dependencies run: | pip install pytest diff --git a/.github/workflows/setup_build_env.sh b/.github/workflows/setup_build_env.sh index d3232ffc..d9244cd8 100755 --- a/.github/workflows/setup_build_env.sh +++ b/.github/workflows/setup_build_env.sh @@ -11,7 +11,8 @@ sudo apt install \ dot -V pip install --upgrade \ pip \ - setuptools + setuptools \ + wheel # note that installing from `requirements.txt` # would also install packages that # may be absent from where `dd` will be installed diff --git a/Makefile b/Makefile index 8e700966..41897fb3 100644 --- a/Makefile +++ b/Makefile @@ -3,49 +3,64 @@ SHELL := bash wheel_file := $(wildcard dist/*.whl) -temp_file := _temp.txt .PHONY: cudd install test -.PHONY: clean clean_all clean_cudd +.PHONY: clean clean_all clean_cudd wheel_deps build_cudd: clean cudd install test -build_sylvan: clean +build_sylvan: clean wheel_deps -pip uninstall -y dd - python setup.py install --sylvan + export DD_SYLVAN=1; \ + pip install . -vvv --use-pep517 --no-build-isolation pip install pytest make test -sdist_test: clean - python setup.py sdist --cudd --buddy - cd dist; \ +sdist_test: clean wheel_deps + pip install -U build cython + export DD_CUDD=1 DD_BUDDY=1; \ + python -m build --sdist --no-isolation + pushd dist; \ pip install dd*.tar.gz; \ - tar -zxf dd*.tar.gz + tar -zxf dd*.tar.gz && \ + popd pip install pytest make -C dist/dd*/ -f ../../Makefile test -sdist_test_cudd: clean - pip install cython ply - python setup.py sdist --cudd --buddy +sdist_test_cudd: clean wheel_deps + pip install build cython ply + export DD_CUDD=1 DD_BUDDY=1; \ + python -m build --sdist --no-isolation yes | pip uninstall cython ply - cd dist; \ + pushd dist; \ tar -zxf dd*.tar.gz; \ - cd dd*; \ - python setup.py install --fetch --cudd + pushd dd*/; \ + export DD_FETCH=1 DD_CUDD=1; \ + pip install . -vvv --use-pep517 --no-build-isolation && \ + popd && popd pip install pytest make -C dist/dd*/ -f ../../Makefile test # use to create source distributions for PyPI -sdist: clean +sdist: clean wheel_deps -rm dist/*.tar.gz - python setup.py sdist --cudd --buddy --sylvan + pip install -U build cython + export DD_CUDD=1 DD_BUDDY=1 DD_SYLVAN=1; \ + python -m build --sdist --no-isolation + +wheel_deps: + pip install --upgrade pip setuptools wheel # use to create binary distributions for PyPI -wheel: clean +wheel: clean wheel_deps -rm dist/*.whl -rm wheelhouse/*.whl - python setup.py bdist_wheel --cudd --cudd_zdd + export DD_CUDD=1 DD_CUDD_ZDD=1; \ + pip wheel . \ + -vvv \ + --wheel-dir dist \ + --no-deps @echo "-------------" auditwheel show dist/*.whl @echo "-------------" @@ -53,24 +68,25 @@ wheel: clean @echo "-------------" auditwheel show wheelhouse/*.whl -install: - python setup.py install --cudd +install: wheel_deps + export DD_CUDD=1; \ + pip install . -vvv --use-pep517 --no-build-isolation -reinstall: uninstall - python setup.py install --cudd --cudd_zdd --sylvan +reinstall: uninstall wheel_deps + export DD_CUDD=1 DD_CUDD_ZDD DD_SYLVAN; \ + pip install . -vvv --use-pep517 --no-build-isolation -reinstall_buddy: uninstall - echo ". --install-option='--buddy'" \ - > $(temp_file) - pip install -vvv -r $(temp_file) +reinstall_buddy: uninstall wheel_deps + export DD_BUDDY=1; \ + pip install . -vvv --use-pep517 --no-build-isolation -reinstall_cudd: uninstall - python setup.py install --cudd --cudd_zdd +reinstall_cudd: uninstall wheel_deps + export DD_CUDD=1 DD_CUDD_ZDD=1; \ + pip install . -vvv --use-pep517 --no-build-isolation -reinstall_sylvan: uninstall - echo ". --install-option='--sylvan'" \ - > $(temp_file) - pip install -vvv -r $(temp_file) +reinstall_sylvan: uninstall wheel_deps + export DD_SYLVAN=1; \ + pip install . -vvv --use-pep517 --no-build-isolation uninstall: pip uninstall -y dd diff --git a/README.md b/README.md index b3232361..f169199b 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ package installer [`pip`](https://pip.pypa.io): pip install dd ``` -Locally: +or from the directory of source files: ```shell pip install . @@ -190,103 +190,47 @@ For Python 2.7, use `dd == 0.5.7`. ## Cython bindings - -### Wheel files with compiled CUDD - - -As of `dd` version 0.5.3, [`manylinux2014_x86_64`]( - https://www.python.org/dev/peps/pep-0599/) -[wheel files](https://www.python.org/dev/peps/pep-0427/) are -[available from PyPI](https://pypi.org/project/dd/#files) for some Python -versions. These wheel files contain the module `dd.cudd` with the CUDD -library compiled and linked. -If you have a Linux system and Python version compatible with one of the -available wheels, then `pip install dd` will install `dd.cudd`, so you need -not compile CUDD. Otherwise, read below. - - -### `dd` fetching CUDD - -By default, the package installs only the Python modules. -You can select to install any Cython extensions using -the `setup.py` options: - -- `--cudd`: build module of CUDD BDDs -- `--cudd_zdd`: build module of CUDD ZDDs -- `--sylvan`: build module of Sylvan BDDs -- `--buddy`: build module of BuDDy BDDs - -Pass `--fetch` to `setup.py` to tell it to download, unpack, and -`make` CUDD v3.0.0. For example: - -```shell -pip download dd --no-deps -tar xzf dd-*.tar.gz -cd dd-*/ -python setup.py install --fetch --cudd --cudd_zdd -``` - -The path to an existing CUDD build directory can be passed as an argument: +To compile also the module `dd.cudd` (which interfaces to CUDD) +when installing from PyPI, run: ```shell -python setup.py install --cudd="/home/user/cudd" +pip install --upgrade wheel cython +export DD_FETCH=1 DD_CUDD=1 +pip install dd -vvv --use-pep517 --no-build-isolation ``` -If you prefer defining installation directories, then follow [Cython's instructions]( - https://cython.readthedocs.io/en/latest/src/tutorial/clibraries.html#compiling-and-linking) -to define `CFLAGS` and `LDFLAGS` before running `setup.py`. -You need to have copied `CuddInt.h` to the installation's include location -(CUDD omits it). +(`DD_FETCH=1 DD_CUDD=1 pip install dd` also works, +when the source tarball includes cythonized code.) -If building from the repository, then first install `cython`. For example: +To confirm that the installation succeeded: ```shell -git clone git@github.com:tulip-control/dd -cd dd -pip install cython # not needed if building from PyPI distro -python setup.py install --fetch --cudd -``` - -The above options can be passed to `pip` too, using the [`--install-option`]( - https://pip.pypa.io/en/latest/cli/pip_install/#per-requirement-overrides) -in a requirements file, for example: - -``` -dd >= 0.1.1 --install-option="--fetch" --install-option="--cudd" +python -c 'import dd.cudd' ``` -The command line behavior of `pip` [is currently different]( - https://github.com/pypa/pip/issues/1883), so - -```shell -pip install --install-option="--fetch" dd -``` - -will propagate option `--fetch` to dependencies, and so raise an error. - - -### User installing build dependencies +The [environment variables]( + https://en.wikipedia.org/wiki/Environment_variable) +above mean: +- `DD_FETCH=1`: download CUDD v3.0.0 sources from the internet, + unpack the tarball (after checking its hash), and `make` CUDD. +- `DD_CUDD=1`: build the Cython module `dd.cudd` -If you build and install CUDD, Sylvan, or BuDDy yourself, then ensure that: +More about environment variables that configure the +C extensions of `dd` is described in the file [`doc.md`]( + https://github.com/tulip-control/dd/blob/main/doc.md) -- the header files and libraries are present, and -- suitable compiler, include, linking, and library flags are passed, -either by setting [environment variables]( - https://en.wikipedia.org/wiki/Environment_variable) -prior to calling `pip`, or by editing the file [`download.py`]( - https://github.com/tulip-control/dd/blob/main/download.py). -Currently, `download.py` expects to find Sylvan under `dd/sylvan` and built with [Autotools](https://en.wikipedia.org/wiki/GNU_Build_System) -(for an example, read `.github/workflows/main.yml`). -If the path differs in your environment, remember to update it. +## Wheel files with compiled CUDD -Scripts are available that fetch, build, and install the Cython bindings: -- [`examples/install_dd_cudd.sh`]( - https://github.com/tulip-control/dd/blob/main/examples/install_dd_cudd.sh) -- [`examples/install_dd_sylvan.sh`]( - https://github.com/tulip-control/dd/blob/main/examples/install_dd_sylvan.sh) -- [`examples/install_dd_buddy.sh`]( - https://github.com/tulip-control/dd/blob/main/examples/install_dd_buddy.sh) +[Wheel files]( + https://www.python.org/dev/peps/pep-0427/) +are [available from PyPI]( + https://pypi.org/project/dd/#files), +which contain the module `dd.cudd`, +with the CUDD library compiled and linked. +If you have a Linux system and Python version compatible with +one of the PyPI wheels, +then `pip install dd` will install also `dd.cudd`. ### Licensing of the compiled modules `dd.cudd` and `dd.cudd_zdd` in the wheel @@ -335,46 +279,6 @@ The modules `dd.cudd` and `dd.cudd_zdd` in the wheel dynamically link to the: that is dynamically linked. -## Installing the development version - -For installing the development version of `dd` from the `git` repository, -an alternative to cloning the repository and installing from the cloned -repository is to [use `pip` for doing so]( - https://pip.pypa.io/en/stable/cli/pip_install/#argument-handling): - -```shell -pip install https://github.com/tulip-control/dd/archive/main.tar.gz -``` - -or with [`pip` using `git`]( - https://pip.pypa.io/en/stable/topics/vcs-support/#git) -(this alternative requires that `git` be installed): - -```shell -pip install git+https://github.com/tulip-control/dd -``` - -A `git` URL can be passed also to [`pip download`]( - https://pip.pypa.io/en/stable/cli/pip_download/#overview), -for example: - -```shell -pip download --no-deps https://github.com/tulip-control/dd/archive/main.tar.gz -``` - -The extension `.zip` too can be used for the name of the [archive file]( - https://en.wikipedia.org/wiki/Archive_file) -in the URL. Analogously, with `pip` using `git`: - -```shell -pip download --no-deps git+https://github.com/tulip-control/dd -``` - -Note that the naming of paths *within* the archive file downloaded from -GitHub in this way will differ, depending on whether `https://` or -`git+https://` is used. - - Tests ===== diff --git a/doc.md b/doc.md index a946695d..9414d8e1 100644 --- a/doc.md +++ b/doc.md @@ -22,6 +22,12 @@ - [Example: Reachability analysis](#example-reachability-analysis) - [Syntax for quantified Boolean formulas](#syntax-for-quantified-boolean-formulas) - [Multi-valued decision diagrams (MDD)](#multi-valued-decision-diagrams-mdd) +- [Installation of C extension modules](#installation-of-c-extension-modules) + - [Environment variables that activate C extensions](#environment-variables-that-activate-c-extensions) + - [Alternative: directly running `setup.py`](#alternative-directly-running-setuppy) + - [Using the package `build`](#using-the-package-build) + - [Customizing the C compilation](#customizing-the-c-compilation) +- [Installing the development version](#installing-the-development-version) - [Footnotes](#footnotes) - [Copying](#copying) @@ -1249,6 +1255,159 @@ in the negated value computed for node `y-3` in the next image. ![example_bdd](https://rawgithub.com/johnyf/binaries/main/dd/mdd.png) +## Installation of C extension modules + + +### Environment variables that activate C extensions + +By default, the package `dd` installs only its Python modules. +You can select to install Cython extensions using +environment variables: + +- `DD_FETCH=1`: download CUDD v3.0.0 sources from the internet, + check the tarball's hash, unpack the tarball, and `make` CUDD. +- `DD_CUDD=1`: build module `dd.cudd`, for CUDD BDDs +- `DD_CUDD_ZDD=1`: build module `dd.cudd_zdd`, for CUDD ZDDs +- `DD_SYLVAN=1`: build module `dd.sylvan`, for Sylvan BDDs +- `DD_BUDDY=1`: build module `dd.buddy`, for BuDDy BDDs + +Example scripts are available that fetch and install +the Cython bindings: +- [`examples/install_dd_cudd.sh`]( + https://github.com/tulip-control/dd/blob/main/examples/install_dd_cudd.sh) +- [`examples/install_dd_sylvan.sh`]( + https://github.com/tulip-control/dd/blob/main/examples/install_dd_sylvan.sh) +- [`examples/install_dd_buddy.sh`]( + https://github.com/tulip-control/dd/blob/main/examples/install_dd_buddy.sh) + + +### Alternative: Directly running `setup.py` + +Activating the Cython build by directly running +`python setup.py` is an alternative to +using environment variables (e.g., `export DD_CUDD=1` etc). +The relevant command-line options of `setup.py` are: + +- `--fetch`: same effect as `DD_FETCH=1` +- `--cudd`: same effect as `DD_CUDD=1` +- `--cudd_zdd`: same effect as `DD_CUDD_ZDD=1` +- `--sylvan`: same effect as `DD_SYLVAN=1` +- `--buddy`: same effect as `DD_BUDDY=1` + +These options work for `python setup.py sdist` and +`python setup.py install`, but directly running +`python setup.py` is deprecated by `setuptools >= 58.3.0`. + +Example: + +```shell +pip download dd --no-deps +tar xzf dd-*.tar.gz +pushd dd-*/ + # `pushd` means `cd` +python setup.py install --fetch --cudd --cudd_zdd +popd +``` + +[`pushd directory`]( + https://en.wikipedia.org/wiki/Pushd_and_popd) +is akin to `stack.append(directory)` in +Python, and `popd` to `stack.pop()`. + +The path to an existing CUDD build directory +can be passed as an argument, for example: + +```shell +python setup.py install \ + --fetch \ + --cudd="/home/user/cudd" +``` + + +### Using the package `build` + +The following also works for building source tarballs and wheels: + +```sh +pip install cython +export DD_FETCH=1 DD_CUDD=1 +python -m build --no-isolation +``` + +To build a source tarball: + +```sh +DD_CUDD=1 python -m build --sdist --no-isolation +``` + + +### Customizing the C compilation + +If you build and install CUDD, Sylvan, or BuDDy yourself, then ensure that: + +- the header files and libraries are present, and +- the compiler is configured appropriately (include, + linking, and library configuration), + +either by setting [environment variables]( + https://en.wikipedia.org/wiki/Environment_variable) +prior to calling `pip`, or by editing the file [`download.py`]( + https://github.com/tulip-control/dd/blob/main/download.py). + +Currently, `download.py` expects to find Sylvan under `dd/sylvan` and +built with [Autotools]( + https://en.wikipedia.org/wiki/GNU_Build_System) +(for an example, read `.github/workflows/setup_build_env.sh`). +If the path differs in your environment, remember to update it. + +If you prefer defining installation directories, then follow +[Cython's instructions]( + https://cython.readthedocs.io/en/latest/src/tutorial/clibraries.html#compiling-and-linking) +to define `CFLAGS` and `LDFLAGS` before installing. +You need to have copied `CuddInt.h` to the installation's include location +(CUDD omits it). + + +## Installing the development version + +For installing the development version of `dd` from the `git` repository, +an alternative to cloning the repository and installing from the cloned +repository is to [use `pip` for doing so]( + https://pip.pypa.io/en/stable/cli/pip_install/#argument-handling): + +```shell +pip install https://github.com/tulip-control/dd/archive/main.tar.gz +``` + +or with [`pip` using `git`]( + https://pip.pypa.io/en/stable/topics/vcs-support/#git) +(this alternative requires that `git` be installed): + +```shell +pip install git+https://github.com/tulip-control/dd +``` + +A `git` URL can be passed also to [`pip download`]( + https://pip.pypa.io/en/stable/cli/pip_download/#overview), +for example: + +```shell +pip download --no-deps https://github.com/tulip-control/dd/archive/main.tar.gz +``` + +The extension `.zip` too can be used for the name of the [archive file]( + https://en.wikipedia.org/wiki/Archive_file) +in the URL. Analogously, with `pip` using `git`: + +```shell +pip download --no-deps git+https://github.com/tulip-control/dd +``` + +Note that the naming of paths *within* the archive file downloaded from +GitHub in this way will differ, depending on whether `https://` or +`git+https://` is used. + + ## Footnotes - The `Makefile` contains the rules `sdist` and `wheel` that diff --git a/download.py b/download.py index a331b339..3a58e139 100644 --- a/download.py +++ b/download.py @@ -174,8 +174,12 @@ def _copy_extern_licenses(args): for name in licenses: license = os.path.join(path, name) included = os.path.join('dd', name) - if yes: + if yes and os.path.isfile(license): shutil.copyfile(license, included) + elif yes and not os.path.isfile(license): + print( + f'WARNING: No file: `{license}`, ' + 'skipping file copy.') elif os.path.isfile(included): os.remove(included) diff --git a/examples/README.md b/examples/README.md index 7fdc0278..dc689b65 100644 --- a/examples/README.md +++ b/examples/README.md @@ -37,14 +37,15 @@ The shell scripts show how to install the Cython modules of `dd`: - `install_dd_buddy.sh`: how to install the module `dd.buddy` To install all the above modules, combine the steps contained in -the above shell scripts, and pass to `setup.py` -all the relevant command-line options, i.e., +the above shell scripts, and define all the relevant +environment variables, i.e., ```shell -pip install -r <(echo "dd \ - --install-option='--buddy' \ - --install-option='--fetch' \ - --install-option='--cudd' \ - --install-option='--cudd_zdd' \ - --install-option='--sylvan'") +export \ + DD_BUDDY=1 \ + DD_FETCH=1 \ + DD_CUDD=1 \ + DD_CUDD_ZDD=1 \ + DD_SYLVAN=1 +pip install dd ``` diff --git a/examples/install_dd_buddy.sh b/examples/install_dd_buddy.sh index 0151b4ca..085df501 100755 --- a/examples/install_dd_buddy.sh +++ b/examples/install_dd_buddy.sh @@ -44,7 +44,11 @@ popd # Fetch and install `dd` pip install cython -pip install -r <(echo "dd --install-option='--buddy'") +export DD_BUDDY=1 +pip install dd \ + -vvv \ + --use-pep517 \ + --no-build-isolation # passes `-lbdd` to the C compiler # # fetch `dd` source diff --git a/examples/install_dd_cudd.sh b/examples/install_dd_cudd.sh index fe507f67..454cc3f3 100755 --- a/examples/install_dd_cudd.sh +++ b/examples/install_dd_cudd.sh @@ -36,7 +36,11 @@ pip download \ --no-binary dd tar -xzf dd-*.tar.gz pushd dd-*/ -python setup.py install --fetch --cudd --cudd_zdd +export DD_FETCH=1 DD_CUDD=1 DD_CUDD_ZDD=1 +pip install . \ + -vvv \ + --use-pep517 \ + --no-build-isolation # confirm that `dd.cudd` did get installed pushd tests/ python -c 'import dd.cudd' diff --git a/examples/install_dd_sylvan.sh b/examples/install_dd_sylvan.sh index 6d079315..623d8244 100755 --- a/examples/install_dd_sylvan.sh +++ b/examples/install_dd_sylvan.sh @@ -51,7 +51,11 @@ popd # Fetch and install `dd` -pip install -r <(echo "dd --install-option='--sylvan'") +export DD_SYLVAN=1 +pip install dd \ + -vvv \ + --use-pep517 \ + --no-build-isolation # fetch `dd` source pip download \ --no-deps dd \ diff --git a/setup.py b/setup.py index ef28161d..8ff8eac3 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ """Installation script.""" import argparse import logging +import os import sys from setuptools import setup @@ -51,7 +52,7 @@ 'ply >= 3.4, <= 3.10', 'psutil >= 3.2.2', 'pydot >= 1.2.2', - 'setuptools >= 42.0.0'] + 'setuptools >= 65.6.0'] TESTS_REQUIRE = [ 'pytest >= 4.6.11'] CLASSIFIERS = [ @@ -118,9 +119,30 @@ def parse_args(): return args +def read_env_vars( + ) -> dict: + """Read relevant environment variables.""" + keys = { + k: '' + for k in download.EXTENSIONS} + keys['fetch'] = True + env_vars = { + k: v + for k, v in keys.items() + if f'DD_{k.upper()}' in os.environ} + print('`setup.py` of `dd` read environment variables:') + print(env_vars) + return env_vars + + def run_setup(): """Build parser, get version from `git`, install.""" + env_vars = read_env_vars() args = parse_args() + dargs = vars(args) + for k, v in env_vars.items(): + if dargs[k] in (None, False): + dargs[k] = v if args.fetch: download.fetch_cudd() # build extensions ?