From 6b85c0492dceb80c1191b3b5f9b91cc18a277c99 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 16 Feb 2024 13:18:19 +0100 Subject: [PATCH] update docs for new build system --- CONTRIBUTING.md | 21 +- README.md | 25 +-- docs/source/changelog.md | 20 +- docs/source/howto/build.md | 391 +++++++++++++++++++++++++++++++++++++ docs/source/howto/index.md | 1 + docs/source/index.md | 2 +- perf/perf.py | 2 +- tools/collect_cmake.py | 69 +++++++ 8 files changed, 485 insertions(+), 46 deletions(-) create mode 100644 docs/source/howto/build.md create mode 100644 tools/collect_cmake.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 668590a05..e0b23ebd2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,22 +39,6 @@ Python-3.9.9 | packaged by conda-forge | (main, Dec 20 2021, 02:38:53) [Clang 11.1.0 ] ``` -## Licensing and contributing to PyZMQ - -PyZMQ uses different licenses for different parts of the code. - -The 'core' of PyZMQ (located in zmq/core) is licensed under LGPLv3. -This just means that if you make any changes to how that code works, -you must release those changes under the LGPL. -If you just _use_ pyzmq, then you can use any license you want for your own code. - -We don't feel that the restrictions imposed by the LGPL make sense for the -'non-core' functionality in pyzmq (derivative code must _also_ be LGPL or GPL), -especially for examples and utility code, so we have relicensed all 'non-core' -code under the more permissive BSD (specifically Modified BSD aka New BSD aka -3-clause BSD), where possible. This means that you can copy this code and build -your own apps without needing to license your own code with the LGPL or GPL. - ### Your contributions **Pull Requests are welcome!** @@ -85,10 +69,7 @@ inherits that project's license. - zmq/ssh/forward.py is from [paramiko], and inherits LGPL -- zmq/devices/monitoredqueue.pxd is derived from the zmq_device function in - libzmq, and inherits LGPL - -- perf examples are (c) iMatix, and LGPL +- perf examples are (c) iMatix, and MPL [paramiko]: http://www.lag.net/paramiko [pre-commit]: https://pre-commit.com diff --git a/README.md b/README.md index f93cfe19c..0549526aa 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ building from the repository will require that you install recent Cython. ## Building and installation -For more detail on building pyzmq, see [our Wiki](https://github.com/zeromq/pyzmq/wiki/Building-and-Installing-PyZMQ). +For more detail on building pyzmq, see [our docs](https://pyzmq.readthedocs.io/en/latest/howto/build.html). We build wheels for macOS, Windows, and Linux, so you can get a binary on those platforms with: @@ -50,33 +50,16 @@ pip install pyzmq ``` but compiling from source with `pip install pyzmq` should work in most environments. -Especially on macOS, make sure you are using the latest pip (≥ 8), or it may not find the right wheels. +Make sure you are using the latest pip, or it may not find the right wheels. If the wheel doesn't work for some reason, or you want to force pyzmq to be compiled (this is often preferable if you already have libzmq installed and configured the way you want it), -you can force installation with: +you can force installation from source with: ``` -pip install --no-binary=:all: pyzmq +pip install --no-binary=pyzmq pyzmq ``` -When compiling pyzmq (e.g. installing with pip on Linux), -it is generally recommended that zeromq be installed separately, -via homebrew, apt, yum, etc: - -``` -# Debian-based -sudo apt-get install libzmq3-dev - -# RHEL-based -sudo yum install libzmq3-devel -``` - -If this is not available, pyzmq will _try_ to build libzmq as a Python Extension, -though this is not guaranteed to work. - -Building pyzmq from the git repo (including release tags on GitHub) requires Cython. - ## Old versions pyzmq 16 drops support Python 2.6 and 3.2. diff --git a/docs/source/changelog.md b/docs/source/changelog.md index 2f0f3ac86..73881156e 100644 --- a/docs/source/changelog.md +++ b/docs/source/changelog.md @@ -7,15 +7,27 @@ For a full changelog, consult the [git log](https://github.com/zeromq/pyzmq/comm ## 26 -pyzmq 26 is a small release, but with some big changes nobody should notice. -The Cython backend has been rewritten using Cython 3's pure Python mode. +pyzmq 26 is a small release, but with some big changes _hopefully_ nobody will notice. +The highlights are: + +- The Cython backend has been rewritten using Cython 3's pure Python mode. +- The build system has been rewritten to use CMake via [scikit-build-core] instead of setuptools (setup.py is gone!). +- Bundled libzmq is updated to 4.3.5, which changes its license from LGPL to MPL. + This means: 1. Cython >=3.0 is now a build requirement (if omitted, source distributions _should_ still build from Cython-generated .c files without any Cython present) 1. pyzmq's Cython backend is a single extension module, which should improve install size, import time, compile time, etc. 1. pyzmq's Cython backend is now BSD-licensed, matching the rest of pyzmq. +1. The license of the libzmq library (included in pyzmq wheels) starting with 4.3.5 is now Mozilla Public License 2.0 (MPL-2.0). +1. when building pyzmq from source and it falls back on bundled libzmq, libzmq and libsodium are built as static libraries using their own build systems (CMake for libzmq, autotools for libsodium except on Windows where it uses msbuild) + rather than bundling libzmq with tweetnacl as a Python Extension. + +Since the new build system uses libzmq and libsodium's own build systems, evaluated at install time, building pyzmq with bundled libzmq from source should be much more likely to succeed on a variety of platforms than the previous method, where their build system was skipped and approximated as a Python extension. +But I would also be _very_ surprised if I didn't break anything in the process of replacing 14 years of setup.py from scratch, especially cases like cross-compiling. +Please [report](https://github.com/zeromq/pyzmq/issues/new) any issues you encounter building pyzmq. -The license of the libzmq library (included in pyzmq wheels and sources included but may not be used in pyzmq tarballs) remains unchanged and has its own LGPL license. +See [build docs](building-pyzmq) for more info. __Enhancements__: @@ -30,6 +42,7 @@ __Breaking changes__: `bytes(Frame)` remains unchanged, and utf-8 text strings can still be produced with: `bytes(Frame).decode("utf8")`, which works in all versions of pyzmq and does the same thing. +- Stop building Python 3.7 wheels for manylinux1, which reached EOL in January, 2022. The new build system doesn't seem to be able to find cmake in that environment. ## 25 @@ -1041,3 +1054,4 @@ s.linger [cython-build-requires]: https://groups.google.com/g/cython-users/c/ZqKFQmS0JdA/m/1FrK1ApYBAAJ [pyczmq]: https://github.com/zeromq/pyczmq +[scikit-build-core]: https://scikit-build-core.readthedocs.io diff --git a/docs/source/howto/build.md b/docs/source/howto/build.md new file mode 100644 index 000000000..f5ad8659c --- /dev/null +++ b/docs/source/howto/build.md @@ -0,0 +1,391 @@ +(building-pyzmq)= + +# Building pyzmq + +pyzmq publishes around a hundred wheels for each release, so hopefully very few folks need to build pyzmq from source. + +pyzmq 26 has a whole new build system using CMake via [scikit-build-core]. + +~all options can be specified via environment variables, in order to play nicely with pip. + +## Installing from source + +When compiling pyzmq, it is generally recommended that zeromq be installed separately, via homebrew, apt, yum, etc: + +```bash +# Debian-based +sudo apt-get install libzmq3-dev + +# Fedora-based +sudo yum install libzmq3-devel + +# homebrew +brew install zeromq +``` + +You can install pyzmq from source with `pip` by telling it `--no-binary pyzmq`: + +``` +python3 -m pip install --no-binary pyzmq pyzmq +``` + +or an editable install from a local checkout: + +``` +python3 -m pip install -e . +``` + +Building from source uses CMake via scikit-build-core. +CMake >= 3.28 is required. +scikit-build-core will attempt to download cmake if a satisfactory version is not found. + +## Examples + +First, some quick examples of influencing pyzmq's build. + +Build a wheel against already-installed libzmq: + +```bash +ZMQ_PREFIX=/usr/local +python3 -m pip install --no-binary pyzmq pyzmq +``` + +Force building bundled libzmq with the draft API: + +```bash +export ZMQ_PREFIX=bundled +export ZMQ_BUILD_DRAFT=1 +python3 -m pip install --no-binary pyzmq pyzmq +``` + +## Finding libzmq + +First, pyzmq tries to find libzmq to link against it. + +pyzmq will first try to search using standard CMake methods, followed by pkg-config. + +You can pass through arguments to the build system via the CMAKE_ARGS environment variable. +e.g. + +```bash +CMAKE_ARGS="-DCMAKE_PREFIX_PATH=/path/to/something" +``` + +or + +```bash +PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig" +``` + +If pyzmq doesn't find your libzmq via the default search, or you want to skip the search and tell pyzmq exactly where to look, set ZMQ_PREFIX (this skips cmake/pkgconfig entirely): + +```bash +ZMQ_PREFIX=/path/to/zmq # should contain 'include', 'lib', etc. +``` + +If you tell pyzmq where to look, it will not try to build if it doesn't find libzmq there, rather than falling back on bundled libzmq. + +## Building bundled libzmq + +If pyzmq doesn't find a libzmq to link to, it will fall back on building libzmq itself. +You can tell pyzmq to skip searching for libzmq and always build the bundled version with `ZMQ_PREFIX=bundled`. + +When building a bundled libzmq, pyzmq downloads and builds libzmq and libsodium as static libraries. +These static libraries are then linked to by the pyzmq extension and discarded. + +### Building bundled libsodium + +libsodium is built first, with `configure` most places: + +```bash +./configure --enable-static --disable-shared --with-pic +make +make install +``` + +or `msbuild` on Windows: + +```bat +msbuild /m /v:n /p:Configuration=StaticRelease /pPlatform=x64 builds/msvc/vs2022/libsodium.sln +``` + +You can _add_ arguments to configure with a semicolon-separated list, by specifying `PYZMQ_LIBSODIUM_CONFIGURE_ARGS` environment variable, or `LIBSODIUM_CONFIGURE_ARGS` CMake variable. + +```bash +PYZMQ_LIBSODIUM_CONFIGURE_ARGS="--without-pthread --enable-minimal" +# or +CMAKE_ARGS="-DLIBSODIUM_CONFIGURE_ARGS=--without-pthread;--enable-minimal" +``` + +and `[PYZMQ_]LIBSODIUM_MSBUILD_ARGS` on Windows: + +```bash +PYZMQ_LIBSODIUM_MSBUILD_ARGS="/something /else" +# or +CMAKE_ARGS="-DLIBSODIUM_MSBUILD_ARGS=/something;/else" +``` + +```{note} +command-line arguments from environment variables are expected to be space-separated (`-a -b`), +while CMake variables are expected to be CMake lists (semicolon-separated) (`-a;-b`). +``` + +### Building bundled libzmq + +The `libzmq-static` static library target is imported via [FetchContent], which means the libzmq CMake build is used on all platforms. +This means that configuring the build of libzmq itself is done directly via CMAKE_ARGS, +and all of libzmq's cmake flags should be available. +See [libzmq's install docs](https://github.com/zeromq/libzmq/blob/HEAD/INSTALL) for more. + +For example, to enable OpenPGM: + +```bash +CMAKE_ARGS="-DWITH_OPENPGM=ON" +``` + +### Specifying bundled versions + +You can specify which version of libsodium/libzmq to bundle with: + +``` +-DLIBZMQ_BUNDLED_VERSION=4.3.5 +-DLIBSODIUM_BUNDLED_VERSION=1.0.19 +``` + +or the specify the full URL to download (e.g. to test bundling an unreleased version): + +``` +-DLIBZMQ_BUNDLED_URL="https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz" +-DLIBSODIUM_BUNDLED_URL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.19.tar.gz" +``` + +```{warning} +Only the default versions are supported and there is no guarantee that bundling versions will work, but you are welcome to try! +``` + +### Windows notes + +I'm not at all confident in building things on Windows, but so far things work in CI. +I've done my best to expose options to allow users to override things if they don't work, +but it's not really meant to be customizable; it's meant to allow you to workaround my mistakes without waiting for a release. + +libsodium ships several solutions for msbuild, identified by `/builds/msvc/vs{year}/libsodium.sln`. +pyzmq tries to guess which solution to use based on the MSVC_VERSION CMake variable, +but you can skip the guess by specifying `-D LIBSODIUM_VS_VERSION=2022` to explicitly use the vs2022 solution. + +## Passing arguments + +pyzmq has a few CMake options to influence the build. All options are settable as environment variables, as well. +Other than `ZMQ_PREFIX` and `ZMQ_DRAFT_API`, environment variables for building pyzmq have the prefix `PYZMQ_`. + +The `_ARGS` variables that are meant to pass-through command-line strings accept standard command-line format from environment, or semicolon-separated lists when specified directly to cmake. + +So + +```bash +export ZMQ_PREFIX=bundled +export PYZMQ_LIBZMQ_BUNDLED_VERSION=4.3.4 +export PYZMQ_LIBSODIUM_CONFIGURE_ARGS=--disable-pie --minimal + +python3 -m build . +``` + +is equivalent to + +```bash +export CMAKE_ARGS="-DZMQ_PREFIX=bundled -DLIBZMQ_BUNDLED_VERSION=4.3.4 -DLIBSODIUM_CONFIGURE_ARGS=--disable-pie;--minimal" +python3 -m build . +``` + +Most cmake options can be seen below: + +% regenerate with python tools/collect_cmake.py + +
+ + +`cmake -LH` output for pyzmq, which can be passed via `CMAKE_ARGS`. +Most of these can also be specified via `PYZMQ_` environment variables. + + + +```bash +# Path to a program. +CYTHON:FILEPATH=$PREFIX/bin/cython + +# full URL to download bundled libsodium +LIBSODIUM_BUNDLED_URL:STRING= + +# libsodium version when bundling +LIBSODIUM_BUNDLED_VERSION:STRING=1.0.19 + +# semicolon-separated list of arguments to pass to ./configure for bundled libsodium +LIBSODIUM_CONFIGURE_ARGS:STRING= + +# semicolon-separated list of arguments to pass to msbuild for bundled libsodium +LIBSODIUM_MSBUILD_ARGS:STRING= + +# Visual studio solution version for bundled libsodium (default: detect from MSVC_VERSION) +LIBSODIUM_VS_VERSION:STRING= + +# full URL to download bundled libzmq +LIBZMQ_BUNDLED_URL:STRING= + +# libzmq version when bundling +LIBZMQ_BUNDLED_VERSION:STRING=4.3.5 + +# whether to build the libzmq draft API +ZMQ_DRAFT_API:BOOL=OFF + +# libzmq installation prefix or 'bundled' +ZMQ_PREFIX:STRING=auto + +# The directory containing a CMake configuration file for ZeroMQ. +ZeroMQ_DIR:PATH=$PREFIX/lib/cmake/ZeroMQ +``` + +
+ +
+ + +`cmake -LH` output for libzmq, showing additional arguments +that can be passed to CMAKE_ARGS when building bundled libzmq + + + +```bash +# Path to a program. +A2X_EXECUTABLE:FILEPATH=A2X_EXECUTABLE-NOTFOUND + +# Choose polling system for zmq_poll(er)_*. valid values are +# poll or select [default=poll unless POLLER=select] +API_POLLER:STRING= + +# Whether or not to build the shared object +BUILD_SHARED:BOOL=ON + +# Whether or not to build the static archive +BUILD_STATIC:BOOL=ON + +# Whether or not to build the tests +BUILD_TESTS:BOOL=ON + +# Build with static analysis(make take very long) +ENABLE_ANALYSIS:BOOL=OFF + +# Build with address sanitizer +ENABLE_ASAN:BOOL=OFF + +# Run tests that require sudo and capsh (for cap_net_admin) +ENABLE_CAPSH:BOOL=OFF + +# Include Clang +ENABLE_CLANG:BOOL=ON + +# Enables cpack rules +ENABLE_CPACK:BOOL=ON + +# Enable CURVE security +ENABLE_CURVE:BOOL=OFF + +# Build and install draft classes and methods +ENABLE_DRAFTS:BOOL=ON + +# Enable/disable eventfd +ENABLE_EVENTFD:BOOL=OFF + +# Build using compiler intrinsics for atomic ops +ENABLE_INTRINSICS:BOOL=OFF + +# Automatically close libsodium randombytes. Not threadsafe without getrandom() +ENABLE_LIBSODIUM_RANDOMBYTES_CLOSE:BOOL=ON + +# Build with empty ZMQ_EXPORT macro, bypassing platform-based automated detection +ENABLE_NO_EXPORT:BOOL=OFF + +# Enable precompiled headers, if possible +ENABLE_PRECOMPILED:BOOL=ON + +# Use radix tree implementation to manage subscriptions +ENABLE_RADIX_TREE:BOOL=ON + +# Build with thread sanitizer +ENABLE_TSAN:BOOL=OFF + +# Build with undefined behavior sanitizer +ENABLE_UBSAN:BOOL=OFF + +# Enable WebSocket transport +ENABLE_WS:BOOL=ON + +# +LIBZMQ_PEDANTIC:BOOL=ON + +# +LIBZMQ_WERROR:BOOL=OFF + +# Choose polling system for I/O threads. valid values are +# kqueue, epoll, devpoll, pollset, poll or select [default=autodetect] +POLLER:STRING= + +# Path to a library. +RT_LIBRARY:FILEPATH=RT_LIBRARY-NOTFOUND + +# Build html docs +WITH_DOCS:BOOL=ON + +# Use libbsd instead of builtin strlcpy +WITH_LIBBSD:BOOL=ON + +# Use libsodium +WITH_LIBSODIUM:BOOL=OFF + +# Use static libsodium library +WITH_LIBSODIUM_STATIC:BOOL=OFF + +# Enable militant assertions +WITH_MILITANT:BOOL=OFF + +# Build with support for NORM +WITH_NORM:BOOL=OFF + +# Use NSS instead of builtin sha1 +WITH_NSS:BOOL=OFF + +# Build with support for OpenPGM +WITH_OPENPGM:BOOL=OFF + +# Build with perf-tools +WITH_PERF_TOOL:BOOL=ON + +# Use TLS for WSS support +WITH_TLS:BOOL=ON + +# Build with support for VMware VMCI socket +WITH_VMCI:BOOL=OFF + +# install path for ZeroMQConfig.cmake +ZEROMQ_CMAKECONFIG_INSTALL_DIR:STRING=lib/cmake/ZeroMQ + +# ZeroMQ library +ZEROMQ_LIBRARY:STRING=libzmq + +# Build as OS X framework +ZMQ_BUILD_FRAMEWORK:BOOL=OFF + +# Build the tests for ZeroMQ +ZMQ_BUILD_TESTS:BOOL=ON + +# Choose condition_variable_t implementation. Valid values are +# stl11, win32api, pthreads, none [default=autodetect] +ZMQ_CV_IMPL:STRING=stl11 + +# Output zmq library base name +ZMQ_OUTPUT_BASENAME:STRING=zmq +``` + +
+ +[fetchcontent]: https://cmake.org/cmake/help/latest/module/FetchContent.html +[scikit-build-core]: https://scikit-build-core.readthedocs.io diff --git a/docs/source/howto/index.md b/docs/source/howto/index.md index b6e2da39f..dd978d060 100644 --- a/docs/source/howto/index.md +++ b/docs/source/howto/index.md @@ -4,6 +4,7 @@ --- maxdepth: 2 --- +build.md morethanbindings.rst serialization.rst devices.rst diff --git a/docs/source/index.md b/docs/source/index.md index 7d0a4457c..c5d6bc826 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -32,7 +32,7 @@ as they never received a stable release. Binary distributions (wheels on [PyPI](https://pypi.org/project/pyzmq/)) of PyZMQ ship with the stable version of libzmq at the time of release, built with default configuration, -and include CURVE support provided by tweetnacl. +and include CURVE support provided by libsodium. For pyzmq-{{ release }}, this is {{ target_libzmq }}. # Using PyZMQ diff --git a/perf/perf.py b/perf/perf.py index c7a42ed22..81d6cd67b 100644 --- a/perf/perf.py +++ b/perf/perf.py @@ -4,7 +4,7 @@ # Distributed under the terms of the Modified BSD License. # # Some original test code Copyright (c) 2007-2010 iMatix Corporation, -# Used under LGPLv3 +# Used under MPL-2.0 import argparse import time diff --git a/tools/collect_cmake.py b/tools/collect_cmake.py new file mode 100644 index 000000000..b30027f50 --- /dev/null +++ b/tools/collect_cmake.py @@ -0,0 +1,69 @@ +""" +collect cmake -LH output + +for inclusion in docs +""" + +import sys +from pathlib import Path +from subprocess import PIPE, run +from tempfile import TemporaryDirectory + +here = Path(__file__).parent.absolute() +repo = here.parent +home = str(Path.home()) + + +def summarize_cmake_output(text: str) -> str: + """Summarize cmake -LH output + + Formats help strings nicer, excludes common + """ + text = text.replace(sys.prefix, "$PREFIX") + text = text.replace(home, "~") + chunks = text.split("\n\n") + new_chunks = [] + for chunk in chunks: + if not chunk: + continue + lines = chunk.splitlines() + doc_lines, assignment = lines[:-1], lines[-1] + if assignment.startswith(("CMAKE_", "FETCHCONTENT_")): + continue + doc_lines = [ + "# " + doc_line.lstrip("/ ") + for doc_line in doc_lines + if not doc_line.startswith("--") + ] + new_chunks.append("\n".join(doc_lines + [assignment])) + return "\n\n".join(new_chunks) + + +def summarize_cmake(path: Path) -> str: + """Collect summarized cmake -LH output from a repo""" + path = Path(path).absolute() + with TemporaryDirectory() as td: + p = run( + ["cmake", "-LH", str(path)], + text=True, + stderr=sys.stderr, + stdout=PIPE, + check=False, + cwd=td, + ) + return summarize_cmake_output(p.stdout) + + +def main(): + if len(sys.argv) < 2: + paths = [repo] + else: + paths = sys.argv[1:] + for path in paths: + print(path) + print(summarize_cmake(path)) + print("\n\n") + + +if __name__ == "__main__": + main()