diff --git a/CHANGES.md b/CHANGES.md index 3021b12c6..22d3860fc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,17 @@ # Release Notes +## 2.16.0 + +This release adds support for `--venv-system-site-packages` when +creating a `--venv` PEX and `--system-site-packages` when creating a +venv using the `pex-tools` / `PEX_TOOLS=1` `venv` command or when using +the `pex3 venv create` command. Although this breaks PEX hermeticity, it +can be the most efficient way to ship partial PEX venvs created with +`--exclude`s to machines that have the excluded dependencies already +installed in the site packages of a compatible system interpreter. + +* Support `--system-site-packages` when creating venvs. (#2500) + ## 2.15.0 This release enhances the REPL your PEX drops into when it either diff --git a/pex/bin/pex.py b/pex/bin/pex.py index abb3dcd88..22c7cbc64 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -316,6 +316,14 @@ def configure_clp_pex_options(parser): "problems with tools or libraries that are confused by symlinked source files." ), ) + group.add_argument( + "--venv-system-site-packages", + "--no-venv-system-site-packages", + dest="venv_system_site_packages", + default=False, + action=HandleBoolAction, + help="If --venv is specified, give the venv access to the system site-packages dir.", + ) group.add_argument( "--non-hermetic-venv-scripts", dest="venv_hermetic_scripts", @@ -960,6 +968,7 @@ def build_pex( pex_info.venv_bin_path = options.venv or BinPath.FALSE pex_info.venv_copies = options.venv_copies pex_info.venv_site_packages_copies = options.venv_site_packages_copies + pex_info.venv_system_site_packages = options.venv_system_site_packages pex_info.venv_hermetic_scripts = options.venv_hermetic_scripts pex_info.includes_tools = options.include_tools or options.venv pex_info.pex_path = options.pex_path.split(os.pathsep) if options.pex_path else () diff --git a/pex/cli/commands/venv.py b/pex/cli/commands/venv.py index 3cd29f515..3e8625941 100644 --- a/pex/cli/commands/venv.py +++ b/pex/cli/commands/venv.py @@ -259,6 +259,7 @@ def _create(self): interpreter=target.get_interpreter(), force=installer_configuration.force, copies=installer_configuration.copies, + system_site_packages=installer_configuration.system_site_packages, prompt=installer_configuration.prompt, ) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 327feeb3c..36336b3a4 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -525,6 +525,7 @@ def ensure_venv( venv_dir=venv, interpreter=pex.interpreter, copies=pex_info.venv_copies, + system_site_packages=pex_info.venv_system_site_packages, prompt=os.path.basename(ENV.PEX) if ENV.PEX else None, ) diff --git a/pex/pex_info.py b/pex/pex_info.py index f6dfda9b5..b8c99349c 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -249,6 +249,16 @@ def venv_site_packages_copies(self, value): # type: (bool) -> None self._pex_info["venv_site_packages_copies"] = value + @property + def venv_system_site_packages(self): + # type: () -> bool + return self._pex_info.get("venv_system_site_packages", False) + + @venv_system_site_packages.setter + def venv_system_site_packages(self, value): + # type: (bool) -> None + self._pex_info["venv_system_site_packages"] = value + @property def venv_hermetic_scripts(self): # type: () -> bool diff --git a/pex/tools/commands/venv.py b/pex/tools/commands/venv.py index 082dad9fb..bb31b835a 100644 --- a/pex/tools/commands/venv.py +++ b/pex/tools/commands/venv.py @@ -117,6 +117,7 @@ def run(self, pex): interpreter=pex.interpreter, force=installer_configuration.force, copies=installer_configuration.copies, + system_site_packages=installer_configuration.system_site_packages, prompt=installer_configuration.prompt, ) diff --git a/pex/venv/installer_configuration.py b/pex/venv/installer_configuration.py index 8b3417fcf..c2036ef30 100644 --- a/pex/venv/installer_configuration.py +++ b/pex/venv/installer_configuration.py @@ -24,6 +24,7 @@ class InstallerConfiguration(object): pip = attr.ib(default=False) # type: bool copies = attr.ib(default=False) # type: bool site_packages_copies = attr.ib(default=False) # type: bool + system_site_packages = attr.ib(default=False) # type: bool compile = attr.ib(default=False) # type: bool prompt = attr.ib(default=None) # type: Optional[str] hermetic_scripts = attr.ib(default=False) # type: bool diff --git a/pex/venv/installer_options.py b/pex/venv/installer_options.py index f444dda7c..b04c5c7f9 100644 --- a/pex/venv/installer_options.py +++ b/pex/venv/installer_options.py @@ -75,13 +75,19 @@ def register( "--copies", action="store_true", default=False, - help="Create the venv using copies of system files instead of symlinks", + help="Create the venv using copies of system files instead of symlinks.", ) parser.add_argument( "--site-packages-copies", action="store_true", default=False, - help="Create the venv using copies of distributions instead of links or symlinks", + help="Create the venv using copies of distributions instead of links or symlinks.", + ) + parser.add_argument( + "--system-site-packages", + action="store_true", + default=False, + help="Give the venv access to the system site-packages dir.", ) parser.add_argument( "--compile", @@ -116,6 +122,7 @@ def configure(options): pip=options.pip, copies=options.copies, site_packages_copies=options.site_packages_copies, + system_site_packages=options.system_site_packages, compile=options.compile, prompt=options.prompt, hermetic_scripts=options.hermetic_scripts, diff --git a/pex/venv/virtualenv.py b/pex/venv/virtualenv.py index 3253e8c13..e9cdb0286 100644 --- a/pex/venv/virtualenv.py +++ b/pex/venv/virtualenv.py @@ -334,6 +334,7 @@ def create_atomic( interpreter=None, # type: Optional[PythonInterpreter] force=False, # type: bool copies=False, # type: bool + system_site_packages=False, # type: bool prompt=None, # type: Optional[str] install_pip=InstallationChoice.NO, # type: InstallationChoice.Value install_setuptools=InstallationChoice.NO, # type: InstallationChoice.Value @@ -346,6 +347,7 @@ def create_atomic( interpreter=interpreter, force=force, copies=copies, + system_site_packages=system_site_packages, prompt=prompt, install_pip=install_pip, install_setuptools=install_setuptools, diff --git a/pex/version.py b/pex/version.py index 780e42e28..df6326738 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.15.0" +__version__ = "2.16.0" diff --git a/tests/integration/venv_ITs/test_issue_1973.py b/tests/integration/venv_ITs/test_issue_1973.py new file mode 100644 index 000000000..ce05a671a --- /dev/null +++ b/tests/integration/venv_ITs/test_issue_1973.py @@ -0,0 +1,137 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import shutil +import subprocess + +import colors # vendor:skip +import pytest + +from pex.interpreter import PythonInterpreter +from pex.typing import TYPE_CHECKING +from testing import PY310, ensure_python_distribution, make_env, run_pex_command +from testing.cli import run_pex3 + +if TYPE_CHECKING: + from typing import Any, Text + +COWSAY_REQUIREMENT = "cowsay==6.0" +BLUE_MOO_ARGS = ["-c", "import colors; from cowsay import tux; tux(colors.cyan('Moo?'))"] + + +def assert_blue_moo(output): + # type: (Text) -> None + assert "| {moo} |".format(moo=colors.cyan("Moo?")) in output, output + + +def assert_colors_import_error( + process, # type: subprocess.Popen + python=PythonInterpreter.get(), # type: PythonInterpreter +): + # type: (...) -> None + _, stderr = process.communicate() + assert 0 != process.returncode + if python.version[0] == 2: + assert b"ImportError: No module named colors\n" in stderr, stderr.decode("utf-8") + else: + assert b"ModuleNotFoundError: No module named 'colors'\n" in stderr, stderr.decode("utf-8") + + +@pytest.fixture +def system_python_with_colors(tmpdir): + # type: (Any) -> str + location, _, _, _ = ensure_python_distribution(PY310) + system_python_distribution = os.path.join(str(tmpdir), "py310") + shutil.copytree(location, system_python_distribution) + system_python = os.path.join(system_python_distribution, "bin", "python") + subprocess.check_call(args=[system_python, "-m", "pip", "install", "ansicolors==1.1.8"]) + return system_python + + +def test_system_site_packages_venv_pex( + tmpdir, # type: Any + system_python_with_colors, # type: str +): + # type: (...) -> None + + pex = os.path.join(str(tmpdir), "pex") + pex_root = os.path.join(str(tmpdir), "pex_root") + run_pex_command( + args=[ + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + "--venv", + "--venv-system-site-packages", + COWSAY_REQUIREMENT, + "-o", + pex, + ] + ).assert_success() + assert_colors_import_error(subprocess.Popen(args=[pex] + BLUE_MOO_ARGS, stderr=subprocess.PIPE)) + + shutil.rmtree(pex_root) + assert_blue_moo( + subprocess.check_output(args=[system_python_with_colors, pex] + BLUE_MOO_ARGS).decode( + "utf-8" + ) + ) + + +def test_system_site_packages_pex_tools( + tmpdir, # type: Any + system_python_with_colors, # type: str +): + # type: (...) -> None + + pex = os.path.join(str(tmpdir), "pex") + run_pex_command(args=["--include-tools", COWSAY_REQUIREMENT, "-o", pex]).assert_success() + + venv = os.path.join(str(tmpdir), "venv") + subprocess.check_call( + args=[system_python_with_colors, pex, "venv", venv], env=make_env(PEX_TOOLS=1) + ) + assert_colors_import_error(subprocess.Popen(args=[pex] + BLUE_MOO_ARGS, stderr=subprocess.PIPE)) + + shutil.rmtree(venv) + subprocess.check_call( + args=[system_python_with_colors, pex, "venv", "--system-site-packages", venv], + env=make_env(PEX_TOOLS=1), + ) + assert_blue_moo( + subprocess.check_output(args=[os.path.join(venv, "pex")] + BLUE_MOO_ARGS).decode("utf-8") + ) + + +def test_system_site_packages_pex3_venv( + tmpdir, # type: Any + system_python_with_colors, # type: str +): + # type: (...) -> None + + venv = os.path.join(str(tmpdir), "venv") + venv_python = os.path.join(venv, "bin", "python") + run_pex3( + "venv", "create", "--python", system_python_with_colors, "-d", venv, COWSAY_REQUIREMENT + ).assert_success() + assert_colors_import_error( + subprocess.Popen(args=[venv_python] + BLUE_MOO_ARGS, stderr=subprocess.PIPE), + python=PythonInterpreter.from_binary(venv_python), + ) + + shutil.rmtree(venv) + run_pex3( + "venv", + "create", + "--python", + system_python_with_colors, + "-d", + venv, + COWSAY_REQUIREMENT, + "--system-site-packages", + ).assert_success() + assert_blue_moo(subprocess.check_output(args=[venv_python] + BLUE_MOO_ARGS).decode("utf-8"))