Support --system-site-packages when creating venvs. (#2500)

Fixes #1973

CHANGES.md

@@ -1,5 +1,17 @@
 # Release Notes
 
+## 2.16.0
+
+This release adds support for `--venv-system-site-packages` when
+creating a `--venv` PEX and `--system-site-packages` when creating a
+venv using the `pex-tools` / `PEX_TOOLS=1` `venv` command or when using
+the `pex3 venv create` command. Although this breaks PEX hermeticity, it
+can be the most efficient way to ship partial PEX venvs created with
+`--exclude`s to machines that have the excluded dependencies already
+installed in the site packages of a compatible system interpreter.
+
+* Support `--system-site-packages` when creating venvs. (#2500)
+
 ## 2.15.0
 
 This release enhances the REPL your PEX drops into when it either

pex/bin/pex.py

@@ -316,6 +316,14 @@ def configure_clp_pex_options(parser):
             "problems with tools or libraries that are confused by symlinked source files."
         ),
     )
+    group.add_argument(
+        "--venv-system-site-packages",
+        "--no-venv-system-site-packages",
+        dest="venv_system_site_packages",
+        default=False,
+        action=HandleBoolAction,
+        help="If --venv is specified, give the venv access to the system site-packages dir.",
+    )
     group.add_argument(
         "--non-hermetic-venv-scripts",
         dest="venv_hermetic_scripts",
@@ -960,6 +968,7 @@ def build_pex(
     pex_info.venv_bin_path = options.venv or BinPath.FALSE
     pex_info.venv_copies = options.venv_copies
     pex_info.venv_site_packages_copies = options.venv_site_packages_copies
+    pex_info.venv_system_site_packages = options.venv_system_site_packages
     pex_info.venv_hermetic_scripts = options.venv_hermetic_scripts
     pex_info.includes_tools = options.include_tools or options.venv
     pex_info.pex_path = options.pex_path.split(os.pathsep) if options.pex_path else ()

pex/cli/commands/venv.py

@@ -259,6 +259,7 @@ def _create(self):
                     interpreter=target.get_interpreter(),
                     force=installer_configuration.force,
                     copies=installer_configuration.copies,
+                    system_site_packages=installer_configuration.system_site_packages,
                     prompt=installer_configuration.prompt,
                 )
 

pex/pex_bootstrapper.py

@@ -525,6 +525,7 @@ def ensure_venv(
                 venv_dir=venv,
                 interpreter=pex.interpreter,
                 copies=pex_info.venv_copies,
+                system_site_packages=pex_info.venv_system_site_packages,
                 prompt=os.path.basename(ENV.PEX) if ENV.PEX else None,
             )
 

pex/pex_info.py

@@ -249,6 +249,16 @@ def venv_site_packages_copies(self, value):
         # type: (bool) -> None
         self._pex_info["venv_site_packages_copies"] = value
 
+    @property
+    def venv_system_site_packages(self):
+        # type: () -> bool
+        return self._pex_info.get("venv_system_site_packages", False)
+
+    @venv_system_site_packages.setter
+    def venv_system_site_packages(self, value):
+        # type: (bool) -> None
+        self._pex_info["venv_system_site_packages"] = value
+
     @property
     def venv_hermetic_scripts(self):
         # type: () -> bool

pex/tools/commands/venv.py

@@ -117,6 +117,7 @@ def run(self, pex):
                 interpreter=pex.interpreter,
                 force=installer_configuration.force,
                 copies=installer_configuration.copies,
+                system_site_packages=installer_configuration.system_site_packages,
                 prompt=installer_configuration.prompt,
             )
 

pex/venv/installer_configuration.py

@@ -24,6 +24,7 @@ class InstallerConfiguration(object):
     pip = attr.ib(default=False)  # type: bool
     copies = attr.ib(default=False)  # type: bool
     site_packages_copies = attr.ib(default=False)  # type: bool
+    system_site_packages = attr.ib(default=False)  # type: bool
     compile = attr.ib(default=False)  # type: bool
     prompt = attr.ib(default=None)  # type: Optional[str]
     hermetic_scripts = attr.ib(default=False)  # type: bool

pex/venv/installer_options.py

@@ -75,13 +75,19 @@ def register(
         "--copies",
         action="store_true",
         default=False,
-        help="Create the venv using copies of system files instead of symlinks",
+        help="Create the venv using copies of system files instead of symlinks.",
     )
     parser.add_argument(
         "--site-packages-copies",
         action="store_true",
         default=False,
-        help="Create the venv using copies of distributions instead of links or symlinks",
+        help="Create the venv using copies of distributions instead of links or symlinks.",
+    )
+    parser.add_argument(
+        "--system-site-packages",
+        action="store_true",
+        default=False,
+        help="Give the venv access to the system site-packages dir.",
     )
     parser.add_argument(
         "--compile",
@@ -116,6 +122,7 @@ def configure(options):
         pip=options.pip,
         copies=options.copies,
         site_packages_copies=options.site_packages_copies,
+        system_site_packages=options.system_site_packages,
         compile=options.compile,
         prompt=options.prompt,
         hermetic_scripts=options.hermetic_scripts,

pex/venv/virtualenv.py

@@ -334,6 +334,7 @@ def create_atomic(
         interpreter=None,  # type: Optional[PythonInterpreter]
         force=False,  # type: bool
         copies=False,  # type: bool
+        system_site_packages=False,  # type: bool
         prompt=None,  # type: Optional[str]
         install_pip=InstallationChoice.NO,  # type: InstallationChoice.Value
         install_setuptools=InstallationChoice.NO,  # type: InstallationChoice.Value
@@ -346,6 +347,7 @@ def create_atomic(
             interpreter=interpreter,
             force=force,
             copies=copies,
+            system_site_packages=system_site_packages,
             prompt=prompt,
             install_pip=install_pip,
             install_setuptools=install_setuptools,

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.15.0"
+__version__ = "2.16.0"

tests/integration/venv_ITs/test_issue_1973.py

@@ -0,0 +1,137 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import os.path
+import shutil
+import subprocess
+
+import colors  # vendor:skip
+import pytest
+
+from pex.interpreter import PythonInterpreter
+from pex.typing import TYPE_CHECKING
+from testing import PY310, ensure_python_distribution, make_env, run_pex_command
+from testing.cli import run_pex3
+
+if TYPE_CHECKING:
+    from typing import Any, Text
+
+COWSAY_REQUIREMENT = "cowsay==6.0"
+BLUE_MOO_ARGS = ["-c", "import colors; from cowsay import tux; tux(colors.cyan('Moo?'))"]
+
+
+def assert_blue_moo(output):
+    # type: (Text) -> None
+    assert "| {moo} |".format(moo=colors.cyan("Moo?")) in output, output
+
+
+def assert_colors_import_error(
+    process,  # type: subprocess.Popen
+    python=PythonInterpreter.get(),  # type: PythonInterpreter
+):
+    # type: (...) -> None
+    _, stderr = process.communicate()
+    assert 0 != process.returncode
+    if python.version[0] == 2:
+        assert b"ImportError: No module named colors\n" in stderr, stderr.decode("utf-8")
+    else:
+        assert b"ModuleNotFoundError: No module named 'colors'\n" in stderr, stderr.decode("utf-8")
+
+
+@pytest.fixture
+def system_python_with_colors(tmpdir):
+    # type: (Any) -> str
+    location, _, _, _ = ensure_python_distribution(PY310)
+    system_python_distribution = os.path.join(str(tmpdir), "py310")
+    shutil.copytree(location, system_python_distribution)
+    system_python = os.path.join(system_python_distribution, "bin", "python")
+    subprocess.check_call(args=[system_python, "-m", "pip", "install", "ansicolors==1.1.8"])
+    return system_python
+
+
+def test_system_site_packages_venv_pex(
+    tmpdir,  # type: Any
+    system_python_with_colors,  # type: str
+):
+    # type: (...) -> None
+
+    pex = os.path.join(str(tmpdir), "pex")
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "--venv",
+            "--venv-system-site-packages",
+            COWSAY_REQUIREMENT,
+            "-o",
+            pex,
+        ]
+    ).assert_success()
+    assert_colors_import_error(subprocess.Popen(args=[pex] + BLUE_MOO_ARGS, stderr=subprocess.PIPE))
+
+    shutil.rmtree(pex_root)
+    assert_blue_moo(
+        subprocess.check_output(args=[system_python_with_colors, pex] + BLUE_MOO_ARGS).decode(
+            "utf-8"
+        )
+    )
+
+
+def test_system_site_packages_pex_tools(
+    tmpdir,  # type: Any
+    system_python_with_colors,  # type: str
+):
+    # type: (...) -> None
+
+    pex = os.path.join(str(tmpdir), "pex")
+    run_pex_command(args=["--include-tools", COWSAY_REQUIREMENT, "-o", pex]).assert_success()
+
+    venv = os.path.join(str(tmpdir), "venv")
+    subprocess.check_call(
+        args=[system_python_with_colors, pex, "venv", venv], env=make_env(PEX_TOOLS=1)
+    )
+    assert_colors_import_error(subprocess.Popen(args=[pex] + BLUE_MOO_ARGS, stderr=subprocess.PIPE))
+
+    shutil.rmtree(venv)
+    subprocess.check_call(
+        args=[system_python_with_colors, pex, "venv", "--system-site-packages", venv],
+        env=make_env(PEX_TOOLS=1),
+    )
+    assert_blue_moo(
+        subprocess.check_output(args=[os.path.join(venv, "pex")] + BLUE_MOO_ARGS).decode("utf-8")
+    )
+
+
+def test_system_site_packages_pex3_venv(
+    tmpdir,  # type: Any
+    system_python_with_colors,  # type: str
+):
+    # type: (...) -> None
+
+    venv = os.path.join(str(tmpdir), "venv")
+    venv_python = os.path.join(venv, "bin", "python")
+    run_pex3(
+        "venv", "create", "--python", system_python_with_colors, "-d", venv, COWSAY_REQUIREMENT
+    ).assert_success()
+    assert_colors_import_error(
+        subprocess.Popen(args=[venv_python] + BLUE_MOO_ARGS, stderr=subprocess.PIPE),
+        python=PythonInterpreter.from_binary(venv_python),
+    )
+
+    shutil.rmtree(venv)
+    run_pex3(
+        "venv",
+        "create",
+        "--python",
+        system_python_with_colors,
+        "-d",
+        venv,
+        COWSAY_REQUIREMENT,
+        "--system-site-packages",
+    ).assert_success()
+    assert_blue_moo(subprocess.check_output(args=[venv_python] + BLUE_MOO_ARGS).decode("utf-8"))