diff --git a/pex/build_system/__init__.py b/pex/build_system/__init__.py index 3cfc24085..029046b60 100644 --- a/pex/build_system/__init__.py +++ b/pex/build_system/__init__.py @@ -3,10 +3,26 @@ from __future__ import absolute_import +import json +import os +import subprocess +from textwrap import dedent + +from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode, safe_mkdtemp +from pex.dist_metadata import Distribution +from pex.interpreter import PythonInterpreter +from pex.jobs import Job, SpawnedJob +from pex.pex import PEX +from pex.pex_bootstrapper import VenvPex, ensure_venv +from pex.pex_builder import PEXBuilder +from pex.result import Error from pex.typing import TYPE_CHECKING +from pex.variables import ENV +from pex.venv.bin_path import BinPath +from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Tuple + from typing import Any, Iterable, Mapping, Optional, Tuple, Union import attr # vendor:skip else: @@ -33,3 +49,161 @@ class BuildSystemTable(object): DEFAULT_BUILD_SYSTEM_TABLE = BuildSystemTable( requires=DEFAULT_BUILD_REQUIRES, build_backend=DEFAULT_BUILD_BACKEND ) + + +# Exit code 75 is EX_TEMPFAIL defined in /usr/include/sysexits.h +# this seems an appropriate signal of DNE vs execute and fail. +_HOOK_UNAVAILABLE_EXIT_CODE = 75 + + +@attr.s(frozen=True) +class BuildSystem(object): + @classmethod + def create( + cls, + interpreter, # type: PythonInterpreter + requires, # type: Iterable[str] + resolved, # type: Iterable[Distribution] + build_backend, # type: str + backend_path, # type: Tuple[str, ...] + extra_requirements=None, # type: Optional[Iterable[str]] + use_system_time=False, # type: bool + **extra_env # type: str + ): + # type: (...) -> Union[BuildSystem, Error] + pex_builder = PEXBuilder(copy_mode=CopyMode.SYMLINK) + pex_builder.info.venv = True + pex_builder.info.venv_site_packages_copies = True + pex_builder.info.venv_bin_path = BinPath.PREPEND + # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect. + pex_builder.info.venv_hermetic_scripts = False + for req in requires: + pex_builder.add_requirement(req) + for dist in resolved: + pex_builder.add_distribution(dist) + pex_builder.freeze(bytecode_compile=False) + venv_pex = ensure_venv(PEX(pex_builder.path(), interpreter=interpreter)) + if extra_requirements: + # N.B.: We install extra requirements separately instead of having them resolved and + # handed in with the `resolved` above because there are cases in the wild where the + # build system requires (PEP-518) and the results of PEP-517 `get_requires_for_*` can + # return overlapping requirements. Pip will error for overlaps complaining of duplicate + # requirements if we attempt to resolve all the requirements at once; so we instead + # resolve and install in two phases. This obviously has problems! That said, it is, in + # fact, how Pip's internal PEP-517 build frontend works; so we emulate that. + virtualenv = Virtualenv(venv_pex.venv_dir) + # Python 3.5 comes with Pip 9.0.1 which is pretty broken: it doesn't work with our test + # cases; so we upgrade. + # For Python 2.7 we use virtualenv (there is no -m venv built into Python) and that + # comes with Pip 22.0.2, Python 3.6 comes with Pip 18.1 and Python 3.7 comes with + # Pip 22.04 and the default Pips only get newer with newer version of Pythons. These all + # work well enough for our test cases and, in general, they should work well enough with + # the Python they come paired with. + upgrade_pip = virtualenv.interpreter.version[:2] == (3, 5) + virtualenv.ensure_pip(upgrade=upgrade_pip) + with open(os.devnull, "wb") as dev_null: + _, process = virtualenv.interpreter.open_process( + args=[ + "-m", + "pip", + "install", + "--ignore-installed", + "--no-user", + "--no-warn-script-location", + ] + + list(extra_requirements), + stdout=dev_null, + stderr=subprocess.PIPE, + ) + _, stderr = process.communicate() + if process.returncode != 0: + return Error( + "Failed to install extra requirement in venv at {venv_dir}: " + "{extra_requirements}\nSTDERR:\n{stderr}".format( + venv_dir=venv_pex.venv_dir, + extra_requirements=", ".join(extra_requirements), + stderr=stderr.decode("utf-8"), + ) + ) + + # Ensure all PEX* env vars are stripped except for PEX_ROOT and PEX_VERBOSE. We want folks + # to be able to steer the location of the cache and the logging verbosity, but nothing else. + # We control the entry-point, etc. of the PEP-518 build backend venv for internal use. + with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE)) as env: + if extra_env: + env.update(extra_env) + if backend_path: + env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path)) + if not use_system_time: + env.update(REPRODUCIBLE_BUILDS_ENV) + return cls( + venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env + ) + + venv_pex = attr.ib() # type: VenvPex + build_backend = attr.ib() # type: str + requires = attr.ib() # type: Tuple[str, ...] + env = attr.ib() # type: Mapping[str, str] + + def invoke_build_hook( + self, + project_directory, # type: str + hook_method, # type: str + hook_args=(), # type: Iterable[Any] + hook_kwargs=None, # type: Optional[Mapping[str, Any]] + ): + # type: (...) -> Union[SpawnedJob[Any], Error] + + # The interfaces are spec'd here: https://peps.python.org/pep-0517 + build_backend_module, _, _ = self.build_backend.partition(":") + build_backend_object = self.build_backend.replace(":", ".") + build_hook_result = os.path.join( + safe_mkdtemp(prefix="pex-pep-517."), "build_hook_result.json" + ) + args = self.venv_pex.execute_args( + additional_args=( + "-c", + dedent( + """\ + import json + import sys + + import {build_backend_module} + + + if not hasattr({build_backend_object}, {hook_method!r}): + sys.exit({hook_unavailable_exit_code}) + + result = {build_backend_object}.{hook_method}(*{hook_args!r}, **{hook_kwargs!r}) + with open({result_file!r}, "w") as fp: + json.dump(result, fp) + """ + ).format( + build_backend_module=build_backend_module, + build_backend_object=build_backend_object, + hook_method=hook_method, + hook_args=tuple(hook_args), + hook_kwargs=dict(hook_kwargs) if hook_kwargs else {}, + hook_unavailable_exit_code=_HOOK_UNAVAILABLE_EXIT_CODE, + result_file=build_hook_result, + ), + ) + ) + process = subprocess.Popen( + args=args, + env=self.env, + cwd=project_directory, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return SpawnedJob.file( + Job( + command=args, + process=process, + context="PEP-517:{hook_method} at {project_directory}".format( + hook_method=hook_method, project_directory=project_directory + ), + ), + output_file=build_hook_result, + result_func=lambda file_content: json.loads(file_content.decode("utf-8")), + ) diff --git a/pex/build_system/pep_517.py b/pex/build_system/pep_517.py index e96303d47..99ce8195b 100644 --- a/pex/build_system/pep_517.py +++ b/pex/build_system/pep_517.py @@ -3,14 +3,11 @@ from __future__ import absolute_import -import json import os -import subprocess -from textwrap import dedent from pex import third_party -from pex.build_system import DEFAULT_BUILD_BACKEND -from pex.build_system.pep_518 import BuildSystem, load_build_system +from pex.build_system import DEFAULT_BUILD_BACKEND, BuildSystem +from pex.build_system.pep_518 import load_build_system from pex.common import safe_mkdtemp from pex.dist_metadata import DistMetadata, Distribution, MetadataType from pex.jobs import Job, SpawnedJob @@ -134,67 +131,21 @@ def _invoke_build_hook( ) ) - build_system_or_error = _get_build_system( + result = _get_build_system( target, resolver, project_directory, extra_requirements=hook_extra_requirements, pip_version=pip_version, ) - if isinstance(build_system_or_error, Error): - return build_system_or_error - build_system = build_system_or_error - - # The interfaces are spec'd here: https://peps.python.org/pep-0517 - build_backend_module, _, _ = build_system.build_backend.partition(":") - build_backend_object = build_system.build_backend.replace(":", ".") - build_hook_result = os.path.join(safe_mkdtemp(prefix="pex-pep-517."), "build_hook_result.json") - args = build_system.venv_pex.execute_args( - additional_args=( - "-c", - dedent( - """\ - import json - import sys - - import {build_backend_module} - - - if not hasattr({build_backend_object}, {hook_method!r}): - sys.exit({hook_unavailable_exit_code}) - - result = {build_backend_object}.{hook_method}(*{hook_args!r}, **{hook_kwargs!r}) - with open({result_file!r}, "w") as fp: - json.dump(result, fp) - """ - ).format( - build_backend_module=build_backend_module, - build_backend_object=build_backend_object, - hook_method=hook_method, - hook_args=tuple(hook_args), - hook_kwargs=dict(hook_kwargs) if hook_kwargs else {}, - hook_unavailable_exit_code=_HOOK_UNAVAILABLE_EXIT_CODE, - result_file=build_hook_result, - ), - ) - ) - process = subprocess.Popen( - args=args, - env=build_system.env, - cwd=project_directory, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - return SpawnedJob.file( - Job( - command=args, - process=process, - context="PEP-517:{hook_method} at {project_directory}".format( - hook_method=hook_method, project_directory=project_directory - ), - ), - output_file=build_hook_result, - result_func=lambda file_content: json.loads(file_content.decode("utf-8")), + if isinstance(result, Error): + return result + + return result.invoke_build_hook( + project_directory=project_directory, + hook_method=hook_method, + hook_args=hook_args, + hook_kwargs=hook_kwargs, ) diff --git a/pex/build_system/pep_518.py b/pex/build_system/pep_518.py index ac5db2424..59a5add8c 100644 --- a/pex/build_system/pep_518.py +++ b/pex/build_system/pep_518.py @@ -4,31 +4,22 @@ from __future__ import absolute_import import os.path -import subprocess from pex import toml -from pex.build_system import DEFAULT_BUILD_BACKEND, DEFAULT_BUILD_SYSTEM_TABLE, BuildSystemTable -from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode -from pex.dist_metadata import Distribution -from pex.interpreter import PythonInterpreter -from pex.pex import PEX -from pex.pex_bootstrapper import VenvPex, ensure_venv -from pex.pex_builder import PEXBuilder +from pex.build_system import ( + DEFAULT_BUILD_BACKEND, + DEFAULT_BUILD_SYSTEM_TABLE, + BuildSystem, + BuildSystemTable, +) from pex.resolve.resolvers import Resolver from pex.result import Error from pex.targets import LocalInterpreter, Target, Targets from pex.tracer import TRACER from pex.typing import TYPE_CHECKING -from pex.variables import ENV -from pex.venv.bin_path import BinPath -from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Iterable, Mapping, Optional, Tuple, Union - - import attr # vendor:skip -else: - from pex.third_party import attr + from typing import Iterable, Optional, Union def _read_build_system_table( @@ -62,96 +53,6 @@ def _read_build_system_table( ) -@attr.s(frozen=True) -class BuildSystem(object): - @classmethod - def create( - cls, - interpreter, # type: PythonInterpreter - requires, # type: Iterable[str] - resolved, # type: Iterable[Distribution] - build_backend, # type: str - backend_path, # type: Tuple[str, ...] - extra_requirements=None, # type: Optional[Iterable[str]] - use_system_time=False, # type: bool - **extra_env # type: str - ): - # type: (...) -> Union[BuildSystem, Error] - pex_builder = PEXBuilder(copy_mode=CopyMode.SYMLINK) - pex_builder.info.venv = True - pex_builder.info.venv_site_packages_copies = True - pex_builder.info.venv_bin_path = BinPath.PREPEND - # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect. - pex_builder.info.venv_hermetic_scripts = False - for req in requires: - pex_builder.add_requirement(req) - for dist in resolved: - pex_builder.add_distribution(dist) - pex_builder.freeze(bytecode_compile=False) - venv_pex = ensure_venv(PEX(pex_builder.path(), interpreter=interpreter)) - if extra_requirements: - # N.B.: We install extra requirements separately instead of having them resolved and - # handed in with the `resolved` above because there are cases in the wild where the - # build system requires (PEP-518) and the results of PEP-517 `get_requires_for_*` can - # return overlapping requirements. Pip will error for overlaps complaining of duplicate - # requirements if we attempt to resolve all the requirements at once; so we instead - # resolve and install in two phases. This obviously has problems! That said, it is, in - # fact, how Pip's internal PEP-517 build frontend works; so we emulate that. - virtualenv = Virtualenv(venv_pex.venv_dir) - # Python 3.5 comes with Pip 9.0.1 which is pretty broken: it doesn't work with our test - # cases; so we upgrade. - # For Python 2.7 we use virtualenv (there is no -m venv built into Python) and that - # comes with Pip 22.0.2, Python 3.6 comes with Pip 18.1 and Python 3.7 comes with - # Pip 22.04 and the default Pips only get newer with newer version of Pythons. These all - # work well enough for our test cases and, in general, they should work well enough with - # the Python they come paired with. - upgrade_pip = virtualenv.interpreter.version[:2] == (3, 5) - virtualenv.ensure_pip(upgrade=upgrade_pip) - with open(os.devnull, "wb") as dev_null: - _, process = virtualenv.interpreter.open_process( - args=[ - "-m", - "pip", - "install", - "--ignore-installed", - "--no-user", - "--no-warn-script-location", - ] - + list(extra_requirements), - stdout=dev_null, - stderr=subprocess.PIPE, - ) - _, stderr = process.communicate() - if process.returncode != 0: - return Error( - "Failed to install extra requirement in venv at {venv_dir}: " - "{extra_requirements}\nSTDERR:\n{stderr}".format( - venv_dir=venv_pex.venv_dir, - extra_requirements=", ".join(extra_requirements), - stderr=stderr.decode("utf-8"), - ) - ) - - # Ensure all PEX* env vars are stripped except for PEX_ROOT and PEX_VERBOSE. We want folks - # to be able to steer the location of the cache and the logging verbosity, but nothing else. - # We control the entry-point, etc. of the PEP-518 build backend venv for internal use. - with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE)) as env: - if extra_env: - env.update(extra_env) - if backend_path: - env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path)) - if not use_system_time: - env.update(REPRODUCIBLE_BUILDS_ENV) - return cls( - venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env - ) - - venv_pex = attr.ib() # type: VenvPex - build_backend = attr.ib() # type: str - requires = attr.ib() # type: Tuple[str, ...] - env = attr.ib() # type: Mapping[str, str] - - def _maybe_load_build_system_table(project_directory): # type: (str) -> Union[Optional[BuildSystemTable], Error] diff --git a/pex/resolve/lock_downloader.py b/pex/resolve/lock_downloader.py new file mode 100644 index 000000000..44159aa47 --- /dev/null +++ b/pex/resolve/lock_downloader.py @@ -0,0 +1,379 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os +import shutil +from collections import OrderedDict +from multiprocessing.pool import ThreadPool + +from pex import resolver +from pex.atomic_directory import FileLockStyle +from pex.auth import PasswordDatabase, PasswordEntry +from pex.common import pluralize +from pex.compatibility import cpu_count +from pex.dist_metadata import Requirement +from pex.network_configuration import NetworkConfiguration +from pex.pep_503 import ProjectName +from pex.pip.local_project import digest_local_project +from pex.pip.tool import PackageIndexConfiguration +from pex.pip.vcs import digest_vcs_archive +from pex.pip.version import PipVersionValue +from pex.resolve.downloads import ArtifactDownloader +from pex.resolve.locked_resolve import ( + DownloadableArtifact, + FileArtifact, + LocalProjectArtifact, + LockConfiguration, + VCSArtifact, +) +from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager +from pex.resolve.lockfile.model import Lockfile +from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion +from pex.resolve.resolvers import MAX_PARALLEL_DOWNLOADS, Resolver +from pex.result import Error, catch +from pex.targets import Target, Targets +from pex.typing import TYPE_CHECKING +from pex.variables import ENV, Variables + +if TYPE_CHECKING: + from typing import Dict, Iterable, Mapping, Optional, Sequence, Tuple, Union + + import attr # vendor:skip + + from pex.hashing import HintedDigest +else: + from pex.third_party import attr + + +class FileArtifactDownloadManager(DownloadManager[FileArtifact]): + def __init__( + self, + file_lock_style, # type: FileLockStyle.Value + downloader, # type: ArtifactDownloader + pex_root=ENV, # type: Union[str, Variables] + ): + super(FileArtifactDownloadManager, self).__init__( + pex_root=pex_root, file_lock_style=file_lock_style + ) + self._downloader = downloader + + def save( + self, + artifact, # type: FileArtifact + project_name, # type: ProjectName + dest_dir, # type: str + digest, # type: HintedDigest + ): + # type: (...) -> Union[str, Error] + return self._downloader.download(artifact=artifact, dest_dir=dest_dir, digest=digest) + + +class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]): + def __init__( + self, + target, # type: Target + file_lock_style, # type: FileLockStyle.Value + indexes=None, # type: Optional[Sequence[str]] + find_links=None, # type: Optional[Sequence[str]] + resolver_version=None, # type: Optional[ResolverVersion.Value] + network_configuration=None, # type: Optional[NetworkConfiguration] + password_entries=(), # type: Iterable[PasswordEntry] + cache=None, # type: Optional[str] + build_configuration=BuildConfiguration(), # type: BuildConfiguration + pex_root=ENV, # type: Union[str, Variables] + pip_version=None, # type: Optional[PipVersionValue] + resolver=None, # type: Optional[Resolver] + use_pip_config=False, # type: bool + extra_pip_requirements=(), # type: Tuple[Requirement, ...] + ): + super(VCSArtifactDownloadManager, self).__init__( + pex_root=pex_root, file_lock_style=file_lock_style + ) + self._target = target + self._indexes = indexes + self._find_links = find_links + self._resolver_version = resolver_version + self._network_configuration = network_configuration + self._password_entries = password_entries + self._cache = cache + + # Since a VCSArtifactDownloadManager is only used for VCS requirements, a build is both + # required and preferred by the user. + self._build_configuration = attr.evolve( + build_configuration, allow_builds=True, prefer_older_binary=False + ) + + self._pip_version = pip_version + self._resolver = resolver + self._use_pip_config = use_pip_config + self._extra_pip_requirements = extra_pip_requirements + + def save( + self, + artifact, # type: VCSArtifact + project_name, # type: ProjectName + dest_dir, # type: str + digest, # type: HintedDigest + ): + # type: (...) -> Union[str, Error] + + requirement = artifact.as_unparsed_requirement(project_name) + downloaded_vcs = resolver.download( + targets=Targets.from_target(self._target), + requirements=[requirement], + transitive=False, + indexes=self._indexes, + find_links=self._find_links, + resolver_version=self._resolver_version, + network_configuration=self._network_configuration, + password_entries=self._password_entries, + build_configuration=self._build_configuration, + max_parallel_jobs=1, + pip_version=self._pip_version, + resolver=self._resolver, + use_pip_config=self._use_pip_config, + extra_pip_requirements=self._extra_pip_requirements, + ) + if len(downloaded_vcs.local_distributions) != 1: + return Error( + "Expected 1 artifact for an intransitive download of {requirement}, found " + "{count}:\n" + "{downloads}".format( + requirement=requirement, + count=len(downloaded_vcs.local_distributions), + downloads="\n".join( + "{index}. {download}".format(index=index, download=download.path) + for index, download in enumerate( + downloaded_vcs.local_distributions, start=1 + ) + ), + ) + ) + + local_distribution = downloaded_vcs.local_distributions[0] + filename = os.path.basename(local_distribution.path) + digest_vcs_archive( + archive_path=local_distribution.path, + vcs=artifact.vcs, + digest=digest, + ) + shutil.move(local_distribution.path, os.path.join(dest_dir, filename)) + return filename + + +class LocalProjectDownloadManager(DownloadManager[LocalProjectArtifact]): + def __init__( + self, + target, # type: Target + file_lock_style, # type: FileLockStyle.Value + resolver, # type: Resolver + pip_version=None, # type: Optional[PipVersionValue] + pex_root=ENV, # type: Union[str, Variables] + ): + super(LocalProjectDownloadManager, self).__init__( + pex_root=pex_root, file_lock_style=file_lock_style + ) + self._target = target + self._pip_version = pip_version + self._resolver = resolver + + def save( + self, + artifact, # type: LocalProjectArtifact + project_name, # type: ProjectName + dest_dir, # type: str + digest, # type: HintedDigest + ): + # type: (...) -> Union[str, Error] + source_dir_or_error = digest_local_project( + directory=artifact.directory, + digest=digest, + pip_version=self._pip_version, + target=self._target, + resolver=self._resolver, + dest_dir=dest_dir, + ) + if isinstance(source_dir_or_error, Error): + return source_dir_or_error + return os.path.basename(source_dir_or_error) + + +@attr.s(frozen=True) +class LockDownloader(object): + @classmethod + def create( + cls, + targets, # type: Iterable[Target] + lock, # type: Lockfile + resolver, # type: Resolver + indexes=None, # type: Optional[Sequence[str]] + find_links=None, # type: Optional[Sequence[str]] + max_parallel_jobs=None, # type: Optional[int] + pip_version=None, # type: Optional[PipVersionValue] + resolver_version=None, # type: Optional[ResolverVersion.Value] + network_configuration=None, # type: Optional[NetworkConfiguration] + password_entries=(), # type: Iterable[PasswordEntry] + build_configuration=BuildConfiguration(), # type: BuildConfiguration + use_pip_config=False, # type: bool + extra_pip_requirements=(), # type: Tuple[Requirement, ...] + ): + # type: (...) -> LockDownloader + + # Since the download managers are stored to via a thread pool, we need to use BSD style locks. + # These locks are not as portable as POSIX style locks but work with threading unlike POSIX + # locks which are subject to threading-unaware deadlock detection per the standard. Linux, in + # fact, implements deadlock detection for POSIX locks; so we can run afoul of false EDEADLCK + # errors under the right interleaving of processes and threads and download artifact targets. + file_lock_style = FileLockStyle.BSD + + file_download_managers_by_target = { + target: FileArtifactDownloadManager( + file_lock_style=file_lock_style, + downloader=ArtifactDownloader( + resolver=resolver, + lock_configuration=LockConfiguration( + style=lock.style, + requires_python=lock.requires_python, + target_systems=lock.target_systems, + lock_build_systems=lock.lock_build_systems, + ), + target=target, + package_index_configuration=PackageIndexConfiguration.create( + pip_version=pip_version, + resolver_version=resolver_version, + indexes=indexes, + find_links=find_links, + network_configuration=network_configuration, + password_entries=( + PasswordDatabase.from_netrc().append(password_entries).entries + ), + use_pip_config=use_pip_config, + extra_pip_requirements=extra_pip_requirements, + ), + max_parallel_jobs=max_parallel_jobs, + ), + ) + for target in targets + } + + vcs_download_managers_by_target = { + target: VCSArtifactDownloadManager( + target=target, + file_lock_style=file_lock_style, + indexes=indexes, + find_links=find_links, + resolver_version=resolver_version, + network_configuration=network_configuration, + password_entries=password_entries, + build_configuration=build_configuration, + pip_version=pip_version, + resolver=resolver, + use_pip_config=use_pip_config, + extra_pip_requirements=extra_pip_requirements, + ) + for target in targets + } + + local_project_download_managers_by_target = { + target: LocalProjectDownloadManager( + file_lock_style=file_lock_style, + pip_version=pip_version, + target=target, + resolver=resolver, + ) + for target in targets + } + + return cls( + file_download_managers_by_target, + vcs_download_managers_by_target, + local_project_download_managers_by_target, + max_parallel_jobs, + ) + + file_download_managers_by_target = ( + attr.ib() + ) # type: Mapping[Target, FileArtifactDownloadManager] + vcs_download_managers_by_target = attr.ib() # type: Mapping[Target, VCSArtifactDownloadManager] + local_project_download_managers_by_target = ( + attr.ib() + ) # type: Mapping[Target, LocalProjectDownloadManager] + max_parallel_jobs = attr.ib(default=None) # type: Optional[int] + + def download_artifact(self, downloadable_artifact_and_target): + # type: (Tuple[DownloadableArtifact, Target]) -> Union[DownloadedArtifact, Error] + downloadable_artifact, target = downloadable_artifact_and_target + if isinstance(downloadable_artifact.artifact, VCSArtifact): + return catch( + self.vcs_download_managers_by_target[target].store, + downloadable_artifact.artifact, + downloadable_artifact.pin.project_name, + ) + + if isinstance(downloadable_artifact.artifact, FileArtifact): + return catch( + self.file_download_managers_by_target[target].store, + downloadable_artifact.artifact, + downloadable_artifact.pin.project_name, + ) + + return catch( + self.local_project_download_managers_by_target[target].store, + downloadable_artifact.artifact, + downloadable_artifact.pin.project_name, + ) + + def download_artifacts(self, downloadable_artifacts_and_targets): + # type: (Sequence[Tuple[DownloadableArtifact, Target]]) -> Union[Dict[DownloadableArtifact, DownloadedArtifact], Error] + max_threads = min( + len(downloadable_artifacts_and_targets) or 1, + min(MAX_PARALLEL_DOWNLOADS, 4 * (self.max_parallel_jobs or cpu_count() or 1)), + ) + pool = ThreadPool(processes=max_threads) + try: + download_results = tuple( + zip( + tuple( + downloadable_artifact + for downloadable_artifact, _ in downloadable_artifacts_and_targets + ), + pool.map(self.download_artifact, downloadable_artifacts_and_targets), + ) + ) + finally: + pool.close() + pool.join() + + downloaded_artifacts = {} # type: Dict[DownloadableArtifact, DownloadedArtifact] + download_errors = OrderedDict() # type: OrderedDict[DownloadableArtifact, Error] + for downloadable_artifact, download_result in download_results: + if isinstance(download_result, DownloadedArtifact): + downloaded_artifacts[downloadable_artifact] = download_result + else: + download_errors[downloadable_artifact] = download_result + + if download_errors: + error_count = len(download_errors) + return Error( + "There {were} {count} {errors} downloading required artifacts:\n" + "{error_details}".format( + were="was" if error_count == 1 else "were", + count=error_count, + errors=pluralize(download_errors, "error"), + error_details="\n".join( + "{index}. {pin} from {url}\n {error}".format( + index=index, + pin=downloadable_artifact.pin, + url=downloadable_artifact.artifact.url.download_url, + error="\n ".join(str(error).splitlines()), + ) + for index, (downloadable_artifact, error) in enumerate( + download_errors.items(), start=1 + ) + ), + ) + ) + + return downloaded_artifacts diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index 4ce1b8497..b821f2011 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -3,238 +3,222 @@ from __future__ import absolute_import -import functools -import os.path -import shutil -from collections import OrderedDict -from multiprocessing.pool import ThreadPool +import hashlib +import os +import tarfile +from collections import OrderedDict, defaultdict -from pex import resolver -from pex.atomic_directory import FileLockStyle from pex.auth import PasswordDatabase, PasswordEntry -from pex.common import pluralize -from pex.compatibility import cpu_count +from pex.build_system import BuildSystem, BuildSystemTable +from pex.cache.dirs import CacheDir +from pex.common import open_zip, safe_mkdtemp from pex.dependency_configuration import DependencyConfiguration -from pex.dist_metadata import Requirement, is_wheel +from pex.dist_metadata import Distribution, Requirement, is_sdist, is_tar_sdist, is_wheel +from pex.exceptions import production_assert +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.interpreter import PythonInterpreter from pex.network_configuration import NetworkConfiguration -from pex.orderedset import OrderedSet -from pex.pep_427 import InstallableType -from pex.pep_503 import ProjectName -from pex.pip.local_project import digest_local_project +from pex.pep_427 import InstallableType, install_wheel_chroot from pex.pip.tool import PackageIndexConfiguration -from pex.pip.vcs import digest_vcs_archive from pex.pip.version import PipVersionValue -from pex.resolve.downloads import ArtifactDownloader +from pex.resolve.lock_downloader import LockDownloader from pex.resolve.locked_resolve import ( DownloadableArtifact, - FileArtifact, LocalProjectArtifact, - LockConfiguration, - VCSArtifact, + LockedResolve, + Resolved, ) -from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager +from pex.resolve.lockfile.download_manager import DownloadedArtifact from pex.resolve.lockfile.model import Lockfile -from pex.resolve.lockfile.subset import subset +from pex.resolve.lockfile.subset import subset, subset_for_target from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion -from pex.resolve.resolvers import MAX_PARALLEL_DOWNLOADS, Resolver, ResolveResult +from pex.resolve.resolvers import ResolvedDistribution, Resolver, ResolveResult from pex.resolver import BuildAndInstallRequest, BuildRequest, InstallRequest -from pex.result import Error, catch, try_ +from pex.result import Error, try_ +from pex.sorted_tuple import SortedTuple from pex.targets import Target, Targets from pex.tracer import TRACER -from pex.typing import TYPE_CHECKING -from pex.variables import ENV, Variables +from pex.typing import TYPE_CHECKING, cast +from pex.util import CacheHelper if TYPE_CHECKING: - from typing import Dict, Iterable, Mapping, Optional, Sequence, Tuple, Union + from typing import DefaultDict, Dict, Iterable, List, Optional, Sequence, Tuple, Union import attr # vendor:skip - - from pex.hashing import HintedDigest else: from pex.third_party import attr -class FileArtifactDownloadManager(DownloadManager[FileArtifact]): - def __init__( - self, - file_lock_style, # type: FileLockStyle.Value - downloader, # type: ArtifactDownloader - pex_root=ENV, # type: Union[str, Variables] - ): - super(FileArtifactDownloadManager, self).__init__( - pex_root=pex_root, file_lock_style=file_lock_style - ) - self._downloader = downloader - - def save( - self, - artifact, # type: FileArtifact - project_name, # type: ProjectName - dest_dir, # type: str - digest, # type: HintedDigest - ): - # type: (...) -> Union[str, Error] - return self._downloader.download(artifact=artifact, dest_dir=dest_dir, digest=digest) - - -class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]): - def __init__( - self, - target, # type: Target - file_lock_style, # type: FileLockStyle.Value - indexes=None, # type: Optional[Sequence[str]] - find_links=None, # type: Optional[Sequence[str]] - resolver_version=None, # type: Optional[ResolverVersion.Value] - network_configuration=None, # type: Optional[NetworkConfiguration] - password_entries=(), # type: Iterable[PasswordEntry] - cache=None, # type: Optional[str] - build_configuration=BuildConfiguration(), # type: BuildConfiguration - pex_root=ENV, # type: Union[str, Variables] - pip_version=None, # type: Optional[PipVersionValue] - resolver=None, # type: Optional[Resolver] - use_pip_config=False, # type: bool - extra_pip_requirements=(), # type: Tuple[Requirement, ...] - ): - super(VCSArtifactDownloadManager, self).__init__( - pex_root=pex_root, file_lock_style=file_lock_style - ) - self._target = target - self._indexes = indexes - self._find_links = find_links - self._resolver_version = resolver_version - self._network_configuration = network_configuration - self._password_entries = password_entries - self._cache = cache - - # Since a VCSArtifactDownloadManager is only used for VCS requirements, a build is both - # required and preferred by the user. - self._build_configuration = attr.evolve( - build_configuration, allow_builds=True, prefer_older_binary=False - ) - - self._pip_version = pip_version - self._resolver = resolver - self._use_pip_config = use_pip_config - self._extra_pip_requirements = extra_pip_requirements +@attr.s(frozen=True) +class LockedSourceDistribution(object): + target = attr.ib() # type: Target + source_artifact = attr.ib() # type: DownloadedArtifact + build_system_table = attr.ib() # type: BuildSystemTable + locked_resolves = attr.ib() # type: Tuple[LockedResolve, ...] - def save( - self, - artifact, # type: VCSArtifact - project_name, # type: ProjectName - dest_dir, # type: str - digest, # type: HintedDigest - ): - # type: (...) -> Union[str, Error] - requirement = artifact.as_unparsed_requirement(project_name) - downloaded_vcs = resolver.download( - targets=Targets.from_target(self._target), - requirements=[requirement], - transitive=False, - indexes=self._indexes, - find_links=self._find_links, - resolver_version=self._resolver_version, - network_configuration=self._network_configuration, - password_entries=self._password_entries, - build_configuration=self._build_configuration, - max_parallel_jobs=1, - pip_version=self._pip_version, - resolver=self._resolver, - use_pip_config=self._use_pip_config, - extra_pip_requirements=self._extra_pip_requirements, +def build_locked_source_distribution( + locked_source_distribution, # type: LockedSourceDistribution + install_requests, # type: Iterable[InstallRequest] + result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value +): + # type: (...) -> Union[ResolvedDistribution, Error] + + installed_wheels_dir = CacheDir.INSTALLED_WHEELS.path() + build_system_distributions = [] # type: List[Distribution] + for install_request in install_requests: + install_result = install_request.result(installed_wheels_dir) + installed_wheel = install_wheel_chroot( + wheel_path=install_request.wheel_path, destination=install_result.build_chroot ) - if len(downloaded_vcs.local_distributions) != 1: - return Error( - "Expected 1 artifact for an intransitive download of {requirement}, found " - "{count}:\n" - "{downloads}".format( - requirement=requirement, - count=len(downloaded_vcs.local_distributions), - downloads="\n".join( - "{index}. {download}".format(index=index, download=download.path) - for index, download in enumerate( - downloaded_vcs.local_distributions, start=1 - ) - ), - ) - ) - - local_distribution = downloaded_vcs.local_distributions[0] - filename = os.path.basename(local_distribution.path) - digest_vcs_archive( - archive_path=local_distribution.path, - vcs=artifact.vcs, - digest=digest, + build_system_distributions.append(Distribution.load(installed_wheel.prefix_dir)) + + result = BuildSystem.create( + interpreter=PythonInterpreter.get(), + requires=locked_source_distribution.build_system_table.requires, + resolved=build_system_distributions, + build_backend=locked_source_distribution.build_system_table.build_backend, + backend_path=locked_source_distribution.build_system_table.backend_path, + ) + if isinstance(result, Error): + return result + + source_artifact_path = locked_source_distribution.source_artifact.path + if is_sdist(source_artifact_path): + chroot = safe_mkdtemp() + if is_tar_sdist(source_artifact_path): + with tarfile.open(source_artifact_path) as tar_fp: + tar_fp.extractall(chroot) + else: + with open_zip(source_artifact_path) as zip_fp: + zip_fp.extractall(chroot) + for root, _, files in os.walk(chroot, topdown=True): + if any(f in ("setup.py", "setup.cfg", "pyproject.toml") for f in files): + project_directory = root + break + else: + return Error("TODO(John Sirois): XXX: Can't happen!") + else: + project_directory = source_artifact_path + + build_dir = os.path.join(safe_mkdtemp(), "build") + os.mkdir(build_dir) + spawned_job = try_( + result.invoke_build_hook( + project_directory, + hook_method="build_wheel", + hook_args=[build_dir], ) - shutil.move(local_distribution.path, os.path.join(dest_dir, filename)) - return filename - - -class LocalProjectDownloadManager(DownloadManager[LocalProjectArtifact]): - def __init__( - self, - target, # type: Target - file_lock_style, # type: FileLockStyle.Value - resolver, # type: Resolver - pip_version=None, # type: Optional[PipVersionValue] - pex_root=ENV, # type: Union[str, Variables] - ): - super(LocalProjectDownloadManager, self).__init__( - pex_root=pex_root, file_lock_style=file_lock_style + ) + distribution = spawned_job.map(lambda _: Distribution.load(build_dir)).await_result() + build_wheel_fingerprint = CacheHelper.hash(distribution.location, hasher=hashlib.sha256) + if result_type is InstallableType.INSTALLED_WHEEL_CHROOT: + install_request = InstallRequest( + target=locked_source_distribution.target, + wheel_path=distribution.location, + fingerprint=build_wheel_fingerprint, ) - self._target = target - self._pip_version = pip_version - self._resolver = resolver - - def save( - self, - artifact, # type: LocalProjectArtifact - project_name, # type: ProjectName - dest_dir, # type: str - digest, # type: HintedDigest - ): - # type: (...) -> Union[str, Error] - source_dir_or_error = digest_local_project( - directory=artifact.directory, - digest=digest, - pip_version=self._pip_version, - target=self._target, - resolver=self._resolver, - dest_dir=dest_dir, + install_result = install_request.result(installed_wheels_dir) + installed_wheel = install_wheel_chroot( + wheel_path=install_request.wheel_path, destination=install_result.build_chroot ) - if isinstance(source_dir_or_error, Error): - return source_dir_or_error - return os.path.basename(source_dir_or_error) + distribution = Distribution.load(installed_wheel.prefix_dir) + + return ResolvedDistribution( + target=locked_source_distribution.target, + fingerprinted_distribution=FingerprintedDistribution( + distribution=distribution, fingerprint=build_wheel_fingerprint + ), + direct_requirements=SortedTuple(), + ) -def download_artifact( - downloadable_artifact_and_target, # type: Tuple[DownloadableArtifact, Target] - file_download_managers_by_target, # type: Mapping[Target, FileArtifactDownloadManager] - vcs_download_managers_by_target, # type: Mapping[Target, VCSArtifactDownloadManager] - local_project_download_managers_by_target, # type: Mapping[Target, LocalProjectDownloadManager] +def build_locked_source_distributions( + locked_source_distributions, # type: Sequence[LockedSourceDistribution] + lock_downloader, # type: LockDownloader + result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value + build_configuration=BuildConfiguration(), # type: BuildConfiguration + dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): - # type: (...) -> Union[DownloadedArtifact, Error] - downloadable_artifact, target = downloadable_artifact_and_target - if isinstance(downloadable_artifact.artifact, VCSArtifact): - return catch( - vcs_download_managers_by_target[target].store, - downloadable_artifact.artifact, - downloadable_artifact.pin.project_name, + # type: (...) -> Union[Iterable[ResolvedDistribution], Error] + + downloadable_artifacts_by_locked_source_distribution = ( + {} + ) # type: Dict[LockedSourceDistribution, Tuple[DownloadableArtifact, ...]] + subset_errors = OrderedDict() # type: OrderedDict[LockedSourceDistribution, Tuple[Error, ...]] + for locked_source_distribution in locked_source_distributions: + subset_result = subset_for_target( + target=locked_source_distribution.target, + locked_resolves=locked_source_distribution.locked_resolves, + requirements_to_resolve=tuple( + Requirement.parse(req) + for req in locked_source_distribution.build_system_table.requires + ), + build_configuration=build_configuration, + dependency_configuration=dependency_configuration, ) - - if isinstance(downloadable_artifact.artifact, FileArtifact): - return catch( - file_download_managers_by_target[target].store, - downloadable_artifact.artifact, - downloadable_artifact.pin.project_name, + if isinstance(subset_result, Resolved): + downloadable_artifacts_by_locked_source_distribution[ + locked_source_distribution + ] = subset_result.downloadable_artifacts + elif isinstance(subset_result, tuple) and subset_result: + subset_errors[locked_source_distribution] = subset_result + if subset_errors: + return Error("TODO(John Sirois): XXX: build a subset errors message") + + downloaded_artifacts = try_( + lock_downloader.download_artifacts( + tuple( + (downloadable_artifact, locked_source_distribution.target) + for locked_source_distribution, downloadable_artifacts in downloadable_artifacts_by_locked_source_distribution.items() + for downloadable_artifact in downloadable_artifacts + ) ) - - return catch( - local_project_download_managers_by_target[target].store, - downloadable_artifact.artifact, - downloadable_artifact.pin.project_name, ) + install_requests_by_locked_source_distribution = defaultdict( + list + ) # type: DefaultDict[LockedSourceDistribution, List[InstallRequest]] + resolve_errors = defaultdict( + list + ) # type: DefaultDict[LockedSourceDistribution, List[DownloadedArtifact]] + for ( + locked_source_distribution, + downloadable_artifacts, + ) in downloadable_artifacts_by_locked_source_distribution.items(): + for downloadable_artifact in downloadable_artifacts: + downloaded_artifact = downloaded_artifacts[downloadable_artifact] + if is_wheel(downloaded_artifact.path): + install_requests_by_locked_source_distribution[locked_source_distribution].append( + InstallRequest( + target=locked_source_distribution.target, + wheel_path=downloaded_artifact.path, + fingerprint=downloaded_artifact.fingerprint, + ) + ) + else: + resolve_errors[locked_source_distribution].append(downloaded_artifact) + if resolve_errors: + return Error("TODO(John Sirois): XXX: build a resolve errors message") + + # TODO(John Sirois): now we have a list of install requests needed per each source distribution + # build system, parallelize install + create pip venv + build wheel + built_distributions = [] # type: List[ResolvedDistribution] + build_errors = [] # type: List[Error] + for ( + locked_source_distribution, + install_requests, + ) in install_requests_by_locked_source_distribution.items(): + build_result = build_locked_source_distribution( + locked_source_distribution, install_requests, result_type + ) + if isinstance(build_result, ResolvedDistribution): + built_distributions.append(build_result) + else: + build_errors.append(build_result) + if build_errors: + return Error("TODO(John Sirois): XXX: build a build errors message") + return built_distributions def resolve_from_lock( @@ -278,78 +262,26 @@ def resolve_from_lock( dependency_configuration=dependency_configuration, ) ) - downloadable_artifacts_and_targets = OrderedSet( + downloadable_artifacts_and_targets = tuple( (downloadable_artifact, resolved_subset.target) for resolved_subset in subset_result.subsets for downloadable_artifact in resolved_subset.resolved.downloadable_artifacts ) - # Since the download managers are stored to via a thread pool, we need to use BSD style locks. - # These locks are not as portable as POSIX style locks but work with threading unlike POSIX - # locks which are subject to threading-unaware deadlock detection per the standard. Linux, in - # fact, implements deadlock detection for POSIX locks; so we can run afoul of false EDEADLCK - # errors under the right interleaving of processes and threads and download artifact targets. - file_lock_style = FileLockStyle.BSD - - file_download_managers_by_target = { - resolved_subset.target: FileArtifactDownloadManager( - file_lock_style=file_lock_style, - downloader=ArtifactDownloader( - resolver=resolver, - lock_configuration=LockConfiguration( - style=lock.style, - requires_python=lock.requires_python, - target_systems=lock.target_systems, - lock_build_systems=lock.lock_build_systems, - ), - target=resolved_subset.target, - package_index_configuration=PackageIndexConfiguration.create( - pip_version=pip_version, - resolver_version=resolver_version, - indexes=indexes, - find_links=find_links, - network_configuration=network_configuration, - password_entries=PasswordDatabase.from_netrc().append(password_entries).entries, - use_pip_config=use_pip_config, - extra_pip_requirements=extra_pip_requirements, - ), - max_parallel_jobs=max_parallel_jobs, - ), - ) - for resolved_subset in subset_result.subsets - } - - vcs_download_managers_by_target = { - resolved_subset.target: VCSArtifactDownloadManager( - target=resolved_subset.target, - file_lock_style=file_lock_style, - indexes=indexes, - find_links=find_links, - resolver_version=resolver_version, - network_configuration=network_configuration, - password_entries=password_entries, - build_configuration=build_configuration, - pip_version=pip_version, - resolver=resolver, - use_pip_config=use_pip_config, - extra_pip_requirements=extra_pip_requirements, - ) - for resolved_subset in subset_result.subsets - } - - local_project_download_managers_by_target = { - resolved_subset.target: LocalProjectDownloadManager( - file_lock_style=file_lock_style, - pip_version=pip_version, - target=resolved_subset.target, - resolver=resolver, - ) - for resolved_subset in subset_result.subsets - } - - max_threads = min( - len(downloadable_artifacts_and_targets) or 1, - min(MAX_PARALLEL_DOWNLOADS, 4 * (max_parallel_jobs or cpu_count() or 1)), + lock_downloader = LockDownloader.create( + targets=tuple(resolved_subset.target for resolved_subset in subset_result.subsets), + lock=lock, + resolver=resolver, + indexes=indexes, + find_links=find_links, + max_parallel_jobs=max_parallel_jobs, + pip_version=pip_version, + resolver_version=resolver_version, + network_configuration=network_configuration, + password_entries=password_entries, + build_configuration=build_configuration, + use_pip_config=use_pip_config, + extra_pip_requirements=extra_pip_requirements, ) with TRACER.timed( "Downloading {url_count} distributions to satisfy {requirement_count} requirements".format( @@ -357,62 +289,14 @@ def resolve_from_lock( requirement_count=len(subset_result.requirements), ) ): - pool = ThreadPool(processes=max_threads) - try: - download_results = tuple( - zip( - tuple( - downloadable_artifact - for downloadable_artifact, _ in downloadable_artifacts_and_targets - ), - pool.map( - functools.partial( - download_artifact, - file_download_managers_by_target=file_download_managers_by_target, - vcs_download_managers_by_target=vcs_download_managers_by_target, - local_project_download_managers_by_target=local_project_download_managers_by_target, - ), - downloadable_artifacts_and_targets, - ), - ) - ) - finally: - pool.close() - pool.join() - - with TRACER.timed("Categorizing {} downloaded artifacts".format(len(download_results))): - downloaded_artifacts = {} # type: Dict[DownloadableArtifact, DownloadedArtifact] - download_errors = OrderedDict() # type: OrderedDict[DownloadableArtifact, Error] - for downloadable_artifact, download_result in download_results: - if isinstance(download_result, DownloadedArtifact): - downloaded_artifacts[downloadable_artifact] = download_result - else: - download_errors[downloadable_artifact] = download_result - - if download_errors: - error_count = len(download_errors) - return Error( - "There {were} {count} {errors} downloading required artifacts:\n" - "{error_details}".format( - were="was" if error_count == 1 else "were", - count=error_count, - errors=pluralize(download_errors, "error"), - error_details="\n".join( - "{index}. {pin} from {url}\n {error}".format( - index=index, - pin=downloadable_artifact.pin, - url=downloadable_artifact.artifact.url.download_url, - error="\n ".join(str(error).splitlines()), - ) - for index, (downloadable_artifact, error) in enumerate( - download_errors.items(), start=1 - ) - ), - ) - ) + downloaded_artifacts = try_( + lock_downloader.download_artifacts(downloadable_artifacts_and_targets) + ) - build_requests = [] - install_requests = [] + with TRACER.timed("Categorizing {} downloaded artifacts".format(len(downloaded_artifacts))): + build_requests = [] # type: List[BuildRequest] + locked_build_requests = [] # type: List[LockedSourceDistribution] + install_requests = [] # type: List[InstallRequest] for resolved_subset in subset_result.subsets: for downloadable_artifact in resolved_subset.resolved.downloadable_artifacts: downloaded_artifact = downloaded_artifacts[downloadable_artifact] @@ -424,6 +308,20 @@ def resolve_from_lock( fingerprint=downloaded_artifact.fingerprint, ) ) + elif lock.lock_build_systems: + production_assert(downloadable_artifact.build_system_table is not None) + build_system_table = cast( + BuildSystemTable, downloadable_artifact.build_system_table + ) + locked_build_system_resolves = lock.build_systems[build_system_table] + locked_build_requests.append( + LockedSourceDistribution( + target=resolved_subset.target, + source_artifact=downloaded_artifact, + build_system_table=build_system_table, + locked_resolves=locked_build_system_resolves, + ) + ) else: build_requests.append( BuildRequest( @@ -432,12 +330,30 @@ def resolve_from_lock( fingerprint=downloaded_artifact.fingerprint, ) ) + build_request_count = len(build_requests) + locked_build_request_count = len(locked_build_requests) + production_assert( + ((build_request_count > 0) ^ (locked_build_request_count > 0)) + or (build_request_count == 0 and locked_build_request_count == 0) + ) with TRACER.timed( "Building {} artifacts and installing {}".format( - len(build_requests), len(build_requests) + len(install_requests) + build_request_count, build_request_count + len(install_requests) ) ): + distributions = list( + try_( + build_locked_source_distributions( + locked_build_requests, + lock_downloader, + result_type=result_type, + build_configuration=build_configuration, + dependency_configuration=dependency_configuration, + ) + ) + ) + build_and_install_request = BuildAndInstallRequest( build_requests=build_requests, install_requests=install_requests, @@ -472,7 +388,7 @@ def resolve_from_lock( # `LockedResolve.resolve` above and need not waste time (~O(100ms)) doing this again. ignore_errors = True - distributions = ( + distributions.extend( build_and_install_request.install_distributions( ignore_errors=ignore_errors, max_parallel_jobs=max_parallel_jobs, diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index c81bbb266..914170111 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -454,6 +454,11 @@ def create( artifact = attr.ib() # type: Union[FileArtifact, LocalProjectArtifact, VCSArtifact] satisfied_direct_requirements = attr.ib(default=SortedTuple()) # type: SortedTuple[Requirement] + @property + def build_system_table(self): + # type: () -> Optional[BuildSystemTable] + return self.artifact.build_system_table + @attr.s(frozen=True) class Resolved(object): diff --git a/pex/resolve/lockfile/subset.py b/pex/resolve/lockfile/subset.py index 0d8e3a49d..7505a8b68 100644 --- a/pex/resolve/lockfile/subset.py +++ b/pex/resolve/lockfile/subset.py @@ -12,7 +12,7 @@ from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.requirements import LocalProjectRequirement, parse_requirement_strings -from pex.resolve.locked_resolve import Resolved +from pex.resolve.locked_resolve import LockedResolve, Resolved from pex.resolve.lockfile.model import Lockfile from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolver_configuration import BuildConfiguration @@ -43,6 +43,41 @@ class SubsetResult(object): subsets = attr.ib() # type: Tuple[Subset, ...] +def subset_for_target( + target, # type: Target + locked_resolves, # type: Iterable[LockedResolve] + requirements_to_resolve, # type: Iterable[Requirement] + constraints=(), # type: Iterable[Requirement] + source=None, # type: Optional[str] + build_configuration=BuildConfiguration(), # type: BuildConfiguration + transitive=True, # type: bool + include_all_matches=False, # type: bool + dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration +): + # type: (...) -> Union[Resolved, Tuple[Error, ...]] + resolveds = [] + errors = [] + for locked_resolve in locked_resolves: + resolve_result = locked_resolve.resolve( + target, + requirements_to_resolve, + constraints=constraints, + source=source, + build_configuration=build_configuration, + transitive=transitive, + include_all_matches=include_all_matches, + dependency_configuration=dependency_configuration, + # TODO(John Sirois): Plumb `--ignore-errors` to support desired but technically + # invalid `pip-legacy-resolver` locks: + # https://github.com/pex-tool/pex/issues/1652 + ) + if isinstance(resolve_result, Resolved): + resolveds.append(resolve_result) + else: + errors.append(resolve_result) + return Resolved.most_specific(resolveds) if resolveds else tuple(errors) + + def subset( targets, # type: Targets lock, # type: Lockfile @@ -105,31 +140,21 @@ def subset( ) ): for target in targets.unique_targets(): - resolveds = [] - errors = [] - for locked_resolve in lock.locked_resolves: - resolve_result = locked_resolve.resolve( - target, - requirements_to_resolve, - constraints=constraints, - source=lock.source, - build_configuration=build_configuration, - transitive=transitive, - include_all_matches=include_all_matches, - dependency_configuration=dependency_configuration, - # TODO(John Sirois): Plumb `--ignore-errors` to support desired but technically - # invalid `pip-legacy-resolver` locks: - # https://github.com/pex-tool/pex/issues/1652 - ) - if isinstance(resolve_result, Resolved): - resolveds.append(resolve_result) - else: - errors.append(resolve_result) - - if resolveds: - resolved_by_target[target] = Resolved.most_specific(resolveds) - elif errors: - errors_by_target[target] = tuple(errors) + result = subset_for_target( + target, + locked_resolves=lock.locked_resolves, + requirements_to_resolve=requirements_to_resolve, + constraints=constraints, + source=lock.source, + build_configuration=build_configuration, + transitive=transitive, + include_all_matches=include_all_matches, + dependency_configuration=dependency_configuration, + ) + if isinstance(result, Resolved): + resolved_by_target[target] = result + elif len(result) > 0: + errors_by_target[target] = result if errors_by_target: return Error( diff --git a/tests/build_system/test_pep_518.py b/tests/build_system/test_pep_518.py index 677b1bc6e..13cf317ad 100644 --- a/tests/build_system/test_pep_518.py +++ b/tests/build_system/test_pep_518.py @@ -5,8 +5,7 @@ import subprocess from textwrap import dedent -from pex.build_system import pep_518 -from pex.build_system.pep_518 import BuildSystem +from pex.build_system import BuildSystem, pep_518 from pex.common import touch from pex.pep_503 import ProjectName from pex.resolve.configured_resolver import ConfiguredResolver diff --git a/tests/integration/build_system/test_pep_518.py b/tests/integration/build_system/test_pep_518.py index fadf61f89..5cdb83188 100644 --- a/tests/integration/build_system/test_pep_518.py +++ b/tests/integration/build_system/test_pep_518.py @@ -4,7 +4,8 @@ import os.path import subprocess -from pex.build_system.pep_518 import BuildSystem, load_build_system +from pex.build_system import BuildSystem +from pex.build_system.pep_518 import load_build_system from pex.pip.version import PipVersion from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.resolver_configuration import PipConfiguration, ReposConfiguration