Plumb deep excludes: the rest with tests WIP.

pex/bin/pex.py

@@ -27,6 +27,7 @@
 from pex.dependency_manager import DependencyManager
 from pex.docs.command import serve_html_docs
 from pex.enum import Enum
+from pex.exclude_configuration import ExcludeConfiguration
 from pex.inherit_path import InheritPath
 from pex.interpreter_constraints import InterpreterConstraints
 from pex.layout import Layout, ensure_installed
@@ -897,6 +898,7 @@ def build_pex(
             )
             excluded.extend(requirements_pex_info.excluded)
 
+    exclude_configuration = ExcludeConfiguration.create(excluded)
     with TRACER.timed(
         "Resolving distributions for requirements: {}".format(
             " ".join(
@@ -911,6 +913,7 @@ def build_pex(
     ):
         try:
             dependency_manager.add_from_resolved(
+                requirement_configuration,
                 resolve(
                     targets=targets,
                     requirement_configuration=requirement_configuration,
@@ -922,13 +925,14 @@ def build_pex(
                         if options.pre_install_wheels
                         else InstallableType.WHEEL_FILE
                     ),
-                )
+                    exclude_configuration=exclude_configuration,
+                ),
             )
         except Unsatisfiable as e:
             die(str(e))
 
     with TRACER.timed("Configuring PEX dependencies"):
-        dependency_manager.configure(pex_builder, excluded=excluded)
+        dependency_manager.configure(pex_builder, exclude_configuration=exclude_configuration)
 
     if options.entry_point:
         pex_builder.set_entry_point(options.entry_point)

pex/dependency_manager.py

@@ -3,24 +3,18 @@
 
 from __future__ import absolute_import
 
-from collections import defaultdict
-
-from pex import pex_warnings
 from pex.dist_metadata import Requirement
 from pex.environment import PEXEnvironment
 from pex.exclude_configuration import ExcludeConfiguration
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.orderedset import OrderedSet
-from pex.pep_503 import ProjectName
 from pex.pex_builder import PEXBuilder
 from pex.pex_info import PexInfo
+from pex.resolve.requirement_configuration import RequirementConfiguration
 from pex.resolve.resolvers import ResolveResult
-from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import DefaultDict, Iterable, Iterator
-
     import attr  # vendor:skip
 else:
     from pex.third_party import attr
@@ -48,77 +42,34 @@ def add_from_pex(
 
         return pex_info
 
-    def add_from_resolved(self, resolved):
-        # type: (ResolveResult) -> None
+    def add_from_resolved(
+        self,
+        requirement_configuration,  # type: RequirementConfiguration
+        resolved,  # type: ResolveResult
+    ):
+        # type: (...) -> None
 
+        # TODO(John Sirois): XXX: What about:
+        #  pex local/project --exclude the-local-project-name-as-it-turns-out
+        #  _
+        #  The local/project dist will not be included in resolved, so we'll have no way to include
+        #  its name in self._requirements (we would have grabbed from
+        #  resolved_dist.direct_requirements).
         for resolved_dist in resolved.distributions:
             self._requirements.update(resolved_dist.direct_requirements)
             self._distributions.add(resolved_dist.fingerprinted_distribution)
 
     def configure(
         self,
         pex_builder,  # type: PEXBuilder
-        excluded=(),  # type: Iterable[str]
+        exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
     ):
         # type: (...) -> None
 
-        exclude_configuration = ExcludeConfiguration.create(excluded)
         exclude_configuration.configure(pex_builder.info)
-
-        dists_by_project_name = defaultdict(
-            OrderedSet
-        )  # type: DefaultDict[ProjectName, OrderedSet[FingerprintedDistribution]]
-        for dist in self._distributions:
-            dists_by_project_name[dist.distribution.metadata.project_name].add(dist)
-
-        root_requirements_by_project_name = defaultdict(
-            OrderedSet
-        )  # type: DefaultDict[ProjectName, OrderedSet[Requirement]]
-        for root_req in self._requirements:
-            root_requirements_by_project_name[root_req.project_name].add(root_req)
-
-        def iter_non_excluded_distributions(requirements):
-            # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution]
-            for req in requirements:
-                candidate_dists = dists_by_project_name[req.project_name]
-                for candidate_dist in tuple(candidate_dists):
-                    if candidate_dist.distribution not in req:
-                        continue
-                    candidate_dists.discard(candidate_dist)
-
-                    excluded_by = exclude_configuration.excluded_by(candidate_dist.distribution)
-                    if excluded_by:
-                        excludes = " and ".join(map(str, excluded_by))
-                        TRACER.log(
-                            "Skipping adding {candidate}: excluded by {excludes}".format(
-                                candidate=candidate_dist.distribution, excludes=excludes
-                            )
-                        )
-                        for root_req in root_requirements_by_project_name[
-                            candidate_dist.distribution.metadata.project_name
-                        ]:
-                            if candidate_dist.distribution in root_req:
-                                pex_warnings.warn(
-                                    "The distribution {dist} was required by the input requirement "
-                                    "{root_req} but excluded by configured excludes: "
-                                    "{excludes}".format(
-                                        dist=candidate_dist.distribution,
-                                        root_req=root_req,
-                                        excludes=excludes,
-                                    )
-                                )
-                        continue
-
-                    yield candidate_dist
-                    for dep in iter_non_excluded_distributions(
-                        candidate_dist.distribution.requires()
-                    ):
-                        yield dep
-
-        for fingerprinted_dist in iter_non_excluded_distributions(self._requirements):
+        for requirement in self._requirements:
+            pex_builder.add_requirement(requirement)
+        for fingerprinted_dist in self._distributions:
             pex_builder.add_distribution(
                 dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint
             )
-
-        for requirement in self._requirements:
-            pex_builder.add_requirement(requirement)

pex/environment.py

@@ -459,8 +459,12 @@ def _resolve_requirement(
             ):
                 yield not_found
 
-    def _root_requirements_iter(self, reqs):
-        # type: (Iterable[Requirement]) -> Iterator[QualifiedRequirementOrNotFound]
+    def _root_requirements_iter(
+        self,
+        reqs,  # type: Iterable[Requirement]
+        exclude_configuration,  # type: ExcludeConfiguration
+    ):
+        # type: (...) -> Iterator[QualifiedRequirementOrNotFound]
 
         # We want to pick one requirement for each key (required project) to then resolve
         # recursively.
@@ -477,6 +481,16 @@ def _root_requirements_iter(self, reqs):
             OrderedDict()
         )  # type: OrderedDict[ProjectName, List[_QualifiedRequirement]]
         for req in reqs:
+            excluded_by = exclude_configuration.excluded_by(req)
+            if excluded_by:
+                TRACER.log(
+                    "Skipping resolving {requirement}: excluded by {excludes}".format(
+                        requirement=req,
+                        excludes=" and ".join(map(str, excluded_by)),
+                    )
+                )
+                continue
+
             required = self._evaluate_marker(req)
             if not required:
                 continue
@@ -591,7 +605,7 @@ def record_unresolved(dist_not_found):
         resolved_dists_by_key = (
             OrderedDict()
         )  # type: OrderedDict[_RequirementKey, FingerprintedDistribution]
-        for qualified_req_or_not_found in self._root_requirements_iter(reqs):
+        for qualified_req_or_not_found in self._root_requirements_iter(reqs, exclude_configuration):
             if isinstance(qualified_req_or_not_found, _DistributionNotFound):
                 record_unresolved(qualified_req_or_not_found)
                 continue

pex/pip/tool.py

@@ -18,11 +18,12 @@
 from pex.auth import PasswordEntry
 from pex.common import safe_mkdir, safe_mkdtemp
 from pex.compatibility import get_stderr_bytes_buffer, shlex_quote, urlparse
+from pex.exclude_configuration import ExcludeConfiguration
 from pex.interpreter import PythonInterpreter
 from pex.jobs import Job
 from pex.network_configuration import NetworkConfiguration
 from pex.pep_427 import install_wheel_interpreter
-from pex.pip import foreign_platform
+from pex.pip import excludes, foreign_platform
 from pex.pip.download_observer import DownloadObserver, PatchSet
 from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage, LogAnalyzer, LogScrapeJob
 from pex.pip.tailer import Tailer
@@ -452,6 +453,7 @@ def spawn_download_distributions(
         package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
         build_configuration=BuildConfiguration(),  # type: BuildConfiguration
         observer=None,  # type: Optional[DownloadObserver]
+        exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
         preserve_log=False,  # type: bool
     ):
         # type: (...) -> Job
@@ -502,7 +504,7 @@ def spawn_download_distributions(
 
         log_analyzers = []  # type: List[LogAnalyzer]
         patch_set = PatchSet()
-        for obs in (foreign_platform_observer, observer):
+        for obs in (foreign_platform_observer, observer, excludes.patch(exclude_configuration)):
             if obs:
                 if obs.analyzer:
                     log_analyzers.append(obs.analyzer)

pex/resolve/configured_resolve.py

@@ -3,6 +3,7 @@
 
 from __future__ import absolute_import
 
+from pex.exclude_configuration import ExcludeConfiguration
 from pex.pep_427 import InstallableType
 from pex.resolve.configured_resolver import ConfiguredResolver
 from pex.resolve.lock_resolver import resolve_from_lock
@@ -30,6 +31,7 @@ def resolve(
     compile_pyc=False,  # type: bool
     ignore_errors=False,  # type: bool
     result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
+    exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
 ):
     # type: (...) -> ResolveResult
     if isinstance(resolver_configuration, LockRepositoryConfiguration):
@@ -58,6 +60,7 @@ def resolve(
                     pip_version=lock.pip_version,
                     use_pip_config=pip_configuration.use_pip_config,
                     result_type=result_type,
+                    exclude_configuration=exclude_configuration,
                 )
             )
     elif isinstance(resolver_configuration, PexRepositoryConfiguration):
@@ -76,6 +79,7 @@ def resolve(
                 transitive=resolver_configuration.transitive,
                 ignore_errors=ignore_errors,
                 result_type=result_type,
+                exclude_configuration=exclude_configuration,
             )
     else:
         with TRACER.timed("Resolving requirements."):
@@ -100,4 +104,5 @@ def resolve(
                 resolver=ConfiguredResolver(pip_configuration=resolver_configuration),
                 use_pip_config=resolver_configuration.use_pip_config,
                 result_type=result_type,
+                exclude_configuration=exclude_configuration,
             )

pex/resolve/lock_resolver.py

@@ -14,6 +14,7 @@
 from pex.auth import PasswordDatabase, PasswordEntry
 from pex.common import pluralize
 from pex.compatibility import cpu_count
+from pex.exclude_configuration import ExcludeConfiguration
 from pex.network_configuration import NetworkConfiguration
 from pex.orderedset import OrderedSet
 from pex.pep_427 import InstallableType
@@ -247,6 +248,7 @@ def resolve_from_lock(
     pip_version=None,  # type: Optional[PipVersionValue]
     use_pip_config=False,  # type: bool
     result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
+    exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
 ):
     # type: (...) -> Union[ResolveResult, Error]
 
@@ -262,6 +264,7 @@ def resolve_from_lock(
             network_configuration=network_configuration,
             build_configuration=build_configuration,
             transitive=transitive,
+            exclude_configuration=exclude_configuration,
         )
     )
     downloadable_artifacts_and_targets = OrderedSet(
@@ -439,6 +442,7 @@ def resolve_from_lock(
             verify_wheels=verify_wheels,
             pip_version=pip_version,
             resolver=resolver,
+            exclude_configuration=exclude_configuration,
         )
 
         local_project_directory_to_sdist = {

pex/resolve/locked_resolve.py

@@ -11,6 +11,7 @@
 from pex.common import pluralize
 from pex.dist_metadata import DistMetadata, Requirement
 from pex.enum import Enum
+from pex.exclude_configuration import ExcludeConfiguration
 from pex.orderedset import OrderedSet
 from pex.pep_425 import CompatibilityTags, TagRank
 from pex.pep_503 import ProjectName
@@ -27,6 +28,7 @@
 from pex.result import Error
 from pex.sorted_tuple import SortedTuple
 from pex.targets import Target
+from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -602,6 +604,7 @@ def resolve(
         transitive=True,  # type: bool
         build_configuration=BuildConfiguration(),  # type: BuildConfiguration
         include_all_matches=False,  # type: bool
+        exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
     ):
         # type: (...) -> Union[Resolved, Error]
 
@@ -652,7 +655,22 @@ def request_resolve(requests):
         for project_name, resolve_requests in required.items():
             reasons = []  # type: List[str]
             compatible_artifacts = []  # type: List[_ResolvedArtifact]
+            excluded_by = None  # type: Optional[Tuple[Requirement, ...]]
             for locked_requirement in repository[project_name]:
+                excluded_by = exclude_configuration.excluded_by(
+                    locked_requirement.pin.as_project_name_and_version()
+                )
+                if excluded_by:
+                    TRACER.log(
+                        "Locked requirement {requirement} from {platform} lock excluded by "
+                        "{exclude} {excluded_by}.".format(
+                            requirement=locked_requirement.pin.as_requirement(),
+                            exclude=pluralize(excluded_by, "exclude"),
+                            excluded_by=" and ".join(map(str, excluded_by)),
+                            platform=self.target_platform,
+                        )
+                    )
+                    break
 
                 def attributed_reason(reason):
                     # type: (str) -> str
@@ -744,6 +762,9 @@ def attributed_reason(reason):
                     for ranked_artifact in ranked_artifacts
                 )
 
+            if excluded_by:
+                continue
+
             if not compatible_artifacts:
                 if reasons:
                     errors.append(

pex/resolve/lockfile/subset.py

@@ -7,6 +7,7 @@
 from collections import OrderedDict
 
 from pex.common import pluralize
+from pex.exclude_configuration import ExcludeConfiguration
 from pex.network_configuration import NetworkConfiguration
 from pex.orderedset import OrderedSet
 from pex.requirements import LocalProjectRequirement, parse_requirement_strings
@@ -49,6 +50,7 @@ def subset(
     build_configuration=BuildConfiguration(),  # type: BuildConfiguration
     transitive=True,  # type: bool
     include_all_matches=False,  # type: bool
+    exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
 ):
     # type: (...) -> Union[SubsetResult, Error]
 
@@ -89,6 +91,7 @@ def subset(
                     build_configuration=build_configuration,
                     transitive=transitive,
                     include_all_matches=include_all_matches,
+                    exclude_configuration=exclude_configuration,
                     # TODO(John Sirois): Plumb `--ignore-errors` to support desired but technically
                     #  invalid `pip-legacy-resolver` locks:
                     #  https://github.com/pex-tool/pex/issues/1652