feat: python plugin file listing

craft_parts/plugins/base.py

@@ -37,7 +37,7 @@
     from craft_parts import infos
 
 
-@dataclass(frozen=True, slots=True)
+@dataclass(frozen=True, slots=True, order=True)
 class Package:
     """A dataclass that uniquely identifies a package."""
 

craft_parts/plugins/python_plugin.py

@@ -16,10 +16,14 @@
 
 """The python plugin."""
 
+import csv
 import shlex
+from email.parser import HeaderParser
 from typing import Literal
 
-from .base import BasePythonPlugin
+from overrides import override
+
+from .base import BasePythonPlugin, Package, PackageFiles
 from .properties import PluginProperties
 
 
@@ -73,3 +77,38 @@ def _get_package_install_commands(self) -> list[str]:
         )
 
         return commands
+
+    @override
+    def get_files(self) -> PackageFiles:
+        # https://packaging.python.org/en/latest/specifications/binary-distribution-format/
+        # Could also add the pkginfo library for this
+
+        venvdir = self._get_venv_directory()
+        python_path = venvdir / "bin/python"
+        python_version = python_path.resolve().name
+        site_pkgs_dir = venvdir / "lib" / python_version / "site-packages"
+
+        ret = {}
+        for pkg_dir in site_pkgs_dir.glob("*.dist-info"):
+            # We only care about the metadata dirs
+            if not pkg_dir.name.endswith(".dist-info"):
+                continue
+
+            # Get package name and version from from METADATA file.
+            # https://packaging.python.org/en/latest/specifications/core-metadata/
+            parser = HeaderParser()
+            with open(pkg_dir / "METADATA") as f:
+                pkg_metadata = parser.parse(f)
+            pkg_name = pkg_metadata["Name"]
+            pkg_version = pkg_metadata["Version"]
+
+            # Read the RECORD file
+            record_file = pkg_dir / "RECORD"
+            with open(record_file) as record_file_obj:
+                csvreader = csv.reader(record_file_obj)
+
+                # First column is files.  These are relative, resolve() to get
+                # rid of all the ".." that leads up to the bin dir.
+                pkg_files = {(site_pkgs_dir / f[0]).resolve() for f in csvreader}
+                ret[Package(pkg_name, pkg_version)] = pkg_files
+        return ret

tests/integration/plugins/test_npm.py

@@ -198,7 +198,7 @@ def test_npm_plugin_include_node(create_fake_package_with_node, new_dir, partiti
 
 
 @pytest.mark.slow
-def test_npm_plugin_get_file_list(create_fake_package_with_node, new_dir, partitions):
+def test_npm_plugin_get_files(create_fake_package_with_node, new_dir, partitions):
     parts = create_fake_package_with_node()
     lifecycle = LifecycleManager(
         parts,

tests/integration/plugins/test_python.py

@@ -19,10 +19,11 @@
 import textwrap
 from pathlib import Path
 
-import craft_parts.plugins.plugins
 import pytest
 import yaml
 from craft_parts import LifecycleManager, Step, errors, plugins
+from craft_parts.plugins.base import Package
+from craft_parts.plugins.python_plugin import PythonPlugin
 from overrides import override
 
 
@@ -123,7 +124,7 @@ def test_python_plugin_symlink(new_dir, partitions):
 def test_python_plugin_override_get_system_interpreter(new_dir, partitions):
     """Override the system interpreter, link should use it."""
 
-    class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin):
+    class MyPythonPlugin(PythonPlugin):
         @override
         def _get_system_python_interpreter(self) -> str | None:
             return "use-this-python"
@@ -159,7 +160,7 @@ def test_python_plugin_no_system_interpreter(
 ):
     """Check that the build fails if a payload interpreter is needed but not found."""
 
-    class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin):
+    class MyPythonPlugin(PythonPlugin):
         @override
         def _get_system_python_interpreter(self) -> str | None:
             return None
@@ -194,7 +195,7 @@ def _should_remove_symlinks(self) -> bool:
 def test_python_plugin_remove_symlinks(new_dir, partitions):
     """Override symlink removal."""
 
-    class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin):
+    class MyPythonPlugin(PythonPlugin):
         @override
         def _should_remove_symlinks(self) -> bool:
             return True
@@ -250,7 +251,7 @@ def test_python_plugin_fix_shebangs(new_dir, partitions):
 def test_python_plugin_override_shebangs(new_dir, partitions):
     """Override what we want in script shebang lines."""
 
-    class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin):
+    class MyPythonPlugin(PythonPlugin):
         @override
         def _get_script_interpreter(self) -> str:
             return "#!/my/script/interpreter"
@@ -298,7 +299,7 @@ def test_find_payload_python_bad_version(new_dir, partitions):
     """Test that the build fails if a payload interpreter is needed but it's the
     wrong Python version."""
 
-    class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin):
+    class MyPythonPlugin(PythonPlugin):
         @override
         def _get_system_python_interpreter(self) -> str | None:
             # To have the build fail after failing to find the payload interpreter
@@ -374,7 +375,7 @@ def test_find_payload_python_good_version(new_dir, partitions):
 def test_no_shebangs(new_dir, partitions):
     """Test that building a Python part with no scripts works."""
 
-    class ScriptlessPlugin(craft_parts.plugins.plugins.PythonPlugin):
+    class ScriptlessPlugin(PythonPlugin):
         @override
         def _get_package_install_commands(self) -> list[str]:
             return [
@@ -408,3 +409,66 @@ def _get_package_install_commands(self) -> list[str]:
 
     primed_script = lf.project_info.prime_dir / "bin/mytest"
     assert not primed_script.exists()
+
+
+@pytest.mark.slow
+def test_python_plugin_get_files(new_dir, partitions):
+    parts_yaml = textwrap.dedent(
+        """\
+        parts:
+          foo:
+            plugin: python
+            source: .
+            python-packages: [flask==3.1.0]
+        """
+    )
+    parts = yaml.safe_load(parts_yaml)
+
+    lifecycle = LifecycleManager(
+        parts,
+        application_name="test_python",
+        cache_dir=new_dir,
+        partitions=partitions,
+    )
+    actions = lifecycle.plan(Step.BUILD)
+
+    with lifecycle.action_executor() as ctx:
+        ctx.execute(actions)
+
+    part_name = list(parts["parts"].keys())[0]
+    actual_file_list = lifecycle._executor._handler[part_name]._plugin.get_files()
+    part_install_dir = lifecycle._executor._part_list[0].part_install_dir
+
+    # Make sure all the expected packages were installed.
+    # We can't assert the exact set of keys because the pip version will change
+    # over time.  And we can't assert the number of keys because py3.10 installs
+    # setuptools as a separate package.
+    assert Package(name="Flask", version="3.1.0") in actual_file_list
+
+    # Can't assert specific versions here because flask has >= versions for its
+    # dependencies.
+    seeking_pkgs = {
+        pkgname: False
+        for pkgname in [
+            "Jinja2",
+            "MarkupSafe",
+            "Werkzeug",
+            "blinker",
+            "click",
+            "itsdangerous",
+        ]
+    }
+    for found_pkg in actual_file_list:
+        for sought_pkg in seeking_pkgs:
+            if found_pkg.name == sought_pkg:
+                seeking_pkgs[sought_pkg] = True
+                break
+    sought_collapsed = set(seeking_pkgs.values())
+    success = len(sought_collapsed) == 1 and sought_collapsed.pop()
+    assert success, f"Didn't find one or more expected packages: {seeking_pkgs}"
+
+    # Check a few specifics to make sure we got package contents correctly
+    assert part_install_dir / "bin/flask" in actual_file_list[Package("Flask", "3.1.0")]
+    assert len(actual_file_list[Package("Jinja2", "3.1.4")]) == 57
+    assert len(actual_file_list[Package("MarkupSafe", "3.0.2")]) == 14
+    assert len(actual_file_list[Package("Werkzeug", "3.1.3")]) == 116

tests/unit/plugins/test_python_plugin.py

@@ -14,11 +14,15 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import csv
+import email.policy
+from email.message import EmailMessage
 from pathlib import Path
 from textwrap import dedent
 
 import pytest
 from craft_parts import Part, PartInfo, ProjectInfo
+from craft_parts.plugins.base import Package
 from craft_parts.plugins.python_plugin import PythonPlugin
 from pydantic import ValidationError
 
@@ -193,3 +197,71 @@ def test_call_should_remove_symlinks(plugin, new_dir, mocker):
         f"[ -f setup.py ] || [ -f pyproject.toml ] && {new_dir}/parts/p1/install/bin/pip install  -U .",
         *get_build_commands(new_dir, should_remove_symlinks=True),
     ]
+
+
+def test_get_file_list(new_dir):
+    part_info = PartInfo(
+        project_info=ProjectInfo(application_name="test", cache_dir=new_dir),
+        part=Part("my-part", {}),
+    )
+    properties = PythonPlugin.properties_class.unmarshal({"source": "."})
+    plugin = PythonPlugin(properties=properties, part_info=part_info)
+
+    # Build a fake file tree, emulating what real package installs look like.
+    # Integration tests actually install stuff and check a subset of the
+    # large installed trees.
+    pkgs_install_dir = plugin._part_info.part_install_dir / "lib/python/site-packages"
+    bins_dir = plugin._part_info.part_install_dir / "bin"
+    pkgs_install_dir.mkdir(parents=True)
+    bins_dir.mkdir()
+
+    (bins_dir / "doit").touch()
+
+    (pkgs_install_dir / "fakeee").mkdir()
+    (pkgs_install_dir / "fakeee/a_file.py").touch()
+    (pkgs_install_dir / "fakeee/things").mkdir()
+    (pkgs_install_dir / "fakeee/things/stuff.py").touch()
+    (pkgs_install_dir / "fakeee/things/nothing.py").touch()
+
+    (pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info").mkdir()
+    (pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt").touch()
+    (pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/REQUESTED").touch()
+
+    metadata = EmailMessage(policy=email.policy.compat32)
+    metadata.add_header("Name", "fakeee")
+    metadata.add_header("Version", "1.2.3-deb_ian")
+    with open(pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/METADATA", "w+") as f:
+        f.write(metadata.as_string())
+
+    file_data = [
+        ["../../../bin/doit", None, None],
+        ["fakeee/a_file.py", None, None],
+        ["fakeee/things/stuff.py", None, None],
+        ["fakeee/things/nothing.py", None, None],
+        ["fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt", None, None],
+        ["fakeee-1.2.3-deb_ian.dist-info/METADATA", None, None],
+        ["fakeee-1.2.3-deb_ian.dist-info/RECORD", None, None],
+        ["fakeee-1.2.3-deb_ian.dist-info/REQUESTED", None, None],
+    ]
+    with open(
+        pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/RECORD", "w+", newline=""
+    ) as f:
+        csvwriter = csv.writer(f)
+        csvwriter.writerows(file_data)
+
+    expected = {
+        Package("fakeee", "1.2.3-deb_ian"): {
+            bins_dir / "doit",
+            pkgs_install_dir / "fakeee/a_file.py",
+            pkgs_install_dir / "fakeee/things/stuff.py",
+            pkgs_install_dir / "fakeee/things/nothing.py",
+            pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt",
+            pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/METADATA",
+            pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/RECORD",
+            pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/REQUESTED",
+        },
+    }
+
+    actual = plugin.get_files()
+
+    assert expected == actual