From 34a6bbf85603c1db407c7bea1866a51f7852e6f0 Mon Sep 17 00:00:00 2001 From: Matt Culler Date: Fri, 20 Dec 2024 10:42:38 -0500 Subject: [PATCH] feat(plugins/python): get package files (#943) --- craft_parts/plugins/base.py | 2 +- craft_parts/plugins/python_plugin.py | 37 +++++++- tests/integration/plugins/test_npm.py | 2 +- tests/integration/plugins/test_python.py | 90 +++++++++++++++++-- tests/unit/plugins/test_python_plugin.py | 37 ++++++++ .../plugins/testfiles/python/install/bin/doit | 2 + .../LICENSE.txt | 1 + .../fakeee-1.2.3-deb_ian.dist-info/METADATA | 2 + .../fakeee-1.2.3-deb_ian.dist-info/RECORD | 8 ++ .../fakeee-1.2.3-deb_ian.dist-info/REQUESTED | 0 .../lib/python/site-packages/fakeee/a_file.py | 2 + .../site-packages/fakeee/things/nothing.py | 2 + .../site-packages/fakeee/things/stuff.py | 2 + 13 files changed, 177 insertions(+), 10 deletions(-) create mode 100755 tests/unit/plugins/testfiles/python/install/bin/doit create mode 100644 tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt create mode 100644 tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/METADATA create mode 100644 tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/RECORD create mode 100644 tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/REQUESTED create mode 100644 tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/a_file.py create mode 100644 tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/things/nothing.py create mode 100644 tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/things/stuff.py diff --git a/craft_parts/plugins/base.py b/craft_parts/plugins/base.py index e8de155fb..c23f79f7c 100644 --- a/craft_parts/plugins/base.py +++ b/craft_parts/plugins/base.py @@ -37,7 +37,7 @@ from craft_parts import infos -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, order=True) class Package: """A dataclass that uniquely identifies a package.""" diff --git a/craft_parts/plugins/python_plugin.py b/craft_parts/plugins/python_plugin.py index b7b170397..32ae64f63 100644 --- a/craft_parts/plugins/python_plugin.py +++ b/craft_parts/plugins/python_plugin.py @@ -16,10 +16,14 @@ """The python plugin.""" +import csv import shlex +from email.parser import HeaderParser from typing import Literal -from .base import BasePythonPlugin +from overrides import override + +from .base import BasePythonPlugin, Package, PackageFiles from .properties import PluginProperties @@ -73,3 +77,34 @@ def _get_package_install_commands(self) -> list[str]: ) return commands + + @override + def get_files(self) -> PackageFiles: + # https://packaging.python.org/en/latest/specifications/binary-distribution-format/ + # Could also add the pkginfo library for this + + venvdir = self._get_venv_directory() + python_path = venvdir / "bin/python" + python_version = python_path.resolve().name + site_pkgs_dir = venvdir / "lib" / python_version / "site-packages" + + ret = {} + for pkg_dir in site_pkgs_dir.glob("*.dist-info"): + # Get package name and version from from METADATA file. + # https://packaging.python.org/en/latest/specifications/core-metadata/ + parser = HeaderParser() + with open(pkg_dir / "METADATA") as f: + pkg_metadata = parser.parse(f) + pkg_name = pkg_metadata["Name"] + pkg_version = pkg_metadata["Version"] + + # Read the RECORD file + record_file = pkg_dir / "RECORD" + with open(record_file) as record_file_obj: + csvreader = csv.reader(record_file_obj) + + # First column is files. These are relative, resolve() to get + # rid of all the ".." that leads up to the bin dir. + pkg_files = {(site_pkgs_dir / f[0]).resolve() for f in csvreader} + ret[Package(pkg_name, pkg_version)] = pkg_files + return ret diff --git a/tests/integration/plugins/test_npm.py b/tests/integration/plugins/test_npm.py index 1e49df6f4..1ab138ac8 100644 --- a/tests/integration/plugins/test_npm.py +++ b/tests/integration/plugins/test_npm.py @@ -198,7 +198,7 @@ def test_npm_plugin_include_node(create_fake_package_with_node, new_dir, partiti @pytest.mark.slow -def test_npm_plugin_get_file_list(create_fake_package_with_node, new_dir, partitions): +def test_npm_plugin_get_files(create_fake_package_with_node, new_dir, partitions): parts = create_fake_package_with_node() lifecycle = LifecycleManager( parts, diff --git a/tests/integration/plugins/test_python.py b/tests/integration/plugins/test_python.py index b063d6389..6ea364287 100644 --- a/tests/integration/plugins/test_python.py +++ b/tests/integration/plugins/test_python.py @@ -19,10 +19,13 @@ import textwrap from pathlib import Path -import craft_parts.plugins.plugins import pytest import yaml from craft_parts import LifecycleManager, Step, errors, plugins +from craft_parts.infos import PartInfo, ProjectInfo +from craft_parts.parts import Part +from craft_parts.plugins.base import Package +from craft_parts.plugins.python_plugin import PythonPlugin from overrides import override @@ -123,7 +126,7 @@ def test_python_plugin_symlink(new_dir, partitions): def test_python_plugin_override_get_system_interpreter(new_dir, partitions): """Override the system interpreter, link should use it.""" - class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin): + class MyPythonPlugin(PythonPlugin): @override def _get_system_python_interpreter(self) -> str | None: return "use-this-python" @@ -159,7 +162,7 @@ def test_python_plugin_no_system_interpreter( ): """Check that the build fails if a payload interpreter is needed but not found.""" - class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin): + class MyPythonPlugin(PythonPlugin): @override def _get_system_python_interpreter(self) -> str | None: return None @@ -194,7 +197,7 @@ def _should_remove_symlinks(self) -> bool: def test_python_plugin_remove_symlinks(new_dir, partitions): """Override symlink removal.""" - class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin): + class MyPythonPlugin(PythonPlugin): @override def _should_remove_symlinks(self) -> bool: return True @@ -250,7 +253,7 @@ def test_python_plugin_fix_shebangs(new_dir, partitions): def test_python_plugin_override_shebangs(new_dir, partitions): """Override what we want in script shebang lines.""" - class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin): + class MyPythonPlugin(PythonPlugin): @override def _get_script_interpreter(self) -> str: return "#!/my/script/interpreter" @@ -298,7 +301,7 @@ def test_find_payload_python_bad_version(new_dir, partitions): """Test that the build fails if a payload interpreter is needed but it's the wrong Python version.""" - class MyPythonPlugin(craft_parts.plugins.plugins.PythonPlugin): + class MyPythonPlugin(PythonPlugin): @override def _get_system_python_interpreter(self) -> str | None: # To have the build fail after failing to find the payload interpreter @@ -374,7 +377,7 @@ def test_find_payload_python_good_version(new_dir, partitions): def test_no_shebangs(new_dir, partitions): """Test that building a Python part with no scripts works.""" - class ScriptlessPlugin(craft_parts.plugins.plugins.PythonPlugin): + class ScriptlessPlugin(PythonPlugin): @override def _get_package_install_commands(self) -> list[str]: return [ @@ -408,3 +411,76 @@ def _get_package_install_commands(self) -> list[str]: primed_script = lf.project_info.prime_dir / "bin/mytest" assert not primed_script.exists() + + +@pytest.mark.slow +def test_python_plugin_get_files(new_dir, partitions): + parts_yaml = textwrap.dedent( + """\ + parts: + foo: + plugin: python + source: . + python-packages: [flask==3.1.0] + """ + ) + parts = yaml.safe_load(parts_yaml) + + lifecycle = LifecycleManager( + parts, + application_name="test_python", + cache_dir=new_dir, + partitions=partitions, + ) + actions = lifecycle.plan(Step.BUILD) + + with lifecycle.action_executor() as ctx: + ctx.execute(actions) + + part_name = list(parts["parts"].keys())[0] + actual_file_list = lifecycle._executor._handler[part_name]._plugin.get_files() + part_install_dir = lifecycle._executor._part_list[0].part_install_dir + + # Real quick instantiate another copy of the plugin to ensure statelessness. + properties2 = PythonPlugin.properties_class.unmarshal(parts["parts"]["foo"]) + part_info = PartInfo( + project_info=ProjectInfo( + application_name="test", cache_dir=new_dir, partitions=partitions + ), + part=Part("foo", {"source": "."}, partitions=partitions), + ) + plugin2 = PythonPlugin(properties=properties2, part_info=part_info) + assert plugin2.get_files() == actual_file_list + + # Make sure all the expected packages were installed. + # We can't assert the exact set of keys because the pip version will change + # over time. And we can't assert the number of keys because py3.10 installs + # setuptools as a separate package. + + assert Package(name="Flask", version="3.1.0") in actual_file_list + assert part_install_dir / "bin/flask" in actual_file_list[Package("Flask", "3.1.0")] + + # Can't assert specific versions here because flask has >= versions for its + # dependencies. + seeking_pkgs = { + pkgname: False + for pkgname in [ + "Jinja2", + "Werkzeug", + "blinker", + "click", + "itsdangerous", + ] + } + for found_pkg in actual_file_list: + # Make sure we got some contents. + # Wheels have at least a METADATA, a RECORD, and we can assume at least + # one actual source file. + if found_pkg.name in seeking_pkgs: + assert len(actual_file_list[found_pkg]) >= 3 + + if found_pkg.name in seeking_pkgs: + seeking_pkgs[found_pkg.name] = True + assert all( + seeking_pkgs.values() + ), f"Didn't find one or more expected packages:\n{seeking_pkgs}" diff --git a/tests/unit/plugins/test_python_plugin.py b/tests/unit/plugins/test_python_plugin.py index b0e9339fd..875bfb1e3 100644 --- a/tests/unit/plugins/test_python_plugin.py +++ b/tests/unit/plugins/test_python_plugin.py @@ -14,11 +14,13 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +import shutil from pathlib import Path from textwrap import dedent import pytest from craft_parts import Part, PartInfo, ProjectInfo +from craft_parts.plugins.base import Package from craft_parts.plugins.python_plugin import PythonPlugin from pydantic import ValidationError @@ -193,3 +195,38 @@ def test_call_should_remove_symlinks(plugin, new_dir, mocker): f"[ -f setup.py ] || [ -f pyproject.toml ] && {new_dir}/parts/p1/install/bin/pip install -U .", *get_build_commands(new_dir, should_remove_symlinks=True), ] + + +def test_get_files(new_dir): + part_info = PartInfo( + project_info=ProjectInfo(application_name="test", cache_dir=new_dir), + part=Part("my-part", {}), + ) + properties = PythonPlugin.properties_class.unmarshal({"source": "."}) + plugin = PythonPlugin(properties=properties, part_info=part_info) + + root = plugin._part_info.part_install_dir + bins_dir = root / "bin" + pkgs_install_dir = root / "lib/python/site-packages" + + # Copy in a fake file tree that emulates a real package installs. + # (Integration tests actually install stuff and check a subset of the + # large installed trees.) + shutil.copytree(Path(__file__).parent / "testfiles/python/install", root) + + expected = { + Package("fakeee", "1.2.3-deb_ian"): { + bins_dir / "doit", + pkgs_install_dir / "fakeee/a_file.py", + pkgs_install_dir / "fakeee/things/stuff.py", + pkgs_install_dir / "fakeee/things/nothing.py", + pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt", + pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/METADATA", + pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/RECORD", + pkgs_install_dir / "fakeee-1.2.3-deb_ian.dist-info/REQUESTED", + }, + } + + actual = plugin.get_files() + + assert expected == actual diff --git a/tests/unit/plugins/testfiles/python/install/bin/doit b/tests/unit/plugins/testfiles/python/install/bin/doit new file mode 100755 index 000000000..aa6ed5bd7 --- /dev/null +++ b/tests/unit/plugins/testfiles/python/install/bin/doit @@ -0,0 +1,2 @@ +#!/bin/sh +echo it is done diff --git a/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt new file mode 100644 index 000000000..de5cc1ebf --- /dev/null +++ b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt @@ -0,0 +1 @@ +This software is covered by the fake license. The terms of this license do not apply to the software. diff --git a/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/METADATA b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/METADATA new file mode 100644 index 000000000..93bee9932 --- /dev/null +++ b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/METADATA @@ -0,0 +1,2 @@ +Name: fakeee +Version: 1.2.3-deb_ian diff --git a/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/RECORD b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/RECORD new file mode 100644 index 000000000..57858b0a9 --- /dev/null +++ b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/RECORD @@ -0,0 +1,8 @@ +../../../bin/doit,, +fakeee/a_file.py,, +fakeee/things/stuff.py,, +fakeee/things/nothing.py,, +fakeee-1.2.3-deb_ian.dist-info/LICENSE.txt,, +fakeee-1.2.3-deb_ian.dist-info/METADATA,, +fakeee-1.2.3-deb_ian.dist-info/RECORD,, +fakeee-1.2.3-deb_ian.dist-info/REQUESTED,, diff --git a/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/REQUESTED b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee-1.2.3-deb_ian.dist-info/REQUESTED new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/a_file.py b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/a_file.py new file mode 100644 index 000000000..0f068ea85 --- /dev/null +++ b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/a_file.py @@ -0,0 +1,2 @@ +def func(): + return True diff --git a/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/things/nothing.py b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/things/nothing.py new file mode 100644 index 000000000..61cdba489 --- /dev/null +++ b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/things/nothing.py @@ -0,0 +1,2 @@ +def lying_function(): + return "lying_function was not executed" diff --git a/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/things/stuff.py b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/things/stuff.py new file mode 100644 index 000000000..809d485b0 --- /dev/null +++ b/tests/unit/plugins/testfiles/python/install/lib/python/site-packages/fakeee/things/stuff.py @@ -0,0 +1,2 @@ +def dontdoit(a): + print(f"Not done: {a}")