Merge pull request #19 from kurtmckee/test-bundling

Create a framework for testing bundling and importing

.pre-commit-config.yaml

@@ -4,6 +4,8 @@ ci:
 default_language_version:
   python: "python3.12"
 
+exclude: tests/installed-projects
+
 repos:
   - repo: "meta"
     hooks:

changelog.d/20241029_234739_kurtmckee_test_bundling.rst

@@ -0,0 +1,4 @@
+Development
+-----------
+
+*   Create a framework for testing bundling and importing.

changelog.d/20241030_004631_kurtmckee_test_bundling.rst

@@ -0,0 +1,4 @@
+Fixed
+-----
+
+*   Fix a bug that prevented databases bundled on Windows from finding package metadata.

pyproject.toml

@@ -47,7 +47,7 @@ source = [
 
 [tool.coverage.report]
 skip_covered = true
-fail_under = 27
+fail_under = 52
 
 
 # mypy

src/sqliteimport/accessor.py

@@ -58,7 +58,7 @@ def add_file(self, directory: pathlib.Path, file: pathlib.Path) -> None:
             """,
             (
                 fullname.replace("/", ".").replace("\\", "."),
-                str(file),
+                str(pathlib.PurePosixPath(file)),
                 is_package,
                 (directory / file).read_text(),
             ),

src/sqliteimport/bundler.py

@@ -0,0 +1,30 @@
+import pathlib
+
+import sqliteimport.accessor
+
+
+def bundle(directory: pathlib.Path, accessor: sqliteimport.accessor.Accessor) -> None:
+    """Bundle files in a directory into a database."""
+
+    paths: list[pathlib.Path] = list(directory.glob("*"))
+    files = []
+    for path in paths:
+        rel_path = path.relative_to(directory)
+        if rel_path.suffix in {".so"}:
+            continue
+        if rel_path.name == "__pycache__":
+            continue
+        if str(rel_path) == "bin":
+            continue
+        if path.is_dir():
+            files.append(rel_path)
+            paths.extend(path.glob("*"))
+        else:
+            files.append(rel_path)
+
+    for file in sorted(files):
+        is_package = (directory / file / "__init__.py").exists()
+        print(f"{'* ' if is_package else '  '} {file}")
+        if (directory / file).is_dir():
+            continue
+        accessor.add_file(directory, file)

src/sqliteimport/cli.py

@@ -3,6 +3,7 @@
 import sys
 import textwrap
 
+from . import bundler
 from .accessor import Accessor
 
 try:
@@ -43,32 +44,9 @@ def bundle(directory: pathlib.Path, database: pathlib.Path) -> None:
         pip install --target=DIRECTORY --requirement=path/to/requirements.txt
     """
 
-    paths: list[pathlib.Path] = list(directory.glob("*"))
-    files = []
-    for path in paths:
-        rel_path = path.relative_to(directory)
-        if rel_path.suffix in {".so"}:
-            continue
-        if rel_path.name == "__pycache__":
-            continue
-        if str(rel_path) == "bin":
-            continue
-        if path.is_dir():
-            files.append(rel_path)
-            paths.extend(path.glob("*"))
-        else:
-            files.append(rel_path)
+    with sqlite3.connect(database) as connection:
+        accessor = Accessor(connection)
+        accessor.initialize_database()
 
-    connection = sqlite3.connect(database)
-    accessor = Accessor(connection)
-    accessor.initialize_database()
-
-    for file in sorted(files):
-        is_package = (directory / file / "__init__.py").exists()
-        print(f"{'* ' if is_package else '  '} {file}")
-        if (directory / file).is_dir():
-            continue
-        accessor.add_file(directory, file)
-
-    connection.commit()
-    connection.close()
+        bundler.bundle(directory, accessor)
+        connection.commit()

tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/METADATA

@@ -0,0 +1,25 @@
+Metadata-Version: 2.1
+Name: module-filesystem
+Version: 1.1.1
+Summary: 
+License: MIT
+Author: Kurt McKee
+Author-email: [email protected]
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Description-Content-Type: text/x-rst
+
+A single-file module project.
+

tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/RECORD

@@ -0,0 +1,7 @@
+module_filesystem-1.1.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+module_filesystem-1.1.1.dist-info/METADATA,sha256=q63m78qbu8_bgx_cRJQ2R3AN_YP3KN5IVzcsv5S7vgY,911
+module_filesystem-1.1.1.dist-info/RECORD,,
+module_filesystem-1.1.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+module_filesystem-1.1.1.dist-info/WHEEL,sha256=iAMR_6Qh95yyjYIwRxyjpiFq4FhDPemrEV-SyWIQB3U,92
+module_filesystem-1.1.1.dist-info/direct_url.json,,
+module_filesystem.py,sha256=bUu1TcGpMlFUpoB9EAQVyK4Mb-dhj5n5mNBaqYZrjfM,13

tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.1
+Root-Is-Purelib: true
+Tag: py2.py3-none-any

tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/direct_url.json

@@ -0,0 +1 @@
+{"archive_info": {"hash": "sha256=a2ffda1c72d29cd9563a596a60723546ba61ec3c252195831c9b7c892e46110f", "hashes": {"sha256": "a2ffda1c72d29cd9563a596a60723546ba61ec3c252195831c9b7c892e46110f"}}, "url": "file:///module_filesystem-1.1.1-py2.py3-none-any.whl"}

tests/installed-projects/filesystem/module_filesystem.py

@@ -0,0 +1 @@
+x = "module"

tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/METADATA

@@ -0,0 +1,25 @@
+Metadata-Version: 2.1
+Name: module-sqlite
+Version: 2.2.2
+Summary: 
+License: MIT
+Author: Kurt McKee
+Author-email: [email protected]
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Description-Content-Type: text/x-rst
+
+A single-file module project.
+

tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/RECORD

@@ -0,0 +1,7 @@
+module_sqlite-2.2.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+module_sqlite-2.2.2.dist-info/METADATA,sha256=Fo-Tnd_dSLq4KSJQM0RuZo_SZYHdBk3Uob6oZrQ-fxc,907
+module_sqlite-2.2.2.dist-info/RECORD,,
+module_sqlite-2.2.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+module_sqlite-2.2.2.dist-info/WHEEL,sha256=iAMR_6Qh95yyjYIwRxyjpiFq4FhDPemrEV-SyWIQB3U,92
+module_sqlite-2.2.2.dist-info/direct_url.json,,
+module_sqlite.py,sha256=bUu1TcGpMlFUpoB9EAQVyK4Mb-dhj5n5mNBaqYZrjfM,13

tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.1
+Root-Is-Purelib: true
+Tag: py2.py3-none-any

tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/direct_url.json

@@ -0,0 +1 @@
+{"archive_info": {"hash": "sha256=20b638c1c7902169760aaebbe6d1c0ed83ac2b36f894fe02eed3fd8580ae1ac5", "hashes": {"sha256": "20b638c1c7902169760aaebbe6d1c0ed83ac2b36f894fe02eed3fd8580ae1ac5"}}, "url": "file:///module_sqlite-2.2.2-py2.py3-none-any.whl"}

tests/installed-projects/sqlite/module_sqlite.py

@@ -0,0 +1 @@
+x = "module"

tests/regenerate-test-projects.py

@@ -0,0 +1,103 @@
+import json
+import pathlib
+import shutil
+import subprocess
+import sys
+
+
+def main() -> int:
+    root_directory = pathlib.Path(__file__).absolute().parent.parent
+
+    filesystem_dist_directory = root_directory.parent / "dist/test-suite-filesystem"
+    wipe_directory(filesystem_dist_directory)
+    sqlite_dist_directory = root_directory.parent / "dist/test-suite-sqlite"
+    wipe_directory(sqlite_dist_directory)
+
+    tests_directory = root_directory / "tests"
+    source_projects = tests_directory / "source-projects"
+    for source_project in source_projects.glob("*/"):
+        print(f"Regenerating {source_project.name}/filesystem...")
+        build_wheel(source_project / "filesystem", filesystem_dist_directory)
+        print(f"Regenerating {source_project.name}/sqlite...")
+        build_wheel(source_project / "sqlite", sqlite_dist_directory)
+
+    installed_projects = tests_directory / "installed-projects"
+    wipe_directory(installed_projects)
+
+    install_wheels(installed_projects / "filesystem", filesystem_dist_directory)
+    install_wheels(installed_projects / "sqlite", sqlite_dist_directory)
+
+    sanitize_dist_info_records(installed_projects)
+
+    return 0
+
+
+def wipe_directory(directory: pathlib.Path) -> None:
+    """Erase the given directory and then re-create it."""
+
+    if directory.exists():
+        if not directory.is_dir():
+            raise OSError(f"{directory} is not a directory.")
+        shutil.rmtree(directory)
+
+    directory.mkdir(parents=True, exist_ok=True)
+
+
+def build_wheel(directory: pathlib.Path, output: pathlib.Path) -> None:
+    """Build a wheel using Poetry."""
+
+    command = f"poetry build --format=wheel --directory={directory} --output={output}"
+    subprocess.check_call(command.split())
+
+
+def install_wheels(target: pathlib.Path, dist_directory: pathlib.Path) -> None:
+    """Install all wheels in the given directory."""
+
+    # Install the wheels in the target directory.
+    command = [
+        *f"{sys.executable} -m pip install --target={target}".split(),
+        *[str(path) for path in dist_directory.glob("*.whl")],
+    ]
+    subprocess.check_call(command)
+
+
+def sanitize_dist_info_records(directory: pathlib.Path) -> None:
+    """Sanitize information in `.dist-info/` metadata directories.
+
+    *   The `url` keys in "direct_url.json" files are modified.
+        https://peps.python.org/pep-0610/
+
+    *   The `direct_url.json` lines in "RECORD" files are modified.
+        https://packaging.python.org/en/latest/specifications/recording-installed-packages/
+
+    Modifying these files reduces unnecessary information entering version control.
+    """
+
+    # Modify `direct_url.json` files.
+    for json_file in directory.rglob("**/*.dist-info/direct_url.json"):
+        document = json.loads(json_file.read_text())
+        if "url" in document:
+            url = document["url"]
+            if not url.startswith("file://"):
+                raise ValueError(f"Unexpected URL value in {json_file}: {url}")
+            document["url"] = f"file:///{pathlib.Path(url).name}"
+            json_file.write_text(json.dumps(document))
+
+    # Modify `RECORD` files.
+    for record_file in directory.rglob("**/*.dist-info/RECORD"):
+        with open(record_file) as file:
+            lines = file.read().splitlines()
+            newline = file.newlines
+        output: list[str] = []
+        for line in lines:
+            if "__pycache__" in line:
+                continue
+            if ".dist-info/direct_url.json" in line:
+                direct_url_json_path, _, _ = line.partition(",")
+                line = f"{direct_url_json_path},,"
+            output.append(line)
+        record_file.write_text("\n".join(output), newline=newline)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/source-projects/single-file-module/README.rst

@@ -0,0 +1 @@
+A single-file module project.

tests/source-projects/single-file-module/filesystem/README.rst

@@ -0,0 +1 @@
+A single-file module project.

tests/source-projects/single-file-module/filesystem/module_filesystem.py

@@ -0,0 +1 @@
+x = "module"

tests/source-projects/single-file-module/filesystem/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# poetry
+# ------
+
+[tool.poetry]
+name = "module-filesystem"
+version = "1.1.1"
+description = ""
+authors = ["Kurt McKee <[email protected]>"]
+license = "MIT"
+readme = "README.rst"

tests/source-projects/single-file-module/sqlite/README.rst

@@ -0,0 +1 @@
+A single-file module project.

tests/source-projects/single-file-module/sqlite/module_sqlite.py

@@ -0,0 +1 @@
+x = "module"

tests/source-projects/single-file-module/sqlite/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# poetry
+# ------
+
+[tool.poetry]
+name = "module-sqlite"
+version = "2.2.2"
+description = ""
+authors = ["Kurt McKee <[email protected]>"]
+license = "MIT"
+readme = "README.rst"

tests/test_importing.py

@@ -0,0 +1,36 @@
+import importlib.metadata
+import pathlib
+import sqlite3
+import sys
+
+import pytest
+
+import sqliteimport
+import sqliteimport.accessor
+import sqliteimport.bundler
+
+installed_projects = pathlib.Path(__file__).parent / "installed-projects"
+sys.path.append(str(installed_projects / "filesystem"))
+
+
+@pytest.fixture(scope="module")
+def database():
+    with sqlite3.connect(":memory:") as connection:
+        accessor = sqliteimport.accessor.Accessor(connection)
+        accessor.initialize_database()
+        sqliteimport.bundler.bundle(installed_projects / "sqlite", accessor)
+        sqliteimport.load(connection)
+
+        yield connection
+
+
+def test_module(database):
+    import module_filesystem
+
+    assert module_filesystem.x == "module"
+    assert importlib.metadata.version("module_filesystem") == "1.1.1"
+
+    import module_sqlite
+
+    assert module_sqlite.x == "module"
+    assert importlib.metadata.version("module_sqlite") == "2.2.2"