diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bfd9e4d..245d099 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,8 @@ ci: default_language_version: python: "python3.12" +exclude: tests/installed-projects + repos: - repo: "meta" hooks: diff --git a/changelog.d/20241029_234739_kurtmckee_test_bundling.rst b/changelog.d/20241029_234739_kurtmckee_test_bundling.rst new file mode 100644 index 0000000..9eb7ce6 --- /dev/null +++ b/changelog.d/20241029_234739_kurtmckee_test_bundling.rst @@ -0,0 +1,4 @@ +Development +----------- + +* Create a framework for testing bundling and importing. diff --git a/changelog.d/20241030_004631_kurtmckee_test_bundling.rst b/changelog.d/20241030_004631_kurtmckee_test_bundling.rst new file mode 100644 index 0000000..588233a --- /dev/null +++ b/changelog.d/20241030_004631_kurtmckee_test_bundling.rst @@ -0,0 +1,4 @@ +Fixed +----- + +* Fix a bug that prevented databases bundled on Windows from finding package metadata. diff --git a/pyproject.toml b/pyproject.toml index a7d33e0..8447ecc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ source = [ [tool.coverage.report] skip_covered = true -fail_under = 27 +fail_under = 52 # mypy diff --git a/src/sqliteimport/accessor.py b/src/sqliteimport/accessor.py index 95a38eb..4b2c125 100644 --- a/src/sqliteimport/accessor.py +++ b/src/sqliteimport/accessor.py @@ -58,7 +58,7 @@ def add_file(self, directory: pathlib.Path, file: pathlib.Path) -> None: """, ( fullname.replace("/", ".").replace("\\", "."), - str(file), + str(pathlib.PurePosixPath(file)), is_package, (directory / file).read_text(), ), diff --git a/src/sqliteimport/bundler.py b/src/sqliteimport/bundler.py new file mode 100644 index 0000000..ce332ac --- /dev/null +++ b/src/sqliteimport/bundler.py @@ -0,0 +1,30 @@ +import pathlib + +import sqliteimport.accessor + + +def bundle(directory: pathlib.Path, accessor: sqliteimport.accessor.Accessor) -> None: + """Bundle files in a directory into a database.""" + + paths: list[pathlib.Path] = list(directory.glob("*")) + files = [] + for path in paths: + rel_path = path.relative_to(directory) + if rel_path.suffix in {".so"}: + continue + if rel_path.name == "__pycache__": + continue + if str(rel_path) == "bin": + continue + if path.is_dir(): + files.append(rel_path) + paths.extend(path.glob("*")) + else: + files.append(rel_path) + + for file in sorted(files): + is_package = (directory / file / "__init__.py").exists() + print(f"{'* ' if is_package else ' '} {file}") + if (directory / file).is_dir(): + continue + accessor.add_file(directory, file) diff --git a/src/sqliteimport/cli.py b/src/sqliteimport/cli.py index 2c79823..2ff1ef1 100644 --- a/src/sqliteimport/cli.py +++ b/src/sqliteimport/cli.py @@ -3,6 +3,7 @@ import sys import textwrap +from . import bundler from .accessor import Accessor try: @@ -43,32 +44,9 @@ def bundle(directory: pathlib.Path, database: pathlib.Path) -> None: pip install --target=DIRECTORY --requirement=path/to/requirements.txt """ - paths: list[pathlib.Path] = list(directory.glob("*")) - files = [] - for path in paths: - rel_path = path.relative_to(directory) - if rel_path.suffix in {".so"}: - continue - if rel_path.name == "__pycache__": - continue - if str(rel_path) == "bin": - continue - if path.is_dir(): - files.append(rel_path) - paths.extend(path.glob("*")) - else: - files.append(rel_path) + with sqlite3.connect(database) as connection: + accessor = Accessor(connection) + accessor.initialize_database() - connection = sqlite3.connect(database) - accessor = Accessor(connection) - accessor.initialize_database() - - for file in sorted(files): - is_package = (directory / file / "__init__.py").exists() - print(f"{'* ' if is_package else ' '} {file}") - if (directory / file).is_dir(): - continue - accessor.add_file(directory, file) - - connection.commit() - connection.close() + bundler.bundle(directory, accessor) + connection.commit() diff --git a/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/INSTALLER b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/METADATA b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/METADATA new file mode 100644 index 0000000..dbb92cc --- /dev/null +++ b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/METADATA @@ -0,0 +1,25 @@ +Metadata-Version: 2.1 +Name: module-filesystem +Version: 1.1.1 +Summary: +License: MIT +Author: Kurt McKee +Author-email: contactme@kurtmckee.org +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Description-Content-Type: text/x-rst + +A single-file module project. + diff --git a/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/RECORD b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/RECORD new file mode 100644 index 0000000..fb4722d --- /dev/null +++ b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/RECORD @@ -0,0 +1,7 @@ +module_filesystem-1.1.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +module_filesystem-1.1.1.dist-info/METADATA,sha256=q63m78qbu8_bgx_cRJQ2R3AN_YP3KN5IVzcsv5S7vgY,911 +module_filesystem-1.1.1.dist-info/RECORD,, +module_filesystem-1.1.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +module_filesystem-1.1.1.dist-info/WHEEL,sha256=iAMR_6Qh95yyjYIwRxyjpiFq4FhDPemrEV-SyWIQB3U,92 +module_filesystem-1.1.1.dist-info/direct_url.json,, +module_filesystem.py,sha256=bUu1TcGpMlFUpoB9EAQVyK4Mb-dhj5n5mNBaqYZrjfM,13 \ No newline at end of file diff --git a/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/REQUESTED b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/REQUESTED new file mode 100644 index 0000000..e69de29 diff --git a/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/WHEEL b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/WHEEL new file mode 100644 index 0000000..3fce7be --- /dev/null +++ b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: poetry-core 1.9.1 +Root-Is-Purelib: true +Tag: py2.py3-none-any diff --git a/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/direct_url.json b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/direct_url.json new file mode 100644 index 0000000..7ece39b --- /dev/null +++ b/tests/installed-projects/filesystem/module_filesystem-1.1.1.dist-info/direct_url.json @@ -0,0 +1 @@ +{"archive_info": {"hash": "sha256=a2ffda1c72d29cd9563a596a60723546ba61ec3c252195831c9b7c892e46110f", "hashes": {"sha256": "a2ffda1c72d29cd9563a596a60723546ba61ec3c252195831c9b7c892e46110f"}}, "url": "file:///module_filesystem-1.1.1-py2.py3-none-any.whl"} \ No newline at end of file diff --git a/tests/installed-projects/filesystem/module_filesystem.py b/tests/installed-projects/filesystem/module_filesystem.py new file mode 100644 index 0000000..53ce041 --- /dev/null +++ b/tests/installed-projects/filesystem/module_filesystem.py @@ -0,0 +1 @@ +x = "module" diff --git a/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/INSTALLER b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/METADATA b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/METADATA new file mode 100644 index 0000000..cc284b1 --- /dev/null +++ b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/METADATA @@ -0,0 +1,25 @@ +Metadata-Version: 2.1 +Name: module-sqlite +Version: 2.2.2 +Summary: +License: MIT +Author: Kurt McKee +Author-email: contactme@kurtmckee.org +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Description-Content-Type: text/x-rst + +A single-file module project. + diff --git a/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/RECORD b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/RECORD new file mode 100644 index 0000000..183fd2d --- /dev/null +++ b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/RECORD @@ -0,0 +1,7 @@ +module_sqlite-2.2.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +module_sqlite-2.2.2.dist-info/METADATA,sha256=Fo-Tnd_dSLq4KSJQM0RuZo_SZYHdBk3Uob6oZrQ-fxc,907 +module_sqlite-2.2.2.dist-info/RECORD,, +module_sqlite-2.2.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +module_sqlite-2.2.2.dist-info/WHEEL,sha256=iAMR_6Qh95yyjYIwRxyjpiFq4FhDPemrEV-SyWIQB3U,92 +module_sqlite-2.2.2.dist-info/direct_url.json,, +module_sqlite.py,sha256=bUu1TcGpMlFUpoB9EAQVyK4Mb-dhj5n5mNBaqYZrjfM,13 \ No newline at end of file diff --git a/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/REQUESTED b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/REQUESTED new file mode 100644 index 0000000..e69de29 diff --git a/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/WHEEL b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/WHEEL new file mode 100644 index 0000000..3fce7be --- /dev/null +++ b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: poetry-core 1.9.1 +Root-Is-Purelib: true +Tag: py2.py3-none-any diff --git a/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/direct_url.json b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/direct_url.json new file mode 100644 index 0000000..28384ed --- /dev/null +++ b/tests/installed-projects/sqlite/module_sqlite-2.2.2.dist-info/direct_url.json @@ -0,0 +1 @@ +{"archive_info": {"hash": "sha256=20b638c1c7902169760aaebbe6d1c0ed83ac2b36f894fe02eed3fd8580ae1ac5", "hashes": {"sha256": "20b638c1c7902169760aaebbe6d1c0ed83ac2b36f894fe02eed3fd8580ae1ac5"}}, "url": "file:///module_sqlite-2.2.2-py2.py3-none-any.whl"} \ No newline at end of file diff --git a/tests/installed-projects/sqlite/module_sqlite.py b/tests/installed-projects/sqlite/module_sqlite.py new file mode 100644 index 0000000..53ce041 --- /dev/null +++ b/tests/installed-projects/sqlite/module_sqlite.py @@ -0,0 +1 @@ +x = "module" diff --git a/tests/regenerate-test-projects.py b/tests/regenerate-test-projects.py new file mode 100644 index 0000000..3ffed9d --- /dev/null +++ b/tests/regenerate-test-projects.py @@ -0,0 +1,103 @@ +import json +import pathlib +import shutil +import subprocess +import sys + + +def main() -> int: + root_directory = pathlib.Path(__file__).absolute().parent.parent + + filesystem_dist_directory = root_directory.parent / "dist/test-suite-filesystem" + wipe_directory(filesystem_dist_directory) + sqlite_dist_directory = root_directory.parent / "dist/test-suite-sqlite" + wipe_directory(sqlite_dist_directory) + + tests_directory = root_directory / "tests" + source_projects = tests_directory / "source-projects" + for source_project in source_projects.glob("*/"): + print(f"Regenerating {source_project.name}/filesystem...") + build_wheel(source_project / "filesystem", filesystem_dist_directory) + print(f"Regenerating {source_project.name}/sqlite...") + build_wheel(source_project / "sqlite", sqlite_dist_directory) + + installed_projects = tests_directory / "installed-projects" + wipe_directory(installed_projects) + + install_wheels(installed_projects / "filesystem", filesystem_dist_directory) + install_wheels(installed_projects / "sqlite", sqlite_dist_directory) + + sanitize_dist_info_records(installed_projects) + + return 0 + + +def wipe_directory(directory: pathlib.Path) -> None: + """Erase the given directory and then re-create it.""" + + if directory.exists(): + if not directory.is_dir(): + raise OSError(f"{directory} is not a directory.") + shutil.rmtree(directory) + + directory.mkdir(parents=True, exist_ok=True) + + +def build_wheel(directory: pathlib.Path, output: pathlib.Path) -> None: + """Build a wheel using Poetry.""" + + command = f"poetry build --format=wheel --directory={directory} --output={output}" + subprocess.check_call(command.split()) + + +def install_wheels(target: pathlib.Path, dist_directory: pathlib.Path) -> None: + """Install all wheels in the given directory.""" + + # Install the wheels in the target directory. + command = [ + *f"{sys.executable} -m pip install --target={target}".split(), + *[str(path) for path in dist_directory.glob("*.whl")], + ] + subprocess.check_call(command) + + +def sanitize_dist_info_records(directory: pathlib.Path) -> None: + """Sanitize information in `.dist-info/` metadata directories. + + * The `url` keys in "direct_url.json" files are modified. + https://peps.python.org/pep-0610/ + + * The `direct_url.json` lines in "RECORD" files are modified. + https://packaging.python.org/en/latest/specifications/recording-installed-packages/ + + Modifying these files reduces unnecessary information entering version control. + """ + + # Modify `direct_url.json` files. + for json_file in directory.rglob("**/*.dist-info/direct_url.json"): + document = json.loads(json_file.read_text()) + if "url" in document: + url = document["url"] + if not url.startswith("file://"): + raise ValueError(f"Unexpected URL value in {json_file}: {url}") + document["url"] = f"file:///{pathlib.Path(url).name}" + json_file.write_text(json.dumps(document)) + + # Modify `RECORD` files. + for record_file in directory.rglob("**/*.dist-info/RECORD"): + with open(record_file) as file: + lines = file.read().splitlines() + newline = file.newlines + output: list[str] = [] + for line in lines: + if "__pycache__" in line: + continue + if ".dist-info/direct_url.json" in line: + direct_url_json_path, _, _ = line.partition(",") + line = f"{direct_url_json_path},," + output.append(line) + record_file.write_text("\n".join(output), newline=newline) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/source-projects/single-file-module/README.rst b/tests/source-projects/single-file-module/README.rst new file mode 100644 index 0000000..a6e9714 --- /dev/null +++ b/tests/source-projects/single-file-module/README.rst @@ -0,0 +1 @@ +A single-file module project. diff --git a/tests/source-projects/single-file-module/filesystem/README.rst b/tests/source-projects/single-file-module/filesystem/README.rst new file mode 100644 index 0000000..a6e9714 --- /dev/null +++ b/tests/source-projects/single-file-module/filesystem/README.rst @@ -0,0 +1 @@ +A single-file module project. diff --git a/tests/source-projects/single-file-module/filesystem/module_filesystem.py b/tests/source-projects/single-file-module/filesystem/module_filesystem.py new file mode 100644 index 0000000..53ce041 --- /dev/null +++ b/tests/source-projects/single-file-module/filesystem/module_filesystem.py @@ -0,0 +1 @@ +x = "module" diff --git a/tests/source-projects/single-file-module/filesystem/pyproject.toml b/tests/source-projects/single-file-module/filesystem/pyproject.toml new file mode 100644 index 0000000..29b10c3 --- /dev/null +++ b/tests/source-projects/single-file-module/filesystem/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +# poetry +# ------ + +[tool.poetry] +name = "module-filesystem" +version = "1.1.1" +description = "" +authors = ["Kurt McKee "] +license = "MIT" +readme = "README.rst" diff --git a/tests/source-projects/single-file-module/sqlite/README.rst b/tests/source-projects/single-file-module/sqlite/README.rst new file mode 100644 index 0000000..a6e9714 --- /dev/null +++ b/tests/source-projects/single-file-module/sqlite/README.rst @@ -0,0 +1 @@ +A single-file module project. diff --git a/tests/source-projects/single-file-module/sqlite/dist/module_sqlite-2.2.2-py2.py3-none-any.whl b/tests/source-projects/single-file-module/sqlite/dist/module_sqlite-2.2.2-py2.py3-none-any.whl new file mode 100644 index 0000000..9914a04 Binary files /dev/null and b/tests/source-projects/single-file-module/sqlite/dist/module_sqlite-2.2.2-py2.py3-none-any.whl differ diff --git a/tests/source-projects/single-file-module/sqlite/dist/module_sqlite-2.2.2.tar.gz b/tests/source-projects/single-file-module/sqlite/dist/module_sqlite-2.2.2.tar.gz new file mode 100644 index 0000000..467ebdc Binary files /dev/null and b/tests/source-projects/single-file-module/sqlite/dist/module_sqlite-2.2.2.tar.gz differ diff --git a/tests/source-projects/single-file-module/sqlite/module_sqlite.py b/tests/source-projects/single-file-module/sqlite/module_sqlite.py new file mode 100644 index 0000000..53ce041 --- /dev/null +++ b/tests/source-projects/single-file-module/sqlite/module_sqlite.py @@ -0,0 +1 @@ +x = "module" diff --git a/tests/source-projects/single-file-module/sqlite/pyproject.toml b/tests/source-projects/single-file-module/sqlite/pyproject.toml new file mode 100644 index 0000000..cbf9f02 --- /dev/null +++ b/tests/source-projects/single-file-module/sqlite/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +# poetry +# ------ + +[tool.poetry] +name = "module-sqlite" +version = "2.2.2" +description = "" +authors = ["Kurt McKee "] +license = "MIT" +readme = "README.rst" diff --git a/tests/test_importing.py b/tests/test_importing.py new file mode 100644 index 0000000..b613ccc --- /dev/null +++ b/tests/test_importing.py @@ -0,0 +1,36 @@ +import importlib.metadata +import pathlib +import sqlite3 +import sys + +import pytest + +import sqliteimport +import sqliteimport.accessor +import sqliteimport.bundler + +installed_projects = pathlib.Path(__file__).parent / "installed-projects" +sys.path.append(str(installed_projects / "filesystem")) + + +@pytest.fixture(scope="module") +def database(): + with sqlite3.connect(":memory:") as connection: + accessor = sqliteimport.accessor.Accessor(connection) + accessor.initialize_database() + sqliteimport.bundler.bundle(installed_projects / "sqlite", accessor) + sqliteimport.load(connection) + + yield connection + + +def test_module(database): + import module_filesystem + + assert module_filesystem.x == "module" + assert importlib.metadata.version("module_filesystem") == "1.1.1" + + import module_sqlite + + assert module_sqlite.x == "module" + assert importlib.metadata.version("module_sqlite") == "2.2.2" diff --git a/tox.ini b/tox.ini index 00132ef..e149619 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,17 @@ commands = # Run pre-commit immediately, but ignore its exit code - pre-commit run -a +[testenv:regenerate_test_projects] +base_python = py3.13 +description = Regenerate installed projects used by the test suite +recreate = true +skip_install = true +deps = + poetry +commands = + python tests/regenerate-test-projects.py + + [flake8] max-line-length = 88 extend-ignore = E203