Skip to content

Commit

Permalink
Add transpiler to produce pip requirements file
Browse files Browse the repository at this point in the history
  • Loading branch information
valentin-krasontovitsch committed Aug 29, 2023
1 parent d953b02 commit 34bba01
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 5 deletions.
60 changes: 59 additions & 1 deletion komodo/release_transpiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import argparse
import itertools
import os
from typing import List, Optional, Sequence
from typing import Dict, List, Optional, Sequence

import yaml
from packaging import version as version_parser
Expand Down Expand Up @@ -153,6 +153,31 @@ def transpile_releases(matrix_file: str, output_folder: str, matrix: dict) -> No
write_to_file(release_dict, os.path.join(output_folder, filename))


def transpile_releases_for_pip(
matrix_file: str, output_folder: str, repository_file: str, matrix: dict
) -> None:
rhel_versions = matrix["rhel"]
python_versions = matrix["py"]
release_base = os.path.splitext(os.path.basename(matrix_file))[0]
release_folder = os.path.dirname(matrix_file)
release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml")
repository = load_yaml(repository_file)
for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions):
release_dict = _pick_package_versions_for_release(
release_matrix, py_ver, rhel_ver
)
pip_packages = [
f"{pkg}=={version}"
for pkg, version in release_dict.items()
if repository[pkg][version].get("make") == "pip"
]
filename = f"{format_release(release_base, rhel_ver, py_ver)}.req"
with open(
os.path.join(output_folder, filename), mode="w", encoding="utf-8"
) as filehandler:
filehandler.write("\n".join(pip_packages))


def combine(args):
build_matrix_file(
args.release_base,
Expand All @@ -166,6 +191,12 @@ def transpile(args):
transpile_releases(args.matrix_file, args.output_folder, args.matrix_coordinates)


def transpile_for_pip(args: Dict):
transpile_releases_for_pip(
args.matrix_file, args.output_folder, args.repo, args.matrix_coordinates
)


def main():
parser = argparse.ArgumentParser(
description="Build release files.",
Expand Down Expand Up @@ -259,6 +290,33 @@ def comma_delimited_python_versions(python_versions: str) -> List[str]:
required=False,
default="{rhel: ['7'], py: ['3.8']}",
)
transpile_for_pip_parser = subparsers.add_parser(
"transpile-for-pip",
description="transpile a matrix file into separate pip requirement files.",
)
transpile_for_pip_parser.set_defaults(func=transpile_for_pip)
transpile_for_pip_parser.add_argument(
"--matrix-file",
required=True,
help="Yaml file describing the release matrix",
)
transpile_for_pip_parser.add_argument(
"--repo",
required=True,
help="A Komodo repository file, in YAML format.",
)
transpile_for_pip_parser.add_argument(
"--output-folder",
required=True,
help="Folder to output new release files",
)
transpile_for_pip_parser.add_argument(
"--matrix-coordinates",
help="Matrix to be transpiled, expected yaml format string.",
type=yaml.safe_load,
required=False,
default="{rhel: ['7'], py: ['3.8']}",
)
args = parser.parse_args()
args.func(args)

Expand Down
29 changes: 25 additions & 4 deletions tests/data/test_repository.yml
Original file line number Diff line number Diff line change
@@ -1,40 +1,61 @@
lib1:
1.2.3+builtin:
source: pypi
make: sh
0.1.2+builtin:
source: pypi
make: sh
1.2.3:
source: pypi
make: pip
0.1.2:
source: pypi
make: pip
0.0.2:
source: pypi

make: pip

lib2:
2.3.4:
make: pip
depends:
- lib1
1.2.3:
make: pip
depends:
- lib1
0.0.2:
make: pip
depends:
- lib1

lib3:
3.4.6:
make: rsync
depends:
- lib1
3.4.5:
make: rsync
depends:
- lib1
2.3.4:
make: rsync
depends:
- lib1
master:
make: rsync
depends:
- lib1

lib4: # comment to be preserved
3.4.5:
make: pip
depends:
- lib3
- lib2

lib5: # comment should be gone
1.2.3:
make: pip
depends:
- lib4
- lib4
7 changes: 7 additions & 0 deletions tests/test_release_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,27 +16,34 @@
expected_result = """lib1:
1.2.3:
source: pypi
make: pip
0.1.2:
source: pypi
make: pip
lib2:
2.3.4:
make: pip
depends:
- lib1
1.2.3:
make: pip
depends:
- lib1
lib3:
3.4.5:
make: rsync
depends:
- lib1
2.3.4:
make: rsync
depends:
- lib1
lib4: # comment to be preserved
3.4.5:
make: pip
depends:
- lib3
- lib2
Expand Down
19 changes: 19 additions & 0 deletions tests/test_release_transpiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
build_matrix_file,
get_py_coords,
transpile_releases,
transpile_releases_for_pip,
)
from tests import _get_test_root

Expand Down Expand Up @@ -102,3 +103,21 @@ def test_get_py_coords():
release_base = "2020.01.a1"
py_coords = get_py_coords(release_base, release_folder)
assert py_coords == ["py27", "py36", "py38"]


def test_transpile_for_pip(tmpdir):
release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml")
repo_file = os.path.join(_get_test_root(), "data", "test_repository.yml")
release_base = os.path.basename(release_file).strip(".yml")
not_pip_pkg = "lib3"
expected_line = "lib2==2.3.4"
with tmpdir.as_cwd():
transpile_releases_for_pip(release_file, os.getcwd(), repo_file)
for rhel_ver in ("rhel7",):
for py_ver in ("py38",):
filename = f"{release_base}-{py_ver}-{rhel_ver}.req"
assert os.path.isfile(filename)
with open(filename, mode="r", encoding="utf-8") as fil:
file_lines = fil.read().splitlines()
assert all([not line.startswith(not_pip_pkg) for line in file_lines])
assert expected_line in file_lines

0 comments on commit 34bba01

Please sign in to comment.