diff --git a/komodo/release_transpiler.py b/komodo/release_transpiler.py index 4d88df111..c979c5923 100755 --- a/komodo/release_transpiler.py +++ b/komodo/release_transpiler.py @@ -3,7 +3,7 @@ import argparse import itertools import os -from typing import List, Optional, Sequence +from typing import Dict, List, Optional, Sequence import yaml from packaging import version as version_parser @@ -153,6 +153,31 @@ def transpile_releases(matrix_file: str, output_folder: str, matrix: dict) -> No write_to_file(release_dict, os.path.join(output_folder, filename)) +def transpile_releases_for_pip( + matrix_file: str, output_folder: str, repository_file: str, matrix: dict +) -> None: + rhel_versions = matrix["rhel"] + python_versions = matrix["py"] + release_base = os.path.splitext(os.path.basename(matrix_file))[0] + release_folder = os.path.dirname(matrix_file) + release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") + repository = load_yaml(repository_file) + for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions): + release_dict = _pick_package_versions_for_release( + release_matrix, py_ver, rhel_ver + ) + pip_packages = [ + f"{pkg}=={version}" + for pkg, version in release_dict.items() + if repository[pkg][version].get("make") == "pip" + ] + filename = f"{format_release(release_base, rhel_ver, py_ver)}.req" + with open( + os.path.join(output_folder, filename), mode="w", encoding="utf-8" + ) as filehandler: + filehandler.write("\n".join(pip_packages)) + + def combine(args): build_matrix_file( args.release_base, @@ -166,6 +191,12 @@ def transpile(args): transpile_releases(args.matrix_file, args.output_folder, args.matrix_coordinates) +def transpile_for_pip(args: Dict): + transpile_releases_for_pip( + args.matrix_file, args.output_folder, args.repo, args.matrix_coordinates + ) + + def main(): parser = argparse.ArgumentParser( description="Build release files.", @@ -259,6 +290,33 @@ def comma_delimited_python_versions(python_versions: str) -> List[str]: required=False, default="{rhel: ['7'], py: ['3.8']}", ) + transpile_for_pip_parser = subparsers.add_parser( + "transpile-for-pip", + description="transpile a matrix file into separate pip requirement files.", + ) + transpile_for_pip_parser.set_defaults(func=transpile_for_pip) + transpile_for_pip_parser.add_argument( + "--matrix-file", + required=True, + help="Yaml file describing the release matrix", + ) + transpile_for_pip_parser.add_argument( + "--repo", + required=True, + help="A Komodo repository file, in YAML format.", + ) + transpile_for_pip_parser.add_argument( + "--output-folder", + required=True, + help="Folder to output new release files", + ) + transpile_for_pip_parser.add_argument( + "--matrix-coordinates", + help="Matrix to be transpiled, expected yaml format string.", + type=yaml.safe_load, + required=False, + default="{rhel: ['7'], py: ['3.8']}", + ) args = parser.parse_args() args.func(args) diff --git a/tests/data/test_repository.yml b/tests/data/test_repository.yml index 5f41df32c..2fd7c0585 100644 --- a/tests/data/test_repository.yml +++ b/tests/data/test_repository.yml @@ -1,40 +1,61 @@ lib1: + 1.2.3+builtin: + source: pypi + make: sh + 0.1.2+builtin: + source: pypi + make: sh 1.2.3: source: pypi + make: pip 0.1.2: source: pypi + make: pip 0.0.2: source: pypi - + make: pip + lib2: 2.3.4: + make: pip depends: - lib1 1.2.3: + make: pip depends: - lib1 0.0.2: + make: pip depends: - lib1 - + lib3: + 3.4.6: + make: rsync + depends: + - lib1 3.4.5: + make: rsync depends: - lib1 2.3.4: + make: rsync depends: - lib1 master: + make: rsync depends: - lib1 - + lib4: # comment to be preserved 3.4.5: + make: pip depends: - lib3 - lib2 lib5: # comment should be gone 1.2.3: + make: pip depends: - - lib4 \ No newline at end of file + - lib4 diff --git a/tests/test_release_cleanup.py b/tests/test_release_cleanup.py index 1a1cb97ed..571c83faa 100644 --- a/tests/test_release_cleanup.py +++ b/tests/test_release_cleanup.py @@ -16,27 +16,34 @@ expected_result = """lib1: 1.2.3: source: pypi + make: pip 0.1.2: source: pypi + make: pip lib2: 2.3.4: + make: pip depends: - lib1 1.2.3: + make: pip depends: - lib1 lib3: 3.4.5: + make: rsync depends: - lib1 2.3.4: + make: rsync depends: - lib1 lib4: # comment to be preserved 3.4.5: + make: pip depends: - lib3 - lib2 diff --git a/tests/test_release_transpiler.py b/tests/test_release_transpiler.py index 60fb10927..0723c2fe8 100644 --- a/tests/test_release_transpiler.py +++ b/tests/test_release_transpiler.py @@ -7,6 +7,7 @@ build_matrix_file, get_py_coords, transpile_releases, + transpile_releases_for_pip, ) from tests import _get_test_root @@ -102,3 +103,21 @@ def test_get_py_coords(): release_base = "2020.01.a1" py_coords = get_py_coords(release_base, release_folder) assert py_coords == ["py27", "py36", "py38"] + + +def test_transpile_for_pip(tmpdir): + release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") + repo_file = os.path.join(_get_test_root(), "data", "test_repository.yml") + release_base = os.path.basename(release_file).strip(".yml") + not_pip_pkg = "lib3" + expected_line = "lib2==2.3.4" + with tmpdir.as_cwd(): + transpile_releases_for_pip(release_file, os.getcwd(), repo_file) + for rhel_ver in ("rhel7",): + for py_ver in ("py38",): + filename = f"{release_base}-{py_ver}-{rhel_ver}.req" + assert os.path.isfile(filename) + with open(filename, mode="r", encoding="utf-8") as fil: + file_lines = fil.read().splitlines() + assert all([not line.startswith(not_pip_pkg) for line in file_lines]) + assert expected_line in file_lines