From 78b171bd22e0725b61cc6f9506a98aff6c3f0fa1 Mon Sep 17 00:00:00 2001 From: hnformentin Date: Mon, 23 Jan 2023 15:07:42 +0100 Subject: [PATCH] Add argument --coords for defining rhel and py versions to build in a release matrix The arguments are optional and the default is python 3.8 and rhel 7. --- komodo/matrix.py | 20 ++++----- komodo/release_transpiler.py | 72 +++++++++++++++++++++++++++----- tests/test_release_transpiler.py | 37 +++++++++++++++- 3 files changed, 107 insertions(+), 22 deletions(-) diff --git a/komodo/matrix.py b/komodo/matrix.py index 3ee4b5a92..badb7f7e0 100644 --- a/komodo/matrix.py +++ b/komodo/matrix.py @@ -3,15 +3,13 @@ handling an arbitrary large and funky matrix, without having to guess and/or repeat itself.""" import itertools +import re -RH_VERSIONS = ["7"] -PY_VERSIONS = ["3.8"] - -def get_matrix(): +def get_matrix(rhel_versions, py_versions): """Return tuples of RHEL version and Python version, representing the current release matrix.""" - for product in itertools.product(RH_VERSIONS, PY_VERSIONS): + for product in itertools.product(rhel_versions, py_versions): rh_ver, py_ver = product yield (f"rhel{rh_ver}", f"py{py_ver.replace('.', '')}") @@ -24,10 +22,12 @@ def format_release(base, rhel_ver, py_ver): def get_matrix_base(release): """Return the base (e.g. matrix part of a concrete release). Should be the - inverse of format_release for actual, concrete matrix releases.""" - for rhel_ver, py_ver in get_matrix(): - suffix = format_release("", rhel_ver, py_ver) - if release.endswith(suffix): - return release.split(suffix, 1)[0] + inverse of format_release for actual, concrete matrix releases. + Hard-coded the suffix pattern '-py..-rhel.' or '-py...-rhel.'""" + + if re.search("-py..-rhel.", release): + return re.split("-py..-rhel.", release)[0] + elif re.search("-py...-rhel.", release): # for python>3.9 + return re.split("-py...-rhel.", release)[0] # no matrix suffix at all return release diff --git a/komodo/release_transpiler.py b/komodo/release_transpiler.py index 786aa82df..1e47da594 100755 --- a/komodo/release_transpiler.py +++ b/komodo/release_transpiler.py @@ -4,13 +4,16 @@ import itertools import os +import yaml + from komodo.matrix import format_release, get_matrix from komodo.prettier import load_yaml, write_to_file -def build_matrix_file(release_base, release_folder, builtins): +def build_matrix_file(release_base, release_folder, builtins, coords): + files = {} - py_keys = [py_ver for _, py_ver in get_matrix()] + py_keys = [f"py{py_version.replace('.', '')}" for py_version in coords["py"]] for key in py_keys: files[key] = load_yaml(f"{release_folder}/{release_base}-{key}.yml") @@ -32,9 +35,37 @@ def build_matrix_file(release_base, release_folder, builtins): write_to_file(compiled, f"{release_base}.yml", False) -def _build(packages, py_ver, rhel_ver): +def _check_consistency_in_versions(versions, pkg_name, rhel_ver, py_ver): + """Check the coordinates `rhel_ver` and `py_ver` inputed as arguments to + build a release against the release matrix file.""" + first_level_versions = [] + for version in versions: + first_level_versions.append(version) + if "rhel" in first_level_versions[0]: + # Both rhel and python versions can have different versions + if rhel_ver not in first_level_versions: + raise KeyError( + f"Rhel version {rhel_ver} is not found for the package {pkg_name}." + ) + second_level_versions = [] + for version_py in versions[rhel_ver]: + second_level_versions.append(version_py) + if py_ver not in second_level_versions: + raise KeyError( + f"Python version {py_ver} is not found for the package {pkg_name} in rhel version {rhel_ver}." + ) + elif "py" in first_level_versions[0]: + # Only python has different versions + if py_ver not in first_level_versions: + raise KeyError( + f"Python version {py_ver} is not found for the package {pkg_name}." + ) + + +def _build(packages, rhel_ver, py_ver): release_dict = {} - for p, versions in packages.items(): + for pkg_name, versions in packages.items(): + _check_consistency_in_versions(versions, pkg_name, rhel_ver, py_ver) if rhel_ver in versions: version = versions[rhel_ver][py_ver] elif py_ver in versions: @@ -43,31 +74,38 @@ def _build(packages, py_ver, rhel_ver): version = versions if version: - release_dict[p] = version + release_dict[pkg_name] = version return release_dict -def transpile_releases(matrix_file, output_folder): +def transpile_releases(matrix_file, output_folder, coords): + rhel_versions = coords["rhel"] + python_versions = coords["py"] + release_base = os.path.splitext(os.path.basename(matrix_file))[0] release_folder = os.path.dirname(matrix_file) release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") - for rhel_ver, py_ver in get_matrix(): - release_dict = _build(release_matrix, py_ver, rhel_ver) + for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions): + release_dict = _build(release_matrix, rhel_ver, py_ver) filename = f"{format_release(release_base, rhel_ver, py_ver)}.yml" write_to_file(release_dict, os.path.join(output_folder, filename)) def combine(args): build_matrix_file( - args.release_base, args.release_folder, load_yaml(args.override_mapping) + args.release_base, + args.release_folder, + load_yaml(args.override_mapping), + args.coords, ) def transpile(args): - transpile_releases(args.matrix_file, args.output_folder) + transpile_releases(args.matrix_file, args.output_folder, args.coords) def main(): + default_coords = "{rhel: ['7'], py: ['3.8']}" parser = argparse.ArgumentParser( description="Build release files.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -112,6 +150,13 @@ def main(): required=True, help="File containing explicit matrix packages (default: None)", ) + matrix_parser.add_argument( + "--coords", + help="Coordinates of the matrix to be combined.", + type=yaml.safe_load, + required=False, + default=default_coords, + ) transpile_parser = subparsers.add_parser( "transpile", description="Transpile a matrix file into separate release files." @@ -127,6 +172,13 @@ def main(): required=True, help="Folder to output new release files", ) + transpile_parser.add_argument( + "--coords", + help="Coordinates of the matrix to be transpiled.", + type=yaml.safe_load, + required=False, + default=default_coords, + ) args = parser.parse_args() args.func(args) diff --git a/tests/test_release_transpiler.py b/tests/test_release_transpiler.py index 839ef4978..8a100fa9a 100644 --- a/tests/test_release_transpiler.py +++ b/tests/test_release_transpiler.py @@ -1,5 +1,6 @@ import os +import pytest import yaml from komodo.release_transpiler import build_matrix_file, transpile_releases @@ -21,7 +22,9 @@ def test_build_release_matrix(tmpdir): release_base = "2020.01.a1" release_folder = os.path.join(_get_test_root(), "data/test_releases/") with tmpdir.as_cwd(): - build_matrix_file(release_base, release_folder, builtins) + build_matrix_file( + release_base, release_folder, builtins, {"rhel": ["7"], "py": ["3.8"]} + ) new_release_file = "{}.yml".format(release_base) assert os.path.isfile(new_release_file) with open(new_release_file) as f: @@ -36,9 +39,39 @@ def test_transpile(tmpdir): release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") release_base = os.path.basename(release_file).strip(".yml") with tmpdir.as_cwd(): - transpile_releases(release_file, os.getcwd()) + transpile_releases(release_file, os.getcwd(), {"py": ["3.8"], "rhel": ["7"]}) for rhel_ver in ("rhel7",): for py_ver in ("py38",): assert os.path.isfile( "{}-{}-{}.yml".format(release_base, py_ver, rhel_ver) ) + + +@pytest.mark.parametrize("coords", [({"py": ["3.8"], "rhel": ["7"]})]) +def test_transpile_add_argument(tmpdir, coords): + release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") + release_base = os.path.basename(release_file).strip(".yml") + with tmpdir.as_cwd(): + transpile_releases(release_file, os.getcwd(), coords) + rhel_ver_clean = f"rhel{coords['rhel'][0]}" + py_ver_clean = f"py{coords['py'][0].replace('.', '')}" + + assert os.path.isfile( + "{}-{}-{}.yml".format(release_base, py_ver_clean, rhel_ver_clean) + ) + + +@pytest.mark.parametrize( + "coords", + [ + ({"py": ["3.8"], "rhel": ["7"]}), + ({"py": ["3.7"], "rhel": ["7"]}), + ({"py": ["3.6"], "rhel": ["5"]}), + ], +) +def test_validate_versions_in_args_with_release_file(coords): + release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") + try: + transpile_releases(release_file, os.getcwd(), coords) + except (KeyError) as exception_info: + assert "not found for the package" in str(exception_info)