diff --git a/komodo/release_transpiler.py b/komodo/release_transpiler.py index 786aa82df..650ae4b8b 100755 --- a/komodo/release_transpiler.py +++ b/komodo/release_transpiler.py @@ -4,6 +4,8 @@ import itertools import os +from typing import Dict + from komodo.matrix import format_release, get_matrix from komodo.prettier import load_yaml, write_to_file @@ -57,6 +59,27 @@ def transpile_releases(matrix_file, output_folder): write_to_file(release_dict, os.path.join(output_folder, filename)) +def transpile_releases_for_pip( + matrix_file: str, output_folder: str, repository_file: str +): + release_base = os.path.splitext(os.path.basename(matrix_file))[0] + release_folder = os.path.dirname(matrix_file) + release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") + repository = load_yaml(repository_file) + for rhel_ver, py_ver in get_matrix(): + release_dict = _build(release_matrix, py_ver, rhel_ver) + pip_packages = [ + f"{pkg}=={version}" + for pkg, version in release_dict.items() + if repository[pkg][version].get("make") == "pip" + ] + filename = f"{format_release(release_base, rhel_ver, py_ver)}.req" + with open( + os.path.join(output_folder, filename), mode="w", encoding="utf-8" + ) as filehandler: + filehandler.write("\n".join(pip_packages)) + + def combine(args): build_matrix_file( args.release_base, args.release_folder, load_yaml(args.override_mapping) @@ -67,6 +90,10 @@ def transpile(args): transpile_releases(args.matrix_file, args.output_folder) +def transpile_for_pip(args: Dict): + transpile_releases_for_pip(args.matrix_file, args.output_folder, args.repo) + + def main(): parser = argparse.ArgumentParser( description="Build release files.", @@ -127,6 +154,26 @@ def main(): required=True, help="Folder to output new release files", ) + transpile_for_pip_parser = subparsers.add_parser( + "transpile-for-pip", + description="transpile a matrix file into separate pip requirement files.", + ) + transpile_for_pip_parser.set_defaults(func=transpile_for_pip) + transpile_for_pip_parser.add_argument( + "--matrix-file", + required=True, + help="Yaml file describing the release matrix", + ) + transpile_for_pip_parser.add_argument( + "--repo", + required=True, + help="A Komodo repository file, in YAML format.", + ) + transpile_for_pip_parser.add_argument( + "--output-folder", + required=True, + help="Folder to output new release files", + ) args = parser.parse_args() args.func(args) diff --git a/komodo/snyk_reporting.py b/komodo/snyk_reporting.py index ea66dbe55..f74cbef67 100644 --- a/komodo/snyk_reporting.py +++ b/komodo/snyk_reporting.py @@ -112,8 +112,26 @@ def find_vulnerabilities( for release_name, packages in releases.items(): pip_packages = filter_pip_packages(packages=packages, repository=repository) snyk_search_string = create_snyk_search_string(pip_packages) + # 149: fail + # 180: fail + # 180:-25 - fail + # 200:-25 - fail + # 260:-25 - worked + # 200:261 - failed + # 230:261 - worked + # 200:230 - worked + # 220:261 - worked + # 200:241 - worked + # 200:251 - failed + # 210:251 - worked + # 205:251 - worked + # snyk_search_string = "\n".join(snyk_search_string.splitlines()[202:251]) + # snyk_search_string = "soupsieve==2.3.2.post1" + print(snyk_search_string) snyk_result = org.test_pipfile(snyk_search_string) + print(snyk_result) vulnerability_issues = get_unique_issues(snyk_result.issues.vulnerabilities) + print(vulnerability_issues) result[release_name] = vulnerability_issues return result diff --git a/tests/data/test_repository.yml b/tests/data/test_repository.yml index 5f41df32c..2fd7c0585 100644 --- a/tests/data/test_repository.yml +++ b/tests/data/test_repository.yml @@ -1,40 +1,61 @@ lib1: + 1.2.3+builtin: + source: pypi + make: sh + 0.1.2+builtin: + source: pypi + make: sh 1.2.3: source: pypi + make: pip 0.1.2: source: pypi + make: pip 0.0.2: source: pypi - + make: pip + lib2: 2.3.4: + make: pip depends: - lib1 1.2.3: + make: pip depends: - lib1 0.0.2: + make: pip depends: - lib1 - + lib3: + 3.4.6: + make: rsync + depends: + - lib1 3.4.5: + make: rsync depends: - lib1 2.3.4: + make: rsync depends: - lib1 master: + make: rsync depends: - lib1 - + lib4: # comment to be preserved 3.4.5: + make: pip depends: - lib3 - lib2 lib5: # comment should be gone 1.2.3: + make: pip depends: - - lib4 \ No newline at end of file + - lib4 diff --git a/tests/test_release_cleanup.py b/tests/test_release_cleanup.py index a829ba624..7ffc8555c 100644 --- a/tests/test_release_cleanup.py +++ b/tests/test_release_cleanup.py @@ -15,27 +15,34 @@ expected_result = """lib1: 1.2.3: source: pypi + make: pip 0.1.2: source: pypi + make: pip lib2: 2.3.4: + make: pip depends: - lib1 1.2.3: + make: pip depends: - lib1 lib3: 3.4.5: + make: rsync depends: - lib1 2.3.4: + make: rsync depends: - lib1 lib4: # comment to be preserved 3.4.5: + make: pip depends: - lib3 - lib2 @@ -98,10 +105,16 @@ def test_write_to_file(tmpdir): os.path.join(_get_test_root(), "data/test_releases/2020.01.a1-py36.yml"), ] used_versions = load_all_releases(files) + print() + print() + print(used_versions) repository = _load_yaml( os.path.join(_get_test_root(), "data/test_repository.yml") ) unused_versions = find_unused_versions(used_versions, repository) + print(unused_versions) + print() + print() remove_unused_versions(repository, unused_versions) write_to_file(repository, "output_repo.yml") with open("output_repo.yml") as output: diff --git a/tests/test_release_transpiler.py b/tests/test_release_transpiler.py index 839ef4978..e14d13c72 100644 --- a/tests/test_release_transpiler.py +++ b/tests/test_release_transpiler.py @@ -2,7 +2,11 @@ import yaml -from komodo.release_transpiler import build_matrix_file, transpile_releases +from komodo.release_transpiler import ( + build_matrix_file, + transpile_releases, + transpile_releases_for_pip, +) from tests import _get_test_root builtins = { @@ -42,3 +46,21 @@ def test_transpile(tmpdir): assert os.path.isfile( "{}-{}-{}.yml".format(release_base, py_ver, rhel_ver) ) + + +def test_transpile_for_pip(tmpdir): + release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") + repo_file = os.path.join(_get_test_root(), "data", "test_repository.yml") + release_base = os.path.basename(release_file).strip(".yml") + not_pip_pkg = "lib3" + expected_line = "lib2==2.3.4" + with tmpdir.as_cwd(): + transpile_releases_for_pip(release_file, os.getcwd(), repo_file) + for rhel_ver in ("rhel7",): + for py_ver in ("py38",): + filename = f"{release_base}-{py_ver}-{rhel_ver}.req" + assert os.path.isfile(filename) + with open(filename, mode="r", encoding="utf-8") as fil: + file_lines = fil.read().splitlines() + assert all([not line.startswith(not_pip_pkg) for line in file_lines]) + assert expected_line in file_lines