Add transpiler to produce pip requirements file

komodo/release_transpiler.py

@@ -4,6 +4,8 @@
 import itertools
 import os
 
+from typing import Dict
+
 from komodo.matrix import format_release, get_matrix
 from komodo.prettier import load_yaml, write_to_file
 
@@ -57,6 +59,27 @@ def transpile_releases(matrix_file, output_folder):
         write_to_file(release_dict, os.path.join(output_folder, filename))
 
 
+def transpile_releases_for_pip(
+    matrix_file: str, output_folder: str, repository_file: str
+):
+    release_base = os.path.splitext(os.path.basename(matrix_file))[0]
+    release_folder = os.path.dirname(matrix_file)
+    release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml")
+    repository = load_yaml(repository_file)
+    for rhel_ver, py_ver in get_matrix():
+        release_dict = _build(release_matrix, py_ver, rhel_ver)
+        pip_packages = [
+            f"{pkg}=={version}"
+            for pkg, version in release_dict.items()
+            if repository[pkg][version].get("make") == "pip"
+        ]
+        filename = f"{format_release(release_base, rhel_ver, py_ver)}.req"
+        with open(
+            os.path.join(output_folder, filename), mode="w", encoding="utf-8"
+        ) as filehandler:
+            filehandler.write("\n".join(pip_packages))
+
+
 def combine(args):
     build_matrix_file(
         args.release_base, args.release_folder, load_yaml(args.override_mapping)
@@ -67,6 +90,10 @@ def transpile(args):
     transpile_releases(args.matrix_file, args.output_folder)
 
 
+def transpile_for_pip(args: Dict):
+    transpile_releases_for_pip(args.matrix_file, args.output_folder, args.repo)
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="Build release files.",
@@ -127,6 +154,26 @@ def main():
         required=True,
         help="Folder to output new release files",
     )
+    transpile_for_pip_parser = subparsers.add_parser(
+        "transpile-for-pip",
+        description="transpile a matrix file into separate pip requirement files.",
+    )
+    transpile_for_pip_parser.set_defaults(func=transpile_for_pip)
+    transpile_for_pip_parser.add_argument(
+        "--matrix-file",
+        required=True,
+        help="Yaml file describing the release matrix",
+    )
+    transpile_for_pip_parser.add_argument(
+        "--repo",
+        required=True,
+        help="A Komodo repository file, in YAML format.",
+    )
+    transpile_for_pip_parser.add_argument(
+        "--output-folder",
+        required=True,
+        help="Folder to output new release files",
+    )
     args = parser.parse_args()
     args.func(args)
 

komodo/snyk_reporting.py

@@ -112,8 +112,26 @@ def find_vulnerabilities(
     for release_name, packages in releases.items():
         pip_packages = filter_pip_packages(packages=packages, repository=repository)
         snyk_search_string = create_snyk_search_string(pip_packages)
+        # 149: fail
+        # 180: fail
+        # 180:-25 - fail
+        # 200:-25 - fail
+        # 260:-25 - worked
+        # 200:261 - failed
+        # 230:261 - worked
+        # 200:230 - worked
+        # 220:261 - worked
+        # 200:241 - worked
+        # 200:251 - failed
+        # 210:251 - worked
+        # 205:251 - worked
+        # snyk_search_string = "\n".join(snyk_search_string.splitlines()[202:251])
+        # snyk_search_string = "soupsieve==2.3.2.post1"
+        print(snyk_search_string)
         snyk_result = org.test_pipfile(snyk_search_string)
+        print(snyk_result)
         vulnerability_issues = get_unique_issues(snyk_result.issues.vulnerabilities)
+        print(vulnerability_issues)
         result[release_name] = vulnerability_issues
 
     return result

tests/data/test_repository.yml

@@ -1,40 +1,61 @@
 lib1:
+  1.2.3+builtin:
+    source: pypi
+    make: sh
+  0.1.2+builtin:
+    source: pypi
+    make: sh
   1.2.3:
     source: pypi
+    make: pip
   0.1.2:
     source: pypi
+    make: pip
   0.0.2:
     source: pypi
-
+    make: pip
+
 lib2:
   2.3.4:
+    make: pip
     depends:
     - lib1
   1.2.3:
+    make: pip
     depends:
     - lib1
   0.0.2:
+    make: pip
     depends:
     - lib1
-    
+
 lib3:
+  3.4.6:
+    make: rsync
+    depends:
+    - lib1
   3.4.5:
+    make: rsync
     depends:
     - lib1
   2.3.4:
+    make: rsync
     depends:
     - lib1
   master:
+    make: rsync
     depends:
     - lib1
-    
+
 lib4: # comment to be preserved
   3.4.5:
+    make: pip
     depends:
     - lib3
     - lib2
 
 lib5: # comment should be gone
   1.2.3:
+    make: pip
     depends:
-    - lib4
+    - lib4

tests/test_release_cleanup.py

@@ -15,27 +15,34 @@
 expected_result = """lib1:
   1.2.3:
     source: pypi
+    make: pip
   0.1.2:
     source: pypi
+    make: pip
 
 lib2:
   2.3.4:
+    make: pip
     depends:
       - lib1
   1.2.3:
+    make: pip
     depends:
       - lib1
 
 lib3:
   3.4.5:
+    make: rsync
     depends:
       - lib1
   2.3.4:
+    make: rsync
     depends:
       - lib1
 
 lib4: # comment to be preserved
   3.4.5:
+    make: pip
     depends:
       - lib3
       - lib2
@@ -98,10 +105,16 @@ def test_write_to_file(tmpdir):
             os.path.join(_get_test_root(), "data/test_releases/2020.01.a1-py36.yml"),
         ]
         used_versions = load_all_releases(files)
+        print()
+        print()
+        print(used_versions)
         repository = _load_yaml(
             os.path.join(_get_test_root(), "data/test_repository.yml")
         )
         unused_versions = find_unused_versions(used_versions, repository)
+        print(unused_versions)
+        print()
+        print()
         remove_unused_versions(repository, unused_versions)
         write_to_file(repository, "output_repo.yml")
         with open("output_repo.yml") as output:

tests/test_release_transpiler.py

@@ -2,7 +2,11 @@
 
 import yaml
 
-from komodo.release_transpiler import build_matrix_file, transpile_releases
+from komodo.release_transpiler import (
+    build_matrix_file,
+    transpile_releases,
+    transpile_releases_for_pip,
+)
 from tests import _get_test_root
 
 builtins = {
@@ -42,3 +46,21 @@ def test_transpile(tmpdir):
                 assert os.path.isfile(
                     "{}-{}-{}.yml".format(release_base, py_ver, rhel_ver)
                 )
+
+
+def test_transpile_for_pip(tmpdir):
+    release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml")
+    repo_file = os.path.join(_get_test_root(), "data", "test_repository.yml")
+    release_base = os.path.basename(release_file).strip(".yml")
+    not_pip_pkg = "lib3"
+    expected_line = "lib2==2.3.4"
+    with tmpdir.as_cwd():
+        transpile_releases_for_pip(release_file, os.getcwd(), repo_file)
+        for rhel_ver in ("rhel7",):
+            for py_ver in ("py38",):
+                filename = f"{release_base}-{py_ver}-{rhel_ver}.req"
+                assert os.path.isfile(filename)
+                with open(filename, mode="r", encoding="utf-8") as fil:
+                    file_lines = fil.read().splitlines()
+                assert all([not line.startswith(not_pip_pkg) for line in file_lines])
+                assert expected_line in file_lines