Refine rendering logic

catalog_reader/app_utils.py

@@ -1,3 +1,8 @@
+import contextlib
+import os
+import yaml
+
+
 def get_app_details_base(retrieve_complete_item_keys: bool = True) -> dict:
     return {
         'app_readme': None,
@@ -35,3 +40,12 @@ def get_default_questions_context() -> dict:
         'system.general.config': {'timezone': 'America/Los_Angeles'},
         'unused_ports': [i for i in range(1025, 65535)],
     }
+
+
+def get_app_basic_details(app_path: str) -> dict:
+    # This just retrieves app name and app version from app path
+    with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
+        app_config = yaml.safe_load(open(os.path.join(app_path, 'app.yaml')))
+        return {k: app_config[k] for k in ('name', 'train', 'version', 'lib_version')}
+
+    return {}

catalog_templating/render.py

@@ -1,58 +1,78 @@
 import collections
 import importlib
 import os
+import pathlib
 import shutil
 
 from jinja2 import Environment, FileSystemLoader
 
-from .utils import RE_MODULE_VERSION
+from apps_validation.exceptions import ValidationError
+from catalog_reader.app_utils import get_app_basic_details
 
 
-def render_templates(template_path: str, test_values: dict, lib_version: str):
+def render_templates(app_version_path: str, test_values: dict) -> dict:
+    app_details = get_app_basic_details(app_version_path)
+    if not app_details:
+        raise ValidationError('app_version_path', 'Unable to retrieve app metadata from specified app version path')
+
+    template_path = os.path.join(app_version_path, 'templates')
+    if not pathlib.Path(os.path.join(template_path, 'library')).is_dir():
+        return {}
+
+    template_libs = import_library(os.path.join(template_path, 'library'), app_details)
     file_loader = FileSystemLoader(template_path)
     env = Environment(loader=file_loader)
-    # This how filters can be introduced currently not sure how to handle it
-    env.filters['make_capital'] = lambda st: st.upper()
-    templates = env.get_template('docker_compose.yaml')
-    libs = import_library(os.path.join(template_path, 'library'), lib_version)
-    context = {
-        **test_values,
-        'libs': libs
-    }
-    return templates.render(context)
-
-
-def import_library(library_path: str, lib_version: str):
+    rendered_templates = {}
+    for to_render_file in filter(
+        lambda f: f.is_file() and f.name.endswith('.yaml'), pathlib.Path(template_path).iterdir()
+    ):
+        # TODO: Let's look to adding dynamic filter support in the future
+        # env.filters['make_capital'] = lambda st: st.upper()
+        rendered_templates[to_render_file.name] = env.get_template(
+            to_render_file.name
+        ).render(test_values | template_libs)
+
+    return rendered_templates
+
+
+def import_library(library_path: str, app_config) -> dict:
     modules_context = collections.defaultdict(dict)
-    for train_name in os.listdir(library_path):
-        if not RE_MODULE_VERSION.findall(train_name):
-            modules_context[train_name] = collections.defaultdict(dict)
-            for app_name in os.listdir(os.path.join(library_path, train_name)):
-                modules_context[train_name][app_name] = import_app_modules(
-                    os.path.join(library_path, train_name, app_name, lib_version), lib_version)
-        else:
-            base_name = train_name.replace(f'_{RE_MODULE_VERSION.findall(train_name)[0]}', '')
-            modules_context[base_name] = import_app_modules(
-                os.path.join(library_path, train_name), train_name)
+    # 2 dirs which we want to import from
+    global_base_lib = os.path.join(library_path, f'base_v{app_config["lib_version"].replace(".", "_")}')
+    app_lib = os.path.join(
+        library_path, app_config['train'], app_config['name'], f'v{app_config["version"].replace(".", "_")}'
+    )
+    if pathlib.Path(global_base_lib).is_dir():
+        modules_context['base'] = import_app_modules(global_base_lib, os.path.basename(global_base_lib))  # base_v1_0_0
+    if pathlib.Path(app_lib).is_dir():
+        modules_context[app_config['train']] = {
+            app_config['name']: import_app_modules(app_lib, os.path.basename(app_lib))  # v1_0_1
+        }
 
     return modules_context
 
 
-def import_app_modules(modules_path: str, parent_module_name):
+def import_app_modules(modules_path: str, parent_module_name) -> dict:
     def import_module_context(module_name, file_path):
         try:
             spec = importlib.util.spec_from_file_location(module_name, file_path)
             module = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(module)
         except Exception as e:
-            raise Exception(f'Some changes might be introduce in this '
-                            f'package {module_name.split(".")[0]} please update the package versions error: {e}')
+            raise Exception(
+                f'Unable to import module {module_name!r} from {file_path!r}: {e!r}.\n\n'
+                'This could be due to various reasons with primary being:\n1) The module is not a valid python module '
+                'which can be imported i.e might have syntax errors\n2) The module has already been imported and '
+                'then has been changed but the version for the module has not been bumped.'
+            )
         return module
+
     sub_modules_context = {}
     try:
         importlib.sys.path.append(os.path.dirname(modules_path))
-        for sub_modules_file in filter(lambda p: os.path.isfile(os.path.join(modules_path, p)),
-                                       os.listdir(modules_path)):
+        for sub_modules_file in filter(
+            lambda p: os.path.isfile(os.path.join(modules_path, p)) and p.endswith('.py'), os.listdir(modules_path)
+        ):
             sub_modules = sub_modules_file.removesuffix('.py')
             sub_modules_context[sub_modules] = import_module_context(
                 f'{parent_module_name}.{sub_modules}', os.path.join(modules_path, sub_modules_file)
@@ -65,6 +85,7 @@ def import_module_context(module_name, file_path):
 
 
 def remove_pycache(library_path: str):
-    for modules in filter(lambda p: os.path.exists(os.path.join(library_path, p, '__pycache__')),
-                          os.listdir(library_path)):
+    for modules in filter(
+        lambda p: os.path.exists(os.path.join(library_path, p, '__pycache__')), os.listdir(library_path)
+    ):
         shutil.rmtree(os.path.join(library_path, modules, '__pycache__'))

catalog_templating/utils.py

@@ -1,4 +0,0 @@
-import re
-
-
-RE_MODULE_VERSION = re.compile(r'\d+_\d+_\d+|\d+$')