GH actions simple

.github/workflows/deploy.yml

@@ -12,10 +12,18 @@ jobs:
         name: Loadmaster
         steps:
             - uses: actions/checkout@v4
-
-            - name: Package step
-              uses: ./
-              id: build-package 
+
+            - uses: actions/setup-python@v2
+              with:
+                python-version: 3.9
+            - uses: snok/install-poetry@v1
+              with:
+                version: 1.2.0
+            - name: Install dependencies
+              run: poetry install
+            - name: Run my script
+            run: |
+                poetry run python build.py
 
 
 

Dockerfile

@@ -17,7 +17,7 @@ RUN poetry export -f requirements.txt --output requirements.txt --without-hashes
 RUN pip3 install --target=/app -r requirements.txt --no-deps
 
 # Keep the same folder structure for imports
-COPY cognite/loadmaster /app/cognite/loadmaster
+COPY cognite/loadmaster/ /app/cognite/loadmaster/
 
 
 # A distroless container image with Python and some basics like SSL certificates

cognite/loadmaster/build.py → build.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 import argparse
+import os
 import logging
 from dotenv import load_dotenv
 from scripts.templates import build_config
+import pathlib
 
 log = logging.getLogger(__name__)
 
@@ -14,8 +16,9 @@
 
 def run(build_dir: str) -> None:
     print(
-        f"Building config files from templates in ./modules and ./common into {build_dir}..."
+        f"Building config files from templates in into {build_dir}..."
     )
+
     build_config(build_dir)
 
 

scripts/templates.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+import os
+import pathlib
+import shutil
+import yaml
+from typing import Any
+from pathlib import Path
+import glob
+
+# Directory paths for YAML and JSON files
+YAML_DIRS = ["./"]
+TMPL_DIRS = ["./common", "./modules"]
+# Add any other files below that should be included in a build
+EXCL_FILES = ["README.md"]
+
+
+def read_yaml_files(yaml_dirs):
+    """Read all YAML files in the given directories and return a dictionary
+
+    This function will not traverse into sub-directories.
+
+    yaml_dirs: list of directories to read YAML files from
+    """
+
+    data = {}
+    for directory in yaml_dirs:
+        print(f"Reading YAML files from {directory}")
+
+        if not Path(directory).exists():
+            raise FileNotFoundError(f"Directory {directory} does not exist")
+
+        for yaml_file in Path(directory).glob("config.yaml"):
+            print(f"Reading {yaml_file}")
+            try:
+                config_data = yaml.safe_load(yaml_file.read_text())
+            except yaml.YAMLError:
+                print(f"Error reading {yaml_file}")
+                continue
+            data.update(config_data)
+    # Replace env variables of ${ENV_VAR} with actual value from environment
+    for k, v in os.environ.items():
+        for k2, v2 in data.items():
+            if f"${{{k}}}" in v2:
+                if isinstance(data[k2], list):
+                    for i in range(len(data[k2])):
+                        data[k2][i] = data[k2][i].replace(f"${{{k}}}", v)
+                else:
+                    data[k2] = data[k2].replace(f"${{{k}}}", v)
+    return data
+
+
+
+def process_config_files(dirs, yaml_data, build_dir="./build"):
+
+        path = Path(build_dir)
+        if path.exists():
+            shutil.rmtree(path)
+        path.mkdir()
+
+        local_yaml_path = ""
+        yaml_local = {}
+        for directory in dirs:
+            for dirpath, _, filenames in os.walk(directory):
+                # When we have traversed out of the module, reset the local yaml config
+                if local_yaml_path not in dirpath:
+                    local_yaml_path == ""
+                    yaml_local = {}
+                for file in filenames:
+                    if file in EXCL_FILES:
+                        continue
+                    # Skip the config.yaml file
+                    if file == "config.yaml":
+                        # Pick up this local yaml files
+                        local_yaml_path = dirpath
+                        yaml_local = read_yaml_files([dirpath])
+                        continue
+                    with open(dirpath + "/" + file, "rt") as f:
+                        content = f.read()
+                    # Replace the local yaml variables
+                    for k, v in yaml_local.items():
+                        # assuming template variables are in the format {{key}}
+                        content = content.replace(f"{{{{{k}}}}}", str(v))
+                    # Replace the root yaml variables
+                    for k, v in yaml_data.items():
+                        # assuming template variables are in the format {{key}}
+                        content = content.replace(f"{{{{{k}}}}}", str(v))
+
+                    split_path = dirpath.split("/")
+                    cdf_path = split_path[len(split_path) - 1]
+                    new_path = Path(f"{build_dir}/{cdf_path}")
+                    new_path.mkdir(exist_ok=True, parents=True)
+                    with open(new_path / file, "w") as f:
+                        f.write(content)
+
+
+
+def build_config(dir: str = "./build"):
+    # TODO #13 Add support for global.yaml and local.yaml configurations of modules and packages to pick up
+
+    try:
+
+        print(pathlib.Path(dir).resolve())
+        print(__file__)
+        dir = os.path.dirname(__file__)
+        dirs = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))]
+
+        print("listdirs", dirs)
+        print("YAML_DIRS", YAML_DIRS)
+
+
+        yaml_files = read_yaml_files(yaml_dirs=YAML_DIRS)
+
+        print(f"Found {yaml_files} YAML files")
+
+        process_config_files(
+            dirs=TMPL_DIRS,
+            yaml_data=yaml_files,
+            build_dir=dir,
+        )
+
+    except Exception as e:
+        print(f"Error building config files: {e}")
+        raise e