Merge pull request #29 from Thymis-io/feat/custom-modules

DEV-62: Custom Modules

.github/workflows/build.yml

@@ -7,18 +7,6 @@ on:
       - master
 
 jobs:
-  # - name: run nix-eval-jobs
-  #   run: nix run nixpkgs#nix-eval-jobs -- --gc-roots-dir gcroots --flake .#all-download-images | tee jobs.json
-  # - name: convert jobs json
-  #   run: nix run nixpkgs#jq -- -r 'select(.system == "x86_64-linux") | .attr + " " + .drvPath + " " + .name' < jobs.json > jobs
-  # - name: build jobs
-  #   run: |
-  #     mkdir -p results
-  #     while read -r attr drvPath name; do
-  #       nix build "$drvPath^*" --out-link "results/$attr-$name" --print-build-logs
-  #     done < jobs
-  # - name: check if nix-eval-jobs produced errors
-  #   run: jq -e 'select(.error)' < jobs.json && exit 1 || true
   build-iso:
     runs-on: ubuntu-latest
     steps:

controller/default.nix

@@ -1,4 +1,8 @@
-{ poetry2nix, git, makeWrapper, lib
+{ poetry2nix
+, git
+, nixpkgs-fmt
+, makeWrapper
+, lib
 ,
 }:
 poetry2nix.mkPoetryApplication {
@@ -11,6 +15,6 @@ poetry2nix.mkPoetryApplication {
   nativeBuildInputs = [ makeWrapper ];
   postInstall = ''
     wrapProgram $out/bin/thymis-controller \
-      --prefix PATH : ${lib.makeBinPath [ git ]}
+      --prefix PATH : ${lib.makeBinPath [ git nixpkgs-fmt ]}
   '';
 }

controller/thymis_controller/crud/project.py

@@ -1,5 +1,4 @@
 import json
-from pydoc import locate
 from pathlib import Path
 from thymis_controller.models.state import State
 import os
@@ -17,6 +16,7 @@ def initialize(self):
         # TODO git reop init - git config user and email
         state = State(
             version="0.0.1",
+            repositories={},
             tags=[],
             devices=[],
         )
@@ -25,7 +25,9 @@ def initialize(self):
     def load_state_from_file(self):
         with open(Path(self.path) / "state.json", "r", encoding="utf-8") as f:
             state_dict = json.load(f)
-        return State.load_from_dict(state_dict)
+        state = State.load_from_dict(state_dict)
+        state.set_repositories_in_python_path(self.path)
+        return state
 
     def update_state(self, state):
         old_state = self.load_state_from_file()

controller/thymis_controller/migration/migrate.py

@@ -1,9 +1,13 @@
 from packaging import version
 from thymis_controller.migration.to_0_0_2 import to_0_0_2
+from thymis_controller.migration.to_0_0_3 import to_0_0_3
+from thymis_controller.migration.to_0_0_4 import to_0_0_4
 
 KNOWN_VERSIONS = [
     "0.0.1",
     "0.0.2",
+    "0.0.3",
+    "0.0.4",
 ]  # TODO: remove this, replace with dynamic versioning
 
 
@@ -13,4 +17,10 @@ def migrate(state: dict):
     if version.parse(state["version"]) == version.parse("0.0.1"):
         state = to_0_0_2(state)
 
+    if version.parse(state["version"]) == version.parse("0.0.2"):
+        state = to_0_0_3(state)
+
+    if version.parse(state["version"]) == version.parse("0.0.3"):
+        state = to_0_0_4(state)
+
     return state

controller/thymis_controller/migration/to_0_0_3.py

@@ -0,0 +1,5 @@
+def to_0_0_3(state: dict):
+    state["version"] = "0.0.3"
+    state["repositories"] = {}
+
+    return state

controller/thymis_controller/migration/to_0_0_4.py

@@ -0,0 +1,7 @@
+def to_0_0_4(state: dict):
+    state["version"] = "0.0.4"
+
+    for repo in state["repositories"]:
+        state["repositories"][repo]["inputs_follows"] = {}
+
+    return state

controller/thymis_controller/models/__init__.py

@@ -3,4 +3,5 @@
 from .modules import ALL_MODULES
 from .tag import Tag
 from .device import Device
+from .repo import Repo
 from .state import State

controller/thymis_controller/models/flake.nix.j2

@@ -2,82 +2,82 @@
   description = "Thymis";
 
   inputs = {
-    thymis.url = "github:thymis-io/thymis";
-    # nixpkgs.url = "nixpkgs/nixos-23.05";
-    nixpkgs.follows = "thymis/nixpkgs";
-    home-manager.url = "github:nix-community/home-manager/release-23.11";
-    home-manager.inputs.nixpkgs.follows = "nixpkgs";
-    nixos-generators.url = "github:nix-community/nixos-generators";
-    nixos-generators.inputs.nixpkgs.follows = "nixpkgs";
-    nixos-hardware.url = "github:NixOS/nixos-hardware";
-    flake-utils.url = "github:numtide/flake-utils";
-    poetry2nix = {
-      url = "github:nix-community/poetry2nix/1.42.1";
-      inputs.nixpkgs.follows = "nixpkgs";
-      inputs.flake-utils.follows = "flake-utils";
-    };
-  };
+    {% for name, repository in repositories.items() %}
+    {{ name }} = {
+      {% if repository.url %}
+      url = "{{ repository.url }}";
+      {% endif %}{% if repository.follows %}
+      follows = "{{ repository.follows }}";
+      {% endif %}{% if repository.input_follows %}
+      {% for input_name, input_repository in repository.input_follows.items() %}
+      inputs.{{ input_name }}.follows = "{{ input_repository }}";
+      {% endfor %}{% endif %}
+      };{% endfor %}
+      };
 
-  outputs = inputs@{ self, nixpkgs, home-manager, poetry2nix, flake-utils, thymis, ... }:
-    let
-      eachSystem = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ];
-      state-json = builtins.fromJSON (builtins.readFile ./state.json);
-      device-to-nixosConfigurations = d:
+      outputs =
+        inputs@{ self
+          #{% for name, repository in repositories.items() %}
+        , {{ name
+        }}
+        {% endfor %}
+        , ...
+        }:
         let
-          # device-modules are all files in ./hosts/<identifier> that end with .nix
-          device-modules = nixpkgs.lib.mapAttrsToList
-            (path: type: ./hosts/${d.identifier}/${path})
-            (nixpkgs.lib.filterAttrs
-              (f: t: t == "regular" && nixpkgs.lib.hasSuffix ".nix" (builtins.toString f))
-              (
-                builtins.readDir ./hosts/${d.identifier}
-              ));
-          # for all tags, get them. For each tag, all files in ./tags/<tag> that end with .nix
-          tag-modules = builtins.concatMap
-            (tag: nixpkgs.lib.mapAttrsToList
-              (path: type: ./tags/${tag}/${path})
-              (nixpkgs.lib.filterAttrs
-                (f: t: t == "regular" && nixpkgs.lib.hasSuffix ".nix" (builtins.toString f))
-                (
-                  builtins.readDir ./tags/${tag}
-                ))
-            )
-            d.tags;
-        in
-        # {
-          #   modules = builtins.concatLists [
-          #     device-modules
-          #     tag-modules
-          #   ];
-          # };
-        {
-          name = d.identifier;
-          value = nixpkgs.lib.nixosSystem {
-            modules = device-modules
-              ++ tag-modules
-              ++ [
-              thymis.nixosModules.thymis
-            ];
-            specialArgs = {
-              inherit inputs;
+          eachSystem = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ];
+          state-json = builtins.fromJSON (builtins.readFile ./state.json);
+          device-to-nixosConfigurations = d:
+            let
+              # device-modules are all files in ./hosts/<identifier> that end with .nix
+              device-modules = nixpkgs.lib.mapAttrsToList
+                (path: type: ./hosts/${d.identifier}/${path})
+                (nixpkgs.lib.filterAttrs
+                  (f: t: t == "regular" && nixpkgs.lib.hasSuffix ".nix" (builtins.toString f))
+                  (
+                    builtins.readDir ./hosts/${d.identifier}
+                  ));
+              # for all tags, get them. For each tag, all files in ./tags/<tag> that end with .nix
+              tag-modules = builtins.concatMap
+                (tag: nixpkgs.lib.mapAttrsToList
+                  (path: type: ./tags/${tag}/${path})
+                  (nixpkgs.lib.filterAttrs
+                    (f: t: t == "regular" && nixpkgs.lib.hasSuffix ".nix" (builtins.toString f))
+                    (
+                      builtins.readDir ./tags/${tag}
+                    ))
+                )
+                d.tags;
+            in
+            {
+              name = d.identifier;
+              value = nixpkgs.lib.nixosSystem {
+                modules = device-modules
+                  ++ tag-modules
+                  ++ [
+                  thymis.nixosModules.thymis
+                ];
+                specialArgs = {
+                  inherit inputs;
+                };
+              };
             };
-          };
-        };
-    in
-    rec {
-      nixosConfigurations = builtins.listToAttrs
-        (builtins.map device-to-nixosConfigurations state-json.devices);
-      packages = eachSystem (system:
-        let
-          pkgs = import nixpkgs { inherit system; };
+          nixosConfigurations = builtins.listToAttrs
+            (builtins.map device-to-nixosConfigurations state-json.devices);
         in
         {
-          thymis = pkgs.writeText "thymis.json"
-            (builtins.toJSON (
-              builtins.mapAttrs
-                (name: nixosCfg: nixosCfg.config.system.build.toplevel)
-                nixosConfigurations
-            ));
-        });
-    };
-}
+          nixosConfigurations = nixosConfigurations;
+          packages = eachSystem (system:
+            let
+              pkgs = import nixpkgs { inherit system; };
+            in
+            {
+              thymis = pkgs.writeText "thymis.json"
+                (builtins.toJSON (
+                  builtins.mapAttrs
+                    (name: nixosCfg: nixosCfg.config.system.build.toplevel)
+                    nixosConfigurations
+                ));
+            });
+          inputs = inputs;
+        };
+    }

controller/thymis_controller/models/modules/__init__.py

@@ -17,3 +17,5 @@
     NodeRedModule(),
     MqttxModule(),
 ]
+
+ALL_MODULES_START = ALL_MODULES.copy()

controller/thymis_controller/models/repo.py

@@ -0,0 +1,74 @@
+from typing import Dict, List, Optional
+from pydantic import BaseModel
+import sys
+import os
+import pkgutil
+import importlib
+from .module import Module
+import thymis_controller.models.modules
+
+
+from thymis_controller.nix import get_input_out_path
+
+
+class Repo(BaseModel):
+    url: Optional[str] = None
+    follows: Optional[str] = None
+    inputs_follows: Dict[str, str] = {}
+
+
+startup_python_path = sys.path.copy()
+lockfile = None
+
+
+def load_repositories(flake_path: os.PathLike, repositories: Dict[str, Repo]):
+    # only run if lockfile changed
+    global lockfile
+    # lockfile sits at path/flake.lock
+    lockfile_path = os.path.join(flake_path, "flake.lock")
+    with open(lockfile_path, "r") as f:
+        new_lockfile = f.read()
+    if new_lockfile == lockfile:
+        return
+    lockfile = new_lockfile
+    # for each repository: get_input_out_path from the flake.nix in the path
+    input_out_paths = {}
+    for name, repo in repositories.items():
+        if not repo.url:
+            continue
+        path = get_input_out_path(flake_path, name)
+        if path is None:
+            continue
+        # check wether path / README.md exists and contains the string "contains thymis modules"
+        if not os.path.exists(os.path.join(path, "README.md")):
+            print(f"Repository {name} does not contain a README.md")
+            print(f"Skipping {name}")
+            continue
+        with open(os.path.join(path, "README.md"), "r") as f:
+            if "contains thymis modules" not in f.read():
+                print(f"Repository {name} contains no thymis modules")
+                print(f"Skipping {name}")
+                continue
+        input_out_paths[name] = path
+    # add the paths to sys.path
+    sys.path = startup_python_path.copy()
+    # for path in input_out_paths.values():
+    modules_found = []
+    for name, path in input_out_paths.items():
+        print(f"Adding {name} at {path} to sys.path")
+        sys.path.append(path)
+        # print modules in path
+        # print(list(pkgutil.iter_modules([path])))
+        for module in pkgutil.walk_packages([path]):
+            imported_module = importlib.import_module(module.name)
+            for cls in imported_module.__dict__.values():
+                if not isinstance(cls, type):
+                    continue
+                if issubclass(cls, Module) and cls != Module:
+                    module_obj = cls()
+                    modules_found.append(module_obj)
+                    print(f"Found module {module_obj.type}")
+    thymis_controller.models.modules.ALL_MODULES = (
+        thymis_controller.models.modules.ALL_MODULES_START
+    )
+    thymis_controller.models.modules.ALL_MODULES.extend(modules_found)