python(feat): Report templates service

python/examples/report_templates/report_template_with_python_config/.env-example

@@ -0,0 +1,8 @@
+# Retrieve the BASE_URI from the Sift team
+# Be sure to exclude "https://" from the BASE_URI
+#
+# BASE URIs and further details can be found here:
+#   https://docs.siftstack.com/ingestion/overview
+BASE_URI=""
+
+SIFT_API_KEY=""

python/examples/report_templates/report_template_with_python_config/expression_modules/kinematics.yml

@@ -0,0 +1,4 @@
+kinetic_energy_gt:
+  0.5 * $mass * $1 * $1 > $threshold
+rod_torque_gt:
+  (1 / 12) * $mass * $rod_length * $rod_length * $1

python/examples/report_templates/report_template_with_python_config/expression_modules/string.yml

@@ -0,0 +1,2 @@
+log_substring_contains:
+  contains($1, $sub_string)

python/examples/report_templates/report_template_with_python_config/main.py

@@ -0,0 +1,48 @@
+import os
+
+from dotenv import load_dotenv
+from report_template_config import load_rules, nostromos_report_template
+from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel
+from sift_py.report_templates.service import ReportTemplateService
+from sift_py.rule.service import RuleService
+
+if __name__ == "__main__":
+    load_dotenv()
+
+    apikey = os.getenv("SIFT_API_KEY")
+    assert apikey, "Missing 'SIFT_API_KEY' environment variable."
+
+    base_uri = os.getenv("BASE_URI")
+    assert base_uri, "Missing 'BASE_URI' environment variable."
+
+    # Create a gRPC transport channel for the Sift API
+    sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey)
+
+    with use_sift_channel(sift_channel_config) as channel:
+        # First create rules
+        rule_service = RuleService(channel)
+        rules = load_rules()  # Load rules from python
+        [rule_service.create_or_update_rule(rule) for rule in rules]
+
+        # Now create report template
+        report_template_service = ReportTemplateService(channel)
+        report_template = nostromos_report_template()
+        report_template.rule_client_keys = [
+            rule.rule_client_key for rule in rules if rule.rule_client_key
+        ]  # Add the rules we just created
+        report_template_service.create_or_update_report_template(report_template)
+
+        # Then make some updates to the template we created (for the sake of example)
+        rules = [rule for rule in rules if "overheating" not in rule.name]  # Remove some rules
+        # Get the report template (for the sake of example)
+        report_template_to_update = report_template_service.get_report_template(
+            client_key=report_template.template_client_key
+        )
+        if report_template_to_update:  # Make some other changes
+            report_template_to_update.rule_client_keys = [
+                rule.rule_client_key for rule in rules if rule.rule_client_key
+            ]
+            report_template_to_update.description = (
+                "A report template for the Nostromo without overheating rule"
+            )
+            report_template_service.create_or_update_report_template(report_template_to_update)

python/examples/report_templates/report_template_with_python_config/report_template_config.py

@@ -0,0 +1,118 @@
+from pathlib import Path
+from typing import List
+
+from sift_py.ingestion.channel import (
+    ChannelConfig,
+    ChannelDataType,
+    ChannelEnumType,
+)
+from sift_py.ingestion.config.yaml.load import load_named_expression_modules
+from sift_py.ingestion.rule.config import (
+    RuleActionCreateDataReviewAnnotation,
+    RuleConfig,
+)
+from sift_py.report_templates.config import ReportTemplateConfig
+
+EXPRESSION_MODULES_DIR = Path().joinpath("expression_modules")
+
+
+def load_rules() -> List[RuleConfig]:
+    named_expressions = load_named_expression_modules(
+        [
+            EXPRESSION_MODULES_DIR.joinpath("kinematics.yml"),
+            EXPRESSION_MODULES_DIR.joinpath("string.yml"),
+        ]
+    )
+
+    log_channel = ChannelConfig(
+        name="log",
+        data_type=ChannelDataType.STRING,
+        description="asset logs",
+    )
+    voltage_channel = ChannelConfig(
+        name="voltage",
+        data_type=ChannelDataType.INT_32,
+        description="voltage at source",
+        unit="Volts",
+    )
+    vehicle_state_channel = ChannelConfig(
+        name="vehicle_state",
+        data_type=ChannelDataType.ENUM,
+        description="vehicle state",
+        enum_types=[
+            ChannelEnumType(name="Accelerating", key=0),
+            ChannelEnumType(name="Decelerating", key=1),
+            ChannelEnumType(name="Stopped", key=2),
+        ],
+    )
+
+    rules = [
+        RuleConfig(
+            name="overheating",
+            description="Checks for vehicle overheating",
+            expression='$1 == "Accelerating" && $2 > 80',
+            rule_client_key="overheating-rule",
+            asset_names=["NostromoLV2024"],
+            channel_references=[
+                # INFO: Can use either "channel_identifier" or "channel_config"
+                {
+                    "channel_reference": "$1",
+                    "channel_identifier": vehicle_state_channel.fqn(),
+                },
+                {
+                    "channel_reference": "$2",
+                    "channel_config": voltage_channel,
+                },
+            ],
+            action=RuleActionCreateDataReviewAnnotation(),
+        ),
+        RuleConfig(
+            name="kinetic_energy",
+            description="Tracks high energy output while in motion",
+            expression=named_expressions["kinetic_energy_gt"],
+            rule_client_key="kinetic-energy-rule",
+            asset_names=["NostromoLV2024"],
+            channel_references=[
+                {
+                    "channel_reference": "$1",
+                    "channel_config": voltage_channel,
+                },
+            ],
+            sub_expressions={
+                "$mass": 10,
+                "$threshold": 470,
+            },
+            action=RuleActionCreateDataReviewAnnotation(
+                tags=["nostromo"],
+            ),
+        ),
+        RuleConfig(
+            name="failure",
+            description="Checks for failures reported by logs",
+            expression=named_expressions["log_substring_contains"],
+            rule_client_key="failure-rule",
+            asset_names=["NostromoLV2024"],
+            channel_references=[
+                {
+                    "channel_reference": "$1",
+                    "channel_config": log_channel,
+                },
+            ],
+            sub_expressions={
+                "$sub_string": "failure",
+            },
+            action=RuleActionCreateDataReviewAnnotation(
+                tags=["nostromo", "failure"],
+            ),
+        ),
+    ]
+    return rules
+
+
+def nostromos_report_template() -> ReportTemplateConfig:
+    return ReportTemplateConfig(
+        name="Nostromo Report Template",
+        template_client_key="nostromo-report-template-test",
+        description="A report template for the Nostromo",
+        rule_client_keys=[],
+    )

python/examples/report_templates/report_template_with_python_config/requirements.txt

@@ -0,0 +1,2 @@
+python-dotenv
+sift-stack-py

python/examples/report_templates/report_template_with_yaml_config/.env-example

@@ -0,0 +1,8 @@
+# Retrieve the BASE_URI from the Sift team
+# Be sure to exclude "https://" from the BASE_URI
+#
+# BASE URIs and further details can be found here:
+#   https://docs.siftstack.com/ingestion/overview
+BASE_URI=""
+
+SIFT_API_KEY=""

python/examples/report_templates/report_template_with_yaml_config/expression_modules/kinematics.yml

@@ -0,0 +1,4 @@
+kinetic_energy_gt:
+  0.5 * $mass * $1 * $1 > $threshold
+rod_torque_gt:
+  (1 / 12) * $mass * $rod_length * $rod_length * $1

python/examples/report_templates/report_template_with_yaml_config/expression_modules/string.yml

@@ -0,0 +1,2 @@
+log_substring_contains:
+  contains($1, $sub_string)

python/examples/report_templates/report_template_with_yaml_config/main.py

@@ -0,0 +1,58 @@
+import os
+from datetime import datetime
+from pathlib import Path
+
+from dotenv import load_dotenv
+from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel
+from sift_py.ingestion.config.yaml.load import load_named_expression_modules
+from sift_py.report_templates.service import ReportTemplateService
+from sift_py.rule.service import RuleService, SubExpression
+
+REPORT_TEMPLATES_DIR = Path().joinpath("report_templates")
+RULE_MODULES_DIR = Path().joinpath("rule_modules")
+EXPRESSION_MODULES_DIR = Path().joinpath("expression_modules")
+
+if __name__ == "__main__":
+    load_dotenv()
+
+    apikey = os.getenv("SIFT_API_KEY")
+    assert apikey, "Missing 'SIFT_API_KEY' environment variable."
+
+    base_uri = os.getenv("BASE_URI")
+    assert base_uri, "Missing 'BASE_URI' environment variable."
+
+    # Create a gRPC transport channel configured specifically for the Sift API
+    sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey)
+
+    # Paths to your rules, named expressions, and report template
+    report_templates = REPORT_TEMPLATES_DIR.joinpath("nostromo_report_template.yml")
+    rule_modules = RULE_MODULES_DIR.joinpath("rules.yml")
+    named_expressions = load_named_expression_modules(
+        [
+            EXPRESSION_MODULES_DIR.joinpath("kinematics.yml"),
+            EXPRESSION_MODULES_DIR.joinpath("string.yml"),
+        ]
+    )
+
+    with use_sift_channel(sift_channel_config) as channel:
+        # First create rules
+        rule_service = RuleService(channel)
+        rules = rule_service.load_rules_from_yaml(
+            paths=[rule_modules],
+            sub_expressions=[
+                SubExpression("kinetic_energy", named_expressions),
+                SubExpression("failure", named_expressions),
+            ],
+        )
+
+        # Now create report templates
+        report_template_service = ReportTemplateService(channel)
+        report_template_service.load_report_templates_from_yaml([report_templates])
+
+        # Archive one template, for the sake of example
+        report_template_to_update = report_template_service.get_report_template(
+            client_key="nostromo-report-template-1"
+        )
+        if report_template_to_update:
+            report_template_to_update.archived_date = datetime.now()
+            report_template_service.create_or_update_report_template(report_template_to_update)

python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml

@@ -0,0 +1,13 @@
+report_templates:
+  - name: Nostromo Report Template 1
+    description: A report tempate for the Nostromo
+    template_client_key: nostromo-report-template-1
+    rule_client_keys:
+      - kinetic-energy-rule
+      - failure-rule
+
+  - name: Nostromo Report Template Overheating
+    description: A report tempate for the Nostromo
+    template_client_key: nostromo-report-template-overheating
+    rule_client_keys:
+      - overheating-rule

python/examples/report_templates/report_template_with_yaml_config/requirements.txt

@@ -0,0 +1,2 @@
+python-dotenv
+sift-stack-py

python/examples/report_templates/report_template_with_yaml_config/rule_modules/nostromo.yml

@@ -0,0 +1,40 @@
+namespace: nostromo
+
+rules:
+  - name: overheating
+    description: Checks for vehicle overheating
+    expression: $1 == "Accelerating" && $2 > 80
+    rule_client_key: overheating-rule
+    channel_references:
+      - $1: *vehicle_state_channel
+      - $2: *voltage_channel
+    type: review
+
+  - name: kinetic_energy
+    description: Tracks high energy output while in motion
+    type: review
+    expression:
+      name: kinetic_energy_gt
+    rule_client_key: kinetic-energy-rule
+    channel_references:
+      - $1: *velocity_channel
+    sub_expressions:
+      - $mass: 10
+      - $threshold: 470
+    tags:
+        - nostromo
+
+  - name: failure
+    description: Checks for failures reported by logs
+    type: review
+    rule_client_key: failure-rule
+    expression:
+      name: log_substring_contains
+    channel_references:
+      - $1: *log_channel
+    sub_expressions:
+      - $sub_string: failure
+    tags:
+        - failure
+        - nostromo
+

python/lib/sift_py/ingestion/config/yaml/load.py

@@ -16,6 +16,7 @@
     TelemetryConfigYamlSpec,
 )
 from sift_py.ingestion.rule.config import RuleActionAnnotationKind
+from sift_py.yaml.utils import _handle_subdir
 
 _CHANNEL_REFERENCE_REGEX = re.compile(r"^\$\d+$")
 _SUB_EXPRESSION_REGEX = re.compile(r"^\$[a-zA-Z_]+$")
@@ -70,19 +71,11 @@ def update_rule_namespaces(rule_module_path: Path):
                 raise YamlConfigError(
                     f"Encountered rules with identical names being loaded, '{key}'."
                 )
-
         rule_namespaces.update(rule_module)
 
-    def handle_dir(path: Path):
-        for file_in_dir in path.iterdir():
-            if file_in_dir.is_dir():
-                handle_dir(file_in_dir)
-            elif file_in_dir.is_file():
-                update_rule_namespaces(file_in_dir)
-
     for path in paths:
         if path.is_dir():
-            handle_dir(path)
+            _handle_subdir(path, update_rule_namespaces)
         elif path.is_file():
             update_rule_namespaces(path)
 
@@ -118,7 +111,7 @@ def _read_rule_namespace_yaml(path: Path) -> Dict[str, List]:
             )
 
         rules = namespace_rules.get("rules")
-        if not isinstance(namespace, str):
+        if not isinstance(rules, list):
             raise YamlConfigError(
                 f"Expected '{rules}' to be a list in rule namespace yaml: '{path}'"
                 f"{_type_fqn(RuleNamespaceYamlSpec)}"

python/lib/sift_py/ingestion/config/yaml/spec.py

@@ -4,9 +4,9 @@
 
 from __future__ import annotations
 
-from typing import Dict, List, Literal, TypedDict, Union
+from typing import Dict, List, Literal, Union
 
-from typing_extensions import NotRequired
+from typing_extensions import NotRequired, TypedDict
 
 
 class TelemetryConfigYamlSpec(TypedDict):