From c09c81e0edab82ea3eefd9313f0561e9fb6c3617 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Thu, 24 Oct 2024 17:10:14 -0700 Subject: [PATCH 01/32] feat: Initial ReportTemplateService and ReportTemplateConfig --- .../lib/sift_py/report_templates/__init__.py | 0 python/lib/sift_py/report_templates/config.py | 19 +++++++++++ .../lib/sift_py/report_templates/service.py | 33 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 python/lib/sift_py/report_templates/__init__.py create mode 100644 python/lib/sift_py/report_templates/config.py create mode 100644 python/lib/sift_py/report_templates/service.py diff --git a/python/lib/sift_py/report_templates/__init__.py b/python/lib/sift_py/report_templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py new file mode 100644 index 00000000..025c995a --- /dev/null +++ b/python/lib/sift_py/report_templates/config.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import Dict, List, Optional + +from sift_py._internal.convert.json import AsJson +from sift_py.ingestion.config.yaml.spec import RuleYamlSpec +from sift_py.ingestion.rule.config import RuleConfig + + +class ReportTemplateConfig(AsJson): + """ + TODO: A nice doc + """ + name: str + template_client_key: str + tags: Optional[List[str]] + description: Optional[str] + rules: List[RuleConfig] + namespaces: Dict[str, List[RuleYamlSpec]] diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py new file mode 100644 index 00000000..7c48531a --- /dev/null +++ b/python/lib/sift_py/report_templates/service.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import Optional, cast + +from sift.report_templates.v1.report_templates_pb2 import ( + GetReportTemplateRequest, + GetReportTemplateResponse, + ReportTemplate, +) +from sift.report_templates.v1.report_templates_pb2_grpc import ReportTemplateServiceStub + +from sift_py.grpc.transport import SiftChannel +from sift_py.report_templates.config import ReportTemplateConfig + + +class ReportTemplateService(): + _report_template_service_stub: ReportTemplateServiceStub + + def __init__(self, channel: SiftChannel): + self._report_template_service_stub = ReportTemplateServiceStub(channel) + + def create_or_update_report_template(self, config: ReportTemplateConfig): + if not config.template_client_key: + raise Exception(f"Report template {config.name} requires a template_client_key") + + def _get_report_template_by_client_key(self, client_key: str) -> Optional[ReportTemplate]: + req = GetReportTemplateRequest(client_key=client_key) + try: + res = cast(GetReportTemplateResponse, self._report_template_service_stub.GetReportTemplate(req)) + return res.report_template or None + except: + return None + From df2743237b4376e4cc5a4c78e1270188992dbfd1 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Fri, 25 Oct 2024 17:22:07 -0700 Subject: [PATCH 02/32] feat: Add create & update methods to service --- python/lib/sift_py/report_templates/config.py | 1 + .../lib/sift_py/report_templates/service.py | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index 025c995a..bb9981a3 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -13,6 +13,7 @@ class ReportTemplateConfig(AsJson): """ name: str template_client_key: str + organization_id: str = "" tags: Optional[List[str]] description: Optional[str] rules: List[RuleConfig] diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 7c48531a..4de44c9a 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -3,9 +3,14 @@ from typing import Optional, cast from sift.report_templates.v1.report_templates_pb2 import ( + CreateReportTemplateRequest, + CreateReportTemplateRequestClientKeys, GetReportTemplateRequest, GetReportTemplateResponse, ReportTemplate, + ReportTemplateRule, + ReportTemplateTag, + UpdateReportTemplateRequest, ) from sift.report_templates.v1.report_templates_pb2_grpc import ReportTemplateServiceStub @@ -22,6 +27,9 @@ def __init__(self, channel: SiftChannel): def create_or_update_report_template(self, config: ReportTemplateConfig): if not config.template_client_key: raise Exception(f"Report template {config.name} requires a template_client_key") + if self._get_report_template_by_client_key(config.template_client_key): + self._update_report_template(config) + self._create_report_template(config) def _get_report_template_by_client_key(self, client_key: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(client_key=client_key) @@ -31,3 +39,43 @@ def _get_report_template_by_client_key(self, client_key: str) -> Optional[Report except: return None + def _create_report_template(self, config: ReportTemplateConfig): + rule_client_keys = self._get_rule_client_keys(config) # TODO only like this for reuse + client_keys_req = CreateReportTemplateRequestClientKeys(rule_client_keys=rule_client_keys) + req = CreateReportTemplateRequest( + name=config.name, + client_key=config.template_client_key, + description=config.description, + tag_names=config.tags, + organization_id=config.organization_id, + rule_client_keys=client_keys_req, + ) + self._report_template_service_stub.CreateReportTemplate(req) + + def _update_report_template(self, config: ReportTemplateConfig): + tags = [] + if config.tags: + tags = [ReportTemplateTag(tag_name=tag) for tag in config.tags] + + rule_client_keys = self._get_rule_client_keys(config) + rules = [ReportTemplateRule(client_key=client_key) for client_key in rule_client_keys] + + report_template = ReportTemplate( + name=config.name, + client_key=config.template_client_key, + description=config.description, + tags=tags, + organization_id=config.organization_id, + rules=rules, + ) + self._report_template_service_stub.UpdateReportTemplate(UpdateReportTemplateRequest(report_template=report_template)) + + def _get_rule_client_keys(self, config: ReportTemplateConfig) -> list[str]: + client_keys = [] + for rule in config.rules: + client_key = rule.rule_client_key + if not client_key: + raise Exception(f"Rule {rule.name} requires a rule_client_key") + client_keys.append(client_key) + + return client_keys From d967468f6272a605bdc469d2188e17cbf4fb12f7 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 4 Nov 2024 16:29:25 -0800 Subject: [PATCH 03/32] test: ReportTemplateConfig init and start adding service tests --- .../sift_py/report_templates/_service_test.py | 32 +++++++++++++++++++ python/lib/sift_py/report_templates/config.py | 32 ++++++++++++++++++- .../lib/sift_py/report_templates/service.py | 2 +- 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 python/lib/sift_py/report_templates/_service_test.py diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py new file mode 100644 index 00000000..750b384e --- /dev/null +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -0,0 +1,32 @@ +import pytest + +from unittest import mock + +from sift_py._internal.test_util.channel import MockChannel +from sift_py.report_templates.config import ReportTemplateConfig +from sift_py.report_templates.service import ReportTemplateService + +""" +TODO: +- Create template, called once +- Update template, called once +- Missing key + +- Get rule client keys from config +- Some rules missing keys +""" + +@pytest.fixture +def report_template_service(): + return ReportTemplateService(MockChannel()) + + +def test_report_template_service_create_report_template(report_template_service): #, mock_create_report_template): #, mock_get_report_template_by_client_key): + report_template_config = ReportTemplateConfig( + name="report-template", + template_client_key="template-client-key", + ) + + with mock.patch.object(ReportTemplateService, "_create_report_template") as mock_create_report_template: + report_template_service.create_or_update_report_template(report_template_config) + mock_create_report_template.assert_called_once_with(report_template_config) diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index bb9981a3..5678e9f3 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from sift_py._internal.convert.json import AsJson from sift_py.ingestion.config.yaml.spec import RuleYamlSpec @@ -18,3 +18,33 @@ class ReportTemplateConfig(AsJson): description: Optional[str] rules: List[RuleConfig] namespaces: Dict[str, List[RuleYamlSpec]] + + def __init__( + self, + name: str, + template_client_key: str, + organization_id: str = "", + tags: Optional[List[str]] = None, + description: Optional[str] = None, + rules: List[RuleConfig] = [], + namespaces: Dict[str, List[RuleYamlSpec]] = {}, + ): + self.name = name + self.template_client_key = template_client_key + self.organization_id = organization_id + self.tags = tags + self.description = description + self.rules = rules + self.namespaces = namespaces + + def as_json(self) -> Any: + hash_map: Dict[str, Any] = { # TODO how to confirm the format here + "name": self.name, + "template_client_key": self.template_client_key, + "organization_id": self.organization_id, + "tags": self.tags, + "description": self.description, + "rules": [rule.as_json() for rule in self.rules], + "namespaces": self.namespaces, + } + return hash_map diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 4de44c9a..7432cb8d 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -40,7 +40,7 @@ def _get_report_template_by_client_key(self, client_key: str) -> Optional[Report return None def _create_report_template(self, config: ReportTemplateConfig): - rule_client_keys = self._get_rule_client_keys(config) # TODO only like this for reuse + rule_client_keys = self._get_rule_client_keys(config) client_keys_req = CreateReportTemplateRequestClientKeys(rule_client_keys=rule_client_keys) req = CreateReportTemplateRequest( name=config.name, From 3d2591dbfb38aed6c53758f4def3bd7c4d4f0486 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 5 Nov 2024 09:26:05 -0800 Subject: [PATCH 04/32] chore: import order fix --- python/lib/sift_py/report_templates/_service_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py index 750b384e..11219505 100644 --- a/python/lib/sift_py/report_templates/_service_test.py +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -1,7 +1,7 @@ -import pytest - from unittest import mock +import pytest + from sift_py._internal.test_util.channel import MockChannel from sift_py.report_templates.config import ReportTemplateConfig from sift_py.report_templates.service import ReportTemplateService @@ -21,7 +21,7 @@ def report_template_service(): return ReportTemplateService(MockChannel()) -def test_report_template_service_create_report_template(report_template_service): #, mock_create_report_template): #, mock_get_report_template_by_client_key): +def test_report_template_service_create_report_template(report_template_service): report_template_config = ReportTemplateConfig( name="report-template", template_client_key="template-client-key", From f2e908314c3d31fe2d1ecec45cf4f9b1c9be0e1f Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 5 Nov 2024 17:28:30 -0800 Subject: [PATCH 05/32] test: Add service tests --- .../sift_py/report_templates/_service_test.py | 83 +++++++++++++++++-- .../lib/sift_py/report_templates/service.py | 1 + 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py index 11219505..f6acc0fa 100644 --- a/python/lib/sift_py/report_templates/_service_test.py +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -5,16 +5,8 @@ from sift_py._internal.test_util.channel import MockChannel from sift_py.report_templates.config import ReportTemplateConfig from sift_py.report_templates.service import ReportTemplateService +from sift_py.rule.config import RuleConfig -""" -TODO: -- Create template, called once -- Update template, called once -- Missing key - -- Get rule client keys from config -- Some rules missing keys -""" @pytest.fixture def report_template_service(): @@ -30,3 +22,76 @@ def test_report_template_service_create_report_template(report_template_service) with mock.patch.object(ReportTemplateService, "_create_report_template") as mock_create_report_template: report_template_service.create_or_update_report_template(report_template_config) mock_create_report_template.assert_called_once_with(report_template_config) + + +def test_report_template_service_update_report_template(report_template_service): + report_template_config = ReportTemplateConfig( + name="report-template", + template_client_key="template-client-key", + ) + + report_template_config_update = ReportTemplateConfig( + name="report-template-updated", + template_client_key="template-client-key", + ) + + with mock.patch.object(ReportTemplateService, "_update_report_template") as mock_update_report_template: + with mock.patch.object(ReportTemplateService, "_get_report_template_by_client_key") as mock_get_report_template_by_client_key: + mock_get_report_template_by_client_key.return_value = report_template_config + report_template_service.create_or_update_report_template(report_template_config_update) + mock_update_report_template.assert_called_once_with(report_template_config_update) + + +def test_report_template_service_missing_template_client_key(report_template_service): + report_template_config = ReportTemplateConfig( + name="report-template", + template_client_key="", + ) + + with pytest.raises(Exception): + report_template_service.create_or_update_report_template(report_template_config, match="Report template report-template requires a template_client_key") + + +def test_report_template_service__get_rule_client_keys(report_template_service): + report_template_config = ReportTemplateConfig( + name="report-template", + template_client_key="template-client-key", + rules=[ + RuleConfig( + name="rule-1", + rule_client_key="rule-client-key-1", + channel_references=[], + ), + RuleConfig( + name="rule-2", + rule_client_key="rule-client-key-2", + channel_references=[], + ), + ], + ) + + rule_client_keys = report_template_service._get_rule_client_keys(report_template_config) + assert rule_client_keys == ["rule-client-key-1", "rule-client-key-2"] + + +def test_report_template_service__get_rule_client_keys_missing_rule_client_key(report_template_service): + report_template_config = ReportTemplateConfig( + name="report-template", + template_client_key="template-client-key", + rules=[ + RuleConfig( + name="rule-1", + rule_client_key="rule-client-key-1", + channel_references=[], + ), + RuleConfig( + name="rule-2", + rule_client_key="", + channel_references=[], + ), + ], + ) + + with pytest.raises(Exception): + report_template_service._get_rule_client_keys(report_template_config, match="rule rule-2 requires a rule_client_key") + diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 7432cb8d..ae1aee0d 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -29,6 +29,7 @@ def create_or_update_report_template(self, config: ReportTemplateConfig): raise Exception(f"Report template {config.name} requires a template_client_key") if self._get_report_template_by_client_key(config.template_client_key): self._update_report_template(config) + return self._create_report_template(config) def _get_report_template_by_client_key(self, client_key: str) -> Optional[ReportTemplate]: From d11cf48b58d121a9dc90b6ac9dbfadf2ae4894f8 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 12 Nov 2024 14:35:43 -0800 Subject: [PATCH 06/32] chore: Added get_report_template() --- .../sift_py/report_templates/_service_test.py | 21 +++++++++++++++++++ .../lib/sift_py/report_templates/service.py | 15 +++++++++++++ 2 files changed, 36 insertions(+) diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py index f6acc0fa..b1c4ce62 100644 --- a/python/lib/sift_py/report_templates/_service_test.py +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -13,6 +13,27 @@ def report_template_service(): return ReportTemplateService(MockChannel()) +def test_report_template_service_get_report_template_by_client_key(report_template_service): + report_template_client_key = "report-template-client-key" + + with mock.patch.object(ReportTemplateService, "_get_report_template_by_client_key") as mock_get_report_template_by_client_key: + report_template_service.get_report_template(client_key=report_template_client_key) + mock_get_report_template_by_client_key.assert_called_once_with(report_template_client_key) + + +def test_report_template_service_get_report_template_by_id(report_template_service): + report_template_id = "report-template-id" + + with mock.patch.object(ReportTemplateService, "_get_report_template_by_id") as mock_get_report_template_by_id: + report_template_service.get_report_template(report_template_id=report_template_id) + mock_get_report_template_by_id.assert_called_once_with(report_template_id) + + +def test_report_template_service_get_report_template_missing_client_key_and_id(report_template_service): + with pytest.raises(ValueError, match="Either client_key or report_template_id must be provided"): + report_template_service.get_report_template() + + def test_report_template_service_create_report_template(report_template_service): report_template_config = ReportTemplateConfig( name="report-template", diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index ae1aee0d..5c43df08 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -32,6 +32,21 @@ def create_or_update_report_template(self, config: ReportTemplateConfig): return self._create_report_template(config) + def get_report_template(self, client_key: str = "", report_template_id: str = "") -> Optional[ReportTemplate]: + if client_key: + return self._get_report_template_by_client_key(client_key) + if report_template_id: + return self._get_report_template_by_id(report_template_id) + raise ValueError("Either client_key or report_template_id must be provided") + + def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: + req = GetReportTemplateRequest(report_template_id=report_template_id) + try: + res = cast(GetReportTemplateResponse, self._report_template_service_stub.GetReportTemplate(req)) + return res.report_template or None + except: + return None + def _get_report_template_by_client_key(self, client_key: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(client_key=client_key) try: From 77dc9116c8e7c3e4443312a0042d839b486466cc Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 12 Nov 2024 14:36:00 -0800 Subject: [PATCH 07/32] chore: dev check --- .../sift_py/report_templates/_service_test.py | 42 ++++++++++++++----- python/lib/sift_py/report_templates/config.py | 1 + .../lib/sift_py/report_templates/service.py | 18 +++++--- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py index b1c4ce62..32c83bee 100644 --- a/python/lib/sift_py/report_templates/_service_test.py +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -16,7 +16,9 @@ def report_template_service(): def test_report_template_service_get_report_template_by_client_key(report_template_service): report_template_client_key = "report-template-client-key" - with mock.patch.object(ReportTemplateService, "_get_report_template_by_client_key") as mock_get_report_template_by_client_key: + with mock.patch.object( + ReportTemplateService, "_get_report_template_by_client_key" + ) as mock_get_report_template_by_client_key: report_template_service.get_report_template(client_key=report_template_client_key) mock_get_report_template_by_client_key.assert_called_once_with(report_template_client_key) @@ -24,13 +26,19 @@ def test_report_template_service_get_report_template_by_client_key(report_templa def test_report_template_service_get_report_template_by_id(report_template_service): report_template_id = "report-template-id" - with mock.patch.object(ReportTemplateService, "_get_report_template_by_id") as mock_get_report_template_by_id: + with mock.patch.object( + ReportTemplateService, "_get_report_template_by_id" + ) as mock_get_report_template_by_id: report_template_service.get_report_template(report_template_id=report_template_id) mock_get_report_template_by_id.assert_called_once_with(report_template_id) -def test_report_template_service_get_report_template_missing_client_key_and_id(report_template_service): - with pytest.raises(ValueError, match="Either client_key or report_template_id must be provided"): +def test_report_template_service_get_report_template_missing_client_key_and_id( + report_template_service, +): + with pytest.raises( + ValueError, match="Either client_key or report_template_id must be provided" + ): report_template_service.get_report_template() @@ -40,7 +48,9 @@ def test_report_template_service_create_report_template(report_template_service) template_client_key="template-client-key", ) - with mock.patch.object(ReportTemplateService, "_create_report_template") as mock_create_report_template: + with mock.patch.object( + ReportTemplateService, "_create_report_template" + ) as mock_create_report_template: report_template_service.create_or_update_report_template(report_template_config) mock_create_report_template.assert_called_once_with(report_template_config) @@ -56,8 +66,12 @@ def test_report_template_service_update_report_template(report_template_service) template_client_key="template-client-key", ) - with mock.patch.object(ReportTemplateService, "_update_report_template") as mock_update_report_template: - with mock.patch.object(ReportTemplateService, "_get_report_template_by_client_key") as mock_get_report_template_by_client_key: + with mock.patch.object( + ReportTemplateService, "_update_report_template" + ) as mock_update_report_template: + with mock.patch.object( + ReportTemplateService, "_get_report_template_by_client_key" + ) as mock_get_report_template_by_client_key: mock_get_report_template_by_client_key.return_value = report_template_config report_template_service.create_or_update_report_template(report_template_config_update) mock_update_report_template.assert_called_once_with(report_template_config_update) @@ -70,7 +84,10 @@ def test_report_template_service_missing_template_client_key(report_template_ser ) with pytest.raises(Exception): - report_template_service.create_or_update_report_template(report_template_config, match="Report template report-template requires a template_client_key") + report_template_service.create_or_update_report_template( + report_template_config, + match="Report template report-template requires a template_client_key", + ) def test_report_template_service__get_rule_client_keys(report_template_service): @@ -95,7 +112,9 @@ def test_report_template_service__get_rule_client_keys(report_template_service): assert rule_client_keys == ["rule-client-key-1", "rule-client-key-2"] -def test_report_template_service__get_rule_client_keys_missing_rule_client_key(report_template_service): +def test_report_template_service__get_rule_client_keys_missing_rule_client_key( + report_template_service, +): report_template_config = ReportTemplateConfig( name="report-template", template_client_key="template-client-key", @@ -114,5 +133,6 @@ def test_report_template_service__get_rule_client_keys_missing_rule_client_key(r ) with pytest.raises(Exception): - report_template_service._get_rule_client_keys(report_template_config, match="rule rule-2 requires a rule_client_key") - + report_template_service._get_rule_client_keys( + report_template_config, match="rule rule-2 requires a rule_client_key" + ) diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index 5678e9f3..e71e6e3b 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -11,6 +11,7 @@ class ReportTemplateConfig(AsJson): """ TODO: A nice doc """ + name: str template_client_key: str organization_id: str = "" diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 5c43df08..e34cad94 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -18,7 +18,7 @@ from sift_py.report_templates.config import ReportTemplateConfig -class ReportTemplateService(): +class ReportTemplateService: _report_template_service_stub: ReportTemplateServiceStub def __init__(self, channel: SiftChannel): @@ -32,7 +32,9 @@ def create_or_update_report_template(self, config: ReportTemplateConfig): return self._create_report_template(config) - def get_report_template(self, client_key: str = "", report_template_id: str = "") -> Optional[ReportTemplate]: + def get_report_template( + self, client_key: str = "", report_template_id: str = "" + ) -> Optional[ReportTemplate]: if client_key: return self._get_report_template_by_client_key(client_key) if report_template_id: @@ -42,7 +44,9 @@ def get_report_template(self, client_key: str = "", report_template_id: str = "" def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(report_template_id=report_template_id) try: - res = cast(GetReportTemplateResponse, self._report_template_service_stub.GetReportTemplate(req)) + res = cast( + GetReportTemplateResponse, self._report_template_service_stub.GetReportTemplate(req) + ) return res.report_template or None except: return None @@ -50,7 +54,9 @@ def _get_report_template_by_id(self, report_template_id: str) -> Optional[Report def _get_report_template_by_client_key(self, client_key: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(client_key=client_key) try: - res = cast(GetReportTemplateResponse, self._report_template_service_stub.GetReportTemplate(req)) + res = cast( + GetReportTemplateResponse, self._report_template_service_stub.GetReportTemplate(req) + ) return res.report_template or None except: return None @@ -84,7 +90,9 @@ def _update_report_template(self, config: ReportTemplateConfig): organization_id=config.organization_id, rules=rules, ) - self._report_template_service_stub.UpdateReportTemplate(UpdateReportTemplateRequest(report_template=report_template)) + self._report_template_service_stub.UpdateReportTemplate( + UpdateReportTemplateRequest(report_template=report_template) + ) def _get_rule_client_keys(self, config: ReportTemplateConfig) -> list[str]: client_keys = [] From 1d805adf9f49d97a6d350035d629d01bda63ea78 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 12 Nov 2024 17:15:30 -0800 Subject: [PATCH 08/32] chore: Switch ReportTemplateConfig to pydantic model --- .../.env-example | 8 +++ .../main.py | 0 .../requirements.txt | 2 + .../lib/sift_py/ingestion/config/yaml/spec.py | 4 +- .../sift_py/report_templates/_service_test.py | 2 +- python/lib/sift_py/report_templates/config.py | 60 ++++++++----------- 6 files changed, 39 insertions(+), 37 deletions(-) create mode 100644 python/examples/report_templates/report_template_with_python_config/.env-example create mode 100644 python/examples/report_templates/report_template_with_python_config/main.py create mode 100644 python/examples/report_templates/report_template_with_python_config/requirements.txt diff --git a/python/examples/report_templates/report_template_with_python_config/.env-example b/python/examples/report_templates/report_template_with_python_config/.env-example new file mode 100644 index 00000000..20b3bec2 --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/.env-example @@ -0,0 +1,8 @@ +# Retrieve the BASE_URI from the Sift team +# Be sure to exclude "https://" from the BASE_URI +# +# BASE URIs and further details can be found here: +# https://docs.siftstack.com/ingestion/overview +BASE_URI="" + +SIFT_API_KEY="" \ No newline at end of file diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py new file mode 100644 index 00000000..e69de29b diff --git a/python/examples/report_templates/report_template_with_python_config/requirements.txt b/python/examples/report_templates/report_template_with_python_config/requirements.txt new file mode 100644 index 00000000..2dda90fe --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/requirements.txt @@ -0,0 +1,2 @@ +python-dotenv +sift-stack-py diff --git a/python/lib/sift_py/ingestion/config/yaml/spec.py b/python/lib/sift_py/ingestion/config/yaml/spec.py index 9396c9f9..698331e3 100644 --- a/python/lib/sift_py/ingestion/config/yaml/spec.py +++ b/python/lib/sift_py/ingestion/config/yaml/spec.py @@ -4,9 +4,9 @@ from __future__ import annotations -from typing import Dict, List, Literal, TypedDict, Union +from typing import Dict, List, Literal, Union -from typing_extensions import NotRequired +from typing_extensions import NotRequired, TypedDict class TelemetryConfigYamlSpec(TypedDict): diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py index 32c83bee..e89125de 100644 --- a/python/lib/sift_py/report_templates/_service_test.py +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -78,7 +78,7 @@ def test_report_template_service_update_report_template(report_template_service) def test_report_template_service_missing_template_client_key(report_template_service): - report_template_config = ReportTemplateConfig( + report_template_config = ReportTemplateConfig.construct( # Without model validation name="report-template", template_client_key="", ) diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index e71e6e3b..1e58dba8 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -2,50 +2,42 @@ from typing import Any, Dict, List, Optional -from sift_py._internal.convert.json import AsJson +from pydantic import BaseModel, ConfigDict, model_validator +from pydantic_core import PydanticCustomError +from typing_extensions import Self + from sift_py.ingestion.config.yaml.spec import RuleYamlSpec from sift_py.ingestion.rule.config import RuleConfig -class ReportTemplateConfig(AsJson): +class ReportTemplateConfig(BaseModel): """ TODO: A nice doc """ + model_config = ConfigDict(arbitrary_types_allowed=True) + name: str template_client_key: str organization_id: str = "" - tags: Optional[List[str]] - description: Optional[str] - rules: List[RuleConfig] - namespaces: Dict[str, List[RuleYamlSpec]] - - def __init__( - self, - name: str, - template_client_key: str, - organization_id: str = "", - tags: Optional[List[str]] = None, - description: Optional[str] = None, - rules: List[RuleConfig] = [], - namespaces: Dict[str, List[RuleYamlSpec]] = {}, - ): - self.name = name - self.template_client_key = template_client_key - self.organization_id = organization_id - self.tags = tags - self.description = description - self.rules = rules - self.namespaces = namespaces + tags: Optional[List[str]] = None + description: Optional[str] = None + rules: List[RuleConfig] = [] + namespaces: Dict[str, List[RuleYamlSpec]] = {} + + @model_validator(mode="after") + def validate_config(self) -> Self: + if not self.name: + raise PydanticCustomError("invalid_config_error", "Empty 'name'") + if not self.template_client_key: + raise PydanticCustomError("invalid_config_error", "Empty 'template_client_key'") + return self def as_json(self) -> Any: - hash_map: Dict[str, Any] = { # TODO how to confirm the format here - "name": self.name, - "template_client_key": self.template_client_key, - "organization_id": self.organization_id, - "tags": self.tags, - "description": self.description, - "rules": [rule.as_json() for rule in self.rules], - "namespaces": self.namespaces, - } - return hash_map + return self.model_dump_json() + + def to_dict(self) -> Dict[str, Any]: + return self.model_dump() + + def from_dict(self, data: Dict[str, Any]) -> ReportTemplateConfig: + return self.parse_obj(data) From 0594e5bdc511bfa2d78cd344df1b689c8f32b910 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Wed, 13 Nov 2024 13:59:59 -0800 Subject: [PATCH 09/32] test: ReportTemplateConfig unit tests --- .../sift_py/report_templates/_config_test.py | 33 +++++++++++++++++++ python/lib/sift_py/report_templates/config.py | 13 +------- 2 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 python/lib/sift_py/report_templates/_config_test.py diff --git a/python/lib/sift_py/report_templates/_config_test.py b/python/lib/sift_py/report_templates/_config_test.py new file mode 100644 index 00000000..fdb73166 --- /dev/null +++ b/python/lib/sift_py/report_templates/_config_test.py @@ -0,0 +1,33 @@ +from pydantic_core import ValidationError +import pytest + +from sift_py.report_templates.config import ReportTemplateConfig + +@pytest.fixture +def report_template_dict() -> dict: + return { + "name": "report-template", + "template_client_key": "template-client-key", + } + + +def test_report_template_config(report_template_dict): + report_template_config = ReportTemplateConfig(**report_template_dict) + assert report_template_config.name == "report-template" + assert report_template_config.template_client_key == "template-client-key" + assert report_template_config.organization_id == "" + assert report_template_config.tags is None + assert report_template_config.description is None + assert report_template_config.rules == [] + assert report_template_config.namespaces == {} + + +def test_report_template_config_validation(report_template_dict): + report_template_dict.pop("name") + with pytest.raises(ValidationError, match="Field required"): + ReportTemplateConfig(**report_template_dict) + + report_template_dict["name"] = "report-template" + report_template_dict.pop("template_client_key") + with pytest.raises(ValidationError, match="Field required"): + ReportTemplateConfig(**report_template_dict) diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index 1e58dba8..27f49e54 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -12,7 +12,7 @@ class ReportTemplateConfig(BaseModel): """ - TODO: A nice doc + Configuration for a report template. """ model_config = ConfigDict(arbitrary_types_allowed=True) @@ -25,19 +25,8 @@ class ReportTemplateConfig(BaseModel): rules: List[RuleConfig] = [] namespaces: Dict[str, List[RuleYamlSpec]] = {} - @model_validator(mode="after") - def validate_config(self) -> Self: - if not self.name: - raise PydanticCustomError("invalid_config_error", "Empty 'name'") - if not self.template_client_key: - raise PydanticCustomError("invalid_config_error", "Empty 'template_client_key'") - return self - def as_json(self) -> Any: return self.model_dump_json() def to_dict(self) -> Dict[str, Any]: return self.model_dump() - - def from_dict(self, data: Dict[str, Any]) -> ReportTemplateConfig: - return self.parse_obj(data) From b6e66aeadb2b108639e3e169116ff2bddf73796f Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Fri, 15 Nov 2024 17:07:31 -0800 Subject: [PATCH 10/32] feat: Add report template initial example --- .../main.py | 73 ++++++++ .../report_template_config.py | 120 +++++++++++++ .../rule_modules/velocity.yml | 20 +++ .../rule_modules/voltage.yml | 20 +++ .../simulator.py | 169 ++++++++++++++++++ .../telemetry_configs/nostromo_lv_426.yml | 73 ++++++++ .../sift_py/report_templates/_config_test.py | 3 +- python/lib/sift_py/report_templates/config.py | 4 +- 8 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 python/examples/report_templates/report_template_with_python_config/report_template_config.py create mode 100644 python/examples/report_templates/report_template_with_python_config/rule_modules/velocity.yml create mode 100644 python/examples/report_templates/report_template_with_python_config/rule_modules/voltage.yml create mode 100644 python/examples/report_templates/report_template_with_python_config/simulator.py create mode 100644 python/examples/report_templates/report_template_with_python_config/telemetry_configs/nostromo_lv_426.yml diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index e69de29b..2fb20104 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -0,0 +1,73 @@ +import os +from pathlib import Path + +from dotenv import load_dotenv +from python.build.lib.sift_py.report_templates.service import ReportTemplateService +from report_template_config import load_rules, nostromos_report_template +from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel +from sift_py.rule.service import RuleService, SubExpression + +TELEMETRY_CONFIGS_DIR = Path().joinpath("telemetry_configs") +RULE_MODULES_DIR = Path().joinpath("rule_modules") + + +if __name__ == "__main__": + load_dotenv() + + apikey = os.getenv("SIFT_API_KEY") + assert apikey, "Missing 'SIFT_API_KEY' environment variable." + + base_uri = os.getenv("BASE_URI") + assert base_uri, "Missing 'BASE_URI' environment variable." + + # Create a gRPC transport channel configured specifically for the Sift API + sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey) + + with use_sift_channel(sift_channel_config) as channel: + # First create rules + rule_service = RuleService(channel) + rules = load_rules() # Load rules from python + rules += rule_service.load_rules_from_yaml( # Load rules from yaml + paths=[ + RULE_MODULES_DIR.joinpath("voltage.yml"), + RULE_MODULES_DIR.joinpath("velocity.yml"), + ], + sub_expressions=[ + SubExpression("voltage.overvoltage", {"$1": 75}), + SubExpression("voltage.undervoltage", {"$1": 30}), + SubExpression( + "velocity.vehicle_stuck", + { + "$1": "vehicle_state", + "$2": "mainmotor.velocity", + }, + ), + SubExpression( + "velocity.vehicle_not_stopped", + { + "$1": "vehicle_state", + "$2": "10", + }, + ), + ], + ) + + # Now create report template + report_template_service = ReportTemplateService(channel) + report_template_to_create = nostromos_report_template() + report_template_to_create.rules = rules # Add the rules we just created + report_template_service.create_or_update_report_template(report_template_to_create) + + # Then make some updates to the template we created (for the sake of the example) + report_template_to_update = report_template_service.get_report_template( + client_key="nostromo-report-template" + ) + if report_template_to_update: + rules = [ + rule for rule in report_template_to_update.rules if rule.name != "overheating" + ] # Remove some rules + report_template_to_update.rules = rules + report_template_to_update.description = ( + "A report template for the Nostromo without overheating rule" + ) + report_template_service.create_or_update_report_template(report_template_to_update) diff --git a/python/examples/report_templates/report_template_with_python_config/report_template_config.py b/python/examples/report_templates/report_template_with_python_config/report_template_config.py new file mode 100644 index 00000000..f048f4c2 --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/report_template_config.py @@ -0,0 +1,120 @@ +from pathlib import Path +from typing import List + +from sift_py.ingestion.channel import ( + ChannelConfig, + ChannelDataType, + ChannelEnumType, +) +from sift_py.ingestion.config.yaml.load import load_named_expression_modules +from sift_py.ingestion.rule.config import ( + RuleActionCreateDataReviewAnnotation, + RuleConfig, +) +from sift_py.report_templates.config import ReportTemplateConfig + +EXPRESSION_MODULES_DIR = Path().joinpath("expression_modules") +RULE_NAMESPACES_DIR = Path().joinpath("rule_modules") + + +def load_rules() -> List[RuleConfig]: + named_expressions = load_named_expression_modules( + [ + EXPRESSION_MODULES_DIR.joinpath("kinematics.yml"), + EXPRESSION_MODULES_DIR.joinpath("string.yml"), + ] + ) + + log_channel = ChannelConfig( + name="log", + data_type=ChannelDataType.STRING, + description="asset logs", + ) + voltage_channel = ChannelConfig( + name="voltage", + data_type=ChannelDataType.INT_32, + description="voltage at source", + unit="Volts", + ) + vehicle_state_channel = ChannelConfig( + name="vehicle_state", + data_type=ChannelDataType.ENUM, + description="vehicle state", + enum_types=[ + ChannelEnumType(name="Accelerating", key=0), + ChannelEnumType(name="Decelerating", key=1), + ChannelEnumType(name="Stopped", key=2), + ], + ) + + rules = [ + RuleConfig( + name="overheating", + description="Checks for vehicle overheating", + expression='$1 == "Accelerating" && $2 > 80', + rule_client_key="overheating-rule", + channel_references=[ + # INFO: Can use either "channel_identifier" or "channel_config" + { + "channel_reference": "$1", + "channel_identifier": vehicle_state_channel.fqn(), + }, + { + "channel_reference": "$2", + "channel_config": voltage_channel, + }, + ], + action=RuleActionCreateDataReviewAnnotation(), + ), + RuleConfig( + name="kinetic_energy", + description="Tracks high energy output while in motion", + expression=named_expressions["kinetic_energy_gt"], + rule_client_key="kinetic-energy-rule", + channel_references=[ + { + "channel_reference": "$1", + "channel_config": voltage_channel, + }, + ], + sub_expressions={ + "$mass": 10, + "$threshold": 470, + }, + action=RuleActionCreateDataReviewAnnotation( + # User in your organization to notify + # assignee="ellen.ripley@weylandcorp.com", + tags=["nostromo"], + ), + ), + RuleConfig( + name="failure", + description="Checks for failures reported by logs", + expression=named_expressions["log_substring_contains"], + rule_client_key="failure-rule", + channel_references=[ + { + "channel_reference": "$1", + "channel_config": log_channel, + }, + ], + sub_expressions={ + "$sub_string": "failure", + }, + action=RuleActionCreateDataReviewAnnotation( + # User in your organization to notify + # assignee="ellen.ripley@weylandcorp.com", + tags=["nostromo", "failure"], + ), + ), + ] + return rules + + +def nostromos_report_template() -> ReportTemplateConfig: + return ReportTemplateConfig( + name="Nostromo Report Template", + template_client_key="nostromo-report-template", + description="A report template for the Nostromo", + rules=[], + ) diff --git a/python/examples/report_templates/report_template_with_python_config/rule_modules/velocity.yml b/python/examples/report_templates/report_template_with_python_config/rule_modules/velocity.yml new file mode 100644 index 00000000..e1ff07d9 --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/rule_modules/velocity.yml @@ -0,0 +1,20 @@ +namespace: velocity + +rules: + - name: vehicle_stuck + rule_client_key: vehicle-stuck-key + description: Triggers if the vehicle velocity is not 0 for 5s after entering accelerating state + expression: $1 == "Accelerating" && persistence($2 == 0, 5) + type: review + assignee: benjamin@siftstack.com + asset_names: + - NostromoLV2024 + + - name: vehicle_not_stopped + rule_client_key: vehicle-not-stopped-key + description: Triggers if the vehicle velocity does not remain 0 while stopped + expression: $1 == "Stopped" && $2 > 0 + type: review + assignee: benjamin@siftstack.com + asset_names: + - NostromoLV2024 diff --git a/python/examples/report_templates/report_template_with_python_config/rule_modules/voltage.yml b/python/examples/report_templates/report_template_with_python_config/rule_modules/voltage.yml new file mode 100644 index 00000000..096495d9 --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/rule_modules/voltage.yml @@ -0,0 +1,20 @@ +namespace: voltage + +rules: + - name: overvoltage + rule_client_key: overvoltage-rule + description: Checks for overvoltage while accelerating + expression: vehicle_state == "Accelerating" && voltage > $1 + type: review + assignee: benjamin@siftstack.com + asset_names: + - NostromoLV2024 + + - name: undervoltage + rule_client_key: undervoltage-rule + description: Checks for undervoltage while accelerating + expression: vehicle_state == "Accelerating" && voltage < $1 + type: review + assignee: benjamin@siftstack.com + asset_names: + - NostromoLV2024 diff --git a/python/examples/report_templates/report_template_with_python_config/simulator.py b/python/examples/report_templates/report_template_with_python_config/simulator.py new file mode 100644 index 00000000..ccd5dc5c --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/simulator.py @@ -0,0 +1,169 @@ +import logging +import random +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import List + +from sift_py.ingestion.channel import ( + bit_field_value, + double_value, + enum_value, + int32_value, + string_value, +) +from sift_py.ingestion.service import IngestionService + +READINGS_FREQUENCY_HZ = 1.5 +LOGS_FREQUENCY_HZ = 2 +PARTIAL_READINGS_WITH_LOG_FREQUENCY_HZ = 0.5 + + +class Simulator: + """ + Telemeters sample data for 60 seconds for various combinations of flows + at various frequencies. + """ + + sample_bit_field_values: List[bytes] + sample_logs: List[str] + ingestion_service: IngestionService + logger: logging.Logger + + def __init__(self, ingestion_service: IngestionService): + self.ingestion_service = ingestion_service + + logging.basicConfig(level=logging.DEBUG) + self.logger = logging.getLogger(__name__) + + sample_bit_field_values = ["00001001", "00100011", "00001101", "11000001"] + self.sample_bit_field_values = [bytes([int(byte, 2)]) for byte in sample_bit_field_values] + + sample_logs = Path().joinpath("sample_data").joinpath("sample_logs.txt") + + with open(sample_logs, "r") as file: + self.sample_logs = file.readlines() + + def run(self): + """ + Send data for different combination of flows at different frequencies. + """ + + asset_name = self.ingestion_service.asset_name + run_id = self.ingestion_service.run_id + + if run_id is not None: + self.logger.info(f"Beginning simulation for '{asset_name}' with run ({run_id})") + else: + self.logger.info(f"Beginning simulation for '{asset_name}'") + + start_time = time.time() + end_time = start_time + 60 + + last_reading_time = start_time + last_log_time = start_time + last_partial_readings_time = start_time + + readings_interval_s = 1 / READINGS_FREQUENCY_HZ + logs_interval_s = 1 / LOGS_FREQUENCY_HZ + partial_readings_with_log_interval_s = 1 / PARTIAL_READINGS_WITH_LOG_FREQUENCY_HZ + + # Buffer size configured to 1 for the purposes of showing data come in live for + # this low ingestion-rate stream. + with self.ingestion_service.buffered_ingestion(buffer_size=1) as buffered_ingestion: + while time.time() < end_time: + current_time = time.time() + + # Send date for readings flow + if current_time - last_reading_time >= readings_interval_s: + timestamp = datetime.now(timezone.utc) + + buffered_ingestion.try_ingest_flows( + { + "flow_name": "readings", + "timestamp": timestamp, + "channel_values": [ + { + "channel_name": "velocity", + "component": "mainmotor", + "value": double_value(random.randint(1, 10)), + }, + { + "channel_name": "voltage", + "value": int32_value(random.randint(1, 10)), + }, + { + "channel_name": "vehicle_state", + "value": enum_value(random.randint(0, 2)), + }, + { + "channel_name": "gpio", + "value": bit_field_value( + random.choice(self.sample_bit_field_values) + ), + }, + ], + } + ) + logging.info(f"{timestamp} Emitted data for 'readings' flow") + last_reading_time = current_time + + # Send date for logs flow + if current_time - last_log_time >= logs_interval_s: + timestamp = datetime.now(timezone.utc) + + buffered_ingestion.try_ingest_flows( + { + "flow_name": "logs", + "timestamp": timestamp, + "channel_values": [ + { + "channel_name": "log", + "value": string_value(random.choice(self.sample_logs).strip()), + }, + ], + } + ) + logging.info(f"{timestamp} Emitted data for 'logs' flow") + last_log_time = current_time + + # Send partial data for readings flow and full data for logs flow + if ( + current_time - last_partial_readings_time + >= partial_readings_with_log_interval_s + ): + timestamp = datetime.now(timezone.utc) + + buffered_ingestion.try_ingest_flows( + { + "flow_name": "readings", + "timestamp": timestamp, + "channel_values": [ + { + "channel_name": "velocity", + "component": "mainmotor", + "value": double_value(random.randint(1, 10)), + }, + { + "channel_name": "voltage", + "value": int32_value(random.randint(1, 10)), + }, + ], + }, + { + "flow_name": "logs", + "timestamp": timestamp, + "channel_values": [ + { + "channel_name": "log", + "value": string_value(random.choice(self.sample_logs).strip()), + }, + ], + }, + ) + logging.info( + f"{timestamp} Emitted log for 'logs' flow and partial data for 'readings' flow" + ) + last_partial_readings_time = current_time + + self.logger.info("Completed simulation.") diff --git a/python/examples/report_templates/report_template_with_python_config/telemetry_configs/nostromo_lv_426.yml b/python/examples/report_templates/report_template_with_python_config/telemetry_configs/nostromo_lv_426.yml new file mode 100644 index 00000000..80b1d84d --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/telemetry_configs/nostromo_lv_426.yml @@ -0,0 +1,73 @@ +asset_name: NostromoLV2024 +ingestion_client_key: nostromo_lv_2024 + +channels: + log_channel: &log_channel + name: log + data_type: string + description: asset logs + + velocity_channel: &velocity_channel + name: velocity + data_type: double + description: speed + unit: Miles Per Hour + component: mainmotor + + voltage_channel: &voltage_channel + name: voltage + data_type: int32 + description: voltage at the source + unit: Volts + + vehicle_state_channel: &vehicle_state_channel + name: vehicle_state + data_type: enum + description: vehicle state + unit: vehicle state + enum_types: + - name: Accelerating + key: 0 + - name: Decelerating + key: 1 + - name: Stopped + key: 2 + + gpio_channel: &gpio_channel + name: gpio + data_type: bit_field + description: on/off values for pins on gpio + bit_field_elements: + - name: 12v + index: 0 + bit_count: 1 + - name: charge + index: 1 + bit_count: 2 + - name: led + index: 3 + bit_count: 4 + - name: heater + index: 7 + bit_count: 1 + +flows: + - name: readings + channels: + - <<: *velocity_channel + - <<: *voltage_channel + - <<: *vehicle_state_channel + - <<: *gpio_channel + + - name: voltage + channels: + - <<: *voltage_channel + + - name: gpio_channel + channels: + - <<: *gpio_channel + + - name: logs + channels: + - <<: *log_channel + diff --git a/python/lib/sift_py/report_templates/_config_test.py b/python/lib/sift_py/report_templates/_config_test.py index fdb73166..6e77535e 100644 --- a/python/lib/sift_py/report_templates/_config_test.py +++ b/python/lib/sift_py/report_templates/_config_test.py @@ -1,8 +1,9 @@ -from pydantic_core import ValidationError import pytest +from pydantic_core import ValidationError from sift_py.report_templates.config import ReportTemplateConfig + @pytest.fixture def report_template_dict() -> dict: return { diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index 27f49e54..91bcc38b 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -2,9 +2,7 @@ from typing import Any, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, model_validator -from pydantic_core import PydanticCustomError -from typing_extensions import Self +from pydantic import BaseModel, ConfigDict from sift_py.ingestion.config.yaml.spec import RuleYamlSpec from sift_py.ingestion.rule.config import RuleConfig From 48a9d66f4a45c9701fa9da27265b9df417567069 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 18 Nov 2024 18:12:26 -0800 Subject: [PATCH 11/32] fix: simplify & fix example --- .../expression_modules/kinematics.yml | 4 +++ .../expression_modules/string.yml | 2 ++ .../main.py | 29 ++----------------- .../report_template_config.py | 3 ++ 4 files changed, 12 insertions(+), 26 deletions(-) create mode 100644 python/examples/report_templates/report_template_with_python_config/expression_modules/kinematics.yml create mode 100644 python/examples/report_templates/report_template_with_python_config/expression_modules/string.yml diff --git a/python/examples/report_templates/report_template_with_python_config/expression_modules/kinematics.yml b/python/examples/report_templates/report_template_with_python_config/expression_modules/kinematics.yml new file mode 100644 index 00000000..36e21758 --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/expression_modules/kinematics.yml @@ -0,0 +1,4 @@ +kinetic_energy_gt: + 0.5 * $mass * $1 * $1 > $threshold +rod_torque_gt: + (1 / 12) * $mass * $rod_length * $rod_length * $1 diff --git a/python/examples/report_templates/report_template_with_python_config/expression_modules/string.yml b/python/examples/report_templates/report_template_with_python_config/expression_modules/string.yml new file mode 100644 index 00000000..f2190286 --- /dev/null +++ b/python/examples/report_templates/report_template_with_python_config/expression_modules/string.yml @@ -0,0 +1,2 @@ +log_substring_contains: + contains($1, $sub_string) diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index 2fb20104..219b947f 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -2,7 +2,7 @@ from pathlib import Path from dotenv import load_dotenv -from python.build.lib.sift_py.report_templates.service import ReportTemplateService +from sift_py.report_templates.service import ReportTemplateService from report_template_config import load_rules, nostromos_report_template from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel from sift_py.rule.service import RuleService, SubExpression @@ -21,36 +21,13 @@ assert base_uri, "Missing 'BASE_URI' environment variable." # Create a gRPC transport channel configured specifically for the Sift API - sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey) + sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey, use_ssl=False) with use_sift_channel(sift_channel_config) as channel: # First create rules rule_service = RuleService(channel) rules = load_rules() # Load rules from python - rules += rule_service.load_rules_from_yaml( # Load rules from yaml - paths=[ - RULE_MODULES_DIR.joinpath("voltage.yml"), - RULE_MODULES_DIR.joinpath("velocity.yml"), - ], - sub_expressions=[ - SubExpression("voltage.overvoltage", {"$1": 75}), - SubExpression("voltage.undervoltage", {"$1": 30}), - SubExpression( - "velocity.vehicle_stuck", - { - "$1": "vehicle_state", - "$2": "mainmotor.velocity", - }, - ), - SubExpression( - "velocity.vehicle_not_stopped", - { - "$1": "vehicle_state", - "$2": "10", - }, - ), - ], - ) + [rule_service.create_or_update_rule(rule) for rule in rules] # Now create report template report_template_service = ReportTemplateService(channel) diff --git a/python/examples/report_templates/report_template_with_python_config/report_template_config.py b/python/examples/report_templates/report_template_with_python_config/report_template_config.py index f048f4c2..e66809cd 100644 --- a/python/examples/report_templates/report_template_with_python_config/report_template_config.py +++ b/python/examples/report_templates/report_template_with_python_config/report_template_config.py @@ -53,6 +53,7 @@ def load_rules() -> List[RuleConfig]: description="Checks for vehicle overheating", expression='$1 == "Accelerating" && $2 > 80', rule_client_key="overheating-rule", + asset_names=["NostromoLV2024"], channel_references=[ # INFO: Can use either "channel_identifier" or "channel_config" { @@ -71,6 +72,7 @@ def load_rules() -> List[RuleConfig]: description="Tracks high energy output while in motion", expression=named_expressions["kinetic_energy_gt"], rule_client_key="kinetic-energy-rule", + asset_names=["NostromoLV2024"], channel_references=[ { "channel_reference": "$1", @@ -92,6 +94,7 @@ def load_rules() -> List[RuleConfig]: description="Checks for failures reported by logs", expression=named_expressions["log_substring_contains"], rule_client_key="failure-rule", + asset_names=["NostromoLV2024"], channel_references=[ { "channel_reference": "$1", From a22091647e3d1fbad346cf5f13182f8e2a202527 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 19 Nov 2024 15:41:03 -0800 Subject: [PATCH 12/32] placeholder --- .../main.py | 10 +++--- .../lib/sift_py/report_templates/service.py | 32 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index 219b947f..c23de7af 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -27,13 +27,13 @@ # First create rules rule_service = RuleService(channel) rules = load_rules() # Load rules from python - [rule_service.create_or_update_rule(rule) for rule in rules] + #[rule_service.create_or_update_rule(rule) for rule in rules] # Now create report template report_template_service = ReportTemplateService(channel) - report_template_to_create = nostromos_report_template() - report_template_to_create.rules = rules # Add the rules we just created - report_template_service.create_or_update_report_template(report_template_to_create) + report_template = nostromos_report_template() + report_template.rules = rules # Add the rules we just created + report_template_service.create_or_update_report_template(report_template) # Then make some updates to the template we created (for the sake of the example) report_template_to_update = report_template_service.get_report_template( @@ -47,4 +47,4 @@ report_template_to_update.description = ( "A report template for the Nostromo without overheating rule" ) - report_template_service.create_or_update_report_template(report_template_to_update) + report_template_service.create_or_update_report_template(report_template_to_update) \ No newline at end of file diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index e34cad94..b789ab27 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -2,6 +2,7 @@ from typing import Optional, cast +from google.protobuf.field_mask_pb2 import FieldMask from sift.report_templates.v1.report_templates_pb2 import ( CreateReportTemplateRequest, CreateReportTemplateRequestClientKeys, @@ -27,15 +28,16 @@ def __init__(self, channel: SiftChannel): def create_or_update_report_template(self, config: ReportTemplateConfig): if not config.template_client_key: raise Exception(f"Report template {config.name} requires a template_client_key") - if self._get_report_template_by_client_key(config.template_client_key): - self._update_report_template(config) + report_template = self._get_report_template_by_client_key(config.template_client_key) + if report_template: + self._update_report_template(config, report_template) return self._create_report_template(config) def get_report_template( self, client_key: str = "", report_template_id: str = "" - ) -> Optional[ReportTemplate]: - if client_key: + ) -> Optional[ReportTemplateConfig]: + if client_key: # TODO: return config return self._get_report_template_by_client_key(client_key) if report_template_id: return self._get_report_template_by_id(report_template_id) @@ -47,7 +49,7 @@ def _get_report_template_by_id(self, report_template_id: str) -> Optional[Report res = cast( GetReportTemplateResponse, self._report_template_service_stub.GetReportTemplate(req) ) - return res.report_template or None + return cast(ReportTemplate, res.report_template) or None except: return None @@ -74,7 +76,7 @@ def _create_report_template(self, config: ReportTemplateConfig): ) self._report_template_service_stub.CreateReportTemplate(req) - def _update_report_template(self, config: ReportTemplateConfig): + def _update_report_template(self, config: ReportTemplateConfig, report_template: ReportTemplate): tags = [] if config.tags: tags = [ReportTemplateTag(tag_name=tag) for tag in config.tags] @@ -82,16 +84,16 @@ def _update_report_template(self, config: ReportTemplateConfig): rule_client_keys = self._get_rule_client_keys(config) rules = [ReportTemplateRule(client_key=client_key) for client_key in rule_client_keys] - report_template = ReportTemplate( - name=config.name, - client_key=config.template_client_key, - description=config.description, - tags=tags, - organization_id=config.organization_id, - rules=rules, - ) + # TODO: Flip this around, take report template id and whatever is needed and create new + report_template.name=config.name + report_template.description=config.description + report_template.tags=tags + report_template.organization_id=config.organization_id + report_template.rules=rules + + field_mask = FieldMask(paths=["name", "description", "tags", "rules"]) self._report_template_service_stub.UpdateReportTemplate( - UpdateReportTemplateRequest(report_template=report_template) + UpdateReportTemplateRequest(report_template=report_template, update_mask=field_mask) ) def _get_rule_client_keys(self, config: ReportTemplateConfig) -> list[str]: From 112a2398ce87f1317eb5ce1b27d4a27fcaf8cefe Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Thu, 21 Nov 2024 11:53:39 -0800 Subject: [PATCH 13/32] feat: Clean up update report template function --- .../main.py | 18 +++++-------- .../lib/sift_py/report_templates/service.py | 27 ++++++++----------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index c23de7af..d3747ecc 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -36,15 +36,11 @@ report_template_service.create_or_update_report_template(report_template) # Then make some updates to the template we created (for the sake of the example) - report_template_to_update = report_template_service.get_report_template( - client_key="nostromo-report-template" + rules = [ + rule for rule in report_template.rules if rule.name != "overheating" + ] # Remove some rules + report_template.rules = rules + report_template.description = ( + "A report template for the Nostromo without overheating rule" ) - if report_template_to_update: - rules = [ - rule for rule in report_template_to_update.rules if rule.name != "overheating" - ] # Remove some rules - report_template_to_update.rules = rules - report_template_to_update.description = ( - "A report template for the Nostromo without overheating rule" - ) - report_template_service.create_or_update_report_template(report_template_to_update) \ No newline at end of file + report_template_service.create_or_update_report_template(report_template) \ No newline at end of file diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index b789ab27..00222c20 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -17,6 +17,7 @@ from sift_py.grpc.transport import SiftChannel from sift_py.report_templates.config import ReportTemplateConfig +from sift_py.rule.config import RuleConfig class ReportTemplateService: @@ -34,15 +35,6 @@ def create_or_update_report_template(self, config: ReportTemplateConfig): return self._create_report_template(config) - def get_report_template( - self, client_key: str = "", report_template_id: str = "" - ) -> Optional[ReportTemplateConfig]: - if client_key: # TODO: return config - return self._get_report_template_by_client_key(client_key) - if report_template_id: - return self._get_report_template_by_id(report_template_id) - raise ValueError("Either client_key or report_template_id must be provided") - def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(report_template_id=report_template_id) try: @@ -84,16 +76,19 @@ def _update_report_template(self, config: ReportTemplateConfig, report_template: rule_client_keys = self._get_rule_client_keys(config) rules = [ReportTemplateRule(client_key=client_key) for client_key in rule_client_keys] - # TODO: Flip this around, take report template id and whatever is needed and create new - report_template.name=config.name - report_template.description=config.description - report_template.tags=tags - report_template.organization_id=config.organization_id - report_template.rules=rules + updated_report_template = ReportTemplate( + report_template_id=report_template.report_template_id, + organization_id=report_template.organization_id, + client_key=report_template.client_key, + name=config.name, + description=config.description, + tags=tags, + rules=rules, + ) field_mask = FieldMask(paths=["name", "description", "tags", "rules"]) self._report_template_service_stub.UpdateReportTemplate( - UpdateReportTemplateRequest(report_template=report_template, update_mask=field_mask) + UpdateReportTemplateRequest(report_template=updated_report_template, update_mask=field_mask) ) def _get_rule_client_keys(self, config: ReportTemplateConfig) -> list[str]: From bf7b84e8443dd8ba24f6a88e7dd2057f5060f904 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Thu, 21 Nov 2024 16:14:37 -0800 Subject: [PATCH 14/32] chore: clean up --- .../main.py | 12 ++- .../report_template_config.py | 1 - .../rule_modules/velocity.yml | 20 ----- .../rule_modules/voltage.yml | 20 ----- .../telemetry_configs/nostromo_lv_426.yml | 73 ------------------- .../lib/sift_py/report_templates/service.py | 9 ++- 6 files changed, 11 insertions(+), 124 deletions(-) delete mode 100644 python/examples/report_templates/report_template_with_python_config/rule_modules/velocity.yml delete mode 100644 python/examples/report_templates/report_template_with_python_config/rule_modules/voltage.yml delete mode 100644 python/examples/report_templates/report_template_with_python_config/telemetry_configs/nostromo_lv_426.yml diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index d3747ecc..c0c07a38 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -2,10 +2,10 @@ from pathlib import Path from dotenv import load_dotenv -from sift_py.report_templates.service import ReportTemplateService from report_template_config import load_rules, nostromos_report_template from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel -from sift_py.rule.service import RuleService, SubExpression +from sift_py.report_templates.service import ReportTemplateService +from sift_py.rule.service import RuleService TELEMETRY_CONFIGS_DIR = Path().joinpath("telemetry_configs") RULE_MODULES_DIR = Path().joinpath("rule_modules") @@ -27,7 +27,7 @@ # First create rules rule_service = RuleService(channel) rules = load_rules() # Load rules from python - #[rule_service.create_or_update_rule(rule) for rule in rules] + # [rule_service.create_or_update_rule(rule) for rule in rules] # Now create report template report_template_service = ReportTemplateService(channel) @@ -40,7 +40,5 @@ rule for rule in report_template.rules if rule.name != "overheating" ] # Remove some rules report_template.rules = rules - report_template.description = ( - "A report template for the Nostromo without overheating rule" - ) - report_template_service.create_or_update_report_template(report_template) \ No newline at end of file + report_template.description = "A report template for the Nostromo without overheating rule" + report_template_service.create_or_update_report_template(report_template) diff --git a/python/examples/report_templates/report_template_with_python_config/report_template_config.py b/python/examples/report_templates/report_template_with_python_config/report_template_config.py index e66809cd..505cb6dc 100644 --- a/python/examples/report_templates/report_template_with_python_config/report_template_config.py +++ b/python/examples/report_templates/report_template_with_python_config/report_template_config.py @@ -14,7 +14,6 @@ from sift_py.report_templates.config import ReportTemplateConfig EXPRESSION_MODULES_DIR = Path().joinpath("expression_modules") -RULE_NAMESPACES_DIR = Path().joinpath("rule_modules") def load_rules() -> List[RuleConfig]: diff --git a/python/examples/report_templates/report_template_with_python_config/rule_modules/velocity.yml b/python/examples/report_templates/report_template_with_python_config/rule_modules/velocity.yml deleted file mode 100644 index e1ff07d9..00000000 --- a/python/examples/report_templates/report_template_with_python_config/rule_modules/velocity.yml +++ /dev/null @@ -1,20 +0,0 @@ -namespace: velocity - -rules: - - name: vehicle_stuck - rule_client_key: vehicle-stuck-key - description: Triggers if the vehicle velocity is not 0 for 5s after entering accelerating state - expression: $1 == "Accelerating" && persistence($2 == 0, 5) - type: review - assignee: benjamin@siftstack.com - asset_names: - - NostromoLV2024 - - - name: vehicle_not_stopped - rule_client_key: vehicle-not-stopped-key - description: Triggers if the vehicle velocity does not remain 0 while stopped - expression: $1 == "Stopped" && $2 > 0 - type: review - assignee: benjamin@siftstack.com - asset_names: - - NostromoLV2024 diff --git a/python/examples/report_templates/report_template_with_python_config/rule_modules/voltage.yml b/python/examples/report_templates/report_template_with_python_config/rule_modules/voltage.yml deleted file mode 100644 index 096495d9..00000000 --- a/python/examples/report_templates/report_template_with_python_config/rule_modules/voltage.yml +++ /dev/null @@ -1,20 +0,0 @@ -namespace: voltage - -rules: - - name: overvoltage - rule_client_key: overvoltage-rule - description: Checks for overvoltage while accelerating - expression: vehicle_state == "Accelerating" && voltage > $1 - type: review - assignee: benjamin@siftstack.com - asset_names: - - NostromoLV2024 - - - name: undervoltage - rule_client_key: undervoltage-rule - description: Checks for undervoltage while accelerating - expression: vehicle_state == "Accelerating" && voltage < $1 - type: review - assignee: benjamin@siftstack.com - asset_names: - - NostromoLV2024 diff --git a/python/examples/report_templates/report_template_with_python_config/telemetry_configs/nostromo_lv_426.yml b/python/examples/report_templates/report_template_with_python_config/telemetry_configs/nostromo_lv_426.yml deleted file mode 100644 index 80b1d84d..00000000 --- a/python/examples/report_templates/report_template_with_python_config/telemetry_configs/nostromo_lv_426.yml +++ /dev/null @@ -1,73 +0,0 @@ -asset_name: NostromoLV2024 -ingestion_client_key: nostromo_lv_2024 - -channels: - log_channel: &log_channel - name: log - data_type: string - description: asset logs - - velocity_channel: &velocity_channel - name: velocity - data_type: double - description: speed - unit: Miles Per Hour - component: mainmotor - - voltage_channel: &voltage_channel - name: voltage - data_type: int32 - description: voltage at the source - unit: Volts - - vehicle_state_channel: &vehicle_state_channel - name: vehicle_state - data_type: enum - description: vehicle state - unit: vehicle state - enum_types: - - name: Accelerating - key: 0 - - name: Decelerating - key: 1 - - name: Stopped - key: 2 - - gpio_channel: &gpio_channel - name: gpio - data_type: bit_field - description: on/off values for pins on gpio - bit_field_elements: - - name: 12v - index: 0 - bit_count: 1 - - name: charge - index: 1 - bit_count: 2 - - name: led - index: 3 - bit_count: 4 - - name: heater - index: 7 - bit_count: 1 - -flows: - - name: readings - channels: - - <<: *velocity_channel - - <<: *voltage_channel - - <<: *vehicle_state_channel - - <<: *gpio_channel - - - name: voltage - channels: - - <<: *voltage_channel - - - name: gpio_channel - channels: - - <<: *gpio_channel - - - name: logs - channels: - - <<: *log_channel - diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 00222c20..e03db7de 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -17,7 +17,6 @@ from sift_py.grpc.transport import SiftChannel from sift_py.report_templates.config import ReportTemplateConfig -from sift_py.rule.config import RuleConfig class ReportTemplateService: @@ -68,7 +67,9 @@ def _create_report_template(self, config: ReportTemplateConfig): ) self._report_template_service_stub.CreateReportTemplate(req) - def _update_report_template(self, config: ReportTemplateConfig, report_template: ReportTemplate): + def _update_report_template( + self, config: ReportTemplateConfig, report_template: ReportTemplate + ): tags = [] if config.tags: tags = [ReportTemplateTag(tag_name=tag) for tag in config.tags] @@ -88,7 +89,9 @@ def _update_report_template(self, config: ReportTemplateConfig, report_template: field_mask = FieldMask(paths=["name", "description", "tags", "rules"]) self._report_template_service_stub.UpdateReportTemplate( - UpdateReportTemplateRequest(report_template=updated_report_template, update_mask=field_mask) + UpdateReportTemplateRequest( + report_template=updated_report_template, update_mask=field_mask + ) ) def _get_rule_client_keys(self, config: ReportTemplateConfig) -> list[str]: From f037452f5ce7de57c949e8cc39ae5fbf25c58659 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Thu, 21 Nov 2024 16:47:23 -0800 Subject: [PATCH 15/32] chore: placeholder, starting yaml setup --- .../.env-example | 8 ++++ .../expression_modules/kinematics.yml | 4 ++ .../expression_modules/string.yml | 2 + .../report_template_with_yaml_config/main.py | 44 ++++++++++++++++++ .../nostromo_report_template.yml | 45 +++++++++++++++++++ .../requirements.txt | 2 + .../rule_modules/velocity.yml | 12 +++++ .../rule_modules/voltage.yml | 12 +++++ python/lib/sift_py/report_templates/config.py | 2 +- .../lib/sift_py/report_templates/service.py | 2 +- 10 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 python/examples/report_templates/report_template_with_yaml_config/.env-example create mode 100644 python/examples/report_templates/report_template_with_yaml_config/expression_modules/kinematics.yml create mode 100644 python/examples/report_templates/report_template_with_yaml_config/expression_modules/string.yml create mode 100644 python/examples/report_templates/report_template_with_yaml_config/main.py create mode 100644 python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml create mode 100644 python/examples/report_templates/report_template_with_yaml_config/requirements.txt create mode 100644 python/examples/report_templates/report_template_with_yaml_config/rule_modules/velocity.yml create mode 100644 python/examples/report_templates/report_template_with_yaml_config/rule_modules/voltage.yml diff --git a/python/examples/report_templates/report_template_with_yaml_config/.env-example b/python/examples/report_templates/report_template_with_yaml_config/.env-example new file mode 100644 index 00000000..20b3bec2 --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/.env-example @@ -0,0 +1,8 @@ +# Retrieve the BASE_URI from the Sift team +# Be sure to exclude "https://" from the BASE_URI +# +# BASE URIs and further details can be found here: +# https://docs.siftstack.com/ingestion/overview +BASE_URI="" + +SIFT_API_KEY="" \ No newline at end of file diff --git a/python/examples/report_templates/report_template_with_yaml_config/expression_modules/kinematics.yml b/python/examples/report_templates/report_template_with_yaml_config/expression_modules/kinematics.yml new file mode 100644 index 00000000..36e21758 --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/expression_modules/kinematics.yml @@ -0,0 +1,4 @@ +kinetic_energy_gt: + 0.5 * $mass * $1 * $1 > $threshold +rod_torque_gt: + (1 / 12) * $mass * $rod_length * $rod_length * $1 diff --git a/python/examples/report_templates/report_template_with_yaml_config/expression_modules/string.yml b/python/examples/report_templates/report_template_with_yaml_config/expression_modules/string.yml new file mode 100644 index 00000000..f2190286 --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/expression_modules/string.yml @@ -0,0 +1,2 @@ +log_substring_contains: + contains($1, $sub_string) diff --git a/python/examples/report_templates/report_template_with_yaml_config/main.py b/python/examples/report_templates/report_template_with_yaml_config/main.py new file mode 100644 index 00000000..c0c07a38 --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/main.py @@ -0,0 +1,44 @@ +import os +from pathlib import Path + +from dotenv import load_dotenv +from report_template_config import load_rules, nostromos_report_template +from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel +from sift_py.report_templates.service import ReportTemplateService +from sift_py.rule.service import RuleService + +TELEMETRY_CONFIGS_DIR = Path().joinpath("telemetry_configs") +RULE_MODULES_DIR = Path().joinpath("rule_modules") + + +if __name__ == "__main__": + load_dotenv() + + apikey = os.getenv("SIFT_API_KEY") + assert apikey, "Missing 'SIFT_API_KEY' environment variable." + + base_uri = os.getenv("BASE_URI") + assert base_uri, "Missing 'BASE_URI' environment variable." + + # Create a gRPC transport channel configured specifically for the Sift API + sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey, use_ssl=False) + + with use_sift_channel(sift_channel_config) as channel: + # First create rules + rule_service = RuleService(channel) + rules = load_rules() # Load rules from python + # [rule_service.create_or_update_rule(rule) for rule in rules] + + # Now create report template + report_template_service = ReportTemplateService(channel) + report_template = nostromos_report_template() + report_template.rules = rules # Add the rules we just created + report_template_service.create_or_update_report_template(report_template) + + # Then make some updates to the template we created (for the sake of the example) + rules = [ + rule for rule in report_template.rules if rule.name != "overheating" + ] # Remove some rules + report_template.rules = rules + report_template.description = "A report template for the Nostromo without overheating rule" + report_template_service.create_or_update_report_template(report_template) diff --git a/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml b/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml new file mode 100644 index 00000000..cf5ddfd1 --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml @@ -0,0 +1,45 @@ +report_templates: + - name: Nostromo + description: A report tempate for the Nostromo + template_client_key: nostromo-report-template + rules: + - name: overheating + description: Checks for vehicle overheating + expression: $1 == "Accelerating" && $2 > 80 + channel_references: + - $1: *vehicle_state_channel + - $2: *voltage_channel + type: review + + - name: kinetic_energy + description: Tracks high energy output while in motion + type: review + # assignee: ellen.ripley@weylandcorp.com + expression: + name: kinetic_energy_gt + channel_references: + - $1: *velocity_channel + sub_expressions: + - $mass: 10 + - $threshold: 470 + tags: + - nostromo + + - name: failure + description: Checks for failures reported by logs + type: review + # assignee: ellen.ripley@weylandcorp.com + expression: + name: log_substring_contains + channel_references: + - $1: *log_channel + sub_expressions: + - $sub_string: failure + tags: + - failure + - nostromo + + + namespaces: + - velocity + - voltage diff --git a/python/examples/report_templates/report_template_with_yaml_config/requirements.txt b/python/examples/report_templates/report_template_with_yaml_config/requirements.txt new file mode 100644 index 00000000..2dda90fe --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/requirements.txt @@ -0,0 +1,2 @@ +python-dotenv +sift-stack-py diff --git a/python/examples/report_templates/report_template_with_yaml_config/rule_modules/velocity.yml b/python/examples/report_templates/report_template_with_yaml_config/rule_modules/velocity.yml new file mode 100644 index 00000000..5505580a --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/rule_modules/velocity.yml @@ -0,0 +1,12 @@ +namespace: velocity + +rules: + - name: vehicle_stuck + description: Triggers if the vehicle velocity is not 0 for 5s after entering accelerating state + expression: $1 == "Accelerating" && persistence($2 == 0, 5) + type: review + + - name: vehicle_not_stopped + description: Triggers if the vehicle velocity does not remain 0 while stopped + expression: $1 == "Stopped" && $2 > 0 + type: review diff --git a/python/examples/report_templates/report_template_with_yaml_config/rule_modules/voltage.yml b/python/examples/report_templates/report_template_with_yaml_config/rule_modules/voltage.yml new file mode 100644 index 00000000..b47e7216 --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/rule_modules/voltage.yml @@ -0,0 +1,12 @@ +namespace: voltage + +rules: + - name: overvoltage + description: Checks for overvoltage while accelerating + expression: $1 == "Accelerating" && $2 > 80 + type: review + + - name: undervoltage + description: Checks for undervoltage while accelerating + expression: $1 == "Accelerating" && $2 < 40 + type: review diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index 91bcc38b..34dacaa4 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -20,7 +20,7 @@ class ReportTemplateConfig(BaseModel): organization_id: str = "" tags: Optional[List[str]] = None description: Optional[str] = None - rules: List[RuleConfig] = [] + rules: List[RuleConfig] = [] # TODO: Make this just rule client keys namespaces: Dict[str, List[RuleYamlSpec]] = {} def as_json(self) -> Any: diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index e03db7de..b6faeb96 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -33,7 +33,7 @@ def create_or_update_report_template(self, config: ReportTemplateConfig): self._update_report_template(config, report_template) return self._create_report_template(config) - + # TODO: Can add back get after rule client key change def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(report_template_id=report_template_id) try: From 1951808010d348acea8d756ad3283b3a236e0b89 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Fri, 22 Nov 2024 17:00:12 -0800 Subject: [PATCH 16/32] refactor: Just use rule client keys --- .../report_template_with_python_config/main.py | 6 +++--- python/lib/sift_py/report_templates/config.py | 10 ++++++++-- python/lib/sift_py/report_templates/service.py | 12 +----------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index c0c07a38..15cde3fb 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -32,13 +32,13 @@ # Now create report template report_template_service = ReportTemplateService(channel) report_template = nostromos_report_template() - report_template.rules = rules # Add the rules we just created + report_template.rules = [rule.rule_client_key for rule in rules] # Add the rules we just created report_template_service.create_or_update_report_template(report_template) # Then make some updates to the template we created (for the sake of the example) rules = [ - rule for rule in report_template.rules if rule.name != "overheating" + rule for rule in rules if rule.name != "overheating" ] # Remove some rules - report_template.rules = rules + report_template.rules = [rule.rule_client_key for rule in rules] report_template.description = "A report template for the Nostromo without overheating rule" report_template_service.create_or_update_report_template(report_template) diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index 34dacaa4..d1512b11 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -11,6 +11,13 @@ class ReportTemplateConfig(BaseModel): """ Configuration for a report template. + + - `name`: Name of the report template. + - `template_client_key`: Unique client key to identify the report template. + - `organization_id`: Organization ID that the report template belongs to. + - `tags`: Tags to associate with the report template. + - `description`: Description of the report template. + - `rule_client_keys`: List of rule client keys associated with the report template. """ model_config = ConfigDict(arbitrary_types_allowed=True) @@ -20,8 +27,7 @@ class ReportTemplateConfig(BaseModel): organization_id: str = "" tags: Optional[List[str]] = None description: Optional[str] = None - rules: List[RuleConfig] = [] # TODO: Make this just rule client keys - namespaces: Dict[str, List[RuleYamlSpec]] = {} + rule_client_keys: List[str] = [] def as_json(self) -> Any: return self.model_dump_json() diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index b6faeb96..0f954392 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -92,14 +92,4 @@ def _update_report_template( UpdateReportTemplateRequest( report_template=updated_report_template, update_mask=field_mask ) - ) - - def _get_rule_client_keys(self, config: ReportTemplateConfig) -> list[str]: - client_keys = [] - for rule in config.rules: - client_key = rule.rule_client_key - if not client_key: - raise Exception(f"Rule {rule.name} requires a rule_client_key") - client_keys.append(client_key) - - return client_keys + ) \ No newline at end of file From aee8efb80ff137cffd8073a5914d4b8193b4f6f9 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Sat, 23 Nov 2024 14:30:54 -0800 Subject: [PATCH 17/32] feat: Add back get function and fix unit tests --- .../main.py | 13 ++-- .../sift_py/report_templates/_config_test.py | 3 +- .../sift_py/report_templates/_service_test.py | 61 +++---------------- .../lib/sift_py/report_templates/service.py | 31 ++++++++-- 4 files changed, 42 insertions(+), 66 deletions(-) diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index 15cde3fb..ac5c598c 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -27,7 +27,7 @@ # First create rules rule_service = RuleService(channel) rules = load_rules() # Load rules from python - # [rule_service.create_or_update_rule(rule) for rule in rules] + [rule_service.create_or_update_rule(rule) for rule in rules] # Now create report template report_template_service = ReportTemplateService(channel) @@ -35,10 +35,13 @@ report_template.rules = [rule.rule_client_key for rule in rules] # Add the rules we just created report_template_service.create_or_update_report_template(report_template) - # Then make some updates to the template we created (for the sake of the example) + # Then make some updates to the template we created (for the sake of example) rules = [ rule for rule in rules if rule.name != "overheating" ] # Remove some rules - report_template.rules = [rule.rule_client_key for rule in rules] - report_template.description = "A report template for the Nostromo without overheating rule" - report_template_service.create_or_update_report_template(report_template) + # Get the report template (for the sake of example) + report_template_to_update = report_template_service.get_report_template(client_key=report_template.template_client_key) + if report_template_to_update: + report_template_to_update.rules = [rule.rule_client_key for rule in rules] + report_template_to_update.description = "A report template for the Nostromo without overheating rule" + report_template_service.create_or_update_report_template(report_template_to_update) diff --git a/python/lib/sift_py/report_templates/_config_test.py b/python/lib/sift_py/report_templates/_config_test.py index 6e77535e..7e2425d2 100644 --- a/python/lib/sift_py/report_templates/_config_test.py +++ b/python/lib/sift_py/report_templates/_config_test.py @@ -19,8 +19,7 @@ def test_report_template_config(report_template_dict): assert report_template_config.organization_id == "" assert report_template_config.tags is None assert report_template_config.description is None - assert report_template_config.rules == [] - assert report_template_config.namespaces == {} + assert report_template_config.rule_client_keys == [] def test_report_template_config_validation(report_template_dict): diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py index e89125de..9c356027 100644 --- a/python/lib/sift_py/report_templates/_service_test.py +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -2,6 +2,7 @@ import pytest +from sift.report_templates.v1.report_templates_pb2 import ReportTemplate from sift_py._internal.test_util.channel import MockChannel from sift_py.report_templates.config import ReportTemplateConfig from sift_py.report_templates.service import ReportTemplateService @@ -17,7 +18,7 @@ def test_report_template_service_get_report_template_by_client_key(report_templa report_template_client_key = "report-template-client-key" with mock.patch.object( - ReportTemplateService, "_get_report_template_by_client_key" + ReportTemplateService, "_get_report_template_by_client_key", return_value=ReportTemplate(name="abc") ) as mock_get_report_template_by_client_key: report_template_service.get_report_template(client_key=report_template_client_key) mock_get_report_template_by_client_key.assert_called_once_with(report_template_client_key) @@ -27,9 +28,9 @@ def test_report_template_service_get_report_template_by_id(report_template_servi report_template_id = "report-template-id" with mock.patch.object( - ReportTemplateService, "_get_report_template_by_id" + ReportTemplateService, "_get_report_template_by_id", return_value=ReportTemplate(name="abc") ) as mock_get_report_template_by_id: - report_template_service.get_report_template(report_template_id=report_template_id) + report_template_service.get_report_template(id=report_template_id) mock_get_report_template_by_id.assert_called_once_with(report_template_id) @@ -37,7 +38,7 @@ def test_report_template_service_get_report_template_missing_client_key_and_id( report_template_service, ): with pytest.raises( - ValueError, match="Either client_key or report_template_id must be provided" + ValueError, match="Either client_key or id must be provided" ): report_template_service.get_report_template() @@ -74,11 +75,11 @@ def test_report_template_service_update_report_template(report_template_service) ) as mock_get_report_template_by_client_key: mock_get_report_template_by_client_key.return_value = report_template_config report_template_service.create_or_update_report_template(report_template_config_update) - mock_update_report_template.assert_called_once_with(report_template_config_update) + mock_update_report_template.assert_called_once_with(report_template_config_update, report_template_config) def test_report_template_service_missing_template_client_key(report_template_service): - report_template_config = ReportTemplateConfig.construct( # Without model validation + report_template_config = ReportTemplateConfig.model_construct( # Without model validation name="report-template", template_client_key="", ) @@ -88,51 +89,3 @@ def test_report_template_service_missing_template_client_key(report_template_ser report_template_config, match="Report template report-template requires a template_client_key", ) - - -def test_report_template_service__get_rule_client_keys(report_template_service): - report_template_config = ReportTemplateConfig( - name="report-template", - template_client_key="template-client-key", - rules=[ - RuleConfig( - name="rule-1", - rule_client_key="rule-client-key-1", - channel_references=[], - ), - RuleConfig( - name="rule-2", - rule_client_key="rule-client-key-2", - channel_references=[], - ), - ], - ) - - rule_client_keys = report_template_service._get_rule_client_keys(report_template_config) - assert rule_client_keys == ["rule-client-key-1", "rule-client-key-2"] - - -def test_report_template_service__get_rule_client_keys_missing_rule_client_key( - report_template_service, -): - report_template_config = ReportTemplateConfig( - name="report-template", - template_client_key="template-client-key", - rules=[ - RuleConfig( - name="rule-1", - rule_client_key="rule-client-key-1", - channel_references=[], - ), - RuleConfig( - name="rule-2", - rule_client_key="", - channel_references=[], - ), - ], - ) - - with pytest.raises(Exception): - report_template_service._get_rule_client_keys( - report_template_config, match="rule rule-2 requires a rule_client_key" - ) diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 0f954392..44ba690e 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -33,7 +33,30 @@ def create_or_update_report_template(self, config: ReportTemplateConfig): self._update_report_template(config, report_template) return self._create_report_template(config) - # TODO: Can add back get after rule client key change + + def get_report_template(self, client_key: str = "", id: str = "") -> Optional[ReportTemplateConfig]: + report_template = None + if not client_key and not id: + raise ValueError("Either client_key or id must be provided") + + if id: + report_template = self._get_report_template_by_id(id) + elif client_key: + report_template = self._get_report_template_by_client_key(client_key) + + if not report_template: + raise Exception(f"Report template with client key {client_key} or id {id} not found.") + + return ReportTemplateConfig( + name=report_template.name, + template_client_key=report_template.client_key, + organization_id=report_template.organization_id, + tags=[tag.tag_name for tag in report_template.tags], + description=report_template.description, + rule_client_keys=[rule.client_key for rule in report_template.rules], + ) + + def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(report_template_id=report_template_id) try: @@ -55,8 +78,7 @@ def _get_report_template_by_client_key(self, client_key: str) -> Optional[Report return None def _create_report_template(self, config: ReportTemplateConfig): - rule_client_keys = self._get_rule_client_keys(config) - client_keys_req = CreateReportTemplateRequestClientKeys(rule_client_keys=rule_client_keys) + client_keys_req = CreateReportTemplateRequestClientKeys(rule_client_keys=config.rule_client_keys) req = CreateReportTemplateRequest( name=config.name, client_key=config.template_client_key, @@ -74,8 +96,7 @@ def _update_report_template( if config.tags: tags = [ReportTemplateTag(tag_name=tag) for tag in config.tags] - rule_client_keys = self._get_rule_client_keys(config) - rules = [ReportTemplateRule(client_key=client_key) for client_key in rule_client_keys] + rules = [ReportTemplateRule(client_key=client_key) for client_key in config.rule_client_keys] updated_report_template = ReportTemplate( report_template_id=report_template.report_template_id, From 7851c6c649d8b2e641e1781c47d716932c019cf5 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Sat, 23 Nov 2024 14:48:43 -0800 Subject: [PATCH 18/32] refactor: use client keys instead of full rules in example --- .../report_template_with_python_config/main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index ac5c598c..1f222313 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -27,13 +27,14 @@ # First create rules rule_service = RuleService(channel) rules = load_rules() # Load rules from python - [rule_service.create_or_update_rule(rule) for rule in rules] +# [rule_service.create_or_update_rule(rule) for rule in rules] # Now create report template report_template_service = ReportTemplateService(channel) report_template = nostromos_report_template() - report_template.rules = [rule.rule_client_key for rule in rules] # Add the rules we just created + report_template.rule_client_keys = [rule.rule_client_key for rule in rules if rule.rule_client_key] # Add the rules we just created report_template_service.create_or_update_report_template(report_template) + print(report_template) # Then make some updates to the template we created (for the sake of example) rules = [ @@ -42,6 +43,7 @@ # Get the report template (for the sake of example) report_template_to_update = report_template_service.get_report_template(client_key=report_template.template_client_key) if report_template_to_update: - report_template_to_update.rules = [rule.rule_client_key for rule in rules] + report_template_to_update.rule_client_keys = [rule.rule_client_key for rule in rules if rule.rule_client_key] report_template_to_update.description = "A report template for the Nostromo without overheating rule" + print(report_template_to_update) report_template_service.create_or_update_report_template(report_template_to_update) From d240dfeb9259cee4cc8e4f35df25785291c26452 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Sat, 23 Nov 2024 14:49:56 -0800 Subject: [PATCH 19/32] refactor: use client keys instead of full rules in example --- .../report_templates/report_template_with_python_config/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index 1f222313..52e51a5f 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -27,7 +27,7 @@ # First create rules rule_service = RuleService(channel) rules = load_rules() # Load rules from python -# [rule_service.create_or_update_rule(rule) for rule in rules] + [rule_service.create_or_update_rule(rule) for rule in rules] # Now create report template report_template_service = ReportTemplateService(channel) From 4d49593b4ec9a85acefbefdc72ba28e322559751 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Sat, 23 Nov 2024 14:50:24 -0800 Subject: [PATCH 20/32] chore: dev check --- .../main.py | 20 ++++++++++++------- .../sift_py/report_templates/_service_test.py | 15 +++++++------- python/lib/sift_py/report_templates/config.py | 3 --- .../lib/sift_py/report_templates/service.py | 15 +++++++++----- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index 52e51a5f..9f15d823 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -32,18 +32,24 @@ # Now create report template report_template_service = ReportTemplateService(channel) report_template = nostromos_report_template() - report_template.rule_client_keys = [rule.rule_client_key for rule in rules if rule.rule_client_key] # Add the rules we just created + report_template.rule_client_keys = [ + rule.rule_client_key for rule in rules if rule.rule_client_key + ] # Add the rules we just created report_template_service.create_or_update_report_template(report_template) print(report_template) # Then make some updates to the template we created (for the sake of example) - rules = [ - rule for rule in rules if rule.name != "overheating" - ] # Remove some rules + rules = [rule for rule in rules if rule.name != "overheating"] # Remove some rules # Get the report template (for the sake of example) - report_template_to_update = report_template_service.get_report_template(client_key=report_template.template_client_key) + report_template_to_update = report_template_service.get_report_template( + client_key=report_template.template_client_key + ) if report_template_to_update: - report_template_to_update.rule_client_keys = [rule.rule_client_key for rule in rules if rule.rule_client_key] - report_template_to_update.description = "A report template for the Nostromo without overheating rule" + report_template_to_update.rule_client_keys = [ + rule.rule_client_key for rule in rules if rule.rule_client_key + ] + report_template_to_update.description = ( + "A report template for the Nostromo without overheating rule" + ) print(report_template_to_update) report_template_service.create_or_update_report_template(report_template_to_update) diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py index 9c356027..c947c911 100644 --- a/python/lib/sift_py/report_templates/_service_test.py +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -1,12 +1,11 @@ from unittest import mock import pytest - from sift.report_templates.v1.report_templates_pb2 import ReportTemplate + from sift_py._internal.test_util.channel import MockChannel from sift_py.report_templates.config import ReportTemplateConfig from sift_py.report_templates.service import ReportTemplateService -from sift_py.rule.config import RuleConfig @pytest.fixture @@ -18,7 +17,9 @@ def test_report_template_service_get_report_template_by_client_key(report_templa report_template_client_key = "report-template-client-key" with mock.patch.object( - ReportTemplateService, "_get_report_template_by_client_key", return_value=ReportTemplate(name="abc") + ReportTemplateService, + "_get_report_template_by_client_key", + return_value=ReportTemplate(name="abc"), ) as mock_get_report_template_by_client_key: report_template_service.get_report_template(client_key=report_template_client_key) mock_get_report_template_by_client_key.assert_called_once_with(report_template_client_key) @@ -37,9 +38,7 @@ def test_report_template_service_get_report_template_by_id(report_template_servi def test_report_template_service_get_report_template_missing_client_key_and_id( report_template_service, ): - with pytest.raises( - ValueError, match="Either client_key or id must be provided" - ): + with pytest.raises(ValueError, match="Either client_key or id must be provided"): report_template_service.get_report_template() @@ -75,7 +74,9 @@ def test_report_template_service_update_report_template(report_template_service) ) as mock_get_report_template_by_client_key: mock_get_report_template_by_client_key.return_value = report_template_config report_template_service.create_or_update_report_template(report_template_config_update) - mock_update_report_template.assert_called_once_with(report_template_config_update, report_template_config) + mock_update_report_template.assert_called_once_with( + report_template_config_update, report_template_config + ) def test_report_template_service_missing_template_client_key(report_template_service): diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index d1512b11..3e07c60b 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -4,9 +4,6 @@ from pydantic import BaseModel, ConfigDict -from sift_py.ingestion.config.yaml.spec import RuleYamlSpec -from sift_py.ingestion.rule.config import RuleConfig - class ReportTemplateConfig(BaseModel): """ diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 44ba690e..53f34542 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -34,7 +34,9 @@ def create_or_update_report_template(self, config: ReportTemplateConfig): return self._create_report_template(config) - def get_report_template(self, client_key: str = "", id: str = "") -> Optional[ReportTemplateConfig]: + def get_report_template( + self, client_key: str = "", id: str = "" + ) -> Optional[ReportTemplateConfig]: report_template = None if not client_key and not id: raise ValueError("Either client_key or id must be provided") @@ -56,7 +58,6 @@ def get_report_template(self, client_key: str = "", id: str = "") -> Optional[Re rule_client_keys=[rule.client_key for rule in report_template.rules], ) - def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(report_template_id=report_template_id) try: @@ -78,7 +79,9 @@ def _get_report_template_by_client_key(self, client_key: str) -> Optional[Report return None def _create_report_template(self, config: ReportTemplateConfig): - client_keys_req = CreateReportTemplateRequestClientKeys(rule_client_keys=config.rule_client_keys) + client_keys_req = CreateReportTemplateRequestClientKeys( + rule_client_keys=config.rule_client_keys + ) req = CreateReportTemplateRequest( name=config.name, client_key=config.template_client_key, @@ -96,7 +99,9 @@ def _update_report_template( if config.tags: tags = [ReportTemplateTag(tag_name=tag) for tag in config.tags] - rules = [ReportTemplateRule(client_key=client_key) for client_key in config.rule_client_keys] + rules = [ + ReportTemplateRule(client_key=client_key) for client_key in config.rule_client_keys + ] updated_report_template = ReportTemplate( report_template_id=report_template.report_template_id, @@ -113,4 +118,4 @@ def _update_report_template( UpdateReportTemplateRequest( report_template=updated_report_template, update_mask=field_mask ) - ) \ No newline at end of file + ) From aee589b4d9a22fa322e967f345a6e50f7270cb08 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Sun, 24 Nov 2024 12:43:08 -0800 Subject: [PATCH 21/32] placeholder for yaml --- .../main.py | 2 - .../report_template_config.py | 2 +- .../report_template_with_yaml_config/main.py | 33 +++-- .../report_template_config.py | 124 ++++++++++++++++++ .../nostromo_report_template.yml | 50 ++----- .../rule_modules/nostromo.yml | 40 ++++++ .../rule_modules/velocity.yml | 12 -- .../rule_modules/voltage.yml | 12 -- .../lib/sift_py/ingestion/config/yaml/load.py | 66 ++++++++-- .../sift_py/report_templates/_config_test.py | 1 + python/lib/sift_py/report_templates/config.py | 3 - .../lib/sift_py/report_templates/service.py | 7 +- 12 files changed, 259 insertions(+), 93 deletions(-) create mode 100644 python/examples/report_templates/report_template_with_yaml_config/report_template_config.py create mode 100644 python/examples/report_templates/report_template_with_yaml_config/rule_modules/nostromo.yml delete mode 100644 python/examples/report_templates/report_template_with_yaml_config/rule_modules/velocity.yml delete mode 100644 python/examples/report_templates/report_template_with_yaml_config/rule_modules/voltage.yml diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index 9f15d823..f7a7721d 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -36,7 +36,6 @@ rule.rule_client_key for rule in rules if rule.rule_client_key ] # Add the rules we just created report_template_service.create_or_update_report_template(report_template) - print(report_template) # Then make some updates to the template we created (for the sake of example) rules = [rule for rule in rules if rule.name != "overheating"] # Remove some rules @@ -51,5 +50,4 @@ report_template_to_update.description = ( "A report template for the Nostromo without overheating rule" ) - print(report_template_to_update) report_template_service.create_or_update_report_template(report_template_to_update) diff --git a/python/examples/report_templates/report_template_with_python_config/report_template_config.py b/python/examples/report_templates/report_template_with_python_config/report_template_config.py index 505cb6dc..d6ea2b6b 100644 --- a/python/examples/report_templates/report_template_with_python_config/report_template_config.py +++ b/python/examples/report_templates/report_template_with_python_config/report_template_config.py @@ -118,5 +118,5 @@ def nostromos_report_template() -> ReportTemplateConfig: name="Nostromo Report Template", template_client_key="nostromo-report-template", description="A report template for the Nostromo", - rules=[], + rule_client_keys=[], ) diff --git a/python/examples/report_templates/report_template_with_yaml_config/main.py b/python/examples/report_templates/report_template_with_yaml_config/main.py index c0c07a38..b751187b 100644 --- a/python/examples/report_templates/report_template_with_yaml_config/main.py +++ b/python/examples/report_templates/report_template_with_yaml_config/main.py @@ -27,18 +27,31 @@ # First create rules rule_service = RuleService(channel) rules = load_rules() # Load rules from python - # [rule_service.create_or_update_rule(rule) for rule in rules] + # TODO: Load rules from YAML like agnostic example + # TODO: update namespaced rule definitions + [rule_service.create_or_update_rule(rule) for rule in rules] # Now create report template report_template_service = ReportTemplateService(channel) report_template = nostromos_report_template() - report_template.rules = rules # Add the rules we just created - report_template_service.create_or_update_report_template(report_template) - - # Then make some updates to the template we created (for the sake of the example) - rules = [ - rule for rule in report_template.rules if rule.name != "overheating" - ] # Remove some rules - report_template.rules = rules - report_template.description = "A report template for the Nostromo without overheating rule" + report_template.rule_client_keys = [ + rule.rule_client_key for rule in rules if rule.rule_client_key + ] # Add the rules we just created report_template_service.create_or_update_report_template(report_template) + print(report_template) + + # Then make some updates to the template we created (for the sake of example) + rules = [rule for rule in rules if rule.name != "overheating"] # Remove some rules + # Get the report template (for the sake of example) + report_template_to_update = report_template_service.get_report_template( + client_key=report_template.template_client_key + ) + if report_template_to_update: + report_template_to_update.rule_client_keys = [ + rule.rule_client_key for rule in rules if rule.rule_client_key + ] + report_template_to_update.description = ( + "A report template for the Nostromo without overheating rule" + ) + print(report_template_to_update) + report_template_service.create_or_update_report_template(report_template_to_update) diff --git a/python/examples/report_templates/report_template_with_yaml_config/report_template_config.py b/python/examples/report_templates/report_template_with_yaml_config/report_template_config.py new file mode 100644 index 00000000..b750ba51 --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/report_template_config.py @@ -0,0 +1,124 @@ +from pathlib import Path +from typing import List + +from sift_py.ingestion.channel import ( + ChannelConfig, + ChannelDataType, + ChannelEnumType, +) +from sift_py.ingestion.config.yaml.load import load_named_expression_modules +from sift_py.ingestion.rule.config import ( + RuleActionCreateDataReviewAnnotation, + RuleConfig, +) +from sift_py.report_templates.config import ReportTemplateConfig + +EXPRESSION_MODULES_DIR = Path().joinpath("expression_modules") + + +def load_rules_from_yaml() -> List[RuleConfig]: + # TODO: Update to load from YAML and load subexpressions + # TODO: Update namespace yamls to be like agnostic example? + named_expressions = load_named_expression_modules( + [ + EXPRESSION_MODULES_DIR.joinpath("kinematics.yml"), + EXPRESSION_MODULES_DIR.joinpath("string.yml"), + ] + ) + + log_channel = ChannelConfig( + name="log", + data_type=ChannelDataType.STRING, + description="asset logs", + ) + voltage_channel = ChannelConfig( + name="voltage", + data_type=ChannelDataType.INT_32, + description="voltage at source", + unit="Volts", + ) + vehicle_state_channel = ChannelConfig( + name="vehicle_state", + data_type=ChannelDataType.ENUM, + description="vehicle state", + enum_types=[ + ChannelEnumType(name="Accelerating", key=0), + ChannelEnumType(name="Decelerating", key=1), + ChannelEnumType(name="Stopped", key=2), + ], + ) + + rules = [ + RuleConfig( + name="overheating", + description="Checks for vehicle overheating", + expression='$1 == "Accelerating" && $2 > 80', + rule_client_key="overheating-rule", + asset_names=["NostromoLV2024"], + channel_references=[ + # INFO: Can use either "channel_identifier" or "channel_config" + { + "channel_reference": "$1", + "channel_identifier": vehicle_state_channel.fqn(), + }, + { + "channel_reference": "$2", + "channel_config": voltage_channel, + }, + ], + action=RuleActionCreateDataReviewAnnotation(), + ), + RuleConfig( + name="kinetic_energy", + description="Tracks high energy output while in motion", + expression=named_expressions["kinetic_energy_gt"], + rule_client_key="kinetic-energy-rule", + asset_names=["NostromoLV2024"], + channel_references=[ + { + "channel_reference": "$1", + "channel_config": voltage_channel, + }, + ], + sub_expressions={ + "$mass": 10, + "$threshold": 470, + }, + action=RuleActionCreateDataReviewAnnotation( + # User in your organization to notify + # assignee="ellen.ripley@weylandcorp.com", + tags=["nostromo"], + ), + ), + RuleConfig( + name="failure", + description="Checks for failures reported by logs", + expression=named_expressions["log_substring_contains"], + rule_client_key="failure-rule", + asset_names=["NostromoLV2024"], + channel_references=[ + { + "channel_reference": "$1", + "channel_config": log_channel, + }, + ], + sub_expressions={ + "$sub_string": "failure", + }, + action=RuleActionCreateDataReviewAnnotation( + # User in your organization to notify + # assignee="ellen.ripley@weylandcorp.com", + tags=["nostromo", "failure"], + ), + ), + ] + return rules + + +def nostromos_report_template() -> ReportTemplateConfig: + return ReportTemplateConfig( + name="Nostromo Report Template", + template_client_key="nostromo-report-template", + description="A report template for the Nostromo", + rule_client_keys=[], + ) diff --git a/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml b/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml index cf5ddfd1..cd98fb9e 100644 --- a/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml +++ b/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml @@ -1,45 +1,13 @@ report_templates: - - name: Nostromo + - name: Nostromo Report Template 1 description: A report tempate for the Nostromo template_client_key: nostromo-report-template - rules: - - name: overheating - description: Checks for vehicle overheating - expression: $1 == "Accelerating" && $2 > 80 - channel_references: - - $1: *vehicle_state_channel - - $2: *voltage_channel - type: review + rule_client_keys: + - kinetic-energy-rule + - failure-rule - - name: kinetic_energy - description: Tracks high energy output while in motion - type: review - # assignee: ellen.ripley@weylandcorp.com - expression: - name: kinetic_energy_gt - channel_references: - - $1: *velocity_channel - sub_expressions: - - $mass: 10 - - $threshold: 470 - tags: - - nostromo - - - name: failure - description: Checks for failures reported by logs - type: review - # assignee: ellen.ripley@weylandcorp.com - expression: - name: log_substring_contains - channel_references: - - $1: *log_channel - sub_expressions: - - $sub_string: failure - tags: - - failure - - nostromo - - - namespaces: - - velocity - - voltage + - name: Nostromo Report Template Overheating + description: A report tempate for the Nostromo + template_client_key: nostromo-report-template + rule_client_keys: + - overheating-rule diff --git a/python/examples/report_templates/report_template_with_yaml_config/rule_modules/nostromo.yml b/python/examples/report_templates/report_template_with_yaml_config/rule_modules/nostromo.yml new file mode 100644 index 00000000..d8b4a0b0 --- /dev/null +++ b/python/examples/report_templates/report_template_with_yaml_config/rule_modules/nostromo.yml @@ -0,0 +1,40 @@ +namespace: nostromo + +rules: + - name: overheating + description: Checks for vehicle overheating + expression: $1 == "Accelerating" && $2 > 80 + rule_client_key: overheating-rule + channel_references: + - $1: *vehicle_state_channel + - $2: *voltage_channel + type: review + + - name: kinetic_energy + description: Tracks high energy output while in motion + type: review + expression: + name: kinetic_energy_gt + rule_client_key: kinetic-energy-rule + channel_references: + - $1: *velocity_channel + sub_expressions: + - $mass: 10 + - $threshold: 470 + tags: + - nostromo + + - name: failure + description: Checks for failures reported by logs + type: review + rule_client_key: failure-rule + expression: + name: log_substring_contains + channel_references: + - $1: *log_channel + sub_expressions: + - $sub_string: failure + tags: + - failure + - nostromo + diff --git a/python/examples/report_templates/report_template_with_yaml_config/rule_modules/velocity.yml b/python/examples/report_templates/report_template_with_yaml_config/rule_modules/velocity.yml deleted file mode 100644 index 5505580a..00000000 --- a/python/examples/report_templates/report_template_with_yaml_config/rule_modules/velocity.yml +++ /dev/null @@ -1,12 +0,0 @@ -namespace: velocity - -rules: - - name: vehicle_stuck - description: Triggers if the vehicle velocity is not 0 for 5s after entering accelerating state - expression: $1 == "Accelerating" && persistence($2 == 0, 5) - type: review - - - name: vehicle_not_stopped - description: Triggers if the vehicle velocity does not remain 0 while stopped - expression: $1 == "Stopped" && $2 > 0 - type: review diff --git a/python/examples/report_templates/report_template_with_yaml_config/rule_modules/voltage.yml b/python/examples/report_templates/report_template_with_yaml_config/rule_modules/voltage.yml deleted file mode 100644 index b47e7216..00000000 --- a/python/examples/report_templates/report_template_with_yaml_config/rule_modules/voltage.yml +++ /dev/null @@ -1,12 +0,0 @@ -namespace: voltage - -rules: - - name: overvoltage - description: Checks for overvoltage while accelerating - expression: $1 == "Accelerating" && $2 > 80 - type: review - - - name: undervoltage - description: Checks for undervoltage while accelerating - expression: $1 == "Accelerating" && $2 < 40 - type: review diff --git a/python/lib/sift_py/ingestion/config/yaml/load.py b/python/lib/sift_py/ingestion/config/yaml/load.py index b8c2153f..258e1b72 100644 --- a/python/lib/sift_py/ingestion/config/yaml/load.py +++ b/python/lib/sift_py/ingestion/config/yaml/load.py @@ -1,7 +1,8 @@ import re from pathlib import Path -from typing import Any, Dict, List, Type, cast +from typing import Any, Callable, Dict, List, Type, cast +from sift_py.report_templates.config import ReportTemplateConfig import yaml from sift_py.ingestion.channel import ChannelDataTypeStrRep @@ -30,6 +31,24 @@ def read_and_validate(path: Path) -> TelemetryConfigYamlSpec: raw_config = _read_yaml(path) return _validate_yaml(raw_config) +def load_report_templates(paths: List[Path]) -> List[ReportTemplateConfig]: + """ + Takes in a list of paths to YAML files which contains report templates and processes them into a list of + `ReportTemplateConfig` objects. For more information on report templates see + `sift_py.report_templates.config.ReportTemplateConfig`. + """ + report_templates: List[ReportTemplateConfig] = [] + + def append_report_template(yaml_path: Path): + report_templates += _read_report_template_yaml(yaml_path) + + for path in paths: + if path.is_dir(): + _handle_subdir(path, append_report_template) + elif path.is_file(): + append_report_template(path) + return report_templates + def load_named_expression_modules(paths: List[Path]) -> Dict[str, str]: """ @@ -70,25 +89,26 @@ def update_rule_namespaces(rule_module_path: Path): raise YamlConfigError( f"Encountered rules with identical names being loaded, '{key}'." ) - rule_namespaces.update(rule_module) - def handle_dir(path: Path): - for file_in_dir in path.iterdir(): - if file_in_dir.is_dir(): - handle_dir(file_in_dir) - elif file_in_dir.is_file(): - update_rule_namespaces(file_in_dir) - for path in paths: if path.is_dir(): - handle_dir(path) + _handle_subdir(path, update_rule_namespaces) elif path.is_file(): update_rule_namespaces(path) return rule_namespaces +def _handle_subdir(path: Path, file_handler: Callable): + """The file_handler callable must accept a Path object as its only argument.""" + for file_in_dir in path.iterdir(): + if file_in_dir.is_dir(): + _handle_subdir(file_in_dir, file_handler) + elif file_in_dir.is_file(): + file_handler(file_in_dir) + + def _read_named_expression_module_yaml(path: Path) -> Dict[str, str]: with open(path, "r") as f: named_expressions = cast(Dict[Any, Any], yaml.safe_load(f.read())) @@ -106,6 +126,30 @@ def _read_named_expression_module_yaml(path: Path) -> Dict[str, str]: return cast(Dict[str, str], named_expressions) +def _read_report_template_yaml(path: Path) -> List[ReportTemplateConfig]: + report_templates = [] + with open(path, "r") as f: + report_templates_yaml = cast(Dict[str, Any], yaml.safe_load(f.read())) + + report_template_list = report_templates_yaml.get("report_templates") + if not isinstance(report_template_list, list): + raise YamlConfigError( + f"Expected 'report_templates' to be a list in report template yaml: '{path}'" + ) + + for report_template in report_template_list: + try: + report_template_config = ReportTemplateConfig(**report_template) + report_templates.append(report_template_config) + except Exception as e: + raise YamlConfigError( + f"Error parsing report template '{report_template}'" + ) from e + + return report_templates + + + def _read_rule_namespace_yaml(path: Path) -> Dict[str, List]: with open(path, "r") as f: namespace_rules = cast(Dict[Any, Any], yaml.safe_load(f.read())) @@ -118,7 +162,7 @@ def _read_rule_namespace_yaml(path: Path) -> Dict[str, List]: ) rules = namespace_rules.get("rules") - if not isinstance(namespace, str): + if not isinstance(rules, list): raise YamlConfigError( f"Expected '{rules}' to be a list in rule namespace yaml: '{path}'" f"{_type_fqn(RuleNamespaceYamlSpec)}" diff --git a/python/lib/sift_py/report_templates/_config_test.py b/python/lib/sift_py/report_templates/_config_test.py index 7e2425d2..50172879 100644 --- a/python/lib/sift_py/report_templates/_config_test.py +++ b/python/lib/sift_py/report_templates/_config_test.py @@ -20,6 +20,7 @@ def test_report_template_config(report_template_dict): assert report_template_config.tags is None assert report_template_config.description is None assert report_template_config.rule_client_keys == [] + assert report_template_config.namespaces is None def test_report_template_config_validation(report_template_dict): diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index 3e07c60b..ce74e838 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -28,6 +28,3 @@ class ReportTemplateConfig(BaseModel): def as_json(self) -> Any: return self.model_dump_json() - - def to_dict(self) -> Dict[str, Any]: - return self.model_dump() diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 53f34542..40a2fdff 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import Optional, cast +from pathlib import Path +from typing import List, Optional, cast from google.protobuf.field_mask_pb2 import FieldMask from sift.report_templates.v1.report_templates_pb2 import ( @@ -16,6 +17,7 @@ from sift.report_templates.v1.report_templates_pb2_grpc import ReportTemplateServiceStub from sift_py.grpc.transport import SiftChannel +from sift_py.ingestion.config.yaml.load import load_report_templates from sift_py.report_templates.config import ReportTemplateConfig @@ -58,6 +60,9 @@ def get_report_template( rule_client_keys=[rule.client_key for rule in report_template.rules], ) + def load_report_templates_from_yaml(self, paths: List[Path]) -> List[ReportTemplateConfig]: + return load_report_templates(paths) + def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(report_template_id=report_template_id) try: From 72409c5811b5149458cdb7395504d06d824effa4 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 25 Nov 2024 18:02:30 -0800 Subject: [PATCH 22/32] feat: Add YAML support and examples --- .../main.py | 5 +- .../report_template_with_yaml_config/main.py | 56 ++++---- .../report_template_config.py | 124 ------------------ .../nostromo_report_template.yml | 4 +- .../lib/sift_py/ingestion/config/yaml/load.py | 7 +- 5 files changed, 29 insertions(+), 167 deletions(-) delete mode 100644 python/examples/report_templates/report_template_with_yaml_config/report_template_config.py diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index f7a7721d..9c866ee6 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -7,9 +7,6 @@ from sift_py.report_templates.service import ReportTemplateService from sift_py.rule.service import RuleService -TELEMETRY_CONFIGS_DIR = Path().joinpath("telemetry_configs") -RULE_MODULES_DIR = Path().joinpath("rule_modules") - if __name__ == "__main__": load_dotenv() @@ -21,7 +18,7 @@ assert base_uri, "Missing 'BASE_URI' environment variable." # Create a gRPC transport channel configured specifically for the Sift API - sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey, use_ssl=False) + sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey) with use_sift_channel(sift_channel_config) as channel: # First create rules diff --git a/python/examples/report_templates/report_template_with_yaml_config/main.py b/python/examples/report_templates/report_template_with_yaml_config/main.py index b751187b..4ef0a626 100644 --- a/python/examples/report_templates/report_template_with_yaml_config/main.py +++ b/python/examples/report_templates/report_template_with_yaml_config/main.py @@ -2,14 +2,14 @@ from pathlib import Path from dotenv import load_dotenv -from report_template_config import load_rules, nostromos_report_template +from sift_py.ingestion.config.yaml.load import load_named_expression_modules, load_report_templates from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel from sift_py.report_templates.service import ReportTemplateService -from sift_py.rule.service import RuleService +from sift_py.rule.service import RuleService, SubExpression -TELEMETRY_CONFIGS_DIR = Path().joinpath("telemetry_configs") +REPORT_TEMPLATES_DIR = Path().joinpath("report_templates") RULE_MODULES_DIR = Path().joinpath("rule_modules") - +EXPRESSION_MODULES_DIR = Path().joinpath("expression_modules") if __name__ == "__main__": load_dotenv() @@ -21,37 +21,29 @@ assert base_uri, "Missing 'BASE_URI' environment variable." # Create a gRPC transport channel configured specifically for the Sift API - sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey, use_ssl=False) - + sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey) + + # Paths to your rules, named expressions, and report template + report_templates = REPORT_TEMPLATES_DIR.joinpath("nostromo_report_template.yml") + rule_modules = RULE_MODULES_DIR.joinpath("rules.yml") + named_expressions = load_named_expression_modules( + [ + EXPRESSION_MODULES_DIR.joinpath("kinematics.yml"), + EXPRESSION_MODULES_DIR.joinpath("string.yml"), + ] + ) with use_sift_channel(sift_channel_config) as channel: # First create rules rule_service = RuleService(channel) - rules = load_rules() # Load rules from python - # TODO: Load rules from YAML like agnostic example - # TODO: update namespaced rule definitions - [rule_service.create_or_update_rule(rule) for rule in rules] + rules = rule_service.load_rules_from_yaml( + paths=[rule_modules], + sub_expressions=[ + SubExpression("kinetic_energy", named_expressions), + SubExpression("failure", named_expressions), + ], + ) # Now create report template report_template_service = ReportTemplateService(channel) - report_template = nostromos_report_template() - report_template.rule_client_keys = [ - rule.rule_client_key for rule in rules if rule.rule_client_key - ] # Add the rules we just created - report_template_service.create_or_update_report_template(report_template) - print(report_template) - - # Then make some updates to the template we created (for the sake of example) - rules = [rule for rule in rules if rule.name != "overheating"] # Remove some rules - # Get the report template (for the sake of example) - report_template_to_update = report_template_service.get_report_template( - client_key=report_template.template_client_key - ) - if report_template_to_update: - report_template_to_update.rule_client_keys = [ - rule.rule_client_key for rule in rules if rule.rule_client_key - ] - report_template_to_update.description = ( - "A report template for the Nostromo without overheating rule" - ) - print(report_template_to_update) - report_template_service.create_or_update_report_template(report_template_to_update) + report_templates_loaded = load_report_templates([report_templates]) + [report_template_service.create_or_update_report_template(report_template) for report_template in report_templates_loaded] diff --git a/python/examples/report_templates/report_template_with_yaml_config/report_template_config.py b/python/examples/report_templates/report_template_with_yaml_config/report_template_config.py deleted file mode 100644 index b750ba51..00000000 --- a/python/examples/report_templates/report_template_with_yaml_config/report_template_config.py +++ /dev/null @@ -1,124 +0,0 @@ -from pathlib import Path -from typing import List - -from sift_py.ingestion.channel import ( - ChannelConfig, - ChannelDataType, - ChannelEnumType, -) -from sift_py.ingestion.config.yaml.load import load_named_expression_modules -from sift_py.ingestion.rule.config import ( - RuleActionCreateDataReviewAnnotation, - RuleConfig, -) -from sift_py.report_templates.config import ReportTemplateConfig - -EXPRESSION_MODULES_DIR = Path().joinpath("expression_modules") - - -def load_rules_from_yaml() -> List[RuleConfig]: - # TODO: Update to load from YAML and load subexpressions - # TODO: Update namespace yamls to be like agnostic example? - named_expressions = load_named_expression_modules( - [ - EXPRESSION_MODULES_DIR.joinpath("kinematics.yml"), - EXPRESSION_MODULES_DIR.joinpath("string.yml"), - ] - ) - - log_channel = ChannelConfig( - name="log", - data_type=ChannelDataType.STRING, - description="asset logs", - ) - voltage_channel = ChannelConfig( - name="voltage", - data_type=ChannelDataType.INT_32, - description="voltage at source", - unit="Volts", - ) - vehicle_state_channel = ChannelConfig( - name="vehicle_state", - data_type=ChannelDataType.ENUM, - description="vehicle state", - enum_types=[ - ChannelEnumType(name="Accelerating", key=0), - ChannelEnumType(name="Decelerating", key=1), - ChannelEnumType(name="Stopped", key=2), - ], - ) - - rules = [ - RuleConfig( - name="overheating", - description="Checks for vehicle overheating", - expression='$1 == "Accelerating" && $2 > 80', - rule_client_key="overheating-rule", - asset_names=["NostromoLV2024"], - channel_references=[ - # INFO: Can use either "channel_identifier" or "channel_config" - { - "channel_reference": "$1", - "channel_identifier": vehicle_state_channel.fqn(), - }, - { - "channel_reference": "$2", - "channel_config": voltage_channel, - }, - ], - action=RuleActionCreateDataReviewAnnotation(), - ), - RuleConfig( - name="kinetic_energy", - description="Tracks high energy output while in motion", - expression=named_expressions["kinetic_energy_gt"], - rule_client_key="kinetic-energy-rule", - asset_names=["NostromoLV2024"], - channel_references=[ - { - "channel_reference": "$1", - "channel_config": voltage_channel, - }, - ], - sub_expressions={ - "$mass": 10, - "$threshold": 470, - }, - action=RuleActionCreateDataReviewAnnotation( - # User in your organization to notify - # assignee="ellen.ripley@weylandcorp.com", - tags=["nostromo"], - ), - ), - RuleConfig( - name="failure", - description="Checks for failures reported by logs", - expression=named_expressions["log_substring_contains"], - rule_client_key="failure-rule", - asset_names=["NostromoLV2024"], - channel_references=[ - { - "channel_reference": "$1", - "channel_config": log_channel, - }, - ], - sub_expressions={ - "$sub_string": "failure", - }, - action=RuleActionCreateDataReviewAnnotation( - # User in your organization to notify - # assignee="ellen.ripley@weylandcorp.com", - tags=["nostromo", "failure"], - ), - ), - ] - return rules - - -def nostromos_report_template() -> ReportTemplateConfig: - return ReportTemplateConfig( - name="Nostromo Report Template", - template_client_key="nostromo-report-template", - description="A report template for the Nostromo", - rule_client_keys=[], - ) diff --git a/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml b/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml index cd98fb9e..accd8146 100644 --- a/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml +++ b/python/examples/report_templates/report_template_with_yaml_config/report_templates/nostromo_report_template.yml @@ -1,13 +1,13 @@ report_templates: - name: Nostromo Report Template 1 description: A report tempate for the Nostromo - template_client_key: nostromo-report-template + template_client_key: nostromo-report-template-1 rule_client_keys: - kinetic-energy-rule - failure-rule - name: Nostromo Report Template Overheating description: A report tempate for the Nostromo - template_client_key: nostromo-report-template + template_client_key: nostromo-report-template-overheating rule_client_keys: - overheating-rule diff --git a/python/lib/sift_py/ingestion/config/yaml/load.py b/python/lib/sift_py/ingestion/config/yaml/load.py index 258e1b72..e9a5091e 100644 --- a/python/lib/sift_py/ingestion/config/yaml/load.py +++ b/python/lib/sift_py/ingestion/config/yaml/load.py @@ -39,14 +39,11 @@ def load_report_templates(paths: List[Path]) -> List[ReportTemplateConfig]: """ report_templates: List[ReportTemplateConfig] = [] - def append_report_template(yaml_path: Path): - report_templates += _read_report_template_yaml(yaml_path) - for path in paths: if path.is_dir(): - _handle_subdir(path, append_report_template) + report_templates += _handle_subdir(path, _read_report_template_yaml) elif path.is_file(): - append_report_template(path) + report_templates += _read_report_template_yaml(path) return report_templates From bf4fa05d74b3139d33619c4100f8ad98ad2f446b Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 26 Nov 2024 11:33:51 -0800 Subject: [PATCH 23/32] chore: Cleanups --- .../main.py | 8 +- .../report_template_config.py | 4 - .../simulator.py | 169 ------------------ .../report_template_with_yaml_config/main.py | 8 +- .../lib/sift_py/ingestion/config/yaml/load.py | 15 +- python/lib/sift_py/report_templates/config.py | 2 +- .../lib/sift_py/report_templates/service.py | 9 +- 7 files changed, 22 insertions(+), 193 deletions(-) delete mode 100644 python/examples/report_templates/report_template_with_python_config/simulator.py diff --git a/python/examples/report_templates/report_template_with_python_config/main.py b/python/examples/report_templates/report_template_with_python_config/main.py index 9c866ee6..fde1e3ec 100644 --- a/python/examples/report_templates/report_template_with_python_config/main.py +++ b/python/examples/report_templates/report_template_with_python_config/main.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from dotenv import load_dotenv from report_template_config import load_rules, nostromos_report_template @@ -7,7 +6,6 @@ from sift_py.report_templates.service import ReportTemplateService from sift_py.rule.service import RuleService - if __name__ == "__main__": load_dotenv() @@ -17,7 +15,7 @@ base_uri = os.getenv("BASE_URI") assert base_uri, "Missing 'BASE_URI' environment variable." - # Create a gRPC transport channel configured specifically for the Sift API + # Create a gRPC transport channel for the Sift API sift_channel_config = SiftChannelConfig(uri=base_uri, apikey=apikey) with use_sift_channel(sift_channel_config) as channel: @@ -35,12 +33,12 @@ report_template_service.create_or_update_report_template(report_template) # Then make some updates to the template we created (for the sake of example) - rules = [rule for rule in rules if rule.name != "overheating"] # Remove some rules + rules = [rule for rule in rules if "overheating" not in rule.name] # Remove some rules # Get the report template (for the sake of example) report_template_to_update = report_template_service.get_report_template( client_key=report_template.template_client_key ) - if report_template_to_update: + if report_template_to_update: # Make some other changes report_template_to_update.rule_client_keys = [ rule.rule_client_key for rule in rules if rule.rule_client_key ] diff --git a/python/examples/report_templates/report_template_with_python_config/report_template_config.py b/python/examples/report_templates/report_template_with_python_config/report_template_config.py index d6ea2b6b..607f0d7b 100644 --- a/python/examples/report_templates/report_template_with_python_config/report_template_config.py +++ b/python/examples/report_templates/report_template_with_python_config/report_template_config.py @@ -83,8 +83,6 @@ def load_rules() -> List[RuleConfig]: "$threshold": 470, }, action=RuleActionCreateDataReviewAnnotation( - # User in your organization to notify - # assignee="ellen.ripley@weylandcorp.com", tags=["nostromo"], ), ), @@ -104,8 +102,6 @@ def load_rules() -> List[RuleConfig]: "$sub_string": "failure", }, action=RuleActionCreateDataReviewAnnotation( - # User in your organization to notify - # assignee="ellen.ripley@weylandcorp.com", tags=["nostromo", "failure"], ), ), diff --git a/python/examples/report_templates/report_template_with_python_config/simulator.py b/python/examples/report_templates/report_template_with_python_config/simulator.py deleted file mode 100644 index ccd5dc5c..00000000 --- a/python/examples/report_templates/report_template_with_python_config/simulator.py +++ /dev/null @@ -1,169 +0,0 @@ -import logging -import random -import time -from datetime import datetime, timezone -from pathlib import Path -from typing import List - -from sift_py.ingestion.channel import ( - bit_field_value, - double_value, - enum_value, - int32_value, - string_value, -) -from sift_py.ingestion.service import IngestionService - -READINGS_FREQUENCY_HZ = 1.5 -LOGS_FREQUENCY_HZ = 2 -PARTIAL_READINGS_WITH_LOG_FREQUENCY_HZ = 0.5 - - -class Simulator: - """ - Telemeters sample data for 60 seconds for various combinations of flows - at various frequencies. - """ - - sample_bit_field_values: List[bytes] - sample_logs: List[str] - ingestion_service: IngestionService - logger: logging.Logger - - def __init__(self, ingestion_service: IngestionService): - self.ingestion_service = ingestion_service - - logging.basicConfig(level=logging.DEBUG) - self.logger = logging.getLogger(__name__) - - sample_bit_field_values = ["00001001", "00100011", "00001101", "11000001"] - self.sample_bit_field_values = [bytes([int(byte, 2)]) for byte in sample_bit_field_values] - - sample_logs = Path().joinpath("sample_data").joinpath("sample_logs.txt") - - with open(sample_logs, "r") as file: - self.sample_logs = file.readlines() - - def run(self): - """ - Send data for different combination of flows at different frequencies. - """ - - asset_name = self.ingestion_service.asset_name - run_id = self.ingestion_service.run_id - - if run_id is not None: - self.logger.info(f"Beginning simulation for '{asset_name}' with run ({run_id})") - else: - self.logger.info(f"Beginning simulation for '{asset_name}'") - - start_time = time.time() - end_time = start_time + 60 - - last_reading_time = start_time - last_log_time = start_time - last_partial_readings_time = start_time - - readings_interval_s = 1 / READINGS_FREQUENCY_HZ - logs_interval_s = 1 / LOGS_FREQUENCY_HZ - partial_readings_with_log_interval_s = 1 / PARTIAL_READINGS_WITH_LOG_FREQUENCY_HZ - - # Buffer size configured to 1 for the purposes of showing data come in live for - # this low ingestion-rate stream. - with self.ingestion_service.buffered_ingestion(buffer_size=1) as buffered_ingestion: - while time.time() < end_time: - current_time = time.time() - - # Send date for readings flow - if current_time - last_reading_time >= readings_interval_s: - timestamp = datetime.now(timezone.utc) - - buffered_ingestion.try_ingest_flows( - { - "flow_name": "readings", - "timestamp": timestamp, - "channel_values": [ - { - "channel_name": "velocity", - "component": "mainmotor", - "value": double_value(random.randint(1, 10)), - }, - { - "channel_name": "voltage", - "value": int32_value(random.randint(1, 10)), - }, - { - "channel_name": "vehicle_state", - "value": enum_value(random.randint(0, 2)), - }, - { - "channel_name": "gpio", - "value": bit_field_value( - random.choice(self.sample_bit_field_values) - ), - }, - ], - } - ) - logging.info(f"{timestamp} Emitted data for 'readings' flow") - last_reading_time = current_time - - # Send date for logs flow - if current_time - last_log_time >= logs_interval_s: - timestamp = datetime.now(timezone.utc) - - buffered_ingestion.try_ingest_flows( - { - "flow_name": "logs", - "timestamp": timestamp, - "channel_values": [ - { - "channel_name": "log", - "value": string_value(random.choice(self.sample_logs).strip()), - }, - ], - } - ) - logging.info(f"{timestamp} Emitted data for 'logs' flow") - last_log_time = current_time - - # Send partial data for readings flow and full data for logs flow - if ( - current_time - last_partial_readings_time - >= partial_readings_with_log_interval_s - ): - timestamp = datetime.now(timezone.utc) - - buffered_ingestion.try_ingest_flows( - { - "flow_name": "readings", - "timestamp": timestamp, - "channel_values": [ - { - "channel_name": "velocity", - "component": "mainmotor", - "value": double_value(random.randint(1, 10)), - }, - { - "channel_name": "voltage", - "value": int32_value(random.randint(1, 10)), - }, - ], - }, - { - "flow_name": "logs", - "timestamp": timestamp, - "channel_values": [ - { - "channel_name": "log", - "value": string_value(random.choice(self.sample_logs).strip()), - }, - ], - }, - ) - logging.info( - f"{timestamp} Emitted log for 'logs' flow and partial data for 'readings' flow" - ) - last_partial_readings_time = current_time - - self.logger.info("Completed simulation.") diff --git a/python/examples/report_templates/report_template_with_yaml_config/main.py b/python/examples/report_templates/report_template_with_yaml_config/main.py index 4ef0a626..6d2c19c8 100644 --- a/python/examples/report_templates/report_template_with_yaml_config/main.py +++ b/python/examples/report_templates/report_template_with_yaml_config/main.py @@ -2,8 +2,8 @@ from pathlib import Path from dotenv import load_dotenv -from sift_py.ingestion.config.yaml.load import load_named_expression_modules, load_report_templates from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel +from sift_py.ingestion.config.yaml.load import load_named_expression_modules, load_report_templates from sift_py.report_templates.service import ReportTemplateService from sift_py.rule.service import RuleService, SubExpression @@ -32,6 +32,7 @@ EXPRESSION_MODULES_DIR.joinpath("string.yml"), ] ) + with use_sift_channel(sift_channel_config) as channel: # First create rules rule_service = RuleService(channel) @@ -43,7 +44,6 @@ ], ) - # Now create report template + # Now create report templates report_template_service = ReportTemplateService(channel) - report_templates_loaded = load_report_templates([report_templates]) - [report_template_service.create_or_update_report_template(report_template) for report_template in report_templates_loaded] + report_template_service.load_report_templates_from_yaml([report_templates]) diff --git a/python/lib/sift_py/ingestion/config/yaml/load.py b/python/lib/sift_py/ingestion/config/yaml/load.py index e9a5091e..744790dd 100644 --- a/python/lib/sift_py/ingestion/config/yaml/load.py +++ b/python/lib/sift_py/ingestion/config/yaml/load.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Type, cast -from sift_py.report_templates.config import ReportTemplateConfig import yaml from sift_py.ingestion.channel import ChannelDataTypeStrRep @@ -17,6 +16,7 @@ TelemetryConfigYamlSpec, ) from sift_py.ingestion.rule.config import RuleActionAnnotationKind +from sift_py.report_templates.config import ReportTemplateConfig _CHANNEL_REFERENCE_REGEX = re.compile(r"^\$\d+$") _SUB_EXPRESSION_REGEX = re.compile(r"^\$[a-zA-Z_]+$") @@ -31,6 +31,7 @@ def read_and_validate(path: Path) -> TelemetryConfigYamlSpec: raw_config = _read_yaml(path) return _validate_yaml(raw_config) + def load_report_templates(paths: List[Path]) -> List[ReportTemplateConfig]: """ Takes in a list of paths to YAML files which contains report templates and processes them into a list of @@ -39,11 +40,14 @@ def load_report_templates(paths: List[Path]) -> List[ReportTemplateConfig]: """ report_templates: List[ReportTemplateConfig] = [] + def update_report_templates(path: Path): + report_templates.extend(_read_report_template_yaml(path)) + for path in paths: if path.is_dir(): - report_templates += _handle_subdir(path, _read_report_template_yaml) + _handle_subdir(path, update_report_templates) elif path.is_file(): - report_templates += _read_report_template_yaml(path) + update_report_templates(path) return report_templates @@ -139,14 +143,11 @@ def _read_report_template_yaml(path: Path) -> List[ReportTemplateConfig]: report_template_config = ReportTemplateConfig(**report_template) report_templates.append(report_template_config) except Exception as e: - raise YamlConfigError( - f"Error parsing report template '{report_template}'" - ) from e + raise YamlConfigError(f"Error parsing report template '{report_template}'") from e return report_templates - def _read_rule_namespace_yaml(path: Path) -> Dict[str, List]: with open(path, "r") as f: namespace_rules = cast(Dict[Any, Any], yaml.safe_load(f.read())) diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index ce74e838..a3996deb 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional +from typing import Any, List, Optional from pydantic import BaseModel, ConfigDict diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 40a2fdff..079385a9 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -30,11 +30,12 @@ def __init__(self, channel: SiftChannel): def create_or_update_report_template(self, config: ReportTemplateConfig): if not config.template_client_key: raise Exception(f"Report template {config.name} requires a template_client_key") + report_template = self._get_report_template_by_client_key(config.template_client_key) if report_template: self._update_report_template(config, report_template) - return - self._create_report_template(config) + else: + self._create_report_template(config) def get_report_template( self, client_key: str = "", id: str = "" @@ -61,7 +62,9 @@ def get_report_template( ) def load_report_templates_from_yaml(self, paths: List[Path]) -> List[ReportTemplateConfig]: - return load_report_templates(paths) + report_templates = load_report_templates(paths) + [self.create_or_update_report_template(report_template) for report_template in report_templates] + return report_templates def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: req = GetReportTemplateRequest(report_template_id=report_template_id) From 1c2c81f1fad4f69d4c3635467ff1824c4b3c3535 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 26 Nov 2024 11:37:06 -0800 Subject: [PATCH 24/32] chore: lint --- .../report_template_with_yaml_config/main.py | 2 +- python/lib/sift_py/report_templates/service.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python/examples/report_templates/report_template_with_yaml_config/main.py b/python/examples/report_templates/report_template_with_yaml_config/main.py index 6d2c19c8..0e86f22e 100644 --- a/python/examples/report_templates/report_template_with_yaml_config/main.py +++ b/python/examples/report_templates/report_template_with_yaml_config/main.py @@ -3,7 +3,7 @@ from dotenv import load_dotenv from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel -from sift_py.ingestion.config.yaml.load import load_named_expression_modules, load_report_templates +from sift_py.ingestion.config.yaml.load import load_named_expression_modules from sift_py.report_templates.service import ReportTemplateService from sift_py.rule.service import RuleService, SubExpression diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 079385a9..1a5bd0ad 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -63,7 +63,10 @@ def get_report_template( def load_report_templates_from_yaml(self, paths: List[Path]) -> List[ReportTemplateConfig]: report_templates = load_report_templates(paths) - [self.create_or_update_report_template(report_template) for report_template in report_templates] + [ + self.create_or_update_report_template(report_template) + for report_template in report_templates + ] return report_templates def _get_report_template_by_id(self, report_template_id: str) -> Optional[ReportTemplate]: From b26756a7888fb08701ec418e97ca8fa4b64e8c10 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 26 Nov 2024 11:41:07 -0800 Subject: [PATCH 25/32] test: Fix config refactor --- python/lib/sift_py/report_templates/_config_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/lib/sift_py/report_templates/_config_test.py b/python/lib/sift_py/report_templates/_config_test.py index 50172879..7e2425d2 100644 --- a/python/lib/sift_py/report_templates/_config_test.py +++ b/python/lib/sift_py/report_templates/_config_test.py @@ -20,7 +20,6 @@ def test_report_template_config(report_template_dict): assert report_template_config.tags is None assert report_template_config.description is None assert report_template_config.rule_client_keys == [] - assert report_template_config.namespaces is None def test_report_template_config_validation(report_template_dict): From f27306570571fba13639defb47bb96c82c86dc02 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 26 Nov 2024 15:16:19 -0800 Subject: [PATCH 26/32] feat: Add archived_date --- .../report_template_config.py | 2 +- .../report_template_with_yaml_config/main.py | 7 +++++++ python/lib/sift_py/report_templates/_config_test.py | 1 + python/lib/sift_py/report_templates/config.py | 4 ++++ python/lib/sift_py/report_templates/service.py | 8 +++++++- 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/python/examples/report_templates/report_template_with_python_config/report_template_config.py b/python/examples/report_templates/report_template_with_python_config/report_template_config.py index 607f0d7b..a7d24925 100644 --- a/python/examples/report_templates/report_template_with_python_config/report_template_config.py +++ b/python/examples/report_templates/report_template_with_python_config/report_template_config.py @@ -112,7 +112,7 @@ def load_rules() -> List[RuleConfig]: def nostromos_report_template() -> ReportTemplateConfig: return ReportTemplateConfig( name="Nostromo Report Template", - template_client_key="nostromo-report-template", + template_client_key="nostromo-report-template-test", description="A report template for the Nostromo", rule_client_keys=[], ) diff --git a/python/examples/report_templates/report_template_with_yaml_config/main.py b/python/examples/report_templates/report_template_with_yaml_config/main.py index 0e86f22e..6425271d 100644 --- a/python/examples/report_templates/report_template_with_yaml_config/main.py +++ b/python/examples/report_templates/report_template_with_yaml_config/main.py @@ -1,3 +1,4 @@ +from datetime import datetime import os from pathlib import Path @@ -47,3 +48,9 @@ # Now create report templates report_template_service = ReportTemplateService(channel) report_template_service.load_report_templates_from_yaml([report_templates]) + + # Archive one template, for the sake of example + report_template_to_update = report_template_service.get_report_template(client_key="nostromo-report-template-1") + if report_template_to_update: + report_template_to_update.archived_date = datetime.now() + report_template_service.create_or_update_report_template(report_template_to_update) diff --git a/python/lib/sift_py/report_templates/_config_test.py b/python/lib/sift_py/report_templates/_config_test.py index 7e2425d2..f76dea9e 100644 --- a/python/lib/sift_py/report_templates/_config_test.py +++ b/python/lib/sift_py/report_templates/_config_test.py @@ -20,6 +20,7 @@ def test_report_template_config(report_template_dict): assert report_template_config.tags is None assert report_template_config.description is None assert report_template_config.rule_client_keys == [] + assert report_template_config.archived_date is None def test_report_template_config_validation(report_template_dict): diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index a3996deb..1fdb9ed2 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime from typing import Any, List, Optional from pydantic import BaseModel, ConfigDict @@ -15,6 +16,8 @@ class ReportTemplateConfig(BaseModel): - `tags`: Tags to associate with the report template. - `description`: Description of the report template. - `rule_client_keys`: List of rule client keys associated with the report template. + - `archived_date`: Date when the report template was archived. Setting this field + will archive the report template, and unsetting it will unarchive the report template. """ model_config = ConfigDict(arbitrary_types_allowed=True) @@ -25,6 +28,7 @@ class ReportTemplateConfig(BaseModel): tags: Optional[List[str]] = None description: Optional[str] = None rule_client_keys: List[str] = [] + archived_date: Optional[datetime] = None def as_json(self) -> Any: return self.model_dump_json() diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 1a5bd0ad..7c99d01b 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -16,6 +16,7 @@ ) from sift.report_templates.v1.report_templates_pb2_grpc import ReportTemplateServiceStub +from sift_py._internal.time import to_timestamp_pb from sift_py.grpc.transport import SiftChannel from sift_py.ingestion.config.yaml.load import load_report_templates from sift_py.report_templates.config import ReportTemplateConfig @@ -110,6 +111,10 @@ def _update_report_template( if config.tags: tags = [ReportTemplateTag(tag_name=tag) for tag in config.tags] + archived_date = None + if config.archived_date: + archived_date = to_timestamp_pb(config.archived_date) + rules = [ ReportTemplateRule(client_key=client_key) for client_key in config.rule_client_keys ] @@ -122,9 +127,10 @@ def _update_report_template( description=config.description, tags=tags, rules=rules, + archived_date=archived_date, ) - field_mask = FieldMask(paths=["name", "description", "tags", "rules"]) + field_mask = FieldMask(paths=["name", "description", "tags", "rules", "archived_date"]) self._report_template_service_stub.UpdateReportTemplate( UpdateReportTemplateRequest( report_template=updated_report_template, update_mask=field_mask From 739478ec1792fe18ea85688ec6527ebdf17a24d7 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 26 Nov 2024 15:16:47 -0800 Subject: [PATCH 27/32] chore: fmt --- .../report_template_with_yaml_config/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/examples/report_templates/report_template_with_yaml_config/main.py b/python/examples/report_templates/report_template_with_yaml_config/main.py index 6425271d..172ada0c 100644 --- a/python/examples/report_templates/report_template_with_yaml_config/main.py +++ b/python/examples/report_templates/report_template_with_yaml_config/main.py @@ -1,5 +1,5 @@ -from datetime import datetime import os +from datetime import datetime from pathlib import Path from dotenv import load_dotenv @@ -50,7 +50,9 @@ report_template_service.load_report_templates_from_yaml([report_templates]) # Archive one template, for the sake of example - report_template_to_update = report_template_service.get_report_template(client_key="nostromo-report-template-1") + report_template_to_update = report_template_service.get_report_template( + client_key="nostromo-report-template-1" + ) if report_template_to_update: report_template_to_update.archived_date = datetime.now() report_template_service.create_or_update_report_template(report_template_to_update) From a8a060e79501ef61e0c1dd2f082190afa62c8f33 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 26 Nov 2024 15:40:27 -0800 Subject: [PATCH 28/32] fix: Include archived_date when fetching report templates --- python/lib/sift_py/report_templates/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index 7c99d01b..e192d4f2 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -16,7 +16,7 @@ ) from sift.report_templates.v1.report_templates_pb2_grpc import ReportTemplateServiceStub -from sift_py._internal.time import to_timestamp_pb +from sift_py._internal.time import to_timestamp_nanos, to_timestamp_pb from sift_py.grpc.transport import SiftChannel from sift_py.ingestion.config.yaml.load import load_report_templates from sift_py.report_templates.config import ReportTemplateConfig @@ -60,6 +60,7 @@ def get_report_template( tags=[tag.tag_name for tag in report_template.tags], description=report_template.description, rule_client_keys=[rule.client_key for rule in report_template.rules], + archived_date=to_timestamp_nanos(report_template.archived_date).to_pydatetime(), ) def load_report_templates_from_yaml(self, paths: List[Path]) -> List[ReportTemplateConfig]: From 570db75e9d9b11d8453e0a727994f0203f1cef64 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 26 Nov 2024 16:03:41 -0800 Subject: [PATCH 29/32] doc: Report template yaml spec --- .../lib/sift_py/ingestion/config/yaml/spec.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/python/lib/sift_py/ingestion/config/yaml/spec.py b/python/lib/sift_py/ingestion/config/yaml/spec.py index 698331e3..c45f4f67 100644 --- a/python/lib/sift_py/ingestion/config/yaml/spec.py +++ b/python/lib/sift_py/ingestion/config/yaml/spec.py @@ -4,6 +4,7 @@ from __future__ import annotations +from datetime import datetime from typing import Dict, List, Literal, Union from typing_extensions import NotRequired, TypedDict @@ -90,6 +91,28 @@ class FlowYamlSpec(TypedDict): channels: List[ChannelConfigYamlSpec] +class ReportTemplateYamlSpec(TypedDict): + """ + Formal spec for a report template. + + `name`: Name of the report template. + `template_client_key`: Unique client key to identify the report template. + `organization_id`: Organization ID that the report template belongs to. + `tags`: Tags to associate with the report template. + `description`: Description of the report template. + `rule_client_keys`: List of rule client keys associated with the report template. + `archived_date`: Date when the report template was archived. Setting this field + will archive the report template, and unsetting it will unarchive the report template. + """ + name: str + template_client_key: str + organization_id: str + tags: NotRequired[List[str]] + description: NotRequired[str] + rule_client_keys: List[str] + archived_date: NotRequired[datetime] + + class RuleNamespaceYamlSpec(TypedDict): """ The formal definition of what a rule namespace looks like in YAML. From 16cbd1b82243484175799023c7e4cb26858221fe Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Tue, 26 Nov 2024 16:04:20 -0800 Subject: [PATCH 30/32] chore: fmt --- python/lib/sift_py/ingestion/config/yaml/spec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/lib/sift_py/ingestion/config/yaml/spec.py b/python/lib/sift_py/ingestion/config/yaml/spec.py index c45f4f67..a75903dd 100644 --- a/python/lib/sift_py/ingestion/config/yaml/spec.py +++ b/python/lib/sift_py/ingestion/config/yaml/spec.py @@ -104,6 +104,7 @@ class ReportTemplateYamlSpec(TypedDict): `archived_date`: Date when the report template was archived. Setting this field will archive the report template, and unsetting it will unarchive the report template. """ + name: str template_client_key: str organization_id: str From 2820bec25c299576358f58434162e25cd1d6127f Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 2 Dec 2024 11:21:58 -0800 Subject: [PATCH 31/32] chore: PR feedback --- .../lib/sift_py/ingestion/config/yaml/load.py | 41 ----------- .../lib/sift_py/ingestion/config/yaml/spec.py | 24 ------ .../sift_py/report_templates/_service_test.py | 6 +- python/lib/sift_py/report_templates/config.py | 4 +- .../lib/sift_py/report_templates/service.py | 64 ++++++++++++---- python/lib/sift_py/yaml/__init__.py | 0 python/lib/sift_py/yaml/report_templates.py | 73 +++++++++++++++++++ 7 files changed, 128 insertions(+), 84 deletions(-) create mode 100644 python/lib/sift_py/yaml/__init__.py create mode 100644 python/lib/sift_py/yaml/report_templates.py diff --git a/python/lib/sift_py/ingestion/config/yaml/load.py b/python/lib/sift_py/ingestion/config/yaml/load.py index 744790dd..bb600f63 100644 --- a/python/lib/sift_py/ingestion/config/yaml/load.py +++ b/python/lib/sift_py/ingestion/config/yaml/load.py @@ -16,7 +16,6 @@ TelemetryConfigYamlSpec, ) from sift_py.ingestion.rule.config import RuleActionAnnotationKind -from sift_py.report_templates.config import ReportTemplateConfig _CHANNEL_REFERENCE_REGEX = re.compile(r"^\$\d+$") _SUB_EXPRESSION_REGEX = re.compile(r"^\$[a-zA-Z_]+$") @@ -32,25 +31,6 @@ def read_and_validate(path: Path) -> TelemetryConfigYamlSpec: return _validate_yaml(raw_config) -def load_report_templates(paths: List[Path]) -> List[ReportTemplateConfig]: - """ - Takes in a list of paths to YAML files which contains report templates and processes them into a list of - `ReportTemplateConfig` objects. For more information on report templates see - `sift_py.report_templates.config.ReportTemplateConfig`. - """ - report_templates: List[ReportTemplateConfig] = [] - - def update_report_templates(path: Path): - report_templates.extend(_read_report_template_yaml(path)) - - for path in paths: - if path.is_dir(): - _handle_subdir(path, update_report_templates) - elif path.is_file(): - update_report_templates(path) - return report_templates - - def load_named_expression_modules(paths: List[Path]) -> Dict[str, str]: """ Takes in a list of paths to YAML files which contains named expressions and processes them into a `dict`. @@ -127,27 +107,6 @@ def _read_named_expression_module_yaml(path: Path) -> Dict[str, str]: return cast(Dict[str, str], named_expressions) -def _read_report_template_yaml(path: Path) -> List[ReportTemplateConfig]: - report_templates = [] - with open(path, "r") as f: - report_templates_yaml = cast(Dict[str, Any], yaml.safe_load(f.read())) - - report_template_list = report_templates_yaml.get("report_templates") - if not isinstance(report_template_list, list): - raise YamlConfigError( - f"Expected 'report_templates' to be a list in report template yaml: '{path}'" - ) - - for report_template in report_template_list: - try: - report_template_config = ReportTemplateConfig(**report_template) - report_templates.append(report_template_config) - except Exception as e: - raise YamlConfigError(f"Error parsing report template '{report_template}'") from e - - return report_templates - - def _read_rule_namespace_yaml(path: Path) -> Dict[str, List]: with open(path, "r") as f: namespace_rules = cast(Dict[Any, Any], yaml.safe_load(f.read())) diff --git a/python/lib/sift_py/ingestion/config/yaml/spec.py b/python/lib/sift_py/ingestion/config/yaml/spec.py index a75903dd..698331e3 100644 --- a/python/lib/sift_py/ingestion/config/yaml/spec.py +++ b/python/lib/sift_py/ingestion/config/yaml/spec.py @@ -4,7 +4,6 @@ from __future__ import annotations -from datetime import datetime from typing import Dict, List, Literal, Union from typing_extensions import NotRequired, TypedDict @@ -91,29 +90,6 @@ class FlowYamlSpec(TypedDict): channels: List[ChannelConfigYamlSpec] -class ReportTemplateYamlSpec(TypedDict): - """ - Formal spec for a report template. - - `name`: Name of the report template. - `template_client_key`: Unique client key to identify the report template. - `organization_id`: Organization ID that the report template belongs to. - `tags`: Tags to associate with the report template. - `description`: Description of the report template. - `rule_client_keys`: List of rule client keys associated with the report template. - `archived_date`: Date when the report template was archived. Setting this field - will archive the report template, and unsetting it will unarchive the report template. - """ - - name: str - template_client_key: str - organization_id: str - tags: NotRequired[List[str]] - description: NotRequired[str] - rule_client_keys: List[str] - archived_date: NotRequired[datetime] - - class RuleNamespaceYamlSpec(TypedDict): """ The formal definition of what a rule namespace looks like in YAML. diff --git a/python/lib/sift_py/report_templates/_service_test.py b/python/lib/sift_py/report_templates/_service_test.py index c947c911..d695b05c 100644 --- a/python/lib/sift_py/report_templates/_service_test.py +++ b/python/lib/sift_py/report_templates/_service_test.py @@ -35,11 +35,13 @@ def test_report_template_service_get_report_template_by_id(report_template_servi mock_get_report_template_by_id.assert_called_once_with(report_template_id) -def test_report_template_service_get_report_template_missing_client_key_and_id( +def test_report_template_service_get_report_template_missing_or_both_client_key_and_id( report_template_service, ): - with pytest.raises(ValueError, match="Either client_key or id must be provided"): + with pytest.raises(ValueError, match="One of client_key or id must be provided"): report_template_service.get_report_template() + with pytest.raises(ValueError, match="One of client_key or id must be provided"): + report_template_service.get_report_template(client_key="abc", id="abc") def test_report_template_service_create_report_template(report_template_service): diff --git a/python/lib/sift_py/report_templates/config.py b/python/lib/sift_py/report_templates/config.py index 1fdb9ed2..30288488 100644 --- a/python/lib/sift_py/report_templates/config.py +++ b/python/lib/sift_py/report_templates/config.py @@ -5,8 +5,10 @@ from pydantic import BaseModel, ConfigDict +from sift_py._internal.convert.json import AsJson -class ReportTemplateConfig(BaseModel): + +class ReportTemplateConfig(BaseModel, AsJson): """ Configuration for a report template. diff --git a/python/lib/sift_py/report_templates/service.py b/python/lib/sift_py/report_templates/service.py index e192d4f2..43fa7169 100644 --- a/python/lib/sift_py/report_templates/service.py +++ b/python/lib/sift_py/report_templates/service.py @@ -16,19 +16,31 @@ ) from sift.report_templates.v1.report_templates_pb2_grpc import ReportTemplateServiceStub -from sift_py._internal.time import to_timestamp_nanos, to_timestamp_pb +from sift_py._internal.time import to_timestamp_pb from sift_py.grpc.transport import SiftChannel -from sift_py.ingestion.config.yaml.load import load_report_templates from sift_py.report_templates.config import ReportTemplateConfig +from sift_py.yaml.report_templates import load_report_templates class ReportTemplateService: + """ + A service for managing report templates. Allows for creating, updating, and retrieving report + templates. + """ + _report_template_service_stub: ReportTemplateServiceStub def __init__(self, channel: SiftChannel): self._report_template_service_stub = ReportTemplateServiceStub(channel) def create_or_update_report_template(self, config: ReportTemplateConfig): + """ + Create or update a report template via a ReportTemplateConfig. The config must contain a + template_client_key, otherwise an exception will be raised. If a report template with the + same client key exists, it will be updated. Otherwise, a new report template will be created. + See `sift_py.report_templates.config.ReportTemplateConfig` for more information on available + fields to configure. + """ if not config.template_client_key: raise Exception(f"Report template {config.name} requires a template_client_key") @@ -39,31 +51,46 @@ def create_or_update_report_template(self, config: ReportTemplateConfig): self._create_report_template(config) def get_report_template( - self, client_key: str = "", id: str = "" + self, client_key: Optional[str] = None, id: Optional[str] = None ) -> Optional[ReportTemplateConfig]: + """ + Retrieve a report template by client key or id. Only one of client_key or id should be + provided, otherwise an exception will be raised. If a report template is found, it will be + returned as a ReportTemplateConfig object. + """ report_template = None - if not client_key and not id: - raise ValueError("Either client_key or id must be provided") + if (not client_key and not id) or (client_key and id): + raise ValueError("One of client_key or id must be provided") if id: report_template = self._get_report_template_by_id(id) elif client_key: report_template = self._get_report_template_by_client_key(client_key) - if not report_template: - raise Exception(f"Report template with client key {client_key} or id {id} not found.") - - return ReportTemplateConfig( - name=report_template.name, - template_client_key=report_template.client_key, - organization_id=report_template.organization_id, - tags=[tag.tag_name for tag in report_template.tags], - description=report_template.description, - rule_client_keys=[rule.client_key for rule in report_template.rules], - archived_date=to_timestamp_nanos(report_template.archived_date).to_pydatetime(), + return ( + ReportTemplateConfig( + name=report_template.name, + template_client_key=report_template.client_key, + organization_id=report_template.organization_id, + tags=[tag.tag_name for tag in report_template.tags], + description=report_template.description, + rule_client_keys=[rule.client_key for rule in report_template.rules], + archived_date=report_template.archived_date.ToDatetime() + if report_template.archived_date + else None, + ) + if report_template + else None ) def load_report_templates_from_yaml(self, paths: List[Path]) -> List[ReportTemplateConfig]: + """ + Load report templates from YAML definitions. The YAML defined report template must have + a client key. If the report template with the given client key exists, it will be updated, + otherwise a new report template will be created. + See `sift_py.yaml.report_templates.load_report_templates` for more information on the YAML + spec for report templates. + """ report_templates = load_report_templates(paths) [ self.create_or_update_report_template(report_template) @@ -108,6 +135,11 @@ def _create_report_template(self, config: ReportTemplateConfig): def _update_report_template( self, config: ReportTemplateConfig, report_template: ReportTemplate ): + """ + Uses the report template id, organization id, and client key from the existing report + template. Updates the name, description, tags, rules, and archived date from the config, + if they are provided. Always passes all fields to the field mask for updating. + """ tags = [] if config.tags: tags = [ReportTemplateTag(tag_name=tag) for tag in config.tags] diff --git a/python/lib/sift_py/yaml/__init__.py b/python/lib/sift_py/yaml/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/sift_py/yaml/report_templates.py b/python/lib/sift_py/yaml/report_templates.py new file mode 100644 index 00000000..740a9160 --- /dev/null +++ b/python/lib/sift_py/yaml/report_templates.py @@ -0,0 +1,73 @@ +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, cast + +import yaml +from typing_extensions import NotRequired, TypedDict + +from sift_py.ingestion.config.yaml.error import YamlConfigError +from sift_py.ingestion.config.yaml.load import _handle_subdir +from sift_py.report_templates.config import ReportTemplateConfig + + +def load_report_templates(paths: List[Path]) -> List[ReportTemplateConfig]: + """ + Takes in a list of paths to YAML files which contains report templates and processes them into a list of + `ReportTemplateConfig` objects. For more information on report templates see + `sift_py.report_templates.config.ReportTemplateConfig`. + """ + report_templates: List[ReportTemplateConfig] = [] + + def update_report_templates(path: Path): + report_templates.extend(_read_report_template_yaml(path)) + + for path in paths: + if path.is_dir(): + _handle_subdir(path, update_report_templates) + elif path.is_file(): + update_report_templates(path) + return report_templates + + +def _read_report_template_yaml(path: Path) -> List[ReportTemplateConfig]: + report_templates = [] + with open(path, "r") as f: + report_templates_yaml = cast(Dict[str, Any], yaml.safe_load(f.read())) + + report_template_list = report_templates_yaml.get("report_templates") + if not isinstance(report_template_list, list): + raise YamlConfigError( + f"Expected 'report_templates' to be a list in report template yaml: '{path}'" + ) + + for report_template in report_template_list: + try: + report_template_config = ReportTemplateConfig(**report_template) + report_templates.append(report_template_config) + except Exception as e: + raise YamlConfigError(f"Error parsing report template '{report_template}'") from e + + return report_templates + + +class ReportTemplateYamlSpec(TypedDict): + """ + Formal spec for a report template. + + `name`: Name of the report template. + `template_client_key`: Unique client key to identify the report template. + `organization_id`: Organization ID that the report template belongs to. + `tags`: Tags to associate with the report template. + `description`: Description of the report template. + `rule_client_keys`: List of rule client keys associated with the report template. + `archived_date`: Date when the report template was archived. Setting this field + will archive the report template, and unsetting it will unarchive the report template. + """ + + name: str + template_client_key: str + organization_id: NotRequired[str] + tags: NotRequired[List[str]] + description: NotRequired[str] + rule_client_keys: List[str] + archived_date: NotRequired[datetime] From 7f860d84840e26a49f916b3d0fefc1e4a21bc081 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 2 Dec 2024 11:28:05 -0800 Subject: [PATCH 32/32] chore: Move common yaml util --- python/lib/sift_py/ingestion/config/yaml/load.py | 12 ++---------- python/lib/sift_py/yaml/report_templates.py | 2 +- python/lib/sift_py/yaml/utils.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 python/lib/sift_py/yaml/utils.py diff --git a/python/lib/sift_py/ingestion/config/yaml/load.py b/python/lib/sift_py/ingestion/config/yaml/load.py index bb600f63..27b85ed8 100644 --- a/python/lib/sift_py/ingestion/config/yaml/load.py +++ b/python/lib/sift_py/ingestion/config/yaml/load.py @@ -1,6 +1,6 @@ import re from pathlib import Path -from typing import Any, Callable, Dict, List, Type, cast +from typing import Any, Dict, List, Type, cast import yaml @@ -16,6 +16,7 @@ TelemetryConfigYamlSpec, ) from sift_py.ingestion.rule.config import RuleActionAnnotationKind +from sift_py.yaml.utils import _handle_subdir _CHANNEL_REFERENCE_REGEX = re.compile(r"^\$\d+$") _SUB_EXPRESSION_REGEX = re.compile(r"^\$[a-zA-Z_]+$") @@ -81,15 +82,6 @@ def update_rule_namespaces(rule_module_path: Path): return rule_namespaces -def _handle_subdir(path: Path, file_handler: Callable): - """The file_handler callable must accept a Path object as its only argument.""" - for file_in_dir in path.iterdir(): - if file_in_dir.is_dir(): - _handle_subdir(file_in_dir, file_handler) - elif file_in_dir.is_file(): - file_handler(file_in_dir) - - def _read_named_expression_module_yaml(path: Path) -> Dict[str, str]: with open(path, "r") as f: named_expressions = cast(Dict[Any, Any], yaml.safe_load(f.read())) diff --git a/python/lib/sift_py/yaml/report_templates.py b/python/lib/sift_py/yaml/report_templates.py index 740a9160..defeb0aa 100644 --- a/python/lib/sift_py/yaml/report_templates.py +++ b/python/lib/sift_py/yaml/report_templates.py @@ -6,8 +6,8 @@ from typing_extensions import NotRequired, TypedDict from sift_py.ingestion.config.yaml.error import YamlConfigError -from sift_py.ingestion.config.yaml.load import _handle_subdir from sift_py.report_templates.config import ReportTemplateConfig +from sift_py.yaml.utils import _handle_subdir def load_report_templates(paths: List[Path]) -> List[ReportTemplateConfig]: diff --git a/python/lib/sift_py/yaml/utils.py b/python/lib/sift_py/yaml/utils.py new file mode 100644 index 00000000..528260db --- /dev/null +++ b/python/lib/sift_py/yaml/utils.py @@ -0,0 +1,11 @@ +from pathlib import Path +from typing import Callable + + +def _handle_subdir(path: Path, file_handler: Callable): + """The file_handler callable must accept a Path object as its only argument.""" + for file_in_dir in path.iterdir(): + if file_in_dir.is_dir(): + _handle_subdir(file_in_dir, file_handler) + elif file_in_dir.is_file(): + file_handler(file_in_dir)