diff --git a/octopoes/octopoes/models/ooi/reports.py b/octopoes/octopoes/models/ooi/reports.py index e3af38fe23e..525ad009b57 100644 --- a/octopoes/octopoes/models/ooi/reports.py +++ b/octopoes/octopoes/models/ooi/reports.py @@ -58,6 +58,7 @@ class ReportRecipe(OOI): subreport_name_format: str | None = None input_recipe: dict[str, Any] # can contain a query which maintains a live set of OOIs or manually picked OOIs. + parent_report_type: str | None = None report_types: list[str] cron_expression: str diff --git a/rocky/reports/runner/report_runner.py b/rocky/reports/runner/report_runner.py index aaaf3992e6a..9fa6454f576 100644 --- a/rocky/reports/runner/report_runner.py +++ b/rocky/reports/runner/report_runner.py @@ -6,10 +6,11 @@ from octopoes.connector.octopoes import OctopoesAPIConnector from octopoes.models import Reference +from reports.report_types.aggregate_organisation_report.report import AggregateOrganisationReport, aggregate_reports from reports.report_types.definitions import report_plugins_union from reports.report_types.helpers import get_report_by_id from reports.runner.models import ReportRunner -from reports.views.mixins import collect_reports, save_report_data +from reports.views.mixins import collect_reports, save_aggregate_report_data, save_report_data from rocky.bytes_client import BytesClient from rocky.scheduler import ReportTask @@ -24,51 +25,94 @@ def run(self, report_task: ReportTask) -> None: connector = OctopoesAPIConnector(settings.OCTOPOES_API, report_task.organisation_id) recipe = connector.get(Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), valid_time) + report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types] + oois_count = len(recipe.input_recipe["input_oois"]) + oois = [] + now = datetime.now(timezone.utc) - error_reports, report_data = collect_reports( - valid_time, connector, recipe.input_recipe["input_oois"], report_types - ) + for ooi_id in recipe.input_recipe["input_oois"]: + ooi = connector.get(Reference.from_str(ooi_id), valid_time) + oois.append(ooi) self.bytes_client.organization = report_task.organisation_id - subreport_names = [] - oois_count = len(recipe.input_recipe["input_oois"]) - for report_type_id, data in report_data.items(): - report_type = get_report_by_id(report_type_id) + if recipe.parent_report_type == AggregateOrganisationReport.id: + parent_report_name = now.strftime( + Template(recipe.report_name_format).safe_substitute( + report_type=str(AggregateOrganisationReport.name), oois_count=str(oois_count) + ) + ) + report_type_ids = [report.id for report in report_types] - for ooi in data: + if "${ooi}" in parent_report_name and oois_count == 1: + ooi = recipe.input_recipe["input_oois"][0] ooi_human_readable = Reference.from_str(ooi).human_readable - subreport_name = Template(recipe.subreport_name_format).safe_substitute( - ooi=ooi_human_readable, report_type=str(report_type.name) - ) - subreport_names.append((subreport_name, subreport_name)) - - parent_report_name = Template(recipe.report_name_format).safe_substitute(oois_count=str(oois_count)) - - if "${ooi}" in parent_report_name and oois_count == 1: - ooi = recipe.input_recipe["input_oois"][0] - ooi_human_readable = Reference.from_str(ooi).human_readable - parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable) - if "${report_type}" in parent_report_name and len(report_types) == 1: - report_type = get_report_by_id(recipe.report_types[0]) - parent_report_name = Template(parent_report_name).safe_substitute(report_type=str(report_type.name)) - - save_report_data( - self.bytes_client, - valid_time, - connector, - Organization.objects.get(code=report_task.organisation_id), - { - "input_data": { - "input_oois": recipe.input_recipe["input_oois"], - "report_types": recipe.report_types, - "plugins": report_plugins_union(report_types), - } - }, - report_data, - subreport_names, - parent_report_name, - ) - - self.bytes_client.organization = None + parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable) + + aggregate_report, post_processed_data, report_data, report_errors = aggregate_reports( + connector, oois, report_type_ids, valid_time, report_task.organisation_id + ) + save_aggregate_report_data( + self.bytes_client, + connector, + Organization.objects.get(code=report_task.organisation_id), + valid_time, + recipe.input_recipe["input_oois"], + { + "input_data": { + "input_oois": recipe.input_recipe["input_oois"], + "report_types": recipe.report_types, + "plugins": report_plugins_union(report_types), + } + }, + parent_report_name, + report_data, + post_processed_data, + aggregate_report, + ) + else: + subreport_names = [] + error_reports, report_data = collect_reports( + valid_time, connector, recipe.input_recipe["input_oois"], report_types + ) + + for report_type_id, data in report_data.items(): + report_type = get_report_by_id(report_type_id) + + for ooi in data: + ooi_human_readable = Reference.from_str(ooi).human_readable + subreport_name = now.strftime( + Template(recipe.subreport_name_format).safe_substitute( + ooi=ooi_human_readable, report_type=str(report_type.name) + ) + ) + subreport_names.append((subreport_name, subreport_name)) + + parent_report_name = now.strftime( + Template(recipe.report_name_format).safe_substitute(oois_count=str(oois_count)) + ) + + if "${ooi}" in parent_report_name and oois_count == 1: + ooi = recipe.input_recipe["input_oois"][0] + ooi_human_readable = Reference.from_str(ooi).human_readable + parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable) + + save_report_data( + self.bytes_client, + valid_time, + connector, + Organization.objects.get(code=report_task.organisation_id), + { + "input_data": { + "input_oois": recipe.input_recipe["input_oois"], + "report_types": recipe.report_types, + "plugins": report_plugins_union(report_types), + } + }, + report_data, + subreport_names, + parent_report_name, + ) + + self.bytes_client.organization = None diff --git a/rocky/reports/templates/report_overview/scheduled_reports_table.html b/rocky/reports/templates/report_overview/scheduled_reports_table.html index ced610163e0..b301a50e253 100644 --- a/rocky/reports/templates/report_overview/scheduled_reports_table.html +++ b/rocky/reports/templates/report_overview/scheduled_reports_table.html @@ -25,14 +25,20 @@ {{ schedule.recipe.report_name_format }} {{ schedule.deadline_at }} diff --git a/rocky/reports/views/base.py b/rocky/reports/views/base.py index a9866b8db63..2d8514a3f18 100644 --- a/rocky/reports/views/base.py +++ b/rocky/reports/views/base.py @@ -267,12 +267,15 @@ def is_scheduled_report(self) -> bool: recurrence_choice = self.request.POST.get("choose_recurrence", "once") return recurrence_choice == "repeat" - def create_report_recipe(self, report_name_format: str, subreport_name_format: str, schedule: str) -> ReportRecipe: + def create_report_recipe( + self, report_name_format: str, subreport_name_format: str, parent_report_type: str | None, schedule: str + ) -> ReportRecipe: report_recipe = ReportRecipe( recipe_id=uuid4(), report_name_format=report_name_format, subreport_name_format=subreport_name_format, input_recipe={"input_oois": self.get_ooi_pks()}, + parent_report_type=parent_report_type, report_types=self.get_report_type_ids(), cron_expression=schedule, ) @@ -532,13 +535,20 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: elif self.is_scheduled_report(): report_name_format = request.POST.get("parent_report_name", "") subreport_name_format = request.POST.get("child_report_name", "") - recurrence = request.POST.get("recurrence", "") deadline_at = request.POST.get("start_date", datetime.now(timezone.utc).date()) + parent_report_type = None + if self.report_type is not None: + parent_report_type = self.report_type.id + elif not self.report_type and subreport_name_format: + parent_report_type = ConcatenatedReport.id + schedule = self.convert_recurrence_to_cron_expressions(recurrence) - report_recipe = self.create_report_recipe(report_name_format, subreport_name_format, schedule) + report_recipe = self.create_report_recipe( + report_name_format, subreport_name_format, parent_report_type, schedule + ) self.create_report_schedule(report_recipe, deadline_at) diff --git a/rocky/reports/views/mixins.py b/rocky/reports/views/mixins.py index 46c31a1dffb..259c691a68e 100644 --- a/rocky/reports/views/mixins.py +++ b/rocky/reports/views/mixins.py @@ -1,4 +1,5 @@ from datetime import datetime, timezone +from string import Template from typing import Any from uuid import uuid4 @@ -76,7 +77,8 @@ def save_report_data( raw_id = bytes_client.upload_raw( raw=ReportDataDict(input_data).model_dump_json().encode(), manual_mime_types={"openkat/report"} ) - name = now.strftime(parent_report_name.replace("${report_type}", str(ConcatenatedReport.name))) + + name = now.strftime(Template(parent_report_name).safe_substitute(report_type=str(ConcatenatedReport.name))) if not name or name.isspace(): name = ConcatenatedReport.name @@ -164,7 +166,7 @@ def save_report_data( manual_mime_types={"openkat/report"}, ) report_type = get_report_by_id(report_type_id) - name = now.strftime(parent_report_name.replace("${report_type}", str(report_type.name))) + name = now.strftime(Template(parent_report_name).safe_substitute(report_type=str(report_type.name))) if not name or name.isspace(): name = ConcatenatedReport.name @@ -189,6 +191,59 @@ def save_report_data( return parent_report_ooi +def save_aggregate_report_data( + bytes_client, + octopoes_api_connector, + organization, + get_observed_at, + ooi_pks, + input_data: dict, + parent_report_name, + report_data, + post_processed_data, + aggregate_report, +) -> Report: + observed_at = get_observed_at + + now = datetime.utcnow() + + # Create the report + report_data_raw_id = bytes_client.upload_raw( + raw=ReportDataDict(post_processed_data | input_data).model_dump_json().encode(), + manual_mime_types={"openkat/report"}, + ) + report_type = type(aggregate_report) + name = now.strftime(parent_report_name) + if not name or name.isspace(): + name = report_type.name + + report_ooi = Report( + name=str(name), + report_type=str(report_type.id), + template=report_type.template_path, + report_id=uuid4(), + organization_code=organization.code, + organization_name=organization.name, + organization_tags=list(organization.tags.all()), + data_raw_id=report_data_raw_id, + date_generated=datetime.now(timezone.utc), + input_oois=ooi_pks, + observed_at=observed_at, + parent_report=None, + has_parent=False, + ) + create_ooi(octopoes_api_connector, bytes_client, report_ooi, observed_at) + + # Save the child reports to bytes + for ooi, types in report_data.items(): + for report_type, data in types.items(): + bytes_client.upload_raw( + raw=ReportDataDict(data | input_data).model_dump_json().encode(), manual_mime_types={"openkat/report"} + ) + + return report_ooi + + class SaveGenerateReportMixin(BaseReportView): def save_report(self, report_names: list) -> Report | None: error_reports, report_data = collect_reports( @@ -224,7 +279,6 @@ def save_report(self, report_names: list) -> Report | None: class SaveAggregateReportMixin(BaseReportView): def save_report(self, report_names: list) -> Report: - organization = self.organization aggregate_report, post_processed_data, report_data, report_errors = aggregate_reports( self.octopoes_api_connector, self.get_oois(), @@ -243,47 +297,18 @@ def save_report(self, report_names: list) -> Report: } messages.add_message(self.request, messages.ERROR, error_message) - observed_at = self.get_observed_at() - - now = datetime.utcnow() - bytes_client = self.bytes_client - - # Create the report - report_data_raw_id = bytes_client.upload_raw( - raw=ReportDataDict(post_processed_data | self.get_input_data()).model_dump_json().encode(), - manual_mime_types={"openkat/report"}, - ) - report_type = type(aggregate_report) - name = now.strftime(report_names[0][1]) - if not name or name.isspace(): - name = report_type.name - - report_ooi = Report( - name=str(name), - report_type=str(report_type.id), - template=report_type.template_path, - report_id=uuid4(), - organization_code=organization.code, - organization_name=organization.name, - organization_tags=list(organization.tags.all()), - data_raw_id=report_data_raw_id, - date_generated=datetime.now(timezone.utc), - input_oois=self.get_ooi_pks(), - observed_at=observed_at, - parent_report=None, - has_parent=False, + return save_aggregate_report_data( + self.bytes_client, + self.octopoes_api_connector, + self.organization, + self.get_observed_at(), + self.get_ooi_pks(), + self.get_input_data(), + report_names[0][1], + report_data, + post_processed_data, + aggregate_report, ) - create_ooi(self.octopoes_api_connector, bytes_client, report_ooi, observed_at) - - # Save the child reports to bytes - for ooi, types in report_data.items(): - for report_type, data in types.items(): - bytes_client.upload_raw( - raw=ReportDataDict(data | self.get_input_data()).model_dump_json().encode(), - manual_mime_types={"openkat/report"}, - ) - - return report_ooi class SaveMultiReportMixin(BaseReportView):