Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix scheduled Aggregate Report naming #3748

Merged
merged 18 commits into from
Nov 1, 2024
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions octopoes/octopoes/models/ooi/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ class ReportRecipe(OOI):
subreport_name_format: str | None = None

input_recipe: dict[str, Any] # can contain a query which maintains a live set of OOIs or manually picked OOIs.
parent_report_type: str | None = None
report_types: list[str]

cron_expression: str
Expand Down
4 changes: 2 additions & 2 deletions rocky/reports/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,13 @@ class CustomReportScheduleForm(BaseRockyForm):

class ParentReportNameForm(BaseRockyForm):
parent_report_name = forms.CharField(
label=_("Report name format"), required=False, initial="{report type} for {oois_count} objects"
label=_("Report name format"), required=False, initial="${report_type} for ${oois_count} objects"
)


class ChildReportNameForm(BaseRockyForm):
child_report_name = forms.CharField(
label=_("Subreports name format"), required=True, initial="{report type} for {ooi}"
label=_("Subreports name format"), required=True, initial="${report_type} for ${ooi}"
)


Expand Down
121 changes: 88 additions & 33 deletions rocky/reports/runner/report_runner.py
madelondohmen marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
from datetime import datetime, timezone
from string import Template

from django.conf import settings
from tools.models import Organization

from octopoes.connector.octopoes import OctopoesAPIConnector
from octopoes.models import Reference
from reports.report_types.aggregate_organisation_report.report import AggregateOrganisationReport, aggregate_reports
from reports.report_types.definitions import report_plugins_union
from reports.report_types.helpers import get_report_by_id
from reports.runner.models import ReportRunner
from reports.views.mixins import collect_reports, save_report_data
from reports.views.mixins import collect_reports, save_aggregate_report_data, save_report_data
from rocky.bytes_client import BytesClient
from rocky.scheduler import ReportTask

Expand All @@ -23,41 +25,94 @@ def run(self, report_task: ReportTask) -> None:

connector = OctopoesAPIConnector(settings.OCTOPOES_API, report_task.organisation_id)
recipe = connector.get(Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), valid_time)

report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types]
oois_count = len(recipe.input_recipe["input_oois"])
oois = []
now = datetime.now(timezone.utc)

error_reports, report_data = collect_reports(
valid_time, connector, recipe.input_recipe["input_oois"], report_types
)
for ooi_id in recipe.input_recipe["input_oois"]:
ooi = connector.get(Reference.from_str(ooi_id), valid_time)
oois.append(ooi)

self.bytes_client.organization = report_task.organisation_id
report_names = []
oois_count = 0

for report_type_id, data in report_data.items():
oois_count += len(data)
report_type = get_report_by_id(report_type_id)

for ooi in data:
report_name = recipe.subreport_name_format.replace("{ooi}", ooi).replace(
"{report type}", str(report_type.name)
if recipe.parent_report_type == AggregateOrganisationReport.id:
parent_report_name = now.strftime(
Template(recipe.report_name_format).safe_substitute(
report_type=str(AggregateOrganisationReport.name), oois_count=str(oois_count)
)
report_names.append((report_name, report_name))

save_report_data(
self.bytes_client,
valid_time,
connector,
Organization.objects.get(code=report_task.organisation_id),
{
"input_data": {
"input_oois": recipe.input_recipe["input_oois"],
"report_types": recipe.report_types,
"plugins": report_plugins_union(report_types),
}
},
report_data,
report_names,
recipe.report_name_format.replace("{oois_count}", str(oois_count)),
)

self.bytes_client.organization = None
)
report_type_ids = [report.id for report in report_types]

if "${ooi}" in parent_report_name and oois_count == 1:
ooi = recipe.input_recipe["input_oois"][0]
ooi_human_readable = Reference.from_str(ooi).human_readable
parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable)

aggregate_report, post_processed_data, report_data, report_errors = aggregate_reports(
connector, oois, report_type_ids, valid_time, report_task.organisation_id
)
save_aggregate_report_data(
self.bytes_client,
connector,
Organization.objects.get(code=report_task.organisation_id),
valid_time,
recipe.input_recipe["input_oois"],
{
"input_data": {
"input_oois": recipe.input_recipe["input_oois"],
"report_types": recipe.report_types,
"plugins": report_plugins_union(report_types),
}
},
parent_report_name,
report_data,
post_processed_data,
aggregate_report,
)
else:
subreport_names = []
error_reports, report_data = collect_reports(
valid_time, connector, recipe.input_recipe["input_oois"], report_types
)

for report_type_id, data in report_data.items():
report_type = get_report_by_id(report_type_id)

for ooi in data:
ooi_human_readable = Reference.from_str(ooi).human_readable
subreport_name = now.strftime(
Template(recipe.subreport_name_format).safe_substitute(
ooi=ooi_human_readable, report_type=str(report_type.name)
)
)
subreport_names.append((subreport_name, subreport_name))

parent_report_name = now.strftime(
Template(recipe.report_name_format).safe_substitute(oois_count=str(oois_count))
)

if "${ooi}" in parent_report_name and oois_count == 1:
ooi = recipe.input_recipe["input_oois"][0]
ooi_human_readable = Reference.from_str(ooi).human_readable
parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable)

save_report_data(
self.bytes_client,
valid_time,
connector,
Organization.objects.get(code=report_task.organisation_id),
{
"input_data": {
"input_oois": recipe.input_recipe["input_oois"],
"report_types": recipe.report_types,
"plugins": report_plugins_union(report_types),
}
},
report_data,
subreport_names,
parent_report_name,
)

self.bytes_client.organization = None
6 changes: 3 additions & 3 deletions rocky/reports/templates/partials/report_names_header.html
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ <h2>{% translate "Report name" %}</h2>
{% blocktranslate trimmed %}
To make the report names more descriptive, you can include placeholders for the
object name, the report type and/or the reference date. For subreports and reports over a single object,
use the placeholder "{ooi}" for the object name, "{report type}" for the report type and use a
use the placeholder "${ooi}" for the object name, "${report_type}" for the report type and use a
<a href="https://strftime.org/" target="_blank" rel="noopener">Python strftime code</a> for the reference
date. For reports over multiple objects, use "{oois_count}" for the number of objects in the report.
date. For reports over multiple objects, use "${oois_count}" for the number of objects in the report.
{% endblocktranslate %}
</p>
<p>
{% blocktranslate trimmed %}
For example, the format "{report type} for {ooi} at %x" could generate:
For example, the format "${report_type} for ${ooi} at %x" could generate:
"DNS Report for example.com at 01/01/25".
{% endblocktranslate %}
</p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,20 @@
<td class="nowrap">{{ schedule.recipe.report_name_format }}</td>
<td>
<ul class="tags horizontal-view">
{% for report_type in schedule.recipe.report_types %}
{% if forloop.counter0 < 2 %}
<li class="label tags-color-{{ report_type|get_report_type_label_style }}">{{ report_type|get_report_type_name }}</li>
{% endif %}
{% if forloop.counter0 == 2 %}
<li class="label tags-color-grey-2">+ {{ schedule.recipe.report_types|length|add:"-2" }}</li>
{% endif %}
{% endfor %}
{% if schedule.recipe.parent_report_type == "aggregate-organisation-report" %}
<li class="label tags-color-{{ schedule.recipe.parent_report_type|get_report_type_label_style }}">
{{ schedule.recipe.parent_report_type|get_report_type_name }}
</li>
{% else %}
{% for report_type in schedule.recipe.report_types %}
{% if forloop.counter0 < 2 %}
<li class="label tags-color-{{ report_type|get_report_type_label_style }}">{{ report_type|get_report_type_name }}</li>
{% endif %}
{% if forloop.counter0 == 2 %}
<li class="label tags-color-grey-2">+ {{ schedule.recipe.report_types|length|add:"-2" }}</li>
{% endif %}
{% endfor %}
{% endif %}
</ul>
</td>
<td class="nowrap">{{ schedule.deadline_at }}</td>
Expand Down
1 change: 1 addition & 0 deletions rocky/reports/views/aggregate_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,3 +116,4 @@ class SaveAggregateReportView(SaveAggregateReportMixin, BreadcrumbsAggregateRepo
template_name = "aggregate_report.html"
breadcrumbs_step = 6
current_step = 5
report_type = AggregateOrganisationReport
16 changes: 13 additions & 3 deletions rocky/reports/views/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,12 +258,15 @@ def is_scheduled_report(self) -> bool:
recurrence_choice = self.request.POST.get("choose_recurrence", "once")
return recurrence_choice == "repeat"

def create_report_recipe(self, report_name_format: str, subreport_name_format: str, schedule: str) -> ReportRecipe:
def create_report_recipe(
self, report_name_format: str, subreport_name_format: str, parent_report_type: str | None, schedule: str
) -> ReportRecipe:
report_recipe = ReportRecipe(
recipe_id=uuid4(),
report_name_format=report_name_format,
subreport_name_format=subreport_name_format,
input_recipe={"input_oois": self.get_ooi_pks()},
parent_report_type=parent_report_type,
report_types=self.get_report_type_ids(),
cron_expression=schedule,
)
Expand Down Expand Up @@ -523,13 +526,20 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
elif self.is_scheduled_report():
report_name_format = request.POST.get("parent_report_name", "")
subreport_name_format = request.POST.get("child_report_name", "")

recurrence = request.POST.get("recurrence", "")
deadline_at = request.POST.get("start_date", datetime.now(timezone.utc).date())

parent_report_type = None
if self.report_type == AggregateOrganisationReport:
parent_report_type = AggregateOrganisationReport.id
elif not self.report_type and subreport_name_format:
parent_report_type = ConcatenatedReport.id

madelondohmen marked this conversation as resolved.
Show resolved Hide resolved
schedule = self.convert_recurrence_to_cron_expressions(recurrence)

report_recipe = self.create_report_recipe(report_name_format, subreport_name_format, schedule)
report_recipe = self.create_report_recipe(
report_name_format, subreport_name_format, parent_report_type, schedule
)

self.create_report_schedule(report_recipe, deadline_at)

Expand Down
Loading
Loading