Skip to content

Commit

Permalink
Sub reports for Aggregate Report (#3852)
Browse files Browse the repository at this point in the history 
Co-authored-by: ammar92 <[email protected]>
  • Loading branch information
@Rieven @ammar92
Rieven and ammar92 authored Nov 21, 2024
1 parent 5f62018 commit 1a0299d
Show file tree
Hide file tree
Showing 4 changed files with 297 additions and 21 deletions.
5 changes: 5 additions & 0 deletions rocky/reports/report_types/definitions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ class ReportPlugins(TypedDict):
optional: set[str]


class SubReportPlugins(TypedDict):
required: list[str]
optional: list[str]


def report_plugins_union(report_types: list[type["BaseReport"]]) -> ReportPlugins:
"""Take the union of the required and optional plugin sets and remove optional plugins that are required"""

Expand Down
67 changes: 46 additions & 21 deletions rocky/reports/views/mixins.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from octopoes.models.ooi.reports import Report
from reports.report_types.aggregate_organisation_report.report import aggregate_reports
from reports.report_types.concatenated_report.report import ConcatenatedReport
from reports.report_types.definitions import BaseReport, SubReportPlugins
from reports.report_types.helpers import REPORTS, get_report_by_id
from reports.report_types.multi_organization_report.report import MultiOrganizationReport, collect_report_data
from reports.views.base import BaseReportView, ReportDataDict
Expand Down Expand Up @@ -56,6 +57,22 @@ def collect_reports(observed_at: datetime, octopoes_connector: OctopoesAPIConnec
return error_reports, report_data


def get_child_input_data(input_data: dict[str, Any], ooi: str, report_type: type[BaseReport]):
required_plugins = list(input_data["input_data"]["plugins"]["required"])
optional_plugins = list(input_data["input_data"]["plugins"]["optional"])

child_plugins: SubReportPlugins = {"required": [], "optional": []}

child_plugins["required"] = [
plugin_id for plugin_id in required_plugins if plugin_id in report_type.plugins["required"]
]
child_plugins["optional"] = [
plugin_id for plugin_id in optional_plugins if plugin_id in report_type.plugins["optional"]
]

return {"input_data": {"input_oois": [ooi], "report_types": [report_type.id], "plugins": child_plugins}}


def save_report_data(
bytes_client,
observed_at,
Expand Down Expand Up @@ -116,21 +133,7 @@ def save_report_data(
name_to_save = updated_name
break

required_plugins = list(input_data["input_data"]["plugins"]["required"])
optional_plugins = list(input_data["input_data"]["plugins"]["optional"])

child_plugins: dict[str, list[str]] = {"required": [], "optional": []}

child_plugins["required"] = [
plugin_id for plugin_id in required_plugins if plugin_id in report_type.plugins["required"]
]
child_plugins["optional"] = [
plugin_id for plugin_id in optional_plugins if plugin_id in report_type.plugins["optional"]
]

child_input_data = {
"input_data": {"input_oois": [ooi], "report_types": [report_type_id], "plugins": child_plugins}
}
child_input_data = get_child_input_data(input_data, ooi, report_type)

raw_id = bytes_client.upload_raw(
raw=ReportDataDict({"report_data": data["data"]} | child_input_data).model_dump_json().encode(),
Expand Down Expand Up @@ -208,8 +211,7 @@ def save_aggregate_report_data(
report_recipe: Reference | None = None,
) -> Report:
observed_at = get_observed_at

now = datetime.utcnow()
now = datetime.now(timezone.utc)

# Create the report
report_data_raw_id = bytes_client.upload_raw(
Expand All @@ -230,7 +232,7 @@ def save_aggregate_report_data(
organization_name=organization.name,
organization_tags=[tag.name for tag in organization.tags.all()],
data_raw_id=report_data_raw_id,
date_generated=datetime.now(timezone.utc),
date_generated=now,
input_oois=ooi_pks,
observed_at=observed_at,
parent_report=None,
Expand All @@ -240,12 +242,35 @@ def save_aggregate_report_data(
create_ooi(octopoes_api_connector, bytes_client, report_ooi, observed_at)

# Save the child reports to bytes

for ooi, types in report_data.items():
for report_type, data in types.items():
bytes_client.upload_raw(
raw=ReportDataDict(data | input_data).model_dump_json().encode(), manual_mime_types={"openkat/report"}
for report_type_id, data in types.items():
report_type = get_report_by_id(report_type_id)
child_input_data = get_child_input_data(input_data, ooi, report_type)

raw_id = bytes_client.upload_raw(
raw=ReportDataDict({"report_data": data} | child_input_data).model_dump_json().encode(),
manual_mime_types={"openkat/report"},
)

aggregate_sub_report_ooi = Report(
name=str(report_type.name),
report_type=report_type_id,
template=report_type.template_path,
report_id=uuid4(),
organization_code=organization.code,
organization_name=organization.name,
organization_tags=[tag.name for tag in organization.tags.all()],
data_raw_id=raw_id,
date_generated=now,
input_oois=[ooi],
observed_at=observed_at,
parent_report=report_ooi.reference,
has_parent=True,
)

create_ooi(octopoes_api_connector, bytes_client, aggregate_sub_report_ooi, observed_at)

return report_ooi


Expand Down
214 changes: 214 additions & 0 deletions rocky/tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2340,3 +2340,217 @@ def get_report_input_data_from_bytes():
}
}
return json.dumps(input_data).encode("utf-8")


@pytest.fixture
def aggregate_report_with_sub_reports():
aggregate_report: Paginated[tuple[Report, list[Report | None]]] = Paginated(
count=1,
items=[
(
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|23820a64-db8f-41b7-b045-031338fbb91d",
name="Aggregate Report",
report_type="aggregate-organisation-report",
template="aggregate_organisation_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("23820a64-db8f-41b7-b045-031338fbb91d"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="3a362cd7-6348-4e91-8a6f-4cd83f9f6a83",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=None,
report_recipe=None,
has_parent=False,
),
[
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|28c7b15e-6dda-49e8-a101-41df3124287e",
name="Mail Report",
report_type="mail-report",
template="mail_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("28c7b15e-6dda-49e8-a101-41df3124287e"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="a534b4d5-5dba-4ddc-9b77-970675ae4b1c",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|2a56737f-492f-424b-88cc-0029ce2a444b",
name="IPv6 Report",
report_type="ipv6-report",
template="ipv6_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("2a56737f-492f-424b-88cc-0029ce2a444b"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="0bdea8eb-7ac0-46ef-ad14-ea3b0bfe1030",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|4ec12350-7552-40de-8c9f-f75ac04b99cb",
name="RPKI Report",
report_type="rpki-report",
template="rpki_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("4ec12350-7552-40de-8c9f-f75ac04b99cb"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="53d5452c-9e67-42d2-9cb0-3b684d8967a2",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|8137a050-f897-45ce-a695-fd21c63e2e5c",
name="Safe Connections Report",
report_type="safe-connections-report",
template="safe_connections_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("8137a050-f897-45ce-a695-fd21c63e2e5c"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="a218ca79-47de-4473-a93d-54d14baadd98",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|9ca7ad01-e19e-42c9-9361-751db4399b94",
name="Web System Report",
report_type="web-system-report",
template="web_system_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("9ca7ad01-e19e-42c9-9361-751db4399b94"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="3779f5b0-3adf-41c8-9630-8eed8a857ae6",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|a76878ba-55e0-4971-b645-63cfdfd34e78",
name="Open Ports Report",
report_type="open-ports-report",
template="open_ports_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("a76878ba-55e0-4971-b645-63cfdfd34e78"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="851feeab-7036-48f6-81ef-599467c52457",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|ad33bbf1-bd35-4cb4-a61d-ebe1409e2f67",
name="Vulnerability Report",
report_type="vulnerability-report",
template="vulnerability_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("ad33bbf1-bd35-4cb4-a61d-ebe1409e2f67"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="1e259fce-3cd7-436f-b233-b4ae24a8f11b",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|bd26a0c0-92c2-4323-977d-a10bd90619e7",
name="System Report",
report_type="systems-report",
template="systems_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("bd26a0c0-92c2-4323-977d-a10bd90619e7"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="50a9e4df-3b69-4ad8-b798-df626162db5a",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|d8fcaa8f-65ca-4304-a18c-078767b37bcb",
name="Name Server Report",
report_type="name-server-report",
template="name_server_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("d8fcaa8f-65ca-4304-a18c-078767b37bcb"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="5faa3364-c8b2-4b9c-8cc8-99d8f19ccf8a",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
],
)
],
)
return aggregate_report
32 changes: 32 additions & 0 deletions rocky/tests/reports/test_report_overview.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,35 @@ def test_report_overview_rerun_reports(
assert list(request._messages)[0].message == "Rerun successful"

assertContains(response, concatenated_report.name)


def test_aggregate_report_has_sub_reports(
rf, client_member, mock_organization_view_octopoes, mock_bytes_client, aggregate_report_with_sub_reports
):
mock_organization_view_octopoes().list_reports.return_value = aggregate_report_with_sub_reports

aggregate_report, subreports = aggregate_report_with_sub_reports.items[0]

response = ReportHistoryView.as_view()(
setup_request(rf.get("report_history"), client_member.user), organization_code=client_member.organization.code
)

assert response.status_code == 200

assertContains(response, "Nov. 21, 2024")
assertContains(response, "Nov. 21, 2024, 10:07 a.m.")

assertContains(response, "expando-button icon ti-chevron-down")

assertContains(
response, f"This report consist of {len(subreports)} subreports with the following report types and objects."
)

assertContains(response, f"Subreports (5/{len(subreports)})", html=True)

assertContains(response, aggregate_report.name)

for subreport in subreports:
assertContains(response, subreport.name)

assertContains(response, "View all subreports")

0 comments on commit 1a0299d

Please sign in to comment.