Add live set (filter/query) to ReportRecipe (#3769)

Co-authored-by: ammar92 <[email protected]>
Co-authored-by: stephanie0x00 <[email protected]>

docs/source/manual/reports.rst

@@ -183,7 +183,7 @@ The table below gives an overview of the elements that can be found in each repo
 
 Report flow
 ===========
-On the Reports page you can generate new reports and get an overview of all generated reports. With the button 'Generate report' you get into the Report flow wizard, which can be used to choose your report, objects and plugins that are required for the report. Please note that enabling plugins during the report flow wizard will result in inaccurate data, as the plugins will take some time before they have gathered and analyzed all data. Check the Tasks page to verify that all tasks have completed.
+On the Reports page you can generate new reports and get an overview of all generated reports. With the button 'Generate report' you get into the Report flow wizard, which can be used to choose your report, objects and plugins that are required for the report. There are two ways to select objects. You can manually select objects, which will be static. Or you can select a live set of objects by continuing with the selected filters. The selected objects will then always be based on the selected filters at the time of generating the report. and  Please note that enabling plugins during the report flow wizard will result in inaccurate data, as the plugins will take some time before they have gathered and analyzed all data. Check the Tasks page to verify that all tasks have completed.
 
 
 Plugins

rocky/reports/runner/report_runner.py

@@ -5,7 +5,8 @@
 from tools.models import Organization
 
 from octopoes.connector.octopoes import OctopoesAPIConnector
-from octopoes.models import Reference
+from octopoes.models import Reference, ScanLevel, ScanProfileType
+from octopoes.models.types import type_by_name
 from reports.report_types.aggregate_organisation_report.report import AggregateOrganisationReport, aggregate_reports
 from reports.report_types.definitions import report_plugins_union
 from reports.report_types.helpers import get_report_by_id
@@ -29,13 +30,30 @@ def run(self, report_task: ReportTask) -> None:
         recipe = connector.get(Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), valid_time)
 
         report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types]
-        oois_count = len(recipe.input_recipe["input_oois"])
         oois = []
         now = datetime.now(timezone.utc)
 
-        for ooi_id in recipe.input_recipe["input_oois"]:
-            ooi = connector.get(Reference.from_str(ooi_id), valid_time)
-            oois.append(ooi)
+        if input_oois := recipe.input_recipe.get("input_oois"):
+            for ooi_id in input_oois:
+                ooi = connector.get(Reference.from_str(ooi_id), valid_time)
+                oois.append(ooi)
+        elif query := recipe.input_recipe.get("query"):
+            types = {type_by_name(t) for t in query["ooi_types"]}
+            scan_level = {ScanLevel(cl) for cl in query["scan_level"]}
+            scan_type = {ScanProfileType(t) for t in query["scan_type"]}
+
+            oois = connector.list_objects(
+                types=types,
+                valid_time=datetime.now(tz=timezone.utc),
+                scan_level=scan_level,
+                scan_profile_type=scan_type,
+                search_string=query["search_string"],
+                order_by=query["order_by"],
+                asc_desc=query["asc_desc"],
+            ).items
+
+        oois_count = len(oois)
+        ooi_pks = [ooi.primary_key for ooi in oois]
 
         self.bytes_client.organization = report_task.organisation_id
 
@@ -48,9 +66,7 @@ def run(self, report_task: ReportTask) -> None:
             report_type_ids = [report.id for report in report_types]
 
             if "${ooi}" in parent_report_name and oois_count == 1:
-                ooi = recipe.input_recipe["input_oois"][0]
-                ooi_human_readable = Reference.from_str(ooi).human_readable
-                parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable)
+                parent_report_name = Template(parent_report_name).safe_substitute(ooi=oois[0].human_readable)
 
             aggregate_report, post_processed_data, report_data, report_errors = aggregate_reports(
                 connector, oois, report_type_ids, valid_time, report_task.organisation_id
@@ -60,10 +76,10 @@ def run(self, report_task: ReportTask) -> None:
                 connector,
                 Organization.objects.get(code=report_task.organisation_id),
                 valid_time,
-                recipe.input_recipe["input_oois"],
+                ooi_pks,
                 {
                     "input_data": {
-                        "input_oois": recipe.input_recipe["input_oois"],
+                        "input_oois": ooi_pks,
                         "report_types": recipe.report_types,
                         "plugins": report_plugins_union(report_types),
                     }
@@ -75,9 +91,7 @@ def run(self, report_task: ReportTask) -> None:
             )
         else:
             subreport_names = []
-            error_reports, report_data = collect_reports(
-                valid_time, connector, recipe.input_recipe["input_oois"], report_types
-            )
+            error_reports, report_data = collect_reports(valid_time, connector, ooi_pks, report_types)
 
             for report_type_id, data in report_data.items():
                 report_type = get_report_by_id(report_type_id)
@@ -96,9 +110,7 @@ def run(self, report_task: ReportTask) -> None:
             )
 
             if "${ooi}" in parent_report_name and oois_count == 1:
-                ooi = recipe.input_recipe["input_oois"][0]
-                ooi_human_readable = Reference.from_str(ooi).human_readable
-                parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi_human_readable)
+                parent_report_name = Template(parent_report_name).safe_substitute(ooi=ooi[0].human_readable)
 
             save_report_data(
                 self.bytes_client,
@@ -107,7 +119,7 @@ def run(self, report_task: ReportTask) -> None:
                 Organization.objects.get(code=report_task.organisation_id),
                 {
                     "input_data": {
-                        "input_oois": recipe.input_recipe["input_oois"],
+                        "input_oois": ooi_pks,
                         "report_types": recipe.report_types,
                         "plugins": report_plugins_union(report_types),
                     }

rocky/reports/templates/forms/report_form_fields.html

@@ -25,3 +25,4 @@
            name="choose_recurrence"
            value="{{ request.POST.choose_recurrence }}">
 {% endif %}
+<input type="hidden" name="object_selection" value="{{ object_selection }}">

rocky/reports/templates/partials/report_ooi_list.html

@@ -10,9 +10,40 @@ <h2>
                 Select objects ({{ total_oois }})
             {% endblocktranslate %}
         </h2>
-        <p>{% translate "Select which objects you want to include in your report." %}</p>
+        <p>
+            {% blocktranslate %}
+                Select which objects you want to include in your report. You can either continue
+                with a live set or you can select the objects manually from the table below.
+            {% endblocktranslate %}
+        </p>
+        <p>
+            {% blocktranslate %}
+                A live set is a set of objects based on the applied filters.
+                Any object that matches this applied filter (now or in the future) will be used as
+                input for the scheduled report. If your live set filter (e.g. 'hostnames' with
+                'L2 clearance' that are 'declared') shows 2 hostnames that match the filter today,
+                the scheduled report will run for those 2 hostnames. If you add 3 more hostnames
+                tomorrow (with the same filter criteria), your next scheduled report will contain
+                5 hostnames. Your live set will update as you go.
+            {% endblocktranslate %}
+        </p>
         {% include "partials/ooi_list_filters.html" %}
 
+        <form novalidate
+              method="post"
+              action="{{ next }}"
+              class="inline layout-wide checkboxes_required">
+            {% csrf_token %}
+            {% include "forms/report_form_fields.html" %}
+
+            <input type="hidden" name="ooi" value="all">
+            <button type="submit"
+                    class="button select_all_objects_element"
+                    name="object_selection"
+                    value="query">
+                {% translate "Continue with live set" %}<span class="icon ti-chevron-right"></span>
+            </button>
+        </form>
         {% if not ooi_list %}
             <p>{% translate "No objects found." %}</p>
             <div class="button-container">
@@ -51,7 +82,8 @@ <h2>
             <form novalidate
                   method="post"
                   action="{{ next }}"
-                  class="inline layout-wide checkboxes_required">
+                  class="inline layout-wide checkboxes_required"
+                  id="object_table_form">
                 {% csrf_token %}
                 {% if all_oois_selected %}
                     {% include "forms/report_form_fields.html" %}
@@ -95,10 +127,13 @@ <h2>
                             {% endfor %}
                         </tbody>
                     </table>
+                    <button type="submit"
+                            class="button ghost"
+                            name="object_selection"
+                            value="selection">
+                        {% translate "Continue with selection" %}<span class="icon ti-chevron-right"></span>
+                    </button>
                 </div>
-                <button type="submit" class="button">
-                    {% translate "Continue with selection" %}<span class="icon ti-chevron-right"></span>
-                </button>
             </form>
             {% include "partials/list_paginator.html" %}
 

rocky/reports/views/base.py

@@ -153,7 +153,7 @@ def format_plugin_data(report_type_plugins: dict[str, list[Plugin]]):
     ]
 
 
-class BaseReportView(OOIFilterView):
+class BaseReportView(OOIFilterView, ReportBreadcrumbs):
     """
     This view is the base for the report creation wizard.
     All the necessary functions and variables needed.
@@ -268,13 +268,25 @@ def is_scheduled_report(self) -> bool:
         return recurrence_choice == "repeat"
 
     def create_report_recipe(
-        self, report_name_format: str, subreport_name_format: str, parent_report_type: str | None, schedule: str
+        self,
+        report_name_format: str,
+        subreport_name_format: str,
+        parent_report_type: str | None,
+        schedule: str,
+        query: dict[str, Any] | None,
     ) -> ReportRecipe:
+        input_recipe: dict[str, Any] = {}
+
+        if query:
+            input_recipe = {"query": query}
+        else:
+            input_recipe = {"input_oois": self.get_ooi_pks()}
+
         report_recipe = ReportRecipe(
             recipe_id=uuid4(),
             report_name_format=report_name_format,
             subreport_name_format=subreport_name_format,
-            input_recipe={"input_oois": self.get_ooi_pks()},
+            input_recipe=input_recipe,
             parent_report_type=parent_report_type,
             report_types=self.get_report_type_ids(),
             cron_expression=schedule,
@@ -302,6 +314,7 @@ def get_context_data(self, **kwargs):
         context["all_oois_selected"] = self.all_oois_selected()
         context["selected_oois"] = self.selected_oois
         context["selected_report_types"] = self.selected_report_types
+        context["object_selection"] = self.request.POST.get("object_selection", "")
 
         return context
 
@@ -311,6 +324,13 @@ class OOISelectionView(BaseReportView, BaseOOIListView):
     Shows a list of OOIs to select from and handles OOIs selection requests.
     """
 
+    def setup(self, request, *args, **kwargs):
+        super().setup(request, *args, **kwargs)
+        object_selection = request.GET.get("object_selection", "")
+
+        if object_selection == "query":
+            return PostRedirect(self.get_next())
+
     def post(self, request, *args, **kwargs):
         if not (self.get_ooi_selection() or self.all_oois_selected()):
             messages.error(request, self.NONE_OOI_SELECTION_MESSAGE)
@@ -322,7 +342,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-class ReportTypeSelectionView(BaseReportView, ReportBreadcrumbs):
+class ReportTypeSelectionView(BaseReportView, TemplateView):
     """
     Shows report types and handles selections and requests.
     """
@@ -332,7 +352,8 @@ def setup(self, request, *args, **kwargs):
         self.available_report_types, self.counted_report_types = self.get_available_report_types()
 
     def post(self, request, *args, **kwargs):
-        if not (self.get_ooi_selection() or self.all_oois_selected()):
+        object_selection = request.GET.get("object_selection", "")
+        if not (self.get_ooi_selection() or self.all_oois_selected()) and object_selection != "query":
             return PostRedirect(self.get_previous())
         return self.get(request, *args, **kwargs)
 
@@ -349,8 +370,11 @@ def get_context_data(self, **kwargs):
 
         return context
 
+    def all_oois_selected(self) -> bool:
+        return "all" in self.request.POST.getlist("ooi", [])
+
 
-class ReportPluginView(BaseReportView, ReportBreadcrumbs, TemplateView):
+class ReportPluginView(BaseReportView, TemplateView):
     """
     This view shows the required and optional plugins together with the summary per report type.
     """
@@ -446,7 +470,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-class ReportFinalSettingsView(BaseReportView, ReportBreadcrumbs, SchedulerView, TemplateView):
+class ReportFinalSettingsView(BaseReportView, SchedulerView, TemplateView):
     report_type: type[BaseReport] | None = None
     task_type = "report"
     is_a_scheduled_report = False
@@ -515,7 +539,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-class SaveReportView(BaseReportView, ReportBreadcrumbs, SchedulerView):
+class SaveReportView(BaseReportView, SchedulerView):
     task_type = "report"
 
     def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
@@ -537,6 +561,24 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
             subreport_name_format = request.POST.get("child_report_name", "")
             recurrence = request.POST.get("recurrence", "")
             deadline_at = request.POST.get("start_date", datetime.now(timezone.utc).date())
+            object_selection = request.POST.get("object_selection", "")
+
+            query = {}
+            if object_selection == "query":
+                query = {
+                    "ooi_types": [t.__name__ for t in self.get_ooi_types()],
+                    "scan_level": self.get_ooi_scan_levels(),
+                    "scan_type": self.get_ooi_profile_types(),
+                    "search_string": self.search_string,
+                    "order_by": self.order_by,
+                    "asc_desc": self.sorting_order,
+                }
+
+            parent_report_type = None
+            if self.report_type == AggregateOrganisationReport:
+                parent_report_type = AggregateOrganisationReport.id
+            elif not self.report_type and subreport_name_format:
+                parent_report_type = ConcatenatedReport.id
 
             parent_report_type = None
             if self.report_type is not None:
@@ -547,7 +589,7 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
             schedule = self.convert_recurrence_to_cron_expressions(recurrence)
 
             report_recipe = self.create_report_recipe(
-                report_name_format, subreport_name_format, parent_report_type, schedule
+                report_name_format, subreport_name_format, parent_report_type, schedule, query
             )
 
             self.create_report_schedule(report_recipe, deadline_at)

rocky/rocky/locale/django.pot

@@ -9,7 +9,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-11-06 09:54+0000\n"
+"POT-Creation-Date: 2024-11-07 10:53+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"
@@ -3862,7 +3862,36 @@ msgstr[0] ""
 msgstr[1] ""
 
 #: reports/templates/partials/report_ooi_list.html
-msgid "Select which objects you want to include in your report."
+msgid ""
+"\n"
+"                Select which objects you want to include in your report. You "
+"can either continue\n"
+"                with a live set or you can select the objects manually from "
+"the table below.\n"
+"            "
+msgstr ""
+
+#: reports/templates/partials/report_ooi_list.html
+msgid ""
+"\n"
+"                A live set is a set of objects based on the applied "
+"filters.\n"
+"                Any object that matches this applied filter (now or in the "
+"future) will be used as\n"
+"                input for the scheduled report. If your live set filter (e."
+"g. 'hostnames' with\n"
+"                'L2 clearance' that are 'declared') shows 2 hostnames that "
+"match the filter today,\n"
+"                the scheduled report will run for those 2 hostnames. If you "
+"add 3 more hostnames\n"
+"                tomorrow (with the same filter criteria), your next "
+"scheduled report will contain\n"
+"                5 hostnames. Your live set will update as you go.\n"
+"            "
+msgstr ""
+
+#: reports/templates/partials/report_ooi_list.html
+msgid "Continue with live set"
 msgstr ""
 
 #: reports/templates/partials/report_ooi_list.html

rocky/tests/reports/test_base_report.py

@@ -35,9 +35,9 @@ def test_aggregate_report_choose_report_types(
 ):
     mocker.patch("reports.views.base.get_katalogus")
     kwargs = {"organization_code": client_member.organization.code}
-    url = reverse("aggregate_report_select_oois", kwargs=kwargs)
+    url = reverse("aggregate_report_select_report_types", kwargs=kwargs)
 
-    request = rf.get(url, {"observed_at": valid_time.strftime("%Y-%m-%d"), "ooi": "all"})
+    request = rf.post(url, {"observed_at": valid_time.strftime("%Y-%m-%d"), "ooi": "all"})
     request.resolver_match = resolve(url)
 
     setup_request(request, client_member.user)