Merge branch 'main' into feature/mula/restructure-folders

* main:
  Increase max number of PostgreSQL connections (#3889)
  Fix for task id as valid UUID (#3744)
  Add `auto_calculate_deadline` attribute to Scheduler (#3869)
  Ignore specific url parameters when following location headers (#3856)
  Let mailserver inherit l1 (#3704)
  Change plugins enabling in report flow to checkboxes (#3747)
  Fix rocky katalogus tests and delete unused fixtures (#3884)
  Enable/disable scheduled reports (#3871)

boefjes/boefjes/plugins/kat_answer_parser/normalize.py

@@ -10,4 +10,4 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
 
     bit_id = data["schema"].removeprefix("/bit/")
 
-    yield Config(ooi=input_ooi["primary_key"], bit_id=bit_id, config=data["answer"])
+    yield Config(ooi=data["answer_ooi"], bit_id=bit_id, config=data["answer"])

boefjes/tests/plugins/test_answer_parser.py

@@ -13,10 +13,10 @@ def test_config_yielded(normalizer_runner):
         normalizer_runner.run(meta, bytes(raw, "UTF-8"))
 
     with pytest.raises(ValidationError):
-        raw = '{"schema": "/bit/port-classification-ip", "answer": [{"key": "test"}]}'
+        raw = '{"schema": "/bit/port-classification-ip", "answer": [{"key": "test"}], "answer_ooi": "Network|internet"}'
         normalizer_runner.run(meta, bytes(raw, "UTF-8"))
 
-    raw = '{"schema": "/bit/port-classification-ip", "answer": {"key": "test"}}'
+    raw = '{"schema": "/bit/port-classification-ip", "answer": {"key": "test"}, "answer_ooi": "Network|internet"}'
     output = normalizer_runner.run(meta, bytes(raw, "UTF-8"))
 
     assert len(output.observations) == 1

docker-compose.yml

@@ -23,6 +23,9 @@ services:
       test: ["CMD", "gosu", "postgres", "pg_isready"]
       interval: 10s
       retries: 10
+    # Django runserver does not limit the number of threads. We need to increase
+    # the maximum number of connection to make sure that we don't hit the limit.
+    command: -c max_connections=500
     volumes:
       - postgres-data:/var/lib/postgresql/data
       - ./init-user-db.sh:/docker-entrypoint-initdb.d/init-user-db.sh

mula/scheduler/schedulers/scheduler.py

@@ -63,6 +63,7 @@ def __init__(
         callback: Callable[..., None] | None = None,
         max_tries: int = -1,
         create_schedule: bool = False,
+        auto_calculate_deadline: bool = True,
     ):
         """Initialize the Scheduler.
 
@@ -90,6 +91,7 @@ def __init__(
         self.max_tries: int = max_tries
         self.enabled: bool = True
         self.create_schedule: bool = create_schedule
+        self.auto_calculate_deadline: bool = auto_calculate_deadline
         self._last_activity: datetime | None = None
 
         # Queue
@@ -329,7 +331,7 @@ def post_push(self, item: models.Task) -> models.Task:
         # based on the item.
         if schedule_db.schedule is not None:
             schedule_db.deadline_at = cron.next_run(schedule_db.schedule)
-        else:
+        elif self.auto_calculate_deadline:
             schedule_db.deadline_at = self.calculate_deadline(item)
 
         self.ctx.datastores.schedule_store.update_schedule(schedule_db)

mula/scheduler/schedulers/schedulers/boefje.py

@@ -68,7 +68,14 @@ def __init__(
             pq_store=ctx.datastores.pq_store,
         )
 
-        super().__init__(ctx=ctx, queue=self.queue, scheduler_id=scheduler_id, callback=callback, create_schedule=True)
+        super().__init__(
+            ctx=ctx,
+            queue=self.queue,
+            scheduler_id=scheduler_id,
+            callback=callback,
+            create_schedule=True,
+            auto_calculate_deadline=True,
+        )
 
         # Priority ranker
         self.priority_ranker = BoefjeRanker(self.ctx)

mula/scheduler/schedulers/schedulers/normalizer.py

@@ -38,7 +38,6 @@ def __init__(
     ):
         self.logger: structlog.BoundLogger = structlog.getLogger(__name__)
         self.organisation: Organisation = organisation
-        self.create_schedule = False
 
         self.queue = queue or PriorityQueue(
             pq_id=scheduler_id,
@@ -48,7 +47,14 @@ def __init__(
             pq_store=ctx.datastores.pq_store,
         )
 
-        super().__init__(ctx=ctx, queue=self.queue, scheduler_id=scheduler_id, callback=callback)
+        super().__init__(
+            ctx=ctx,
+            queue=self.queue,
+            scheduler_id=scheduler_id,
+            callback=callback,
+            create_schedule=False,
+            auto_calculate_deadline=False,
+        )
 
         self.ranker = NormalizerRanker(ctx=self.ctx)
 

mula/scheduler/schedulers/schedulers/report.py

@@ -36,7 +36,14 @@ def __init__(
             pq_store=ctx.datastores.pq_store,
         )
 
-        super().__init__(ctx=ctx, queue=self.queue, scheduler_id=scheduler_id, callback=callback, create_schedule=True)
+        super().__init__(
+            ctx=ctx,
+            queue=self.queue,
+            scheduler_id=scheduler_id,
+            callback=callback,
+            create_schedule=True,
+            auto_calculate_deadline=False,
+        )
 
     def run(self) -> None:
         # Rescheduling

octopoes/bits/ask_url_params_to_ignore/ask_url_params_to_ignore.py

@@ -0,0 +1,17 @@
+import json
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from octopoes.models import OOI
+from octopoes.models.ooi.network import Network
+from octopoes.models.ooi.question import Question
+
+
+def run(input_ooi: Network, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]:
+    network = input_ooi
+
+    with (Path(__file__).parent / "question_schema.json").open() as f:
+        schema = json.load(f)
+
+    yield Question(ooi=network.reference, schema_id=schema["$id"], json_schema=json.dumps(schema))

octopoes/bits/ask_url_params_to_ignore/bit.py

@@ -0,0 +1,10 @@
+from bits.definitions import BitDefinition
+from octopoes.models.ooi.network import Network
+
+BIT = BitDefinition(
+    id="ask_url_params_to_ignore",
+    consumes=Network,
+    parameters=[],
+    min_scan_level=0,
+    module="bits.ask_url_params_to_ignore.ask_url_params_to_ignore",
+)

octopoes/bits/ask_url_params_to_ignore/question_schema.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema",
+  "$id": "/bit/oois-in-headers",
+  "type": "object",
+  "default": {},
+  "required": [
+    "ignored_url_parameters"
+  ],
+  "properties": {
+    "ignored_url_parameters": {
+      "description": "Comma separated list of url parameters that are ignored when following location headers.",
+      "type": "string",
+      "pattern": "^(\\s*(,*)[^,]+,?\\s*)*$",
+      "default": "session_id, phpsessid, jsessionid, cifd, cftoken, asp.net_sessionid"
+    }
+  }
+}

octopoes/bits/oois_in_headers/bit.py

@@ -2,5 +2,9 @@
 from octopoes.models.types import HTTPHeader
 
 BIT = BitDefinition(
-    id="oois-in-headers", consumes=HTTPHeader, parameters=[], module="bits.oois_in_headers.oois_in_headers"
+    id="oois-in-headers",
+    consumes=HTTPHeader,
+    parameters=[],
+    module="bits.oois_in_headers.oois_in_headers",
+    config_ooi_relation_path="HTTPHeader.resource.website.hostname.network",
 )

octopoes/bits/oois_in_headers/oois_in_headers.py

@@ -1,7 +1,7 @@
 import re
 from collections.abc import Iterator
 from typing import Any
-from urllib.parse import urljoin, urlparse
+from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse
 
 from pydantic import ValidationError
 
@@ -16,12 +16,33 @@ def is_url(input_str):
     return bool(result.scheme)
 
 
+def get_ignored_url_params(config: dict, config_key: str, default: list) -> list[str]:
+    ignored_url_params = config.get(config_key)
+    if ignored_url_params is None:
+        return default
+    return [param.strip() for param in ignored_url_params.split(",")] if ignored_url_params else []
+
+
+def remove_ignored_params(url: str, ignored_params: list[str]) -> str:
+    parsed_url = urlparse(url)
+    query_params = parse_qs(parsed_url.query)
+    if not query_params:
+        return url
+    filtered_params = {k: v for k, v in query_params.items() if k.lower() not in ignored_params}
+    new_query = urlencode(filtered_params, doseq=True)
+    new_url = urlunparse(
+        (parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, new_query, parsed_url.fragment)
+    )
+    return new_url
+
+
 def run(input_ooi: HTTPHeader, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]:
     network = Network(name="internet")
 
     if input_ooi.key.lower() == "location":
+        ignored_url_params = get_ignored_url_params(config, "ignored_url_parameters", [])
         if is_url(input_ooi.value):
-            u = URL(raw=input_ooi.value, network=network.reference)
+            u = URL(raw=remove_ignored_params(input_ooi.value, ignored_url_params), network=network.reference)
         else:
             # url is not a url but a relative path
             http_url = input_ooi.reference.tokenized.resource.web_url

octopoes/octopoes/models/ooi/dns/records.py

@@ -50,7 +50,7 @@ class DNSMXRecord(DNSRecord):
     object_type: Literal["DNSMXRecord"] = "DNSMXRecord"
     dns_record_type: Literal["MX"] = "MX"
 
-    mail_hostname: Reference | None = ReferenceField(Hostname, default=None)
+    mail_hostname: Reference | None = ReferenceField(Hostname, default=None, max_inherit_scan_level=1)
     preference: int | None = None
 
     _reverse_relation_names = {"hostname": "dns_mx_records", "mail_hostname": "mail_server_of"}

octopoes/tests/integration/test_ooi_deletion.py

@@ -131,9 +131,9 @@ def test_events_created_in_worker_during_handling(
         xtdb_octopoes_service.process_event(event)
     xtdb_octopoes_service.commit()
 
-    assert len(event_manager.queue) == 7  # Handling OOI delete event triggers Origin delete event
+    assert len(event_manager.queue) == 8  # Handling OOI delete event triggers Origin delete event
 
-    event = event_manager.queue[6]  # OOID]elete event
+    event = event_manager.queue[7]  # OOIDelete event
 
     assert isinstance(event, OriginDBEvent)
     assert event.operation_type.value == "delete"
@@ -229,7 +229,7 @@ def test_deletion_events_after_nxdomain(
     event_manager.complete_process_events(xtdb_octopoes_service)
 
     assert len(list(filter(lambda x: x.operation_type.value == "delete", event_manager.queue))) == 0
-    assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 7
+    assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 8
 
     nxd = NXDOMAIN(hostname=hostname.reference)
     xtdb_octopoes_service.ooi_repository.save(nxd, valid_time)
@@ -250,7 +250,7 @@ def test_deletion_events_after_nxdomain(
     event_manager.complete_process_events(xtdb_octopoes_service)
 
     assert len(list(filter(lambda x: x.operation_type.value == "delete", event_manager.queue))) >= 3
-    assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 5
+    assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 6
 
 
 @pytest.mark.xfail(reason="Wappalyzer works on wrong input objects (to be addressed)")

rocky/assets/css/components/system-tag.scss

@@ -0,0 +1,43 @@
+.system-tag,
+span.system-tag {
+  background-color: var(--colors-white);
+  border: 2px solid;
+  border-radius: var(--border-radius-xl);
+  padding: var(--spacing-grid-025) var(--spacing-grid-100);
+  box-sizing: border-box;
+  width: auto;
+
+  &::before {
+    content: none;
+  }
+
+  &.color-1 {
+    color: var(--colors-blue-600);
+    border-color: var(--colors-blue-600);
+  }
+
+  &.color-2 {
+    color: var(--colors-green-600);
+    border-color: var(--colors-green-600);
+  }
+
+  &.color-3 {
+    color: var(--colors-ochre-500);
+    border-color: var(--colors-ochre-500);
+  }
+
+  &.color-4 {
+    color: var(--colors-orange-600);
+    border-color: var(--colors-orange-600);
+  }
+
+  &.color-5 {
+    color: var(--colors-red-600);
+    border-color: var(--colors-red-600);
+  }
+
+  &.color-6 {
+    color: var(--colors-purrple-600);
+    border-color: var(--colors-purrple-600);
+  }
+}

rocky/assets/css/main.scss

@@ -67,6 +67,7 @@
 @import "components/sticky";
 @import "components/sticky-column";
 @import "components/state-tags";
+@import "components/system-tag";
 @import "components/table";
 @import "components/toggle";
 @import "components/toolbar";

rocky/assets/css/themes/soft/fundamentals/border-radii.scss

@@ -20,6 +20,10 @@
   border-radius: var(--border-radius-l);
 }
 
+.border-radius-xl {
+  border-radius: var(--border-radius-xl);
+}
+
 .border-radius-round {
   border-radius: var(--border-radius-round);
 }

rocky/components/modal/README.md

@@ -10,7 +10,7 @@ This outlines the basic usages and provides a code block example below, of how t
 
 ### Instantiate
 
-First you need to add `{% load component_tags %}` at the top of your template. Next you need to add the following code block at the bottom, to include the corresponding JS (if you haven't already you also need to add `{% load compress %}`).
+First you need to add `{% load component_tags %}` at the top of your template. Next you need to add the following code block at the bottom, to include the corresponding JS (if you haven't already you also need to add `{% load compress %}` and `{% load static %}`).
 
 ```
 {% block html_at_end_body %}
@@ -58,7 +58,7 @@ Including `{% component_css_dependencies %}` is needed to inject the reference t
 	{% fill "content" %}
 		<form  id="content-form"  class="horizontal-view"  action=""  method="post">
 			{% csrf_token %}
-			{% blocktranslate %}
+			{% blocktranslate with context_data_variable=context_data_variable %}
 				<p>You can use {{ context_data_variable }} and HTML here <code>valid_time</code>!</p>
 			{% endblocktranslate %}
 		</form>

rocky/reports/templates/forms/report_form_fields.html

@@ -18,7 +18,7 @@
     {% endfor %}
 {% endif %}
 {% for required_optional_plugin, plugins_ in plugins.items %}
-    {% for plugin in plugins_ %}<input type="hidden" name="plugin" value="{{ plugin.id }}">{% endfor %}
+    {% for plugin in plugins_ %}<input type="hidden" name="all_plugins" value="{{ plugin.id }}">{% endfor %}
 {% endfor %}
 {% if request.POST.choose_recurrence %}
     <input type="hidden"

rocky/reports/templates/partials/report_setup_scan.html

@@ -52,8 +52,13 @@ <h3>{% translate "Required plugins" %}</h3>
                     </div>
                     <div class="column-4 tiles plugins images-cover">
                         {% for required_plugin in plugins.required|dictsort:"enabled" %}
-                            {% include "partials/plugin_tile.html" with plugin=required_plugin plugin_report_types=plugin_data.plugin_report_types show_report_types="yes" plugin_report_types=plugin_data.plugin_report_types %}
+                            {% if required_plugin.enabled %}
+                                {% include "partials/plugin_tile.html" with form_id="continue-to-configuration" plugin_report_types=plugin_data.plugin_report_types show_report_types="yes" plugin=required_plugin remove_action_buttons="yes" add_checkbox="yes" %}
 
+                            {% else %}
+                                {% include "partials/plugin_tile.html" with form_id="continue-to-configuration" plugin_report_types=plugin_data.plugin_report_types show_report_types="yes" plugin=required_plugin remove_action_buttons="yes" add_checkbox="yes" checked="yes" %}
+
+                            {% endif %}
                         {% endfor %}
                     </div>
                 </div>
@@ -76,7 +81,7 @@ <h3>{% translate "Suggested plugins" %}</h3>
                     </div>
                     <div class="column-4 tiles plugins images-cover">
                         {% for optional_plugin in plugins.optional|dictsort:"enabled" %}
-                            {% include "partials/plugin_tile.html" with plugin=optional_plugin form_id="continue-to-configurationt" show_report_types="yes" plugin_report_types=plugin_data.plugin_report_types %}
+                            {% include "partials/plugin_tile.html" with form_id="continue-to-configuration" plugin_report_types=plugin_data.plugin_report_types show_report_types="yes" plugin=optional_plugin remove_action_buttons="yes" add_checkbox="yes" %}
 
                         {% endfor %}
                     </div>
@@ -98,7 +103,7 @@ <h3>{% translate "Suggested plugins" %}</h3>
                 {% include "forms/report_form_fields.html" %}
 
                 <button type="submit">
-                    {% translate "Continue" %}<span class="icon ti-chevron-right"></span>
+                    {% translate "Enable plugins and continue" %}<span class="icon ti-chevron-right"></span>
                 </button>
             </form>
         {% else %}

rocky/reports/templates/report_overview/modal_partials/enable_disable_schedule_modal.html

@@ -0,0 +1,24 @@
+{% load i18n %}
+
+{% component "modal" size="dialog-small" modal_id=modal_id %}
+{% fill "header" %}
+{% translate "Disable schedule" %}
+{% endfill %}
+{% fill "content" %}
+<form id="content-form"  class="horizontal-view"  action=""  method="post">
+    {% csrf_token %}
+    <p>
+        {% blocktranslate with report_name=schedule.recipe.report_name_format %}
+            Are you sure you want to disable the schedule for {{ report_name }}?
+            The recipe will still exist and the schedule can be enabled later on.
+        {% endblocktranslate %}
+    </p>
+</form>
+{% endfill %}
+{% fill "footer_buttons" %}
+<a class="button destructive"
+   href="{% url "enable_disable_scheduled_reports" organization.code %}?schedule_id={{ schedule.schedule_id }}">{% translate "Disable schedule" %}</a>
+<button class="ghost close-modal-button">{% translate "Cancel" %}</button>
+{% endfill %}
+{% endcomponent %}
+{% component_css_dependencies %}