Add new Boefje (#3400)

Signed-off-by: Donny Peeters <[email protected]>
Co-authored-by: Donny Peeters <[email protected]>
Co-authored-by: Donny Peeters <[email protected]>
Co-authored-by: Jan Klopper <[email protected]>

boefjes/boefjes/api.py

@@ -13,9 +13,10 @@
 from boefjes.clients.bytes_client import BytesAPIClient
 from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus
 from boefjes.config import settings
+from boefjes.dependencies.plugins import PluginService, get_plugin_service
 from boefjes.job_handler import get_environment_settings, get_octopoes_api_connector
 from boefjes.job_models import BoefjeMeta
-from boefjes.local_repository import LocalPluginRepository, get_local_repository
+from boefjes.models import PluginType
 from boefjes.plugins.models import _default_mime_types
 from octopoes.models import Reference
 from octopoes.models.exception import ObjectNotFoundException
@@ -88,14 +89,15 @@ async def root():
 def boefje_input(
     task_id: UUID,
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
-    local_repository: LocalPluginRepository = Depends(get_local_repository),
+    plugin_service: PluginService = Depends(get_plugin_service),
 ):
     task = get_task(task_id, scheduler_client)
 
     if task.status is not TaskStatus.RUNNING:
         raise HTTPException(status_code=403, detail="Task does not have status running")
 
-    boefje_meta = create_boefje_meta(task, local_repository)
+    plugin = plugin_service.by_plugin_id(task.data.boefje.id, task.data.organization)
+    boefje_meta = create_boefje_meta(task, plugin)
 
     output_url = str(settings.api).rstrip("/") + f"/api/v0/tasks/{task_id}"
     return BoefjeInput(task_id=task_id, output_url=output_url, boefje_meta=boefje_meta)
@@ -107,22 +109,23 @@ def boefje_output(
     boefje_output: BoefjeOutput,
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
     bytes_client: BytesAPIClient = Depends(get_bytes_client),
-    local_repository: LocalPluginRepository = Depends(get_local_repository),
+    plugin_service: PluginService = Depends(get_plugin_service),
 ):
     task = get_task(task_id, scheduler_client)
 
     if task.status is not TaskStatus.RUNNING:
         raise HTTPException(status_code=403, detail="Task does not have status running")
 
-    boefje_meta = create_boefje_meta(task, local_repository)
+    plugin = plugin_service.by_plugin_id(task.data.boefje.id, task.data.organization)
+    boefje_meta = create_boefje_meta(task, plugin)
     boefje_meta.started_at = task.modified_at
     boefje_meta.ended_at = datetime.now(timezone.utc)
 
     bytes_client.login()
     bytes_client.save_boefje_meta(boefje_meta)
 
     if boefje_output.files:
-        mime_types = _default_mime_types(task.data.boefje)
+        mime_types = _default_mime_types(boefje_meta.boefje).union(plugin.produces)
         for file in boefje_output.files:
             raw = base64.b64decode(file.content)
             # when supported, also save file.name to Bytes
@@ -148,14 +151,10 @@ def get_task(task_id, scheduler_client):
     return task
 
 
-def create_boefje_meta(task, local_repository):
-    boefje = task.data.boefje
-    boefje_resource = local_repository.by_id(boefje.id)
-    environment = get_environment_settings(task.data, boefje_resource.schema)
-
+def create_boefje_meta(task, plugin: PluginType) -> BoefjeMeta:
     organization = task.data.organization
     input_ooi = task.data.input_ooi
-    arguments = {"oci_arguments": boefje_resource.oci_arguments}
+    arguments = {"oci_arguments": plugin.oci_arguments}
 
     if input_ooi:
         reference = Reference.from_str(input_ooi)
@@ -168,10 +167,10 @@ def create_boefje_meta(task, local_repository):
 
     boefje_meta = BoefjeMeta(
         id=task.id,
-        boefje=boefje,
+        boefje=task.data.boefje,
         input_ooi=input_ooi,
         arguments=arguments,
         organization=organization,
-        environment=environment,
+        environment=get_environment_settings(task.data, plugin.schema),
     )
     return boefje_meta

boefjes/boefjes/app.py

@@ -8,13 +8,18 @@
 import structlog
 from httpx import HTTPError
 from pydantic import ValidationError
+from sqlalchemy.orm import sessionmaker
 
 from boefjes.clients.scheduler_client import SchedulerAPIClient, SchedulerClientInterface, Task, TaskStatus
 from boefjes.config import Settings
+from boefjes.dependencies.plugins import PluginService
 from boefjes.job_handler import BoefjeHandler, NormalizerHandler, bytes_api_client
 from boefjes.local import LocalBoefjeJobRunner, LocalNormalizerJobRunner
 from boefjes.local_repository import get_local_repository
 from boefjes.runtime_interfaces import Handler, WorkerManager
+from boefjes.sql.config_storage import create_config_storage
+from boefjes.sql.db import get_engine
+from boefjes.sql.plugin_storage import create_plugin_storage
 
 logger = structlog.get_logger(__name__)
 
@@ -256,9 +261,17 @@ def _start_working(
 
 def get_runtime_manager(settings: Settings, queue: WorkerManager.Queue, log_level: str) -> WorkerManager:
     local_repository = get_local_repository()
+
+    session = sessionmaker(bind=get_engine())()
+    plugin_service = PluginService(
+        create_plugin_storage(session),
+        create_config_storage(session),
+        local_repository,
+    )
+
     item_handler: Handler
     if queue is WorkerManager.Queue.BOEFJES:
-        item_handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), local_repository, bytes_api_client)
+        item_handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), plugin_service, bytes_api_client)
     else:
         item_handler = NormalizerHandler(
             LocalNormalizerJobRunner(local_repository), bytes_api_client, settings.scan_profile_whitelist

boefjes/boefjes/dependencies/plugins.py

@@ -204,7 +204,7 @@ def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> Plugi
         return plugin
 
 
-def get_plugin_service(organisation_id: str) -> Iterator[PluginService]:
+def get_plugin_service() -> Iterator[PluginService]:
     def closure(session: Session):
         return PluginService(
             create_plugin_storage(session),

boefjes/boefjes/job_handler.py

@@ -12,9 +12,9 @@
 
 from boefjes.clients.bytes_client import BytesAPIClient
 from boefjes.config import settings
+from boefjes.dependencies.plugins import PluginService
 from boefjes.docker_boefjes_runner import DockerBoefjesRunner
 from boefjes.job_models import BoefjeMeta, NormalizerMeta
-from boefjes.local_repository import LocalPluginRepository
 from boefjes.plugins.models import _default_mime_types
 from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner
 from boefjes.storage.interfaces import SettingsNotConformingToSchema
@@ -79,26 +79,30 @@ class BoefjeHandler(Handler):
     def __init__(
         self,
         job_runner: BoefjeJobRunner,
-        local_repository: LocalPluginRepository,
+        plugin_service: PluginService,
         bytes_client: BytesAPIClient,
     ):
         self.job_runner = job_runner
-        self.local_repository = local_repository
+        self.plugin_service = plugin_service
         self.bytes_client = bytes_client
 
     def handle(self, boefje_meta: BoefjeMeta) -> None:
         logger.info("Handling boefje %s[task_id=%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 
         # Check if this boefje is container-native, if so, continue using the Docker boefjes runner
-        boefje_resource = self.local_repository.by_id(boefje_meta.boefje.id)
-        if boefje_resource.oci_image:
+        plugin = self.plugin_service.by_plugin_id(boefje_meta.boefje.id, boefje_meta.organization)
+
+        if plugin.type != "boefje":
+            raise ValueError("Plugin id does not belong to a boefje")
+
+        if plugin.oci_image:
             logger.info(
                 "Delegating boefje %s[task_id=%s] to Docker runner with OCI image [%s]",
                 boefje_meta.boefje.id,
                 str(boefje_meta.id),
-                boefje_resource.oci_image,
+                plugin.oci_image,
             )
-            docker_runner = DockerBoefjesRunner(boefje_resource, boefje_meta)
+            docker_runner = DockerBoefjesRunner(plugin, boefje_meta)
             return docker_runner.run()
 
         if boefje_meta.input_ooi:
@@ -112,10 +116,10 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
 
             boefje_meta.arguments["input"] = ooi.serialize()
 
-        boefje_meta.runnable_hash = boefje_resource.runnable_hash
-        boefje_meta.environment = get_environment_settings(boefje_meta, boefje_resource.schema)
+        boefje_meta.runnable_hash = plugin.runnable_hash
+        boefje_meta.environment = get_environment_settings(boefje_meta, plugin.schema)
 
-        mime_types = _default_mime_types(boefje_meta.boefje)
+        mime_types = _default_mime_types(boefje_meta.boefje).union(plugin.produces)
 
         logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 

boefjes/tests/test_api.py

@@ -1,7 +1,10 @@
 from pathlib import Path
+from unittest import mock
 
 import boefjes.api
 from boefjes.clients.scheduler_client import TaskStatus
+from boefjes.dependencies.plugins import PluginService
+from boefjes.local_repository import get_local_repository
 from tests.conftest import MockSchedulerClient
 from tests.loading import get_dummy_data
 
@@ -26,6 +29,11 @@ def test_boefje_input_running(api, tmp_path):
     task = scheduler_client.pop_item("boefje")
     scheduler_client.patch_task(task.id, TaskStatus.RUNNING)
     api.app.dependency_overrides[boefjes.api.get_scheduler_client] = lambda: scheduler_client
+    api.app.dependency_overrides[boefjes.api.get_plugin_service] = lambda: PluginService(
+        mock.MagicMock(),
+        mock.MagicMock(),
+        get_local_repository(),
+    )
 
     boefjes.api.get_environment_settings = lambda *_: {}
     response = api.get("/api/v0/tasks/70da7d4f-f41f-4940-901b-d98a92e9014b")

boefjes/tests/test_tasks.py

@@ -8,12 +8,15 @@
 
 import pytest
 
+from boefjes.dependencies.plugins import PluginService
 from boefjes.job_handler import BoefjeHandler
 from boefjes.job_models import BoefjeMeta, InvalidReturnValueNormalizer, NormalizerMeta
 from boefjes.local import LocalBoefjeJobRunner, LocalNormalizerJobRunner
 from boefjes.local_repository import LocalPluginRepository
 from boefjes.models import Bit, Boefje, Normalizer, PluginType
 from boefjes.runtime_interfaces import JobRuntimeError
+from boefjes.sql.config_storage import create_config_storage
+from boefjes.sql.plugin_storage import create_plugin_storage
 from tests.loading import get_dummy_data
 
 
@@ -106,11 +109,19 @@ def test_handle_boefje_with_exception(self, mock_get_octopoes_api_connector, moc
             arguments={},
             organization="_dev",
         )
-
         local_repository = LocalPluginRepository(Path(__file__).parent / "modules")
 
+        mock_session = mock.MagicMock()
+        mock_session.query.all.return_value = []
+
+        plugin_service = PluginService(
+            create_plugin_storage(mock_session),
+            create_config_storage(mock_session),
+            local_repository,
+        )
+
         with pytest.raises(RuntimeError):  # Bytes still saves exceptions before they are reraised
-            BoefjeHandler(LocalBoefjeJobRunner(local_repository), local_repository, mock_bytes_api_client).handle(meta)
+            BoefjeHandler(LocalBoefjeJobRunner(local_repository), plugin_service, mock_bytes_api_client).handle(meta)
 
         mock_bytes_api_client.save_boefje_meta.assert_called_once_with(meta)
         mock_bytes_api_client.save_raw.assert_called_once()

boefjes/tools/run_boefje.py

@@ -8,6 +8,12 @@
 from pathlib import Path
 
 import click
+from sqlalchemy.orm import sessionmaker
+
+from boefjes.dependencies.plugins import PluginService
+from boefjes.sql.config_storage import create_config_storage
+from boefjes.sql.db import get_engine
+from boefjes.sql.plugin_storage import create_plugin_storage
 
 sys.path.append(str(Path(__file__).resolve().parent.parent))
 
@@ -31,7 +37,14 @@ def run_boefje(start_pdb, organization_code, boefje_id, input_ooi):
 
     local_repository = get_local_repository()
 
-    handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), local_repository, bytes_api_client)
+    session = sessionmaker(bind=get_engine())()
+    plugin_service = PluginService(
+        create_plugin_storage(session),
+        create_config_storage(session),
+        local_repository,
+    )
+
+    handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), plugin_service, bytes_api_client)
     try:
         handler.handle(meta)
     except Exception:

rocky/katalogus/client.py

@@ -4,6 +4,7 @@
 import structlog
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
+from httpx import codes
 from jsonschema.exceptions import SchemaError
 from jsonschema.validators import Draft202012Validator
 from pydantic import BaseModel, Field, field_serializer
@@ -38,10 +39,13 @@ def can_scan(self, member) -> bool:
 
 class Boefje(Plugin):
     scan_level: SCAN_LEVEL
-    consumes: set[type[OOI]]
+    consumes: set[type[OOI]] = Field(default_factory=set)
+    produces: set[str] = Field(default_factory=set)
     options: list[str] | None = None
     runnable_hash: str | None = None
-    produces: set[str]
+    schema: dict | None = None
+    oci_image: str | None = None
+    oci_arguments: list[str] = Field(default_factory=list)
 
     # use a custom field_serializer for `consumes`
     @field_serializer("consumes")
@@ -204,6 +208,19 @@ def get_cover(self, boefje_id: str) -> BytesIO:
         response.raise_for_status()
         return BytesIO(response.content)
 
+    def create_plugin(self, plugin: Plugin) -> None:
+        response = self.session.post(
+            f"{self.organization_uri}/plugins",
+            headers={"Content-Type": "application/json"},
+            content=plugin.model_dump_json(exclude_none=True),
+        )
+        response.raise_for_status()
+
+        if response.status_code == codes.CREATED:
+            logger.info("Plugin %s", plugin.name)
+        else:
+            logger.info("Plugin %s could not be created", plugin.name)
+
 
 def parse_boefje(boefje: dict) -> Boefje:
     scan_level = SCAN_LEVEL(boefje["scan_level"])
@@ -225,6 +242,9 @@ def parse_boefje(boefje: dict) -> Boefje:
         scan_level=scan_level,
         consumes=consumes,
         produces=boefje["produces"],
+        schema=boefje.get("schema"),
+        oci_image=boefje.get("oci_image"),
+        oci_arguments=boefje.get("oci_arguments", []),
     )
 
 

rocky/katalogus/templates/about_plugins.html

@@ -23,8 +23,9 @@ <h1>{% translate "About plugins" %}</h1>
                     <h2>Boefjes</h2>
                     <p>
                         {% blocktranslate trimmed %}
-                            Boefjes gather factual information, such as by calling an external scanning
-                            tool like nmap or using a database like shodan.
+                            Boefjes are used to scan for objects. They detect vulnerabilities,
+                            security issues, and give insight. Each boefje is a separate scan that
+                            can run on a selection of objects.
                         {% endblocktranslate %}
                     </p>
                     <h2>Normalizers</h2>