{% translate "Boefje setup" %}
++ {% blocktranslate %} + You can create a new Boefje. If you want more information on this, + you can check out the documentation. + {% endblocktranslate%} +
+ +diff --git a/boefjes/boefjes/api.py b/boefjes/boefjes/api.py index 22542898dfd..6919b331027 100644 --- a/boefjes/boefjes/api.py +++ b/boefjes/boefjes/api.py @@ -13,9 +13,10 @@ from boefjes.clients.bytes_client import BytesAPIClient from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus from boefjes.config import settings +from boefjes.dependencies.plugins import PluginService, get_plugin_service from boefjes.job_handler import get_environment_settings, get_octopoes_api_connector from boefjes.job_models import BoefjeMeta -from boefjes.local_repository import LocalPluginRepository, get_local_repository +from boefjes.models import PluginType from boefjes.plugins.models import _default_mime_types from octopoes.models import Reference from octopoes.models.exception import ObjectNotFoundException @@ -88,14 +89,15 @@ async def root(): def boefje_input( task_id: UUID, scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client), - local_repository: LocalPluginRepository = Depends(get_local_repository), + plugin_service: PluginService = Depends(get_plugin_service), ): task = get_task(task_id, scheduler_client) if task.status is not TaskStatus.RUNNING: raise HTTPException(status_code=403, detail="Task does not have status running") - boefje_meta = create_boefje_meta(task, local_repository) + plugin = plugin_service.by_plugin_id(task.data.boefje.id, task.data.organization) + boefje_meta = create_boefje_meta(task, plugin) output_url = str(settings.api).rstrip("/") + f"/api/v0/tasks/{task_id}" return BoefjeInput(task_id=task_id, output_url=output_url, boefje_meta=boefje_meta) @@ -107,14 +109,15 @@ def boefje_output( boefje_output: BoefjeOutput, scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client), bytes_client: BytesAPIClient = Depends(get_bytes_client), - local_repository: LocalPluginRepository = Depends(get_local_repository), + plugin_service: PluginService = Depends(get_plugin_service), ): task = get_task(task_id, scheduler_client) if task.status is not TaskStatus.RUNNING: raise HTTPException(status_code=403, detail="Task does not have status running") - boefje_meta = create_boefje_meta(task, local_repository) + plugin = plugin_service.by_plugin_id(task.data.boefje.id, task.data.organization) + boefje_meta = create_boefje_meta(task, plugin) boefje_meta.started_at = task.modified_at boefje_meta.ended_at = datetime.now(timezone.utc) @@ -122,7 +125,7 @@ def boefje_output( bytes_client.save_boefje_meta(boefje_meta) if boefje_output.files: - mime_types = _default_mime_types(task.data.boefje) + mime_types = _default_mime_types(boefje_meta.boefje).union(plugin.produces) for file in boefje_output.files: raw = base64.b64decode(file.content) # when supported, also save file.name to Bytes @@ -148,14 +151,10 @@ def get_task(task_id, scheduler_client): return task -def create_boefje_meta(task, local_repository): - boefje = task.data.boefje - boefje_resource = local_repository.by_id(boefje.id) - environment = get_environment_settings(task.data, boefje_resource.schema) - +def create_boefje_meta(task, plugin: PluginType) -> BoefjeMeta: organization = task.data.organization input_ooi = task.data.input_ooi - arguments = {"oci_arguments": boefje_resource.oci_arguments} + arguments = {"oci_arguments": plugin.oci_arguments} if input_ooi: reference = Reference.from_str(input_ooi) @@ -168,10 +167,10 @@ def create_boefje_meta(task, local_repository): boefje_meta = BoefjeMeta( id=task.id, - boefje=boefje, + boefje=task.data.boefje, input_ooi=input_ooi, arguments=arguments, organization=organization, - environment=environment, + environment=get_environment_settings(task.data, plugin.schema), ) return boefje_meta diff --git a/boefjes/boefjes/app.py b/boefjes/boefjes/app.py index 731cbd7e19c..c12f1e15bc9 100644 --- a/boefjes/boefjes/app.py +++ b/boefjes/boefjes/app.py @@ -8,13 +8,18 @@ import structlog from httpx import HTTPError from pydantic import ValidationError +from sqlalchemy.orm import sessionmaker from boefjes.clients.scheduler_client import SchedulerAPIClient, SchedulerClientInterface, Task, TaskStatus from boefjes.config import Settings +from boefjes.dependencies.plugins import PluginService from boefjes.job_handler import BoefjeHandler, NormalizerHandler, bytes_api_client from boefjes.local import LocalBoefjeJobRunner, LocalNormalizerJobRunner from boefjes.local_repository import get_local_repository from boefjes.runtime_interfaces import Handler, WorkerManager +from boefjes.sql.config_storage import create_config_storage +from boefjes.sql.db import get_engine +from boefjes.sql.plugin_storage import create_plugin_storage logger = structlog.get_logger(__name__) @@ -256,9 +261,17 @@ def _start_working( def get_runtime_manager(settings: Settings, queue: WorkerManager.Queue, log_level: str) -> WorkerManager: local_repository = get_local_repository() + + session = sessionmaker(bind=get_engine())() + plugin_service = PluginService( + create_plugin_storage(session), + create_config_storage(session), + local_repository, + ) + item_handler: Handler if queue is WorkerManager.Queue.BOEFJES: - item_handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), local_repository, bytes_api_client) + item_handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), plugin_service, bytes_api_client) else: item_handler = NormalizerHandler( LocalNormalizerJobRunner(local_repository), bytes_api_client, settings.scan_profile_whitelist diff --git a/boefjes/boefjes/dependencies/plugins.py b/boefjes/boefjes/dependencies/plugins.py index 080ff6d7da7..99a978f0fab 100644 --- a/boefjes/boefjes/dependencies/plugins.py +++ b/boefjes/boefjes/dependencies/plugins.py @@ -204,7 +204,7 @@ def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> Plugi return plugin -def get_plugin_service(organisation_id: str) -> Iterator[PluginService]: +def get_plugin_service() -> Iterator[PluginService]: def closure(session: Session): return PluginService( create_plugin_storage(session), diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index 7ad0f7247ad..51cdc12a09f 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -12,9 +12,9 @@ from boefjes.clients.bytes_client import BytesAPIClient from boefjes.config import settings +from boefjes.dependencies.plugins import PluginService from boefjes.docker_boefjes_runner import DockerBoefjesRunner from boefjes.job_models import BoefjeMeta, NormalizerMeta -from boefjes.local_repository import LocalPluginRepository from boefjes.plugins.models import _default_mime_types from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner from boefjes.storage.interfaces import SettingsNotConformingToSchema @@ -79,26 +79,30 @@ class BoefjeHandler(Handler): def __init__( self, job_runner: BoefjeJobRunner, - local_repository: LocalPluginRepository, + plugin_service: PluginService, bytes_client: BytesAPIClient, ): self.job_runner = job_runner - self.local_repository = local_repository + self.plugin_service = plugin_service self.bytes_client = bytes_client def handle(self, boefje_meta: BoefjeMeta) -> None: logger.info("Handling boefje %s[task_id=%s]", boefje_meta.boefje.id, str(boefje_meta.id)) # Check if this boefje is container-native, if so, continue using the Docker boefjes runner - boefje_resource = self.local_repository.by_id(boefje_meta.boefje.id) - if boefje_resource.oci_image: + plugin = self.plugin_service.by_plugin_id(boefje_meta.boefje.id, boefje_meta.organization) + + if plugin.type != "boefje": + raise ValueError("Plugin id does not belong to a boefje") + + if plugin.oci_image: logger.info( "Delegating boefje %s[task_id=%s] to Docker runner with OCI image [%s]", boefje_meta.boefje.id, str(boefje_meta.id), - boefje_resource.oci_image, + plugin.oci_image, ) - docker_runner = DockerBoefjesRunner(boefje_resource, boefje_meta) + docker_runner = DockerBoefjesRunner(plugin, boefje_meta) return docker_runner.run() if boefje_meta.input_ooi: @@ -112,10 +116,10 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: boefje_meta.arguments["input"] = ooi.serialize() - boefje_meta.runnable_hash = boefje_resource.runnable_hash - boefje_meta.environment = get_environment_settings(boefje_meta, boefje_resource.schema) + boefje_meta.runnable_hash = plugin.runnable_hash + boefje_meta.environment = get_environment_settings(boefje_meta, plugin.schema) - mime_types = _default_mime_types(boefje_meta.boefje) + mime_types = _default_mime_types(boefje_meta.boefje).union(plugin.produces) logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) diff --git a/boefjes/tests/test_api.py b/boefjes/tests/test_api.py index e1e95b7b50e..f6cac54d1f3 100644 --- a/boefjes/tests/test_api.py +++ b/boefjes/tests/test_api.py @@ -1,7 +1,10 @@ from pathlib import Path +from unittest import mock import boefjes.api from boefjes.clients.scheduler_client import TaskStatus +from boefjes.dependencies.plugins import PluginService +from boefjes.local_repository import get_local_repository from tests.conftest import MockSchedulerClient from tests.loading import get_dummy_data @@ -26,6 +29,11 @@ def test_boefje_input_running(api, tmp_path): task = scheduler_client.pop_item("boefje") scheduler_client.patch_task(task.id, TaskStatus.RUNNING) api.app.dependency_overrides[boefjes.api.get_scheduler_client] = lambda: scheduler_client + api.app.dependency_overrides[boefjes.api.get_plugin_service] = lambda: PluginService( + mock.MagicMock(), + mock.MagicMock(), + get_local_repository(), + ) boefjes.api.get_environment_settings = lambda *_: {} response = api.get("/api/v0/tasks/70da7d4f-f41f-4940-901b-d98a92e9014b") diff --git a/boefjes/tests/test_tasks.py b/boefjes/tests/test_tasks.py index 93b37e382d6..c311b25de76 100644 --- a/boefjes/tests/test_tasks.py +++ b/boefjes/tests/test_tasks.py @@ -8,12 +8,15 @@ import pytest +from boefjes.dependencies.plugins import PluginService from boefjes.job_handler import BoefjeHandler from boefjes.job_models import BoefjeMeta, InvalidReturnValueNormalizer, NormalizerMeta from boefjes.local import LocalBoefjeJobRunner, LocalNormalizerJobRunner from boefjes.local_repository import LocalPluginRepository from boefjes.models import Bit, Boefje, Normalizer, PluginType from boefjes.runtime_interfaces import JobRuntimeError +from boefjes.sql.config_storage import create_config_storage +from boefjes.sql.plugin_storage import create_plugin_storage from tests.loading import get_dummy_data @@ -106,11 +109,19 @@ def test_handle_boefje_with_exception(self, mock_get_octopoes_api_connector, moc arguments={}, organization="_dev", ) - local_repository = LocalPluginRepository(Path(__file__).parent / "modules") + mock_session = mock.MagicMock() + mock_session.query.all.return_value = [] + + plugin_service = PluginService( + create_plugin_storage(mock_session), + create_config_storage(mock_session), + local_repository, + ) + with pytest.raises(RuntimeError): # Bytes still saves exceptions before they are reraised - BoefjeHandler(LocalBoefjeJobRunner(local_repository), local_repository, mock_bytes_api_client).handle(meta) + BoefjeHandler(LocalBoefjeJobRunner(local_repository), plugin_service, mock_bytes_api_client).handle(meta) mock_bytes_api_client.save_boefje_meta.assert_called_once_with(meta) mock_bytes_api_client.save_raw.assert_called_once() diff --git a/boefjes/tools/run_boefje.py b/boefjes/tools/run_boefje.py index 2b578143383..73686f67437 100755 --- a/boefjes/tools/run_boefje.py +++ b/boefjes/tools/run_boefje.py @@ -8,6 +8,12 @@ from pathlib import Path import click +from sqlalchemy.orm import sessionmaker + +from boefjes.dependencies.plugins import PluginService +from boefjes.sql.config_storage import create_config_storage +from boefjes.sql.db import get_engine +from boefjes.sql.plugin_storage import create_plugin_storage sys.path.append(str(Path(__file__).resolve().parent.parent)) @@ -31,7 +37,14 @@ def run_boefje(start_pdb, organization_code, boefje_id, input_ooi): local_repository = get_local_repository() - handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), local_repository, bytes_api_client) + session = sessionmaker(bind=get_engine())() + plugin_service = PluginService( + create_plugin_storage(session), + create_config_storage(session), + local_repository, + ) + + handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), plugin_service, bytes_api_client) try: handler.handle(meta) except Exception: diff --git a/rocky/katalogus/client.py b/rocky/katalogus/client.py index ebb614b0b9e..33dccb5f6c8 100644 --- a/rocky/katalogus/client.py +++ b/rocky/katalogus/client.py @@ -4,6 +4,7 @@ import structlog from django.conf import settings from django.utils.translation import gettext_lazy as _ +from httpx import codes from jsonschema.exceptions import SchemaError from jsonschema.validators import Draft202012Validator from pydantic import BaseModel, Field, field_serializer @@ -38,10 +39,13 @@ def can_scan(self, member) -> bool: class Boefje(Plugin): scan_level: SCAN_LEVEL - consumes: set[type[OOI]] + consumes: set[type[OOI]] = Field(default_factory=set) + produces: set[str] = Field(default_factory=set) options: list[str] | None = None runnable_hash: str | None = None - produces: set[str] + schema: dict | None = None + oci_image: str | None = None + oci_arguments: list[str] = Field(default_factory=list) # use a custom field_serializer for `consumes` @field_serializer("consumes") @@ -204,6 +208,19 @@ def get_cover(self, boefje_id: str) -> BytesIO: response.raise_for_status() return BytesIO(response.content) + def create_plugin(self, plugin: Plugin) -> None: + response = self.session.post( + f"{self.organization_uri}/plugins", + headers={"Content-Type": "application/json"}, + content=plugin.model_dump_json(exclude_none=True), + ) + response.raise_for_status() + + if response.status_code == codes.CREATED: + logger.info("Plugin %s", plugin.name) + else: + logger.info("Plugin %s could not be created", plugin.name) + def parse_boefje(boefje: dict) -> Boefje: scan_level = SCAN_LEVEL(boefje["scan_level"]) @@ -225,6 +242,9 @@ def parse_boefje(boefje: dict) -> Boefje: scan_level=scan_level, consumes=consumes, produces=boefje["produces"], + schema=boefje.get("schema"), + oci_image=boefje.get("oci_image"), + oci_arguments=boefje.get("oci_arguments", []), ) diff --git a/rocky/katalogus/templates/about_plugins.html b/rocky/katalogus/templates/about_plugins.html index ac295f89512..94b1ee65957 100644 --- a/rocky/katalogus/templates/about_plugins.html +++ b/rocky/katalogus/templates/about_plugins.html @@ -23,8 +23,9 @@
{% blocktranslate trimmed %} - Boefjes gather factual information, such as by calling an external scanning - tool like nmap or using a database like shodan. + Boefjes are used to scan for objects. They detect vulnerabilities, + security issues, and give insight. Each boefje is a separate scan that + can run on a selection of objects. {% endblocktranslate %}
{{ plugin.description }}
{% endif %}- {% translate "Description" %} -
-{{ plugin.description }}
- {% endif %}+ {% if plugin.produces %} +
+ {% blocktranslate trimmed with plugin_name=plugin.name %} + {{ plugin_name }} can produce the following output: + {% endblocktranslate %} +
++
- {{ plugin.produce }} -
- {% include "tasks/plugin_detail_task_list.html" %} - -
+ {% endif %}+ {% blocktranslate %} + You can create a new Boefje. If you want more information on this, + you can check out the documentation. + {% endblocktranslate%} +
+ +{% blocktranslate trimmed %} - Boefjes gather factual information, such as by calling an - external scanning tool like nmap or using a database like shodan. + Boefjes are used to scan for objects. They detect vulnerabilities, + security issues, and give insight. Each boefje is a separate scan that + can run on a selection of objects. {% endblocktranslate %}
{{ object_list|length }} Boefje{{ object_list|pluralize:"s" }} {% translate "available" %}
+ {% translate "The container image for this Boefje is:" %} {{ plugin.oci_image }} +
++ {% blocktranslate %} + Boefje variants that use the same container image. For more + information about Boefje variants you can read the documentation. + {% endblocktranslate %} +
+{% translate "Name" %} | +{% translate "Scan level" %} | +{% translate "Published by" %} | +{% translate "Status" %} | ++ |
---|---|---|---|---|
{{ variant.name }}name | +{{ variant.scan_level }}scan_level | +{{ variant.published_by }}published_by | +{{ variant.status }}status | ++ + | +
+ {% translate "Arguments" %}+{% translate "The following arguments are used for this Boefje variant." %} +
+
+ Some code example + |
+
+ {% translate "This Boefje has no variants yet." %} + {% blocktranslate trimmed %} + You can make a variant and change the arguments and JSON Schema + to customize it to fit your needs. + {% endblocktranslate %} +
+ {% endif %} +{% translate "Settings" %}
-{% translate "Variable" %} | +{% translate "Value" %} | +{% translate "Required" %} | +|||||
---|---|---|---|---|---|---|---|
{% translate "Name" %} | -{% translate "Value" %} | -{% translate "Required" %} | -{% translate "Action" %} | -||||
{{ setting.name }} | -- {% if setting.value is None %} - {% translate "Unset" %} - {% elif setting.secret %} - ••••••••••••• - {% else %} - {{ setting.value }} - {% endif %} - | -- {% if setting.required %} - {% translate "Yes" %} - {% else %} - {% translate "No" %} - {% endif %} - | +{{ setting.name }} | +{% if setting.value is None %} - | - {% translate "Add" %} - | -+ - + {% elif setting.secret %} + ••••••••••••• {% else %} - | - {% translate "Edit" %} - | + {{ setting.value }} {% endif %} -
A description of the boefje explaining in short what it can do. This will " +"both be displayed inside the KAT-alogus and on the Boefje details page.
" +msgstr "" + +#: tools/forms/settings.py +msgid "" +"If any other settings are needed for your Boefje, add these as a JSON " +"Schema, otherwise, leave the field empty or 'null'.
This JSON is " +"used as the basis for a form for the user. When the user enables this Boefje " +"they can get the option to give extra information. For example, it can " +"contain an API key that the script requires.
More information about " +"what the schema.json file looks like can be found here.
" +msgstr "" + +#: tools/forms/settings.py +msgid "" +"Select the object type that your Boefje consumes.
This object type " +"triggers the Boefje to run. Whenever this OOI gets added, this Boefje will " +"run with that OOI.
" +msgstr "" + +#: tools/forms/settings.py +msgid "" +"Add a set of mime types that are produced by this Boefje, separated by " +"commas. For example: 'text/html', 'image/jpeg' or 'boefje/" +"{boefje-id}'
These output mime types will be shown on the Boefje " +"detail page as information for other users.
" +msgstr "" + +#: tools/forms/settings.py +msgid "" +"Select a clearance level for your Boefje. For more information about the " +"different clearance levels please check the documentation." +"
" +msgstr "" + #: tools/forms/settings.py msgid "Depth of the tree." msgstr "" @@ -5551,6 +5693,12 @@ msgstr "" msgid "Assigned clearance level" msgstr "" +#: rocky/templates/organizations/organization_member_list.html +#: rocky/templates/organizations/organization_settings.html +#: rocky/views/ooi_edit.py rocky/views/organization_edit.py +msgid "Edit" +msgstr "" + #: rocky/templates/organizations/organization_member_list.html msgid "Super user" msgstr "" diff --git a/rocky/rocky/templates/partials/form/field_input.html b/rocky/rocky/templates/partials/form/field_input.html index fcaad77193a..d2d09ec65e7 100644 --- a/rocky/rocky/templates/partials/form/field_input.html +++ b/rocky/rocky/templates/partials/form/field_input.html @@ -1,7 +1,8 @@ {% load i18n %}{{ field.label_tag }}
+{{ field.field.widget.attrs.description }}
{% if form_view != "vertical" %}{% translate "There are no tasks for" %} {{ plugin.name }}
- {% include "tasks/partials/task_filter.html" %} - +{% translate "There are no tasks for" %} {{ plugin.name }}.
{% else %} -{% translate "List of tasks for" %} {{ plugin.name }}
+{% translate "List of tasks for" %} {{ plugin.name }}:
{% include "tasks/partials/task_filter.html" %}