diff --git a/boefjes/.ci/docker-compose.yml b/boefjes/.ci/docker-compose.yml index 407d1678336..bf7ee7aeadd 100644 --- a/boefjes/.ci/docker-compose.yml +++ b/boefjes/.ci/docker-compose.yml @@ -8,6 +8,7 @@ services: command: sh -c 'python -m pytest -v tests/integration' depends_on: - ci_katalogus-db + - ci_katalogus env_file: - .ci/.env.test volumes: @@ -84,7 +85,7 @@ services: - .ci/.env.test ci_xtdb: - image: "ghcr.io/dekkers/xtdb-http-multinode:v1.0.8" + image: "ghcr.io/dekkers/xtdb-http-multinode:v1.1.0" ci_octopoes_api_worker: build: @@ -103,8 +104,15 @@ services: hard: 262144 ci_katalogus: - image: "docker.io/wiremock/wiremock:2.34.0" - volumes: - - .ci/wiremock:/home/wiremock + build: + context: .. + dockerfile: boefjes/Dockerfile + args: + - ENVIRONMENT=dev + command: uvicorn boefjes.katalogus.root:app --host 0.0.0.0 --port 8080 + depends_on: + - ci_katalogus-db env_file: - .ci/.env.test + volumes: + - .:/app/boefjes diff --git a/boefjes/.ci/wiremock/mappings/organisations.json b/boefjes/.ci/wiremock/mappings/organisations.json deleted file mode 100644 index 13124e6222f..00000000000 --- a/boefjes/.ci/wiremock/mappings/organisations.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "request": { - "method": "GET", - "url": "/v1/organisations" - }, - "response": { - "status": 200, - "jsonBody": { - "_dev": { - "id": "_dev", - "name": "Development Organisation" - } - } - } -} diff --git a/boefjes/boefjes/api.py b/boefjes/boefjes/api.py index 72ac9ccc96c..8209a14f991 100644 --- a/boefjes/boefjes/api.py +++ b/boefjes/boefjes/api.py @@ -2,6 +2,7 @@ import multiprocessing from datetime import datetime, timezone from enum import Enum +from multiprocessing.context import ForkContext, ForkProcess from uuid import UUID import structlog @@ -13,18 +14,20 @@ from boefjes.clients.bytes_client import BytesAPIClient from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus from boefjes.config import settings +from boefjes.dependencies.plugins import PluginService, get_plugin_service from boefjes.job_handler import get_environment_settings, get_octopoes_api_connector from boefjes.job_models import BoefjeMeta -from boefjes.local_repository import LocalPluginRepository, get_local_repository +from boefjes.models import PluginType from boefjes.plugins.models import _default_mime_types from octopoes.models import Reference from octopoes.models.exception import ObjectNotFoundException app = FastAPI(title="Boefje API") logger = structlog.get_logger(__name__) +ctx: ForkContext = multiprocessing.get_context("fork") -class UvicornServer(multiprocessing.Process): +class UvicornServer(ForkProcess): def __init__(self, config: Config): super().__init__() self.server = Server(config=config) @@ -58,7 +61,7 @@ class StatusEnum(str, Enum): class File(BaseModel): name: str | None = None - content: str = Field(..., contentEncoding="base64") + content: str = Field(json_schema_extra={"contentEncoding": "base64"}) tags: list[str] | None = None @@ -88,14 +91,15 @@ async def root(): def boefje_input( task_id: UUID, scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client), - local_repository: LocalPluginRepository = Depends(get_local_repository), + plugin_service: PluginService = Depends(get_plugin_service), ): task = get_task(task_id, scheduler_client) if task.status is not TaskStatus.RUNNING: raise HTTPException(status_code=403, detail="Task does not have status running") - boefje_meta = create_boefje_meta(task, local_repository) + plugin = plugin_service.by_plugin_id(task.data.boefje.id, task.data.organization) + boefje_meta = create_boefje_meta(task, plugin) output_url = str(settings.api).rstrip("/") + f"/api/v0/tasks/{task_id}" return BoefjeInput(task_id=task_id, output_url=output_url, boefje_meta=boefje_meta) @@ -107,14 +111,15 @@ def boefje_output( boefje_output: BoefjeOutput, scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client), bytes_client: BytesAPIClient = Depends(get_bytes_client), - local_repository: LocalPluginRepository = Depends(get_local_repository), + plugin_service: PluginService = Depends(get_plugin_service), ): task = get_task(task_id, scheduler_client) if task.status is not TaskStatus.RUNNING: raise HTTPException(status_code=403, detail="Task does not have status running") - boefje_meta = create_boefje_meta(task, local_repository) + plugin = plugin_service.by_plugin_id(task.data.boefje.id, task.data.organization) + boefje_meta = create_boefje_meta(task, plugin) boefje_meta.started_at = task.modified_at boefje_meta.ended_at = datetime.now(timezone.utc) @@ -122,7 +127,7 @@ def boefje_output( bytes_client.save_boefje_meta(boefje_meta) if boefje_output.files: - mime_types = _default_mime_types(task.data.boefje) + mime_types = _default_mime_types(boefje_meta.boefje) for file in boefje_output.files: raw = base64.b64decode(file.content) # when supported, also save file.name to Bytes @@ -148,15 +153,10 @@ def get_task(task_id, scheduler_client): return task -def create_boefje_meta(task, local_repository): - boefje = task.data.boefje - boefje_resource = local_repository.by_id(boefje.id) - env_keys = boefje_resource.environment_keys - environment = get_environment_settings(task.data, env_keys) if env_keys else {} - +def create_boefje_meta(task, plugin: PluginType) -> BoefjeMeta: organization = task.data.organization input_ooi = task.data.input_ooi - arguments = {"oci_arguments": boefje_resource.oci_arguments} + arguments = {"oci_arguments": plugin.oci_arguments} if input_ooi: reference = Reference.from_str(input_ooi) @@ -169,10 +169,10 @@ def create_boefje_meta(task, local_repository): boefje_meta = BoefjeMeta( id=task.id, - boefje=boefje, + boefje=task.data.boefje, input_ooi=input_ooi, arguments=arguments, organization=organization, - environment=environment, + environment=get_environment_settings(task.data, plugin.boefje_schema), ) return boefje_meta diff --git a/boefjes/boefjes/app.py b/boefjes/boefjes/app.py index 731cbd7e19c..15c3fd2376b 100644 --- a/boefjes/boefjes/app.py +++ b/boefjes/boefjes/app.py @@ -1,22 +1,30 @@ -import multiprocessing as mp +import multiprocessing import os import signal import sys import time +from multiprocessing.context import ForkContext +from multiprocessing.process import BaseProcess from queue import Queue import structlog from httpx import HTTPError from pydantic import ValidationError +from sqlalchemy.orm import sessionmaker from boefjes.clients.scheduler_client import SchedulerAPIClient, SchedulerClientInterface, Task, TaskStatus from boefjes.config import Settings +from boefjes.dependencies.plugins import PluginService from boefjes.job_handler import BoefjeHandler, NormalizerHandler, bytes_api_client from boefjes.local import LocalBoefjeJobRunner, LocalNormalizerJobRunner from boefjes.local_repository import get_local_repository from boefjes.runtime_interfaces import Handler, WorkerManager +from boefjes.sql.config_storage import create_config_storage +from boefjes.sql.db import get_engine +from boefjes.sql.plugin_storage import create_plugin_storage logger = structlog.get_logger(__name__) +ctx: ForkContext = multiprocessing.get_context("fork") class SchedulerWorkerManager(WorkerManager): @@ -31,11 +39,11 @@ def __init__( self.scheduler_client = scheduler_client self.settings = settings - manager = mp.Manager() + manager = ctx.Manager() self.task_queue = manager.Queue() # multiprocessing.Queue() will not work on macOS, see mp.Queue.qsize() self.handling_tasks = manager.dict() - self.workers: list[mp.Process] = [] + self.workers: list[BaseProcess] = [] logger.setLevel(log_level) @@ -45,13 +53,13 @@ def run(self, queue_type: WorkerManager.Queue) -> None: logger.info("Created worker pool for queue '%s'", queue_type.value) self.workers = [ - mp.Process(target=_start_working, args=self._worker_args()) for _ in range(self.settings.pool_size) + ctx.Process(target=_start_working, args=self._worker_args()) for _ in range(self.settings.pool_size) ] for worker in self.workers: worker.start() - signal.signal(signal.SIGINT, lambda signum, _: self.exit(queue_type, signum)) - signal.signal(signal.SIGTERM, lambda signum, _: self.exit(queue_type, signum)) + signal.signal(signal.SIGINT, lambda signum, _: self.exit(signum)) + signal.signal(signal.SIGTERM, lambda signum, _: self.exit(signum)) while True: try: @@ -68,7 +76,7 @@ def run(self, queue_type: WorkerManager.Queue) -> None: # been called yet. if not self.exited: logger.exception("Exiting worker...") - self.exit(queue_type) + self.exit() raise @@ -93,18 +101,18 @@ def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue): all_queues_empty = True - for queue_type in queues: - logger.debug("Popping from queue %s", queue_type.id) + for queue in queues: + logger.debug("Popping from queue %s", queue.id) try: - p_item = self.scheduler_client.pop_item(queue_type.id) + p_item = self.scheduler_client.pop_item(queue.id) except (HTTPError, ValidationError): logger.exception("Popping task from scheduler failed, sleeping 10 seconds") time.sleep(10) continue if not p_item: - logger.debug("Queue %s empty", queue_type.id) + logger.debug("Queue %s empty", queue.id) continue all_queues_empty = False @@ -153,13 +161,13 @@ def _check_workers(self) -> None: self._cleanup_pending_worker_task(worker) worker.close() - new_worker = mp.Process(target=_start_working, args=self._worker_args()) + new_worker = ctx.Process(target=_start_working, args=self._worker_args()) new_worker.start() new_workers.append(new_worker) self.workers = new_workers - def _cleanup_pending_worker_task(self, worker: mp.Process) -> None: + def _cleanup_pending_worker_task(self, worker: BaseProcess) -> None: if worker.pid not in self.handling_tasks: logger.debug("No pending task found for Worker[pid=%s, %s]", worker.pid, _format_exit_code(worker.exitcode)) return @@ -169,7 +177,7 @@ def _cleanup_pending_worker_task(self, worker: mp.Process) -> None: try: task = self.scheduler_client.get_task(handling_task_id) - if task.status is TaskStatus.DISPATCHED: + if task.status is TaskStatus.DISPATCHED or task.status is TaskStatus.RUNNING: try: self.scheduler_client.patch_task(task.id, TaskStatus.FAILED) logger.warning("Set status to failed in the scheduler for task[id=%s]", handling_task_id) @@ -181,7 +189,7 @@ def _cleanup_pending_worker_task(self, worker: mp.Process) -> None: def _worker_args(self) -> tuple: return self.task_queue, self.item_handler, self.scheduler_client, self.handling_tasks - def exit(self, queue_type: WorkerManager.Queue, signum: int | None = None): + def exit(self, signum: int | None = None): try: if signum: logger.info("Received %s, exiting", signal.Signals(signum).name) @@ -191,7 +199,7 @@ def exit(self, queue_type: WorkerManager.Queue, signum: int | None = None): for p_item in items: try: - self.scheduler_client.push_item(queue_type.value, p_item) + self.scheduler_client.push_item(p_item) except HTTPError: logger.exception("Rescheduling task failed[id=%s]", p_item.id) @@ -226,7 +234,7 @@ def _format_exit_code(exitcode: int | None) -> str: def _start_working( - task_queue: mp.Queue, + task_queue: multiprocessing.Queue, handler: Handler, scheduler_client: SchedulerClientInterface, handling_tasks: dict[int, str], @@ -239,6 +247,7 @@ def _start_working( handling_tasks[os.getpid()] = str(p_item.id) try: + scheduler_client.patch_task(p_item.id, TaskStatus.RUNNING) handler.handle(p_item.data) status = TaskStatus.COMPLETED except Exception: # noqa @@ -248,17 +257,27 @@ def _start_working( raise finally: try: - scheduler_client.patch_task(p_item.id, status) # Note: implicitly, we have p_item.id == task_id - logger.info("Set status to %s in the scheduler for task[id=%s]", status, p_item.data.id) + if scheduler_client.get_task(p_item.id).status == TaskStatus.RUNNING: + # The docker runner could have handled this already + scheduler_client.patch_task(p_item.id, status) # Note that implicitly, we have p_item.id == task_id + logger.info("Set status to %s in the scheduler for task[id=%s]", status, p_item.data.id) except HTTPError: logger.exception("Could not patch scheduler task to %s", status.value) def get_runtime_manager(settings: Settings, queue: WorkerManager.Queue, log_level: str) -> WorkerManager: local_repository = get_local_repository() + + session = sessionmaker(bind=get_engine())() + plugin_service = PluginService( + create_plugin_storage(session), + create_config_storage(session), + local_repository, + ) + item_handler: Handler if queue is WorkerManager.Queue.BOEFJES: - item_handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), local_repository, bytes_api_client) + item_handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), plugin_service, bytes_api_client) else: item_handler = NormalizerHandler( LocalNormalizerJobRunner(local_repository), bytes_api_client, settings.scan_profile_whitelist diff --git a/boefjes/boefjes/clients/bytes_client.py b/boefjes/boefjes/clients/bytes_client.py index c2698523183..b96369065c1 100644 --- a/boefjes/boefjes/clients/bytes_client.py +++ b/boefjes/boefjes/clients/bytes_client.py @@ -1,5 +1,6 @@ import typing import uuid +from base64 import b64encode from collections.abc import Callable, Set from functools import wraps from typing import Any @@ -73,7 +74,7 @@ def _get_token(self) -> str: @retry_with_login def save_boefje_meta(self, boefje_meta: BoefjeMeta) -> None: - response = self._session.post("/bytes/boefje_meta", content=boefje_meta.json(), headers=self.headers) + response = self._session.post("/bytes/boefje_meta", content=boefje_meta.model_dump_json(), headers=self.headers) self._verify_response(response) @@ -86,7 +87,9 @@ def get_boefje_meta(self, boefje_meta_id: str) -> BoefjeMeta: @retry_with_login def save_normalizer_meta(self, normalizer_meta: NormalizerMeta) -> None: - response = self._session.post("/bytes/normalizer_meta", content=normalizer_meta.json(), headers=self.headers) + response = self._session.post( + "/bytes/normalizer_meta", content=normalizer_meta.model_dump_json(), headers=self.headers + ) self._verify_response(response) @@ -99,17 +102,25 @@ def get_normalizer_meta(self, normalizer_meta_id: uuid.UUID) -> NormalizerMeta: @retry_with_login def save_raw(self, boefje_meta_id: str, raw: str | bytes, mime_types: Set[str] = frozenset()) -> UUID: - headers = {"content-type": "application/octet-stream"} - headers.update(self.headers) + file_name = "raw" # The name provides a key for all ids returned, so this is arbitrary as we only upload 1 file + response = self._session.post( "/bytes/raw", - content=raw, - headers=headers, - params={"mime_types": list(mime_types), "boefje_meta_id": boefje_meta_id}, + json={ + "files": [ + { + "name": file_name, + "content": b64encode(raw if isinstance(raw, bytes) else raw.encode()).decode(), + "tags": list(mime_types), + } + ] + }, + headers=self.headers, + params={"boefje_meta_id": str(boefje_meta_id)}, ) - self._verify_response(response) - return UUID(response.json()["id"]) + + return UUID(response.json()[file_name]) @retry_with_login def get_raw(self, raw_data_id: str) -> bytes: diff --git a/boefjes/boefjes/clients/scheduler_client.py b/boefjes/boefjes/clients/scheduler_client.py index 5e07d83d0be..bb342001475 100644 --- a/boefjes/boefjes/clients/scheduler_client.py +++ b/boefjes/boefjes/clients/scheduler_client.py @@ -42,7 +42,7 @@ class SchedulerClientInterface: def get_queues(self) -> list[Queue]: raise NotImplementedError() - def pop_item(self, queue: str) -> Task | None: + def pop_item(self, queue_id: str) -> Task | None: raise NotImplementedError() def patch_task(self, task_id: uuid.UUID, status: TaskStatus) -> None: @@ -51,7 +51,7 @@ def patch_task(self, task_id: uuid.UUID, status: TaskStatus) -> None: def get_task(self, task_id: uuid.UUID) -> Task: raise NotImplementedError() - def push_item(self, queue_id: str, p_item: Task) -> None: + def push_item(self, p_item: Task) -> None: raise NotImplementedError() @@ -69,14 +69,14 @@ def get_queues(self) -> list[Queue]: return TypeAdapter(list[Queue]).validate_json(response.content) - def pop_item(self, queue: str) -> Task | None: - response = self._session.post(f"/queues/{queue}/pop") + def pop_item(self, queue_id: str) -> Task | None: + response = self._session.post(f"/queues/{queue_id}/pop") self._verify_response(response) return TypeAdapter(Task | None).validate_json(response.content) - def push_item(self, queue_id: str, p_item: Task) -> None: - response = self._session.post(f"/queues/{queue_id}/push", content=p_item.json()) + def push_item(self, p_item: Task) -> None: + response = self._session.post(f"/queues/{p_item.scheduler_id}/push", content=p_item.model_dump_json()) self._verify_response(response) def patch_task(self, task_id: uuid.UUID, status: TaskStatus) -> None: diff --git a/boefjes/boefjes/dependencies/plugins.py b/boefjes/boefjes/dependencies/plugins.py index ccb3187c0e2..20dc9b7dc65 100644 --- a/boefjes/boefjes/dependencies/plugins.py +++ b/boefjes/boefjes/dependencies/plugins.py @@ -16,7 +16,8 @@ from boefjes.sql.plugin_storage import create_plugin_storage from boefjes.storage.interfaces import ( ConfigStorage, - ExistingPluginId, + DuplicatePlugin, + IntegrityError, NotFound, PluginNotFound, PluginStorage, @@ -98,7 +99,7 @@ def clone_settings_to_organisation(self, from_organisation: str, to_organisation self.set_enabled_by_id(plugin_id, to_organisation, enabled=True) def upsert_settings(self, settings: dict, organisation_id: str, plugin_id: str): - self._assert_settings_match_schema(settings, organisation_id, plugin_id) + self._assert_settings_match_schema(settings, plugin_id) self._put_boefje(plugin_id) return self.config_storage.upsert(organisation_id, plugin_id, settings=settings) @@ -106,30 +107,58 @@ def upsert_settings(self, settings: dict, organisation_id: str, plugin_id: str): def create_boefje(self, boefje: Boefje) -> None: try: self.local_repo.by_id(boefje.id) - raise ExistingPluginId(boefje.id) + raise DuplicatePlugin("id") except KeyError: - self.plugin_storage.create_boefje(boefje) + try: + plugin = self.local_repo.by_name(boefje.name) + + if plugin.type == "boefje": + raise DuplicatePlugin("name") + else: + try: + with self.plugin_storage as storage: + storage.create_boefje(boefje) + except IntegrityError as error: + raise DuplicatePlugin(self._translate_duplicate_plugin(error.message)) + except KeyError: + try: + with self.plugin_storage as storage: + storage.create_boefje(boefje) + except IntegrityError as error: + raise DuplicatePlugin(self._translate_duplicate_plugin(error.message)) + + def _translate_duplicate_plugin(self, error_message): + translations = {"boefje_plugin_id": "id", "boefje_name": "name"} + return next((value for key, value in translations.items() if key in error_message), None) def create_normalizer(self, normalizer: Normalizer) -> None: try: self.local_repo.by_id(normalizer.id) - raise ExistingPluginId(normalizer.id) + raise DuplicatePlugin("id") except KeyError: - self.plugin_storage.create_normalizer(normalizer) + try: + plugin = self.local_repo.by_name(normalizer.name) + + if plugin.types == "normalizer": + raise DuplicatePlugin("name") + else: + self.plugin_storage.create_normalizer(normalizer) + except KeyError: + self.plugin_storage.create_normalizer(normalizer) def _put_boefje(self, boefje_id: str) -> None: """Check existence of a boefje, and insert a database entry if it concerns a local boefje""" try: self.plugin_storage.boefje_by_id(boefje_id) - except PluginNotFound: + except PluginNotFound as e: try: plugin = self.local_repo.by_id(boefje_id) except KeyError: - raise + raise e if plugin.type != "boefje": - raise + raise e self.plugin_storage.create_boefje(plugin) def _put_normalizer(self, normalizer_id: str) -> None: @@ -150,18 +179,13 @@ def _put_normalizer(self, normalizer_id: str) -> None: def delete_settings(self, organisation_id: str, plugin_id: str): self.config_storage.delete(organisation_id, plugin_id) - try: - self._assert_settings_match_schema({}, organisation_id, plugin_id) - except SettingsNotConformingToSchema: - logger.warning("Making sure %s is disabled for %s because settings are deleted", plugin_id, organisation_id) - - self.set_enabled_by_id(plugin_id, organisation_id, False) + # We don't check the schema anymore because we can provide entries through the global environment as well def schema(self, plugin_id: str) -> dict | None: try: boefje = self.plugin_storage.boefje_by_id(plugin_id) - return boefje.schema + return boefje.boefje_schema except PluginNotFound: return self.local_repo.schema(plugin_id) @@ -184,9 +208,7 @@ def description(self, plugin_id: str, organisation_id: str) -> str: return "" def set_enabled_by_id(self, plugin_id: str, organisation_id: str, enabled: bool): - if enabled: - all_settings = self.get_all_settings(organisation_id, plugin_id) - self._assert_settings_match_schema(all_settings, organisation_id, plugin_id) + # We don't check the schema anymore because we can provide entries through the global environment as well try: self._put_boefje(plugin_id) @@ -195,14 +217,14 @@ def set_enabled_by_id(self, plugin_id: str, organisation_id: str, enabled: bool) self.config_storage.upsert(organisation_id, plugin_id, enabled=enabled) - def _assert_settings_match_schema(self, all_settings: dict, organisation_id: str, plugin_id: str): + def _assert_settings_match_schema(self, all_settings: dict, plugin_id: str): schema = self.schema(plugin_id) if schema: # No schema means that there is nothing to assert try: validate(instance=all_settings, schema=schema) except ValidationError as e: - raise SettingsNotConformingToSchema(organisation_id, plugin_id, e.message) from e + raise SettingsNotConformingToSchema(plugin_id, e.message) from e def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> PluginType: with contextlib.suppress(KeyError, NotFound): @@ -211,7 +233,7 @@ def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> Plugi return plugin -def get_plugin_service(organisation_id: str) -> Iterator[PluginService]: +def get_plugin_service() -> Iterator[PluginService]: def closure(session: Session): return PluginService( create_plugin_storage(session), @@ -231,5 +253,6 @@ def get_plugins_filter_parameters( ids: list[str] | None = Query(None), plugin_type: Literal["boefje", "normalizer", "bit"] | None = None, state: bool | None = None, + oci_image: str | None = None, ) -> FilterParameters: - return FilterParameters(q=q, ids=ids, type=plugin_type, state=state) + return FilterParameters(q=q, ids=ids, type=plugin_type, state=state, oci_image=oci_image) diff --git a/boefjes/boefjes/docker_boefjes_runner.py b/boefjes/boefjes/docker_boefjes_runner.py index f28c64055ae..fd5527acdf8 100644 --- a/boefjes/boefjes/docker_boefjes_runner.py +++ b/boefjes/boefjes/docker_boefjes_runner.py @@ -36,7 +36,6 @@ def run(self) -> None: stderr_mime_types = boefjes.plugins.models._default_mime_types(self.boefje_meta.boefje) task_id = self.boefje_meta.id - self.scheduler_client.patch_task(task_id, TaskStatus.RUNNING) self.boefje_meta.started_at = datetime.now(timezone.utc) try: @@ -63,7 +62,11 @@ def run(self) -> None: # have to raise exception to prevent _start_working function from setting status to completed raise RuntimeError("Boefje did not call output API endpoint") except ContainerError as e: - logger.exception("Container error") + logger.error( + "Container for task %s failed and returned exit status %d, stderr saved to bytes", + task_id, + e.exit_status, + ) # save container log (stderr) to bytes self.bytes_api_client.login() @@ -75,9 +78,9 @@ def run(self) -> None: logger.error("Failed to save boefje meta to bytes, continuing anyway") self.bytes_api_client.save_raw(task_id, e.stderr, stderr_mime_types) self.scheduler_client.patch_task(task_id, TaskStatus.FAILED) - # have to raise exception to prevent _start_working function from setting status to completed - raise e - except (APIError, ImageNotFound) as e: - logger.exception("API error or image not found") + except ImageNotFound: + logger.error("Docker image %s not found", self.boefje_resource.oci_image) + self.scheduler_client.patch_task(task_id, TaskStatus.FAILED) + except APIError as e: + logger.error("Docker API error: %s", e) self.scheduler_client.patch_task(task_id, TaskStatus.FAILED) - raise e diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index a7e0d4feca1..038842a4090 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -7,14 +7,17 @@ import httpx import structlog from httpx import HTTPError +from jsonschema.exceptions import ValidationError +from jsonschema.validators import validate from boefjes.clients.bytes_client import BytesAPIClient from boefjes.config import settings +from boefjes.dependencies.plugins import PluginService from boefjes.docker_boefjes_runner import DockerBoefjesRunner from boefjes.job_models import BoefjeMeta, NormalizerMeta -from boefjes.local_repository import LocalPluginRepository from boefjes.plugins.models import _default_mime_types from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner +from boefjes.storage.interfaces import SettingsNotConformingToSchema from octopoes.api.models import Affirmation, Declaration, Observation from octopoes.connector.octopoes import OctopoesAPIConnector from octopoes.models import Reference, ScanLevel @@ -35,7 +38,7 @@ def get_octopoes_api_connector(org_code: str) -> OctopoesAPIConnector: return OctopoesAPIConnector(str(settings.octopoes_api), org_code) -def get_environment_settings(boefje_meta: BoefjeMeta, environment_keys: list[str]) -> dict[str, str]: +def get_environment_settings(boefje_meta: BoefjeMeta, schema: dict | None = None) -> dict[str, str]: try: katalogus_api = str(settings.katalogus_api).rstrip("/") response = httpx.get( @@ -43,47 +46,63 @@ def get_environment_settings(boefje_meta: BoefjeMeta, environment_keys: list[str timeout=30, ) response.raise_for_status() - environment = response.json() - - # Add prefixed BOEFJE_* global environment variables - for key, value in os.environ.items(): - if key.startswith("BOEFJE_"): - katalogus_key = key.split("BOEFJE_", 1)[1] - # Only pass the environment variable if it is not explicitly set through the katalogus, - # if and only if they are defined in boefje.json - if katalogus_key in environment_keys and katalogus_key not in environment: - environment[katalogus_key] = value - - return {k: str(v) for k, v in environment.items() if k in environment_keys} except HTTPError: logger.exception("Error getting environment settings") raise + allowed_keys = schema.get("properties", []) if schema else [] + new_env = { + key.split("BOEFJE_", 1)[1]: value + for key, value in os.environ.items() + if key.startswith("BOEFJE_") and key in allowed_keys + } + + settings_from_katalogus = response.json() + + for key, value in settings_from_katalogus.items(): + if key in allowed_keys: + new_env[key] = value + + # The schema, besides dictating that a boefje cannot run if it is not matched, also provides an extra safeguard: + # it is possible to inject code if arguments are passed that "escape" the call to a tool. Hence, we should enforce + # the schema somewhere and make the schema as strict as possible. + if schema is not None: + try: + validate(instance=new_env, schema=schema) + except ValidationError as e: + raise SettingsNotConformingToSchema(boefje_meta.boefje.id, e.message) from e + + return new_env + class BoefjeHandler(Handler): def __init__( self, job_runner: BoefjeJobRunner, - local_repository: LocalPluginRepository, + plugin_service: PluginService, bytes_client: BytesAPIClient, ): self.job_runner = job_runner - self.local_repository = local_repository + self.plugin_service = plugin_service self.bytes_client = bytes_client def handle(self, boefje_meta: BoefjeMeta) -> None: logger.info("Handling boefje %s[task_id=%s]", boefje_meta.boefje.id, str(boefje_meta.id)) # Check if this boefje is container-native, if so, continue using the Docker boefjes runner - boefje_resource = self.local_repository.by_id(boefje_meta.boefje.id) - if boefje_resource.oci_image: + plugin = self.plugin_service.by_plugin_id(boefje_meta.boefje.id, boefje_meta.organization) + + if plugin.type != "boefje": + raise ValueError("Plugin id does not belong to a boefje") + + if plugin.oci_image: logger.info( "Delegating boefje %s[task_id=%s] to Docker runner with OCI image [%s]", boefje_meta.boefje.id, str(boefje_meta.id), - boefje_resource.oci_image, + plugin.oci_image, ) - docker_runner = DockerBoefjesRunner(boefje_resource, boefje_meta) + docker_runner = DockerBoefjesRunner(plugin, boefje_meta) return docker_runner.run() if boefje_meta.input_ooi: @@ -97,18 +116,14 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: boefje_meta.arguments["input"] = ooi.serialize() - env_keys = boefje_resource.environment_keys - - boefje_meta.runnable_hash = boefje_resource.runnable_hash - boefje_meta.environment = get_environment_settings(boefje_meta, env_keys) if env_keys else {} - - mime_types = _default_mime_types(boefje_meta.boefje) + boefje_meta.runnable_hash = plugin.runnable_hash + boefje_meta.environment = get_environment_settings(boefje_meta, plugin.boefje_schema) logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) boefje_meta.started_at = datetime.now(timezone.utc) - boefje_results: list[tuple[set, bytes | str]] + boefje_results: list[tuple[set, bytes | str]] = [] try: boefje_results = self.job_runner.run(boefje_meta, boefje_meta.environment) @@ -135,7 +150,9 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: ) else: valid_mimetypes.add(mimetype) - raw_file_id = self.bytes_client.save_raw(boefje_meta.id, output, mime_types.union(valid_mimetypes)) + raw_file_id = self.bytes_client.save_raw( + boefje_meta.id, output, _default_mime_types(boefje_meta.boefje).union(valid_mimetypes) + ) logger.info( "Saved raw file %s for boefje %s[%s]", raw_file_id, boefje_meta.boefje.id, boefje_meta.id ) @@ -209,6 +226,23 @@ def handle(self, normalizer_meta: NormalizerMeta) -> None: ) ) + if ( + normalizer_meta.raw_data.boefje_meta.input_ooi # No input OOI means no deletion propagation + and not (results.observations or results.declarations or results.affirmations) + ): + # There were no results found, which we still need to signal to Octopoes for deletion propagation + + connector.save_observation( + Observation( + method=normalizer_meta.normalizer.id, + source=Reference.from_str(normalizer_meta.raw_data.boefje_meta.input_ooi), + source_method=normalizer_meta.raw_data.boefje_meta.boefje.id, + task_id=normalizer_meta.id, + valid_time=normalizer_meta.raw_data.boefje_meta.ended_at, + result=[], + ) + ) + corrected_scan_profiles = [] for profile in results.scan_profiles: profile.level = ScanLevel( diff --git a/boefjes/boefjes/katalogus/plugins.py b/boefjes/boefjes/katalogus/plugins.py index 134243ad963..87065119feb 100644 --- a/boefjes/boefjes/katalogus/plugins.py +++ b/boefjes/boefjes/katalogus/plugins.py @@ -1,8 +1,11 @@ import datetime from functools import partial +import structlog +from croniter import croniter from fastapi import APIRouter, Body, Depends, HTTPException, status from fastapi.responses import FileResponse, JSONResponse, Response +from jsonschema.exceptions import SchemaError from jsonschema.validators import Draft202012Validator from pydantic import BaseModel, Field, field_validator @@ -15,7 +18,7 @@ from boefjes.katalogus.organisations import check_organisation_exists from boefjes.models import FilterParameters, PaginationParameters, PluginType from boefjes.sql.plugin_storage import get_plugin_storage -from boefjes.storage.interfaces import PluginStorage +from boefjes.storage.interfaces import DuplicatePlugin, IntegrityError, NotAllowed, PluginStorage router = APIRouter( prefix="/organisations/{organisation_id}", @@ -23,6 +26,8 @@ dependencies=[Depends(check_organisation_exists)], ) +logger = structlog.get_logger(__name__) + # check if query matches plugin id, name or description def _plugin_matches_query(plugin: PluginType, query: str) -> bool: @@ -65,6 +70,10 @@ def list_plugins( if filter_params.state is not None: plugins = filter(lambda x: x.enabled is filter_params.state, plugins) + # filter plugins by oci_image + if filter_params.oci_image is not None: + plugins = filter(lambda x: x.type == "boefje" and x.oci_image == filter_params.oci_image, plugins) + # filter plugins by scan level for boefje plugins plugins = list(filter(lambda x: x.type != "boefje" or x.scan_level >= filter_params.scan_level, plugins)) @@ -90,14 +99,17 @@ def get_plugin( @router.post("/plugins", status_code=status.HTTP_201_CREATED) def add_plugin(plugin: PluginType, plugin_service: PluginService = Depends(get_plugin_service)): - with plugin_service as service: - plugin.static = False # Creation through the API implies that these cannot be static + try: + with plugin_service as service: + plugin.static = False # Creation through the API implies that these cannot be static - if plugin.type == "boefje": - return service.create_boefje(plugin) + if plugin.type == "boefje": + return service.create_boefje(plugin) - if plugin.type == "normalizer": - return service.create_normalizer(plugin) + if plugin.type == "normalizer": + return service.create_normalizer(plugin) + except DuplicatePlugin as error: + raise HTTPException(status.HTTP_400_BAD_REQUEST, error.message) raise HTTPException(status.HTTP_400_BAD_REQUEST, "Creation of Bits is not supported") @@ -123,22 +135,33 @@ class BoefjeIn(BaseModel): version: str | None = None created: datetime.datetime | None = None description: str | None = None - environment_keys: list[str] = Field(default_factory=list) scan_level: int = 1 consumes: set[str] = Field(default_factory=set) produces: set[str] = Field(default_factory=set) - schema: dict | None = None + boefje_schema: dict | None = None + cron: str | None = None + interval: int | None = None oci_image: str | None = None oci_arguments: list[str] = Field(default_factory=list) - @field_validator("schema") + @field_validator("boefje_schema") @classmethod def json_schema_valid(cls, schema: dict | None) -> dict | None: if schema is not None: - Draft202012Validator.check_schema(schema) - return schema + try: + Draft202012Validator.check_schema(schema) + except SchemaError as e: + raise ValueError("The schema field is not a valid JSON schema") from e + + return schema - return None + @field_validator("cron") + @classmethod + def cron_valid(cls, cron: str | None) -> str | None: + if cron is not None: + croniter(cron) # Raises a ValueError + + return cron @router.patch("/boefjes/{boefje_id}", status_code=status.HTTP_204_NO_CONTENT) @@ -147,8 +170,15 @@ def update_boefje( boefje: BoefjeIn, storage: PluginStorage = Depends(get_plugin_storage), ): - with storage as p: - p.update_boefje(boefje_id, boefje.model_dump(exclude_unset=True)) + # todo: update boefje should be done in the plugin service + try: + with storage as p: + try: + p.update_boefje(boefje_id, boefje.model_dump(exclude_unset=True)) + except NotAllowed: + raise HTTPException(status.HTTP_403_FORBIDDEN, "Updating a static plugin is not allowed") + except IntegrityError as error: + raise HTTPException(status.HTTP_400_BAD_REQUEST, error.message) @router.delete("/boefjes/{boefje_id}", status_code=status.HTTP_204_NO_CONTENT) @@ -167,7 +197,6 @@ class NormalizerIn(BaseModel): version: str | None = None created: datetime.datetime | None = None description: str | None = None - environment_keys: list[str] = Field(default_factory=list) consumes: list[str] = Field(default_factory=list) # mime types (and/ or boefjes) produces: list[str] = Field(default_factory=list) # oois diff --git a/boefjes/boefjes/katalogus/root.py b/boefjes/boefjes/katalogus/root.py index 8aa0c1683c5..771adffb945 100644 --- a/boefjes/boefjes/katalogus/root.py +++ b/boefjes/boefjes/katalogus/root.py @@ -5,7 +5,6 @@ import structlog from fastapi import APIRouter, FastAPI, Request, status from fastapi.responses import JSONResponse, RedirectResponse -from jsonschema.exceptions import SchemaError from opentelemetry import trace from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor @@ -20,7 +19,7 @@ from boefjes.katalogus import organisations, plugins from boefjes.katalogus import settings as settings_router from boefjes.katalogus.version import __version__ -from boefjes.storage.interfaces import NotAllowed, NotFound, StorageError +from boefjes.storage.interfaces import IntegrityError, NotAllowed, NotFound, StorageError with settings.log_cfg.open() as f: logging.config.dictConfig(json.load(f)) @@ -89,19 +88,19 @@ def not_allowed_handler(request: Request, exc: NotAllowed): ) -@app.exception_handler(StorageError) -def storage_error_handler(request: Request, exc: StorageError): +@app.exception_handler(IntegrityError) +def integrity_error_handler(request: Request, exc: IntegrityError): return JSONResponse( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + status_code=status.HTTP_400_BAD_REQUEST, content={"message": exc.message}, ) -@app.exception_handler(SchemaError) -def schema_error_handler(request: Request, exc: StorageError): +@app.exception_handler(StorageError) +def storage_error_handler(request: Request, exc: StorageError): return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={"message": "Invalid jsonschema provided"}, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"message": exc.message}, ) diff --git a/boefjes/boefjes/local_repository.py b/boefjes/boefjes/local_repository.py index a29fb92ba35..719b26f06a8 100644 --- a/boefjes/boefjes/local_repository.py +++ b/boefjes/boefjes/local_repository.py @@ -47,6 +47,19 @@ def by_id(self, plugin_id: str) -> PluginType: raise KeyError(f"Can't find plugin {plugin_id}") + def by_name(self, plugin_name: str) -> PluginType: + boefjes = {resource.boefje.name: resource for resource in self.resolve_boefjes().values()} + + if plugin_name in boefjes: + return boefjes[plugin_name].boefje + + normalizers = {resource.normalizer.name: resource for resource in self.resolve_normalizers().values()} + + if plugin_name in normalizers: + return normalizers[plugin_name].normalizer + + raise KeyError(f"Can't find plugin {plugin_name}") + def schema(self, id_: str) -> dict | None: boefjes = self.resolve_boefjes() diff --git a/boefjes/boefjes/logging.json b/boefjes/boefjes/logging.json index 5ab07731cae..489c18d9681 100644 --- a/boefjes/boefjes/logging.json +++ b/boefjes/boefjes/logging.json @@ -3,8 +3,7 @@ "disable_existing_loggers": 0, "formatters": { "default": { - "format": "%(asctime)s [%(process)d] [%(levelname)s] [%(module)s] %(message)s", - "datefmt": "[%Y-%m-%d %H:%M:%S %z]" + "format": "%(message)s" } }, "handlers": { diff --git a/boefjes/boefjes/migrations/versions/5be152459a7b_introduce_schema_field_to_boefje_model.py b/boefjes/boefjes/migrations/versions/5be152459a7b_introduce_schema_field_to_boefje_model.py index 2cd63145aa5..470385e1879 100644 --- a/boefjes/boefjes/migrations/versions/5be152459a7b_introduce_schema_field_to_boefje_model.py +++ b/boefjes/boefjes/migrations/versions/5be152459a7b_introduce_schema_field_to_boefje_model.py @@ -10,11 +10,9 @@ import sqlalchemy as sa from alembic import op -from sqlalchemy.orm import sessionmaker +from sqlalchemy import text from boefjes.local_repository import get_local_repository -from boefjes.sql.plugin_storage import create_plugin_storage -from boefjes.storage.interfaces import PluginNotFound # revision identifiers, used by Alembic. revision = "5be152459a7b" @@ -30,29 +28,21 @@ def upgrade() -> None: op.add_column("boefje", sa.Column("schema", sa.JSON(), nullable=True)) local_repo = get_local_repository() - session = sessionmaker(bind=op.get_bind())() + connection = op.get_bind() - with create_plugin_storage(session) as storage: + with connection.begin(): plugins = local_repo.get_all() logger.info("Found %s plugins", len(plugins)) for plugin in local_repo.get_all(): schema = local_repo.schema(plugin.id) - if schema: - try: - # This way we avoid the safeguard that updating static boefjes is not allowed - instance = storage._db_boefje_instance_by_id(plugin.id) - instance.schema = schema - storage.session.add(instance) - logger.info("Updated database entry for plugin %s", plugin.id) - except PluginNotFound: - logger.info("No database entry for plugin %s", plugin.id) - continue + query = text("UPDATE boefje SET schema = :schema WHERE plugin_id = :plugin_id") # noqa: S608 + connection.execute(query, {"schema": schema, "plugin_id": plugin.id}) + logger.info("Updated any database entries for plugin %s", plugin.id) else: logger.info("No schema present for plugin %s", plugin.id) - session.close() # ### end Alembic commands ### diff --git a/boefjes/boefjes/migrations/versions/870fc302b852_remove_environment_keys_field.py b/boefjes/boefjes/migrations/versions/870fc302b852_remove_environment_keys_field.py new file mode 100644 index 00000000000..7bdfbd9e024 --- /dev/null +++ b/boefjes/boefjes/migrations/versions/870fc302b852_remove_environment_keys_field.py @@ -0,0 +1,37 @@ +"""Remove environment keys field + +Revision ID: 870fc302b852 +Revises: 5be152459a7b +Create Date: 2024-08-20 06:08:20.943924 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "870fc302b852" +down_revision = "5be152459a7b" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("boefje", "environment_keys") + op.drop_column("normalizer", "environment_keys") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "normalizer", + sa.Column("environment_keys", postgresql.ARRAY(sa.VARCHAR(length=128)), autoincrement=False, nullable=False), + ) + op.add_column( + "boefje", + sa.Column("environment_keys", postgresql.ARRAY(sa.VARCHAR(length=128)), autoincrement=False, nullable=False), + ) + # ### end Alembic commands ### diff --git a/boefjes/boefjes/migrations/versions/9f48560b0000_add_schedule_interval_fields.py b/boefjes/boefjes/migrations/versions/9f48560b0000_add_schedule_interval_fields.py new file mode 100644 index 00000000000..901020b2f65 --- /dev/null +++ b/boefjes/boefjes/migrations/versions/9f48560b0000_add_schedule_interval_fields.py @@ -0,0 +1,30 @@ +"""Add cron field + +Revision ID: 9f48560b0000 +Revises: 870fc302b852 +Create Date: 2024-09-18 13:12:40.926394 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "9f48560b0000" +down_revision = "a2c8d54b0124" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("boefje", sa.Column("cron", sa.String(length=128), nullable=True)) + op.add_column("boefje", sa.Column("interval", sa.Integer(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("boefje", "interval") + op.drop_column("boefje", "cron") + # ### end Alembic commands ### diff --git a/boefjes/boefjes/migrations/versions/a2c8d54b0124_unique_plugin_names.py b/boefjes/boefjes/migrations/versions/a2c8d54b0124_unique_plugin_names.py new file mode 100644 index 00000000000..84759f49cff --- /dev/null +++ b/boefjes/boefjes/migrations/versions/a2c8d54b0124_unique_plugin_names.py @@ -0,0 +1,29 @@ +"""Unique plugin names + +Revision ID: a2c8d54b0124 +Revises: 870fc302b852 +Create Date: 2024-09-18 14:46:00.881022 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a2c8d54b0124" +down_revision = "870fc302b852" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint("unique_boefje_name", "boefje", ["name"]) + op.create_unique_constraint("unique_normalizer_name", "normalizer", ["name"]) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("unique_normalizer_name", "normalizer", type_="unique") + op.drop_constraint("unique_boefje_name", "boefje", type_="unique") + # ### end Alembic commands ### diff --git a/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py b/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py index 40a44d504d3..d46f360b703 100644 --- a/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py +++ b/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py @@ -95,7 +95,7 @@ def upgrade() -> None: str(boefje.scan_level), list(boefje.consumes), list(boefje.produces), - boefje.environment_keys, + ["TEST_KEY"], boefje.oci_image, boefje.oci_arguments, boefje.version, @@ -137,7 +137,7 @@ def upgrade() -> None: str(boefje.scan_level), list(boefje.consumes), list(boefje.produces), - boefje.environment_keys, + ["TEST_KEY"], boefje.oci_image, boefje.oci_arguments, boefje.version, @@ -177,7 +177,7 @@ def upgrade() -> None: normalizer.description, normalizer.consumes, normalizer.produces, - normalizer.environment_keys, + ["TEST_KEY"], normalizer.version, ) for normalizer in normalizers_to_insert diff --git a/boefjes/boefjes/models.py b/boefjes/boefjes/models.py index 4881b26008a..752de2aa298 100644 --- a/boefjes/boefjes/models.py +++ b/boefjes/boefjes/models.py @@ -2,6 +2,8 @@ from enum import Enum from typing import Literal +from croniter import croniter +from jsonschema.exceptions import SchemaError from jsonschema.validators import Draft202012Validator from pydantic import BaseModel, Field, field_validator @@ -13,11 +15,10 @@ class Organisation(BaseModel): class Plugin(BaseModel): id: str - name: str | None = None + name: str version: str | None = None created: datetime.datetime | None = None description: str | None = None - environment_keys: list[str] = Field(default_factory=list) enabled: bool = False static: bool = True # We need to differentiate between local and remote plugins to know which ones can be deleted @@ -30,17 +31,31 @@ class Boefje(Plugin): scan_level: int = 1 consumes: set[str] = Field(default_factory=set) produces: set[str] = Field(default_factory=set) - schema: dict | None = None + boefje_schema: dict | None = None + cron: str | None = None + interval: int | None = None runnable_hash: str | None = None oci_image: str | None = None oci_arguments: list[str] = Field(default_factory=list) - @field_validator("schema") + @field_validator("boefje_schema") @classmethod def json_schema_valid(cls, schema: dict) -> dict: if schema is not None: - Draft202012Validator.check_schema(schema) - return schema + try: + Draft202012Validator.check_schema(schema) + except SchemaError as e: + raise ValueError("The schema field is not a valid JSON schema") from e + + return schema + + @field_validator("cron") + @classmethod + def cron_valid(cls, cron: str | None) -> str | None: + if cron is not None: + croniter(cron) # Raises a ValueError + + return cron class Config: validate_assignment = True @@ -80,3 +95,4 @@ class FilterParameters(BaseModel): ids: list[str] | None = None state: bool | None = None scan_level: int = 0 + oci_image: str | None = None diff --git a/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json b/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json index f840cded2ad..d342474d1e8 100644 --- a/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json +++ b/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json @@ -1,7 +1,7 @@ { "id": "adr-validator-normalize", "name": "API Design Rules validator", - "description": "TODO", + "description": "Parses and validates the API Design Rules (ADR). https://www.forumstandaardisatie.nl/open-standaarden/rest-api-design-rules", "consumes": [ "boefje/adr-validator" ], diff --git a/boefjes/boefjes/plugins/kat_answer_parser/normalizer.json b/boefjes/boefjes/plugins/kat_answer_parser/normalizer.json index 922b333697f..e10ab9a92c6 100644 --- a/boefjes/boefjes/plugins/kat_answer_parser/normalizer.json +++ b/boefjes/boefjes/plugins/kat_answer_parser/normalizer.json @@ -1,7 +1,7 @@ { "id": "kat_answer_parser", "name": "Answer Parser", - "description": "Parses the answers from Config objects.", + "description": "Parses the answers from 'Config' objects. Config OOIs are used when your policies and objects need different treatment from the usual setup.", "consumes": [ "answer" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/boefje.json b/boefjes/boefjes/plugins/kat_binaryedge/boefje.json index 9dc2a85d8fb..e7d90e4ee98 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/boefje.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/boefje.json @@ -6,8 +6,5 @@ "IPAddressV4", "IPAddressV6" ], - "environment_keys": [ - "BINARYEDGE_API" - ], "scan_level": 2 } diff --git a/boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json index 086ce350160..288b50127c7 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_binaryedge_containers", "name": "BinaryEdge containers", + "description": "Parse BinaryEdge data to check if Kubernetes hosts have any vulnerabilities. Creates 'VERIFIED-VULNERABILITY' findings.", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json index 2af3f47f891..d2336234281 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_binaryedge_databases", "name": "BinaryEdge databases", + "description": "Parses BinaryEdge data to check if any Cassandra, ElasticSearch, Memcached, MongoDB and Redis servers are identified and parses the version number. Create 'EXPOSED-SOFTWARE' findings.", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json index f5cafc7560a..03f77c3fe73 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_binaryedge_http_web", "name": "BinaryEdge Websites", + "description": "Parses BinaryEdge data to check for AWS secrets, F5 BIG IP loadbalancers and Citrix NetScaler.", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json index caa59b56f4b..3953e6f9b13 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_binaryedge_message_queues", "name": "BinaryEdge message queues", + "description": "Parses BinaryEdge data to check for message queues (mqtt) servers. Creates the finding 'EXPOSED-SOFTWARE' if mqtt servers are found.", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json index 30d0f02963e..f6e89f9fb3d 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_binaryedge_protocols", - "name": "BinaryEdge protocols", + "name": "BinaryEdge SSL/TLS protocols", + "description": "Parses BinaryEdge data to check for various vulnerabilities within SSL/TLS protocols, such as Heartbleed, Secure Renegotiation and SSL Compression.", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json index 80e1837a499..5cc15031b65 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_binaryedge_remote_desktop", "name": "Binary Edge remote desktop", + "description": "Parses BinaryEdge data to check for remote desktop services such as RDP, VNC and X11. Creates 'EXPOSED-SOFTWARE' findings.", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json index d451a79b150..dc92d458b6c 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_binaryedge_service_identification", "name": "BinaryEdge service identification", + "description": "Parses BinaryEdge data to check if Software is present that is known for malware.", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json index 57a0f8dac16..89b0b22c057 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_binaryedge_services", "name": "BinaryEdge services", + "description": "Parses BinaryEdge data to check for services such as SSH, rsync, FTP, telnet and SMB.", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_burpsuite/normalize.py b/boefjes/boefjes/plugins/kat_burpsuite/normalize.py index a476695d471..e938e150578 100644 --- a/boefjes/boefjes/plugins/kat_burpsuite/normalize.py +++ b/boefjes/boefjes/plugins/kat_burpsuite/normalize.py @@ -7,7 +7,6 @@ from defusedxml import minidom from boefjes.job_models import NormalizerOutput -from octopoes.models import Reference from octopoes.models.ooi.dns.zone import Hostname from octopoes.models.ooi.findings import CAPECFindingType, CVEFindingType, CWEFindingType, Finding from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, Protocol @@ -15,16 +14,22 @@ from octopoes.models.ooi.web import URL, HostnameHTTPURL, HTTPHeader, HTTPResource, IPAddressHTTPURL, WebScheme, Website +def find_network(data: dict) -> dict: + if "network" in data: + return data["network"] + for key, value in data.items(): + if isinstance(value, dict): + result = find_network(value) + if result is not None: + return result + # Return internet if network is not found + return {"name": "internet"} + + def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: - parser = minidom.parse(raw) - - # assume that input ooi is none or a HostnameHTTPURL - if input_ooi: - ooi = Reference.from_str(input_ooi["primary_key"]) - network = Network(name=ooi.tokenized.netloc.network.name) - else: - network = Network(name="internet") - yield network + parser = minidom.parseString(raw.decode("UTF-8")) + + network = Network(name=find_network(input_ooi).get("name", "internet")) tcp_protocol = Protocol.TCP diff --git a/boefjes/boefjes/plugins/kat_censys/boefje.json b/boefjes/boefjes/plugins/kat_censys/boefje.json index 6aadac16fba..034d79a0034 100644 --- a/boefjes/boefjes/plugins/kat_censys/boefje.json +++ b/boefjes/boefjes/plugins/kat_censys/boefje.json @@ -1,14 +1,10 @@ { "id": "censys", "name": "Censys", - "description": "Use Censys to discover open ports, services and certificates. Requires and API key.", + "description": "Use Censys to discover open ports, services and certificates. Requires an API key.", "consumes": [ "IPAddressV4", "IPAddressV6" ], - "environment_keys": [ - "CENSYS_API_ID", - "CENSYS_API_SECRET" - ], "scan_level": 1 } diff --git a/boefjes/boefjes/plugins/kat_censys/normalizer.json b/boefjes/boefjes/plugins/kat_censys/normalizer.json index 809fc7d7174..c160577458c 100644 --- a/boefjes/boefjes/plugins/kat_censys/normalizer.json +++ b/boefjes/boefjes/plugins/kat_censys/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_censys_normalize", "name": "Censys", + "description": "Parses Cencys data into objects that can be used by other boefjes and normalizers. Can create ports, certificates, software, websites and headers. Doesn't create findings.", "consumes": [ "boefje/censys" ], diff --git a/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json b/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json index 280ea27e565..ee0be42b6e5 100644 --- a/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json +++ b/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json @@ -1,13 +1,10 @@ { "id": "cve-finding-types", "name": "CVE Finding Types", - "description": "Hydrate information of Common Vulnerabilities and Exposures (CVE) finding types from the CVE API", + "description": "Hydrate information of Common Vulnerabilities and Exposures (CVE) finding types from the CVE API.", "consumes": [ "CVEFindingType" ], - "environment_keys": [ - "CVEAPI_URL" - ], "scan_level": 0, "enabled": true } diff --git a/boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json b/boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json index 6ae5590562d..31d217e7658 100644 --- a/boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json +++ b/boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json @@ -1,7 +1,7 @@ { "id": "kat_cve_finding_types_normalize", "name": "CVE finding types", - "description": "Parses CVE findings.", + "description": "Parses Common Vulnerability Exposures (CVE) into findings.", "consumes": [ "boefje/cve-finding-types" ], diff --git a/boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json b/boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json index abeeaa7d9d0..edcc6cae7c7 100644 --- a/boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json +++ b/boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json @@ -1,7 +1,7 @@ { "id": "cwe-finding-types", "name": "CWE Finding Types", - "description": "Hydrate information of Common Weakness Enumeration (CWE) finding types", + "description": "Hydrate information of Common Weakness Enumeration (CWE) finding types.", "consumes": [ "CWEFindingType" ], diff --git a/boefjes/boefjes/plugins/kat_cwe_finding_types/normalizer.json b/boefjes/boefjes/plugins/kat_cwe_finding_types/normalizer.json index 9b939d07df5..2b97ac2d404 100644 --- a/boefjes/boefjes/plugins/kat_cwe_finding_types/normalizer.json +++ b/boefjes/boefjes/plugins/kat_cwe_finding_types/normalizer.json @@ -1,7 +1,7 @@ { "id": "kat_cwe_finding_types_normalize", "name": "CWE finding", - "description": "Parses CWE findings.", + "description": "Parses Common Weakness Enumeration (CWE) into findings.", "consumes": [ "boefje/cwe-finding-types" ], diff --git a/boefjes/boefjes/plugins/kat_dicom/normalizer.json b/boefjes/boefjes/plugins/kat_dicom/normalizer.json index 74519e6e96c..f829c80bced 100644 --- a/boefjes/boefjes/plugins/kat_dicom/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dicom/normalizer.json @@ -1,7 +1,7 @@ { "id": "kat_dicom_normalize", "name": "DICOM servers", - "description": "Parses DICOM output into findings and identified software.", + "description": "Parses medical imaging data (DICOM) into findings and identified software.", "consumes": [ "boefje/dicom" ], diff --git a/boefjes/boefjes/plugins/kat_dns/boefje.json b/boefjes/boefjes/plugins/kat_dns/boefje.json index 53391f0155d..5773364b9b6 100644 --- a/boefjes/boefjes/plugins/kat_dns/boefje.json +++ b/boefjes/boefjes/plugins/kat_dns/boefje.json @@ -5,9 +5,5 @@ "consumes": [ "Hostname" ], - "environment_keys": [ - "RECORD_TYPES", - "REMOTE_NS" - ], "scan_level": 1 } diff --git a/boefjes/boefjes/plugins/kat_dns/normalizer.json b/boefjes/boefjes/plugins/kat_dns/normalizer.json index fa9c8a73fa6..a534d08e04b 100644 --- a/boefjes/boefjes/plugins/kat_dns/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dns/normalizer.json @@ -1,7 +1,7 @@ { "id": "kat_dns_normalize", "name": "DNS records", - "description": "Parses the DNS records.", + "description": "Parses DNS records. Can parse A, AAAA, CAA, CNAME, MX, NS, SOA, TXT, DKIM and DMARC data.", "consumes": [ "boefje/dns-records" ], diff --git a/boefjes/boefjes/plugins/kat_dns_version/normalizer.json b/boefjes/boefjes/plugins/kat_dns_version/normalizer.json index 0252c4fe250..1ee372c81f5 100644 --- a/boefjes/boefjes/plugins/kat_dns_version/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dns_version/normalizer.json @@ -1,5 +1,7 @@ { "id": "dns-bind-version-normalize", + "name": "DNS bind version normalizer", + "description": "Parses DNS Bind data into Software version objects.", "consumes": [ "boefje/dns-bind-version" ], diff --git a/boefjes/boefjes/plugins/kat_dns_zone/boefje.json b/boefjes/boefjes/plugins/kat_dns_zone/boefje.json index cc03e079bd1..04fd37c4826 100644 --- a/boefjes/boefjes/plugins/kat_dns_zone/boefje.json +++ b/boefjes/boefjes/plugins/kat_dns_zone/boefje.json @@ -1,7 +1,7 @@ { "id": "dns-zone", "name": "DNS zone", - "description": "Fetch the parent DNS zone of a DNS zone", + "description": "Fetch the parent DNS zone of a DNS zone.", "consumes": [ "DNSZone" ], diff --git a/boefjes/boefjes/plugins/kat_dnssec/boefje.json b/boefjes/boefjes/plugins/kat_dnssec/boefje.json index 8b4d156396e..a772cbb6cf0 100644 --- a/boefjes/boefjes/plugins/kat_dnssec/boefje.json +++ b/boefjes/boefjes/plugins/kat_dnssec/boefje.json @@ -1,7 +1,7 @@ { "id": "dns-sec", "name": "DNSSEC", - "description": "Validates DNSSec of a hostname", + "description": "Validates DNSSEC of a hostname by checking the cryptographic signatures.", "consumes": [ "Hostname" ], diff --git a/boefjes/boefjes/plugins/kat_dnssec/main.py b/boefjes/boefjes/plugins/kat_dnssec/main.py index 3c1b93328a0..950225b1829 100644 --- a/boefjes/boefjes/plugins/kat_dnssec/main.py +++ b/boefjes/boefjes/plugins/kat_dnssec/main.py @@ -17,4 +17,4 @@ def run(boefje_meta: dict): cmd = ["/usr/bin/drill", "-DT", domain] output = subprocess.run(cmd, capture_output=True) - return [(set(), output.stdout)] + return [({"openkat/dnssec-output"}, output.stdout)] diff --git a/boefjes/boefjes/plugins/kat_dnssec/normalizer.json b/boefjes/boefjes/plugins/kat_dnssec/normalizer.json index 670f16592f4..7a605294fc9 100644 --- a/boefjes/boefjes/plugins/kat_dnssec/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dnssec/normalizer.json @@ -3,7 +3,8 @@ "name": "DNS records", "description": "Parses DNSSEC data into findings.", "consumes": [ - "boefje/dns-sec" + "boefje/dns-sec", + "openkat/dnssec-output" ], "produces": [ "KATFindingType", diff --git a/boefjes/boefjes/plugins/kat_external_db/boefje.json b/boefjes/boefjes/plugins/kat_external_db/boefje.json index 4de34b597ac..da2043f7db9 100644 --- a/boefjes/boefjes/plugins/kat_external_db/boefje.json +++ b/boefjes/boefjes/plugins/kat_external_db/boefje.json @@ -1,16 +1,9 @@ { "id": "external_db", "name": "External database host fetcher", - "description": "Fetch hostnames and IP addresses/netblocks from an external database with API. See `description.md` for more information. Useful if you have a large network.", + "description": "Fetch hostnames and IP addresses/netblocks from an external database with API. See `description.md` for more information. Useful if you have a large network and wish to add all your hosts. You can also upload hosts through the CSV upload functionality.", "consumes": [ "Network" ], - "environment_keys": [ - "DB_URL", - "DB_ACCESS_TOKEN", - "DB_ORGANIZATION_IDENTIFIER", - "DB_ENDPOINT_FORMAT", - "REQUESTS_CA_BUNDLE" - ], "scan_level": 0 } diff --git a/boefjes/boefjes/plugins/kat_external_db/normalizer.json b/boefjes/boefjes/plugins/kat_external_db/normalizer.json index 2d9e72d56e9..9988c541f68 100644 --- a/boefjes/boefjes/plugins/kat_external_db/normalizer.json +++ b/boefjes/boefjes/plugins/kat_external_db/normalizer.json @@ -1,7 +1,7 @@ { "id": "kat_external_db_normalize", "name": "External database hosts fetcher", - "description": "Parse data the fetched host data from the external database into hostnames and IP-addresses.", + "description": "Parse the fetched host data from the external database into hostnames and IP-addresses.", "consumes": [ "boefje/external_db" ], diff --git a/boefjes/boefjes/plugins/kat_fierce/boefje.json b/boefjes/boefjes/plugins/kat_fierce/boefje.json index 1f7d5c677db..9e1ab8182ef 100644 --- a/boefjes/boefjes/plugins/kat_fierce/boefje.json +++ b/boefjes/boefjes/plugins/kat_fierce/boefje.json @@ -1,7 +1,7 @@ { "id": "fierce", "name": "Fierce", - "description": "Perform DNS reconnaissance using Fierce, to help locate non-contiguous IP space and hostnames against specified hostnames. No exploitation is performed.", + "description": "Perform DNS reconnaissance using Fierce. Helps to locate non-contiguous IP space and hostnames against specified hostnames. No exploitation is performed.", "consumes": [ "Hostname" ], diff --git a/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json b/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json index 70adfd46c47..2acc3c9c210 100644 --- a/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json +++ b/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_generic_finding_normalize", "name": "Finding types", + "description": "Parses data to create (CVE) Findings.", "consumes": [ "openkat/finding" ], diff --git a/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json b/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json index 92434280db5..714628e5587 100644 --- a/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json +++ b/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_green_hosting_normalize", "description": "Parses the Green Hosting output into findings.", + "name": "Green Hosting", "consumes": [ "boefje/green-hosting" ], diff --git a/boefjes/boefjes/plugins/kat_kat_finding_types/boefje.json b/boefjes/boefjes/plugins/kat_kat_finding_types/boefje.json index 559724289f9..e653f45622d 100644 --- a/boefjes/boefjes/plugins/kat_kat_finding_types/boefje.json +++ b/boefjes/boefjes/plugins/kat_kat_finding_types/boefje.json @@ -1,7 +1,7 @@ { "id": "kat-finding-types", "name": "KAT Finding Types", - "description": "Hydrate information of KAT finding types", + "description": "Hydrate information of KAT finding types.", "consumes": [ "KATFindingType" ], diff --git a/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json b/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json index 23ec1b70da7..a23c8d04f2f 100644 --- a/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json +++ b/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json @@ -13,14 +13,14 @@ "impact": "The usage possibility of JavaScript is not limited by the website. If the website contains a cross-site scripting vulnerability, then JavaScript code can be injected into the web page. This code is then executed by the browser of the victim. If a well-established Content Security Policy is active, the attacker can inject JavaScript code into the browser of the victim, but then the code will not get executed by the browser. A good configured Content Security Policy is a strong protection against cross-site scripting vulnerabilities.", "recommendation": "1. Set the Content-Security-Policy HTTP header in all HTTP answers. 2. Make sure that when the Content Security Policy is violated by a browser, that this violation is logged and monitored. Point the content security violation variable report-uri to a server-side log script. 3. Implement a process that periodically analyses these logs for programming errors and hack attacks." }, - "KAT-NO-X-PERMITTED-CROSS-DOMAIN-POLICIES": { + "KAT-X-PERMITTED-CROSS-DOMAIN-POLICIES": { "description": "The HTTP header X-Permitted-Cross-Domain- Policies is missing in HTTP responses. This header is not officially supported by Mozilla MDN.", "source": "https://owasp.org/www-project-secure-headers/#div-headers", "risk": "recommendation", "impact": "When the value of this header is not set to master- only, Adobe Flash or Adobe Acrobat (and possibly other software) can also look at cross-domain configuration files hosted at the web server.", "recommendation": "This header is not supported by default by Mozilla. If this header is required for your environment: Set the HTTP header X-Permitted-Cross- Domain-Policies: none in all HTTP responses. Use value master-only if a Flash or Acrobat cross- domain configuration file is used that is placed in the root of the web server" }, - "KAT-NO-EXPLICIT-XSS-PROTECTION": { + "KAT-EXPLICIT-XSS-PROTECTION": { "description": "The 'X-XSS-Protection' header is a deprecated header previously used to prevent against Cross-Site-Scripting attacks. Support in modern browsers could introduce XSS attacks again.", "source": "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection", "risk": "recommendation", @@ -34,14 +34,14 @@ "impact": "There is a change that clickjacking is possible. This is an attack technique in which the website is invisibly loaded. On top of the original website, another malicious website is loaded that contains specially placed buttons or links. When the victim clicks on those buttons or links, the mouse click and thus its corresponding action is performed on the original website (which is made invisible). If the victim is logged in, then this click can perform an unauthorized action.", "recommendation": "1. Set the HTTP header X-Frame- Options with value deny (safest) or sameorigin in every HTTP answer for older browsers. 2. Set the frame-ancestors variable in the Content-Security-Policy header for modern browsers. 3. Add JavaScript code to all pages to ensure that these web pages may not be loaded within an