Merge branch 'main' into fix/bytes-s3 and fix conflicts

.env-dist

@@ -65,6 +65,8 @@ BYTES_DB_URI=postgresql://${BYTES_DB_USER}:${BYTES_DB_PASSWORD}@postgres:5432/${
 # --- Octopoes --- #
 # See `octopoes/octopoes/config/settings.py`
 
+# Number of Celery workers (for the Octopoes API worker) that need to be started
+CELERY_WORKER_CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-4}
 
 # --- Mula --- #
 # See `mula/scheduler/config/settings.py`

.gitignore

@@ -444,3 +444,4 @@ nl-kat-*
 /boefjes/boefjes/plugins/kat_rpki/rpki-meta.json
 
 *.pstat
+**/.cache*

.pre-commit-config.yaml

@@ -76,7 +76,7 @@ repos:
     rev: 1.16.0
     hooks:
       - id: django-upgrade
-        args: [--target-version, "4.2"]
+        args: [--target-version, "5.0"]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.9.0
@@ -91,11 +91,13 @@ repos:
           - httpx
           - types-python-dateutil
           - types-requests
+          - types-croniter
         exclude: |
           (?x)(
           ^boefjes/tools |
           ^keiko/templates |
           ^mula/whitelist\.py$ |
+          ^mula/scripts |
           ^octopoes/tools |
           ^rocky/whitelist\.py$ |
           /tests/ |
@@ -108,7 +110,7 @@ repos:
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
-        args: [-L, lama]
+        args: ["-L", "lama", "--ignore-regex", ".{1024}|.*codespell-ignore.*"]
         exclude: |
           (?x)(
           \.po$ |

boefjes/.ci/.env.test

@@ -1,11 +1,42 @@
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
-POSTGRES_DB=ci_katalogus
+POSTGRES_DB=test
 
-KATALOGUS_DB_URI=postgresql://postgres:postgres@ci_katalogus-db:5432/ci_katalogus
+KATALOGUS_DB_URI=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@ci_katalogus-db:5432/${POSTGRES_DB}
 
 CI=1
 
-ENCRYPTION_MIDDLEWARE=NACL_SEALBOX
+BOEFJES_ENCRYPTION_MIDDLEWARE=NACL_SEALBOX
 KATALOGUS_PRIVATE_KEY_B64=Vpb0g34rGFbnoUuiSjkFr8TKh278AViSJEdjII5DvQY=
 KATALOGUS_PUBLIC_KEY_B64=iR/vPrBVrx0LXOiwK6DMB3QCggjzQXDtj/hyVK7mpy8=
+BOEFJES_API=http://placeholder:1234
+
+
+# Benchmark setup
+RABBITMQ_DEFAULT_VHOST=kat
+RABBITMQ_DEFAULT_USER=ci_user
+RABBITMQ_DEFAULT_PASS=ci_pass
+
+QUEUE_URI=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@ci_rabbitmq:5672/${RABBITMQ_DEFAULT_VHOST}
+
+KATALOGUS_API=http://ci_katalogus:8080
+OCTOPOES_API=http://ci_octopoes:80
+XTDB_URI=http://ci_xtdb:3000
+BYTES_API=http://ci_bytes:8000
+SCHEDULER_API=http://placeholder:8000
+
+CI=1
+
+# CI Bytes configuration
+
+BYTES_SECRET=3d54f6e4e65723aa678d17d8fd22aba5234136d3e44e5a77305dedaa8ce13f45
+BYTES_ACCESS_TOKEN_EXPIRE_MINUTES=1000
+BYTES_USERNAME=test
+BYTES_PASSWORD=secret
+BYTES_ENCRYPTION_MIDDLEWARE=IDENTITY
+
+BYTES_DB_URI=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@ci_bytes-db:5432/${POSTGRES_DB}
+BYTES_LOG_FILE=/var/log/bytes-test.log
+BYTES_FILE_PERMISSION=555
+
+BYTES_METRICS_TTL_SECONDS=0

boefjes/.ci/docker-compose.yml

@@ -5,9 +5,10 @@ services:
       dockerfile: boefjes/Dockerfile
       args:
         - ENVIRONMENT=dev
-    command: sh -c 'python -m pytest -v boefjes/katalogus/tests/integration'
+    command: sh -c 'python -m pytest -v tests/integration'
     depends_on:
       - ci_katalogus-db
+      - ci_katalogus
     env_file:
       - .ci/.env.test
     volumes:
@@ -17,3 +18,101 @@ services:
     image: docker.io/library/postgres:15
     env_file:
       - .ci/.env.test
+
+  migration_bench:
+    build:
+      context: ..
+      dockerfile: boefjes/Dockerfile
+      args:
+        - ENVIRONMENT=dev
+    command: bash -c "python -m cProfile -o .ci/bench_$(date +%Y_%m_%d-%H:%M:%S).pstat -m pytest -v -m slow tests/integration"
+    depends_on:
+      - ci_bytes
+      - ci_octopoes
+      - ci_katalogus-db
+    env_file:
+      - .ci/.env.test
+    volumes:
+      - .:/app/boefjes
+    environment:
+      - DATABASE_MIGRATION=1
+
+  ci_bytes:
+    build:
+      context: ../bytes
+      args:
+        ENVIRONMENT: dev
+    command: uvicorn bytes.api:app --host 0.0.0.0
+    depends_on:
+      ci_rabbitmq:
+        condition: service_healthy
+      ci_bytes-db:
+        condition: service_started
+    env_file:
+      - .ci/.env.test
+    environment:
+      - DATABASE_MIGRATION=1
+
+  ci_bytes-db:
+    image: docker.io/library/postgres:15
+    env_file:
+      - .ci/.env.test
+
+  ci_octopoes:
+    build:
+      context: ../octopoes
+    command: uvicorn octopoes.api.api:app --host 0.0.0.0 --port 80
+    depends_on:
+      ci_rabbitmq:
+        condition: service_healthy
+      ci_xtdb:
+        condition: service_started
+      ci_katalogus:
+        condition: service_started
+      ci_octopoes_api_worker:
+        condition: service_started
+    env_file:
+      - .ci/.env.test
+
+  ci_rabbitmq:
+    restart: on-failure
+    image: "docker.io/library/rabbitmq:3.12-management"
+    healthcheck:
+      test: ["CMD", "rabbitmqctl", "status"]
+      interval: 5s
+      retries: 4
+    env_file:
+      - .ci/.env.test
+
+  ci_xtdb:
+    image: "ghcr.io/dekkers/xtdb-http-multinode:v1.0.8"
+
+  ci_octopoes_api_worker:
+    build:
+      context: ../octopoes
+    command: celery -A octopoes.tasks.tasks worker -E --loglevel=INFO
+    depends_on:
+      ci_rabbitmq:
+        condition: service_healthy
+      ci_xtdb:
+        condition: service_started
+    env_file:
+      - .ci/.env.test
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+
+  ci_katalogus:
+    build:
+      context: ..
+      dockerfile: boefjes/Dockerfile
+      args:
+        - ENVIRONMENT=dev
+    command: uvicorn boefjes.katalogus.root:app --host 0.0.0.0 --port 8080
+    depends_on:
+      - ci_katalogus-db
+    env_file:
+      - .ci/.env.test
+    volumes:
+      - .:/app/boefjes

boefjes/Dockerfile

@@ -6,8 +6,8 @@ ARG USER_GID=1000
 
 ENTRYPOINT ["/app/boefjes/entrypoint.sh"]
 
-RUN groupadd --gid $USER_GID nonroot
-RUN adduser --disabled-password --gecos '' --uid $USER_UID --gid $USER_GID nonroot
+RUN groupadd --gid "$USER_GID" nonroot
+RUN adduser --disabled-password --gecos '' --uid "$USER_UID" --gid "$USER_GID" nonroot
 
 WORKDIR /app/boefjes
 ENV PATH=/home/nonroot/.local/bin:${PATH}
@@ -20,10 +20,10 @@ RUN --mount=type=cache,target=/root/.cache \
     pip install --upgrade pip \
     && if [ "$ENVIRONMENT" = "dev" ]; \
     then \
-    grep -v git+https:// requirements-dev.txt | pip install -r /dev/stdin ; \
+    grep -v git+https:// requirements-dev.txt | pip install -r /dev/stdin && \
     grep git+https:// requirements-dev.txt | pip install -r /dev/stdin ; \
     else \
-    grep -v git+https:// requirements.txt | pip install -r /dev/stdin ;\
+    grep -v git+https:// requirements.txt | pip install -r /dev/stdin && \
     grep git+https:// requirements.txt | pip install -r /dev/stdin ; \
     fi
 

boefjes/Makefile

@@ -77,6 +77,12 @@ itest: ## Run the integration tests.
 	$(ci-docker-compose) run --rm katalogus_integration
 	$(ci-docker-compose) down
 
+bench:  ## Run the report benchmark.
+	$(ci-docker-compose) build
+	$(ci-docker-compose) down --remove-orphans
+	$(ci-docker-compose) run --rm migration_bench
+	$(ci-docker-compose) stop
+
 debian12:
 	docker run --rm \
 	--env PKG_NAME=kat-boefjes \

boefjes/boefjes/__main__.py

@@ -2,6 +2,7 @@
 import logging.config
 
 import click
+import structlog
 
 from boefjes.app import get_runtime_manager
 from boefjes.config import settings
@@ -10,7 +11,27 @@
 with settings.log_cfg.open() as f:
     logging.config.dictConfig(json.load(f))
 
-logger = logging.getLogger(__name__)
+structlog.configure(
+    processors=[
+        structlog.contextvars.merge_contextvars,
+        structlog.processors.add_log_level,
+        structlog.processors.StackInfoRenderer(),
+        structlog.dev.set_exc_info,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.processors.TimeStamper("iso", utc=False),
+        (
+            structlog.dev.ConsoleRenderer(colors=True, pad_level=False)
+            if settings.logging_format == "text"
+            else structlog.processors.JSONRenderer()
+        ),
+    ],
+    context_class=dict,
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    wrapper_class=structlog.stdlib.BoundLogger,
+    cache_logger_on_first_use=True,
+)
+
+logger = structlog.get_logger(__name__)
 
 
 @click.command()

boefjes/boefjes/api.py

@@ -1,10 +1,10 @@
 import base64
-import logging
 import multiprocessing
 from datetime import datetime, timezone
 from enum import Enum
 from uuid import UUID
 
+import structlog
 from fastapi import Depends, FastAPI, HTTPException, Response
 from httpx import HTTPError, HTTPStatusError
 from pydantic import BaseModel, ConfigDict, Field
@@ -13,15 +13,15 @@
 from boefjes.clients.bytes_client import BytesAPIClient
 from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus
 from boefjes.config import settings
-from boefjes.job_handler import get_environment_settings, get_octopoes_api_connector, serialize_ooi
+from boefjes.job_handler import get_environment_settings, get_octopoes_api_connector
 from boefjes.job_models import BoefjeMeta
-from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
+from boefjes.local_repository import LocalPluginRepository, get_local_repository
 from boefjes.plugins.models import _default_mime_types
 from octopoes.models import Reference
 from octopoes.models.exception import ObjectNotFoundException
 
 app = FastAPI(title="Boefje API")
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
 
 
 class UvicornServer(multiprocessing.Process):
@@ -85,7 +85,7 @@ async def root():
 
 
 @app.get("/api/v0/tasks/{task_id}", response_model=BoefjeInput)
-async def boefje_input(
+def boefje_input(
     task_id: UUID,
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
     local_repository: LocalPluginRepository = Depends(get_local_repository),
@@ -102,7 +102,7 @@ async def boefje_input(
 
 
 @app.post("/api/v0/tasks/{task_id}")
-async def boefje_output(
+def boefje_output(
     task_id: UUID,
     boefje_output: BoefjeOutput,
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
@@ -122,7 +122,7 @@ async def boefje_output(
     bytes_client.save_boefje_meta(boefje_meta)
 
     if boefje_output.files:
-        mime_types = _default_mime_types(task.p_item.data.boefje)
+        mime_types = _default_mime_types(task.data.boefje)
         for file in boefje_output.files:
             raw = base64.b64decode(file.content)
             # when supported, also save file.name to Bytes
@@ -149,13 +149,12 @@ def get_task(task_id, scheduler_client):
 
 
 def create_boefje_meta(task, local_repository):
-    boefje = task.p_item.data.boefje
+    boefje = task.data.boefje
     boefje_resource = local_repository.by_id(boefje.id)
-    env_keys = boefje_resource.environment_keys
-    environment = get_environment_settings(task.p_item.data, env_keys) if env_keys else {}
+    environment = get_environment_settings(task.data, boefje_resource.schema)
 
-    organization = task.p_item.data.organization
-    input_ooi = task.p_item.data.input_ooi
+    organization = task.data.organization
+    input_ooi = task.data.input_ooi
     arguments = {"oci_arguments": boefje_resource.oci_arguments}
 
     if input_ooi:
@@ -165,7 +164,7 @@ def create_boefje_meta(task, local_repository):
         except ObjectNotFoundException as e:
             raise ObjectNotFoundException(f"Object {reference} not found in Octopoes") from e
 
-        arguments["input"] = serialize_ooi(ooi)
+        arguments["input"] = ooi.serialize()
 
     boefje_meta = BoefjeMeta(
         id=task.id,