Merge branch 'main' into feature/plugin-sources

.env-dist

@@ -33,6 +33,7 @@ RABBITMQ_DEFAULT_VHOST=kat
 RABBITMQ_DEFAULT_USER={%QUEUE_USERNAME}
 RABBITMQ_DEFAULT_PASS={%QUEUE_PASSWORD}
 
+# Boefjes and Normalizers
 QUEUE_NAME_BOEFJES=boefjes
 QUEUE_NAME_NORMALIZERS=normalizers
 QUEUE_URI=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/${RABBITMQ_DEFAULT_VHOST}

.github/scripts/commit_sign_push.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#GITHUB_TOKEN should be ${{ secrets.GITHUB_TOKEN }}
+#DESTINATION_BRANCH should be ${{ github.ref }}
+
+FILES=$(git diff --name-only)
+for FILE in $FILES; do
+  CONTENT=$(base64 -i "$FILE")
+  SHA=$(git rev-parse "$DESTINATION_BRANCH":"$FILE")
+  gh api --method PUT /repos/:owner/:repo/contents/"$FILE" \
+    --field message="Update $FILE" \
+    --field content="$CONTENT" \
+    --field encoding="base64" \
+    --field branch="$DESTINATION_BRANCH" \
+    --field sha="$SHA"
+done

.github/workflows/build-debian-docker-image.yml

@@ -2,10 +2,16 @@ name: Create and publish Docker image for building Debian packages
 
 on:
   workflow_dispatch: {}
-  pull_request:
+  push:
+    branches:
+      - 'main'
     paths:
       - "packaging"
       - ".github/workflows/build-debian-docker-image.yml"
+  pull_request:
+    paths:
+      - "packaging/**"
+      - ".github/workflows/build-debian-docker-image.yml"
 
 env:
   REGISTRY: ghcr.io

.github/workflows/check_poetry_dependencies.yml

@@ -3,20 +3,26 @@ name: Check Poetry dependencies
 on:
   push:
     paths:
-      - requirements.txt
-      - requirements-dev.txt
-      - pyproject.toml
-      - poetry.lock
+      - '**/requirements.txt'
+      - '**/requirements-dev.txt'
+      - '**/pyproject.toml'
+      - '**/poetry.lock'
   pull_request:
     paths:
-      - requirements.txt
-      - requirements-dev.txt
-      - pyproject.toml
-      - poetry.lock
+      - '**/requirements.txt'
+      - '**/requirements-dev.txt'
+      - '**/pyproject.toml'
+      - '**/poetry.lock'
 
 
 jobs:
   poetry-dependencies:
+
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the
+      # added or changed files to the repository.
+      contents: write
+
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -29,10 +35,18 @@ jobs:
           cache: pip
 
       - name: Install pip dependencies
-        run: pip install poetry poetry-plugin-export
+        run: pip install poetry
 
       - name: Check, lock, and export Poetry dependencies
         run: make poetry-dependencies
 
       - name: Check if there are any changed files
+        if: ${{ github.actor != 'dependabot[bot]' }}
         run: git diff --exit-code
+
+      - name: Commit, sign, and push changes
+        if: ${{ github.actor == 'dependabot[bot]' }}
+        run: sh .github/scripts/commit_sign_push.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DESTINATION_BRANCH: ${{ github.ref }}

.github/workflows/debian_package.yml

@@ -27,6 +27,9 @@ jobs:
           bytes:
             - 'bytes/**'
             - '.github/workflows/debian_package.yml'
+          cveapi:
+            - 'cveapi/**'
+            - '.github/workflows/debian_package.yml'
           keiko:
             - 'keiko/**'
             - '.github/workflows/debian_package.yml'
@@ -49,7 +52,12 @@ jobs:
       matrix:
         dist: [debian11, debian12, ubuntu22.04]
         # On main, release branches and tags we always want to build all the packages
-        package: ${{ github.event_name == 'push' && fromJSON('["boefjes", "bytes", "keiko", "mula", "octopoes", "rocky"]') || fromJSON(needs.changes.outputs.packages) }}
+        package: ${{ github.event_name == 'push' && fromJSON('["boefjes", "bytes", "cveapi", "keiko", "mula", "octopoes", "rocky"]') || fromJSON(needs.changes.outputs.packages) }}
+        exclude:
+          - package: cveapi
+            dist: debian11
+          - package: cveapi
+            dist: ubuntu22.04
     runs-on: ubuntu-22.04
     env:
       PKG_NAME: kat-${{ matrix.package }}
@@ -89,4 +97,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ${{env.PKG_NAME}}_${{ env.RELEASE_VERSION }}_${{ matrix.dist }}.deb
-          path: ${{matrix.package}}/build/${{env.PKG_NAME}}_${{ env.RELEASE_VERSION }}_amd64.deb
+          path: ${{matrix.package}}/build/${{env.PKG_NAME}}_${{ env.RELEASE_VERSION }}_${{ matrix.package == 'cveapi' && 'all' || 'amd64' }}.deb

.github/workflows/masscan_container_image.yml

@@ -0,0 +1,61 @@
+name: Create and publish Masscan Docker image
+
+on:
+  workflow_dispatch: {}
+  push:
+    branches: [ main ]
+    paths:
+      - 'boefjes/images/masscan/Dockerfile'
+      - '.github/workflows/masscan_container_image.yml'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'boefjes/images/masscan/Dockerfile'
+      - '.github/workflows/masscan_container_image.yml'
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build-masscan-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        id: buildx
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@12cce9efe0d49980455aaaca9b071c0befcdd702
+        with:
+          images: |
+            ghcr.io/${{ github.repository_owner }}/nl-kat-masscan-build-image
+          tags: |
+            type=ref,event=branch
+            type=ref,event=tag
+            type=ref,event=pr
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=sha
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: ./boefjes/images/masscan
+          push: true
+          platforms: linux/arm64,linux/amd64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.pre-commit-config.yaml

@@ -71,14 +71,17 @@ repos:
     hooks:
       - id: codespell
         additional_dependencies: ['tomli']
+        args: [-L, lama]
         exclude: |
           (?x)(
           \.po$ |
           \.xml$ |
           poetry.lock$ |
+          pyproject.toml$ |
           requirements-.*.txt$ |
           retirejs.json$ |
           ^boefjes/boefjes/plugins/kat_fierce/lists |
+          ^boefjes/tests/examples/inputs/cve-result-without-cvss.json |
           ^keiko/glossaries |
           ^keiko/templates/.*/template.tex$ |
           ^rocky/assets/js/vendor |

Makefile

@@ -100,11 +100,15 @@ docs:
 	sphinx-build -b html docs/source docs/_build
 
 poetry-dependencies:
-	for path in . keiko octopoes
+	for path in . keiko octopoes boefjes bytes mula rocky
 	do
 		echo $$path
 		poetry check -C $$path
 		poetry lock --check -C $$path
 		poetry export -C $$path --without=dev -f requirements.txt -o $$path/requirements.txt
 		poetry export -C $$path --with=dev -f requirements.txt -o $$path/requirements-dev.txt
 	done
+
+	# NOTE: pip does not yet support hash verification for git dependencies;
+	# rocky's requirements-dev.txt unfortunately has no hashing until then
+	sed -i '/--hash/d; s/ \\$$//' rocky/requirements-dev.txt

boefjes/.env-dist

@@ -11,3 +11,4 @@ BYTES_PASSWORD=secret
 
 KATALOGUS_API=
 KATALOGUS_DB_URI=
+WORKER_HEARTBEAT=

boefjes/Dockerfile

@@ -5,7 +5,7 @@ COPY boefjes/boefjes ./boefjes
 COPY boefjes/requirements.txt ./boefjes
 
 # The echo since cat does not add a newline
-RUN find ./boefjes -name 'requirements.txt' -execdir sh -c "cat {} && echo" \; | sort -u > /tmp/boefjes-requirements.txt
+RUN find ./boefjes/plugins -name 'requirements.txt' -execdir sh -c "cat {} && echo" \; | sort -u > /tmp/boefjes-requirements.txt
 
 FROM python:$PYTHON_VERSION AS dev
 
@@ -29,6 +29,7 @@ RUN --mount=type=cache,target=/root/.cache \
     pip install --upgrade pip \
     && pip install -r /tmp/boefjes-requirements.txt \
     && rm /tmp/boefjes-requirements.txt \
+    && pip install -r requirements.txt \
     && if [ "$ENVIRONMENT" = "dev" ]; then pip install -r requirements-dev.txt; fi
 
 FROM dev

boefjes/boefjes/__main__.py

@@ -5,7 +5,7 @@
 
 from boefjes.app import get_runtime_manager
 from boefjes.config import settings
-from boefjes.runtime_interfaces import RuntimeManager
+from boefjes.runtime_interfaces import WorkerManager
 
 with settings.log_cfg.open() as f:
     logging.config.dictConfig(json.load(f))
@@ -14,7 +14,7 @@
 
 
 @click.command()
-@click.argument("worker_type", type=click.Choice([q.value for q in RuntimeManager.Queue]))
+@click.argument("worker_type", type=click.Choice([q.value for q in WorkerManager.Queue]))
 @click.option(
     "--log-level",
     type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]),
@@ -25,7 +25,7 @@ def cli(worker_type: str, log_level: str):
     logger.setLevel(log_level)
     logger.info("Starting runtime for %s", worker_type)
 
-    queue = RuntimeManager.Queue(worker_type)
+    queue = WorkerManager.Queue(worker_type)
     runtime = get_runtime_manager(settings, queue, log_level)
     runtime.run(queue)