Merge branch 'main' into fix/report-names-2

.github/workflows/build-rdo-package.yml

@@ -16,6 +16,9 @@ env:
 jobs:
   build:
     runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python_version: ["3.10", "3.11"]
 
     # Generic bits
     steps:
@@ -26,7 +29,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python_version }}
           cache: "pip"
 
       - name: Prep Install requirements
@@ -48,18 +51,18 @@ jobs:
 
       - name: Octopoes Build whl package
         run: |
-          python3.10 -m pip install build
-          python3.10 -m build
+          python${{ matrix.python_version }} -m pip install build
+          python${{ matrix.python_version }} -m build
         working-directory: ./octopoes
 
       - name: Octopoes Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Octopoes Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
 
       - name: Octopoes Create venv archive
-        run: tar -zcvf ${{ env.PKGDIR }}/octopoes_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/octopoes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       - name: Octopoes Enable production logging config
         run: mv prod.logging.yml logging.yml
@@ -84,13 +87,13 @@ jobs:
         working-directory: ./rocky
 
       - name: Rocky Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Rocky Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
 
       - name: Rocky Create rocky_venv tarball
-        run: tar -zcvf ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       - name: Rocky Run nvm install
         run: nvm install v16
@@ -113,7 +116,7 @@ jobs:
         working-directory: ./rocky
 
       - name: Rocky Compilemessages
-        run: /var/www/html/.venv/bin/python3.10 manage.py collectstatic && /var/www/html/.venv/bin/python3.10 manage.py compress && /var/www/html/.venv/bin/python3.10 manage.py compilemessages
+        run: /var/www/html/.venv/bin/python${{ matrix.python_version }} manage.py collectstatic && /var/www/html/.venv/bin/python${{ matrix.python_version }} manage.py compress && /var/www/html/.venv/bin/python${{ matrix.python_version }} manage.py compilemessages
         working-directory: ./rocky
         env:
           BYTES_API: http://bytes:8000
@@ -138,7 +141,7 @@ jobs:
         working-directory: ./bytes
 
       - name: Bytes Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Bytes Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
@@ -148,7 +151,7 @@ jobs:
         working-directory: ./bytes
 
       - name: Bytes Create bytes venv package
-        run: tar -zcvf ${{ env.PKGDIR }}/bytes_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/bytes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       # Mula / Scheduler
       - name: Mula Create /var/lib/html
@@ -159,21 +162,21 @@ jobs:
         working-directory: ./mula
 
       - name: Mula Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Create scheduler release archive
         run: tar -cvzf ${{ env.PKGDIR }}/scheduler_${{ env.RELEASE_VERSION }}.tar.gz --exclude=./.git* --exclude=Makefile --exclude=Dockerfile --exclude=requirements* --exclude=tests .
         working-directory: ./mula
 
       - name: Create virtual env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
         working-directory: /var/www/html
 
       - name: Create venv archive
-        run: tar -zcvf ${{ env.PKGDIR }}/scheduler_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/scheduler_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       # Boefjes
       - name: Boefjes Create /var/lib/html
@@ -184,14 +187,14 @@ jobs:
         working-directory: ./boefjes
 
       - name: Boefjes Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
         working-directory: /var/www/html
 
       - name: Create archive
-        run: tar -zcvf ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       - name: Create boefjes release
         run: tar -cvzf ${{ env.PKGDIR }}/boefjes_${{ env.RELEASE_VERSION }}.tar.gz .
@@ -206,14 +209,14 @@ jobs:
         working-directory: ./keiko
 
       - name: Keiko Create env
-        run: python3.10 -m venv /var/www/html/.venv
+        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
 
       - name: Keiko Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; find . -name requirements.txt | xargs -L 1 pip install -r
         working-directory: /var/www/html
 
       - name: Keiko Create archive
-        run: tar -zcvf ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
+        run: tar -zcvf ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
 
       - name: Keiko Create katkeiko release
         run: tar -cvzf ${{ env.PKGDIR }}/katkeiko_${{ env.RELEASE_VERSION }}.tar.gz .
@@ -229,8 +232,8 @@ jobs:
       - name: Octopoes Upload venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: octopoes_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/octopoes_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: octopoes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/octopoes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Octopoes Upload octopoes tar
         uses: actions/upload-artifact@v4
@@ -247,8 +250,8 @@ jobs:
       - name: Rocky Upload venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: rocky_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: rocky_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Bytes Upload bytes tar
         uses: actions/upload-artifact@v4
@@ -259,8 +262,8 @@ jobs:
       - name: Bytes Upload bytes venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: bytes_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/bytes_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: bytes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/bytes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Upload scheduler release
         uses: actions/upload-artifact@v4
@@ -271,8 +274,8 @@ jobs:
       - name: Upload venv archive
         uses: actions/upload-artifact@v4
         with:
-          name: scheduler_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/scheduler_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: scheduler_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/scheduler_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Upload boefjes tar
         uses: actions/upload-artifact@v4
@@ -283,8 +286,8 @@ jobs:
       - name: Upload venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: boefjes_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: boefjes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
 
       - name: Keiko Upload tar
         uses: actions/upload-artifact@v4
@@ -295,5 +298,5 @@ jobs:
       - name: Keiko Upload venv tar
         uses: actions/upload-artifact@v4
         with:
-          name: katkeiko_venv_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}.tar.gz
+          name: katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
+          path: ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz

.github/workflows/build_docs_on_pr.yml

@@ -3,7 +3,14 @@ name: Build docs artifact for PR
 on:
   pull_request:
     paths:
+      # We generate documentation for boefje, bytes, keiko, mula and octopoes
+      # from code so the workflow should also depend on it.
+      - "boefje/**"
+      - "bytes/**"
       - "docs/**"
+      - "keiko/**"
+      - "mula/**"
+      - "octopoes/**"
       - "requirements.txt"
       - ".github/workflows/build_docs_on_pr.yml"
 

.gitignore

@@ -437,7 +437,14 @@ nl-kat-*
 /_version.py
 
 # Automatically generated markdown files for the environment variables
-/docs/source/technical_design/environment_settings/*.md
+docs/source/installation-and-deployment/environment-settings/boefjes.md
+docs/source/installation-and-deployment/environment-settings/bytes.md
+docs/source/installation-and-deployment/environment-settings/keiko.md
+docs/source/installation-and-deployment/environment-settings/mula.md
+docs/source/installation-and-deployment/environment-settings/octopoes.md
+
+docs/source/_static/d3.min.js
+docs/source/_static/mermaid.min.js
 
 # rpki cache
 /boefjes/boefjes/plugins/kat_rpki/rpki.json

.pre-commit-config.yaml

@@ -188,5 +188,6 @@ repos:
           \.json$ |
           \.min\.js$ |
           ^rocky/assets/css/themes/soft/fonts |
-          ^rocky/assets/vendors
+          ^rocky/assets/vendors |
+          ^docs/source/_static
           )

Makefile

@@ -12,12 +12,14 @@ UNAME := $(shell uname)
 export DOCKER_BUILDKIT=1
 export COMPOSE_DOCKER_CLI_BUILD=1
 
+# We can't really return an error here, so if settings-doc fails we delete the
+# file which will result in sphinx-build returning an error later on
 define build-settings-doc
-	echo "# $(4)" > docs/source/installation_and_deployment/environment_settings/$(3).md
+	echo "# $(4)" > docs/source/installation-and-deployment/environment-settings/$(3).md
 	DOCS=True PYTHONPATH=./$(1) settings-doc generate \
 	-f markdown -m $(2) \
 	--templates docs/settings-doc-templates \
-	>> docs/source/installation_and_deployment/environment_settings/$(3).md
+	>> docs/source/installation-and-deployment/environment-settings/$(3).md || exit 1
 endef
 
 
@@ -114,7 +116,12 @@ docs:
 	$(call build-settings-doc,bytes,bytes.config,bytes,Bytes)
 	$(call build-settings-doc,mula/scheduler,config.settings,mula,Mula)
 
-	PYTHONPATH=$(PYTHONPATH):boefjes/:bytes/:keiko/:mula/:octopoes/ sphinx-build -b html docs/source docs/_build
+	curl -sL -o - https://registry.npmjs.org/d3/-/d3-7.9.0.tgz | tar -Oxzf - package/dist/d3.min.js > docs/source/_static/d3.min.js
+	curl -sL -o - https://registry.npmjs.org/mermaid/-/mermaid-11.3.0.tgz | tar -Oxzf - package/dist/mermaid.min.js > docs/source/_static/mermaid.min.js
+	echo "f2094bbf6141b359722c4fe454eb6c4b0f0e42cc10cc7af921fc158fceb86539  docs/source/_static/d3.min.js" | sha256sum --quiet --check || exit 1
+	echo "0d2b6f2361e7e0ce466a6ed458e03daa5584b42ef6926c3beb62eb64670ca261  docs/source/_static/mermaid.min.js" | sha256sum --quiet --check || exit 1
+
+	PYTHONPATH=$(PYTHONPATH):boefjes/:bytes/:keiko/:mula/:octopoes/ sphinx-build -b html --fail-on-warning docs/source docs/_build
 
 
 poetry-dependencies:

boefjes/boefjes/plugins/helpers.py

@@ -66,3 +66,12 @@ def get_file_from_container(container: docker.models.containers.Container, path:
         return None
 
     return extracted_file.read()
+
+
+def cpe_to_name_version(cpe: str) -> tuple[str | None, str | None]:
+    """Fetch the software name and version from a CPE string."""
+    cpe_split = cpe.split(":")
+    cpe_split_len = len(cpe_split)
+    name = None if cpe_split_len < 4 else cpe_split[3]
+    version = None if cpe_split_len < 5 else cpe_split[4]
+    return name, version

boefjes/boefjes/plugins/kat_binaryedge/http_web/normalize.py

@@ -3,7 +3,7 @@
 from collections.abc import Iterable
 
 from boefjes.job_models import NormalizerOutput
-from boefjes.plugins.kat_binaryedge.services.normalize import get_name_from_cpe
+from boefjes.plugins.helpers import cpe_to_name_version
 from octopoes.models import Reference
 from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software, SoftwareInstance
@@ -50,7 +50,8 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
 
             for app in response.get("apps", {}):
                 if "cpe" in app:
-                    software_ooi = Software(name=get_name_from_cpe(app["cpe"]), cpe=app["cpe"])
+                    name, version = cpe_to_name_version(cpe=app["cpe"])
+                    software_ooi = Software(name=name, version=version, cpe=app["cpe"])
                     yield software_ooi
                     yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference)
                 else:
@@ -74,9 +75,8 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
             for potential_software in data:
                 # Check all values for 'cpe'
                 if isinstance(potential_software, dict) and "cpe" in potential_software:
-                    software_ooi = Software(
-                        name=get_name_from_cpe(potential_software["cpe"]), cpe=potential_software["cpe"]
-                    )
+                    name, version = cpe_to_name_version(cpe=potential_software["cpe"])
+                    software_ooi = Software(name=name, version=version, cpe=potential_software["cpe"])
                     yield software_ooi
                     yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference)
 

boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalize.py

@@ -3,7 +3,7 @@
 from collections.abc import Iterable
 
 from boefjes.job_models import NormalizerOutput
-from boefjes.plugins.kat_binaryedge.services.normalize import get_name_from_cpe
+from boefjes.plugins.helpers import cpe_to_name_version
 from octopoes.models import Reference
 from octopoes.models.ooi.findings import Finding, KATFindingType
 from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
@@ -51,7 +51,8 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
 
             if "cpe" in service:
                 for cpe in service["cpe"]:
-                    software_ooi = Software(name=get_name_from_cpe(cpe), cpe=cpe)
+                    name, version = cpe_to_name_version(cpe=cpe)
+                    software_ooi = Software(name=name, version=version, cpe=cpe)
                     yield software_ooi
                     software_instance_ooi = SoftwareInstance(
                         ooi=ip_service_ooi.reference, software=software_ooi.reference

boefjes/boefjes/plugins/kat_binaryedge/services/normalize.py

@@ -3,24 +3,12 @@
 from collections.abc import Iterable
 
 from boefjes.job_models import NormalizerOutput
+from boefjes.plugins.helpers import cpe_to_name_version
 from octopoes.models import Reference
 from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software, SoftwareInstance
 
 
-def get_name_from_cpe(cpe: str) -> str:
-    split = []
-    if cpe[0:5] == "cpe:/":
-        split = cpe[5:].split(":")
-    elif cpe[0:8] == "cpe:2.3:":
-        split = cpe[8:].split(":")
-
-    if len(split) > 3:
-        return split[2]
-    else:
-        return cpe
-
-
 def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
     results = json.loads(raw)
     pk_ooi = Reference.from_str(input_ooi["primary_key"])
@@ -73,7 +61,8 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
                 yield software_ooi
                 yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference)
             for cpe in scan.get("result", {}).get("data", {}).get("cpe", []):
-                software_ooi = Software(name=get_name_from_cpe(cpe), cpe=cpe)
+                name, version = cpe_to_name_version(cpe=cpe)
+                software_ooi = Software(name=name, version=version, cpe=cpe)
                 yield software_ooi
                 yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference)
 

boefjes/boefjes/plugins/kat_dnssec/main.py

@@ -17,4 +17,8 @@ def run(boefje_meta: dict):
     cmd = ["/usr/bin/drill", "-DT", domain]
     output = subprocess.run(cmd, capture_output=True)
 
-    return [({"openkat/dnssec-output"}, output.stdout)]
+    output.check_returncode()
+
+    results = [({"openkat/dnssec-output"}, output.stdout)]
+
+    return results

boefjes/boefjes/plugins/kat_nmap_tcp/main.py

@@ -18,4 +18,10 @@ def run(boefje_meta: dict):
         cmd.append("-6")
 
     cmd.extend(["-oX", "-", str(ip)])
-    return [({"openkat/nmap-output"}, subprocess.run(cmd, capture_output=True).stdout.decode())]
+    output = subprocess.run(cmd, capture_output=True)
+
+    output.check_returncode()
+
+    results = [({"openkat/nmap-output"}, output.stdout.decode())]
+
+    return results