Merge branch 'main' into chore/mula/documentation

* main: (25 commits)
  Add sterr to output list (#3649)
  feat: ✨ add Shodan InternetDB boefje (#2615)
  Add search endpoint for schedules for scheduler (#3695)
  Fix/report naming (#3666)
  Add delete schedule functionality for schedules in the scheduler (#3678)
  Fix javascript and component template in prod environments (#3672)
  Fix first order dangling affirmation delete (#3682)
  Do not run dh_strip_nondeterminism in Debian packaging (#3674)
  Github action should trigger if workflow definition changes (#3680)
  Updated packages (#3694)
  Silence KATFindingType not found error in JobHandler (#3686)
  Add docs for xtdb analyze bits. (#3688)
  Fix empty vulnerability reports (#3662)
  Fix jsonb 'contained by' query (#3643)
  Add plugins to findings report (#3657)
  Translations update from Hosted Weblate (#3673)
  Update upload_raw.py (#3645)
  Add rocky worker service to debian packages (#3619)
  Fix/yielded objects (#3669)
  make some things look better (#3661)
  ...

.github/workflows/boefjes_tests.yml

@@ -10,10 +10,12 @@ on:
     paths:
       - boefjes/**
       - octopoes/**
+      - .github/workflows/boefjes_test.yml
   pull_request:
     paths:
       - boefjes/**
       - octopoes/**
+      - .github/workflows/boefjes_test.yml
 
 jobs:
   Tests:

.github/workflows/build_docs_on_pr.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "docs/**"
       - "requirements.txt"
+      - ".github/workflows/build_docs_on_pr.yml"
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:

.github/workflows/bytes_tests.yml

@@ -9,9 +9,11 @@ on:
       - "*"
     paths:
       - bytes/**
+      - .github/workflows/bytes_test.yml
   pull_request:
     paths:
       - bytes/**
+      - .github/workflows/bytes_test.yml
 
 jobs:
   test:

.github/workflows/check_poetry_dependencies.yml

@@ -12,12 +12,14 @@ on:
       - "**/requirements-dev.txt"
       - "**/pyproject.toml"
       - "**/poetry.lock"
+      - ".github/workflows/check_poetry_dependencies.yml"
   pull_request:
     paths:
       - "**/requirements.txt"
       - "**/requirements-dev.txt"
       - "**/pyproject.toml"
       - "**/poetry.lock"
+      - ".github/workflows/check_poetry_dependencies.yml"
 
 jobs:
   poetry-dependencies:

.github/workflows/keiko_itest.yml

@@ -9,9 +9,11 @@ on:
       - "*"
     paths:
       - keiko/**
+      - .github/workflows/keiko_itest.yml
   pull_request:
     paths:
       - keiko/**
+      - .github/workflows/keiko_itest.yml
 
 jobs:
   itest:

.github/workflows/keiko_tex_linters.yml

@@ -9,9 +9,11 @@ on:
       - "*"
     paths:
       - keiko/**
+      - .github/workflows/keiko_tex_linters.yml
   pull_request:
     paths:
       - keiko/**
+      - .github/workflows/keiko_tex_linters.yml
 
 jobs:
   check-latex:

.github/workflows/mula_tests.yml

@@ -9,9 +9,11 @@ on:
       - "*"
     paths:
       - mula/**
+      - .github/workflows/mula_test.yml
   pull_request:
     paths:
       - mula/**
+      - .github/workflows/mula_test.yml
 
 jobs:
   test:

.github/workflows/octopoes_rtest.yml

@@ -9,9 +9,11 @@ on:
       - "*"
     paths:
       - octopoes/**
+      - .github/workflows/octopoes_rtest.yml
   pull_request:
     paths:
       - octopoes/**
+      - .github/workflows/octopoes_rtest.yml
 
 jobs:
   rtest:

.github/workflows/octopoes_tests.yml

@@ -9,9 +9,11 @@ on:
       - "*"
     paths:
       - octopoes/**
+      - .github/workflows/octopoes_test.yml
   pull_request:
     paths:
       - octopoes/**
+      - .github/workflows/octopoes_test.yml
 
 jobs:
   test:

.github/workflows/rocky_tests.yml

@@ -10,10 +10,12 @@ on:
     paths:
       - octopoes/**
       - rocky/**
+      - .github/workflows/rocky_test.yml
   pull_request:
     paths:
       - octopoes/**
       - rocky/**
+      - .github/workflows/rocky_test.yml
 
 jobs:
   test:

.github/workflows/test_debian_packages_on_ubuntu.yml

@@ -213,7 +213,7 @@ jobs:
           sudo sed -i "s/BYTES_PASSWORD=\$/BYTES_PASSWORD=$(sudo grep BYTES_PASSWORD /etc/kat/bytes.conf | awk -F'=' '{ print $2 }')/" /etc/kat/mula.conf
 
       - name: Restart KAT
-        run: sudo systemctl restart kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+        run: sudo systemctl restart kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
       - name: Setup accounts in Rocky
         run: |

README.rst

@@ -2,7 +2,7 @@
 What is OpenKAT?
 ================
 
-OpenKAT aims to monitor, record and analyze the status of information systems. The basic premise is that many of the major security incidents are caused by small errors and known vulnerabilities, and that if you can find them in time your systems and infrastructure become a lot more secure.
+OpenKAT aims to monitor, record and analyze the status of information systems. The basic premise is that many of the major security incidents are caused by small errors and known vulnerabilities, and if you find them and resolve them in time your systems and infrastructure become a lot more secure.
 
 OpenKAT scans, collects, analyzes and reports in an ongoing process:
 
@@ -11,7 +11,7 @@ OpenKAT scans, collects, analyzes and reports in an ongoing process:
 
 OpenKAT scans networks, finds vulnerabilities and creates accessible reports. It integrates the most widely used network tools and scanning software into a modular framework, accesses external databases such as shodan, and combines the information from all these sources into clear reports. It also includes lots of cat hair.
 
-OpenKAT is useful if you want to monitor a complex system and want to know whether it contains known vulnerabilities or configuration errors. Due to its modular structure and extensibility, OpenKAT can be applied in a multitude of situations. You can customize it and put it to your own use.
+OpenKAT is useful if you want to monitor a complex system and know whether it contains known vulnerabilities or configuration errors. Due to its modular structure and extensibility, OpenKAT can be applied in different situations. You can customize it and put it to your own use.
 
 Documentation
 =============
@@ -62,7 +62,7 @@ The tools addressed by OpenKAT may have their own license, from the OS/S domain
 Contact
 =======
 
-There several options to contact the OpenKAT team:
+There are several options to contact the OpenKAT team:
 
 - Direct contact: [email protected]
 - `Github Discussions <https://github.com/minvws/nl-kat-coordination/discussions>`_
@@ -73,10 +73,10 @@ There several options to contact the OpenKAT team:
 Privacy
 =======
 
-OpenKAT is not designed to collect private information and it does not act on any private information that its finds. Some information considered to be personally identifying information might be collected through one or more of OpenKAT's plugins and subsequently stored, but only if that information was accessible to OpenKAT. E.g. a phone number or email address listed on a website might end up being collected as part of OpenKAT normal data collection. This data might then be stored for a long time because OpenKAT stores 'proofs' of its actions. No email or phone number models are present and as such they won't be processed into objects by OpenKAT.
-An OpenKAT installation requires user accounts for users to be able to login. These accounts (and all data OpenKAT works with) are stored only on the OpenKAT installation itself and are not shared with other parties or outside of your OpenKAT install.
+OpenKAT is not designed to collect private information and it does not act on any private information that it finds. Some information considered to be personally identifiable information, may be collected through one or more of OpenKAT's plugins and subsequently stored, but only if that information has been accessible to OpenKAT. For example, a phone number or email address listed on a website might end up being collected as part of OpenKAT normal data collection. These data might then be stored for a long period of time, because OpenKAT stores evidence of its actions. No email or phone number models are present and as such they won't be processed into objects by OpenKAT.
+An OpenKAT installation requires user accounts for users to be able to log in. These accounts (and all data OpenKAT works with) are stored only on the OpenKAT installation itself, and are not shared with any other parties or outside of your OpenKAT install.
 
 Security
 ========
 
-OpenKAT is designed to be secure by default in its production setup. In the development setup some debugging flags are enabled by default and it will not include TLS out of the box. To setup a secure production OpenKAT install, please follow the `Production setup guidelines <https://docs.openkat.nl/installation_and_deployment/install.html#production-environments>`_ and `Hardening guidelines <https://docs.openkat.nl/installation_and_deployment/hardening.html>`_.
+OpenKAT is designed to be secure by default in its production setup. In the development setup some debugging flags are enabled by default and it will not include TLS out of the box. To set up a secure production OpenKAT install, please follow the `Production setup guidelines <https://docs.openkat.nl/installation_and_deployment/install.html#production-environments>`_ and `Hardening guidelines <https://docs.openkat.nl/installation_and_deployment/hardening.html>`_.

boefjes/boefjes/app.py

@@ -90,7 +90,7 @@ def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue):
         except HTTPError:
             # Scheduler is having issues, so make note of it and try again
             logger.exception("Getting the queues from the scheduler failed")
-            time.sleep(10 * self.settings.poll_interval)  # But not immediately
+            time.sleep(self.settings.poll_interval)  # But not immediately
             return
 
         # We do not target a specific queue since we start one runtime for all organisations

boefjes/boefjes/job_handler.py

@@ -103,8 +103,14 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
                 ooi = get_octopoes_api_connector(boefje_meta.organization).get(
                     reference, valid_time=datetime.now(timezone.utc)
                 )
-            except ObjectNotFoundException as e:
-                raise ObjectNotFoundException(f"Object {reference} not found in Octopoes") from e
+            except ObjectNotFoundException:
+                logger.info(
+                    "Can't run boefje because OOI does not exist anymore",
+                    boefje_id=boefje_meta.boefje.id,
+                    ooi=reference,
+                    task_id=boefje_meta.id,
+                )
+                return
 
             boefje_meta.arguments["input"] = ooi.serialize()
 

boefjes/boefjes/plugins/helpers.py

@@ -66,3 +66,12 @@ def get_file_from_container(container: docker.models.containers.Container, path:
         return None
 
     return extracted_file.read()
+
+
+def cpe_to_name_version(cpe: str) -> tuple[str | None, str | None]:
+    """Fetch the software name and version from a CPE string."""
+    cpe_split = cpe.split(":")
+    cpe_split_len = len(cpe_split)
+    name = None if cpe_split_len < 4 else cpe_split[3]
+    version = None if cpe_split_len < 5 else cpe_split[4]
+    return name, version

boefjes/boefjes/plugins/kat_binaryedge/http_web/normalize.py

@@ -3,7 +3,7 @@
 from collections.abc import Iterable
 
 from boefjes.job_models import NormalizerOutput
-from boefjes.plugins.kat_binaryedge.services.normalize import get_name_from_cpe
+from boefjes.plugins.helpers import cpe_to_name_version
 from octopoes.models import Reference
 from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software, SoftwareInstance
@@ -50,7 +50,8 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
 
             for app in response.get("apps", {}):
                 if "cpe" in app:
-                    software_ooi = Software(name=get_name_from_cpe(app["cpe"]), cpe=app["cpe"])
+                    name, version = cpe_to_name_version(cpe=app["cpe"])
+                    software_ooi = Software(name=name, version=version, cpe=app["cpe"])
                     yield software_ooi
                     yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference)
                 else:
@@ -74,9 +75,8 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
             for potential_software in data:
                 # Check all values for 'cpe'
                 if isinstance(potential_software, dict) and "cpe" in potential_software:
-                    software_ooi = Software(
-                        name=get_name_from_cpe(potential_software["cpe"]), cpe=potential_software["cpe"]
-                    )
+                    name, version = cpe_to_name_version(cpe=potential_software["cpe"])
+                    software_ooi = Software(name=name, version=version, cpe=potential_software["cpe"])
                     yield software_ooi
                     yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference)
 

boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalize.py

@@ -3,7 +3,7 @@
 from collections.abc import Iterable
 
 from boefjes.job_models import NormalizerOutput
-from boefjes.plugins.kat_binaryedge.services.normalize import get_name_from_cpe
+from boefjes.plugins.helpers import cpe_to_name_version
 from octopoes.models import Reference
 from octopoes.models.ooi.findings import Finding, KATFindingType
 from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
@@ -51,7 +51,8 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
 
             if "cpe" in service:
                 for cpe in service["cpe"]:
-                    software_ooi = Software(name=get_name_from_cpe(cpe), cpe=cpe)
+                    name, version = cpe_to_name_version(cpe=cpe)
+                    software_ooi = Software(name=name, version=version, cpe=cpe)
                     yield software_ooi
                     software_instance_ooi = SoftwareInstance(
                         ooi=ip_service_ooi.reference, software=software_ooi.reference

boefjes/boefjes/plugins/kat_binaryedge/services/normalize.py

@@ -3,24 +3,12 @@
 from collections.abc import Iterable
 
 from boefjes.job_models import NormalizerOutput
+from boefjes.plugins.helpers import cpe_to_name_version
 from octopoes.models import Reference
 from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software, SoftwareInstance
 
 
-def get_name_from_cpe(cpe: str) -> str:
-    split = []
-    if cpe[0:5] == "cpe:/":
-        split = cpe[5:].split(":")
-    elif cpe[0:8] == "cpe:2.3:":
-        split = cpe[8:].split(":")
-
-    if len(split) > 3:
-        return split[2]
-    else:
-        return cpe
-
-
 def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
     results = json.loads(raw)
     pk_ooi = Reference.from_str(input_ooi["primary_key"])
@@ -73,7 +61,8 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
                 yield software_ooi
                 yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference)
             for cpe in scan.get("result", {}).get("data", {}).get("cpe", []):
-                software_ooi = Software(name=get_name_from_cpe(cpe), cpe=cpe)
+                name, version = cpe_to_name_version(cpe=cpe)
+                software_ooi = Software(name=name, version=version, cpe=cpe)
                 yield software_ooi
                 yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference)
 

boefjes/boefjes/plugins/kat_dnssec/main.py

@@ -17,4 +17,8 @@ def run(boefje_meta: dict):
     cmd = ["/usr/bin/drill", "-DT", domain]
     output = subprocess.run(cmd, capture_output=True)
 
-    return [({"openkat/dnssec-output"}, output.stdout)]
+    output.check_returncode()
+
+    results = [({"openkat/dnssec-output"}, output.stdout)]
+
+    return results

boefjes/boefjes/plugins/kat_nmap_tcp/main.py

@@ -18,4 +18,10 @@ def run(boefje_meta: dict):
         cmd.append("-6")
 
     cmd.extend(["-oX", "-", str(ip)])
-    return [({"openkat/nmap-output"}, subprocess.run(cmd, capture_output=True).stdout.decode())]
+    output = subprocess.run(cmd, capture_output=True)
+
+    output.check_returncode()
+
+    results = [({"openkat/nmap-output"}, output.stdout.decode())]
+
+    return results

boefjes/boefjes/plugins/kat_shodan_internetdb/boefje.json

@@ -0,0 +1,10 @@
+{
+  "id": "shodan_internetdb",
+  "name": "'Shodan InternetDB",
+  "description": "Use Shodan InternetDB to find open ports with vulnerabilities that are found on an IP.",
+  "consumes": [
+    "IPAddressV4",
+    "IPAddressV6"
+  ],
+  "scan_level": 1
+}

boefjes/boefjes/plugins/kat_shodan_internetdb/description.md

@@ -0,0 +1,30 @@
+# Shodan InternetDB
+
+Fast IP Lookups for Open Ports and Vulnerabilities. Only free for non-commercial use. The API gets updated once a week.
+
+See: https://internetdb.shodan.io/, https://internetdb.shodan.io/docs
+
+## Return Schema:
+
+```
+{
+  "cpes": [
+    "string"
+  ],
+  "hostnames": [
+    "string"
+  ],
+  "ip": "string",
+  "ports": [
+    0
+  ],
+  "tags": [
+    "string"
+  ],
+  "vulns": [
+    "string"
+  ]
+}
+```
+
+Tags are discarded in the normalizer.

boefjes/boefjes/plugins/kat_shodan_internetdb/main.py

@@ -0,0 +1,18 @@
+from ipaddress import ip_address
+
+import requests
+
+from boefjes.job_models import BoefjeMeta
+
+REQUEST_TIMEOUT = 60
+
+
+def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
+    """Make request to InternetDB."""
+    ip = boefje_meta.arguments["input"]["address"]
+    if ip_address(ip).is_private:
+        return [({"info/boefje"}, "Skipping private IP address")]
+    response = requests.get(f"https://internetdb.shodan.io/{ip}", timeout=REQUEST_TIMEOUT)
+    response.raise_for_status()
+
+    return [(set(), response.content)]