From 4fa0bd9e63baeb6a879a93d8b73e9ebe17ba6be4 Mon Sep 17 00:00:00 2001 From: noamblitz <43830693+noamblitz@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:15:14 +0200 Subject: [PATCH 01/29] Generic Finding normalizer (#3383) Co-authored-by: Jan Klopper Co-authored-by: Jeroen Dekkers --- .../plugins/kat_cve_2023_34039/main.py | 3 +- .../plugins/kat_cve_2023_34039/normalize.py | 19 -------- .../kat_finding_normalizer/__init__.py | 0 .../kat_finding_normalizer/normalize.py | 38 +++++++++++++++ .../normalizer.json | 4 +- .../tests/test_generic_finding_normalizer.py | 48 +++++++++++++++++++ .../development_tutorial/creating_a_boefje.md | 2 + 7 files changed, 92 insertions(+), 22 deletions(-) delete mode 100644 boefjes/boefjes/plugins/kat_cve_2023_34039/normalize.py create mode 100644 boefjes/boefjes/plugins/kat_finding_normalizer/__init__.py create mode 100644 boefjes/boefjes/plugins/kat_finding_normalizer/normalize.py rename boefjes/boefjes/plugins/{kat_cve_2023_34039 => kat_finding_normalizer}/normalizer.json (56%) create mode 100644 boefjes/tests/test_generic_finding_normalizer.py diff --git a/boefjes/boefjes/plugins/kat_cve_2023_34039/main.py b/boefjes/boefjes/plugins/kat_cve_2023_34039/main.py index f6e580e1468..d5bd7f4795a 100644 --- a/boefjes/boefjes/plugins/kat_cve_2023_34039/main.py +++ b/boefjes/boefjes/plugins/kat_cve_2023_34039/main.py @@ -57,7 +57,8 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, str | bytes]]: "\n".join( (str(coutput), f"{key_file} is allowed access to vRealize Network Insight on {ip}:{port}") ), - ) + ), + ({"openkat/finding"}, "CVE-2023-34039"), ] except Exception: # noqa: S112 diff --git a/boefjes/boefjes/plugins/kat_cve_2023_34039/normalize.py b/boefjes/boefjes/plugins/kat_cve_2023_34039/normalize.py deleted file mode 100644 index b379e8158f1..00000000000 --- a/boefjes/boefjes/plugins/kat_cve_2023_34039/normalize.py +++ /dev/null @@ -1,19 +0,0 @@ -from collections.abc import Iterable - -from boefjes.job_models import NormalizerOutput -from octopoes.models import Reference -from octopoes.models.ooi.findings import CVEFindingType, Finding - - -def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: - ooi = Reference.from_str(input_ooi["primary_key"]) - - if "is allowed access to vRealize Network Insight " in raw.decode(): - finding_type = CVEFindingType(id="CVE-2023-34039") - finding = Finding( - finding_type=finding_type.reference, - ooi=ooi, - description="Service is most likely vulnerable to CVE-2023-34039", - ) - yield finding_type - yield finding diff --git a/boefjes/boefjes/plugins/kat_finding_normalizer/__init__.py b/boefjes/boefjes/plugins/kat_finding_normalizer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/boefjes/boefjes/plugins/kat_finding_normalizer/normalize.py b/boefjes/boefjes/plugins/kat_finding_normalizer/normalize.py new file mode 100644 index 00000000000..e00cc7d08fb --- /dev/null +++ b/boefjes/boefjes/plugins/kat_finding_normalizer/normalize.py @@ -0,0 +1,38 @@ +import re +from collections.abc import Iterable + +from boefjes.job_models import NormalizerOutput +from octopoes.models import Reference +from octopoes.models.ooi.findings import CVEFindingType, Finding, KATFindingType, RetireJSFindingType, SnykFindingType + +CVE_PATTERN = re.compile(r"CVE-\d{4}-\d{4,}") + + +def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: + ooi = Reference.from_str(input_ooi["primary_key"]) + finding_ids_str = raw.decode() + finding_ids_list = [fid.strip().upper() for fid in finding_ids_str.split(",")] + + finding_type_mapping = { + "CVE": CVEFindingType, + "KAT": KATFindingType, + "SNYK": SnykFindingType, + "RETIREJS": RetireJSFindingType, + } + + for finding_id in finding_ids_list: + parts = finding_id.split("-") + prefix = parts[0] + + if prefix in finding_type_mapping: + if prefix == "CVE" and not CVE_PATTERN.match(finding_id): + raise ValueError(f"{finding_id} is not a valid CVE ID") + + finding_type = finding_type_mapping[prefix](id=finding_id) + finding = Finding( + finding_type=finding_type.reference, + ooi=ooi, + description=f"{finding_id} is found on this OOI", + ) + yield finding_type + yield finding diff --git a/boefjes/boefjes/plugins/kat_cve_2023_34039/normalizer.json b/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json similarity index 56% rename from boefjes/boefjes/plugins/kat_cve_2023_34039/normalizer.json rename to boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json index 4cbb1bddda9..ec7e54b209c 100644 --- a/boefjes/boefjes/plugins/kat_cve_2023_34039/normalizer.json +++ b/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json @@ -1,7 +1,7 @@ { - "id": "kat_cve_2023_normalize", + "id": "kat_generic_finding_normalize", "consumes": [ - "boefje/CVE-2023-34039" + "openkat/finding" ], "produces": [ "Finding", diff --git a/boefjes/tests/test_generic_finding_normalizer.py b/boefjes/tests/test_generic_finding_normalizer.py new file mode 100644 index 00000000000..9e8da108984 --- /dev/null +++ b/boefjes/tests/test_generic_finding_normalizer.py @@ -0,0 +1,48 @@ +from unittest import TestCase + +from boefjes.plugins.kat_finding_normalizer.normalize import run +from octopoes.models import Reference +from octopoes.models.ooi.findings import KATFindingType +from octopoes.models.types import CVEFindingType, Finding + + +class CVETest(TestCase): + maxDiff = None + + def test_single(self): + input_ooi = {"primary_key": "Network|internet"} + + oois = list(run(input_ooi, b"CVE-2021-00000")) + + expected = [ + CVEFindingType(id="CVE-2021-00000"), + Finding( + finding_type=CVEFindingType(id="CVE-2021-00000").reference, + ooi=Reference.from_str("Network|internet"), + description="CVE-2021-00000 is found on this OOI", + ), + ] + + self.assertEqual(expected, oois) + + def test_multiple(self): + input_ooi = {"primary_key": "Network|internet"} + + oois = list(run(input_ooi, b"CVE-2021-00000, KAT-MOCK-FINDING")) + + expected = [ + CVEFindingType(id="CVE-2021-00000"), + Finding( + finding_type=CVEFindingType(id="CVE-2021-00000").reference, + ooi=Reference.from_str("Network|internet"), + description="CVE-2021-00000 is found on this OOI", + ), + KATFindingType(id="KAT-MOCK-FINDING"), + Finding( + finding_type=KATFindingType(id="KAT-MOCK-FINDING").reference, + ooi=Reference.from_str("Network|internet"), + description="KAT-MOCK-FINDING is found on this OOI", + ), + ] + + self.assertEqual(expected, oois) diff --git a/docs/source/developer_documentation/development_tutorial/creating_a_boefje.md b/docs/source/developer_documentation/development_tutorial/creating_a_boefje.md index 719c885c7de..dbb2f3bdc70 100644 --- a/docs/source/developer_documentation/development_tutorial/creating_a_boefje.md +++ b/docs/source/developer_documentation/development_tutorial/creating_a_boefje.md @@ -126,6 +126,8 @@ def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]: The most important part is the return value we send back. This is what will be used by our normalizer to create our new OOIs. +For ease of development, we added a generic finding normalizer. When we just want to create a CVE or other type of finding on the input OOI, we can return the CVE ID or KAT ID as a string with `openkat/finding` as mime-type. + --- The final task of creating a boefje is specifying what DockerFile our boefje should use. We can do this inside the file located in `boefjes/Makefile`. From 75a171f1e70bef911979fc0220d369d60d0178e7 Mon Sep 17 00:00:00 2001 From: zcrt <115991818+zcrt@users.noreply.github.com> Date: Wed, 28 Aug 2024 10:09:20 +0200 Subject: [PATCH 02/29] feat: :chart_with_upwards_trend: default katalogus view to boefje (#3394) Co-authored-by: ammar92 Co-authored-by: Jan Klopper --- .../templates/partials/plugins_navigation.html | 6 +++--- rocky/katalogus/urls.py | 2 +- rocky/rocky/locale/django.pot | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/rocky/katalogus/templates/partials/plugins_navigation.html b/rocky/katalogus/templates/partials/plugins_navigation.html index 05a2ba9f403..bd35aeb0adb 100644 --- a/rocky/katalogus/templates/partials/plugins_navigation.html +++ b/rocky/katalogus/templates/partials/plugins_navigation.html @@ -3,15 +3,15 @@
+ {% endif %} + {% block html_at_end_body %} {% compress js %} - {% endcompress %} {% endblock html_at_end_body %} diff --git a/rocky/reports/views/aggregate_report.py b/rocky/reports/views/aggregate_report.py index 2b7c82a37e3..623905542ba 100644 --- a/rocky/reports/views/aggregate_report.py +++ b/rocky/reports/views/aggregate_report.py @@ -190,7 +190,9 @@ class SaveAggregateReportView(SaveAggregateReportMixin, BreadcrumbsAggregateRepo def post(self, request, *args, **kwargs): old_report_names = request.POST.getlist("old_report_name") new_report_names = request.POST.getlist("report_name") - report_names = list(zip(old_report_names, new_report_names)) + reference_dates = request.POST.getlist("reference_date") + + report_names = list(zip(old_report_names, self.finalise_report_names(new_report_names, reference_dates))) report_ooi = self.save_report(report_names) return redirect( diff --git a/rocky/reports/views/base.py b/rocky/reports/views/base.py index cf6e720bff6..f84e12c15c0 100644 --- a/rocky/reports/views/base.py +++ b/rocky/reports/views/base.py @@ -329,6 +329,25 @@ def get_plugin_data(self): return plugin_data + @staticmethod + def finalise_report_names(report_names: list[str], reference_dates: list[str]) -> list[str]: + final_report_names = [] + + if len(report_names) == len(reference_dates): + for index, report_name in enumerate(report_names): + date_format = "" + if reference_dates[index] and reference_dates[index] != "": + date_format = " - " + if reference_dates[index] == "week": + date_format += _("Week %W, %Y") + else: + date_format += reference_dates[index] + final_report_name = f"{report_name} {date_format}".strip() + final_report_names.append(final_report_name) + if not final_report_names: + return report_names + return final_report_names + def save_report_raw(self, data: dict) -> str: report_data_raw_id = self.bytes_client.upload_raw( raw=ReportDataDict(data).model_dump_json().encode(), @@ -350,7 +369,7 @@ def save_report_ooi( if not name or name.isspace(): name = report_type.name report_ooi = ReportOOI( - name=name, + name=str(name), report_type=str(report_type.id), template=report_type.template_path, report_id=uuid4(), diff --git a/rocky/reports/views/generate_report.py b/rocky/reports/views/generate_report.py index 7a6bb11d2fd..cfd37c4ad05 100644 --- a/rocky/reports/views/generate_report.py +++ b/rocky/reports/views/generate_report.py @@ -2,6 +2,7 @@ from typing import Any from django.contrib import messages +from django.core.exceptions import SuspiciousOperation from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect from django.urls import reverse @@ -170,15 +171,20 @@ class SaveGenerateReportView(SaveGenerateReportMixin, BreadcrumbsGenerateReportV def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: old_report_names = request.POST.getlist("old_report_name") - new_report_names = request.POST.getlist("report_name") - report_names = list(zip(old_report_names, new_report_names)) - report_ooi = self.save_report(report_names) - - return redirect( - reverse("view_report", kwargs={"organization_code": self.organization.code}) - + "?" - + urlencode({"report_id": report_ooi.reference}) - ) + report_names = request.POST.getlist("report_name") + reference_dates = request.POST.getlist("reference_date") + + if "" in report_names: + raise SuspiciousOperation(_("Empty name should not be possible.")) + else: + final_report_names = list(zip(old_report_names, self.finalise_report_names(report_names, reference_dates))) + report_ooi = self.save_report(final_report_names) + + return redirect( + reverse("view_report", kwargs={"organization_code": self.organization.code}) + + "?" + + urlencode({"report_id": report_ooi.reference}) + ) def create_report_names(oois_pk, report_types) -> dict[str, str]: diff --git a/rocky/reports/views/mixins.py b/rocky/reports/views/mixins.py index 99a8e5f37bf..cd72d658690 100644 --- a/rocky/reports/views/mixins.py +++ b/rocky/reports/views/mixins.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Any from django.contrib import messages @@ -57,6 +58,7 @@ def save_report(self, report_names: list) -> ReportOOI: number_of_reports += 1 observed_at = self.get_observed_at() + now = datetime.utcnow() # if its not a single report, we need a parent if number_of_reports > 1: @@ -68,18 +70,19 @@ def save_report(self, report_names: list) -> ReportOOI: parent=None, has_parent=False, observed_at=observed_at, - name=report_names[0][1], + name=now.strftime(report_names[0][1]), ) + for report_type, ooi_data in report_data.items(): for ooi, data in ooi_data.items(): + name_to_save = "" + report_type_name = str(get_report_by_id(report_type).name) ooi_name = Reference.from_str(ooi).human_readable for default_name, updated_name in report_names: + # Use default_name to check if we're on the right index in the list to update the name to save. if ooi_name in default_name and report_type_name in default_name: - name = updated_name - break - else: - name = default_name + name_to_save = updated_name break raw_id = self.save_report_raw(data={"report_data": data["data"]}) @@ -91,7 +94,7 @@ def save_report(self, report_names: list) -> ReportOOI: parent=report_ooi.reference, has_parent=True, observed_at=observed_at, - name=name, + name=now.strftime(name_to_save), ) # if its a single report we can just save it as complete else: @@ -108,7 +111,7 @@ def save_report(self, report_names: list) -> ReportOOI: parent=None, has_parent=False, observed_at=observed_at, - name=report_names[0][1], + name=now.strftime(report_names[0][1]), ) # If OOI could not be found or the date is incorrect, it will be shown to the user as a message error if error_reports: @@ -171,6 +174,8 @@ def save_report(self, report_names: list) -> ReportOOI: } ) + now = datetime.utcnow() + # Create the report report_data_raw_id = self.save_report_raw(data=post_processed_data) report_ooi = self.save_report_ooi( @@ -180,7 +185,7 @@ def save_report(self, report_names: list) -> ReportOOI: parent=None, has_parent=False, observed_at=observed_at, - name=report_names[0][1], + name=now.strftime(report_names[0][1]), ) # Save the child reports to bytes diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 12e9687e218..a3916d88b23 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-08-27 13:18+0000\n" +"POT-Creation-Date: 2024-08-28 08:22+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -575,7 +575,6 @@ msgstr "" #: katalogus/templates/change_clearance_level.html #: katalogus/templates/confirmation_clone_settings.html #: katalogus/templates/plugin_settings_delete.html -#: reports/templates/partials/export_report_settings.html #: rocky/templates/oois/ooi_delete.html #: rocky/templates/oois/ooi_mute_finding.html #: rocky/templates/organizations/organization_edit.html @@ -631,7 +630,6 @@ msgid "Sorting options" msgstr "" #: katalogus/forms/plugin_settings.py -#: reports/templates/partials/export_report_settings.html msgid "This field is required." msgstr "" @@ -3379,7 +3377,11 @@ msgid "Report name" msgstr "" #: reports/templates/partials/export_report_settings.html -msgid "You can name individual reports." +msgid "" +"Give your report a custom name and optionally add the reports' reference " +"date to the name. To do so you can select a standard option or use a Python strftime code in " +"the report name." msgstr "" #: reports/templates/partials/export_report_settings.html @@ -3387,21 +3389,7 @@ msgid "Report names:" msgstr "" #: reports/templates/partials/export_report_settings.html -msgid "Edit " -msgstr "" - -#: reports/templates/partials/export_report_settings.html -msgid "" -"\n" -" Give your report " -"a custom name and optionally add the reports' reference date\n" -" to the name. To " -"do so you can select an option below.\n" -" " -msgstr "" - -#: reports/templates/partials/export_report_settings.html -msgid "Report name settings." +msgid "(Required)" msgstr "" #: reports/templates/partials/export_report_settings.html @@ -3409,7 +3397,8 @@ msgid "Add reference date" msgstr "" #: reports/templates/partials/export_report_settings.html -msgid "Select option" +#: rocky/views/scan_profile.py +msgid "Reset" msgstr "" #: reports/templates/partials/export_report_settings.html @@ -3432,10 +3421,6 @@ msgstr "" msgid "Year" msgstr "" -#: reports/templates/partials/export_report_settings.html -msgid "Confirm" -msgstr "" - #: reports/templates/partials/export_report_settings.html #: reports/templates/report_overview/report_overview_header.html #: reports/views/generate_report.py @@ -3946,6 +3931,14 @@ msgstr "" msgid "Report type '%s' does not exist." msgstr "" +#: reports/views/base.py +msgid "Week %W, %Y" +msgstr "" + +#: reports/views/generate_report.py +msgid "Empty name should not be possible." +msgstr "" + #: reports/views/generate_report.py #, python-brace-format msgid "Concatenated Report for {oois_count} objects" @@ -6698,10 +6691,6 @@ msgstr "" msgid "Can not reset scan level. Scan level of {ooi_name} not declared" msgstr "" -#: rocky/views/scan_profile.py -msgid "Reset" -msgstr "" - #: rocky/views/scheduler.py msgid "" "Your task is scheduled and will soon be started in the background. Results " From 77f868f9c28880d6f5d873bfe1e177b467d05ef3 Mon Sep 17 00:00:00 2001 From: HeleenSG Date: Wed, 28 Aug 2024 11:00:47 +0200 Subject: [PATCH 05/29] =?UTF-8?q?Styling=20fixes=20within=20filters,=20hie?= =?UTF-8?q?rarchy=20fix=20on=20organisation=20members=20b=E2=80=A6=20(#332?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rieven Co-authored-by: Jan Klopper Co-authored-by: ammar92 Co-authored-by: Jeroen Dekkers --- .../css/themes/soft/manon/accordion.scss | 2 +- .../assets/css/themes/soft/manon/button.scss | 6 +- .../assets/css/themes/soft/manon/filter.scss | 50 +++++++++++++- rocky/assets/css/themes/soft/manon/form.scss | 6 +- .../css/themes/soft/manon/login-meta.scss | 2 +- .../css/themes/soft/manon/navigation.scss | 2 +- .../css/themes/soft/manon/sidemenu.scss | 6 +- rocky/assets/css/themes/soft/manon/table.scss | 2 +- .../templates/partials/katalogus_filter.html | 15 +++-- rocky/katalogus/views/katalogus.py | 8 +++ rocky/package.json | 3 +- .../templates/partials/report_ooi_list.html | 9 +-- rocky/rocky/locale/django.pot | 66 ++++++++++--------- .../templates/crisis_room/crisis_room.html | 2 +- .../templates/findings/finding_list.html | 2 +- .../templates/findings/findings_filter.html | 20 +++--- rocky/rocky/templates/oois/ooi_list.html | 10 +-- rocky/rocky/templates/oois/ooi_tree.html | 4 +- .../organization_crisis_room.html | 2 +- .../organization_member_list.html | 9 ++- .../elements/ooi_list_settings_form.html | 2 +- .../templates/partials/form/fieldset.html | 2 +- .../templates/partials/list_filters.html | 4 -- .../templates/partials/ooi_list_filters.html | 21 +++--- .../organization_member_list_filters.html | 20 +++--- .../templates/tasks/partials/task_filter.html | 8 ++- rocky/rocky/views/finding_list.py | 10 ++- rocky/rocky/views/ooi_tree.py | 12 +++- rocky/rocky/views/ooi_view.py | 19 ++++-- rocky/rocky/views/scheduler.py | 14 +++- rocky/rocky/views/tasks.py | 1 + rocky/tests/test_crisis_room.py | 10 +-- 32 files changed, 228 insertions(+), 121 deletions(-) diff --git a/rocky/assets/css/themes/soft/manon/accordion.scss b/rocky/assets/css/themes/soft/manon/accordion.scss index f621b8da4b3..fc77c6afbd0 100644 --- a/rocky/assets/css/themes/soft/manon/accordion.scss +++ b/rocky/assets/css/themes/soft/manon/accordion.scss @@ -33,7 +33,7 @@ --accordion-content-gap: 1.5rem; --accordion-content-border-width: 0 0 1px 0; --accordion-content-background-color: var(--colors-white); - --accordion-content-text-color: var(--colors-black); + --accordion-content-text-color: var(--application-base-text-color); --accordion-content-padding: 1.5rem 2rem 2rem 2rem; } diff --git a/rocky/assets/css/themes/soft/manon/button.scss b/rocky/assets/css/themes/soft/manon/button.scss index 8cc1d37ea2b..93c3c716d21 100644 --- a/rocky/assets/css/themes/soft/manon/button.scss +++ b/rocky/assets/css/themes/soft/manon/button.scss @@ -27,18 +27,18 @@ /* Hover */ --button-base-hover-background-color: var(--colors-yellow-300); - --button-base-hover-text-color: var(--colors-black); + --button-base-hover-text-color: var(--application-base-text-color); /* Active */ --button-base-active-background-color: var(--colors-yellow-300); - --button-base-active-text-color: var(--colors-black); + --button-base-active-text-color: var(--application-base-text-color); --button-base-active-border-style: var(--button-base-border-style); --button-base-active-border-color: var(--button-base-active-background-color); --button-base-active-border-width: var(--button-base-border-width); /* Focus */ --button-base-focus-background-color: var(--colors-yellow-300); - --button-base-focus-text-color: var(--colors-black); + --button-base-focus-text-color: var(--application-base-text-color); --button-base-focus-border-style: var(--button-base-border-style); --button-base-focus-border-color: var(--button-base-focus-background-color); --button-base-focus-border-width: var(--button-base-border-width); diff --git a/rocky/assets/css/themes/soft/manon/filter.scss b/rocky/assets/css/themes/soft/manon/filter.scss index f0022e419e5..cc7c320b7b0 100644 --- a/rocky/assets/css/themes/soft/manon/filter.scss +++ b/rocky/assets/css/themes/soft/manon/filter.scss @@ -1,10 +1,54 @@ /* Filter - Variables */ :root { + /* Intro */ + --filter-intro-border-width: 1px 0 0 0; + + /* Button */ --filter-button-font-size: 1rem; - --filter-button-font-weight: normal; - --filter-button-text-color: var(--application-base-accent-color-text-color); - --filter-intro-text-color: var(--application-base-text-color); + --filter-button-font-weight: bold; + --filter-button-text-color: var(--link-text-color); + --filter-button-margin-left: 0; + --filter-button-padding: var(--spacing-grid-150) var(--spacing-grid-100); + + /* Button icon */ --filter-button-icon-before-open-content: "\ea5f"; --filter-button-icon-before-close-content: "\ea62"; } + +.filter form { + background-color: transparent; + padding: 1.5rem 2rem 2rem; + border-bottom: 1px solid var(--colors-grey-200); +} + +/* Temporary fix, this needs to be updated after this issue has been solved in Manon */ +.filter > div button { + &::before, + &::after { + font-family: var(--filter-button-icon-font-family, inherit); + line-height: var(--filter-button-icon-line-height); + font-size: var(--filter-button-icon-font-size, inherit); + font-weight: var(--filter-button-icon-font-weight, inherit); + } + + &[aria-expanded="true"] { + &::before { + content: var(--filter-button-icon-before-close-content); + } + + &::after { + content: var(--filter-button-icon-after-close-content); + } + } + + &[aria-expanded="false"] { + &::before { + content: var(--filter-button-icon-before-open-content); + } + + &::after { + content: var(--filter-button-icon-after-open-content); + } + } +} diff --git a/rocky/assets/css/themes/soft/manon/form.scss b/rocky/assets/css/themes/soft/manon/form.scss index 26bbd7eb2d2..cdfbe4649f3 100644 --- a/rocky/assets/css/themes/soft/manon/form.scss +++ b/rocky/assets/css/themes/soft/manon/form.scss @@ -1,11 +1,13 @@ /* Form - Variables */ :root { + --form-base-text-color: var(--application-base-text-color); + /* Accent color */ --form-accent-color-color: var(--branding-color-2); /* Input */ - --form-base-input-text-color: var(--colors-black); + --form-base-input-text-color: var(--form-base-text-color); --form-input-border-width: 1px; --form-input-border-style: solid; --form-input-border-color: var(--colors-grey-200); @@ -16,7 +18,7 @@ --form-help-button-icon-font-family: var(--icon-font-family); /* Textarea */ - --form-textarea-text-color: var(--form-base-input-text-color); + --form-textarea-text-color: var(--form-base-text-color); --form-textarea-border-color: var(--colors-grey-200); /* Fieldset */ diff --git a/rocky/assets/css/themes/soft/manon/login-meta.scss b/rocky/assets/css/themes/soft/manon/login-meta.scss index 07fc5b6a0ed..9a4bf2e7026 100644 --- a/rocky/assets/css/themes/soft/manon/login-meta.scss +++ b/rocky/assets/css/themes/soft/manon/login-meta.scss @@ -2,5 +2,5 @@ :root { --login-meta-background-color: var(--colors-grey-100); - --login-meta-color: var(--colors-black); + --login-meta-color: var(--application-base-text-color); } diff --git a/rocky/assets/css/themes/soft/manon/navigation.scss b/rocky/assets/css/themes/soft/manon/navigation.scss index 8ee2227eb94..9b6ebc63e51 100644 --- a/rocky/assets/css/themes/soft/manon/navigation.scss +++ b/rocky/assets/css/themes/soft/manon/navigation.scss @@ -15,7 +15,7 @@ /* Collapsing menu */ --navigation-collapsible-menu-collapsing-menu-width: 100%; --navigation-collapsible-menu-list-item-collapsed-hover-background-color: var( - --colors-black + --application-base-text-color ); /* Collapsing menu item */ diff --git a/rocky/assets/css/themes/soft/manon/sidemenu.scss b/rocky/assets/css/themes/soft/manon/sidemenu.scss index a8a6b719872..c8a9c36558e 100644 --- a/rocky/assets/css/themes/soft/manon/sidemenu.scss +++ b/rocky/assets/css/themes/soft/manon/sidemenu.scss @@ -67,7 +67,7 @@ main.sidemenu { a:visited, a:focus, a:active { - color: var(--colors-black); + color: var(--application-base-text-color); text-decoration: none; } @@ -76,7 +76,7 @@ main.sidemenu { a:visited, a:focus, a:active { - color: var(--colors-black); + color: var(--application-base-text-color); text-decoration: none; font-style: italic; word-break: break-all; @@ -112,7 +112,7 @@ main.sidemenu { font-size: var(--sidemenu-toggle-button-font-size); border: 0; border-radius: 0; - color: var(--colors-black); + color: var(--application-base-text-color); /* Transition */ transition: margin-left 1s; diff --git a/rocky/assets/css/themes/soft/manon/table.scss b/rocky/assets/css/themes/soft/manon/table.scss index 850fab9beaa..c186d4cb9e3 100644 --- a/rocky/assets/css/themes/soft/manon/table.scss +++ b/rocky/assets/css/themes/soft/manon/table.scss @@ -50,7 +50,7 @@ a:visited, a:focus, a:hover { - color: var(--colors-black); + color: var(--application-base-text-color); text-decoration: none; } } diff --git a/rocky/katalogus/templates/partials/katalogus_filter.html b/rocky/katalogus/templates/partials/katalogus_filter.html index 6659197e6aa..935ecc39c4a 100644 --- a/rocky/katalogus/templates/partials/katalogus_filter.html +++ b/rocky/katalogus/templates/partials/katalogus_filter.html @@ -1,8 +1,15 @@ -{% extends "partials/list_filters.html" %} - {% load i18n %} -{% block filter_form %} +
+
+ +
{% include "partials/form/fieldset.html" with fields=form %} @@ -11,4 +18,4 @@ {% translate "Clear filter" %}
-{% endblock filter_form %} + diff --git a/rocky/katalogus/views/katalogus.py b/rocky/katalogus/views/katalogus.py index e63748b49a2..f40a3e9e7aa 100644 --- a/rocky/katalogus/views/katalogus.py +++ b/rocky/katalogus/views/katalogus.py @@ -52,10 +52,18 @@ def sort_queryset(self, queryset, sort_options): def sort_alphabetic_ascending(self, queryset): return sorted(queryset, key=lambda item: item.name.lower()) + def count_active_filters(self): + filter_options = self.request.GET.get("filter_options") + sortin_options = self.request.GET.get("sorting_options") + count_filter_options = 1 if filter_options and filter_options != "all" else 0 + count_sorting_options = 1 if sortin_options and sortin_options != "a-z" else 0 + return count_filter_options + count_sorting_options + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["view_type"] = self.kwargs.get("view_type", "grid") context["url_name"] = self.request.resolver_match.url_name + context["active_filters_counter"] = self.count_active_filters() context["breadcrumbs"] = [ { "url": reverse("katalogus", kwargs={"organization_code": self.organization.code}), diff --git a/rocky/package.json b/rocky/package.json index a9c7a0543fa..e1597de7b38 100644 --- a/rocky/package.json +++ b/rocky/package.json @@ -16,5 +16,6 @@ }, "browserslist": [ "last 1 Chrome version" - ] + ], + "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e" } diff --git a/rocky/reports/templates/partials/report_ooi_list.html b/rocky/reports/templates/partials/report_ooi_list.html index 66b1fef5572..255f8d41b3b 100644 --- a/rocky/reports/templates/partials/report_ooi_list.html +++ b/rocky/reports/templates/partials/report_ooi_list.html @@ -11,15 +11,8 @@

{% endblocktranslate %}

{% translate "Select which objects you want to include in your report." %}

- {% translate "Filter" as filter_title %} - {% include "partials/ooi_list_filters.html" with title=filter_title clearance_level_filter_form=clearance_level_filter_form %} + {% include "partials/ooi_list_filters.html" %} -

- {% if active_filters %} - {% translate "Currently filtered on:" %} - {% for filter, value in active_filters.items %}{{ filter }}{{ value|title }} {% endfor %} - {% endif %} -

{% if not ooi_list %}

{% translate "No objects found." %}

diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index a3916d88b23..3bf00674a16 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-08-28 08:22+0000\n" +"POT-Creation-Date: 2024-08-28 08:40+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -874,6 +874,30 @@ msgstr "" msgid "You don't have permission to enable" msgstr "" +#: katalogus/templates/partials/katalogus_filter.html +#: rocky/templates/findings/findings_filter.html +#: rocky/templates/partials/ooi_list_filters.html +#: rocky/templates/partials/organization_member_list_filters.html +#: rocky/templates/tasks/partials/task_filter.html +msgid "Hide filters" +msgstr "" + +#: katalogus/templates/partials/katalogus_filter.html +#: rocky/templates/findings/findings_filter.html +#: rocky/templates/partials/ooi_list_filters.html +#: rocky/templates/partials/organization_member_list_filters.html +#: rocky/templates/tasks/partials/task_filter.html +msgid "Show filters" +msgstr "" + +#: katalogus/templates/partials/katalogus_filter.html +#: rocky/templates/findings/findings_filter.html +#: rocky/templates/partials/ooi_list_filters.html +#: rocky/templates/partials/organization_member_list_filters.html +#: rocky/templates/tasks/partials/task_filter.html +msgid "applied" +msgstr "" + #: katalogus/templates/partials/katalogus_filter.html msgid "Filter plugins" msgstr "" @@ -3513,17 +3537,6 @@ msgstr[1] "" msgid "Select which objects you want to include in your report." msgstr "" -#: reports/templates/partials/report_ooi_list.html -#: rocky/templates/admin/change_list.html rocky/templates/oois/ooi_list.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Filter" -msgstr "" - -#: reports/templates/partials/report_ooi_list.html -#: rocky/templates/oois/ooi_list.html -msgid "Currently filtered on:" -msgstr "" - #: reports/templates/partials/report_ooi_list.html msgid "No objects found." msgstr "" @@ -4380,7 +4393,6 @@ msgstr "" #: tools/view_helpers.py rocky/templates/header.html #: rocky/templates/organizations/organization_list.html #: rocky/templates/organizations/organization_member_list.html -#: rocky/templates/partials/organization_member_list_filters.html #: rocky/views/organization_member_edit.py msgid "Members" msgstr "" @@ -4613,6 +4625,10 @@ msgid_plural "Please correct the errors below." msgstr[0] "" msgstr[1] "" +#: rocky/templates/admin/change_list.html +msgid "Filter" +msgstr "" + #: rocky/templates/admin/change_list.html msgid "Hide counts" msgstr "" @@ -4935,7 +4951,9 @@ msgstr "" msgid "OpenKAT logo, go to the homepage of OpenKAT" msgstr "" -#: rocky/templates/header.html rocky/views/finding_list.py +#: rocky/templates/header.html +#: rocky/templates/organizations/organization_crisis_room.html +#: rocky/views/finding_list.py msgid "Crisis room" msgstr "" @@ -5402,14 +5420,6 @@ msgstr "" msgid "shows the same objects." msgstr "" -#: rocky/templates/oois/ooi_tree.html -msgid "Object tree" -msgstr "" - -#: rocky/templates/organizations/organization_crisis_room.html -msgid "Crisis room:" -msgstr "" - #: rocky/templates/organizations/organization_crisis_room.html #: rocky/templates/organizations/organization_settings.html msgid "indemnification warning" @@ -6142,14 +6152,6 @@ msgstr "" msgid "Download task data" msgstr "" -#: rocky/templates/tasks/partials/task_filter.html -msgid "Hide filters" -msgstr "" - -#: rocky/templates/tasks/partials/task_filter.html -msgid "Show filters" -msgstr "" - #: rocky/templates/two_factor/_wizard_actions.html msgid "Log in" msgstr "" @@ -6559,6 +6561,10 @@ msgstr "" msgid "Graph Visualisation" msgstr "" +#: rocky/views/ooi_view.py +msgid "Observed_at: " +msgstr "" + #: rocky/views/ooi_view.py msgid "OOI types: " msgstr "" diff --git a/rocky/rocky/templates/crisis_room/crisis_room.html b/rocky/rocky/templates/crisis_room/crisis_room.html index 2924b864ba5..f08a8efcccf 100644 --- a/rocky/rocky/templates/crisis_room/crisis_room.html +++ b/rocky/rocky/templates/crisis_room/crisis_room.html @@ -9,7 +9,7 @@
-

{% translate "Crisis Room" %}

+

{% translate "Crisis Room" %} @ {{ observed_at|date:'M d, Y' }}

{% translate "An overview of all (critical) findings OpenKAT found. Check the detail section for additional severity information." %}

diff --git a/rocky/rocky/templates/findings/finding_list.html b/rocky/rocky/templates/findings/finding_list.html index d9c5c47e65d..a23d6eb82b0 100644 --- a/rocky/rocky/templates/findings/finding_list.html +++ b/rocky/rocky/templates/findings/finding_list.html @@ -11,7 +11,7 @@
-

{% translate "Findings @ " %}{{ valid_time|date }}

+

{% translate "Findings @ " %}{{ observed_at|date }}

{% blocktranslate trimmed with organization.name as organization_name %} An overview of all findings OpenKAT found for organization {{ organization_name }}. diff --git a/rocky/rocky/templates/findings/findings_filter.html b/rocky/rocky/templates/findings/findings_filter.html index 599ad8dc3ed..971d3f373ba 100644 --- a/rocky/rocky/templates/findings/findings_filter.html +++ b/rocky/rocky/templates/findings/findings_filter.html @@ -1,13 +1,15 @@ -{% extends "partials/list_filters.html" %} - -{% load static %} {% load i18n %} -{% block title %} - {% if count %}{{ count }}{% endif %} - {{ title }} -{% endblock title %} -{% block filter_form %} +

+
+ +
{% include "partials/form/fieldset.html" with fields=observed_at_form %} @@ -22,4 +24,4 @@ {% translate "Clear filters" %}
-{% endblock filter_form %} +
diff --git a/rocky/rocky/templates/oois/ooi_list.html b/rocky/rocky/templates/oois/ooi_list.html index ac30802eb0b..17f316dba27 100644 --- a/rocky/rocky/templates/oois/ooi_list.html +++ b/rocky/rocky/templates/oois/ooi_list.html @@ -23,16 +23,8 @@

{% endblocktranslate %}

{% include "partials/ooi_list_toolbar.html" %} + {% include "partials/ooi_list_filters.html" %} - {% translate "Filter" as filter_title %} - {% include "partials/ooi_list_filters.html" with title=filter_title clearance_level_filter_form=clearance_level_filter_form %} - -

- {% if active_filters %} - {% translate "Currently filtered on:" %} - {% for filter, value in active_filters.items %}{{ filter }}{{ value|title }} {% endfor %} - {% endif %} -

{% blocktranslate with length=ooi_list|length total=total_oois %}Showing {{ length }} of {{ total }} objects{% endblocktranslate %}

diff --git a/rocky/rocky/templates/oois/ooi_tree.html b/rocky/rocky/templates/oois/ooi_tree.html index 191ca596590..eceed2a3dfd 100644 --- a/rocky/rocky/templates/oois/ooi_tree.html +++ b/rocky/rocky/templates/oois/ooi_tree.html @@ -11,9 +11,7 @@
{% include "partials/ooi_head.html" with ooi=ooi view="ooi_tree" %} - - {% translate "Object tree" as filter_title %} - {% include "partials/ooi_list_filters.html" with title=filter_title ooi_id=ooi.primary_key %} + {% include "partials/ooi_list_filters.html" with ooi_id=ooi.primary_key %} {% if tree_view == "table" %} {% include "partials/elements/ooi_tree_table.html" with list=tree ooi_id=ooi.primary_key %} diff --git a/rocky/rocky/templates/organizations/organization_crisis_room.html b/rocky/rocky/templates/organizations/organization_crisis_room.html index a38cb5e2e5e..9cd0d3e6ee9 100644 --- a/rocky/rocky/templates/organizations/organization_crisis_room.html +++ b/rocky/rocky/templates/organizations/organization_crisis_room.html @@ -9,7 +9,7 @@
-

{% translate "Crisis room:" %} {{ organization.name }}

+

{% translate "Crisis room" %} {{ organization.name }} @ {{ observed_at|date:'M d, Y' }}

{% if not indemnification_present %}

{% translate "Organization:" %} {{ organization.name }}

{% blocktranslate with organization_name=organization.name %} An overview of "{{ organization_name }}" its members. {% endblocktranslate %} -

{% translate "Members" %}

+

+ {% translate "Members" %} + {% if members.count %} + ({{ members.count }}) + {% else %} + (0) + {% endif %} +

{% if perms.tools.add_organizationmember %}
diff --git a/rocky/rocky/templates/partials/elements/ooi_list_settings_form.html b/rocky/rocky/templates/partials/elements/ooi_list_settings_form.html index 498ff9fefc9..16fbbc32e9c 100644 --- a/rocky/rocky/templates/partials/elements/ooi_list_settings_form.html +++ b/rocky/rocky/templates/partials/elements/ooi_list_settings_form.html @@ -13,7 +13,7 @@ class="toggle-all toggle-on ghost" data-toggle-target="ooi_type">{% translate "Toggle all OOI types" %} {% endif %} - {% include "partials/form/fieldset.html" with fields=clearance_level_filter_form %} + {% include "partials/form/fieldset.html" with fields=clearance_level_filter_form fieldset_parent_class="column-2" %} {% include "partials/form/fieldset.html" with fields=ooi_search_form %}
diff --git a/rocky/rocky/templates/partials/form/fieldset.html b/rocky/rocky/templates/partials/form/fieldset.html index d7fc4795ba0..528249768bf 100644 --- a/rocky/rocky/templates/partials/form/fieldset.html +++ b/rocky/rocky/templates/partials/form/fieldset.html @@ -1,6 +1,6 @@ {% load i18n %} -
+
{% if legend %}{{ legend }}{% endif %} {% for field in fields %} {% if not fieldset or field.name in fieldset.split %} diff --git a/rocky/rocky/templates/partials/list_filters.html b/rocky/rocky/templates/partials/list_filters.html index eabfe118f88..ef4af912982 100644 --- a/rocky/rocky/templates/partials/list_filters.html +++ b/rocky/rocky/templates/partials/list_filters.html @@ -3,10 +3,6 @@
-

- {% block title %} - {% endblock title %} -

+
+ {% include "partials/elements/ooi_list_settings_form.html" %} -{% endblock filter_form %} +
diff --git a/rocky/rocky/templates/partials/organization_member_list_filters.html b/rocky/rocky/templates/partials/organization_member_list_filters.html index 2fca6925cf6..6857a3a72bd 100644 --- a/rocky/rocky/templates/partials/organization_member_list_filters.html +++ b/rocky/rocky/templates/partials/organization_member_list_filters.html @@ -1,11 +1,15 @@ -{% extends "partials/list_filters.html" %} - {% load i18n %} -{% block title %} - {{ members.count }} {% translate "Members" %} -{% endblock title %} -{% block filter_form %} +
+
+ +
{% translate "Shown status types" %} @@ -16,7 +20,7 @@ id="option-filter-{{ option.value }}" name="client_status" value="{{ option.value }}" - {% if option.checked %}checked{% endif %}> + {% if option %}checked{% endif %}>
{% endfor %} @@ -39,4 +43,4 @@ -{% endblock filter_form %} +
diff --git a/rocky/rocky/templates/tasks/partials/task_filter.html b/rocky/rocky/templates/tasks/partials/task_filter.html index 92b34d69a09..221c24f414e 100644 --- a/rocky/rocky/templates/tasks/partials/task_filter.html +++ b/rocky/rocky/templates/tasks/partials/task_filter.html @@ -2,9 +2,13 @@
-

{% translate "Filter" %}

+ data-hide-filters-label='{% translate "Hide filters" %}'> + {% translate "Show filters" %} + {% if active_filters_counter > 0 %} + ({{ active_filters_counter }} {% translate "applied" %}) + {% endif %} +
{% include "partials/form/fieldset.html" with fields=task_filter_form %} diff --git a/rocky/rocky/views/finding_list.py b/rocky/rocky/views/finding_list.py index 9b0430d2f7c..9f4e9ea2dc6 100644 --- a/rocky/rocky/views/finding_list.py +++ b/rocky/rocky/views/finding_list.py @@ -1,4 +1,5 @@ from collections.abc import Iterable +from datetime import datetime, timezone from typing import Any import structlog @@ -60,6 +61,12 @@ def setup(self, request, *args, **kwargs): self.exclude_muted = self.muted_findings == "non-muted" self.only_muted = self.muted_findings == "muted" + def count_observed_at_filter(self) -> int: + return 1 if datetime.now(timezone.utc).date() != self.observed_at.date() else 0 + + def count_active_filters(self): + return len(self.severities) + 1 if self.muted_findings else 0 + self.count_observed_at_filter() + def get_queryset(self) -> FindingList: return FindingList( octopoes_connector=self.octopoes_api_connector, @@ -72,10 +79,11 @@ def get_queryset(self) -> FindingList: def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["observed_at_form"] = self.get_connector_form() - context["valid_time"] = self.observed_at + context["observed_at"] = self.observed_at context["severity_filter"] = FindingSeverityMultiSelectForm({"severity": list(self.severities)}) context["muted_findings_filter"] = MutedFindingSelectionForm({"muted_findings": self.muted_findings}) context["only_muted"] = self.only_muted + context["active_filters_counter"] = self.count_active_filters() return context diff --git a/rocky/rocky/views/ooi_tree.py b/rocky/rocky/views/ooi_tree.py index e943980581d..b03151cbe5f 100644 --- a/rocky/rocky/views/ooi_tree.py +++ b/rocky/rocky/views/ooi_tree.py @@ -1,3 +1,5 @@ +from datetime import datetime, timezone + from django.utils.translation import gettext_lazy as _ from django.views.generic import TemplateView from tools.forms.ooi import OoiTreeSettingsForm @@ -18,6 +20,14 @@ def get_filtered_tree(self, tree_dict): filtered_types = self.request.GET.getlist("ooi_type", []) return filter_ooi_tree(tree_dict, filtered_types) + def count_observed_at_filter(self) -> int: + return 1 if datetime.now(timezone.utc).date() != self.observed_at.date() else 0 + + def count_active_filters(self): + count_depth_filter = len(self.request.GET.getlist("depth", [])) + count_ooi_type_filter = len(self.request.GET.getlist("ooi_type", [])) + return self.count_observed_at_filter() + count_depth_filter + count_ooi_type_filter + def get_connector_form_kwargs(self): kwargs = super().get_connector_form_kwargs() @@ -44,7 +54,7 @@ def get_context_data(self, **kwargs): context["tree"] = self.get_filtered_tree(self.get_tree_dict()) context["tree_view"] = self.request.GET.get("view", "condensed") context["observed_at_form"] = self.get_connector_form() - + context["active_filters_counter"] = self.count_active_filters() return context diff --git a/rocky/rocky/views/ooi_view.py b/rocky/rocky/views/ooi_view.py index 3eaca0b7845..55613017936 100644 --- a/rocky/rocky/views/ooi_view.py +++ b/rocky/rocky/views/ooi_view.py @@ -39,8 +39,13 @@ def setup(self, request, *args, **kwargs): self.clearance_types = request.GET.getlist("clearance_type", []) self.search_string = request.GET.get("search", "") + def count_observed_at_filter(self) -> int: + return 1 if datetime.now(timezone.utc).date() != self.observed_at.date() else 0 + def get_active_filters(self) -> dict[str, str]: active_filters = {} + if self.count_observed_at_filter() > 0: + active_filters[_("Observed_at: ")] = self.observed_at.strftime("%Y-%m-%d") if self.filtered_ooi_types: active_filters[_("OOI types: ")] = ", ".join(self.filtered_ooi_types) if self.clearance_levels: @@ -50,6 +55,14 @@ def get_active_filters(self) -> dict[str, str]: active_filters[_("Clearance type: ")] = ", ".join(self.clearance_types) return active_filters + def count_active_filters(self): + return ( + len(self.filtered_ooi_types) + + len(self.clearance_levels) + + len(self.clearance_types) + + self.count_observed_at_filter() + ) + def get_ooi_scan_levels(self) -> set[ScanLevel]: if not self.clearance_levels: return self.scan_levels @@ -69,15 +82,11 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["observed_at"] = self.observed_at context["observed_at_form"] = self.get_connector_form() - context["ooi_types_selection"] = self.filtered_ooi_types - context["clearance_levels_selection"] = self.clearance_levels context["clearance_level_filter_form"] = ClearanceFilterForm(self.request.GET) - context["clearance_types_selection"] = self.clearance_types - - context["active_filters"] = self.get_active_filters() + context["active_filters_counter"] = self.count_active_filters() return context diff --git a/rocky/rocky/views/scheduler.py b/rocky/rocky/views/scheduler.py index 4dd3898d107..1fb75f195a2 100644 --- a/rocky/rocky/views/scheduler.py +++ b/rocky/rocky/views/scheduler.py @@ -34,13 +34,23 @@ def get_task_filters(self) -> dict[str, Any]: "scheduler_id": f"{self.task_type}-{self.organization.code}", "task_type": self.task_type, "plugin_id": None, # plugin_id present and set at plugin detail - **self.get_form_data(), + **self.get_task_filter_form_data(), } - def get_form_data(self) -> dict[str, Any]: + def get_task_filter_form_data(self) -> dict[str, Any]: form_data = self.get_task_filter_form().data.dict() return {k: v for k, v in form_data.items() if v} + def count_active_task_filters(self): + form_data = self.get_task_filter_form_data() + + count = len(form_data) + for task_filter in form_data: + if task_filter in ("observed_at", "ooi_id"): + count -= 1 + + return count + def get_task_filter_form(self) -> TaskFilterForm: return self.task_filter_form(self.request.GET) diff --git a/rocky/rocky/views/tasks.py b/rocky/rocky/views/tasks.py index 8099d037496..698b1e15d7e 100644 --- a/rocky/rocky/views/tasks.py +++ b/rocky/rocky/views/tasks.py @@ -44,6 +44,7 @@ def post(self, request, *args, **kwargs): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["task_filter_form"] = self.get_task_filter_form() + context["active_filters_counter"] = self.count_active_task_filters() context["stats"] = self.get_task_statistics() context["breadcrumbs"] = [ { diff --git a/rocky/tests/test_crisis_room.py b/rocky/tests/test_crisis_room.py index ab5499e8550..fb52dcb483f 100644 --- a/rocky/tests/test_crisis_room.py +++ b/rocky/tests/test_crisis_room.py @@ -1,5 +1,3 @@ -from datetime import datetime, timezone - from crisis_room.views import CrisisRoomView, OrganizationFindingCountPerSeverity from django.urls import resolve, reverse from pytest_django.asserts import assertContains @@ -31,13 +29,17 @@ def test_crisis_room_observed_at(rf, client_member, mock_crisis_room_octopoes): request.resolver_match = resolve(reverse("crisis_room")) response = CrisisRoomView.as_view()(request) assert response.status_code == 200 - assertContains(response, "Jan 01, 2021") + assertContains(response, "Jan 01, 2021") # Next to title crisis room + assertContains(response, "2021-01-01") # Date Widget + +def test_crisis_room_observed_at_bad_format(rf, client_member, mock_crisis_room_octopoes): request = setup_request(rf.get("crisis_room", {"observed_at": "2021-bad-format"}), client_member.user) request.resolver_match = resolve(reverse("crisis_room")) response = CrisisRoomView.as_view()(request) assert response.status_code == 200 - assertContains(response, datetime.now(timezone.utc).date().strftime("%b %d, %Y")) + assertContains(response, "Can not parse date, falling back to show current date.") + assertContains(response, "Enter a valid date.") def test_org_finding_count_total(): From f21217dc86d07400c6cab317c176a1b8fdeb28ad Mon Sep 17 00:00:00 2001 From: noamblitz <43830693+noamblitz@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:51:08 +0200 Subject: [PATCH 06/29] Use better paginator for finding list (#3407) Co-authored-by: Jan Klopper --- rocky/rocky/templates/findings/finding_list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocky/rocky/templates/findings/finding_list.html b/rocky/rocky/templates/findings/finding_list.html index a23d6eb82b0..2e2025a2f56 100644 --- a/rocky/rocky/templates/findings/finding_list.html +++ b/rocky/rocky/templates/findings/finding_list.html @@ -130,7 +130,7 @@

{% translate "Risk score" %}

{% endif %} - {% include "partials/pagination.html" %} + {% include "partials/list_paginator.html" %}
From 9f2b02673e9cd4478b6ee8e42368fabe152444e2 Mon Sep 17 00:00:00 2001 From: zcrt <115991818+zcrt@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:59:18 +0200 Subject: [PATCH 07/29] Generic tasks view refactor (#3389) Co-authored-by: Jan Klopper Co-authored-by: Peter-Paul van Gemerden Co-authored-by: ammar92 --- rocky/assets/js/renderNormalizerOutputOOIs.js | 7 +- .../partials/single_action_form.html | 4 +- rocky/rocky/locale/django.pot | 21 ++++- rocky/rocky/scheduler.py | 84 ++++++++++++------- rocky/rocky/settings.py | 2 +- rocky/rocky/templates/header.html | 5 ++ rocky/rocky/templates/tasks/boefjes.html | 16 +++- rocky/rocky/templates/tasks/normalizers.html | 21 +++-- .../tasks/partials/tab_navigation.html | 12 ++- .../tasks/partials/task_actions.html | 19 +++-- rocky/rocky/urls.py | 10 ++- rocky/rocky/views/tasks.py | 52 +++++++++++- 12 files changed, 196 insertions(+), 57 deletions(-) diff --git a/rocky/assets/js/renderNormalizerOutputOOIs.js b/rocky/assets/js/renderNormalizerOutputOOIs.js index 1e7f71d37ea..e7d7fdcbefe 100644 --- a/rocky/assets/js/renderNormalizerOutputOOIs.js +++ b/rocky/assets/js/renderNormalizerOutputOOIs.js @@ -11,11 +11,14 @@ buttons.forEach((button) => { .closest("tr") .getAttribute("data-task-id") .replace(/-/g, ""); + const organization = + organization_code || + button.closest("tr").getAttribute("data-organization-code"); const json_url = "/" + language + "/" + - organization_code + + organization + "/tasks/normalizers/" + encodeURI(task_id); @@ -79,7 +82,7 @@ buttons.forEach((button) => { "/" + language + "/" + - escapeHTMLEntities(encodeURIComponent(organization_code)); + escapeHTMLEntities(encodeURIComponent(organization)); let object_list = ""; // set the observed at time a fews seconds into the future, as the job finish time is not the same as the ooi-creation time. Due to async reasons the object might be a bit slow. data["timestamp"] = Date.parse(data["valid_time"] + "Z"); diff --git a/rocky/katalogus/templates/partials/single_action_form.html b/rocky/katalogus/templates/partials/single_action_form.html index 7aada262413..cdcd087a872 100644 --- a/rocky/katalogus/templates/partials/single_action_form.html +++ b/rocky/katalogus/templates/partials/single_action_form.html @@ -1,4 +1,6 @@ -
+ {% csrf_token %} {% if key %}{% endif %} diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 3bf00674a16..6d1c6ed2ae2 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -4456,10 +4456,6 @@ msgstr "" msgid "Task list: " msgstr "" -#: rocky/scheduler.py -msgid "Task statistics: " -msgstr "" - #: rocky/settings.py msgid "Blue light" msgstr "" @@ -6088,12 +6084,20 @@ msgstr "" msgid "List of tasks for boefjes" msgstr "" +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +msgid "Organization Code" +msgstr "" + #: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html #: rocky/templates/tasks/ooi_detail_task_list.html #: rocky/templates/tasks/plugin_detail_task_list.html msgid "Created date" msgstr "" +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +msgid "Modified date" +msgstr "" + #: rocky/templates/tasks/normalizers.html msgid "There are no tasks for normalizers" msgstr "" @@ -6704,6 +6708,15 @@ msgid "" "refresh of the page may be needed to show the results." msgstr "" +#: rocky/views/tasks.py +#, python-brace-format +msgid "Fetching tasks failed: no connection with scheduler: {error}" +msgstr "" + +#: rocky/views/tasks.py +msgid "All Tasks" +msgstr "" + #: rocky/views/upload_csv.py msgid "" "For URL object type, a column 'raw' with URL values is required, starting " diff --git a/rocky/rocky/scheduler.py b/rocky/rocky/scheduler.py index 416aebbc648..f6c11d1cea0 100644 --- a/rocky/rocky/scheduler.py +++ b/rocky/rocky/scheduler.py @@ -1,7 +1,8 @@ from __future__ import annotations +import collections import datetime -import json +import logging import uuid from enum import Enum from functools import cached_property @@ -162,6 +163,7 @@ def __getitem__(self, key) -> list[Task]: else: raise TypeError("Invalid slice argument type.") + logging.info("Getting max %s lazy items at offset %s with filter %s", limit, offset, self.kwargs) res = self.scheduler_client.list_tasks( limit=limit, offset=offset, @@ -214,7 +216,7 @@ class SchedulerHTTPError(SchedulerError): class SchedulerClient: - def __init__(self, base_uri: str, organization_code: str): + def __init__(self, base_uri: str, organization_code: str | None): self._client = httpx.Client(base_url=base_uri) self.organization_code = organization_code @@ -223,8 +225,10 @@ def list_tasks( **kwargs, ) -> PaginatedTasksResponse: try: - kwargs = {k: v for k, v in kwargs.items() if v is not None} # filter Nones from kwargs - res = self._client.get("/tasks", params=kwargs) + filter_key = "filters" + params = {k: v for k, v in kwargs.items() if v is not None if k != filter_key} # filter Nones from kwargs + endpoint = "/tasks" + res = self._client.post(endpoint, params=params, json=kwargs.get(filter_key, None)) return PaginatedTasksResponse.model_validate_json(res.content) except ValidationError: raise SchedulerValidationError(extra_message=_("Task list: ")) @@ -232,22 +236,19 @@ def list_tasks( raise SchedulerConnectError(extra_message=_("Task list: ")) def get_task_details(self, task_id: str) -> Task: - try: - res = self._client.get(f"/tasks/{task_id}") - res.raise_for_status() - task_details = Task.model_validate_json(res.content) + if self.organization_code is None: + raise SchedulerTaskNotFound("No organization defined in client.") + task_details = Task.model_validate_json(self._get(f"/tasks/{task_id}", "content")) - if task_details.type == "normalizer": - organization = task_details.data.raw_data.boefje_meta.organization - else: - organization = task_details.data.organization + if task_details.type == "normalizer": + organization = task_details.data.raw_data.boefje_meta.organization + else: + organization = task_details.data.organization - if organization != self.organization_code: - raise SchedulerTaskNotFound() + if organization != self.organization_code: + raise SchedulerTaskNotFound() - return task_details - except ConnectError: - raise SchedulerConnectError() + return task_details def push_task(self, item: Task) -> None: try: @@ -270,24 +271,45 @@ def push_task(self, item: Task) -> None: raise SchedulerError() def health(self) -> ServiceHealth: - try: - health_endpoint = self._client.get("/health") - health_endpoint.raise_for_status() - return ServiceHealth.model_validate_json(health_endpoint.content) - except HTTPError: - raise SchedulerHTTPError() - except ConnectError: - raise SchedulerConnectError() + return ServiceHealth.model_validate_json(self._get("/health", return_type="content")) + + def _get_task_stats(self, scheduler_id: str) -> dict: + """Return task stats for specific scheduler.""" + return self._get(f"/tasks/stats/{scheduler_id}") # type: ignore def get_task_stats(self, task_type: str) -> dict: + """Return task stats for specific task type.""" + return self._get_task_stats(scheduler_id=f"{task_type}-{self.organization_code}") + + @staticmethod + def _merge_stat_dicts(dicts: list[dict]) -> dict: + """Merge multiple stats dicts.""" + stat_sum: dict[str, collections.Counter] = collections.defaultdict(collections.Counter) + for dct in dicts: + for timeslot, counts in dct.items(): + stat_sum[timeslot].update(counts) + return dict(stat_sum) + + def get_combined_schedulers_stats(self, scheduler_ids: list) -> dict: + """Return merged stats for a set of scheduler ids.""" + return SchedulerClient._merge_stat_dicts( + dicts=[self._get_task_stats(scheduler_id=scheduler_id) for scheduler_id in scheduler_ids] + ) + + def _get(self, path: str, return_type: str = "json") -> dict | bytes: + """Helper to do a get request and raise warning for path.""" try: - res = self._client.get(f"/tasks/stats/{task_type}-{self.organization_code}") + res = self._client.get(path) res.raise_for_status() - except ConnectError: - raise SchedulerConnectError(extra_message=_("Task statistics: ")) - task_stats = json.loads(res.content) - return task_stats + except HTTPError as exc: + raise SchedulerError(path) from exc + except ConnectError as exc: + raise SchedulerConnectError(path) from exc + + if return_type == "content": + return res.content + return res.json() -def scheduler_client(organization_code: str) -> SchedulerClient: +def scheduler_client(organization_code: str | None) -> SchedulerClient: return SchedulerClient(settings.SCHEDULER_API, organization_code) diff --git a/rocky/rocky/settings.py b/rocky/rocky/settings.py index 497bb9ee2c1..6040c32ca5c 100644 --- a/rocky/rocky/settings.py +++ b/rocky/rocky/settings.py @@ -78,7 +78,7 @@ "loggers": { "root": { "handlers": ["console"], - "level": "INFO", + "level": env("LOG_LEVEL", default="INFO").upper(), }, }, } diff --git a/rocky/rocky/templates/header.html b/rocky/rocky/templates/header.html index f96979c6c04..ca685e775f4 100644 --- a/rocky/rocky/templates/header.html +++ b/rocky/rocky/templates/header.html @@ -32,6 +32,11 @@ {% translate "Crisis room" %} +
  • + {% url "all_task_list" as index_url %} + {% translate "Tasks" %} +
    • {% else %}
  • {% url "organization_crisis_room" organization.code as index_url %} diff --git a/rocky/rocky/templates/tasks/boefjes.html b/rocky/rocky/templates/tasks/boefjes.html index b9a1920a992..e786c0acba9 100644 --- a/rocky/rocky/templates/tasks/boefjes.html +++ b/rocky/rocky/templates/tasks/boefjes.html @@ -25,9 +25,13 @@

    {% translate "Boefjes" %}

    + {% if not organization.code %} + + {% endif %} + @@ -35,15 +39,21 @@

    {% translate "Boefjes" %}

    {% for task in task_list %} + {% if not organization.code %} + + {% endif %} + - diff --git a/rocky/rocky/templates/tasks/normalizers.html b/rocky/rocky/templates/tasks/normalizers.html index 9062d970129..096327e328c 100644 --- a/rocky/rocky/templates/tasks/normalizers.html +++ b/rocky/rocky/templates/tasks/normalizers.html @@ -26,9 +26,13 @@

    {% translate "Normalizers" %}

    {% translate "Organization Code" %}{% translate "Boefje" %} {% translate "Status" %} {% translate "Created date" %}{% translate "Modified date" %} {% translate "Input Object" %}
    + {{ task.data.organization }} + - {{ task.data.boefje.name }} + {{ task.data.boefje.name }}  {{ task.status.value|capfirst }} {{ task.created_at }}{{ task.modified_at }} - {{ task.data.input_ooi }} + {{ task.data.input_ooi }}
    + {% include "tasks/partials/task_actions.html" %}
    + {% if not organization.code %} + + {% endif %} + @@ -36,19 +40,26 @@

    {% translate "Normalizers" %}

    {% for task in task_list %} - + + {% if not organization %} + + {% endif %} + - diff --git a/rocky/rocky/templates/tasks/partials/tab_navigation.html b/rocky/rocky/templates/tasks/partials/tab_navigation.html index dec34aab028..16754c651a4 100644 --- a/rocky/rocky/templates/tasks/partials/tab_navigation.html +++ b/rocky/rocky/templates/tasks/partials/tab_navigation.html @@ -3,10 +3,18 @@ diff --git a/rocky/rocky/templates/tasks/partials/task_actions.html b/rocky/rocky/templates/tasks/partials/task_actions.html index 1a894e2b364..017cf65b7fc 100644 --- a/rocky/rocky/templates/tasks/partials/task_actions.html +++ b/rocky/rocky/templates/tasks/partials/task_actions.html @@ -13,18 +13,25 @@

    {% translate "Yielded objects" %}

    {% if task.status.value in "completed,failed" %} {% if task.type == "normalizer" %} {% translate "Download meta and raw data" %} - {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} + href="{% url 'bytes_raw' organization_code=task.data.raw_data.boefje_meta.organization boefje_meta_id=task.data.raw_data.boefje_meta.id %}">{% translate "Download meta and raw data" %} + {% url 'task_list' organization_code=task.data.raw_data.boefje_meta.organization as taskurl %} + {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" url=taskurl value=task.id %} {% elif task.type == "boefje" %} {% translate "Download meta and raw data" %} - {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} + href="{% url 'bytes_raw' organization_code=task.data.organization boefje_meta_id=task.id %}">{% translate "Download meta and raw data" %} + {% url 'task_list' organization_code=task.data.organization as taskurl %} + {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" url=taskurl value=task.id %} {% endif %} {% else %} - {% translate "Download task data" %} + {% if task.type == "normalizer" %} + {% translate "Download task data" %} + {% elif task.type == "boefje" %} + {% translate "Download task data" %} + {% endif %} {% endif %} diff --git a/rocky/rocky/urls.py b/rocky/rocky/urls.py index bb1157c8409..1397c325969 100644 --- a/rocky/rocky/urls.py +++ b/rocky/rocky/urls.py @@ -40,7 +40,12 @@ from rocky.views.scan_profile import ScanProfileDetailView, ScanProfileResetView from rocky.views.scans import ScanListView from rocky.views.task_detail import BoefjeTaskDetailView, DownloadTaskDetail, NormalizerTaskJSONView -from rocky.views.tasks import BoefjesTaskListView, NormalizersTaskListView +from rocky.views.tasks import ( + AllBoefjesTaskListView, + AllNormalizersTaskListView, + BoefjesTaskListView, + NormalizersTaskListView, +) from rocky.views.upload_csv import UploadCSV from rocky.views.upload_raw import UploadRaw @@ -72,6 +77,9 @@ PrivacyStatementView.as_view(), name="privacy_statement", ), + path("tasks/", AllBoefjesTaskListView.as_view(), name="all_task_list"), + path("tasks/boefjes", AllBoefjesTaskListView.as_view(), name="all_boefjes_task_list"), + path("tasks/normalizers", AllNormalizersTaskListView.as_view(), name="all_normalizers_task_list"), path( "/settings/indemnifications/", IndemnificationAddView.as_view(), diff --git a/rocky/rocky/views/tasks.py b/rocky/rocky/views/tasks.py index 698b1e15d7e..3f6b9acfa3c 100644 --- a/rocky/rocky/views/tasks.py +++ b/rocky/rocky/views/tasks.py @@ -5,9 +5,10 @@ from django.utils.translation import gettext_lazy as _ from django.views.generic.list import ListView from httpx import HTTPError +from tools.forms.scheduler import TaskFilterForm from rocky.paginator import RockyPaginator -from rocky.scheduler import SchedulerError +from rocky.scheduler import LazyTaskList, SchedulerError, scheduler_client from rocky.views.page_actions import PageActionsView from rocky.views.scheduler import SchedulerView @@ -63,3 +64,52 @@ class BoefjesTaskListView(TaskListView): class NormalizersTaskListView(TaskListView): template_name = "tasks/normalizers.html" task_type = "normalizer" + + +class AllTaskListView(SchedulerListView, PageActionsView): + paginator_class = RockyPaginator + paginate_by = 20 + context_object_name = "task_list" + client = scheduler_client(None) + task_filter_form = TaskFilterForm + + def get_queryset(self): + task_type = self.request.GET.get("type", self.task_type) + self.schedulers = [f"{task_type}-{o.code}" for o in self.request.user.organizations] + form_data = self.task_filter_form(self.request.GET).data.dict() + kwargs = {k: v for k, v in form_data.items() if v} + + try: + return LazyTaskList( + self.client, + task_type=task_type, + filters={"filters": [{"column": "scheduler_id", "operator": "in", "value": self.schedulers}]}, + **kwargs, + ) + + except HTTPError as error: + error_message = _(f"Fetching tasks failed: no connection with scheduler: {error}") + messages.add_message(self.request, messages.ERROR, error_message) + return [] + except SchedulerError as error: + messages.add_message(self.request, messages.ERROR, str(error)) + return [] + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["task_filter_form"] = self.task_filter_form(self.request.GET) + context["stats"] = self.client.get_combined_schedulers_stats(scheduler_ids=self.schedulers) + context["breadcrumbs"] = [ + {"url": reverse("all_task_list", kwargs={}), "text": _("All Tasks")}, + ] + return context + + +class AllBoefjesTaskListView(AllTaskListView): + template_name = "tasks/boefjes.html" + task_type = "boefje" + + +class AllNormalizersTaskListView(AllTaskListView): + template_name = "tasks/normalizers.html" + task_type = "normalizer" From 7a51134590709b645010a426e12f72e548e3691d Mon Sep 17 00:00:00 2001 From: zcrt <115991818+zcrt@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:04:40 +0200 Subject: [PATCH 08/29] feat: :memo: improve pagination (#3393) Co-authored-by: Jan Klopper --- rocky/katalogus/views/plugin_settings_list.py | 2 +- rocky/reports/views/report_overview.py | 4 ++-- rocky/rocky/views/finding_list.py | 2 +- rocky/rocky/views/tasks.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rocky/katalogus/views/plugin_settings_list.py b/rocky/katalogus/views/plugin_settings_list.py index 370e93a0bfc..f27b16f1d43 100644 --- a/rocky/katalogus/views/plugin_settings_list.py +++ b/rocky/katalogus/views/plugin_settings_list.py @@ -14,7 +14,7 @@ class PluginSettingsListView(SinglePluginView): """ paginator_class = RockyPaginator - paginate_by = 10 + paginate_by = 150 context_object_name = "plugin_settings" def get_plugin_settings(self) -> list[dict[str, Any]]: diff --git a/rocky/reports/views/report_overview.py b/rocky/reports/views/report_overview.py index b5f0f5cdca4..d181f3e38ff 100644 --- a/rocky/reports/views/report_overview.py +++ b/rocky/reports/views/report_overview.py @@ -30,7 +30,7 @@ class ReportHistoryView(BreadcrumbsReportOverviewView, OctopoesView, ListView): Shows all the reports that have ever been generated for the organization. """ - paginate_by = 20 + paginate_by = 30 breadcrumbs_step = 2 context_object_name = "reports" paginator = RockyPaginator @@ -53,7 +53,7 @@ class SubreportView(BreadcrumbsReportOverviewView, OctopoesView, ListView): Shows all the subreports that belong to the selected parent report. """ - paginate_by = 20 + paginate_by = 150 breadcrumbs_step = 3 context_object_name = "subreports" paginator = RockyPaginator diff --git a/rocky/rocky/views/finding_list.py b/rocky/rocky/views/finding_list.py index 9f4e9ea2dc6..480c97edba4 100644 --- a/rocky/rocky/views/finding_list.py +++ b/rocky/rocky/views/finding_list.py @@ -89,7 +89,7 @@ def get_context_data(self, **kwargs): class FindingListView(BreadcrumbsMixin, FindingListFilter): template_name = "findings/finding_list.html" - paginate_by = 20 + paginate_by = 150 def build_breadcrumbs(self): return [ diff --git a/rocky/rocky/views/tasks.py b/rocky/rocky/views/tasks.py index 3f6b9acfa3c..9194907a1a3 100644 --- a/rocky/rocky/views/tasks.py +++ b/rocky/rocky/views/tasks.py @@ -24,7 +24,7 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]: class TaskListView(SchedulerView, SchedulerListView, PageActionsView): paginator_class = RockyPaginator - paginate_by = 20 + paginate_by = 150 context_object_name = "task_list" def get_queryset(self): From 876dded8a8ac07fcd7f0be5875fad179af3ba58e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:28:32 +0200 Subject: [PATCH 09/29] Bump myst-parser from 3.0.1 to 4.0.0 (#3346) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ammar Co-authored-by: Jan Klopper --- poetry.lock | 16 ++++++++-------- pyproject.toml | 2 +- requirements.txt | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index 761ba6de69e..1733c92d963 100644 --- a/poetry.lock +++ b/poetry.lock @@ -369,22 +369,22 @@ files = [ [[package]] name = "myst-parser" -version = "3.0.1" +version = "4.0.0" description = "An extended [CommonMark](https://spec.commonmark.org/) compliant parser," optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" files = [ - {file = "myst_parser-3.0.1-py3-none-any.whl", hash = "sha256:6457aaa33a5d474aca678b8ead9b3dc298e89c68e67012e73146ea6fd54babf1"}, - {file = "myst_parser-3.0.1.tar.gz", hash = "sha256:88f0cb406cb363b077d176b51c476f62d60604d68a8dcdf4832e080441301a87"}, + {file = "myst_parser-4.0.0-py3-none-any.whl", hash = "sha256:b9317997552424448c6096c2558872fdb6f81d3ecb3a40ce84a7518798f3f28d"}, + {file = "myst_parser-4.0.0.tar.gz", hash = "sha256:851c9dfb44e36e56d15d05e72f02b80da21a9e0d07cba96baf5e2d476bb91531"}, ] [package.dependencies] -docutils = ">=0.18,<0.22" +docutils = ">=0.19,<0.22" jinja2 = "*" markdown-it-py = ">=3.0,<4.0" -mdit-py-plugins = ">=0.4,<1.0" +mdit-py-plugins = ">=0.4.1,<1.0" pyyaml = "*" -sphinx = ">=6,<8" +sphinx = ">=7,<9" [package.extras] code-style = ["pre-commit (>=3.0,<4.0)"] @@ -950,4 +950,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "71b725c644473a0939fa7fe7025afa25788c8926bf028a58207622d2af7677de" +content-hash = "0f799bca25a57baa2c08a56766cdd6542aadf19c96b32b4c54a05cbf8eb53538" diff --git a/pyproject.toml b/pyproject.toml index ab4e6fd3a40..9ded858460d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,7 +103,7 @@ python = "^3.10" sphinx = "^7.4.7" sphinx_rtd_theme = "2.0.0" sphinxcontrib-mermaid = "^0.9.2" -myst-parser = "^3.0.1" +myst-parser = "^4.0.0" settings-doc = "^4.0.1" colorama = "0.4.6" # Required on all platforms, not just win32 autodoc-pydantic = "^2.2.0" diff --git a/requirements.txt b/requirements.txt index 3c89cdbaeae..c8cfe26ac7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -191,9 +191,9 @@ mdit-py-plugins==0.4.1 ; python_version >= "3.10" and python_version < "4.0" \ mdurl==0.1.2 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba -myst-parser==3.0.1 ; python_version >= "3.10" and python_version < "4.0" \ - --hash=sha256:6457aaa33a5d474aca678b8ead9b3dc298e89c68e67012e73146ea6fd54babf1 \ - --hash=sha256:88f0cb406cb363b077d176b51c476f62d60604d68a8dcdf4832e080441301a87 +myst-parser==4.0.0 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:851c9dfb44e36e56d15d05e72f02b80da21a9e0d07cba96baf5e2d476bb91531 \ + --hash=sha256:b9317997552424448c6096c2558872fdb6f81d3ecb3a40ce84a7518798f3f28d packaging==24.1 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 From c0aef69c3972cd5ceee130d827da644e635338b6 Mon Sep 17 00:00:00 2001 From: HeleenSG Date: Wed, 28 Aug 2024 18:24:18 +0200 Subject: [PATCH 10/29] Feat: Lazy loading on plugin images (#3414) Co-authored-by: Jan Klopper --- rocky/katalogus/templates/boefje_detail.html | 3 ++- rocky/katalogus/templates/normalizer_detail.html | 3 ++- rocky/katalogus/templates/partials/boefje_tile.html | 3 ++- rocky/katalogus/templates/partials/plugin_tile.html | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rocky/katalogus/templates/boefje_detail.html b/rocky/katalogus/templates/boefje_detail.html index a6229151d6f..316262fd2bb 100644 --- a/rocky/katalogus/templates/boefje_detail.html +++ b/rocky/katalogus/templates/boefje_detail.html @@ -30,7 +30,8 @@

    {{ plugin.name }}

    - boefje placeholder image
    diff --git a/rocky/katalogus/templates/normalizer_detail.html b/rocky/katalogus/templates/normalizer_detail.html index dbb3d3cbc37..55d115d8db2 100644 --- a/rocky/katalogus/templates/normalizer_detail.html +++ b/rocky/katalogus/templates/normalizer_detail.html @@ -24,7 +24,8 @@

    {{ plugin.name }}

    - boefje placeholder image
    diff --git a/rocky/katalogus/templates/partials/boefje_tile.html b/rocky/katalogus/templates/partials/boefje_tile.html index 2eedda760df..08ad467167a 100644 --- a/rocky/katalogus/templates/partials/boefje_tile.html +++ b/rocky/katalogus/templates/partials/boefje_tile.html @@ -12,7 +12,8 @@ {% endif %} {% if widget %}{{ item|get_type_name }}{% endif %} - placeholder image for Boefje {{ item.name }} {% with scan_level=item.scan_level.value %}
    diff --git a/rocky/katalogus/templates/partials/plugin_tile.html b/rocky/katalogus/templates/partials/plugin_tile.html index 543b9980421..5110f091bbf 100644 --- a/rocky/katalogus/templates/partials/plugin_tile.html +++ b/rocky/katalogus/templates/partials/plugin_tile.html @@ -13,7 +13,8 @@ value="{{ plugin.id }}" {% if checked %}checked{% endif %} /> {% endif %} - boefje placeholder image

    {{ plugin.name }}{{ plugin.type|title }} From 28d8abd1c4c433ed1241bfa3318a06fdbe986b19 Mon Sep 17 00:00:00 2001 From: Jeroen Dekkers Date: Thu, 29 Aug 2024 13:45:50 +0200 Subject: [PATCH 11/29] Bump django-rest-framework jquery version (#3422) --- rocky/rocky/templates/rest_framework/api.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocky/rocky/templates/rest_framework/api.html b/rocky/rocky/templates/rest_framework/api.html index 47f01103893..e88d7e1043c 100644 --- a/rocky/rocky/templates/rest_framework/api.html +++ b/rocky/rocky/templates/rest_framework/api.html @@ -12,7 +12,7 @@ "csrfToken": "{% if request %}{{ csrf_token }}{% endif %}" } - + From 5dfcfa6a413202921f154c77b156cb4ab479830a Mon Sep 17 00:00:00 2001 From: Madelon Dohmen <99282220+madelondohmen@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:52:35 +0200 Subject: [PATCH 12/29] Fix KAT-alogus navigation (#3415) Co-authored-by: Jan Klopper --- .../templates/partials/plugins_navigation.html | 2 +- rocky/katalogus/urls.py | 12 +++++++++--- rocky/katalogus/views/katalogus.py | 16 ++++++++++++++++ rocky/tests/katalogus/test_katalogus.py | 2 +- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/rocky/katalogus/templates/partials/plugins_navigation.html b/rocky/katalogus/templates/partials/plugins_navigation.html index bd35aeb0adb..d476968331a 100644 --- a/rocky/katalogus/templates/partials/plugins_navigation.html +++ b/rocky/katalogus/templates/partials/plugins_navigation.html @@ -10,7 +10,7 @@ {% translate "Normalizers" %}

  • - {% translate "All" %} + {% translate "All" %}
  • {% translate "About plugins" %} diff --git a/rocky/katalogus/urls.py b/rocky/katalogus/urls.py index 95c5ff69cc7..7a254ce01da 100644 --- a/rocky/katalogus/urls.py +++ b/rocky/katalogus/urls.py @@ -1,7 +1,13 @@ from django.urls import path from katalogus.views.change_clearance_level import ChangeClearanceLevel -from katalogus.views.katalogus import AboutPluginsView, BoefjeListView, KATalogusView, NormalizerListView +from katalogus.views.katalogus import ( + AboutPluginsView, + BoefjeListView, + KATalogusLandingView, + KATalogusView, + NormalizerListView, +) from katalogus.views.katalogus_settings import ConfirmCloneSettingsView, KATalogusSettingsView from katalogus.views.plugin_detail import BoefjeDetailView, NormalizerDetailView, PluginCoverImgView from katalogus.views.plugin_enable_disable import PluginEnableDisableView @@ -9,8 +15,7 @@ from katalogus.views.plugin_settings_delete import PluginSettingsDeleteView urlpatterns = [ - path("", BoefjeListView.as_view(), name="katalogus"), - path("view//", KATalogusView.as_view(), name="katalogus"), + path("", KATalogusLandingView.as_view(), name="katalogus"), path( "settings/", KATalogusSettingsView.as_view(), @@ -36,6 +41,7 @@ NormalizerListView.as_view(), name="normalizers_list", ), + path("plugins/all//", KATalogusView.as_view(), name="all_plugins_list"), path( "plugins/about-plugins/", AboutPluginsView.as_view(), diff --git a/rocky/katalogus/views/katalogus.py b/rocky/katalogus/views/katalogus.py index f40a3e9e7aa..3b6d97f5f37 100644 --- a/rocky/katalogus/views/katalogus.py +++ b/rocky/katalogus/views/katalogus.py @@ -1,6 +1,8 @@ from typing import Any from account.mixins import OrganizationView +from django.http import HttpRequest, HttpResponse +from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext_lazy as _ from django.views.generic import FormView, ListView, TemplateView @@ -9,6 +11,20 @@ from katalogus.forms import KATalogusFilter +class KATalogusLandingView(OrganizationView): + """ + Landing page for KAT-alogus. + """ + + def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: + return redirect( + reverse( + "boefjes_list", + kwargs={"organization_code": self.organization.code, "view_type": self.kwargs.get("view_type", "grid")}, + ) + ) + + class BaseKATalogusView(OrganizationView, ListView, FormView): form_class = KATalogusFilter diff --git a/rocky/tests/katalogus/test_katalogus.py b/rocky/tests/katalogus/test_katalogus.py index 53cf9e612e7..c881109e143 100644 --- a/rocky/tests/katalogus/test_katalogus.py +++ b/rocky/tests/katalogus/test_katalogus.py @@ -118,7 +118,7 @@ def test_katalogus_plugin_listing_no_enable_disable_perm(rf, client_member, mock mock_requests.Client().get.return_value = mock_response mock_response.json.return_value = get_plugins_data() - request = rf.get("/en/test/kat-alogus/") + request = rf.get("/en/test/kat-alogus/plugins/all/grid/") request.resolver_match = resolve(request.path) response = KATalogusView.as_view()( setup_request(request, client_member.user), organization_code=client_member.organization.code From b0d62b91b7d93dc01d0d47f74ca55d03faf5ce22 Mon Sep 17 00:00:00 2001 From: Jeroen Dekkers Date: Thu, 29 Aug 2024 14:01:17 +0200 Subject: [PATCH 13/29] Move variables from utils.js to renderNormalizerOutputOOIs.js (#3412) Co-authored-by: Jan Klopper --- rocky/assets/js/renderNormalizerOutputOOIs.js | 4 +++- rocky/assets/js/utils.js | 5 ----- rocky/rocky/templates/tasks/normalizers.html | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/rocky/assets/js/renderNormalizerOutputOOIs.js b/rocky/assets/js/renderNormalizerOutputOOIs.js index e7d7fdcbefe..3e20cddab3f 100644 --- a/rocky/assets/js/renderNormalizerOutputOOIs.js +++ b/rocky/assets/js/renderNormalizerOutputOOIs.js @@ -1,4 +1,6 @@ -import { language, organization_code } from "./utils.js"; +const htmlElement = document.getElementsByTagName("html")[0]; +const language = htmlElement.getAttribute("lang"); +const organization_code = htmlElement.getAttribute("data-organization-code"); const buttons = document.querySelectorAll( ".expando-button.normalizer-list-table-row", diff --git a/rocky/assets/js/utils.js b/rocky/assets/js/utils.js index 225c265be1c..e69de29bb2d 100644 --- a/rocky/assets/js/utils.js +++ b/rocky/assets/js/utils.js @@ -1,5 +0,0 @@ -const htmlElement = document.getElementsByTagName("html")[0]; -const language = htmlElement.getAttribute("lang"); -const organization_code = htmlElement.getAttribute("data-organization-code"); - -export { language, organization_code }; diff --git a/rocky/rocky/templates/tasks/normalizers.html b/rocky/rocky/templates/tasks/normalizers.html index 096327e328c..340dad142de 100644 --- a/rocky/rocky/templates/tasks/normalizers.html +++ b/rocky/rocky/templates/tasks/normalizers.html @@ -93,6 +93,6 @@

    {% translate "Normalizers" %}

    {% block html_at_end_body %} {{ block.super }} {% compress js %} - + {% endcompress %} {% endblock html_at_end_body %} From e91ef99592e63ffe36d725ff186c7c05ffaa0c66 Mon Sep 17 00:00:00 2001 From: Jeroen Dekkers Date: Fri, 30 Aug 2024 09:44:50 +0200 Subject: [PATCH 14/29] Replace lru_cache with cache (#3413) Co-authored-by: ammar92 Co-authored-by: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> --- bytes/bytes/config.py | 4 ++-- bytes/bytes/database/db.py | 4 ++-- bytes/bytes/rabbitmq.py | 4 ++-- octopoes/bits/definitions.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bytes/bytes/config.py b/bytes/bytes/config.py index 056f6fa3f39..4f9b61f1c97 100644 --- a/bytes/bytes/config.py +++ b/bytes/bytes/config.py @@ -1,6 +1,6 @@ import logging import os -from functools import lru_cache +from functools import cache from pathlib import Path from typing import Any, Literal @@ -155,7 +155,7 @@ def settings_customise_sources( return env_settings, init_settings, file_secret_settings, backwards_compatible_settings -@lru_cache +@cache def get_settings() -> Settings: return Settings() diff --git a/bytes/bytes/database/db.py b/bytes/bytes/database/db.py index 3df22afe107..922ea4ee499 100644 --- a/bytes/bytes/database/db.py +++ b/bytes/bytes/database/db.py @@ -1,4 +1,4 @@ -from functools import lru_cache +from functools import cache import structlog from sqlalchemy import create_engine @@ -10,7 +10,7 @@ SQL_BASE = declarative_base() -@lru_cache(maxsize=1) +@cache def get_engine(db_uri: str, pool_size: int) -> Engine: """Returns database engine according to config settings.""" db_uri_redacted = make_url(name_or_url=str(db_uri)).render_as_string(hide_password=True) diff --git a/bytes/bytes/rabbitmq.py b/bytes/bytes/rabbitmq.py index 75aebc4ab54..4ffa498ddd8 100644 --- a/bytes/bytes/rabbitmq.py +++ b/bytes/bytes/rabbitmq.py @@ -1,4 +1,4 @@ -from functools import lru_cache +from functools import cache import pika import pika.exceptions @@ -61,7 +61,7 @@ def publish(self, event: Event) -> None: pass -@lru_cache(maxsize=1) +@cache def create_event_manager() -> EventManager: settings = get_settings() diff --git a/octopoes/bits/definitions.py b/octopoes/bits/definitions.py index 58e82a19dcb..8c6dac010b9 100644 --- a/octopoes/bits/definitions.py +++ b/octopoes/bits/definitions.py @@ -1,6 +1,6 @@ import importlib import pkgutil -from functools import lru_cache +from functools import cache from logging import getLogger from pathlib import Path from types import ModuleType @@ -29,7 +29,7 @@ class BitDefinition(BaseModel): config_ooi_relation_path: str | None = None -@lru_cache(maxsize=32) +@cache def get_bit_definitions() -> dict[str, BitDefinition]: bit_definitions = {} From c728cac45917ae48d23a9279036266ff20beb365 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Fri, 30 Aug 2024 13:46:49 +0200 Subject: [PATCH 15/29] Kat dns serverversion (#3291) Co-authored-by: ammar92 --- .../plugins/kat_dns_version/__init__.py | 0 .../plugins/kat_dns_version/boefje.json | 9 ++++ .../plugins/kat_dns_version/description.md | 3 ++ .../boefjes/plugins/kat_dns_version/main.py | 42 +++++++++++++++++++ .../plugins/kat_dns_version/normalize.py | 36 ++++++++++++++++ .../plugins/kat_dns_version/normalizer.json | 10 +++++ .../plugins/kat_dns_version/schema.json | 13 ++++++ boefjes/tests/integration/test_api.py | 8 ++-- 8 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 boefjes/boefjes/plugins/kat_dns_version/__init__.py create mode 100644 boefjes/boefjes/plugins/kat_dns_version/boefje.json create mode 100644 boefjes/boefjes/plugins/kat_dns_version/description.md create mode 100644 boefjes/boefjes/plugins/kat_dns_version/main.py create mode 100644 boefjes/boefjes/plugins/kat_dns_version/normalize.py create mode 100644 boefjes/boefjes/plugins/kat_dns_version/normalizer.json create mode 100644 boefjes/boefjes/plugins/kat_dns_version/schema.json diff --git a/boefjes/boefjes/plugins/kat_dns_version/__init__.py b/boefjes/boefjes/plugins/kat_dns_version/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/boefjes/boefjes/plugins/kat_dns_version/boefje.json b/boefjes/boefjes/plugins/kat_dns_version/boefje.json new file mode 100644 index 00000000000..3aa66ca3cfd --- /dev/null +++ b/boefjes/boefjes/plugins/kat_dns_version/boefje.json @@ -0,0 +1,9 @@ +{ + "id": "dns-bind-version", + "name": "DNS software version", + "description": "Uses the DNS VERSION.BIND command to attempt to learn the servers software.", + "consumes": [ + "IPService" + ], + "scan_level": 2 +} diff --git a/boefjes/boefjes/plugins/kat_dns_version/description.md b/boefjes/boefjes/plugins/kat_dns_version/description.md new file mode 100644 index 00000000000..5ac8b8ea5f2 --- /dev/null +++ b/boefjes/boefjes/plugins/kat_dns_version/description.md @@ -0,0 +1,3 @@ +# Fetch DNS Server software version + +This boefje tries to detect the DNS Server version by doing a VERSION.BIND call. diff --git a/boefjes/boefjes/plugins/kat_dns_version/main.py b/boefjes/boefjes/plugins/kat_dns_version/main.py new file mode 100644 index 00000000000..40631e61f69 --- /dev/null +++ b/boefjes/boefjes/plugins/kat_dns_version/main.py @@ -0,0 +1,42 @@ +"""Boefje script for getting namserver version""" + +import json +from os import getenv + +import dns +import dns.message +import dns.query + +from boefjes.job_models import BoefjeMeta + + +def run(boefje_meta: BoefjeMeta) -> list[tuple[set, str | bytes]]: + input_ = boefje_meta.arguments["input"] # input is IPService + ip_port = input_["ip_port"] + if input_["service"]["name"] != "domain": + return [({"boefje/error"}, "Not a DNS service")] + + ip = ip_port["address"]["address"] + port = int(ip_port["port"]) + protocol = ip_port["protocol"] + + timeout = float(getenv("TIMEOUT", 30)) + + method = dns.query.udp if protocol == "udp" else dns.query.tcp + + queries = [ + dns.message.make_query("VERSION.BIND", dns.rdatatype.TXT, dns.rdataclass.CHAOS), + dns.message.make_query("VERSION.SERVER", dns.rdatatype.TXT, dns.rdataclass.CHAOS), + ] + + results = [] + for query in queries: + response = method(query, where=ip, timeout=timeout, port=port) + + try: + answer = response.answer[0] + results.append(answer.to_rdataset().pop().strings[0].decode()) + except IndexError: + pass + + return [(set(), json.dumps(results))] diff --git a/boefjes/boefjes/plugins/kat_dns_version/normalize.py b/boefjes/boefjes/plugins/kat_dns_version/normalize.py new file mode 100644 index 00000000000..b3e805cc1c5 --- /dev/null +++ b/boefjes/boefjes/plugins/kat_dns_version/normalize.py @@ -0,0 +1,36 @@ +import json +from collections.abc import Iterable + +from boefjes.job_models import NormalizerOutput +from octopoes.models import Reference +from octopoes.models.ooi.software import Software, SoftwareInstance + + +def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: + input_ooi_reference = Reference.from_str(input_ooi["primary_key"]) + + results = json.loads(raw) + for version in results: + if version.startswith("bind"): + name = "bind" + version_number = version.split("-")[1] + elif version.startswith("9."): + name = "bind" + version_number = version + elif version.startswith("Microsoft DNS"): + name = "Microsoft DNS" + version_number = version.replace("Microsoft DNS ", "").split(" ")[0] + elif version.startswith("dnsmasq"): + name = "dnsmasq" + version_number = version.split("-")[1] + elif version.startswith("PowerDNS"): + name = "PowerDNS" + version_number = version.replace("PowerDNS Authoritative Server ", "").split(" ")[0] + else: + name = None + version_number = None + + if name and version_number: + software = Software(name=name, version=version_number) + software_instance = SoftwareInstance(ooi=input_ooi_reference, software=software.reference) + yield from [software, software_instance] diff --git a/boefjes/boefjes/plugins/kat_dns_version/normalizer.json b/boefjes/boefjes/plugins/kat_dns_version/normalizer.json new file mode 100644 index 00000000000..0252c4fe250 --- /dev/null +++ b/boefjes/boefjes/plugins/kat_dns_version/normalizer.json @@ -0,0 +1,10 @@ +{ + "id": "dns-bind-version-normalize", + "consumes": [ + "boefje/dns-bind-version" + ], + "produces": [ + "Software", + "SoftwareInstance" + ] +} diff --git a/boefjes/boefjes/plugins/kat_dns_version/schema.json b/boefjes/boefjes/plugins/kat_dns_version/schema.json new file mode 100644 index 00000000000..6a9fbe29348 --- /dev/null +++ b/boefjes/boefjes/plugins/kat_dns_version/schema.json @@ -0,0 +1,13 @@ +{ + "title": "Arguments", + "type": "object", + "properties": { + "TIMEOUT": { + "title": "TIMEOUT", + "type": "integer", + "description": "Timeout for requests to the targeted dns servers", + "default": 30, + "minimum": 0 + } + } +} diff --git a/boefjes/tests/integration/test_api.py b/boefjes/tests/integration/test_api.py index 9c32af42f8a..ab67e466d98 100644 --- a/boefjes/tests/integration/test_api.py +++ b/boefjes/tests/integration/test_api.py @@ -47,9 +47,9 @@ def test_get_local_plugin(self): def test_filter_plugins(self): response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/") - self.assertEqual(len(response.json()), 97) + self.assertEqual(len(response.json()), 99) response = self.client.get(f"/v1/organisations/{self.org.id}/plugins?plugin_type=boefje") - self.assertEqual(len(response.json()), 43) + self.assertEqual(len(response.json()), 44) response = self.client.get(f"/v1/organisations/{self.org.id}/plugins?limit=10") self.assertEqual(len(response.json()), 10) @@ -74,7 +74,7 @@ def test_add_boefje(self): self.assertEqual(response.status_code, 422) response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/?plugin_type=boefje") - self.assertEqual(len(response.json()), 44) + self.assertEqual(len(response.json()), 45) boefje_dict = boefje.dict() boefje_dict["consumes"] = list(boefje_dict["consumes"]) @@ -99,7 +99,7 @@ def test_add_normalizer(self): self.assertEqual(response.status_code, 201) response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/?plugin_type=normalizer") - self.assertEqual(len(response.json()), 55) + self.assertEqual(len(response.json()), 56) response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/test_normalizer") self.assertEqual(response.json(), normalizer.dict()) From 25e4d5c6aef15ef49eeeac19e0a8c63656d60999 Mon Sep 17 00:00:00 2001 From: Rieven Date: Mon, 2 Sep 2024 16:30:35 +0200 Subject: [PATCH 16/29] Redirect to desired view when all plugins are enabled. (#3380) Co-authored-by: ammar92 Co-authored-by: Jeroen Dekkers --- .../templates/partials/return_button.html | 2 +- rocky/reports/views/aggregate_report.py | 16 ++++++++++++-- rocky/reports/views/generate_report.py | 14 +++++++++++-- rocky/tests/conftest.py | 21 +++++++++++++++++++ .../reports/test_aggregate_report_flow.py | 12 ++++++----- .../reports/test_generate_report_flow.py | 9 +++++--- rocky/tools/view_helpers.py | 5 +++++ 7 files changed, 66 insertions(+), 13 deletions(-) diff --git a/rocky/reports/templates/partials/return_button.html b/rocky/reports/templates/partials/return_button.html index ed038f61bd1..b86d6f6d58b 100644 --- a/rocky/reports/templates/partials/return_button.html +++ b/rocky/reports/templates/partials/return_button.html @@ -7,7 +7,7 @@ {% csrf_token %} {% include "forms/report_form_fields.html" %} - diff --git a/rocky/reports/views/aggregate_report.py b/rocky/reports/views/aggregate_report.py index 623905542ba..51d0f9da8b7 100644 --- a/rocky/reports/views/aggregate_report.py +++ b/rocky/reports/views/aggregate_report.py @@ -7,6 +7,7 @@ from django.urls import reverse from django.utils.http import urlencode from django.utils.translation import gettext_lazy as _ +from tools.view_helpers import PostRedirect from reports.report_types.aggregate_organisation_report.report import AggregateOrganisationReport from reports.report_types.definitions import AggregateReport, MultiReport, Report @@ -118,7 +119,7 @@ def setup(self, request, *args, **kwargs): def post(self, request, *args, **kwargs): if not self.selected_oois: messages.error(request, self.NONE_OOI_SELECTION_MESSAGE) - return redirect(self.get_previous()) + return PostRedirect(self.get_previous()) return self.get(request, *args, **kwargs) def get_report_types_for_aggregate_report( @@ -151,9 +152,17 @@ class SetupScanAggregateReportView( current_step = 3 def post(self, request, *args, **kwargs): + # If the user wants to change selection, but all plugins are enabled, it needs to go even further back if not self.selected_report_types: messages.error(request, self.NONE_REPORT_TYPE_SELECTION_MESSAGE) - return redirect(self.get_previous()) + return PostRedirect(self.get_previous()) + + if "return" in self.request.POST and self.plugins_enabled(): + return PostRedirect(self.get_previous()) + + if self.plugins_enabled(): + return PostRedirect(self.get_next()) + return self.get(request, *args, **kwargs) @@ -167,6 +176,9 @@ class ExportSetupAggregateReportView(AggregateReportStepsMixin, BreadcrumbsAggre current_step = 4 def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: + if not self.selected_report_types: + messages.error(request, self.NONE_REPORT_TYPE_SELECTION_MESSAGE) + return PostRedirect(self.get_previous()) return super().get(request, *args, **kwargs) def get_context_data(self, **kwargs): diff --git a/rocky/reports/views/generate_report.py b/rocky/reports/views/generate_report.py index cfd37c4ad05..a21e2557324 100644 --- a/rocky/reports/views/generate_report.py +++ b/rocky/reports/views/generate_report.py @@ -8,6 +8,7 @@ from django.urls import reverse from django.utils.http import urlencode from django.utils.translation import gettext_lazy as _ +from tools.view_helpers import PostRedirect from octopoes.models import Reference from reports.report_types.helpers import get_ooi_types_with_report, get_report_types_for_oois @@ -108,7 +109,7 @@ class ReportTypesSelectionGenerateReportView( def post(self, request, *args, **kwargs): if not self.selected_oois: messages.error(request, self.NONE_OOI_SELECTION_MESSAGE) - return redirect(self.get_previous()) + return PostRedirect(self.get_previous()) return self.get(request, *args, **kwargs) def get_context_data(self, **kwargs): @@ -135,7 +136,13 @@ class SetupScanGenerateReportView( def post(self, request, *args, **kwargs): if not self.selected_report_types: messages.error(request, self.NONE_REPORT_TYPE_SELECTION_MESSAGE) - return redirect(self.get_previous()) + return PostRedirect(self.get_previous()) + + if "return" in self.request.POST and self.plugins_enabled(): + return PostRedirect(self.get_previous()) + + if self.plugins_enabled(): + return PostRedirect(self.get_next()) return self.get(request, *args, **kwargs) @@ -150,6 +157,9 @@ class ExportSetupGenerateReportView(GenerateReportStepsMixin, BreadcrumbsGenerat reports: dict[str, str] = {} def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: + if not self.selected_report_types: + messages.error(request, self.NONE_REPORT_TYPE_SELECTION_MESSAGE) + return PostRedirect(self.get_previous()) self.reports = create_report_names(self.oois_pk, self.report_types) return super().get(request, *args, **kwargs) diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index a00690c900e..13b5c33b230 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -1785,3 +1785,24 @@ def boefje_dns_records(): runnable_hash=None, produces={"boefje/dns-records"}, ) + + +@pytest.fixture +def boefje_nmap_tcp(): + return Boefje( + id="nmap", + name="Nmap TCP", + version=None, + authors=None, + created=None, + description="Defaults to top 250 TCP ports. Includes service detection.", + environment_keys=None, + related=[], + enabled=True, + type="boefje", + scan_level=SCAN_LEVEL.L2, + consumes={IPAddressV4, IPAddressV6}, + options=None, + runnable_hash=None, + produces={"boefje/nmap"}, + ) diff --git a/rocky/tests/reports/test_aggregate_report_flow.py b/rocky/tests/reports/test_aggregate_report_flow.py index f3c6610cbf8..fca4d053401 100644 --- a/rocky/tests/reports/test_aggregate_report_flow.py +++ b/rocky/tests/reports/test_aggregate_report_flow.py @@ -177,7 +177,7 @@ def test_report_types_selection_nothing_selected( response = SetupScanAggregateReportView.as_view()(request, organization_code=client_member.organization.code) - assert response.status_code == 302 + assert response.status_code == 307 assert list(request._messages)[0].message == "Select at least one report type to proceed." @@ -189,6 +189,7 @@ def test_report_types_selection( listed_hostnames, mocker, boefje_dns_records, + boefje_nmap_tcp, rocky_health, mock_bytes_client, ): @@ -197,7 +198,7 @@ def test_report_types_selection( """ katalogus_mocker = mocker.patch("reports.views.base.get_katalogus")() - katalogus_mocker.get_plugins.return_value = [boefje_dns_records] + katalogus_mocker.get_plugins.return_value = [boefje_dns_records, boefje_nmap_tcp] rocky_health_mocker = mocker.patch("reports.report_types.aggregate_organisation_report.report.get_rocky_health")() rocky_health_mocker.return_value = rocky_health @@ -211,16 +212,17 @@ def test_report_types_selection( request = setup_request( rf.post( "aggregate_report_setup_scan", - {"observed_at": valid_time.strftime("%Y-%m-%d"), "report_type": "dns-report"}, + {"observed_at": valid_time.strftime("%Y-%m-%d"), "report_type": ["dns-report", "systems-report"]}, ), client_member.user, ) response = SetupScanAggregateReportView.as_view()(request, organization_code=client_member.organization.code) - assert response.status_code == 200 # if all plugins are enabled the view will auto redirect to generate report + assert response.status_code == 307 # if all plugins are enabled the view will auto redirect to generate report - assertContains(response, '', html=True) + # Redirect to export setup + assert response.headers["Location"] == "/en/test/reports/aggregate-report/export-setup/?" def test_save_aggregate_report_view( diff --git a/rocky/tests/reports/test_generate_report_flow.py b/rocky/tests/reports/test_generate_report_flow.py index 1271a69bff7..2cae8e51990 100644 --- a/rocky/tests/reports/test_generate_report_flow.py +++ b/rocky/tests/reports/test_generate_report_flow.py @@ -177,7 +177,8 @@ def test_report_types_selection_nothing_selected( response = SetupScanGenerateReportView.as_view()(request, organization_code=client_member.organization.code) - assert response.status_code == 302 + assert response.status_code == 307 + assert list(request._messages)[0].message == "Select at least one report type to proceed." @@ -214,8 +215,10 @@ def test_report_types_selection( response = SetupScanGenerateReportView.as_view()(request, organization_code=client_member.organization.code) - assert response.status_code == 200 - assertContains(response, '', html=True) + assert response.status_code == 307 + + # Redirect to export setup, all plugins are then enabled + assert response.headers["Location"] == "/en/test/reports/generate-report/export-setup/?" def test_save_generate_report_view( diff --git a/rocky/tools/view_helpers.py b/rocky/tools/view_helpers.py index e216d944334..9f7008662ed 100644 --- a/rocky/tools/view_helpers.py +++ b/rocky/tools/view_helpers.py @@ -4,6 +4,7 @@ from urllib.parse import urlencode, urlparse, urlunparse from django.http import HttpRequest +from django.http.response import HttpResponseRedirectBase from django.urls.base import reverse, reverse_lazy from django.utils.translation import gettext_lazy as _ @@ -163,3 +164,7 @@ def build_breadcrumbs(self): "text": _("Objects"), } ] + + +class PostRedirect(HttpResponseRedirectBase): + status_code = 307 From 85d3227ca322d2a06daccb00663f7b383a26fa1d Mon Sep 17 00:00:00 2001 From: Rieven Date: Tue, 3 Sep 2024 09:31:01 +0200 Subject: [PATCH 17/29] Fix findings overview overflow (#3439) --- .../report_severity_totals_table.html | 108 +++++++++--------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/rocky/reports/templates/partials/report_severity_totals_table.html b/rocky/reports/templates/partials/report_severity_totals_table.html index 2803dab932f..0074622bfc1 100644 --- a/rocky/reports/templates/partials/report_severity_totals_table.html +++ b/rocky/reports/templates/partials/report_severity_totals_table.html @@ -3,63 +3,61 @@

    {% translate "Findings overview" %}

    -
    -
    -
    • {% translate "Organization Code" %}{% translate "Normalizer" %} {% translate "Status" %} {% translate "Created date" %}{% translate "Modified date" %} {% translate "Boefje" %} {% translate "Boefje input OOI" %}
    + {{ task.data.raw_data.boefje_meta.organization }} + - {{ task.data.normalizer.id }} + {{ task.data.normalizer.id }}  {{ task.status.value|capfirst }} {{ task.created_at }}{{ task.modified_at }} - {{ task.data.raw_data.boefje_meta.boefje.id }} + {{ task.data.raw_data.boefje_meta.boefje.id }} - {{ task.data.raw_data.boefje_meta.input_ooi }} + {{ task.data.raw_data.boefje_meta.input_ooi }}
    + {% include "tasks/partials/task_actions.html" %}
    - - +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    {% translate "Risk level" %}{% translate "Findings" %}{% translate "Occurrences" %}
    + Critical + {{ data.total_by_severity_per_finding_type.critical }}{{ data.total_by_severity.critical }}
    + High + {{ data.total_by_severity_per_finding_type.high }}{{ data.total_by_severity.high }}
    + Medium + {{ data.total_by_severity_per_finding_type.medium }}{{ data.total_by_severity.medium }}
    + Low + {{ data.total_by_severity_per_finding_type.low }}{{ data.total_by_severity.low }}
    + Recommendation + {{ data.total_by_severity_per_finding_type.recommendation }}{{ data.total_by_severity.recommendation }}
    {% translate "Risk level" %}{% translate "Findings" %}{% translate "Occurrences" %}Total{{ data.total_finding_types }}{{ data.total_occurrences }}
    - Critical - {{ data.total_by_severity_per_finding_type.critical }}{{ data.total_by_severity.critical }}
    - High - {{ data.total_by_severity_per_finding_type.high }}{{ data.total_by_severity.high }}
    - Medium - {{ data.total_by_severity_per_finding_type.medium }}{{ data.total_by_severity.medium }}
    - Low - {{ data.total_by_severity_per_finding_type.low }}{{ data.total_by_severity.low }}
    - Recommendation - {{ data.total_by_severity_per_finding_type.recommendation }}{{ data.total_by_severity.recommendation }}
    Total{{ data.total_finding_types }}{{ data.total_occurrences }}
    -
    • + + +
    From 51897dd1b3c99648c336353af4db5deec47e0c12 Mon Sep 17 00:00:00 2001 From: Jeroen Dekkers Date: Tue, 3 Sep 2024 11:21:23 +0200 Subject: [PATCH 18/29] Add indemnification to API (#3423) Co-authored-by: Jan Klopper --- rocky/tests/conftest.py | 9 +++++ rocky/tests/test_api_organization.py | 53 ++++++++++++++++++++++++---- rocky/tools/viewsets.py | 28 +++++++++++++-- 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index 13b5c33b230..f5bbb28dfac 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -1806,3 +1806,12 @@ def boefje_nmap_tcp(): runnable_hash=None, produces={"boefje/nmap"}, ) + + +@pytest.fixture +def drf_admin_client(create_drf_client, admin_user): + client = create_drf_client(admin_user) + # We need to set this so that the test client doesn't throw an + # exception, but will return error in the API we can test + client.raise_request_exception = False + return client diff --git a/rocky/tests/test_api_organization.py b/rocky/tests/test_api_organization.py index 9dbc2170a58..b3aa1155793 100644 --- a/rocky/tests/test_api_organization.py +++ b/rocky/tests/test_api_organization.py @@ -2,13 +2,16 @@ from unittest.mock import patch import pytest +from django.urls import reverse from httpx import HTTPError from pytest_assert_utils import assert_model_attrs from pytest_common_subject import precondition_fixture from pytest_drf import ( + APIViewTest, Returns200, Returns201, Returns204, + Returns409, Returns500, UsesDeleteMethod, UsesDetailEndpoint, @@ -51,13 +54,7 @@ def organizations(self): list_url = lambda_fixture(lambda: url_for("organization-list")) detail_url = lambda_fixture(lambda organization: url_for("organization-detail", organization.pk)) - @pytest.fixture - def client(self, create_drf_client, admin_user): - client = create_drf_client(admin_user) - # We need to set this so that the test client doesn't throw an - # exception, but will return error in the API we can test - client.raise_request_exception = False - return client + client = lambda_fixture("drf_admin_client") class TestList( UsesGetMethod, @@ -280,3 +277,45 @@ def test_it_returns_error(self, json): ], } assert json == expected + + +class TestGetIndemnification(APIViewTest, UsesGetMethod, Returns200): + # The superuser_member fixture creates the indemnification + url = lambda_fixture( + lambda organization, superuser_member: reverse("organization-indemnification", args=[organization.pk]) + ) + client = lambda_fixture("drf_admin_client") + + def test_it_returns_indemnification(self, json, superuser_member): + expected = {"indemnification": True, "user": superuser_member.user.id} + assert json == expected + + +class TestIndemnificationDoesNotExist(APIViewTest, UsesGetMethod, Returns200): + url = lambda_fixture(lambda organization: reverse("organization-indemnification", args=[organization.pk])) + client = lambda_fixture("drf_admin_client") + + def test_it_returns_no_indemnification(self, json): + expected = {"indemnification": False, "user": None} + assert json == expected + + +class TestSetIndemnification(APIViewTest, UsesPostMethod, Returns201): + url = lambda_fixture(lambda organization: reverse("organization-indemnification", args=[organization.pk])) + client = lambda_fixture("drf_admin_client") + + def test_it_sets_indemnification(self, json, admin_user): + expected = {"indemnification": True, "user": admin_user.id} + assert json == expected + + +class TestIndemnificationAlreadyExists(APIViewTest, UsesPostMethod, Returns409): + # The superuser_member fixture creates the indemnification + url = lambda_fixture( + lambda organization, superuser_member: reverse("organization-indemnification", args=[organization.pk]) + ) + client = lambda_fixture("drf_admin_client") + + def test_it_returns_indemnification(self, json, superuser_member): + expected = {"indemnification": True, "user": superuser_member.user.id} + assert json == expected diff --git a/rocky/tools/viewsets.py b/rocky/tools/viewsets.py index 9f3683f7303..52e5d492252 100644 --- a/rocky/tools/viewsets.py +++ b/rocky/tools/viewsets.py @@ -1,6 +1,8 @@ -from rest_framework import viewsets +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response -from tools.models import Organization +from tools.models import Indemnification, Organization from tools.serializers import OrganizationSerializer, OrganizationSerializerReadOnlyCode @@ -16,3 +18,25 @@ def get_serializer_class(self): if self.request.method != "POST": serializer_class = OrganizationSerializerReadOnlyCode return serializer_class + + @action(detail=True) + def indemnification(self, request, pk=None): + organization = self.get_object() + indemnification = Indemnification.objects.filter(organization=organization).first() + + if indemnification: + return Response({"indemnification": True, "user": indemnification.user.pk}) + else: + return Response({"indemnification": False, "user": None}) + + @indemnification.mapping.post + def set_indemnification(self, request, pk=None): + organization = self.get_object() + + indemnification = Indemnification.objects.filter(organization=organization).first() + if indemnification: + return Response({"indemnification": True, "user": indemnification.user.pk}, status=status.HTTP_409_CONFLICT) + + indemnification = Indemnification.objects.create(organization=organization, user=self.request.user) + + return Response({"indemnification": True, "user": indemnification.user.pk}, status=status.HTTP_201_CREATED) From 175bdce257f0da7806b98a20007a1b1662af0b8d Mon Sep 17 00:00:00 2001 From: originalsouth Date: Wed, 4 Sep 2024 09:07:35 +0200 Subject: [PATCH 19/29] Stop yielding network in certain normalizers (#3420) Co-authored-by: Jan Klopper Co-authored-by: ammar92 --- boefjes/boefjes/plugins/kat_crt_sh/normalize.py | 1 - boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py | 1 - boefjes/tests/test_nmap.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/boefjes/boefjes/plugins/kat_crt_sh/normalize.py b/boefjes/boefjes/plugins/kat_crt_sh/normalize.py index aba1315df12..3c430005fb9 100644 --- a/boefjes/boefjes/plugins/kat_crt_sh/normalize.py +++ b/boefjes/boefjes/plugins/kat_crt_sh/normalize.py @@ -16,7 +16,6 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: current = fqdn.lstrip(".") network = Network(name="internet") - yield network network_reference = network.reference unique_domains = set() diff --git a/boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py b/boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py index f4177a32b06..9a60c70e46f 100644 --- a/boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py +++ b/boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py @@ -53,7 +53,6 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: # Relevant network object is received from the normalizer_meta. network = Network(name=input_ooi["network"]["name"]) - yield network netblock_ref = None if "NetBlock" in input_ooi["object_type"]: diff --git a/boefjes/tests/test_nmap.py b/boefjes/tests/test_nmap.py index 8a5b9e29b9e..9d0de572fdf 100644 --- a/boefjes/tests/test_nmap.py +++ b/boefjes/tests/test_nmap.py @@ -9,7 +9,7 @@ class NmapTest(TestCase): def test_normalizer(self): input_ooi = IPAddressV4(network=Network(name="internet").reference, address="134.209.85.72") output = list(run(input_ooi.serialize(), get_dummy_data("raw/nmap_mispoes.xml"))) - self.assertEqual(16, len(output)) + self.assertEqual(15, len(output)) for i, out in enumerate(output[:-1]): if out.object_type == "IPPort" and output[i + 1].object_type == "Service": if out.port == 80: From a28863301cf5cbd7080caea2807776171a518a06 Mon Sep 17 00:00:00 2001 From: noamblitz <43830693+noamblitz@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:10:29 +0200 Subject: [PATCH 20/29] Feature/finding sorting searching (#3405) --- octopoes/octopoes/api/router.py | 6 +++++ octopoes/octopoes/connector/octopoes.py | 10 +++++++- .../octopoes/repositories/ooi_repository.py | 23 ++++++++++++++++--- rocky/rocky/locale/django.pot | 14 +++++++---- .../templates/findings/findings_filter.html | 1 + rocky/rocky/views/finding_list.py | 6 ++++- rocky/rocky/views/mixins.py | 10 ++++++++ rocky/tools/forms/findings.py | 6 +++++ 8 files changed, 66 insertions(+), 10 deletions(-) diff --git a/octopoes/octopoes/api/router.py b/octopoes/octopoes/api/router.py index 6b9c7c11ba7..cb907f2c650 100644 --- a/octopoes/octopoes/api/router.py +++ b/octopoes/octopoes/api/router.py @@ -459,6 +459,9 @@ def list_findings( octopoes: OctopoesService = Depends(octopoes_service), valid_time: datetime = Depends(extract_valid_time), severities: set[RiskLevelSeverity] = Query(DEFAULT_SEVERITY_FILTER), + search_string: str | None = None, + order_by: Literal["score", "finding_type"] = "score", + asc_desc: Literal["asc", "desc"] = "desc", ) -> Paginated[Finding]: return octopoes.ooi_repository.list_findings( severities, @@ -467,6 +470,9 @@ def list_findings( only_muted, offset, limit, + search_string, + order_by, + asc_desc, ) diff --git a/octopoes/octopoes/connector/octopoes.py b/octopoes/octopoes/connector/octopoes.py index 9751fc3563c..0b346164d1e 100644 --- a/octopoes/octopoes/connector/octopoes.py +++ b/octopoes/octopoes/connector/octopoes.py @@ -265,15 +265,23 @@ def list_findings( only_muted: bool = False, offset: int = DEFAULT_OFFSET, limit: int = DEFAULT_LIMIT, + search_string: str | None = None, + order_by: Literal["score", "finding_type"] = "score", + asc_desc: Literal["asc", "desc"] = "desc", ) -> Paginated[Finding]: - params: dict[str, str | int | list[str]] = { + params: dict[str, str | int | list[str] | None] = { "valid_time": str(valid_time), "offset": offset, "limit": limit, "severities": [s.value for s in severities], "exclude_muted": exclude_muted, "only_muted": only_muted, + "search_string": search_string, + "order_by": order_by, + "asc_desc": asc_desc, } + + params = {k: v for k, v in params.items() if v is not None} # filter out None values res = self.session.get(f"/{self.client}/findings", params=params) return TypeAdapter(Paginated[Finding]).validate_json(res.content) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index 2638cd7a638..40f05bc4fba 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -146,6 +146,9 @@ def list_findings( only_muted, offset, limit, + search_string, + order_by, + asc_desc, ) -> Paginated[Finding]: raise NotImplementedError @@ -729,6 +732,9 @@ def list_findings( only_muted=False, offset=DEFAULT_OFFSET, limit=DEFAULT_LIMIT, + search_string: str | None = None, + order_by: Literal["score", "finding_type"] = "score", + asc_desc: Literal["asc", "desc"] = "desc", ) -> Paginated[Finding]: # clause to find risk_severity concrete_finding_types = to_concrete({FindingType}) @@ -751,6 +757,15 @@ def list_findings( elif only_muted: muted_clause = "[?muted_finding :MutedFinding/finding ?finding]" + search_statement = ( + f"""[?finding :xt/id ?id] + [(clojure.string/includes? ?id \"{escape_string(search_string)}\")]""" + if search_string + else "" + ) + + order_statement = f":order-by [[?{order_by} :{asc_desc}]]" + severity_values = ", ".join([str_val(severity.value) for severity in severities]) count_query = f""" @@ -760,6 +775,7 @@ def list_findings( :in [[severities_ ...]] :where [[?finding :object_type "Finding"] [?finding :Finding/finding_type ?finding_type] + {search_statement} [(== ?severity severities_)] {or_severities} {muted_clause}] @@ -776,17 +792,18 @@ def list_findings( finding_query = f""" {{ :query {{ - :find [(pull ?finding [*]) ?score] + :find [(pull ?finding [*]) ?score ?finding_type] :in [[severities_ ...]] :where [[?finding :object_type "Finding"] [?finding :Finding/finding_type ?finding_type] [(== ?severity severities_)] {or_severities} {or_scores} - {muted_clause}] + {muted_clause} + {search_statement}] :limit {limit} :offset {offset} - :order-by [[?score :desc]] + {order_statement} }} :in-args [[{severity_values}]] }} diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 6d1c6ed2ae2..597592e2987 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-08-28 08:40+0000\n" +"POT-Creation-Date: 2024-09-04 08:55+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -4151,6 +4151,14 @@ msgstr "" msgid "Filter by muted findings" msgstr "" +#: tools/forms/findings.py tools/forms/ooi_form.py tools/forms/scheduler.py +msgid "Search" +msgstr "" + +#: tools/forms/findings.py +msgid "Object ID contains (case sensitive)" +msgstr "" + #: tools/forms/ooi.py msgid "Filter types" msgstr "" @@ -4189,10 +4197,6 @@ msgstr "" msgid "Filter by clearance type" msgstr "" -#: tools/forms/ooi_form.py tools/forms/scheduler.py -msgid "Search" -msgstr "" - #: tools/forms/scheduler.py msgid "From" msgstr "" diff --git a/rocky/rocky/templates/findings/findings_filter.html b/rocky/rocky/templates/findings/findings_filter.html index 971d3f373ba..7e9d7553713 100644 --- a/rocky/rocky/templates/findings/findings_filter.html +++ b/rocky/rocky/templates/findings/findings_filter.html @@ -16,6 +16,7 @@
    {% include "partials/form/fieldset.html" with fields=severity_filter fieldset_class="filter-fields-direction column" %} {% include "partials/form/fieldset.html" with fields=muted_findings_filter fieldset_class="filter-fields-direction column" %} + {% include "partials/form/fieldset.html" with fields=finding_search_form %}
    diff --git a/rocky/rocky/views/finding_list.py b/rocky/rocky/views/finding_list.py index 480c97edba4..bb7c346b168 100644 --- a/rocky/rocky/views/finding_list.py +++ b/rocky/rocky/views/finding_list.py @@ -7,7 +7,7 @@ from django.utils.translation import gettext_lazy as _ from django.views.generic import ListView from tools.forms.base import ObservedAtForm -from tools.forms.findings import FindingSeverityMultiSelectForm, MutedFindingSelectionForm +from tools.forms.findings import FindingSearchForm, FindingSeverityMultiSelectForm, MutedFindingSelectionForm from tools.view_helpers import BreadcrumbsMixin from octopoes.models.ooi.findings import RiskLevelSeverity @@ -61,6 +61,8 @@ def setup(self, request, *args, **kwargs): self.exclude_muted = self.muted_findings == "non-muted" self.only_muted = self.muted_findings == "muted" + self.search_string = request.GET.get("search", "") + def count_observed_at_filter(self) -> int: return 1 if datetime.now(timezone.utc).date() != self.observed_at.date() else 0 @@ -74,6 +76,7 @@ def get_queryset(self) -> FindingList: severities=self.severities, exclude_muted=self.exclude_muted, only_muted=self.only_muted, + search_string=self.search_string, ) def get_context_data(self, **kwargs): @@ -82,6 +85,7 @@ def get_context_data(self, **kwargs): context["observed_at"] = self.observed_at context["severity_filter"] = FindingSeverityMultiSelectForm({"severity": list(self.severities)}) context["muted_findings_filter"] = MutedFindingSelectionForm({"muted_findings": self.muted_findings}) + context["finding_search_form"] = FindingSearchForm(self.request.GET) context["only_muted"] = self.only_muted context["active_filters_counter"] = self.count_active_filters() return context diff --git a/rocky/rocky/views/mixins.py b/rocky/rocky/views/mixins.py index 6afe8d5f560..d8bc37268ed 100644 --- a/rocky/rocky/views/mixins.py +++ b/rocky/rocky/views/mixins.py @@ -278,6 +278,9 @@ def __init__( severities: set[RiskLevelSeverity], exclude_muted: bool = True, only_muted: bool = False, + search_string: str | None = None, + order_by: Literal["score", "finding_type"] = "score", + asc_desc: Literal["asc", "desc"] = "desc", ): self.octopoes_connector = octopoes_connector self.valid_time = valid_time @@ -286,6 +289,9 @@ def __init__( self.severities = severities self.exclude_muted = exclude_muted self.only_muted = only_muted + self.search_string = search_string + self.order_by = order_by + self.asc_desc = asc_desc @cached_property def count(self) -> int: @@ -295,6 +301,7 @@ def count(self) -> int: exclude_muted=self.exclude_muted, only_muted=self.only_muted, limit=0, + search_string=self.search_string, ).count def __len__(self): @@ -313,6 +320,9 @@ def __getitem__(self, key: int | slice) -> list[HydratedFinding]: only_muted=self.only_muted, offset=offset, limit=limit, + search_string=self.search_string, + order_by=self.order_by, + asc_desc=self.asc_desc, ).items ooi_references = {finding.ooi for finding in findings} finding_type_references = {finding.finding_type for finding in findings} diff --git a/rocky/tools/forms/findings.py b/rocky/tools/forms/findings.py index b19070fcd57..6d65bc8a775 100644 --- a/rocky/tools/forms/findings.py +++ b/rocky/tools/forms/findings.py @@ -32,3 +32,9 @@ class MutedFindingSelectionForm(BaseRockyForm): choices=MUTED_FINDINGS_CHOICES, widget=forms.RadioSelect, ) + + +class FindingSearchForm(BaseRockyForm): + search = forms.CharField( + label=_("Search"), required=False, max_length=256, help_text=_("Object ID contains (case sensitive)") + ) From 03fee11ef2f6a2924e6ed531e131725a4db5b143 Mon Sep 17 00:00:00 2001 From: HeleenSG Date: Wed, 4 Sep 2024 23:01:00 +0200 Subject: [PATCH 21/29] fix: notification width (#3450) Co-authored-by: Jan Klopper --- rocky/assets/css/main.scss | 1 + .../vendor_overrides/manon/notification.scss | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 rocky/assets/css/vendor_overrides/manon/notification.scss diff --git a/rocky/assets/css/main.scss b/rocky/assets/css/main.scss index fefaa3969a6..0e41f9ea21d 100644 --- a/rocky/assets/css/main.scss +++ b/rocky/assets/css/main.scss @@ -18,6 +18,7 @@ @import "vendor_overrides/manon/layout-fifty-fifty"; @import "vendor_overrides/manon/layout-form"; @import "vendor_overrides/manon/nested-section"; +@import "vendor_overrides/manon/notification"; @import "vendor_overrides/manon/table"; @import "vendor_overrides/manon/tile"; @import "vendor_overrides/two-factor"; diff --git a/rocky/assets/css/vendor_overrides/manon/notification.scss b/rocky/assets/css/vendor_overrides/manon/notification.scss new file mode 100644 index 00000000000..03b895c8acf --- /dev/null +++ b/rocky/assets/css/vendor_overrides/manon/notification.scss @@ -0,0 +1,18 @@ +p, +a, +span, +li, +h1, +h2, +h3, +h4, +h5, +h6 { + &.error, + &.warning, + &.explanation, + &.confirmation, + &.system { + max-width: var(--max-line-length-max-width); + } +} From 5b743b67968513ff96896e60c347782a2290e94a Mon Sep 17 00:00:00 2001 From: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:07:36 +0200 Subject: [PATCH 22/29] Update katalogus boefje descriptions (#3444) Co-authored-by: Ammar Co-authored-by: Jan Klopper --- .../plugins/kat_adr_finding_types/boefje.json | 2 +- .../kat_adr_finding_types/normalizer.json | 2 ++ .../plugins/kat_adr_validator/boefje.json | 2 +- .../plugins/kat_adr_validator/normalizer.json | 2 ++ .../plugins/kat_answer_parser/normalizer.json | 2 ++ .../plugins/kat_binaryedge/boefje.json | 2 +- .../kat_binaryedge/containers/normalizer.json | 1 + .../kat_binaryedge/databases/normalizer.json | 1 + .../kat_binaryedge/http_web/normalizer.json | 1 + .../message_queues/normalizer.json | 1 + .../kat_binaryedge/protocols/normalizer.json | 1 + .../remote_desktop/normalizer.json | 1 + .../service_identification/normalizer.json | 1 + .../kat_binaryedge/services/normalizer.json | 1 + .../plugins/kat_burpsuite/normalizer.json | 2 +- .../plugins/kat_calvin/normalizer.json | 2 ++ .../boefjes/plugins/kat_censys/boefje.json | 2 +- .../plugins/kat_censys/normalizer.json | 1 + .../boefjes/plugins/kat_crt_sh/boefje.json | 2 +- .../plugins/kat_crt_sh/normalizer.json | 2 ++ .../plugins/kat_cve_2023_34039/boefje.json | 4 +-- .../plugins/kat_cve_2023_35078/boefje.json | 4 +-- .../kat_cve_2023_35078/normalizer.json | 2 ++ .../plugins/kat_cve_2024_6387/normalizer.json | 3 ++- .../plugins/kat_cve_finding_types/boefje.json | 2 +- .../kat_cve_finding_types/normalizer.json | 2 ++ .../plugins/kat_cwe_finding_types/boefje.json | 2 +- .../kat_cwe_finding_types/normalizer.json | 2 ++ boefjes/boefjes/plugins/kat_dicom/boefje.json | 2 +- .../boefjes/plugins/kat_dicom/normalizer.json | 2 ++ boefjes/boefjes/plugins/kat_dns/boefje.json | 4 +-- .../boefjes/plugins/kat_dns/normalizer.json | 2 ++ .../boefjes/plugins/kat_dns_zone/boefje.json | 2 +- .../plugins/kat_dns_zone/normalizer.json | 2 ++ .../boefjes/plugins/kat_dnssec/boefje.json | 2 +- .../plugins/kat_dnssec/normalizer.json | 2 ++ .../plugins/kat_external_db/boefje.json | 4 +-- .../plugins/kat_external_db/normalizer.json | 2 ++ .../boefjes/plugins/kat_fierce/boefje.json | 4 +-- .../plugins/kat_fierce/normalizer.json | 2 ++ .../kat_finding_normalizer/normalizer.json | 1 + .../plugins/kat_green_hosting/boefje.json | 2 +- .../plugins/kat_green_hosting/normalizer.json | 1 + .../kat_finding_types.json | 26 +++++++++---------- .../kat_kat_finding_types/normalizer.json | 1 + .../boefjes/plugins/kat_leakix/boefje.json | 2 +- .../plugins/kat_leakix/normalizer.json | 2 ++ .../boefjes/plugins/kat_log4shell/boefje.json | 2 +- .../plugins/kat_manual/csv/normalizer.json | 1 + .../plugins/kat_masscan/normalizer.json | 2 ++ .../plugins/kat_maxmind_geoip/boefje.json | 2 +- .../plugins/kat_maxmind_geoip/normalizer.json | 2 ++ .../plugins/kat_nmap_tcp/normalizer.json | 1 + .../plugins/kat_nuclei_cve/normalizer.json | 1 + .../kat_nuclei_exposed_panels/normalizer.json | 1 + .../kat_nuclei_take_over/normalizer.json | 1 + .../boefjes/plugins/kat_rdns/normalizer.json | 1 + .../plugins/kat_report_data/normalizer.json | 1 + .../normalizer.json | 1 + .../boefjes/plugins/kat_rpki/normalizer.json | 1 + .../normalizer.json | 1 + .../plugins/kat_shodan/normalizer.json | 1 + .../boefjes/plugins/kat_snyk/normalizer.json | 1 + .../kat_snyk_finding_types/normalizer.json | 1 + .../kat_ssl_certificates/normalizer.json | 1 + .../plugins/kat_ssl_scan/normalizer.json | 1 + .../kat_testssl_sh_ciphers/normalizer.json | 1 + .../plugins/kat_wappalyzer/normalizer.json | 1 + .../check_images/normalizer.json | 1 + .../find_images_in_html/normalizer.json | 1 + .../headers/normalizer.json | 1 + .../plugins/kat_wpscan/normalizer.json | 1 + .../plugins/pdio_subfinder/normalizer.json | 1 + boefjes/tests/integration/test_api.py | 2 +- .../test_migration_add_schema_field.py | 8 +++--- 75 files changed, 113 insertions(+), 44 deletions(-) diff --git a/boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json b/boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json index e53d47f3982..93c2ae8ef7d 100644 --- a/boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json +++ b/boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json @@ -1,7 +1,7 @@ { "id": "adr-finding-types", "name": "ADR Finding Types", - "description": "Hydrate information of ADR finding types", + "description": "Hydrate information on API Design Rules (ADR) finding types for common design mistakes.", "consumes": [ "ADRFindingType" ], diff --git a/boefjes/boefjes/plugins/kat_adr_finding_types/normalizer.json b/boefjes/boefjes/plugins/kat_adr_finding_types/normalizer.json index 583b7714bed..fabda504805 100644 --- a/boefjes/boefjes/plugins/kat_adr_finding_types/normalizer.json +++ b/boefjes/boefjes/plugins/kat_adr_finding_types/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_adr_finding_types_normalize", + "name": "API Design Rules (ADR) Finding Types", + "description": "Parse API Design Rules (ADR) finding types.", "consumes": [ "boefje/adr-finding-types" ], diff --git a/boefjes/boefjes/plugins/kat_adr_validator/boefje.json b/boefjes/boefjes/plugins/kat_adr_validator/boefje.json index b782dbb5b5b..9d43ee60519 100644 --- a/boefjes/boefjes/plugins/kat_adr_validator/boefje.json +++ b/boefjes/boefjes/plugins/kat_adr_validator/boefje.json @@ -1,7 +1,7 @@ { "id": "adr-validator", "name": "API Design Rules validator", - "description": "Validate if an API conforms to the API Design Rules", + "description": "Validate if an API conforms to the API Design Rules (ADR).", "consumes": [ "RESTAPI" ], diff --git a/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json b/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json index 52c21e9a03e..f840cded2ad 100644 --- a/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json +++ b/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json @@ -1,5 +1,7 @@ { "id": "adr-validator-normalize", + "name": "API Design Rules validator", + "description": "TODO", "consumes": [ "boefje/adr-validator" ], diff --git a/boefjes/boefjes/plugins/kat_answer_parser/normalizer.json b/boefjes/boefjes/plugins/kat_answer_parser/normalizer.json index 41a89a217b0..922b333697f 100644 --- a/boefjes/boefjes/plugins/kat_answer_parser/normalizer.json +++ b/boefjes/boefjes/plugins/kat_answer_parser/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_answer_parser", + "name": "Answer Parser", + "description": "Parses the answers from Config objects.", "consumes": [ "answer" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/boefje.json b/boefjes/boefjes/plugins/kat_binaryedge/boefje.json index 004015e0570..9dc2a85d8fb 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/boefje.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/boefje.json @@ -1,7 +1,7 @@ { "id": "binaryedge", "name": "BinaryEdge", - "description": "Use BinaryEdge to find open ports with vulnerabilities that are found on that port", + "description": "Use BinaryEdge to find open ports with vulnerabilities. Requires a BinaryEdge API key.", "consumes": [ "IPAddressV4", "IPAddressV6" diff --git a/boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json index 46a034d0d1c..086ce350160 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_binaryedge_containers", + "name": "BinaryEdge containers", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json index 22fd81eb927..2af3f47f891 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_binaryedge_databases", + "name": "BinaryEdge databases", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json index f0e5825f36d..f5cafc7560a 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_binaryedge_http_web", + "name": "BinaryEdge Websites", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json index 15ea3e250b0..caa59b56f4b 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_binaryedge_message_queues", + "name": "BinaryEdge message queues", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json index 34f17a681c1..30d0f02963e 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_binaryedge_protocols", + "name": "BinaryEdge protocols", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json index c28180a88c7..80e1837a499 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_binaryedge_remote_desktop", + "name": "Binary Edge remote desktop", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json index eaea2744052..d451a79b150 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_binaryedge_service_identification", + "name": "BinaryEdge service identification", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json b/boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json index b2671be67a1..57a0f8dac16 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_binaryedge_services", + "name": "BinaryEdge services", "consumes": [ "boefje/binaryedge" ], diff --git a/boefjes/boefjes/plugins/kat_burpsuite/normalizer.json b/boefjes/boefjes/plugins/kat_burpsuite/normalizer.json index 44c8b40ab3e..c0b88e6a857 100644 --- a/boefjes/boefjes/plugins/kat_burpsuite/normalizer.json +++ b/boefjes/boefjes/plugins/kat_burpsuite/normalizer.json @@ -1,7 +1,7 @@ { "id": "kat_burpsuite_normalize", "name": "Burpsuite normalizer", - "description": "Parses Burpsuite XML output (reports). Check https://docs.openkat.nl on how to create the XML file.", + "description": "Parses Burpsuite XML output into findings. Check https://docs.openkat.nl/manual/normalizers.html#burp-suite on how to create the XML file.", "consumes": [ "xml/burp-export" ], diff --git a/boefjes/boefjes/plugins/kat_calvin/normalizer.json b/boefjes/boefjes/plugins/kat_calvin/normalizer.json index 601433e8681..c596dbdcd4a 100644 --- a/boefjes/boefjes/plugins/kat_calvin/normalizer.json +++ b/boefjes/boefjes/plugins/kat_calvin/normalizer.json @@ -1,5 +1,7 @@ { "id": "calvin-normalize", + "name": "Calvin", + "description": "Produces applications and incidents for Calvin.", "consumes": [ "boefje/calvin" ], diff --git a/boefjes/boefjes/plugins/kat_censys/boefje.json b/boefjes/boefjes/plugins/kat_censys/boefje.json index e8c15547c76..6aadac16fba 100644 --- a/boefjes/boefjes/plugins/kat_censys/boefje.json +++ b/boefjes/boefjes/plugins/kat_censys/boefje.json @@ -1,7 +1,7 @@ { "id": "censys", "name": "Censys", - "description": "Use Censys to discover open ports, services and certificates", + "description": "Use Censys to discover open ports, services and certificates. Requires and API key.", "consumes": [ "IPAddressV4", "IPAddressV6" diff --git a/boefjes/boefjes/plugins/kat_censys/normalizer.json b/boefjes/boefjes/plugins/kat_censys/normalizer.json index 446c55cd485..809fc7d7174 100644 --- a/boefjes/boefjes/plugins/kat_censys/normalizer.json +++ b/boefjes/boefjes/plugins/kat_censys/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_censys_normalize", + "name": "Censys", "consumes": [ "boefje/censys" ], diff --git a/boefjes/boefjes/plugins/kat_crt_sh/boefje.json b/boefjes/boefjes/plugins/kat_crt_sh/boefje.json index 72051dbb411..f9aa67e604e 100644 --- a/boefjes/boefjes/plugins/kat_crt_sh/boefje.json +++ b/boefjes/boefjes/plugins/kat_crt_sh/boefje.json @@ -1,7 +1,7 @@ { "id": "certificate-search", "name": "CRT", - "description": "Certificate search", + "description": "Searches for certificates and new hostnames in the transparency logs of crt.sh.", "consumes": [ "DNSZone" ], diff --git a/boefjes/boefjes/plugins/kat_crt_sh/normalizer.json b/boefjes/boefjes/plugins/kat_crt_sh/normalizer.json index 5fd671f9719..130bd2b8301 100644 --- a/boefjes/boefjes/plugins/kat_crt_sh/normalizer.json +++ b/boefjes/boefjes/plugins/kat_crt_sh/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_crt_sh_normalize", + "name": "Certificate Transparency logs (crt.sh)", + "description": "Parses data from certificate transparency logs (crt.sh) into hostnames and X509 certificates.", "consumes": [ "boefje/certificate-search" ], diff --git a/boefjes/boefjes/plugins/kat_cve_2023_34039/boefje.json b/boefjes/boefjes/plugins/kat_cve_2023_34039/boefje.json index 9c82a08ec66..0b0f6b6dc6e 100644 --- a/boefjes/boefjes/plugins/kat_cve_2023_34039/boefje.json +++ b/boefjes/boefjes/plugins/kat_cve_2023_34039/boefje.json @@ -1,7 +1,7 @@ { "id": "CVE-2023-34039", - "name": "CVE_2023_34039", - "description": "Check to see if known keys are usable on VMware CVE-2023-34039", + "name": "CVE-2023-34039 - VMware Aria Operations", + "description": "Checks if there are static SSH keys present that can be used for remote code execution on VWware Aria Operations (CVE-2023-34039). This vulnerability can be used to bypass SSH authentication and gain access to the Aria Operations for Networks CLI.", "consumes": [ "IPService" ], diff --git a/boefjes/boefjes/plugins/kat_cve_2023_35078/boefje.json b/boefjes/boefjes/plugins/kat_cve_2023_35078/boefje.json index 07525502bf3..52c93d41450 100644 --- a/boefjes/boefjes/plugins/kat_cve_2023_35078/boefje.json +++ b/boefjes/boefjes/plugins/kat_cve_2023_35078/boefje.json @@ -1,7 +1,7 @@ { "id": "CVE_2023_35078", - "name": "CVE_2023_35078", - "description": "Use NFIR script to find CVE-2023-35078", + "name": "CVE-2023-35078 - Ivanti EPMM", + "description": "Checks websites for the presents of the Ivanti EPMM interface and whether the interface is vulnerable to the remote unauthenticated API access vulnerability (CVE-2023-35078). Script contribution by NFIR.", "consumes": [ "Website" ], diff --git a/boefjes/boefjes/plugins/kat_cve_2023_35078/normalizer.json b/boefjes/boefjes/plugins/kat_cve_2023_35078/normalizer.json index 0b7413eede9..c735560eae4 100644 --- a/boefjes/boefjes/plugins/kat_cve_2023_35078/normalizer.json +++ b/boefjes/boefjes/plugins/kat_cve_2023_35078/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_CVE_2023_35078_normalize", + "name": "CVE-2023-35078 Ivanti EPMM", + "description": "Checks if the Ivanti EPMM website is vulnerable to CVE-2023-35078. Produces a finding if it is vulnerable.", "consumes": [ "boefje/CVE_2023_35078" ], diff --git a/boefjes/boefjes/plugins/kat_cve_2024_6387/normalizer.json b/boefjes/boefjes/plugins/kat_cve_2024_6387/normalizer.json index 1cf4c49e5bf..0e06b9d2362 100644 --- a/boefjes/boefjes/plugins/kat_cve_2024_6387/normalizer.json +++ b/boefjes/boefjes/plugins/kat_cve_2024_6387/normalizer.json @@ -1,9 +1,10 @@ { "id": "kat_cve_2024_6387_normalize", + "name": "CVE-2024-6387 OpenSSH", + "description": "Checks the service banner for a race condition in OpenSSH server which can result in an unauthenticated remote attacker to trigger that some signals are handled in an unsafe manner (CVE-2024-6387). Requires the Service-Banner-boefje to be enabled.", "consumes": [ "openkat/service-banner" ], - "description": "Checks service banner for CVE-2024-6387, enable service banner boefje to get the service banner", "produces": [ "Finding", "CVEFindingType" diff --git a/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json b/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json index 2b390197290..280ea27e565 100644 --- a/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json +++ b/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json @@ -1,7 +1,7 @@ { "id": "cve-finding-types", "name": "CVE Finding Types", - "description": "Hydrate information of CVE finding types from the CVE API", + "description": "Hydrate information of Common Vulnerabilities and Exposures (CVE) finding types from the CVE API", "consumes": [ "CVEFindingType" ], diff --git a/boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json b/boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json index 6e2d52291aa..6ae5590562d 100644 --- a/boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json +++ b/boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_cve_finding_types_normalize", + "name": "CVE finding types", + "description": "Parses CVE findings.", "consumes": [ "boefje/cve-finding-types" ], diff --git a/boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json b/boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json index a3656aa48c6..abeeaa7d9d0 100644 --- a/boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json +++ b/boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json @@ -1,7 +1,7 @@ { "id": "cwe-finding-types", "name": "CWE Finding Types", - "description": "Hydrate information of CWE finding types", + "description": "Hydrate information of Common Weakness Enumeration (CWE) finding types", "consumes": [ "CWEFindingType" ], diff --git a/boefjes/boefjes/plugins/kat_cwe_finding_types/normalizer.json b/boefjes/boefjes/plugins/kat_cwe_finding_types/normalizer.json index 7b19ddd4c99..9b939d07df5 100644 --- a/boefjes/boefjes/plugins/kat_cwe_finding_types/normalizer.json +++ b/boefjes/boefjes/plugins/kat_cwe_finding_types/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_cwe_finding_types_normalize", + "name": "CWE finding", + "description": "Parses CWE findings.", "consumes": [ "boefje/cwe-finding-types" ], diff --git a/boefjes/boefjes/plugins/kat_dicom/boefje.json b/boefjes/boefjes/plugins/kat_dicom/boefje.json index 437829787a9..6cfd4e76498 100644 --- a/boefjes/boefjes/plugins/kat_dicom/boefje.json +++ b/boefjes/boefjes/plugins/kat_dicom/boefje.json @@ -1,7 +1,7 @@ { "id": "dicom", "name": "DICOM", - "description": "Find exposed DICOM servers.", + "description": "Find exposed DICOM servers. DICOM servers are used to process medical imaging information.", "consumes": [ "IPAddressV4", "IPAddressV6" diff --git a/boefjes/boefjes/plugins/kat_dicom/normalizer.json b/boefjes/boefjes/plugins/kat_dicom/normalizer.json index b8e5f1dd49c..74519e6e96c 100644 --- a/boefjes/boefjes/plugins/kat_dicom/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dicom/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_dicom_normalize", + "name": "DICOM servers", + "description": "Parses DICOM output into findings and identified software.", "consumes": [ "boefje/dicom" ], diff --git a/boefjes/boefjes/plugins/kat_dns/boefje.json b/boefjes/boefjes/plugins/kat_dns/boefje.json index 76c36ae1775..53391f0155d 100644 --- a/boefjes/boefjes/plugins/kat_dns/boefje.json +++ b/boefjes/boefjes/plugins/kat_dns/boefje.json @@ -1,7 +1,7 @@ { "id": "dns-records", - "name": "DnsRecords", - "description": "Fetch the DNS record(s) of a hostname", + "name": "DNS records", + "description": "Fetch the DNS record(s) of a hostname.", "consumes": [ "Hostname" ], diff --git a/boefjes/boefjes/plugins/kat_dns/normalizer.json b/boefjes/boefjes/plugins/kat_dns/normalizer.json index e4a2316eda0..fa9c8a73fa6 100644 --- a/boefjes/boefjes/plugins/kat_dns/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dns/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_dns_normalize", + "name": "DNS records", + "description": "Parses the DNS records.", "consumes": [ "boefje/dns-records" ], diff --git a/boefjes/boefjes/plugins/kat_dns_zone/boefje.json b/boefjes/boefjes/plugins/kat_dns_zone/boefje.json index 25df0af7bcd..cc03e079bd1 100644 --- a/boefjes/boefjes/plugins/kat_dns_zone/boefje.json +++ b/boefjes/boefjes/plugins/kat_dns_zone/boefje.json @@ -1,6 +1,6 @@ { "id": "dns-zone", - "name": "DnsZone", + "name": "DNS zone", "description": "Fetch the parent DNS zone of a DNS zone", "consumes": [ "DNSZone" diff --git a/boefjes/boefjes/plugins/kat_dns_zone/normalizer.json b/boefjes/boefjes/plugins/kat_dns_zone/normalizer.json index c4060c833ec..e9e156f6d2c 100644 --- a/boefjes/boefjes/plugins/kat_dns_zone/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dns_zone/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_dns_zone_normalize", + "name": "DNS zone", + "description": "Parses the parent DNS zone into new hostnames and DNS zones.", "consumes": [ "boefje/dns-zone" ], diff --git a/boefjes/boefjes/plugins/kat_dnssec/boefje.json b/boefjes/boefjes/plugins/kat_dnssec/boefje.json index 7b59b0fae25..8b4d156396e 100644 --- a/boefjes/boefjes/plugins/kat_dnssec/boefje.json +++ b/boefjes/boefjes/plugins/kat_dnssec/boefje.json @@ -1,6 +1,6 @@ { "id": "dns-sec", - "name": "Dnssec", + "name": "DNSSEC", "description": "Validates DNSSec of a hostname", "consumes": [ "Hostname" diff --git a/boefjes/boefjes/plugins/kat_dnssec/normalizer.json b/boefjes/boefjes/plugins/kat_dnssec/normalizer.json index 24877c2e897..670f16592f4 100644 --- a/boefjes/boefjes/plugins/kat_dnssec/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dnssec/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_dnssec_normalize", + "name": "DNS records", + "description": "Parses DNSSEC data into findings.", "consumes": [ "boefje/dns-sec" ], diff --git a/boefjes/boefjes/plugins/kat_external_db/boefje.json b/boefjes/boefjes/plugins/kat_external_db/boefje.json index 1f27e7f9d2e..4de34b597ac 100644 --- a/boefjes/boefjes/plugins/kat_external_db/boefje.json +++ b/boefjes/boefjes/plugins/kat_external_db/boefje.json @@ -1,7 +1,7 @@ { "id": "external_db", - "name": "External Database", - "description": "Fetch hostnames and IP addresses/netblocks from an external database with API. See `description.md` for more information.", + "name": "External database host fetcher", + "description": "Fetch hostnames and IP addresses/netblocks from an external database with API. See `description.md` for more information. Useful if you have a large network.", "consumes": [ "Network" ], diff --git a/boefjes/boefjes/plugins/kat_external_db/normalizer.json b/boefjes/boefjes/plugins/kat_external_db/normalizer.json index 36d425db438..2d9e72d56e9 100644 --- a/boefjes/boefjes/plugins/kat_external_db/normalizer.json +++ b/boefjes/boefjes/plugins/kat_external_db/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_external_db_normalize", + "name": "External database hosts fetcher", + "description": "Parse data the fetched host data from the external database into hostnames and IP-addresses.", "consumes": [ "boefje/external_db" ], diff --git a/boefjes/boefjes/plugins/kat_fierce/boefje.json b/boefjes/boefjes/plugins/kat_fierce/boefje.json index c198875c8a1..1f7d5c677db 100644 --- a/boefjes/boefjes/plugins/kat_fierce/boefje.json +++ b/boefjes/boefjes/plugins/kat_fierce/boefje.json @@ -1,9 +1,9 @@ { "id": "fierce", "name": "Fierce", - "description": "Use a Fierce scan to find subdomains (with their ip)", + "description": "Perform DNS reconnaissance using Fierce, to help locate non-contiguous IP space and hostnames against specified hostnames. No exploitation is performed.", "consumes": [ "Hostname" ], - "scan_level": 3 + "scan_level": 1 } diff --git a/boefjes/boefjes/plugins/kat_fierce/normalizer.json b/boefjes/boefjes/plugins/kat_fierce/normalizer.json index 536944b4995..82589b6565d 100644 --- a/boefjes/boefjes/plugins/kat_fierce/normalizer.json +++ b/boefjes/boefjes/plugins/kat_fierce/normalizer.json @@ -1,5 +1,7 @@ { "id": "kat_fierce_normalize", + "name": "Fierce", + "description": "Parse the DNS reconnaissance data from Fierce into hostnames and/or IP addresses.", "consumes": [ "boefje/fierce" ], diff --git a/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json b/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json index ec7e54b209c..70adfd46c47 100644 --- a/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json +++ b/boefjes/boefjes/plugins/kat_finding_normalizer/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_generic_finding_normalize", + "name": "Finding types", "consumes": [ "openkat/finding" ], diff --git a/boefjes/boefjes/plugins/kat_green_hosting/boefje.json b/boefjes/boefjes/plugins/kat_green_hosting/boefje.json index 9fe34d17ae8..846b05efa33 100644 --- a/boefjes/boefjes/plugins/kat_green_hosting/boefje.json +++ b/boefjes/boefjes/plugins/kat_green_hosting/boefje.json @@ -1,7 +1,7 @@ { "id": "green-hosting", "name": "GreenHosting", - "description": "Use the Green Web Foundation Partner API to check whether the website is hosted on a green server. Meaning it runs on renewable energy and/or offsets its carbon footprint", + "description": "Use the Green Web Foundation Partner API to check whether the website is hosted on a green server. Meaning it runs on renewable energy and/or offsets its carbon footprint. Does not require an API key.", "consumes": [ "Website" ], diff --git a/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json b/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json index 993413d85e4..92434280db5 100644 --- a/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json +++ b/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_green_hosting_normalize", + "description": "Parses the Green Hosting output into findings.", "consumes": [ "boefje/green-hosting" ], diff --git a/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json b/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json index 31ff5bed15b..23ec1b70da7 100644 --- a/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json +++ b/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json @@ -21,14 +21,14 @@ "recommendation": "This header is not supported by default by Mozilla. If this header is required for your environment: Set the HTTP header X-Permitted-Cross- Domain-Policies: none in all HTTP responses. Use value master-only if a Flash or Acrobat cross- domain configuration file is used that is placed in the root of the web server" }, "KAT-NO-EXPLICIT-XSS-PROTECTION": { - "description": "This is a deprecated header previously used to prevent against Cross-Site-Scripting attacks. Support in modern browsers could introduce XSS attacks again.", + "description": "The 'X-XSS-Protection' header is a deprecated header previously used to prevent against Cross-Site-Scripting attacks. Support in modern browsers could introduce XSS attacks again.", "source": "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection", "risk": "recommendation", "impact": "Reflected cross-site scripting attacks may not be blocked.", - "recommendation": "This header is deprecated and should not be used." + "recommendation": "Remove the deprecated header to reduce the chance of XSS attacks." }, "KAT-NO-X-FRAME-OPTIONS": { - "description": "HTTP header 'X-Frame-Options' is missing. It is possible that the website can be loaded via an