From 3ccf16ea8cc09c0eb841981f88a8fda16c71a653 Mon Sep 17 00:00:00 2001 From: zcrt <115991818+zcrt@users.noreply.github.com> Date: Thu, 5 Sep 2024 09:56:28 +0200 Subject: [PATCH 01/28] feat: :boom: recalculate all bits (#3451) Co-authored-by: Jan Klopper Co-authored-by: ammar92 --- rocky/rocky/locale/django.pot | 6 ++- .../organizations/organization_list.html | 23 ++++++++--- rocky/rocky/views/organization_list.py | 40 +++++++++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 597592e2987..67ccc529f3c 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-09-04 08:55+0000\n" +"POT-Creation-Date: 2024-09-04 14:21+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -5468,6 +5468,10 @@ msgstr "" msgid "Save organization" msgstr "" +#: rocky/templates/organizations/organization_list.html +msgid "Rerun all bits for all my organizations" +msgstr "" + #: rocky/templates/organizations/organization_list.html msgid "Add new organization" msgstr "" diff --git a/rocky/rocky/templates/organizations/organization_list.html b/rocky/rocky/templates/organizations/organization_list.html index 8b031603fb8..d85555e985a 100644 --- a/rocky/rocky/templates/organizations/organization_list.html +++ b/rocky/rocky/templates/organizations/organization_list.html @@ -11,11 +11,22 @@

{% translate "Organizations" %}

- {% if perms.tools.add_organization %} -
+
+ {% if perms.tools.can_recalculate_bits %} +
+ {% csrf_token %} + +
+ {% endif %} + {% if perms.tools.add_organization %} {% translate "Add new organization" %} -
- {% endif %} + {% endif %} +
{% if object_list %} @@ -32,10 +43,10 @@

{% translate "Organizations" %}

{% for organization in object_list %} + + + {% endfor %} + +
- {{ organization.name }} + {{ organization.name }} - {{ organization.code }} + {{ organization.code }} {% include "organizations/organization_tags.html" %} diff --git a/rocky/rocky/views/organization_list.py b/rocky/rocky/views/organization_list.py index 6bf12f1956a..596637bd82c 100644 --- a/rocky/rocky/views/organization_list.py +++ b/rocky/rocky/views/organization_list.py @@ -1,9 +1,19 @@ +import logging +from datetime import datetime + from account.models import KATUser +from django.conf import settings +from django.contrib import messages +from django.core.exceptions import PermissionDenied from django.db.models import Count +from django.http import HttpRequest, HttpResponse, HttpResponseBadRequest +from django.utils.translation import gettext_lazy as _ from django.views.generic import ListView from tools.models import Organization from tools.view_helpers import OrganizationBreadcrumbsMixin +from octopoes.connector.octopoes import OctopoesAPIConnector + class OrganizationListView( OrganizationBreadcrumbsMixin, @@ -18,3 +28,33 @@ def get_queryset(self) -> list[Organization]: .prefetch_related("tags") .filter(id__in=[organization.id for organization in user.organizations]) ) + + def post(self, request: HttpRequest, *args, **kwargs) -> HttpResponse: + """Perform actions based on action type""" + if request.POST.get("action") == "recalculate": + if not self.request.user.has_perm("tools.can_recalculate_bits"): + raise PermissionDenied() + organizations = self.request.user.organizations + number_of_bits = 0 + failed = [] + start_time = datetime.now() + for organization in organizations: + try: + number_of_bits += OctopoesAPIConnector(settings.OCTOPOES_API, organization.code).recalculate_bits() + except Exception as exc: + failed.append(f"{organization}, ({str(exc)})") + logging.warning("Failed recalculating bits for %s, %s", organization, exc) + duration = datetime.now() - start_time + n_failed = len(failed) + message = f"Recalculated {number_of_bits} bits for {len(organizations)-n_failed} organizations." + message += f" Duration: {duration}." + if failed: + message += f"\nFailed for {n_failed} organisations: {', '.join(failed)}" + messages.add_message( + request, + messages.INFO, + _(message), + ) + return self.get(request, *args, **kwargs) + else: + raise HttpResponseBadRequest("Unknown action") From 280272c38694339b1f31fc9e73a650e639491fe2 Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:36:15 +0200 Subject: [PATCH 02/28] Add raw SQL migrations (#3457) Signed-off-by: Donny Peeters Co-authored-by: Jan Klopper --- ...88b9cd96aa_remove_the_repository_model.sql | 14 ++++++++ ...introduce_boefje_and_normalizer_models.sql | 32 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 boefjes/export_migrations/0006_7c88b9cd96aa_remove_the_repository_model.sql create mode 100644 boefjes/export_migrations/0007_6f99834a4a5a_introduce_boefje_and_normalizer_models.sql diff --git a/boefjes/export_migrations/0006_7c88b9cd96aa_remove_the_repository_model.sql b/boefjes/export_migrations/0006_7c88b9cd96aa_remove_the_repository_model.sql new file mode 100644 index 00000000000..d435a98c9f5 --- /dev/null +++ b/boefjes/export_migrations/0006_7c88b9cd96aa_remove_the_repository_model.sql @@ -0,0 +1,14 @@ +-- Make sure all plugin_state entries point to the LOCAL repository +DELETE FROM plugin_state ps WHERE EXISTS(SELECT 1 FROM repository r WHERE ps.repository_pk = r.pk AND r.id != 'LOCAL'); + +-- Make the plugin_state entries unique by plugin_id and organisation_pk only, as they all point to the LOCAL repository +ALTER TABLE plugin_state DROP CONSTRAINT unique_plugin_per_repo_per_org; +ALTER TABLE plugin_state ADD CONSTRAINT unique_plugin_id_per_org UNIQUE (plugin_id, organisation_pk); + +-- Remove the repository foreign key for plugin_state +ALTER TABLE plugin_state DROP CONSTRAINT plugin_state_repository_pk_fkey; +ALTER TABLE plugin_state DROP COLUMN repository_pk; + +-- Remove the many-to-many relation to organisations and the repository table itself +DROP TABLE organisation_repository; +DROP TABLE repository; diff --git a/boefjes/export_migrations/0007_6f99834a4a5a_introduce_boefje_and_normalizer_models.sql b/boefjes/export_migrations/0007_6f99834a4a5a_introduce_boefje_and_normalizer_models.sql new file mode 100644 index 00000000000..938486a0806 --- /dev/null +++ b/boefjes/export_migrations/0007_6f99834a4a5a_introduce_boefje_and_normalizer_models.sql @@ -0,0 +1,32 @@ +CREATE TYPE scan_level AS ENUM ('0', '1', '2', '3', '4'); + +CREATE TABLE boefje ( + id SERIAL NOT NULL, + plugin_id VARCHAR(64) NOT NULL, + created TIMESTAMP WITH TIME ZONE, + name VARCHAR(64) NOT NULL, + description TEXT, + scan_level scan_level NOT NULL, + consumes VARCHAR(128)[] NOT NULL, + produces VARCHAR(128)[] NOT NULL, + environment_keys VARCHAR(128)[] NOT NULL, + oci_image VARCHAR(256), + oci_arguments VARCHAR(128)[] NOT NULL, + version VARCHAR(16), + PRIMARY KEY (id), + UNIQUE (plugin_id) +); + +CREATE TABLE normalizer ( + id SERIAL NOT NULL, + plugin_id VARCHAR(64) NOT NULL, + created TIMESTAMP WITH TIME ZONE, + name VARCHAR(64) NOT NULL, + description TEXT, + consumes VARCHAR(128)[] NOT NULL, + produces VARCHAR(128)[] NOT NULL, + environment_keys VARCHAR(128)[] NOT NULL, + version VARCHAR(16), + PRIMARY KEY (id), + UNIQUE (plugin_id) +); From f243e7384ee2101509a897b376c34031758f4a38 Mon Sep 17 00:00:00 2001 From: noamblitz <43830693+noamblitz@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:43:52 +0200 Subject: [PATCH 03/28] Fix add related, fix manual ooi task list, remove redundant octopoes call (#3421) Co-authored-by: ammar92 Co-authored-by: Jan Klopper --- rocky/rocky/locale/django.pot | 6 +- .../oois/ooi_detail_origins_observations.html | 6 +- .../partials/ooi_detail_related_object.html | 2 +- rocky/rocky/views/mixins.py | 22 +++-- rocky/rocky/views/ooi_detail.py | 4 +- .../rocky/views/ooi_detail_related_object.py | 97 ++++++++++--------- rocky/tests/objects/test_objects_detail.py | 12 +-- .../objects/test_objects_scan_profile.py | 10 +- 8 files changed, 86 insertions(+), 73 deletions(-) diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 67ccc529f3c..8d7603aad8a 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-09-04 14:21+0000\n" +"POT-Creation-Date: 2024-09-05 08:44+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -5271,6 +5271,10 @@ msgstr "" msgid "When" msgstr "" +#: rocky/templates/oois/ooi_detail_origins_observations.html +msgid "This scan was manually created." +msgstr "" + #: rocky/templates/oois/ooi_detail_origins_observations.html msgid "The boefje has since been deleted or disabled." msgstr "" diff --git a/rocky/rocky/templates/oois/ooi_detail_origins_observations.html b/rocky/rocky/templates/oois/ooi_detail_origins_observations.html index c4b4e60cc05..e60225d97d6 100644 --- a/rocky/rocky/templates/oois/ooi_detail_origins_observations.html +++ b/rocky/rocky/templates/oois/ooi_detail_origins_observations.html @@ -24,7 +24,11 @@

{% translate "Last observed by" %}

{{ observation.boefje.name }} {% else %} - {% translate "The boefje has since been deleted or disabled." %} + {% if observation.normalizer.raw_data.boefje_meta.boefje.id == "manual" %} + {% translate "This scan was manually created." %} + {% else %} + {% translate "The boefje has since been deleted or disabled." %} + {% endif %} {% endif %} 		diff --git a/rocky/rocky/templates/partials/ooi_detail_related_object.html b/rocky/rocky/templates/partials/ooi_detail_related_object.html index 30ebfcb5467..f70076a4c1e 100644 --- a/rocky/rocky/templates/partials/ooi_detail_related_object.html +++ b/rocky/rocky/templates/partials/ooi_detail_related_object.html @@ -12,7 +12,7 @@

{% translate "Related objects" %}

{% if not ooi_past_due %} {% if not ooi|is_finding and not ooi|is_finding_type %}
- {% translate "Add" %}
{% endif %} diff --git a/rocky/rocky/views/mixins.py b/rocky/rocky/views/mixins.py index d8bc37268ed..116ae4cd234 100644 --- a/rocky/rocky/views/mixins.py +++ b/rocky/rocky/views/mixins.py @@ -172,16 +172,20 @@ def get_origins( boefje_meta = normalizer_data["raw_data"]["boefje_meta"] boefje_id = boefje_meta["boefje"]["id"] if boefje_meta.get("ended_at"): - boefje_meta["ended_at"] = datetime.strptime(boefje_meta["ended_at"], "%Y-%m-%dT%H:%M:%S.%fZ") + try: + boefje_meta["ended_at"] = datetime.strptime(boefje_meta["ended_at"], "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + boefje_meta["ended_at"] = datetime.strptime(boefje_meta["ended_at"], "%Y-%m-%dT%H:%M:%SZ") origin.normalizer = normalizer_data - try: - origin.boefje = katalogus.get_plugin(boefje_id) - except HTTPError as e: - logger.error( - "Could not load boefje: %s from katalogus, error: %s", - boefje_id, - e, - ) + if boefje_id != "manual": + try: + origin.boefje = katalogus.get_plugin(boefje_id) + except HTTPError as e: + logger.error( + "Could not load boefje %s from katalogus: %s", + boefje_id, + e, + ) observations.append(origin) return results diff --git a/rocky/rocky/views/ooi_detail.py b/rocky/rocky/views/ooi_detail.py index fb086b97d28..16b610a4508 100644 --- a/rocky/rocky/views/ooi_detail.py +++ b/rocky/rocky/views/ooi_detail.py @@ -13,14 +13,14 @@ from octopoes.models import Reference from octopoes.models.ooi.question import Question -from rocky.views.ooi_detail_related_object import OOIFindingManager, OOIRelatedObjectAddView +from rocky.views.ooi_detail_related_object import OOIFindingManager, OOIRelatedObjectManager from rocky.views.ooi_view import BaseOOIDetailView from rocky.views.tasks import TaskListView class OOIDetailView( BaseOOIDetailView, - OOIRelatedObjectAddView, + OOIRelatedObjectManager, OOIFindingManager, TaskListView, ): diff --git a/rocky/rocky/views/ooi_detail_related_object.py b/rocky/rocky/views/ooi_detail_related_object.py index 356c189b9c4..3840f235246 100644 --- a/rocky/rocky/views/ooi_detail_related_object.py +++ b/rocky/rocky/views/ooi_detail_related_object.py @@ -3,6 +3,7 @@ from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext_lazy as _ +from django.views.generic.base import TemplateView from tools.ooi_helpers import format_attr_name from tools.view_helpers import existing_ooi_type, get_mandatory_fields, url_with_querystring @@ -29,54 +30,6 @@ def get_related_objects(self, observed_at): related.append(rel) return related - -class OOIFindingManager(SingleOOITreeMixin): - def get_findings(self) -> list[Finding]: - findings = [] - for relation in self.tree.root.children.values(): - for child in relation: - ooi = self.tree.store[str(child.reference)] - if isinstance(ooi, Finding) and ooi.reference != self.tree.root.reference: - findings.append(ooi) - return findings - - def count_findings_per_severity(self) -> Counter: - counter = Counter({severity: 0 for severity in RiskLevelSeverity}) - for finding in self.get_findings(): - finding_type: FindingType | None = self.tree.store.get(str(finding.finding_type), None) - if finding_type is not None and finding_type.risk_severity is not None: - counter.update([finding_type.risk_severity]) - else: - counter.update([RiskLevelSeverity.UNKNOWN]) - return counter - - def get_finding_details_sorted_by_score_desc(self) -> list[tuple[Finding, FindingType]]: - finding_details = self.get_finding_details() - return list(sorted(finding_details, key=lambda x: x[1].risk_score or 0, reverse=True)) - - def get_finding_details(self) -> list[tuple[Finding, FindingType]]: - return [(finding, self.tree.store[str(finding.finding_type)]) for finding in self.get_findings()] - - -class OOIRelatedObjectAddView(OOIRelatedObjectManager): - template_name = "oois/ooi_detail_add_related_object.html" - - def get(self, request, *args, **kwargs): - if "ooi_id" in request.GET: - self.ooi_id = self.get_ooi(pk=request.GET.get("ooi_id")) - - if "add_ooi_type" in request.GET: - ooi_type_choice = self.split_ooi_type_choice(request.GET["add_ooi_type"]) - if existing_ooi_type(ooi_type_choice["ooi_type"]): - return redirect(self.ooi_add_url(self.ooi_id, **ooi_type_choice)) - - if "status_code" in kwargs: - response = super().get(request, *args, **kwargs) - response.status_code = kwargs["status_code"] - return response - - return super().get(request, *args, **kwargs) - def split_ooi_type_choice(self, ooi_type_choice) -> dict[str, str]: ooi_type = ooi_type_choice.split("|", 1) @@ -146,6 +99,54 @@ def get_ooi_types_input_values(self, ooi: OOI) -> list[dict[str, str]]: return input_values + +class OOIFindingManager(SingleOOITreeMixin): + def get_findings(self) -> list[Finding]: + findings = [] + for relation in self.tree.root.children.values(): + for child in relation: + ooi = self.tree.store[str(child.reference)] + if isinstance(ooi, Finding) and ooi.reference != self.tree.root.reference: + findings.append(ooi) + return findings + + def count_findings_per_severity(self) -> Counter: + counter = Counter({severity: 0 for severity in RiskLevelSeverity}) + for finding in self.get_findings(): + finding_type: FindingType | None = self.tree.store.get(str(finding.finding_type), None) + if finding_type is not None and finding_type.risk_severity is not None: + counter.update([finding_type.risk_severity]) + else: + counter.update([RiskLevelSeverity.UNKNOWN]) + return counter + + def get_finding_details_sorted_by_score_desc(self) -> list[tuple[Finding, FindingType]]: + finding_details = self.get_finding_details() + return list(sorted(finding_details, key=lambda x: x[1].risk_score or 0, reverse=True)) + + def get_finding_details(self) -> list[tuple[Finding, FindingType]]: + return [(finding, self.tree.store[str(finding.finding_type)]) for finding in self.get_findings()] + + +class OOIRelatedObjectAddView(OOIRelatedObjectManager, TemplateView): + template_name = "oois/ooi_detail_add_related_object.html" + + def get(self, request, *args, **kwargs): + if "ooi_id" in request.GET: + self.ooi_id = self.get_ooi(pk=request.GET["ooi_id"]) + + if "add_ooi_type" in request.GET: + ooi_type_choice = self.split_ooi_type_choice(request.GET["add_ooi_type"]) + if existing_ooi_type(ooi_type_choice["ooi_type"]): + return redirect(self.ooi_add_url(self.ooi_id, **ooi_type_choice)) + + if "status_code" in kwargs: + response = super().get(request, *args, **kwargs) + response.status_code = kwargs["status_code"] + return response + + return super().get(request, *args, **kwargs) + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["ooi_id"] = self.ooi_id diff --git a/rocky/tests/objects/test_objects_detail.py b/rocky/tests/objects/test_objects_detail.py index 71d5d4d63a1..4bd7b7653e1 100644 --- a/rocky/tests/objects/test_objects_detail.py +++ b/rocky/tests/objects/test_objects_detail.py @@ -71,7 +71,7 @@ def test_ooi_detail( response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) assert response.status_code == 200 - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assertContains(response, "Object") assertContains(response, "Network|testnetwork") @@ -109,7 +109,7 @@ def test_question_detail( response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) assert response.status_code == 200 - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assertContains(response, "Question") assertContains(response, "Rendered Question Form") @@ -142,7 +142,7 @@ def test_answer_question( response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) assertContains(response, "Question has been answered.", status_code=200) - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 def test_answer_question_bad_schema( @@ -217,7 +217,7 @@ def test_ooi_detail_start_scan( ) response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assert response.status_code == 200 @@ -263,7 +263,7 @@ def test_ooi_detail_start_scan_no_indemnification( ) response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assertContains(response, "Object details") assertContains(response, "Indemnification not present") @@ -295,7 +295,7 @@ def test_ooi_detail_start_scan_no_action( ) response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assertContains(response, "Object details") diff --git a/rocky/tests/objects/test_objects_scan_profile.py b/rocky/tests/objects/test_objects_scan_profile.py index 1c2f6fa5aaa..98626ec7407 100644 --- a/rocky/tests/objects/test_objects_scan_profile.py +++ b/rocky/tests/objects/test_objects_scan_profile.py @@ -59,7 +59,7 @@ def test_scan_profile(rf, redteam_member, mock_scheduler, mock_organization_view response = ScanProfileDetailView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 200 - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assertContains(response, "Set clearance level") @@ -125,7 +125,7 @@ def test_scan_profile_no_permissions_acknowledged( response = ScanProfileDetailView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 200 - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assertNotContains(response, "Set clearance level") @@ -146,7 +146,7 @@ def test_scan_profile_no_permissions_trusted( response = ScanProfileDetailView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 200 - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assertNotContains(response, "Set clearance level") @@ -162,7 +162,7 @@ def test_scan_profile_reset_view(rf, redteam_member, mock_scheduler, mock_organi response = ScanProfileResetView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 200 - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assertContains(response, "Set clearance level") assertContains(response, "Yes, set to inherit") @@ -183,5 +183,5 @@ def test_scan_reset_calls_octopoes(rf, redteam_member, mock_scheduler, mock_orga response = ScanProfileResetView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 302 - assert mock_organization_view_octopoes().get_tree.call_count == 2 + assert mock_organization_view_octopoes().get_tree.call_count == 1 assert mock_organization_view_octopoes().save_scan_profile.call_count == 1 From 3369b3bc592a5eec237df5e90c6df26aacccb4d8 Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:26:28 +0200 Subject: [PATCH 04/28] Feature: improve settings and environment logic and phase out redundant environment keys (#3384) Signed-off-by: Donny Peeters Co-authored-by: Jan Klopper Co-authored-by: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> Co-authored-by: ammar92 --- boefjes/.ci/docker-compose.yml | 14 ++++-- .../.ci/wiremock/mappings/organisations.json | 15 ------- boefjes/boefjes/api.py | 3 +- boefjes/boefjes/dependencies/plugins.py | 23 ++++------ boefjes/boefjes/job_handler.py | 45 ++++++++++++------- boefjes/boefjes/katalogus/plugins.py | 2 - ...fc302b852_remove_environment_keys_field.py | 37 +++++++++++++++ ...de6eb7824b_introduce_boefjeconfig_model.py | 6 +-- boefjes/boefjes/models.py | 1 - .../plugins/kat_binaryedge/boefje.json | 3 -- .../boefjes/plugins/kat_censys/boefje.json | 4 -- .../plugins/kat_cve_finding_types/boefje.json | 3 -- boefjes/boefjes/plugins/kat_dns/boefje.json | 4 -- .../plugins/kat_external_db/boefje.json | 7 --- .../boefjes/plugins/kat_leakix/boefje.json | 3 -- .../boefjes/plugins/kat_log4shell/boefje.json | 3 -- .../boefjes/plugins/kat_masscan/boefje.json | 4 -- .../plugins/kat_maxmind_geoip/boefje.json | 6 +-- .../plugins/kat_nmap_ip_range/boefje.json | 6 --- .../plugins/kat_nmap_ports/boefje.json | 3 -- .../boefjes/plugins/kat_nmap_tcp/boefje.json | 3 -- .../boefjes/plugins/kat_nmap_udp/boefje.json | 3 -- .../boefjes/plugins/kat_shodan/boefje.json | 3 -- .../kat_testssl_sh_ciphers/boefje.json | 5 +-- .../plugins/kat_webpage_analysis/boefje.json | 3 -- .../boefjes/plugins/kat_wpscan/boefje.json | 3 -- boefjes/boefjes/sql/db_models.py | 2 - boefjes/boefjes/sql/plugin_storage.py | 4 -- boefjes/boefjes/storage/interfaces.py | 7 +-- boefjes/pyproject.toml | 1 + boefjes/tests/conftest.py | 38 ++++++++++++++-- boefjes/tests/integration/test_bench.py | 4 +- .../tests/integration/test_get_environment.py | 22 +++++++++ .../test_migration_add_schema_field.py | 6 +-- .../integration/test_sql_repositories.py | 2 - .../boefjes_test_dir/kat_test/boefje.json | 1 - .../kat_test/kat_test_2/boefje.json | 1 - .../kat_test/kat_test_4/boefje.json | 1 - .../tests/katalogus/test_plugin_service.py | 30 +++---------- .../tests/modules/dummy_boefje/boefje.json | 3 +- .../dummy_boefje_environment/boefje.json | 3 +- .../boefje.json | 3 +- .../boefje.json | 3 +- .../dummy_boefje_missing_run/boefje.json | 3 +- .../boefje.json | 3 +- .../dummy_oci_boefje_no_main/boefje.json | 1 - boefjes/tests/test_tasks.py | 6 +-- .../source/developer_documentation/boefjes.md | 4 +- .../development_tutorial/creating_a_boefje.md | 25 +++++++++-- docs/source/introduction/makeyourown.rst | 43 ++++++++++++------ mula/scheduler/models/plugin.py | 1 - rocky/katalogus/client.py | 1 - rocky/tests/conftest.py | 1 - rocky/tests/stubs/katalogus_boefjes.json | 9 ---- rocky/tests/stubs/katalogus_normalizers.json | 5 --- rocky/tests/test_indemnification.py | 1 - 56 files changed, 223 insertions(+), 223 deletions(-) delete mode 100644 boefjes/.ci/wiremock/mappings/organisations.json create mode 100644 boefjes/boefjes/migrations/versions/870fc302b852_remove_environment_keys_field.py create mode 100644 boefjes/tests/integration/test_get_environment.py diff --git a/boefjes/.ci/docker-compose.yml b/boefjes/.ci/docker-compose.yml index 407d1678336..f3dc4ecc244 100644 --- a/boefjes/.ci/docker-compose.yml +++ b/boefjes/.ci/docker-compose.yml @@ -8,6 +8,7 @@ services: command: sh -c 'python -m pytest -v tests/integration' depends_on: - ci_katalogus-db + - ci_katalogus env_file: - .ci/.env.test volumes: @@ -103,8 +104,15 @@ services: hard: 262144 ci_katalogus: - image: "docker.io/wiremock/wiremock:2.34.0" - volumes: - - .ci/wiremock:/home/wiremock + build: + context: .. + dockerfile: boefjes/Dockerfile + args: + - ENVIRONMENT=dev + command: uvicorn boefjes.katalogus.root:app --host 0.0.0.0 --port 8080 + depends_on: + - ci_katalogus-db env_file: - .ci/.env.test + volumes: + - .:/app/boefjes diff --git a/boefjes/.ci/wiremock/mappings/organisations.json b/boefjes/.ci/wiremock/mappings/organisations.json deleted file mode 100644 index 13124e6222f..00000000000 --- a/boefjes/.ci/wiremock/mappings/organisations.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "request": { - "method": "GET", - "url": "/v1/organisations" - }, - "response": { - "status": 200, - "jsonBody": { - "_dev": { - "id": "_dev", - "name": "Development Organisation" - } - } - } -} diff --git a/boefjes/boefjes/api.py b/boefjes/boefjes/api.py index 72ac9ccc96c..22542898dfd 100644 --- a/boefjes/boefjes/api.py +++ b/boefjes/boefjes/api.py @@ -151,8 +151,7 @@ def get_task(task_id, scheduler_client): def create_boefje_meta(task, local_repository): boefje = task.data.boefje boefje_resource = local_repository.by_id(boefje.id) - env_keys = boefje_resource.environment_keys - environment = get_environment_settings(task.data, env_keys) if env_keys else {} + environment = get_environment_settings(task.data, boefje_resource.schema) organization = task.data.organization input_ooi = task.data.input_ooi diff --git a/boefjes/boefjes/dependencies/plugins.py b/boefjes/boefjes/dependencies/plugins.py index ccb3187c0e2..080ff6d7da7 100644 --- a/boefjes/boefjes/dependencies/plugins.py +++ b/boefjes/boefjes/dependencies/plugins.py @@ -98,7 +98,7 @@ def clone_settings_to_organisation(self, from_organisation: str, to_organisation self.set_enabled_by_id(plugin_id, to_organisation, enabled=True) def upsert_settings(self, settings: dict, organisation_id: str, plugin_id: str): - self._assert_settings_match_schema(settings, organisation_id, plugin_id) + self._assert_settings_match_schema(settings, plugin_id) self._put_boefje(plugin_id) return self.config_storage.upsert(organisation_id, plugin_id, settings=settings) @@ -122,14 +122,14 @@ def _put_boefje(self, boefje_id: str) -> None: try: self.plugin_storage.boefje_by_id(boefje_id) - except PluginNotFound: + except PluginNotFound as e: try: plugin = self.local_repo.by_id(boefje_id) except KeyError: - raise + raise e if plugin.type != "boefje": - raise + raise e self.plugin_storage.create_boefje(plugin) def _put_normalizer(self, normalizer_id: str) -> None: @@ -150,12 +150,7 @@ def _put_normalizer(self, normalizer_id: str) -> None: def delete_settings(self, organisation_id: str, plugin_id: str): self.config_storage.delete(organisation_id, plugin_id) - try: - self._assert_settings_match_schema({}, organisation_id, plugin_id) - except SettingsNotConformingToSchema: - logger.warning("Making sure %s is disabled for %s because settings are deleted", plugin_id, organisation_id) - - self.set_enabled_by_id(plugin_id, organisation_id, False) + # We don't check the schema anymore because we can provide entries through the global environment as well def schema(self, plugin_id: str) -> dict | None: try: @@ -184,9 +179,7 @@ def description(self, plugin_id: str, organisation_id: str) -> str: return "" def set_enabled_by_id(self, plugin_id: str, organisation_id: str, enabled: bool): - if enabled: - all_settings = self.get_all_settings(organisation_id, plugin_id) - self._assert_settings_match_schema(all_settings, organisation_id, plugin_id) + # We don't check the schema anymore because we can provide entries through the global environment as well try: self._put_boefje(plugin_id) @@ -195,14 +188,14 @@ def set_enabled_by_id(self, plugin_id: str, organisation_id: str, enabled: bool) self.config_storage.upsert(organisation_id, plugin_id, enabled=enabled) - def _assert_settings_match_schema(self, all_settings: dict, organisation_id: str, plugin_id: str): + def _assert_settings_match_schema(self, all_settings: dict, plugin_id: str): schema = self.schema(plugin_id) if schema: # No schema means that there is nothing to assert try: validate(instance=all_settings, schema=schema) except ValidationError as e: - raise SettingsNotConformingToSchema(organisation_id, plugin_id, e.message) from e + raise SettingsNotConformingToSchema(plugin_id, e.message) from e def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> PluginType: with contextlib.suppress(KeyError, NotFound): diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index a7e0d4feca1..8b0d979d256 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -7,6 +7,8 @@ import httpx import structlog from httpx import HTTPError +from jsonschema.exceptions import ValidationError +from jsonschema.validators import validate from boefjes.clients.bytes_client import BytesAPIClient from boefjes.config import settings @@ -15,6 +17,7 @@ from boefjes.local_repository import LocalPluginRepository from boefjes.plugins.models import _default_mime_types from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner +from boefjes.storage.interfaces import SettingsNotConformingToSchema from octopoes.api.models import Affirmation, Declaration, Observation from octopoes.connector.octopoes import OctopoesAPIConnector from octopoes.models import Reference, ScanLevel @@ -35,7 +38,7 @@ def get_octopoes_api_connector(org_code: str) -> OctopoesAPIConnector: return OctopoesAPIConnector(str(settings.octopoes_api), org_code) -def get_environment_settings(boefje_meta: BoefjeMeta, environment_keys: list[str]) -> dict[str, str]: +def get_environment_settings(boefje_meta: BoefjeMeta, schema: dict | None = None) -> dict[str, str]: try: katalogus_api = str(settings.katalogus_api).rstrip("/") response = httpx.get( @@ -43,22 +46,34 @@ def get_environment_settings(boefje_meta: BoefjeMeta, environment_keys: list[str timeout=30, ) response.raise_for_status() - environment = response.json() - - # Add prefixed BOEFJE_* global environment variables - for key, value in os.environ.items(): - if key.startswith("BOEFJE_"): - katalogus_key = key.split("BOEFJE_", 1)[1] - # Only pass the environment variable if it is not explicitly set through the katalogus, - # if and only if they are defined in boefje.json - if katalogus_key in environment_keys and katalogus_key not in environment: - environment[katalogus_key] = value - - return {k: str(v) for k, v in environment.items() if k in environment_keys} except HTTPError: logger.exception("Error getting environment settings") raise + allowed_keys = schema.get("properties", []) if schema else [] + new_env = { + key.split("BOEFJE_", 1)[1]: value + for key, value in os.environ.items() + if key.startswith("BOEFJE_") and key in allowed_keys + } + + settings_from_katalogus = response.json() + + for key, value in settings_from_katalogus.items(): + if key in allowed_keys: + new_env[key] = value + + # The schema, besides dictating that a boefje cannot run if it is not matched, also provides an extra safeguard: + # it is possible to inject code if arguments are passed that "escape" the call to a tool. Hence, we should enforce + # the schema somewhere and make the schema as strict as possible. + if schema is not None: + try: + validate(instance=new_env, schema=schema) + except ValidationError as e: + raise SettingsNotConformingToSchema(boefje_meta.boefje.id, e.message) from e + + return new_env + class BoefjeHandler(Handler): def __init__( @@ -97,10 +112,8 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: boefje_meta.arguments["input"] = ooi.serialize() - env_keys = boefje_resource.environment_keys - boefje_meta.runnable_hash = boefje_resource.runnable_hash - boefje_meta.environment = get_environment_settings(boefje_meta, env_keys) if env_keys else {} + boefje_meta.environment = get_environment_settings(boefje_meta, boefje_resource.schema) mime_types = _default_mime_types(boefje_meta.boefje) diff --git a/boefjes/boefjes/katalogus/plugins.py b/boefjes/boefjes/katalogus/plugins.py index 134243ad963..6ecc4b031db 100644 --- a/boefjes/boefjes/katalogus/plugins.py +++ b/boefjes/boefjes/katalogus/plugins.py @@ -123,7 +123,6 @@ class BoefjeIn(BaseModel): version: str | None = None created: datetime.datetime | None = None description: str | None = None - environment_keys: list[str] = Field(default_factory=list) scan_level: int = 1 consumes: set[str] = Field(default_factory=set) produces: set[str] = Field(default_factory=set) @@ -167,7 +166,6 @@ class NormalizerIn(BaseModel): version: str | None = None created: datetime.datetime | None = None description: str | None = None - environment_keys: list[str] = Field(default_factory=list) consumes: list[str] = Field(default_factory=list) # mime types (and/ or boefjes) produces: list[str] = Field(default_factory=list) # oois diff --git a/boefjes/boefjes/migrations/versions/870fc302b852_remove_environment_keys_field.py b/boefjes/boefjes/migrations/versions/870fc302b852_remove_environment_keys_field.py new file mode 100644 index 00000000000..7bdfbd9e024 --- /dev/null +++ b/boefjes/boefjes/migrations/versions/870fc302b852_remove_environment_keys_field.py @@ -0,0 +1,37 @@ +"""Remove environment keys field + +Revision ID: 870fc302b852 +Revises: 5be152459a7b +Create Date: 2024-08-20 06:08:20.943924 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "870fc302b852" +down_revision = "5be152459a7b" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("boefje", "environment_keys") + op.drop_column("normalizer", "environment_keys") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "normalizer", + sa.Column("environment_keys", postgresql.ARRAY(sa.VARCHAR(length=128)), autoincrement=False, nullable=False), + ) + op.add_column( + "boefje", + sa.Column("environment_keys", postgresql.ARRAY(sa.VARCHAR(length=128)), autoincrement=False, nullable=False), + ) + # ### end Alembic commands ### diff --git a/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py b/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py index 40a44d504d3..d46f360b703 100644 --- a/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py +++ b/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py @@ -95,7 +95,7 @@ def upgrade() -> None: str(boefje.scan_level), list(boefje.consumes), list(boefje.produces), - boefje.environment_keys, + ["TEST_KEY"], boefje.oci_image, boefje.oci_arguments, boefje.version, @@ -137,7 +137,7 @@ def upgrade() -> None: str(boefje.scan_level), list(boefje.consumes), list(boefje.produces), - boefje.environment_keys, + ["TEST_KEY"], boefje.oci_image, boefje.oci_arguments, boefje.version, @@ -177,7 +177,7 @@ def upgrade() -> None: normalizer.description, normalizer.consumes, normalizer.produces, - normalizer.environment_keys, + ["TEST_KEY"], normalizer.version, ) for normalizer in normalizers_to_insert diff --git a/boefjes/boefjes/models.py b/boefjes/boefjes/models.py index 4881b26008a..028d1a1c9a6 100644 --- a/boefjes/boefjes/models.py +++ b/boefjes/boefjes/models.py @@ -17,7 +17,6 @@ class Plugin(BaseModel): version: str | None = None created: datetime.datetime | None = None description: str | None = None - environment_keys: list[str] = Field(default_factory=list) enabled: bool = False static: bool = True # We need to differentiate between local and remote plugins to know which ones can be deleted diff --git a/boefjes/boefjes/plugins/kat_binaryedge/boefje.json b/boefjes/boefjes/plugins/kat_binaryedge/boefje.json index 9dc2a85d8fb..e7d90e4ee98 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/boefje.json +++ b/boefjes/boefjes/plugins/kat_binaryedge/boefje.json @@ -6,8 +6,5 @@ "IPAddressV4", "IPAddressV6" ], - "environment_keys": [ - "BINARYEDGE_API" - ], "scan_level": 2 } diff --git a/boefjes/boefjes/plugins/kat_censys/boefje.json b/boefjes/boefjes/plugins/kat_censys/boefje.json index 6aadac16fba..ef6c3ab9a67 100644 --- a/boefjes/boefjes/plugins/kat_censys/boefje.json +++ b/boefjes/boefjes/plugins/kat_censys/boefje.json @@ -6,9 +6,5 @@ "IPAddressV4", "IPAddressV6" ], - "environment_keys": [ - "CENSYS_API_ID", - "CENSYS_API_SECRET" - ], "scan_level": 1 } diff --git a/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json b/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json index 280ea27e565..f1315d93c33 100644 --- a/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json +++ b/boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json @@ -5,9 +5,6 @@ "consumes": [ "CVEFindingType" ], - "environment_keys": [ - "CVEAPI_URL" - ], "scan_level": 0, "enabled": true } diff --git a/boefjes/boefjes/plugins/kat_dns/boefje.json b/boefjes/boefjes/plugins/kat_dns/boefje.json index 53391f0155d..5773364b9b6 100644 --- a/boefjes/boefjes/plugins/kat_dns/boefje.json +++ b/boefjes/boefjes/plugins/kat_dns/boefje.json @@ -5,9 +5,5 @@ "consumes": [ "Hostname" ], - "environment_keys": [ - "RECORD_TYPES", - "REMOTE_NS" - ], "scan_level": 1 } diff --git a/boefjes/boefjes/plugins/kat_external_db/boefje.json b/boefjes/boefjes/plugins/kat_external_db/boefje.json index 4de34b597ac..cbf7ee0c927 100644 --- a/boefjes/boefjes/plugins/kat_external_db/boefje.json +++ b/boefjes/boefjes/plugins/kat_external_db/boefje.json @@ -5,12 +5,5 @@ "consumes": [ "Network" ], - "environment_keys": [ - "DB_URL", - "DB_ACCESS_TOKEN", - "DB_ORGANIZATION_IDENTIFIER", - "DB_ENDPOINT_FORMAT", - "REQUESTS_CA_BUNDLE" - ], "scan_level": 0 } diff --git a/boefjes/boefjes/plugins/kat_leakix/boefje.json b/boefjes/boefjes/plugins/kat_leakix/boefje.json index 6bbc19c76ee..173f1b3169d 100644 --- a/boefjes/boefjes/plugins/kat_leakix/boefje.json +++ b/boefjes/boefjes/plugins/kat_leakix/boefje.json @@ -7,8 +7,5 @@ "IPAddressV6", "Hostname" ], - "environment_keys": [ - "LEAKIX_API" - ], "scan_level": 1 } diff --git a/boefjes/boefjes/plugins/kat_log4shell/boefje.json b/boefjes/boefjes/plugins/kat_log4shell/boefje.json index 1841a2c6d9a..40a9b6e9991 100644 --- a/boefjes/boefjes/plugins/kat_log4shell/boefje.json +++ b/boefjes/boefjes/plugins/kat_log4shell/boefje.json @@ -5,8 +5,5 @@ "consumes": [ "Hostname" ], - "environment_keys": [ - "REPLY_FQDN" - ], "scan_level": 4 } diff --git a/boefjes/boefjes/plugins/kat_masscan/boefje.json b/boefjes/boefjes/plugins/kat_masscan/boefje.json index b0681d7ba2a..fbdc03a6093 100644 --- a/boefjes/boefjes/plugins/kat_masscan/boefje.json +++ b/boefjes/boefjes/plugins/kat_masscan/boefje.json @@ -5,9 +5,5 @@ "consumes": [ "IPV4NetBlock" ], - "environment_keys": [ - "PORTS", - "MAX_RATE" - ], "scan_level": 2 } diff --git a/boefjes/boefjes/plugins/kat_maxmind_geoip/boefje.json b/boefjes/boefjes/plugins/kat_maxmind_geoip/boefje.json index bd0386c9c20..eaeae4d2095 100644 --- a/boefjes/boefjes/plugins/kat_maxmind_geoip/boefje.json +++ b/boefjes/boefjes/plugins/kat_maxmind_geoip/boefje.json @@ -6,9 +6,5 @@ "IPAddressV4", "IPAddressV6" ], - "scan_level": 1, - "environment_keys": [ - "MAXMIND_USER_ID", - "MAXMIND_LICENCE_KEY" - ] + "scan_level": 1 } diff --git a/boefjes/boefjes/plugins/kat_nmap_ip_range/boefje.json b/boefjes/boefjes/plugins/kat_nmap_ip_range/boefje.json index 4b4dc1a2d06..fe3baf38eb7 100644 --- a/boefjes/boefjes/plugins/kat_nmap_ip_range/boefje.json +++ b/boefjes/boefjes/plugins/kat_nmap_ip_range/boefje.json @@ -6,11 +6,5 @@ "IPV6NetBlock", "IPV4NetBlock" ], - "environment_keys": [ - "TOP_PORTS_TCP", - "TOP_PORTS_UDP", - "MIN_VLSM_IPV4", - "MIN_VLSM_IPV6" - ], "scan_level": 2 } diff --git a/boefjes/boefjes/plugins/kat_nmap_ports/boefje.json b/boefjes/boefjes/plugins/kat_nmap_ports/boefje.json index 506b6ffd478..87fdb592a06 100644 --- a/boefjes/boefjes/plugins/kat_nmap_ports/boefje.json +++ b/boefjes/boefjes/plugins/kat_nmap_ports/boefje.json @@ -6,8 +6,5 @@ "IPAddressV4", "IPAddressV6" ], - "environment_keys": [ - "PORTS" - ], "scan_level": 2 } diff --git a/boefjes/boefjes/plugins/kat_nmap_tcp/boefje.json b/boefjes/boefjes/plugins/kat_nmap_tcp/boefje.json index f3d46684aeb..2ab5f018393 100644 --- a/boefjes/boefjes/plugins/kat_nmap_tcp/boefje.json +++ b/boefjes/boefjes/plugins/kat_nmap_tcp/boefje.json @@ -6,9 +6,6 @@ "IPAddressV4", "IPAddressV6" ], - "environment_keys": [ - "TOP_PORTS" - ], "scan_level": 2, "oci_image": "ghcr.io/minvws/openkat/nmap:latest", "oci_arguments": [ diff --git a/boefjes/boefjes/plugins/kat_nmap_udp/boefje.json b/boefjes/boefjes/plugins/kat_nmap_udp/boefje.json index f9839e53e37..671ff93fe2a 100644 --- a/boefjes/boefjes/plugins/kat_nmap_udp/boefje.json +++ b/boefjes/boefjes/plugins/kat_nmap_udp/boefje.json @@ -6,9 +6,6 @@ "IPAddressV4", "IPAddressV6" ], - "environment_keys": [ - "TOP_PORTS_UDP" - ], "scan_level": 2, "oci_image": "ghcr.io/minvws/openkat/nmap:latest", "oci_arguments": [ diff --git a/boefjes/boefjes/plugins/kat_shodan/boefje.json b/boefjes/boefjes/plugins/kat_shodan/boefje.json index ad1e1c3787e..c1309e7e1ac 100644 --- a/boefjes/boefjes/plugins/kat_shodan/boefje.json +++ b/boefjes/boefjes/plugins/kat_shodan/boefje.json @@ -6,8 +6,5 @@ "IPAddressV4", "IPAddressV6" ], - "environment_keys": [ - "SHODAN_API" - ], "scan_level": 1 } diff --git a/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/boefje.json b/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/boefje.json index 91a7d6f9994..92626c95337 100644 --- a/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/boefje.json +++ b/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/boefje.json @@ -5,8 +5,5 @@ "consumes": [ "IPService" ], - "scan_level": 2, - "environment_keys": [ - "TIMEOUT" - ] + "scan_level": 2 } diff --git a/boefjes/boefjes/plugins/kat_webpage_analysis/boefje.json b/boefjes/boefjes/plugins/kat_webpage_analysis/boefje.json index c6c12b6e73c..e284ad6f61b 100644 --- a/boefjes/boefjes/plugins/kat_webpage_analysis/boefje.json +++ b/boefjes/boefjes/plugins/kat_webpage_analysis/boefje.json @@ -141,8 +141,5 @@ "video/x-msvideo", "video/x-sgi-movie" ], - "environment_keys": [ - "USERAGENT" - ], "scan_level": 2 } diff --git a/boefjes/boefjes/plugins/kat_wpscan/boefje.json b/boefjes/boefjes/plugins/kat_wpscan/boefje.json index 83658bb2d56..9c1d0b343da 100644 --- a/boefjes/boefjes/plugins/kat_wpscan/boefje.json +++ b/boefjes/boefjes/plugins/kat_wpscan/boefje.json @@ -5,8 +5,5 @@ "consumes": [ "SoftwareInstance" ], - "environment_keys": [ - "WP_SCAN_API" - ], "scan_level": 2 } diff --git a/boefjes/boefjes/sql/db_models.py b/boefjes/boefjes/sql/db_models.py index e8e9740f8f4..f9b32a9a9ea 100644 --- a/boefjes/boefjes/sql/db_models.py +++ b/boefjes/boefjes/sql/db_models.py @@ -75,7 +75,6 @@ class BoefjeInDB(SQL_BASE): # Job specifications consumes = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) produces = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) - environment_keys = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) schema = Column(types.JSON(), nullable=True) # Image specifications @@ -99,5 +98,4 @@ class NormalizerInDB(SQL_BASE): # Job specifications consumes = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) produces = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) - environment_keys = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) version = Column(types.String(length=16), nullable=True) diff --git a/boefjes/boefjes/sql/plugin_storage.py b/boefjes/boefjes/sql/plugin_storage.py index c9bcffbaa2f..263d035ab76 100644 --- a/boefjes/boefjes/sql/plugin_storage.py +++ b/boefjes/boefjes/sql/plugin_storage.py @@ -111,7 +111,6 @@ def to_boefje_in_db(boefje: Boefje) -> BoefjeInDB: consumes=boefje.consumes, produces=boefje.produces, schema=boefje.schema, - environment_keys=boefje.environment_keys, oci_image=boefje.oci_image, oci_arguments=boefje.oci_arguments, version=boefje.version, @@ -127,7 +126,6 @@ def to_normalizer_in_db(normalizer: Normalizer) -> NormalizerInDB: description=normalizer.description, consumes=normalizer.consumes, produces=normalizer.produces, - environment_keys=normalizer.environment_keys, version=normalizer.version, static=normalizer.static, ) @@ -144,7 +142,6 @@ def to_boefje(boefje_in_db: BoefjeInDB) -> Boefje: consumes=boefje_in_db.consumes, produces=boefje_in_db.produces, schema=boefje_in_db.schema, - environment_keys=boefje_in_db.environment_keys, oci_image=boefje_in_db.oci_image, oci_arguments=boefje_in_db.oci_arguments, version=boefje_in_db.version, @@ -161,7 +158,6 @@ def to_normalizer(normalizer_in_db: NormalizerInDB) -> Normalizer: description=normalizer_in_db.description, consumes=normalizer_in_db.consumes, produces=normalizer_in_db.produces, - environment_keys=normalizer_in_db.environment_keys, version=normalizer_in_db.version, static=normalizer_in_db.static, ) diff --git a/boefjes/boefjes/storage/interfaces.py b/boefjes/boefjes/storage/interfaces.py index ff19ca7c9f8..3f3d58e4a63 100644 --- a/boefjes/boefjes/storage/interfaces.py +++ b/boefjes/boefjes/storage/interfaces.py @@ -11,11 +11,8 @@ def __init__(self, message: str): class SettingsNotConformingToSchema(StorageError): - def __init__(self, organisation_id: str, plugin_id: str, validation_error: str): - super().__init__( - f"Settings for organisation {organisation_id} and plugin {plugin_id} are not conform the plugin schema: " - f"{validation_error}" - ) + def __init__(self, plugin_id: str, validation_error: str): + super().__init__(f"Settings for plugin {plugin_id} are not conform the plugin schema: {validation_error}") class NotFound(StorageError): diff --git a/boefjes/pyproject.toml b/boefjes/pyproject.toml index 31c11e2cf98..b1ca215be31 100644 --- a/boefjes/pyproject.toml +++ b/boefjes/pyproject.toml @@ -87,6 +87,7 @@ line-length = 120 transform-concats = true [tool.pytest.ini_options] +markers = ["slow: marks tests as slow"] addopts = "-m 'not slow'" env = [ "D:KATALOGUS_DB_URI=postgresql://postgres:postgres@ci_katalogus-db:5432/ci_katalogus", diff --git a/boefjes/tests/conftest.py b/boefjes/tests/conftest.py index d94dc074344..079185699ed 100644 --- a/boefjes/tests/conftest.py +++ b/boefjes/tests/conftest.py @@ -16,12 +16,16 @@ from boefjes.clients.bytes_client import BytesAPIClient from boefjes.clients.scheduler_client import Queue, SchedulerClientInterface, Task, TaskStatus from boefjes.config import Settings, settings +from boefjes.dependencies.plugins import PluginService from boefjes.job_handler import bytes_api_client from boefjes.job_models import BoefjeMeta, NormalizerMeta +from boefjes.local_repository import get_local_repository from boefjes.models import Organisation from boefjes.runtime_interfaces import Handler, WorkerManager +from boefjes.sql.config_storage import SQLConfigStorage, create_encrypter from boefjes.sql.db import SQL_BASE, get_engine from boefjes.sql.organisation_storage import SQLOrganisationStorage +from boefjes.sql.plugin_storage import SQLPluginStorage from octopoes.api.models import Declaration, Observation from octopoes.connector.octopoes import OctopoesAPIConnector from octopoes.models import OOI @@ -142,21 +146,47 @@ def api(tmp_path): @pytest.fixture -def organisation_repository(): +def session(): engine = get_engine() session = sessionmaker(bind=engine)() - yield SQLOrganisationStorage(session, settings) + yield session session.execute(";".join([f"TRUNCATE TABLE {t} CASCADE" for t in SQL_BASE.metadata.tables])) + session.commit() session.close() @pytest.fixture -def organisation(organisation_repository) -> Organisation: +def organisation_storage(session): + return SQLOrganisationStorage(session, settings) + + +@pytest.fixture +def config_storage(session): + return SQLConfigStorage(session, create_encrypter()) + + +@pytest.fixture +def plugin_storage(session): + return SQLPluginStorage(session, settings) + + +@pytest.fixture +def local_repo(): + return get_local_repository() + + +@pytest.fixture +def plugin_service(plugin_storage, config_storage, local_repo): + return PluginService(plugin_storage, config_storage, local_repo) + + +@pytest.fixture +def organisation(organisation_storage) -> Organisation: organisation = Organisation(id="test", name="Test org") - with organisation_repository as repo: + with organisation_storage as repo: repo.create(organisation) return organisation diff --git a/boefjes/tests/integration/test_bench.py b/boefjes/tests/integration/test_bench.py index a77d5732092..14123016904 100644 --- a/boefjes/tests/integration/test_bench.py +++ b/boefjes/tests/integration/test_bench.py @@ -19,7 +19,7 @@ def test_migration( octopoes_api_connector: OctopoesAPIConnector, bytes_client: BytesAPIClient, - organisation_repository: SQLOrganisationStorage, + organisation_storage: SQLOrganisationStorage, valid_time, ): octopoes_api_connector.session._timeout.connect = 60 @@ -87,7 +87,7 @@ def test_migration( bytes_client.save_normalizer_meta(normalizer_meta) total_oois = octopoes_api_connector.list_objects(set(), valid_time).count - total_processed, total_failed = upgrade(organisation_repository, valid_time) + total_processed, total_failed = upgrade(organisation_storage, valid_time) assert total_processed == len(hostname_range) assert total_failed == 0 diff --git a/boefjes/tests/integration/test_get_environment.py b/boefjes/tests/integration/test_get_environment.py new file mode 100644 index 00000000000..5aa209c0493 --- /dev/null +++ b/boefjes/tests/integration/test_get_environment.py @@ -0,0 +1,22 @@ +import pytest + +from boefjes.dependencies.plugins import PluginService +from boefjes.job_handler import get_environment_settings +from boefjes.models import Organisation +from tests.loading import get_boefje_meta + + +@pytest.mark.skipif("os.environ.get('CI') != '1'") +def test_environment_builds_up_correctly(plugin_service: PluginService, organisation: Organisation): + plugin_id = "dns-records" + schema = plugin_service.schema(plugin_id) + environment = get_environment_settings(get_boefje_meta(boefje_id=plugin_id), schema) + + assert environment == {} + + with plugin_service: + plugin_service.upsert_settings({"RECORD_TYPES": "CNAME,AAAA", "WRONG": "3"}, organisation.id, plugin_id) + + environment = get_environment_settings(get_boefje_meta(boefje_id=plugin_id), schema) + + assert environment == {"RECORD_TYPES": "CNAME,AAAA"} diff --git a/boefjes/tests/integration/test_migration_add_schema_field.py b/boefjes/tests/integration/test_migration_add_schema_field.py index bbd66fbafa7..158e99cce34 100644 --- a/boefjes/tests/integration/test_migration_add_schema_field.py +++ b/boefjes/tests/integration/test_migration_add_schema_field.py @@ -30,7 +30,7 @@ def setUp(self) -> None: str(boefje.scan_level), list(sorted(boefje.consumes)), list(sorted(boefje.produces)), - boefje.environment_keys, + ["RECORD_TYPES", "REMOTE_NS"], boefje.oci_image, boefje.oci_arguments, boefje.version, @@ -71,7 +71,7 @@ def test_fail_on_wrong_plugin_ids(self): "2", ["IPAddressV4", "IPAddressV6"], ["boefje/nmap-udp"], - ["TOP_PORTS_UDP"], + ["RECORD_TYPES", "REMOTE_NS"], "ghcr.io/minvws/openkat/nmap:latest", ["--open", "-T4", "-Pn", "-r", "-v10", "-sV", "-sU"], None, @@ -144,7 +144,7 @@ def test_fail_on_wrong_plugin_ids(self): "2", ["IPAddressV4", "IPAddressV6"], ["boefje/nmap-udp"], - ["TOP_PORTS_UDP"], + ["RECORD_TYPES", "REMOTE_NS"], "ghcr.io/minvws/openkat/nmap:latest", ["--open", "-T4", "-Pn", "-r", "-v10", "-sV", "-sU"], None, diff --git a/boefjes/tests/integration/test_sql_repositories.py b/boefjes/tests/integration/test_sql_repositories.py index ad4f804ad10..4dbe5d5c991 100644 --- a/boefjes/tests/integration/test_sql_repositories.py +++ b/boefjes/tests/integration/test_sql_repositories.py @@ -194,7 +194,6 @@ def test_rich_boefje_storage(self): version="v1.09", created=datetime.datetime(2010, 10, 10, 10, 10, 10, tzinfo=datetime.UTC), description="My Boefje", - environment_keys=["api_key", "TOKEN"], scan_level=4, consumes=["Internet"], produces=[ @@ -244,7 +243,6 @@ def test_rich_normalizer_storage(self): version="v1.19", created=datetime.datetime(2010, 10, 10, 10, 10, 10, tzinfo=datetime.UTC), description="My Normalizer", - environment_keys=["api_key", "TOKEN"], scan_level=4, consumes=["Internet"], produces=[ diff --git a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/boefje.json b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/boefje.json index 518d9804e4b..321d8541364 100644 --- a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/boefje.json +++ b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/boefje.json @@ -5,6 +5,5 @@ "consumes": [ "DNSZone" ], - "environment_keys": [], "scan_level": 1 } diff --git a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_2/boefje.json b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_2/boefje.json index 07c01db4d21..67dc80c70bf 100644 --- a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_2/boefje.json +++ b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_2/boefje.json @@ -8,6 +8,5 @@ "produces": [ "text/html" ], - "environment_keys": [], "scan_level": 1 } diff --git a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_4/boefje.json b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_4/boefje.json index b00b312a437..d419ffd073f 100644 --- a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_4/boefje.json +++ b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_4/boefje.json @@ -8,6 +8,5 @@ "produces": [ "text/html" ], - "environment_keys": [], "scan_level": 1 } diff --git a/boefjes/tests/katalogus/test_plugin_service.py b/boefjes/tests/katalogus/test_plugin_service.py index a86b4aa6fbf..acb892eedee 100644 --- a/boefjes/tests/katalogus/test_plugin_service.py +++ b/boefjes/tests/katalogus/test_plugin_service.py @@ -55,27 +55,13 @@ def test_update_by_id(self): def test_update_by_id_bad_schema(self): plugin_id = "kat_test" - with self.assertRaises(SettingsNotConformingToSchema) as ctx: - self.service.set_enabled_by_id(plugin_id, self.organisation, True) - - msg = ( - "Settings for organisation test and plugin kat_test are not conform the plugin schema: 'api_key' is a " - "required property" - ) - self.assertEqual(ctx.exception.message, msg) - self.service.config_storage.upsert(self.organisation, plugin_id, {"api_key": 128 * "a"}) self.service.set_enabled_by_id(plugin_id, self.organisation, True) - value = 129 * "a" - self.service.config_storage.upsert(self.organisation, plugin_id, {"api_key": 129 * "a"}) with self.assertRaises(SettingsNotConformingToSchema) as ctx: - self.service.set_enabled_by_id(plugin_id, self.organisation, True) + self.service.upsert_settings({"api_key": 129 * "a"}, self.organisation, plugin_id) - msg = ( - f"Settings for organisation test and plugin kat_test are not conform the plugin schema: " - f"'{value}' is too long" - ) + msg = f"Settings for plugin kat_test are not conform the plugin schema: '{129 * 'a'}' is too long" self.assertEqual(ctx.exception.message, msg) def test_get_schema(self): @@ -93,7 +79,7 @@ def test_get_schema(self): schema = self.service.schema("kat_test_normalize") self.assertIsNone(schema) - def test_removing_mandatory_setting_disables_plugin(self): + def test_removing_mandatory_setting_does_not_disable_plugin_anymore(self): plugin_id = "kat_test" self.service.config_storage.upsert(self.organisation, plugin_id, {"api_key": 128 * "a"}) @@ -105,20 +91,17 @@ def test_removing_mandatory_setting_disables_plugin(self): self.service.delete_settings(self.organisation, plugin_id) plugin = self.service.by_plugin_id(plugin_id, self.organisation) - self.assertFalse(plugin.enabled) + self.assertTrue(plugin.enabled) def test_adding_integer_settings_within_given_constraints(self): plugin_id = "kat_test_2" - self.service.config_storage.upsert(self.organisation, plugin_id, {"api_key": "24"}) - with self.assertRaises(SettingsNotConformingToSchema) as ctx: - self.service.set_enabled_by_id(plugin_id, self.organisation, True) + self.service.upsert_settings({"api_key": "24"}, self.organisation, plugin_id) self.assertIn("'24' is not of type 'integer'", ctx.exception.message) - self.service.config_storage.upsert(self.organisation, plugin_id, {"api_key": 24}) - + self.service.upsert_settings({"api_key": 24}, self.organisation, plugin_id) self.service.set_enabled_by_id(plugin_id, self.organisation, True) plugin = self.service.by_plugin_id(plugin_id, self.organisation) @@ -155,7 +138,6 @@ def test_clone_many_settings(self): all_settings_1 = {"api_key": "123"} self.service.upsert_settings(all_settings_1, self.organisation, plugin_id_1) - self.service.clone_settings_to_organisation(self.organisation, "org2") all_settings_for_new_org = self.service.get_all_settings("org2", plugin_id_1) diff --git a/boefjes/tests/modules/dummy_boefje/boefje.json b/boefjes/tests/modules/dummy_boefje/boefje.json index 791539e798f..81b94057368 100644 --- a/boefjes/tests/modules/dummy_boefje/boefje.json +++ b/boefjes/tests/modules/dummy_boefje/boefje.json @@ -3,6 +3,5 @@ "name": "dummy", "description": "", "consumes": [], - "produces": [], - "environment_keys": [] + "produces": [] } diff --git a/boefjes/tests/modules/dummy_boefje_environment/boefje.json b/boefjes/tests/modules/dummy_boefje_environment/boefje.json index 838cef1dae7..2ad7fdddefa 100644 --- a/boefjes/tests/modules/dummy_boefje_environment/boefje.json +++ b/boefjes/tests/modules/dummy_boefje_environment/boefje.json @@ -3,6 +3,5 @@ "name": "dummy", "description": "", "consumes": [], - "produces": [], - "environment_keys": [] + "produces": [] } diff --git a/boefjes/tests/modules/dummy_boefje_environment_with_pycache/boefje.json b/boefjes/tests/modules/dummy_boefje_environment_with_pycache/boefje.json index 28f7090b2f0..009f2116c0f 100644 --- a/boefjes/tests/modules/dummy_boefje_environment_with_pycache/boefje.json +++ b/boefjes/tests/modules/dummy_boefje_environment_with_pycache/boefje.json @@ -3,6 +3,5 @@ "name": "dummy", "description": "", "consumes": [], - "produces": [], - "environment_keys": [] + "produces": [] } diff --git a/boefjes/tests/modules/dummy_boefje_invalid_signature/boefje.json b/boefjes/tests/modules/dummy_boefje_invalid_signature/boefje.json index 47f325f09e9..3fca5d08ce7 100644 --- a/boefjes/tests/modules/dummy_boefje_invalid_signature/boefje.json +++ b/boefjes/tests/modules/dummy_boefje_invalid_signature/boefje.json @@ -3,6 +3,5 @@ "name": "dummy", "description": "", "consumes": [], - "produces": [], - "environment_keys": [] + "produces": [] } diff --git a/boefjes/tests/modules/dummy_boefje_missing_run/boefje.json b/boefjes/tests/modules/dummy_boefje_missing_run/boefje.json index 838cef1dae7..2ad7fdddefa 100644 --- a/boefjes/tests/modules/dummy_boefje_missing_run/boefje.json +++ b/boefjes/tests/modules/dummy_boefje_missing_run/boefje.json @@ -3,6 +3,5 @@ "name": "dummy", "description": "", "consumes": [], - "produces": [], - "environment_keys": [] + "produces": [] } diff --git a/boefjes/tests/modules/dummy_boefje_runtime_exception/boefje.json b/boefjes/tests/modules/dummy_boefje_runtime_exception/boefje.json index 197a6aa43f0..e7dcc24f271 100644 --- a/boefjes/tests/modules/dummy_boefje_runtime_exception/boefje.json +++ b/boefjes/tests/modules/dummy_boefje_runtime_exception/boefje.json @@ -3,6 +3,5 @@ "name": "dummy", "description": "", "consumes": [], - "produces": [], - "environment_keys": [] + "produces": [] } diff --git a/boefjes/tests/modules/dummy_oci_boefje_no_main/boefje.json b/boefjes/tests/modules/dummy_oci_boefje_no_main/boefje.json index 2ef96a5e2da..6e8c1bc69b3 100644 --- a/boefjes/tests/modules/dummy_oci_boefje_no_main/boefje.json +++ b/boefjes/tests/modules/dummy_oci_boefje_no_main/boefje.json @@ -4,6 +4,5 @@ "description": "", "consumes": [], "produces": [], - "environment_keys": [], "oci_image": "openkat/test" } diff --git a/boefjes/tests/test_tasks.py b/boefjes/tests/test_tasks.py index f0cbf5fac56..f9c319897a1 100644 --- a/boefjes/tests/test_tasks.py +++ b/boefjes/tests/test_tasks.py @@ -198,6 +198,6 @@ def test_correct_local_runner_hash(self) -> None: assert Path(path / "__pycache__/pytest__init__.cpython-311.pyc").is_file() assert Path(path / "__pycache__/pytest_main.cpython-311.pyc").is_file() - assert boefje_resource_1.runnable_hash == "4bae5e869bd17759bf750bf357fdee1eedff5768d407248b8ddcb63d0abdee19" - assert boefje_resource_2.runnable_hash == "e0c46fb915778b06f69cd5934b2157733cef84d67fc89c563c5bbd965ad52949" - assert boefje_resource_3.runnable_hash == "0185c90d3d1a4dc1490ec918374f84e8a480101f98db14d434638147dd82c626" + assert boefje_resource_1.runnable_hash == "7450ebc13f6856df925e90cd57f2769468a39723f18ba835749982b484564ec9" + assert boefje_resource_2.runnable_hash == "874e154b572a0315cfe4329bd3b756bf9cad77f6a87bb9b9b9bb6296f1d4b520" + assert boefje_resource_3.runnable_hash == "70c0b0ad3b2e70fd79e52dcf043096a50ed69db1359df0011499e66ab1510bbe" diff --git a/docs/source/developer_documentation/boefjes.md b/docs/source/developer_documentation/boefjes.md index 553dfddd74e..2a8c4737138 100644 --- a/docs/source/developer_documentation/boefjes.md +++ b/docs/source/developer_documentation/boefjes.md @@ -51,13 +51,13 @@ although this would be more complicated. ## Environment variables By design, Boefjes do not have access to the host system's environment variables. -If a Boefje requires access to an environment variable (e.g. `HTTP_PROXY` or `USER_AGENT`), it should note as such in its `boefje.json` manifest. +If a Boefje requires access to an environment variable (e.g. `HTTP_PROXY` or `USER_AGENT`), it should note as such in its `schema.json`. The system-wide variables can be set as environment variable to the boefjes runner by prefixing it with `BOEFJE_`. This is to prevent a Boefje from accessing variables it should not have access to, such as secrets. To illustrate: if `BOEFJE_HTTP_PROXY=https://proxy:8080` environment variable is configured, the Boefje can access it as `HTTP_PROXY`. This feature can also be used to set default values for KAT-alogus settings. For example, configuring the `BOEFJE_TOP_PORTS` environment variable will set the default value for the `TOP_PORTS` setting (used by the nmap Boefje). -This default value can be overridden by setting any value for `TOP_PORTS` in the KAT-alogus. +This default value can be overridden per organisation by setting any value for `TOP_PORTS` in the KAT-alogus. ## Technical Design diff --git a/docs/source/developer_documentation/development_tutorial/creating_a_boefje.md b/docs/source/developer_documentation/development_tutorial/creating_a_boefje.md index dbb2f3bdc70..65e4d750f18 100644 --- a/docs/source/developer_documentation/development_tutorial/creating_a_boefje.md +++ b/docs/source/developer_documentation/development_tutorial/creating_a_boefje.md @@ -28,7 +28,6 @@ This file contains information about our boefje. For example, this file contains "name": "Hello Katty", "description": "A simple boefje that can say hello", "consumes": ["IPAddressV4", "IPAddressV6"], - "environment_keys": ["MESSAGE", "NUMBER"], "scan_level": 0, "oci_image": "openkat/hello-katty" } @@ -38,7 +37,6 @@ This file contains information about our boefje. For example, this file contains - **`name`**: A name to display in the KAT-alogus. - **`description`**: A description in the KAT-alogus. - **`consumes`**: A list of OOI types that trigger the boefje to run. Whenever one of these OOIs gets added, this boefje will run with that OOI. In our case, we will run our boefje whenever a new IPAddressV4 or IPAddressV6 gets added. -- **`environment_keys`**: A list of inputs provided by the user. More information about these inputs can be found in `schema.json`. OpenKAT also provides some environment variables. - **`scan_level`**: A scan level that decides how intrusively this boefje will scan the provided OOIs. Since we will not make any external requests our boefje will have a scan level of 0. - **`oci_image`**: The name of the docker image that is provided inside `boefjes/Makefile` @@ -52,8 +50,9 @@ This file contains a description of the boefje to explain to the user what this ## `schema.json` -This JSON is used as the basis for a form for the user. When the user enables this boefje they can get the option to give extra information. For example, it can contain an API key that the script requires. -This is an example of a `schema.json` file: +To allow the user to pass information to a boefje runtime, add a schema.json file to the folder where your boefje is located. +This can be used, for example, to add an API key that the script requires. +It must conform to the https://json-schema.org/ standard, for example: ```json { @@ -78,6 +77,24 @@ This is an example of a `schema.json` file: } ``` +This JSON defines which additional environment variables can be set for the boefje. +There are two ways to do this. +Firstly, using this schema as an example, you could set the `BOEFJE_MESSAGE` environment variable in the boefje runtime. +Prepending the key with `BOEFJE_` provides an extra safeguard. +Note that setting an environment variable means this configuration is applied to _all_ organisations. +Secondly, if you want to avoid setting environment variables or configure it for just one organisation, +it is also possible to set the API key through the KAT-alogus. +Navigate to the boefje detail page of Shodan to find the schema as a form. +These values take precedence over the environment variables. +This is also a way to test whether the schema is properly understood for your boefje. +If encryption has been set up for the KATalogus, all keys provided through this form are stored encrypted in the database. + +Although the Shodan boefje defines an API key, the schema could contain anything your boefje needs. +However, OpenKAT currently officially only supports "string" and "integer" properties that are one level deep. +Because keys may be passed through environment variables, +schema validation does not happen right away when settings are added or boefjes enabled. +Schema validation happens right before spawning a boefje, meaning your tasks will fail if is missing a required variable. + - `title`: This should always contain a string containing 'Arguments'. - `type`: This should always contain a string containing 'object'. - `description`: A description of the boefje explaining in short what it can do. This will both be displayed inside the KAT-alogus and on the boefje's page. diff --git a/docs/source/introduction/makeyourown.rst b/docs/source/introduction/makeyourown.rst index 226b2dfcb5c..04d9979b694 100644 --- a/docs/source/introduction/makeyourown.rst +++ b/docs/source/introduction/makeyourown.rst @@ -15,7 +15,7 @@ What types of plugins are available? There are three types of plugins, deployed by OpenKAT to collect information, translate it into objects for the data model and then analyze it. Boefjes gather facts, Whiskers structure the information for the data model and Bits determine what you want to think about it; they are the business rules. Each action is cut into the smallest possible pieces. -- Boefjes gather factual information, such as by calling an external scanning tool like nmap or using a database like shodan. +- Boefjes gather factual information, such as by calling an external scanning tool like nmap or using a database like Shodan. - Whiskers analyze the information and turn it into objects for the data model in Octopoes. @@ -51,7 +51,7 @@ To make a finding about a CVE to a software version, you need multiple objects: Existing boefjes ================ -The existing boefjes can be viewed via the KATalog in OpenKAT and are on `GitHUB in the boefjes repository. `_ +The existing boefjes can be viewed via the KATalog in OpenKAT and are on `GitHub in the boefjes repository. `_ Object-types, classes and objects. ---------------------------------- @@ -65,7 +65,7 @@ When we talk about objects, we usually mean instance of such a class, or a 'reco Example: the boefje for shodan ------------------------------ -The boefje calling shodan gives a good first impression of its capabilities. The boefje includes the following files. +The boefje calling Shodan gives a good first impression of its capabilities. The boefje includes the following files. - __init.py__, which remains empty - boefje.json, containing the normalizers and object-types in the data model @@ -75,7 +75,7 @@ The boefje calling shodan gives a good first impression of its capabilities. The - normalize.py, the normalizer (whiskers) - normalizer.json, which accepts and supplies the normalizer - requirements.txt, with the requirements for this boefje -- schema.json, settings for the web interface +- schema.json, settings for the web interface formatted as JSON Schema boefje.json *********** @@ -99,7 +99,6 @@ An example: "IPPort", "CVEFindingType" ], - "environment_keys": ["SHODAN_API"], "scan_level": 1 } @@ -111,15 +110,15 @@ Using the template as a base, you can create a boefje.json for your own boefje. NOTE: If your boefje needs object-types that do not exist, you will need to create those. This will be described later in the document. -The boefje also uses variables from the web interface, like the Shodan the API key. There are more possibilities, you can be creative with this and let the end user bring settings from the web interface. Use *environment_keys* for this. The schema.json file defines the metadata for these fields. +The boefje also uses variables from the web interface, like the Shodan the API key. There are more possibilities, you can be creative with this and let the end user bring settings from the web interface. Use environment variables or KATalogus settings for this. The schema.json file defines the metadata for these fields. schema.json *********** -To allow the user to add information through the web interface, add the schema.json file to the folder where your boefje is located. This json is used as the basis for a form for the user. In this case, it can contain an API key, but it can also be something else that your boefje responds to. This Schema must conform to the https://json-schema.org/ standard. - -Currently, however, OpenKAT only understands fairly shallow structures. For example, not all field types are supported, nor does OpenKAT understand references. You can test whether your Schema is neatly understood by checking the settings form in Rocky's KAT catalog for your boefje. +To allow the user to pass information to a boefje runtime, add a schema.json file to the folder where your boefje is located. +This can be used, for example, to add an API key that the script requires. +It must conform to the https://json-schema.org/ standard, see for example the schema.json for Shodan: .. code-block:: json @@ -139,6 +138,24 @@ Currently, however, OpenKAT only understands fairly shallow structures. For exam ] } +This JSON defines which additional environment variables can be set for the boefje. +There are two ways to do this. +Firstly, using the Shodan schema as an example, you could set the ``BOEFJE_SHODAN_API`` environment variable in the boefje runtime. +Prepending the key with ``BOEFJE_`` provides an extra safeguard. +Note that setting an environment variable means this configuration is applied to _all_ organisations. +Secondly, if you want to avoid setting environment variables or configure it for just one organisation, +it is also possible to set the API key through the KAT-alogus. +Navigate to the boefje detail page of Shodan to find the schema as a form. +These values take precedence over the environment variables. +This is also a way to test whether the schema is properly understood for your boefje. +If encryption has been set up for the KATalogus, all keys provided through this form are stored encrypted in the database. + +Although the Shodan boefje defines an API key, the schema could contain anything your boefje needs. +However, OpenKAT currently only officially supports "string" and "integer" properties that are one level deep. +Because keys may be passed through environment variables, +schema validation does not happen right away when settings are added or boefjes enabled. +Schema validation happens right before spawning a boefje, meaning your tasks will fail if is missing a required variable. + main.py ******* @@ -190,8 +207,8 @@ normalizer.json The normalizers translate the output of a boefje into objects that fit the data model. Each normalizer defines what input it accepts and what object-types it provides. -In the case of the shodan normalizer, -it involves the entire output of the shodan boefje (created based on IP address), +In the case of the Shodan normalizer, +it involves the entire output of the Shodan boefje (created based on IP address), where findings and ports come out. The `normalizer.json` defines these: .. code-block:: json @@ -295,7 +312,7 @@ If you want to add an object-type, you need to know with which other object-type Adding object-types to the data model requires an addition in octopus. Here, an object-type can be added if it is connected to other object-types. Visually this is well understood using the `Graph explorer `_. The actual code is `in the Octopoes repo `_. -As with the boefje for shodan, here we again use the example from the functional documentation. A description of an object-type in the data model, in this case an IPPort, looks like this: +As with the boefje for Shodan, here we again use the example from the functional documentation. A description of an object-type in the data model, in this case an IPPort, looks like this: .. code-block:: python @@ -521,4 +538,4 @@ There are a number of ways to add your new boefje to OpenKAT. - Do a commit of your code, after review it can be included - Add an image server in the KAT catalog config file ``*`` -``*`` If you want to add an image server, join the ongoing project to standardize and describe it. The idea is to add an image server in the KAT catalog config file that has artifacts from your boefjes and normalizers as outputted by the Github CI. +``*`` If you want to add an image server, join the ongoing project to standardize and describe it. The idea is to add an image server in the KAT catalog config file that has artifacts from your boefjes and normalizers as outputted by the GitHub CI. diff --git a/mula/scheduler/models/plugin.py b/mula/scheduler/models/plugin.py index f51c4952972..a30908af8a3 100644 --- a/mula/scheduler/models/plugin.py +++ b/mula/scheduler/models/plugin.py @@ -12,7 +12,6 @@ class Plugin(BaseModel): authors: list[str] | None = None created: datetime.datetime | None = None description: str | None = None - environment_keys: list[str] | None = None related: list[str] | None = None scan_level: int | None = None consumes: str | list[str] diff --git a/rocky/katalogus/client.py b/rocky/katalogus/client.py index 9c293a670ba..8bb9f7fc463 100644 --- a/rocky/katalogus/client.py +++ b/rocky/katalogus/client.py @@ -24,7 +24,6 @@ class Plugin(BaseModel): authors: str | None = None created: str | None = None description: str | None = None - environment_keys: list[str] | None = None related: list[str] = Field(default_factory=list) enabled: bool type: str diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index f5bbb28dfac..d9825e7b14e 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -1775,7 +1775,6 @@ def boefje_dns_records(): authors=None, created=None, description="Fetch the DNS record(s) of a hostname", - environment_keys=None, related=[], enabled=True, type="boefje", diff --git a/rocky/tests/stubs/katalogus_boefjes.json b/rocky/tests/stubs/katalogus_boefjes.json index 40c79a6cb1b..d7d6ba63991 100644 --- a/rocky/tests/stubs/katalogus_boefjes.json +++ b/rocky/tests/stubs/katalogus_boefjes.json @@ -5,9 +5,6 @@ "version": null, "created": null, "description": "Use BinaryEdge to find open ports with vulnerabilities that are found on that port", - "environment_keys": [ - "BINARYEDGE_API" - ], "enabled": true, "type": "boefje", "scan_level": 2, @@ -33,7 +30,6 @@ "version": null, "created": null, "description": "Scan SSL certificates of websites", - "environment_keys": [], "enabled": false, "type": "boefje", "scan_level": 1, @@ -51,7 +47,6 @@ "version": null, "created": null, "description": "Scan SSL/TLS versions of websites", - "environment_keys": [], "enabled": false, "type": "boefje", "scan_level": 2, @@ -70,9 +65,6 @@ "version": null, "created": null, "description": "Scan wordpress sites", - "environment_keys": [ - "WP_SCAN_API" - ], "enabled": false, "type": "boefje", "scan_level": 2, @@ -91,7 +83,6 @@ "version": null, "created": null, "description": null, - "environment_keys": [], "enabled": true, "type": "normalizer", "consumes": [ diff --git a/rocky/tests/stubs/katalogus_normalizers.json b/rocky/tests/stubs/katalogus_normalizers.json index 182b5bbb549..dfbca8cb726 100644 --- a/rocky/tests/stubs/katalogus_normalizers.json +++ b/rocky/tests/stubs/katalogus_normalizers.json @@ -5,7 +5,6 @@ "version": null, "created": null, "description": null, - "environment_keys": [], "enabled": true, "type": "normalizer", "consumes": [ @@ -22,7 +21,6 @@ "version": null, "created": null, "description": null, - "environment_keys": [], "enabled": true, "type": "normalizer", "consumes": [ @@ -42,7 +40,6 @@ "version": null, "created": null, "description": null, - "environment_keys": [], "enabled": true, "type": "normalizer", "consumes": [ @@ -59,7 +56,6 @@ "version": null, "created": null, "description": null, - "environment_keys": [], "enabled": true, "type": "normalizer", "consumes": [ @@ -83,7 +79,6 @@ "version": null, "created": null, "description": null, - "environment_keys": [], "enabled": true, "type": "normalizer", "consumes": [ diff --git a/rocky/tests/test_indemnification.py b/rocky/tests/test_indemnification.py index 785679ca7aa..e6d628b323e 100644 --- a/rocky/tests/test_indemnification.py +++ b/rocky/tests/test_indemnification.py @@ -14,7 +14,6 @@ def test_update_clearance_level(rf, client_member, httpx_mock): "authors": None, "created": None, "description": "Use BinaryEdge to find open ports with vulnerabilities that are found on that port", - "environment_keys": ["BINARYEDGE_API"], "related": None, "enabled": True, "type": "boefje", From 1e273324f94cd10ba404e5d234d6ca790e27a8d7 Mon Sep 17 00:00:00 2001 From: HeleenSG Date: Thu, 5 Sep 2024 13:40:26 +0200 Subject: [PATCH 05/28] feat: adds notification styling and icons (#3461) Co-authored-by: Jan Klopper --- rocky/assets/css/components/button-plain.scss | 22 ++++++ .../assets/css/components/notifications.scss | 78 +++++++++++++++++++ rocky/assets/css/main.scss | 1 + .../themes/soft/manon/notification-block.scss | 6 +- 4 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 rocky/assets/css/components/button-plain.scss diff --git a/rocky/assets/css/components/button-plain.scss b/rocky/assets/css/components/button-plain.scss new file mode 100644 index 00000000000..50553d75978 --- /dev/null +++ b/rocky/assets/css/components/button-plain.scss @@ -0,0 +1,22 @@ +button, +a.button, +input[type="button"], +input[type="submit"], +input[type="reset"] { + &.plain { + background-color: transparent; + border: 0; + color: var(--colors-blue-600); + padding: 0; + + &:hover { + color: var(--colors-blue-700); + } + + .icon { + &:hover { + color: var(--colors-blue-700); + } + } + } +} diff --git a/rocky/assets/css/components/notifications.scss b/rocky/assets/css/components/notifications.scss index a6e557fdcb1..83abe718872 100644 --- a/rocky/assets/css/components/notifications.scss +++ b/rocky/assets/css/components/notifications.scss @@ -1,3 +1,25 @@ +/* Block notification */ +div, +section { + &.error, + &.warning, + &.explanation, + &.confirmation, + &.system { + flex-direction: row; + + h1, + h2, + h3, + h4, + h5, + h6 { + font-size: var(--heading-small-font-size); + } + } +} + +/* Help text */ div { &.explanation { &.help-text { @@ -10,3 +32,59 @@ div { margin-top: 0; } } + +/* Inline notification */ +p, +span { + &.error, + &.warning, + &.explanation, + &.confirmation, + &.system { + display: flex; + align-items: center; + } +} + +/* Icons */ +p, +span, +section, +div { + &.error { + &::before { + content: "\eac5"; + color: var(--colors-red-600); + } + } + + &.warning { + &::before { + content: "\ea06"; + color: var(--colors-ochre-500); + } + } + + &.explanation { + &::before { + content: "\eac5"; + color: var(--colors-blue-600); + } + } + + &.confirmation { + &::before { + content: "\ea5e"; + color: var(--colors-green-600); + } + } + + &.system { + background-color: var(--colors-black-05); + + &::before { + content: "\ea06"; + color: var(--colors-blue-600); + } + } +} diff --git a/rocky/assets/css/main.scss b/rocky/assets/css/main.scss index 0e41f9ea21d..d68d91f47e0 100644 --- a/rocky/assets/css/main.scss +++ b/rocky/assets/css/main.scss @@ -29,6 +29,7 @@ /* Components */ @import "components/action-buttons"; @import "components/block-indented"; +@import "components/button-plain"; @import "components/cat-loader"; @import "components/cat-paw-loader"; @import "components/cytoscape"; diff --git a/rocky/assets/css/themes/soft/manon/notification-block.scss b/rocky/assets/css/themes/soft/manon/notification-block.scss index 713d94abde7..45e75d01f6c 100644 --- a/rocky/assets/css/themes/soft/manon/notification-block.scss +++ b/rocky/assets/css/themes/soft/manon/notification-block.scss @@ -1,7 +1,9 @@ /* Notification block - Variables */ :root { - --notification-block-element-padding-right: 0.5rem; - --notification-block-element-padding-left: 0.5rem; + --notification-block-element-padding-top: var(--spacing-grid-300); + --notification-block-element-padding-right: var(--spacing-grid-200); + --notification-block-element-padding-left: var(--spacing-grid-200); + --notification-block-element-padding-bottom: var(--spacing-grid-300); --notification-block-element-gap: 0.5rem; } From 980415375c372e6f32086ce55078e8e5533f456b Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Thu, 5 Sep 2024 15:16:58 +0200 Subject: [PATCH 06/28] Hotfix for normalizer API bug (#3475) Signed-off-by: Donny Peeters --- mula/scheduler/server/handlers/tasks.py | 17 +++++++++++------ mula/tests/integration/test_api.py | 7 +++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/mula/scheduler/server/handlers/tasks.py b/mula/scheduler/server/handlers/tasks.py index db3391dc2d6..6912e7023ab 100644 --- a/mula/scheduler/server/handlers/tasks.py +++ b/mula/scheduler/server/handlers/tasks.py @@ -139,12 +139,16 @@ def list( ] } elif task_type == "normalizer": - f_plugin = storage.filters.Filter( - column="data", - field="normalizer__id", - operator="eq", - value=plugin_id, - ) + f_plugin = { + "and": [ + storage.filters.Filter( + column="data", + field="normalizer__id", + operator="eq", + value=plugin_id, + ) + ] + } else: f_plugin = { "or": [ @@ -182,6 +186,7 @@ def list( detail=f"invalid filter(s) [exception: {exc}]", ) from exc except storage.errors.StorageError as exc: + self.logger.exception(exc) raise fastapi.HTTPException( status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"error occurred while accessing the database [exception: {exc}]", diff --git a/mula/tests/integration/test_api.py b/mula/tests/integration/test_api.py index 0511c200f52..9f6440c5801 100644 --- a/mula/tests/integration/test_api.py +++ b/mula/tests/integration/test_api.py @@ -706,6 +706,13 @@ def test_get_tasks_min_created_at_future(self): self.assertEqual(200, response.status_code) self.assertEqual(0, len(response.json()["results"])) + def test_get_normalizer_filtered(self): + # This used to be a bug where the normalizer filter was missing a nesting on the operator + params = {"task_type": "normalizer", "plugin_id": "kat_nmap_normalize"} + response = self.client.get("/tasks", params=params) + self.assertEqual(200, response.status_code) + self.assertEqual(0, len(response.json()["results"])) + def test_get_tasks_filtered(self): response = self.client.post( "/tasks", From 79bba68f818ee04bec6ee1363ae05a55d8705238 Mon Sep 17 00:00:00 2001 From: HeleenSG Date: Fri, 6 Sep 2024 09:14:58 +0200 Subject: [PATCH 07/28] fix: toggle styling (#3449) Co-authored-by: Jan Klopper --- rocky/assets/css/components/toggle.scss | 70 +++++++++++++++++++ rocky/assets/css/components/toolbar.scss | 1 + rocky/assets/css/main.scss | 1 + .../soft/fundamentals/border-radii.scss | 3 +- .../css/themes/soft/manon/button-icon.scss | 2 + .../assets/css/themes/soft/manon/button.scss | 2 +- .../templates/partials/katalogus_toolbar.html | 6 +- 7 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 rocky/assets/css/components/toggle.scss diff --git a/rocky/assets/css/components/toggle.scss b/rocky/assets/css/components/toggle.scss new file mode 100644 index 00000000000..d2a1dc59717 --- /dev/null +++ b/rocky/assets/css/components/toggle.scss @@ -0,0 +1,70 @@ +button, +a, +input[type="button"], +input[type="submit"], +input[type="reset"] { + $breakpoint: 24rem !default; + + &.toggle-button { + /* Alignment */ + display: flex; + align-items: center; + justify-content: center; + align-self: flex-start; + gap: 0; + margin: 0; + box-sizing: border-box; + + /* Styling */ + border-radius: var(--border-radius-l); + padding: var(--spacing-grid-150); + width: 2.75rem; + height: 2.75rem; + border: 1px solid var(--colors-purrple-300); + + /* Text and icon styling */ + background-color: var(--colors-white); + color: var(--colors-purrple-500); + text-decoration: none; + font-size: 0; + + &::before { + color: var(--colors-purrple-500); + padding: 0; + } + + /* Behaviour */ + cursor: pointer; + overflow-wrap: break-word; + + /* States */ + &[aria-current="page"] { + background-color: var(--colors-purrple-200); + border-color: var(--colors-purrple-200); + + &::before { + color: var(--colors-purrple-700); + } + } + + &:hover, + &:active { + border-color: var(--colors-purrple-500); + } + + &:focus { + outline: 2px solid var(--colors-purrple-500); + outline-offset: 0.125rem; + z-index: 2; + position: relative; + } + } +} + +a { + &.toggle-button { + &:visited { + color: var(--colors-purrple-500); + } + } +} diff --git a/rocky/assets/css/components/toolbar.scss b/rocky/assets/css/components/toolbar.scss index ae148bb349c..f8c3e962c81 100644 --- a/rocky/assets/css/components/toolbar.scss +++ b/rocky/assets/css/components/toolbar.scss @@ -4,6 +4,7 @@ justify-content: flex-end; align-items: center; width: 100%; + overflow: visible; /* Making sure outline is visible on focus */ &.start { justify-content: flex-start; diff --git a/rocky/assets/css/main.scss b/rocky/assets/css/main.scss index d68d91f47e0..f6171506258 100644 --- a/rocky/assets/css/main.scss +++ b/rocky/assets/css/main.scss @@ -65,6 +65,7 @@ @import "components/sticky"; @import "components/state-tags"; @import "components/table"; +@import "components/toggle"; @import "components/toolbar"; @import "components/tree-tables"; @import "components/user-icon"; diff --git a/rocky/assets/css/themes/soft/fundamentals/border-radii.scss b/rocky/assets/css/themes/soft/fundamentals/border-radii.scss index 6ca1b1c20b7..0ac5c9d7060 100644 --- a/rocky/assets/css/themes/soft/fundamentals/border-radii.scss +++ b/rocky/assets/css/themes/soft/fundamentals/border-radii.scss @@ -3,7 +3,8 @@ :root { --border-radius-s: 0.25rem; --border-radius-m: 0.5rem; - --border-radius-l: 1.5rem; + --border-radius-l: 1rem; + --border-radius-xl: 1.5rem; --border-radius-round: 50%; } diff --git a/rocky/assets/css/themes/soft/manon/button-icon.scss b/rocky/assets/css/themes/soft/manon/button-icon.scss index e814863f4a2..d8cff1a5e1e 100644 --- a/rocky/assets/css/themes/soft/manon/button-icon.scss +++ b/rocky/assets/css/themes/soft/manon/button-icon.scss @@ -2,7 +2,9 @@ :root { --button-icon-min-width: 2.75rem; + --button-icon-width: 2.75rem; --button-icon-min-height: 2.75rem; + --button-icon-height: 2.75rem; --button-icon-icon-font-size: 0.8rem; --button-icon-icon-width: 0.75rem; } diff --git a/rocky/assets/css/themes/soft/manon/button.scss b/rocky/assets/css/themes/soft/manon/button.scss index 93c3c716d21..a8260df13ec 100644 --- a/rocky/assets/css/themes/soft/manon/button.scss +++ b/rocky/assets/css/themes/soft/manon/button.scss @@ -4,7 +4,7 @@ /* Button */ --button-base-background-color: var(--branding-color-2-background-color); --button-base-text-color: var(--branding-color-2-text-color); - --button-base-border-radius: 1rem; + --button-base-border-radius: var(--border-radius-l); --button-base-font-family: var(--application-base-font-family); --button-base-padding-top: var(--spacing-grid-150); --button-base-padding-right: var(--spacing-grid-200); diff --git a/rocky/katalogus/templates/partials/katalogus_toolbar.html b/rocky/katalogus/templates/partials/katalogus_toolbar.html index cea35c9210b..96f21fe1c29 100644 --- a/rocky/katalogus/templates/partials/katalogus_toolbar.html +++ b/rocky/katalogus/templates/partials/katalogus_toolbar.html @@ -3,8 +3,10 @@ {% if object_list %}
{% translate "Gridview" %} + class=" icon toggle-button ti-layout-grid" + {% if view_type == "grid" %}aria-current="page"{% endif %}>{% translate "Gridview" %} {% translate "Tableview" %} + class=" icon toggle-button ti-menu-2" + {% if view_type == "table" %}aria-current="page"{% endif %}>{% translate "Tableview" %}
{% endif %} From a7ca00d76b746ae03a2c8328a28793db1611ae5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 10:54:04 +0200 Subject: [PATCH 08/28] Bump cryptography from 42.0.8 to 43.0.1 in /bytes (#3473) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ammar --- boefjes/poetry.lock | 65 +++++++++++++++++------------------- boefjes/pyproject.toml | 2 +- boefjes/requirements-dev.txt | 61 ++++++++++++++++----------------- boefjes/requirements.txt | 61 ++++++++++++++++----------------- bytes/poetry.lock | 65 +++++++++++++++++------------------- bytes/pyproject.toml | 1 + bytes/requirements-dev.txt | 61 ++++++++++++++++----------------- bytes/requirements.txt | 61 ++++++++++++++++----------------- 8 files changed, 174 insertions(+), 203 deletions(-) diff --git a/boefjes/poetry.lock b/boefjes/poetry.lock index fcd021764f5..593f2e7f6d7 100644 --- a/boefjes/poetry.lock +++ b/boefjes/poetry.lock @@ -539,43 +539,38 @@ testing = ["pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "py [[package]] name = "cryptography" -version = "42.0.8" +version = "43.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, - {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, - {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, - {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, - {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, - {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, - {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, + {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, + {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, + {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, + {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, + {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, + {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, + {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, ] [package.dependencies] @@ -588,7 +583,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -3190,4 +3185,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "5065bdc0a65dda4d1d607f6f6fa30f8f3a0bfd6ce11afdc410f044c10d7720e3" +content-hash = "4135f17871ab68dcf30970e2716a8551978ed6782db30b8bbd7858c90e99e0f0" diff --git a/boefjes/pyproject.toml b/boefjes/pyproject.toml index b1ca215be31..2b1f16ad6f9 100644 --- a/boefjes/pyproject.toml +++ b/boefjes/pyproject.toml @@ -45,7 +45,7 @@ python-libnmap = "0.7.3" # required by kat_shodan boefje shodan = "1.25.0" # required by kat_ssl_certificates boefje -cryptography = "^42.0.1" +cryptography = "^43.0.1" # required by kat_webpage_analysis forcediphttpsadapter = "1.1.0" python-wappalyzer = {git = "https://github.com/chorsley/python-Wappalyzer.git", rev = "0.4.0"} diff --git a/boefjes/requirements-dev.txt b/boefjes/requirements-dev.txt index 3d257eb1585..2338137a316 100644 --- a/boefjes/requirements-dev.txt +++ b/boefjes/requirements-dev.txt @@ -288,39 +288,34 @@ colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" \ configparser==7.0.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:af3c618a67aaaedc4d689fd7317d238f566b9aa03cae50102e92d7f0dfe78ba0 \ --hash=sha256:f46d52a12811c637104c6bb8eb33693be0038ab6bf01d69aae009c39ec8c2017 -cryptography==42.0.8 ; python_version >= "3.10" and python_version < "4.0" \ - --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ - --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ - --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ - --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ - --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ - --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ - --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ - --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ - --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ - --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ - --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ - --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ - --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ - --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ - --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ - --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ - --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ - --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ - --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ - --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ - --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ - --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ - --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ - --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ - --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ - --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ - --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ - --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ - --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ - --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ - --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ - --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e +cryptography==43.0.1 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \ + --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \ + --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \ + --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \ + --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \ + --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \ + --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \ + --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \ + --hash=sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84 \ + --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \ + --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \ + --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \ + --hash=sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2 \ + --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \ + --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \ + --hash=sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365 \ + --hash=sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96 \ + --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \ + --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \ + --hash=sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d \ + --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \ + --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \ + --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \ + --hash=sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172 \ + --hash=sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034 \ + --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a \ + --hash=sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289 decorator==5.1.1 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \ --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186 diff --git a/boefjes/requirements.txt b/boefjes/requirements.txt index 4ad7647fb2d..c14689d93c8 100644 --- a/boefjes/requirements.txt +++ b/boefjes/requirements.txt @@ -288,39 +288,34 @@ colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" \ configparser==7.0.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:af3c618a67aaaedc4d689fd7317d238f566b9aa03cae50102e92d7f0dfe78ba0 \ --hash=sha256:f46d52a12811c637104c6bb8eb33693be0038ab6bf01d69aae009c39ec8c2017 -cryptography==42.0.8 ; python_version >= "3.10" and python_version < "4.0" \ - --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ - --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ - --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ - --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ - --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ - --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ - --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ - --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ - --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ - --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ - --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ - --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ - --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ - --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ - --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ - --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ - --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ - --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ - --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ - --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ - --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ - --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ - --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ - --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ - --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ - --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ - --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ - --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ - --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ - --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ - --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ - --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e +cryptography==43.0.1 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \ + --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \ + --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \ + --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \ + --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \ + --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \ + --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \ + --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \ + --hash=sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84 \ + --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \ + --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \ + --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \ + --hash=sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2 \ + --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \ + --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \ + --hash=sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365 \ + --hash=sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96 \ + --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \ + --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \ + --hash=sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d \ + --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \ + --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \ + --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \ + --hash=sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172 \ + --hash=sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034 \ + --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a \ + --hash=sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289 decorator==5.1.1 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \ --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186 diff --git a/bytes/poetry.lock b/bytes/poetry.lock index 03c8c961fe8..f70bba87163 100644 --- a/bytes/poetry.lock +++ b/bytes/poetry.lock @@ -321,43 +321,38 @@ files = [ [[package]] name = "cryptography" -version = "42.0.8" +version = "43.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, - {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, - {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, - {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, - {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, - {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, - {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, + {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, + {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, + {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, + {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, + {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, + {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, + {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, ] [package.dependencies] @@ -370,7 +365,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -1719,4 +1714,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "3c0e90e2a472d71d04b1507109f9ba59c598f04dc6373d87038bc01a9a9f40a2" +content-hash = "b38550771cc3c4f564889d3c83e98d927e7b0db07f0cdbcb01b6c89a877fa2df" diff --git a/bytes/pyproject.toml b/bytes/pyproject.toml index 40842c56ea6..eb8534de8f3 100644 --- a/bytes/pyproject.toml +++ b/bytes/pyproject.toml @@ -40,6 +40,7 @@ opentelemetry-util-http = "^0.47b0" pyjwt = "^2.8.0" fastapi-slim = "^0.111.0" structlog = "^24.2.0" +cryptography = "^43.0.1" [tool.poetry.group.dev.dependencies] pytest = "^8.2.0" diff --git a/bytes/requirements-dev.txt b/bytes/requirements-dev.txt index c7ef092702d..fa977408c9c 100644 --- a/bytes/requirements-dev.txt +++ b/bytes/requirements-dev.txt @@ -194,39 +194,34 @@ click==8.1.7 ; python_version >= "3.10" and python_version < "4.0" \ colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 -cryptography==42.0.8 ; python_version >= "3.10" and python_version < "4.0" \ - --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ - --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ - --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ - --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ - --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ - --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ - --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ - --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ - --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ - --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ - --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ - --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ - --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ - --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ - --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ - --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ - --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ - --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ - --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ - --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ - --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ - --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ - --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ - --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ - --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ - --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ - --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ - --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ - --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ - --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ - --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ - --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e +cryptography==43.0.1 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \ + --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \ + --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \ + --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \ + --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \ + --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \ + --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \ + --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \ + --hash=sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84 \ + --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \ + --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \ + --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \ + --hash=sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2 \ + --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \ + --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \ + --hash=sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365 \ + --hash=sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96 \ + --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \ + --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \ + --hash=sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d \ + --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \ + --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \ + --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \ + --hash=sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172 \ + --hash=sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034 \ + --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a \ + --hash=sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289 deprecated==1.2.14 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \ --hash=sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3 diff --git a/bytes/requirements.txt b/bytes/requirements.txt index d3d077cc7d0..dc6f031361e 100644 --- a/bytes/requirements.txt +++ b/bytes/requirements.txt @@ -194,39 +194,34 @@ click==8.1.7 ; python_version >= "3.10" and python_version < "4.0" \ colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" and platform_system == "Windows" \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 -cryptography==42.0.8 ; python_version >= "3.10" and python_version < "4.0" \ - --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ - --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ - --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ - --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ - --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ - --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ - --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ - --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ - --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ - --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ - --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ - --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ - --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ - --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ - --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ - --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ - --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ - --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ - --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ - --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ - --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ - --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ - --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ - --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ - --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ - --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ - --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ - --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ - --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ - --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ - --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ - --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e +cryptography==43.0.1 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \ + --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \ + --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \ + --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \ + --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \ + --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \ + --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \ + --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \ + --hash=sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84 \ + --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \ + --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \ + --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \ + --hash=sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2 \ + --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \ + --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \ + --hash=sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365 \ + --hash=sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96 \ + --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \ + --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \ + --hash=sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d \ + --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \ + --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \ + --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \ + --hash=sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172 \ + --hash=sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034 \ + --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a \ + --hash=sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289 deprecated==1.2.14 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \ --hash=sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3 From a11a7390612e7a465314e818d848ad8ed9c5c10f Mon Sep 17 00:00:00 2001 From: noamblitz <43830693+noamblitz@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:23:18 +0200 Subject: [PATCH 09/28] Dont yield all snyk findings when no version was found (#3431) Co-authored-by: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> --- .../kat_kat_finding_types/kat_finding_types.json | 5 +++++ boefjes/boefjes/plugins/kat_snyk/normalize.py | 11 ++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json b/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json index 23ec1b70da7..00fbbba47a4 100644 --- a/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json +++ b/boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json @@ -486,5 +486,10 @@ "source": "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers", "impact": "Nonstandard headers may not be supported by all browsers and may not provide the security that is expected.", "recommendation": "Remove the nonstandard headers from the response." + }, + "KAT-SOFTWARE-VERSION-NOT-FOUND": { + "description": "The version of the software is not found.", + "risk": "recommendation", + "recommendation": "There was no version found for this software but there are known vulnerabilities for this software." } } diff --git a/boefjes/boefjes/plugins/kat_snyk/normalize.py b/boefjes/boefjes/plugins/kat_snyk/normalize.py index cd8b5746234..240cc580e0f 100755 --- a/boefjes/boefjes/plugins/kat_snyk/normalize.py +++ b/boefjes/boefjes/plugins/kat_snyk/normalize.py @@ -23,7 +23,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: elif not results["table_vulnerabilities"] and not results["cve_vulnerabilities"]: # no vulnerabilities found return - else: + if software_version: for vuln in results["table_vulnerabilities"]: snyk_ft = SnykFindingType(id=vuln.get("Vuln_href")) yield snyk_ft @@ -40,6 +40,15 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: ooi=pk_ooi, description=vuln.get("Vuln_text"), ) + if not software_version and (results["table_vulnerabilities"] or results["cve_vulnerabilities"]): + kat_ooi = KATFindingType(id="KAT-SOFTWARE-VERSION-NOT-FOUND") + yield kat_ooi + yield Finding( + finding_type=kat_ooi.reference, + ooi=pk_ooi, + description="There was no version found for this software. " + "But there are known vulnerabilities for some versions.", + ) # Check for latest version latest_version = "" From a0c48c8fe2463f18b57bfdf6a092f3ba8b542a5c Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:28:11 +0200 Subject: [PATCH 10/28] Make the "name" field for plugins mandatory (#3471) Signed-off-by: Donny Peeters Co-authored-by: Jan Klopper --- boefjes/boefjes/models.py | 2 +- boefjes/boefjes/plugins/kat_dns_version/normalizer.json | 1 + boefjes/boefjes/plugins/kat_green_hosting/normalizer.json | 1 + .../boefjes/plugins/kat_kat_finding_types/normalizer.json | 1 + .../boefjes/plugins/kat_manual/single_ooi/normalizer.json | 1 + .../kat_test/kat_test_2/kat_test_3/normalizer.json | 1 + .../katalogus/boefjes_test_dir/kat_test/normalizer.json | 1 + .../dummy_bad_normalizer_dict_structure/normalizer.json | 1 + .../dummy_bad_normalizer_return_type/normalizer.json | 1 + boefjes/tests/modules/dummy_normalizer/normalizer.json | 1 + .../modules/dummy_normalizer_import_error/normalizer.json | 1 + boefjes/tests/test_tasks.py | 8 ++++++++ 12 files changed, 19 insertions(+), 1 deletion(-) diff --git a/boefjes/boefjes/models.py b/boefjes/boefjes/models.py index 028d1a1c9a6..58665b8588a 100644 --- a/boefjes/boefjes/models.py +++ b/boefjes/boefjes/models.py @@ -13,7 +13,7 @@ class Organisation(BaseModel): class Plugin(BaseModel): id: str - name: str | None = None + name: str version: str | None = None created: datetime.datetime | None = None description: str | None = None diff --git a/boefjes/boefjes/plugins/kat_dns_version/normalizer.json b/boefjes/boefjes/plugins/kat_dns_version/normalizer.json index 0252c4fe250..4bd2cad202f 100644 --- a/boefjes/boefjes/plugins/kat_dns_version/normalizer.json +++ b/boefjes/boefjes/plugins/kat_dns_version/normalizer.json @@ -1,5 +1,6 @@ { "id": "dns-bind-version-normalize", + "name": "DNS bind version normalizer", "consumes": [ "boefje/dns-bind-version" ], diff --git a/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json b/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json index 92434280db5..714628e5587 100644 --- a/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json +++ b/boefjes/boefjes/plugins/kat_green_hosting/normalizer.json @@ -1,6 +1,7 @@ { "id": "kat_green_hosting_normalize", "description": "Parses the Green Hosting output into findings.", + "name": "Green Hosting", "consumes": [ "boefje/green-hosting" ], diff --git a/boefjes/boefjes/plugins/kat_kat_finding_types/normalizer.json b/boefjes/boefjes/plugins/kat_kat_finding_types/normalizer.json index fc0516ed23e..81130efc8f7 100644 --- a/boefjes/boefjes/plugins/kat_kat_finding_types/normalizer.json +++ b/boefjes/boefjes/plugins/kat_kat_finding_types/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_kat_finding_types_normalize", + "name": "KAT finding types", "description": "Parses KAT finding types.", "consumes": [ "boefje/kat-finding-types" diff --git a/boefjes/boefjes/plugins/kat_manual/single_ooi/normalizer.json b/boefjes/boefjes/plugins/kat_manual/single_ooi/normalizer.json index 193deac6cda..cda7b5d8b97 100644 --- a/boefjes/boefjes/plugins/kat_manual/single_ooi/normalizer.json +++ b/boefjes/boefjes/plugins/kat_manual/single_ooi/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_manual_ooi", + "name": "Manual OOI normalizer", "consumes": [ "manual/ooi" ], diff --git a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_2/kat_test_3/normalizer.json b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_2/kat_test_3/normalizer.json index 0613a485c0a..25d874aa2d0 100644 --- a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_2/kat_test_3/normalizer.json +++ b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/kat_test_2/kat_test_3/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_test_normalize_2", + "name": "Test normalizer 2", "description": "Testing KAT 2", "consumes": [ "text/html" diff --git a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/normalizer.json b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/normalizer.json index 7b7ddc26ade..20e18b975b6 100644 --- a/boefjes/tests/katalogus/boefjes_test_dir/kat_test/normalizer.json +++ b/boefjes/tests/katalogus/boefjes_test_dir/kat_test/normalizer.json @@ -1,5 +1,6 @@ { "id": "kat_test_normalize", + "name": "Test normalizer", "description": "Testing KAT", "consumes": [ "text/html" diff --git a/boefjes/tests/modules/dummy_bad_normalizer_dict_structure/normalizer.json b/boefjes/tests/modules/dummy_bad_normalizer_dict_structure/normalizer.json index cbfb74cfd32..fe23b0855ae 100644 --- a/boefjes/tests/modules/dummy_bad_normalizer_dict_structure/normalizer.json +++ b/boefjes/tests/modules/dummy_bad_normalizer_dict_structure/normalizer.json @@ -1,5 +1,6 @@ { "id": "dummy_bad_normalizer_dict_structure", + "name": "dummy_bad_normalizer_dict_structure", "description": "", "consumes": [], "produces": [] diff --git a/boefjes/tests/modules/dummy_bad_normalizer_return_type/normalizer.json b/boefjes/tests/modules/dummy_bad_normalizer_return_type/normalizer.json index f0060d3b12e..8bcee7c4327 100644 --- a/boefjes/tests/modules/dummy_bad_normalizer_return_type/normalizer.json +++ b/boefjes/tests/modules/dummy_bad_normalizer_return_type/normalizer.json @@ -1,5 +1,6 @@ { "id": "dummy_bad_normalizer_return_type", + "name": "dummy_bad_normalizer_return_type", "description": "", "consumes": [], "produces": [] diff --git a/boefjes/tests/modules/dummy_normalizer/normalizer.json b/boefjes/tests/modules/dummy_normalizer/normalizer.json index a6447d09407..4347a7a0b47 100644 --- a/boefjes/tests/modules/dummy_normalizer/normalizer.json +++ b/boefjes/tests/modules/dummy_normalizer/normalizer.json @@ -1,5 +1,6 @@ { "id": "dummy_normalizer", + "name": "dummy_normalizer", "description": "", "consumes": [], "produces": [] diff --git a/boefjes/tests/modules/dummy_normalizer_import_error/normalizer.json b/boefjes/tests/modules/dummy_normalizer_import_error/normalizer.json index a6447d09407..4347a7a0b47 100644 --- a/boefjes/tests/modules/dummy_normalizer_import_error/normalizer.json +++ b/boefjes/tests/modules/dummy_normalizer_import_error/normalizer.json @@ -1,5 +1,6 @@ { "id": "dummy_normalizer", + "name": "dummy_normalizer", "description": "", "consumes": [], "produces": [] diff --git a/boefjes/tests/test_tasks.py b/boefjes/tests/test_tasks.py index f9c319897a1..93b37e382d6 100644 --- a/boefjes/tests/test_tasks.py +++ b/boefjes/tests/test_tasks.py @@ -22,21 +22,25 @@ def setUp(self) -> None: self.boefjes = [ Boefje( id="test-boefje-1", + name="test-boefje-1", consumes={"SomeOOI"}, produces=["test-boef-1", "test/text"], ), Boefje( id="test-boefje-2", + name="test-boefje-2", consumes={"SomeOOI"}, produces=["test-boef-2", "test/text"], ), Boefje( id="test-boefje-3", + name="test-boefje-3", consumes={"SomeOOI"}, produces=["test-boef-3", "test/plain"], ), Boefje( id="test-boefje-4", + name="test-boefje-4", consumes={"SomeOOI"}, produces=["test-boef-4", "test/and-simple"], ), @@ -44,11 +48,13 @@ def setUp(self) -> None: self.normalizers = [ Normalizer( id="test-normalizer-1", + name="test-normalizer-1", consumes=["test-boef-3", "test/text"], produces=["SomeOOI", "OtherOOI"], ), Normalizer( id="test-normalizer-2", + name="test-normalizer-2", consumes=["test/text"], produces=["SomeOtherOOI"], ), @@ -56,12 +62,14 @@ def setUp(self) -> None: self.bits = [ Bit( id="test-bit-1", + name="test-bit-1", consumes="SomeOOI", produces=["SomeOOI"], parameters=[], ), Bit( id="test-bit-2", + name="test-bit-2", consumes="SomeOOI", produces=["SomeOOI", "SomeOtherOOI"], parameters=[], From 9942c9fab60a096f12f61851494476af697ee013 Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:05:20 +0200 Subject: [PATCH 11/28] Handle empty normalizer results (#3482) Signed-off-by: Donny Peeters Co-authored-by: Jan Klopper --- boefjes/boefjes/job_handler.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index 8b0d979d256..7ad0f7247ad 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -222,6 +222,23 @@ def handle(self, normalizer_meta: NormalizerMeta) -> None: ) ) + if ( + normalizer_meta.raw_data.boefje_meta.input_ooi # No input OOI means no deletion propagation + and not (results.observations or results.declarations or results.affirmations) + ): + # There were no results found, which we still need to signal to Octopoes for deletion propagation + + connector.save_observation( + Observation( + method=normalizer_meta.normalizer.id, + source=Reference.from_str(normalizer_meta.raw_data.boefje_meta.input_ooi), + source_method=normalizer_meta.raw_data.boefje_meta.boefje.id, + task_id=normalizer_meta.id, + valid_time=normalizer_meta.raw_data.boefje_meta.ended_at, + result=[], + ) + ) + corrected_scan_profiles = [] for profile in results.scan_profiles: profile.level = ScanLevel( From b098d8d2ee2f7bdbb4c7d98e7125ae3c16f43bd7 Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:13:48 +0200 Subject: [PATCH 12/28] Fix enabling normalizers from Rocky (#3481) Signed-off-by: Donny Peeters Co-authored-by: Jan Klopper --- rocky/katalogus/client.py | 12 ++--- .../katalogus/views/plugin_enable_disable.py | 49 ++----------------- rocky/katalogus/views/plugin_settings_add.py | 2 +- rocky/rocky/locale/django.pot | 10 +--- 4 files changed, 11 insertions(+), 62 deletions(-) diff --git a/rocky/katalogus/client.py b/rocky/katalogus/client.py index 8bb9f7fc463..ebb614b0b9e 100644 --- a/rocky/katalogus/client.py +++ b/rocky/katalogus/client.py @@ -169,14 +169,14 @@ def get_normalizers(self) -> list[Plugin]: def get_boefjes(self) -> list[Plugin]: return self.get_plugins(plugin_type="boefje") - def enable_boefje(self, plugin: Plugin) -> None: - self._patch_boefje_state(plugin.id, True) + def enable_plugin(self, plugin: Plugin) -> None: + self._patch_plugin_state(plugin.id, True) def enable_boefje_by_id(self, boefje_id: str) -> None: - self.enable_boefje(self.get_plugin(boefje_id)) + self.enable_plugin(self.get_plugin(boefje_id)) - def disable_boefje(self, plugin: Plugin) -> None: - self._patch_boefje_state(plugin.id, False) + def disable_plugin(self, plugin: Plugin) -> None: + self._patch_plugin_state(plugin.id, False) def get_enabled_boefjes(self) -> list[Plugin]: return [plugin for plugin in self.get_boefjes() if plugin.enabled] @@ -184,7 +184,7 @@ def get_enabled_boefjes(self) -> list[Plugin]: def get_enabled_normalizers(self) -> list[Plugin]: return [plugin for plugin in self.get_normalizers() if plugin.enabled] - def _patch_boefje_state(self, boefje_id: str, enabled: bool) -> None: + def _patch_plugin_state(self, boefje_id: str, enabled: bool) -> None: logger.info("Toggle plugin state", plugin_id=boefje_id, enabled=enabled) response = self.session.patch( diff --git a/rocky/katalogus/views/plugin_enable_disable.py b/rocky/katalogus/views/plugin_enable_disable.py index 36db033e530..81b3d7ce36f 100644 --- a/rocky/katalogus/views/plugin_enable_disable.py +++ b/rocky/katalogus/views/plugin_enable_disable.py @@ -1,24 +1,16 @@ from django.contrib import messages from django.http import HttpResponseRedirect -from django.shortcuts import redirect -from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from httpx import HTTPError from katalogus.views.mixins import SinglePluginView class PluginEnableDisableView(SinglePluginView): - def check_required_settings(self, settings: dict): - if self.plugin_schema is None or "required" not in self.plugin_schema: - return True - - return all([field in settings for field in self.plugin_schema["required"]]) - def post(self, request, *args, **kwargs): plugin_state = kwargs["plugin_state"] + if plugin_state == "True": - self.katalogus_client.disable_boefje(self.plugin) + self.katalogus_client.disable_plugin(self.plugin) messages.add_message( self.request, messages.WARNING, @@ -26,43 +18,8 @@ def post(self, request, *args, **kwargs): ) return HttpResponseRedirect(request.POST.get("current_url")) - try: - plugin_settings = self.katalogus_client.get_plugin_settings(self.plugin.id) - except HTTPError: - messages.add_message( - self.request, - messages.ERROR, - _("Failed fetching settings for {}. Is the Katalogus up?").format(self.plugin.name), - ) - return redirect( - reverse( - "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, - ) - ) - - if not self.check_required_settings(plugin_settings): - messages.add_message( - self.request, - messages.INFO, - _("Before enabling, please set the required settings for '{}'.").format(self.plugin.name), - ) - return redirect( - reverse( - "plugin_settings_add", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - "plugin_type": self.plugin.type, - }, - ) - ) - if self.plugin.can_scan(self.organization_member): - self.katalogus_client.enable_boefje(self.plugin) + self.katalogus_client.enable_plugin(self.plugin) messages.add_message( self.request, messages.SUCCESS, diff --git a/rocky/katalogus/views/plugin_settings_add.py b/rocky/katalogus/views/plugin_settings_add.py index 47842b284a5..f89921025cd 100644 --- a/rocky/katalogus/views/plugin_settings_add.py +++ b/rocky/katalogus/views/plugin_settings_add.py @@ -54,7 +54,7 @@ def form_valid(self, form): if "add-enable" in self.request.POST: try: - self.katalogus_client.enable_boefje(self.plugin) + self.katalogus_client.enable_plugin(self.plugin) except HTTPError: messages.add_message(self.request, messages.ERROR, _("Enabling {} failed").format(self.plugin.name)) return redirect(self.get_success_url()) diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 8d7603aad8a..84889204c77 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-09-05 08:44+0000\n" +"POT-Creation-Date: 2024-09-06 08:27+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -1198,14 +1198,6 @@ msgstr "" msgid "{} '{}' disabled." msgstr "" -#: katalogus/views/plugin_enable_disable.py -msgid "Failed fetching settings for {}. Is the Katalogus up?" -msgstr "" - -#: katalogus/views/plugin_enable_disable.py -msgid "Before enabling, please set the required settings for '{}'." -msgstr "" - #: katalogus/views/plugin_enable_disable.py msgid "{} '{}' enabled." msgstr "" From 60283ac02e142055e5ceadb858cf4d3c398e983e Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Tue, 10 Sep 2024 14:13:18 +0200 Subject: [PATCH 13/28] Feature/upload multiple files at once to bytes (#3476) Signed-off-by: Donny Peeters Co-authored-by: ammar92 --- boefjes/boefjes/clients/bytes_client.py | 23 +++-- bytes/bytes/api/models.py | 14 +++- bytes/bytes/api/router.py | 97 +++++++++++++--------- bytes/bytes/models.py | 10 ++- bytes/tests/client.py | 31 +++++-- bytes/tests/integration/test_bytes_api.py | 49 +++++++++-- bytes/tests/integration/test_migrations.py | 6 +- bytes/tests/unit/test_context_mapping.py | 2 +- rocky/rocky/bytes_client.py | 19 ++++- 9 files changed, 179 insertions(+), 72 deletions(-) diff --git a/boefjes/boefjes/clients/bytes_client.py b/boefjes/boefjes/clients/bytes_client.py index c2698523183..b7b66bbc272 100644 --- a/boefjes/boefjes/clients/bytes_client.py +++ b/boefjes/boefjes/clients/bytes_client.py @@ -1,5 +1,6 @@ import typing import uuid +from base64 import b64encode from collections.abc import Callable, Set from functools import wraps from typing import Any @@ -99,17 +100,25 @@ def get_normalizer_meta(self, normalizer_meta_id: uuid.UUID) -> NormalizerMeta: @retry_with_login def save_raw(self, boefje_meta_id: str, raw: str | bytes, mime_types: Set[str] = frozenset()) -> UUID: - headers = {"content-type": "application/octet-stream"} - headers.update(self.headers) + file_name = "raw" # The name provides a key for all ids returned, so this is arbitrary as we only upload 1 file + response = self._session.post( "/bytes/raw", - content=raw, - headers=headers, - params={"mime_types": list(mime_types), "boefje_meta_id": boefje_meta_id}, + json={ + "files": [ + { + "name": file_name, + "content": b64encode(raw if isinstance(raw, bytes) else raw.encode()).decode(), + "tags": list(mime_types), + } + ] + }, + headers=self.headers, + params={"boefje_meta_id": str(boefje_meta_id)}, ) - self._verify_response(response) - return UUID(response.json()["id"]) + + return UUID(response.json()[file_name]) @retry_with_login def get_raw(self, raw_data_id: str) -> bytes: diff --git a/bytes/bytes/api/models.py b/bytes/bytes/api/models.py index af77cb82fc6..fddfc0dc4a3 100644 --- a/bytes/bytes/api/models.py +++ b/bytes/bytes/api/models.py @@ -1,7 +1,17 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field class RawResponse(BaseModel): status: str message: str - id: str | None = None + ids: list[str] | None = None + + +class File(BaseModel): + name: str + content: str = Field(..., contentEncoding="base64") + tags: list[str] = Field(default_factory=list) + + +class BoefjeOutput(BaseModel): + files: list[File] = Field(default_factory=list) diff --git a/bytes/bytes/api/router.py b/bytes/bytes/api/router.py index e88efc6cc06..394f0e959b4 100644 --- a/bytes/bytes/api/router.py +++ b/bytes/bytes/api/router.py @@ -1,13 +1,14 @@ +from base64 import b64decode from uuid import UUID import structlog -from asgiref.sync import async_to_sync from cachetools import TTLCache, cached -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import Response +from httpx import codes from starlette.responses import JSONResponse -from bytes.api.models import RawResponse +from bytes.api.models import BoefjeOutput from bytes.auth import authenticate_token from bytes.config import get_settings from bytes.database.sql_meta_repository import MetaIntegrityError, ObjectNotFoundException, create_meta_data_repository @@ -34,10 +35,11 @@ def create_boefje_meta( meta_repository.save_boefje_meta(boefje_meta) except MetaIntegrityError: return JSONResponse( - {"status": "failed", "message": "Integrity error: object might already exist"}, status_code=400 + {"status": "failed", "message": "Integrity error: object might already exist"}, + status_code=codes.BAD_REQUEST, ) - return JSONResponse({"status": "success"}, status_code=201) + return JSONResponse({"status": "success"}, status_code=codes.CREATED) @router.get("/boefje_meta/{boefje_meta_id}", response_model=BoefjeMeta, tags=[BOEFJE_META_TAG]) @@ -95,10 +97,11 @@ def create_normalizer_meta( meta_repository.save_normalizer_meta(normalizer_meta) except MetaIntegrityError: return JSONResponse( - {"status": "failed", "message": "Integrity error: object might already exist"}, status_code=400 + {"status": "failed", "message": "Integrity error: object might already exist"}, + status_code=codes.BAD_REQUEST, ) - return JSONResponse({"status": "success"}, status_code=201) + return JSONResponse({"status": "success"}, status_code=codes.CREATED) @router.get("/normalizer_meta/{normalizer_meta_id}", response_model=NormalizerMeta, tags=[NORMALIZER_META_TAG]) @@ -109,7 +112,7 @@ def get_normalizer_meta_by_id( try: return meta_repository.get_normalizer_meta_by_id(normalizer_meta_id) except ObjectNotFoundException as error: - raise HTTPException(status_code=404, detail="Normalizer meta not found") from error + raise HTTPException(status_code=codes.NOT_FOUND, detail="Normalizer meta not found") from error @router.get("/normalizer_meta", response_model=list[NormalizerMeta], tags=[NORMALIZER_META_TAG]) @@ -148,42 +151,60 @@ def get_normalizer_meta( @router.post("/raw", tags=[RAW_TAG]) def create_raw( - request: Request, boefje_meta_id: UUID, - mime_types: list[str] | None = Query(None), + boefje_output: BoefjeOutput, meta_repository: MetaDataRepository = Depends(create_meta_data_repository), event_manager: EventManager = Depends(create_event_manager), -) -> RawResponse: - parsed_mime_types = [] if mime_types is None else [MimeType(value=mime_type) for mime_type in mime_types] +) -> dict[str, UUID]: + """Parse all the raw files from the request and return the ids. The ids are ordered according to the order from the + request data, but we assume the `name` field is unique, and hence return a mapping of the file name to the id.""" - try: - meta = meta_repository.get_boefje_meta_by_id(boefje_meta_id) + raw_ids = {} + mime_types_by_id = { + raw.id: set(raw.mime_types) for raw in meta_repository.get_raw(RawDataFilter(boefje_meta_id=boefje_meta_id)) + } + all_parsed_mime_types = list(mime_types_by_id.values()) - if meta_repository.has_raw(meta, parsed_mime_types): - return RawResponse(status="success", message="Raw data already present") + for raw in boefje_output.files: + parsed_mime_types = {MimeType(value=x) for x in raw.tags} - # FastAPI/starlette only has async versions of the Request methods, but - # all our code is sync, so we wrap it in async_to_sync. - data = async_to_sync(request.body)() + if parsed_mime_types in mime_types_by_id.values(): + # Set the id for this file using the precomputed dict that maps existing primary keys to the mime-type set. + raw_ids[raw.name] = list(mime_types_by_id.keys())[list(mime_types_by_id.values()).index(parsed_mime_types)] - raw_data = RawData(value=data, boefje_meta=meta, mime_types=parsed_mime_types) - with meta_repository: - raw_id = meta_repository.save_raw(raw_data) - - event = RawFileReceived( - organization=meta.organization, - raw_data=RawDataMeta( - id=raw_id, - boefje_meta=raw_data.boefje_meta, - mime_types=raw_data.mime_types, - ), - ) - event_manager.publish(event) - except Exception as error: - logger.exception("Error saving raw data") - raise HTTPException(status_code=500, detail="Could not save raw data") from error + continue + + if parsed_mime_types in all_parsed_mime_types: + raise HTTPException( + status_code=codes.BAD_REQUEST, detail="Content types do not define unique sets of mime types." + ) + + try: + meta = meta_repository.get_boefje_meta_by_id(boefje_meta_id) + raw_data = RawData(value=b64decode(raw.content.encode()), boefje_meta=meta, mime_types=parsed_mime_types) + + with meta_repository: + raw_id = meta_repository.save_raw(raw_data) + raw_ids[raw.name] = raw_id + + all_parsed_mime_types.append(parsed_mime_types) + + event = RawFileReceived( + organization=meta.organization, + raw_data=RawDataMeta( + id=raw_id, + boefje_meta=raw_data.boefje_meta, + mime_types=raw_data.mime_types, + ), + ) + event_manager.publish(event) + except Exception as error: + logger.exception("Error saving raw data") + raise HTTPException(status_code=codes.INTERNAL_SERVER_ERROR, detail="Could not save raw data") from error + + all_parsed_mime_types.append(parsed_mime_types) - return RawResponse(status="success", message="Raw data saved", id=raw_id) + return raw_ids @router.get("/raw/{raw_id}", tags=[RAW_TAG]) @@ -194,7 +215,7 @@ def get_raw_by_id( try: raw_data = meta_repository.get_raw_by_id(raw_id) except ObjectNotFoundException as error: - raise HTTPException(status_code=404, detail="No raw data found") from error + raise HTTPException(status_code=codes.NOT_FOUND, detail="No raw data found") from error return Response(raw_data.value, media_type="application/octet-stream") @@ -207,7 +228,7 @@ def get_raw_meta_by_id( try: raw_meta = meta_repository.get_raw_meta_by_id(raw_id) except ObjectNotFoundException as error: - raise HTTPException(status_code=404, detail="No raw data found") from error + raise HTTPException(status_code=codes.NOT_FOUND, detail="No raw data found") from error return raw_meta diff --git a/bytes/bytes/models.py b/bytes/bytes/models.py index ce8cfcd08be..03ae39506aa 100644 --- a/bytes/bytes/models.py +++ b/bytes/bytes/models.py @@ -38,6 +38,12 @@ def _validate_timezone_aware_datetime(value: datetime) -> datetime: class MimeType(BaseModel): value: str + def __hash__(self): + return hash(self.value) + + def __lt__(self, other: MimeType): + return self.value < other.value + class Job(BaseModel): id: UUID @@ -69,7 +75,7 @@ class RawDataMeta(BaseModel): id: UUID boefje_meta: BoefjeMeta - mime_types: list[MimeType] = Field(default_factory=list) + mime_types: set[MimeType] = Field(default_factory=set) # These are set once the raw is saved secure_hash: SecureHash | None = None @@ -80,7 +86,7 @@ class RawDataMeta(BaseModel): class RawData(BaseModel): value: bytes boefje_meta: BoefjeMeta - mime_types: list[MimeType] = Field(default_factory=list) + mime_types: set[MimeType] = Field(default_factory=set) # These are set once the raw is saved secure_hash: SecureHash | None = None diff --git a/bytes/tests/client.py b/bytes/tests/client.py index 6996d1e821b..22405c75ffc 100644 --- a/bytes/tests/client.py +++ b/bytes/tests/client.py @@ -1,4 +1,5 @@ import typing +from base64 import b64encode from collections.abc import Callable from functools import wraps from typing import Any @@ -7,6 +8,7 @@ import httpx from httpx import HTTPError +from bytes.api.models import BoefjeOutput from bytes.models import BoefjeMeta, NormalizerMeta from bytes.repositories.meta_repository import BoefjeMetaFilter, NormalizerMetaFilter, RawDataFilter @@ -126,19 +128,34 @@ def save_raw(self, boefje_meta_id: UUID, raw: bytes, mime_types: list[str] | Non if not mime_types: mime_types = [] - headers = {"content-type": "application/octet-stream"} - + file_name = "raw" # The name provides a key for all ids returned, so this is arbitrary as we only upload 1 file response = self.client.post( "/bytes/raw", - content=raw, - headers=headers, - params={"mime_types": mime_types, "boefje_meta_id": str(boefje_meta_id)}, + json={ + "files": [ + { + "name": file_name, + "content": b64encode(raw).decode(), + "tags": mime_types, + } + ], + }, + params={"boefje_meta_id": str(boefje_meta_id)}, ) + self._verify_response(response) + return response.json()[file_name] + + @retry_with_login + def save_raws(self, boefje_meta_id: UUID, boefje_output: BoefjeOutput) -> dict[str, str]: + response = self.client.post( + "/bytes/raw", + content=boefje_output.model_dump_json(), + params={"boefje_meta_id": str(boefje_meta_id)}, + ) self._verify_response(response) - raw_id = response.json()["id"] - return str(raw_id) + return response.json() @retry_with_login def get_raw(self, raw_id: UUID) -> bytes: diff --git a/bytes/tests/integration/test_bytes_api.py b/bytes/tests/integration/test_bytes_api.py index 95122aaeea0..afc34ec75bd 100644 --- a/bytes/tests/integration/test_bytes_api.py +++ b/bytes/tests/integration/test_bytes_api.py @@ -1,10 +1,12 @@ import uuid +from base64 import b64encode import httpx import pytest from httpx import HTTPError from prometheus_client.parser import text_string_to_metric_families +from bytes.api.models import BoefjeOutput, File from bytes.models import MimeType from bytes.rabbitmq import RabbitMQEventManager from bytes.repositories.meta_repository import BoefjeMetaFilter, NormalizerMetaFilter, RawDataFilter @@ -147,7 +149,10 @@ def test_normalizer_meta(bytes_api_client: BytesAPIClient, event_manager: Rabbit normalizer_meta.raw_data.hash_retrieval_link = retrieved_normalizer_meta.raw_data.hash_retrieval_link normalizer_meta.raw_data.signing_provider_url = retrieved_normalizer_meta.raw_data.signing_provider_url - assert normalizer_meta.dict() == retrieved_normalizer_meta.dict() + normalizer_meta.raw_data.mime_types = sorted(normalizer_meta.raw_data.mime_types) + retrieved_normalizer_meta.raw_data.mime_types = sorted(retrieved_normalizer_meta.raw_data.mime_types) + + assert normalizer_meta.model_dump_json() == retrieved_normalizer_meta.model_dump_json() def test_filtered_normalizer_meta(bytes_api_client: BytesAPIClient) -> None: @@ -255,21 +260,30 @@ def test_save_raw_no_mime_types(bytes_api_client: BytesAPIClient) -> None: boefje_meta = get_boefje_meta(meta_id=uuid.uuid4()) bytes_api_client.save_boefje_meta(boefje_meta) - headers = {"content-type": "application/octet-stream"} bytes_api_client.login() - headers.update(bytes_api_client.client.headers) raw_url = f"{bytes_api_client.client.base_url}/bytes/raw" raw = b"second test 123456" + file_name = "raw" response = httpx.post( - raw_url, content=raw, headers=headers, params={"boefje_meta_id": str(boefje_meta.id)}, timeout=30 + raw_url, + json={ + "files": [ + { + "name": file_name, + "content": b64encode(raw).decode(), + "tags": [], + } + ] + }, + headers=bytes_api_client.client.headers, + params={"boefje_meta_id": str(boefje_meta.id)}, ) - assert response.status_code == 200 get_raw_without_mime_type_response = httpx.get( - f"{raw_url}/{response.json().get('id')}", headers=bytes_api_client.client.headers, timeout=30 + f"{raw_url}/{response.json()[file_name]}", headers=bytes_api_client.client.headers, timeout=30 ) assert get_raw_without_mime_type_response.status_code == 200 @@ -293,13 +307,13 @@ def test_raw_mimes(bytes_api_client: BytesAPIClient) -> None: ) ) assert len(retrieved_raws) == 1 - assert retrieved_raws[0]["mime_types"] == [{"value": value} for value in mime_types] + assert {x["value"] for x in retrieved_raws[0]["mime_types"]} == set(mime_types) retrieved_raws = bytes_api_client.get_raws( RawDataFilter(boefje_meta_id=boefje_meta.id, normalized=False, mime_types=[MimeType(value="text/html")]) ) assert len(retrieved_raws) == 1 - assert retrieved_raws[0]["mime_types"] == [{"value": value} for value in mime_types] + assert {x["value"] for x in retrieved_raws[0]["mime_types"]} == set(mime_types) retrieved_raws = bytes_api_client.get_raws( RawDataFilter(boefje_meta_id=boefje_meta.id, normalized=False, mime_types=[MimeType(value="bad/mime")]) @@ -336,3 +350,22 @@ def test_cannot_overwrite_raw(bytes_api_client: BytesAPIClient) -> None: retrieved_raw = bytes_api_client.get_raw(first_raw_id) assert retrieved_raw == right_raw + + +def test_save_multiple_raw_files(bytes_api_client: BytesAPIClient) -> None: + boefje_meta = get_boefje_meta() + bytes_api_client.save_boefje_meta(boefje_meta) + + first_raw = b"first" + second_raw = b"second" + boefje_output = BoefjeOutput( + files=[ + File(name="first", content=b64encode(first_raw).decode(), tags=[]), + File(name="second", content=b64encode(second_raw).decode(), tags=["mime", "type"]), + ] + ) + + ids = bytes_api_client.save_raws(boefje_meta.id, boefje_output) + + assert bytes_api_client.get_raw(ids["first"]) == first_raw + assert bytes_api_client.get_raw(ids["second"]) == second_raw diff --git a/bytes/tests/integration/test_migrations.py b/bytes/tests/integration/test_migrations.py index eb3a29c0c4f..4967d847470 100644 --- a/bytes/tests/integration/test_migrations.py +++ b/bytes/tests/integration/test_migrations.py @@ -13,15 +13,15 @@ def test_clean_mime_types(meta_repository: SQLMetaDataRepository) -> None: meta_repository.save_boefje_meta(boefje_meta) raw = get_raw_data() - raw.mime_types.append(MimeType(value=raw.boefje_meta.boefje.id)) + raw.mime_types.add(MimeType(value=raw.boefje_meta.boefje.id)) raw_id_1 = meta_repository.save_raw(raw) - raw.mime_types.append( + raw.mime_types.add( MimeType(value=f"boefje/{raw.boefje_meta.boefje.id}-ce293f79fd3c809a300a2837bb1da4f7115fc034a1f78") ) raw_id_2 = meta_repository.save_raw(raw) - raw.mime_types.append( + raw.mime_types.add( MimeType(value=f"boefje/{raw.boefje_meta.boefje.id}-ba293f79fd3c809a300a2837bb1da4f7115fc034a1f78") ) raw_id_3 = meta_repository.save_raw(raw) diff --git a/bytes/tests/unit/test_context_mapping.py b/bytes/tests/unit/test_context_mapping.py index 62f303f2c46..61147f84414 100644 --- a/bytes/tests/unit/test_context_mapping.py +++ b/bytes/tests/unit/test_context_mapping.py @@ -62,7 +62,7 @@ def test_context_mapping_raw() -> None: assert raw_data.hash_retrieval_link == raw_data_in_db.hash_retrieval_link assert raw_data.secure_hash == raw_data_in_db.secure_hash assert raw_data.signing_provider_url is None - assert raw_data.mime_types == [to_mime_type(mime_type) for mime_type in raw_data_in_db.mime_types] + assert raw_data.mime_types == {to_mime_type(mime_type) for mime_type in raw_data_in_db.mime_types} raw_data_new = to_raw_data(raw_data_in_db, raw_data.value) diff --git a/rocky/rocky/bytes_client.py b/rocky/rocky/bytes_client.py index 837d586ebc9..9ddd97ff9d7 100644 --- a/rocky/rocky/bytes_client.py +++ b/rocky/rocky/bytes_client.py @@ -1,4 +1,5 @@ import uuid +from base64 import b64encode from collections.abc import Set from datetime import datetime, timezone @@ -113,15 +114,25 @@ def _save_normalizer_meta(self, normalizer_meta: NormalizerMeta) -> None: response.raise_for_status() def _save_raw(self, boefje_meta_id: uuid.UUID, raw: bytes, mime_types: Set[str] = frozenset()) -> str: + file_name = "raw" # The name provides a key for all ids returned, so this is arbitrary as we only upload 1 file + response = self.session.post( "/bytes/raw", - content=raw, - headers={"content-type": "application/octet-stream"}, - params={"mime_types": list(mime_types), "boefje_meta_id": str(boefje_meta_id)}, + json={ + "files": [ + { + "name": file_name, + "content": b64encode(raw).decode(), + "tags": list(mime_types), + } + ] + }, + params={"boefje_meta_id": str(boefje_meta_id)}, ) response.raise_for_status() - return response.json()["id"] + + return response.json()[file_name] def get_raw(self, raw_id: str) -> bytes: # Note: we assume organization permissions are handled before requesting raw data. From ba809d0ce61c3a0f28940c97bd048582c657481a Mon Sep 17 00:00:00 2001 From: JP Bruins Slot Date: Tue, 10 Sep 2024 14:19:56 +0200 Subject: [PATCH 14/28] Add report scheduler functionality to scheduler (#3352) Co-authored-by: ammar92 Co-authored-by: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> Co-authored-by: Jan Klopper --- mula/scheduler/app.py | 22 ++- mula/scheduler/models/__init__.py | 2 +- mula/scheduler/models/task.py | 13 +- mula/scheduler/schedulers/__init__.py | 1 + mula/scheduler/schedulers/report.py | 162 ++++++++++++++++++ mula/scheduler/server/handlers/schedules.py | 2 + mula/tests/integration/test_api.py | 11 ++ mula/tests/integration/test_app.py | 22 +-- .../integration/test_report_scheduler.py | 162 ++++++++++++++++++ 9 files changed, 383 insertions(+), 14 deletions(-) create mode 100644 mula/scheduler/schedulers/report.py create mode 100644 mula/tests/integration/test_report_scheduler.py diff --git a/mula/scheduler/app.py b/mula/scheduler/app.py index 4e0836bed12..833cb3ec2f5 100644 --- a/mula/scheduler/app.py +++ b/mula/scheduler/app.py @@ -60,7 +60,10 @@ def __init__(self, ctx: context.AppContext) -> None: self.schedulers: dict[ str, - schedulers.Scheduler | schedulers.BoefjeScheduler | schedulers.NormalizerScheduler, + schedulers.Scheduler + | schedulers.BoefjeScheduler + | schedulers.NormalizerScheduler + | schedulers.ReportScheduler, ] = {} self.server: server.Server | None = None @@ -136,12 +139,21 @@ def monitor_organisations(self) -> None: callback=self.remove_scheduler, ) + scheduler_report = schedulers.ReportScheduler( + ctx=self.ctx, + scheduler_id=f"report-{org.id}", + organisation=org, + callback=self.remove_scheduler, + ) + with self.lock: self.schedulers[scheduler_boefje.scheduler_id] = scheduler_boefje self.schedulers[scheduler_normalizer.scheduler_id] = scheduler_normalizer + self.schedulers[scheduler_report.scheduler_id] = scheduler_report scheduler_normalizer.run() scheduler_boefje.run() + scheduler_report.run() if additions: # Flush katalogus caches when new organisations are added @@ -201,6 +213,14 @@ def start_schedulers(self) -> None: ) self.schedulers[normalizer_scheduler.scheduler_id] = normalizer_scheduler + report_scheduler = schedulers.ReportScheduler( + ctx=self.ctx, + scheduler_id=f"report-{org.id}", + organisation=org, + callback=self.remove_scheduler, + ) + self.schedulers[report_scheduler.scheduler_id] = report_scheduler + # Start schedulers for scheduler in self.schedulers.values(): scheduler.run() diff --git a/mula/scheduler/models/__init__.py b/mula/scheduler/models/__init__.py index ed1a7fa177a..a5390ad6ede 100644 --- a/mula/scheduler/models/__init__.py +++ b/mula/scheduler/models/__init__.py @@ -9,4 +9,4 @@ from .queue import Queue from .schedule import Schedule, ScheduleDB from .scheduler import Scheduler -from .task import BoefjeTask, NormalizerTask, Task, TaskDB, TaskStatus +from .task import BoefjeTask, NormalizerTask, ReportTask, Task, TaskDB, TaskStatus diff --git a/mula/scheduler/models/task.py b/mula/scheduler/models/task.py index cf0c9a95834..76c51ad2e79 100644 --- a/mula/scheduler/models/task.py +++ b/mula/scheduler/models/task.py @@ -59,7 +59,7 @@ class Task(BaseModel): hash: str | None = Field(None, max_length=32) - data: dict | None = None + data: dict = Field(default_factory=dict) created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) modified_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) @@ -143,3 +143,14 @@ def hash(self) -> str: return mmh3.hash_bytes(f"{self.input_ooi}-{self.boefje.id}-{self.organization}").hex() return mmh3.hash_bytes(f"{self.boefje.id}-{self.organization}").hex() + + +class ReportTask(BaseModel): + type: ClassVar[str] = "report" + + organisation_id: str + report_recipe_id: str + + @property + def hash(self) -> str: + return mmh3.hash_bytes(f"{self.report_recipe_id}-{self.organisation_id}").hex() diff --git a/mula/scheduler/schedulers/__init__.py b/mula/scheduler/schedulers/__init__.py index 5614508b532..4c82914aee5 100644 --- a/mula/scheduler/schedulers/__init__.py +++ b/mula/scheduler/schedulers/__init__.py @@ -1,3 +1,4 @@ from .boefje import BoefjeScheduler from .normalizer import NormalizerScheduler +from .report import ReportScheduler from .scheduler import Scheduler diff --git a/mula/scheduler/schedulers/report.py b/mula/scheduler/schedulers/report.py new file mode 100644 index 00000000000..b76d79a13b7 --- /dev/null +++ b/mula/scheduler/schedulers/report.py @@ -0,0 +1,162 @@ +from collections.abc import Callable +from concurrent import futures +from datetime import datetime, timezone +from typing import Any + +import structlog +from opentelemetry import trace + +from scheduler import context, queues, storage +from scheduler.models import Organisation, ReportTask, Task +from scheduler.storage import filters + +from .scheduler import Scheduler + +tracer = trace.get_tracer(__name__) + + +class ReportScheduler(Scheduler): + ITEM_TYPE: Any = ReportTask + + def __init__( + self, + ctx: context.AppContext, + scheduler_id: str, + organisation: Organisation, + queue: queues.PriorityQueue | None = None, + callback: Callable[..., None] | None = None, + ): + self.logger: structlog.BoundLogger = structlog.get_logger(__name__) + self.organisation = organisation + self.create_schedule = False + + self.queue = queue or queues.PriorityQueue( + pq_id=scheduler_id, + maxsize=ctx.config.pq_maxsize, + item_type=self.ITEM_TYPE, + allow_priority_updates=True, + pq_store=ctx.datastores.pq_store, + ) + + super().__init__( + ctx=ctx, + queue=self.queue, + scheduler_id=scheduler_id, + callback=callback, + ) + + def run(self) -> None: + # Rescheduling + self.run_in_thread( + name=f"scheduler-{self.scheduler_id}-reschedule", + target=self.push_tasks_for_rescheduling, + interval=60.0, + ) + + @tracer.start_as_current_span(name="report_push_tasks_for_rescheduling") + def push_tasks_for_rescheduling(self): + if self.queue.full(): + self.logger.warning( + "Report queue is full, not populating with new tasks", + queue_qsize=self.queue.qsize(), + organisation_id=self.organisation.id, + scheduler_id=self.scheduler_id, + ) + return + + try: + schedules, _ = self.ctx.datastores.schedule_store.get_schedules( + filters=filters.FilterRequest( + filters=[ + filters.Filter( + column="scheduler_id", + operator="eq", + value=self.scheduler_id, + ), + filters.Filter( + column="deadline_at", + operator="lt", + value=datetime.now(timezone.utc), + ), + filters.Filter( + column="enabled", + operator="eq", + value=True, + ), + ] + ) + ) + except storage.errors.StorageError as exc_db: + self.logger.error( + "Could not get schedules for rescheduling %s", + self.scheduler_id, + scheduler_id=self.scheduler_id, + organisation_id=self.organisation.id, + exc_info=exc_db, + ) + raise exc_db + + with futures.ThreadPoolExecutor( + thread_name_prefix=f"ReportScheduler-TPE-{self.scheduler_id}-rescheduling" + ) as executor: + for schedule in schedules: + report_task = ReportTask.model_validate(schedule.data) + executor.submit( + self.push_report_task, + report_task, + self.push_tasks_for_rescheduling.__name__, + ) + + def push_report_task(self, report_task: ReportTask, caller: str = "") -> None: + self.logger.debug( + "Pushing report task", + task_hash=report_task.hash, + organisation_id=self.organisation.id, + scheduler_id=self.scheduler_id, + caller=caller, + ) + + if self.is_item_on_queue_by_hash(report_task.hash): + self.logger.debug( + "Report task already on queue", + task_hash=report_task.hash, + organisation_id=self.organisation.id, + scheduler_id=self.scheduler_id, + caller=caller, + ) + return + + task = Task( + scheduler_id=self.scheduler_id, + priority=int(datetime.now().timestamp()), + type=self.ITEM_TYPE.type, + hash=report_task.hash, + data=report_task.model_dump(), + ) + + try: + self.push_item_to_queue_with_timeout( + task, + self.max_tries, + ) + except queues.QueueFullError: + self.logger.warning( + "Could not add task %s to queue, queue was full", + report_task.hash, + task_hash=report_task.hash, + queue_qsize=self.queue.qsize(), + queue_maxsize=self.queue.maxsize, + organisation_id=self.organisation.id, + scheduler_id=self.scheduler_id, + caller=caller, + ) + return + + self.logger.info( + "Report task pushed to queue", + task_id=task.id, + task_hash=report_task.hash, + organisation_id=self.organisation.id, + scheduler_id=self.scheduler_id, + caller=caller, + ) diff --git a/mula/scheduler/server/handlers/schedules.py b/mula/scheduler/server/handlers/schedules.py index 33bd17b5aa3..23e44c638b3 100644 --- a/mula/scheduler/server/handlers/schedules.py +++ b/mula/scheduler/server/handlers/schedules.py @@ -63,6 +63,7 @@ def __init__( def list( self, request: fastapi.Request, + scheduler_id: str | None = None, schedule_hash: str | None = None, enabled: bool | None = None, offset: int = 0, @@ -86,6 +87,7 @@ def list( try: results, count = self.ctx.datastores.schedule_store.get_schedules( + scheduler_id=scheduler_id, schedule_hash=schedule_hash, enabled=enabled, min_deadline_at=min_deadline_at, diff --git a/mula/tests/integration/test_api.py b/mula/tests/integration/test_api.py index 9f6440c5801..f72c67705c4 100644 --- a/mula/tests/integration/test_api.py +++ b/mula/tests/integration/test_api.py @@ -837,6 +837,17 @@ def test_list_schedules(self): self.assertEqual(2, response.json()["count"]) self.assertEqual(2, len(response.json()["results"])) + def test_list_schedules_scheduler_id(self): + response = self.client.get(f"/schedules?scheduler_id={self.scheduler.scheduler_id}") + self.assertEqual(200, response.status_code) + self.assertEqual(2, response.json()["count"]) + self.assertEqual(2, len(response.json()["results"])) + + response = self.client.get(f"/schedules?scheduler_id={uuid.uuid4()}") + self.assertEqual(200, response.status_code) + self.assertEqual(0, response.json()["count"]) + self.assertEqual(0, len(response.json()["results"])) + def test_list_schedules_enabled(self): response = self.client.get("/schedules?enabled=true") self.assertEqual(200, response.status_code) diff --git a/mula/tests/integration/test_app.py b/mula/tests/integration/test_app.py index 4fd380ca3ee..0c5ae739c76 100644 --- a/mula/tests/integration/test_app.py +++ b/mula/tests/integration/test_app.py @@ -50,9 +50,9 @@ def test_monitor_orgs_add(self): # Act self.app.monitor_organisations() - # Assert: four schedulers should have been created for two organisations - self.assertEqual(4, len(self.app.schedulers.keys())) - self.assertEqual(4, len(self.app.server.schedulers.keys())) + # Assert: six schedulers should have been created for two organisations + self.assertEqual(6, len(self.app.schedulers.keys())) + self.assertEqual(6, len(self.app.server.schedulers.keys())) scheduler_org_ids = {s.organisation.id for s in self.app.schedulers.values()} self.assertEqual({"org-1", "org-2"}, scheduler_org_ids) @@ -68,9 +68,9 @@ def test_monitor_orgs_remove(self): # Act self.app.monitor_organisations() - # Assert: four schedulers should have been created for two organisations - self.assertEqual(4, len(self.app.schedulers.keys())) - self.assertEqual(4, len(self.app.server.schedulers.keys())) + # Assert: six schedulers should have been created for two organisations + self.assertEqual(6, len(self.app.schedulers.keys())) + self.assertEqual(6, len(self.app.server.schedulers.keys())) scheduler_org_ids = {s.organisation.id for s in self.app.schedulers.values()} self.assertEqual({"org-1", "org-2"}, scheduler_org_ids) @@ -100,9 +100,9 @@ def test_monitor_orgs_add_and_remove(self): # Act self.app.monitor_organisations() - # Assert: four schedulers should have been created for two organisations - self.assertEqual(4, len(self.app.schedulers.keys())) - self.assertEqual(4, len(self.app.server.schedulers.keys())) + # Assert: six schedulers should have been created for two organisations + self.assertEqual(6, len(self.app.schedulers.keys())) + self.assertEqual(6, len(self.app.server.schedulers.keys())) scheduler_org_ids = {s.organisation.id for s in self.app.schedulers.values()} self.assertEqual({"org-1", "org-2"}, scheduler_org_ids) @@ -117,8 +117,8 @@ def test_monitor_orgs_add_and_remove(self): self.app.monitor_organisations() # Assert - self.assertEqual(4, len(self.app.schedulers.keys())) - self.assertEqual(4, len(self.app.server.schedulers.keys())) + self.assertEqual(6, len(self.app.schedulers.keys())) + self.assertEqual(6, len(self.app.server.schedulers.keys())) scheduler_org_ids = {s.organisation.id for s in self.app.schedulers.values()} self.assertEqual({"org-1", "org-3"}, scheduler_org_ids) diff --git a/mula/tests/integration/test_report_scheduler.py b/mula/tests/integration/test_report_scheduler.py new file mode 100644 index 00000000000..f0d93232a47 --- /dev/null +++ b/mula/tests/integration/test_report_scheduler.py @@ -0,0 +1,162 @@ +import unittest +from types import SimpleNamespace +from unittest import mock + +from scheduler import config, models, schedulers, storage + +from tests.factories import OrganisationFactory + + +class ReportSchedulerBaseTestCase(unittest.TestCase): + def setUp(self): + # Application Context + self.mock_ctx = mock.patch("scheduler.context.AppContext").start() + self.mock_ctx.config = config.settings.Settings() + + # Database + self.dbconn = storage.DBConn(str(self.mock_ctx.config.db_uri)) + self.dbconn.connect() + models.Base.metadata.drop_all(self.dbconn.engine) + models.Base.metadata.create_all(self.dbconn.engine) + + self.mock_ctx.datastores = SimpleNamespace( + **{ + storage.TaskStore.name: storage.TaskStore(self.dbconn), + storage.PriorityQueueStore.name: storage.PriorityQueueStore(self.dbconn), + storage.ScheduleStore.name: storage.ScheduleStore(self.dbconn), + } + ) + + # Scheduler + self.organisation = OrganisationFactory() + self.scheduler = schedulers.ReportScheduler( + ctx=self.mock_ctx, + scheduler_id=self.organisation.id, + organisation=self.organisation, + ) + + def tearDown(self): + self.scheduler.stop() + models.Base.metadata.drop_all(self.dbconn.engine) + self.dbconn.engine.dispose() + + +class ReportSchedulerTestCase(ReportSchedulerBaseTestCase): + def setUp(self): + super().setUp() + + self.mock_get_schedules = mock.patch( + "scheduler.context.AppContext.datastores.schedule_store.get_schedules", + ).start() + + def tearDown(self): + mock.patch.stopall() + + def test_enable_scheduler(self): + # Disable scheduler first + self.scheduler.disable() + + # Threads should be stopped + self.assertEqual(0, len(self.scheduler.threads)) + + # Queue should be empty + self.assertEqual(0, self.scheduler.queue.qsize()) + + # Re-enable scheduler + self.scheduler.enable() + + # Threads should be started + self.assertGreater(len(self.scheduler.threads), 0) + + # Scheduler should be enabled + self.assertTrue(self.scheduler.is_enabled()) + + # Stop the scheduler + self.scheduler.stop() + + def test_disable_scheduler(self): + # Disable scheduler + self.scheduler.disable() + + # Threads should be stopped + self.assertEqual(0, len(self.scheduler.threads)) + + # Queue should be empty + self.assertEqual(0, self.scheduler.queue.qsize()) + + # Scheduler should be disabled + self.assertFalse(self.scheduler.is_enabled()) + + def test_push_tasks_for_rescheduling(self): + """When the deadline of schedules have passed, the resulting task should be added to the queue""" + # Arrange + report_task = models.ReportTask( + organisation_id=self.organisation.id, + report_recipe_id="123", + ) + + schedule = models.Schedule( + scheduler_id=self.scheduler.scheduler_id, + hash=report_task.hash, + data=report_task.dict(), + ) + + schedule_db = self.mock_ctx.datastores.schedule_store.create_schedule(schedule) + + # Mocks + self.mock_get_schedules.return_value = ([schedule_db], 1) + + # Act + self.scheduler.push_tasks_for_rescheduling() + + # Assert: new item should be on queue + self.assertEqual(1, self.scheduler.queue.qsize()) + + # Assert: new item is created with a similar task + peek = self.scheduler.queue.peek(0) + self.assertEqual(schedule.hash, peek.hash) + + # Assert: task should be created, and should be the one that is queued + task_db = self.mock_ctx.datastores.task_store.get_task(peek.id) + self.assertIsNotNone(task_db) + self.assertEqual(peek.id, task_db.id) + + def test_push_tasks_for_rescheduling_item_on_queue(self): + """When the deadline of schedules have passed, the resulting task should be added to the queue""" + # Arrange + report_task = models.ReportTask( + organisation_id=self.organisation.id, + report_recipe_id="123", + ) + + schedule = models.Schedule( + scheduler_id=self.scheduler.scheduler_id, + hash=report_task.hash, + data=report_task.dict(), + ) + + schedule_db = self.mock_ctx.datastores.schedule_store.create_schedule(schedule) + + # Mocks + self.mock_get_schedules.return_value = ([schedule_db], 1) + + # Act + self.scheduler.push_tasks_for_rescheduling() + + # Assert: new item should be on queue + self.assertEqual(1, self.scheduler.queue.qsize()) + + # Assert: new item is created with a similar task + peek = self.scheduler.queue.peek(0) + self.assertEqual(schedule.hash, peek.hash) + + # Assert: task should be created, and should be the one that is queued + task_db = self.mock_ctx.datastores.task_store.get_task(peek.id) + self.assertIsNotNone(task_db) + self.assertEqual(peek.id, task_db.id) + + # Act: push again + self.scheduler.push_tasks_for_rescheduling() + + # Should only be one task on queue + self.assertEqual(1, self.scheduler.queue.qsize()) From 4958776f498b2edc9c54df45293315eea0c1da40 Mon Sep 17 00:00:00 2001 From: Rieven Date: Tue, 10 Sep 2024 16:58:50 +0200 Subject: [PATCH 15/28] Fix report types selection not being overriden (#3436) Co-authored-by: Peter-Paul van Gemerden Co-authored-by: ammar92 Co-authored-by: Jan Klopper --- .../templates/partials/report_types_selection.html | 9 +++++---- rocky/reports/views/aggregate_report.py | 4 ++++ rocky/reports/views/generate_report.py | 4 +++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/rocky/reports/templates/partials/report_types_selection.html b/rocky/reports/templates/partials/report_types_selection.html index 3c401605847..066add7c640 100644 --- a/rocky/reports/templates/partials/report_types_selection.html +++ b/rocky/reports/templates/partials/report_types_selection.html @@ -14,7 +14,7 @@

{% translate "Choose report types" %}

{% endblocktranslate %}

{% if not selected_oois %} - {% include "partials/return_button.html" with btn_text="Go back" %} + {% include "partials/return_button.html" with btn_text="Go back" selected_report_types=None %} {% else %}

@@ -40,7 +40,7 @@

action="{{ previous }}" class="inline layout-wide"> {% csrf_token %} - {% include "forms/report_form_fields.html" %} + {% include "forms/report_form_fields.html" with selected_report_types=None %} + + + + + + +{% endblock content %} diff --git a/rocky/katalogus/templates/boefjes.html b/rocky/katalogus/templates/boefjes.html index 91c55b0eb4b..445f0d7e9dc 100644 --- a/rocky/katalogus/templates/boefjes.html +++ b/rocky/katalogus/templates/boefjes.html @@ -16,14 +16,21 @@

Boefjes

{% blocktranslate trimmed %} - Boefjes gather factual information, such as by calling an - external scanning tool like nmap or using a database like shodan. + Boefjes are used to scan for objects. They detect vulnerabilities, + security issues, and give insight. Each boefje is a separate scan that + can run on a selection of objects. {% endblocktranslate %}

{{ object_list|length }} Boefje{{ object_list|pluralize:"s" }} {% translate "available" %}

+ {% if perms.tools.can_set_katalogus_settings %} +
+ {% translate "Add Boefje" %} +
+ {% endif %} {% include "partials/katalogus_filter.html" with form=form %} {% include "partials/katalogus_toolbar.html" %} diff --git a/rocky/katalogus/templates/plugin_container_image.html b/rocky/katalogus/templates/plugin_container_image.html new file mode 100644 index 00000000000..674fcd4a7f4 --- /dev/null +++ b/rocky/katalogus/templates/plugin_container_image.html @@ -0,0 +1,85 @@ +{% load static %} +{% load i18n %} + +
+

{% translate "Container image" %}

+

+ {% translate "The container image for this Boefje is:" %} {{ plugin.oci_image }} +

+
+
+

{% translate "Variants" %}

+

+ {% blocktranslate %} + Boefje variants that use the same container image. For more + information about Boefje variants you can read the documentation. + {% endblocktranslate %} +

+
+ {% if variants %} +
+ {% translate "Add variant" %} +
+ {% endif %} +
+ {% if variants %} +
+
    +
  • + +
    + + + + + + + + + + + + + {% for variant in variants %} + + + + + + + + + + + {% endfor %} + +
    {% translate "Name" %}{% translate "Scan level" %}{% translate "Published by" %}{% translate "Status" %}
    {{ variant.name }}name{{ variant.scan_level }}scan_level{{ variant.published_by }}published_by{{ variant.status }}status + +
    +
    {% translate "Arguments" %}
    +

    {% translate "The following arguments are used for this Boefje variant." %}

    +
    +

    Some code example

    +
    +
    +
    +
    • +
+
+ {% else %} +

+ {% translate "This Boefje has no variants yet." %} + {% blocktranslate trimmed %} + You can make a variant and change the arguments and JSON Schema + to customize it to fit your needs. + {% endblocktranslate %} +

+ {% endif %} +
diff --git a/rocky/katalogus/templates/plugin_settings_list.html b/rocky/katalogus/templates/plugin_settings_list.html index 3ca5ca51dfb..545d88c229e 100644 --- a/rocky/katalogus/templates/plugin_settings_list.html +++ b/rocky/katalogus/templates/plugin_settings_list.html @@ -2,62 +2,56 @@ {% load i18n %} {% if object_list %} -
-
-

{{ plugin.type|title }}{% translate " Details" %}

-

{% translate "Settings" %}

-
- - - +
+
+
+

{% translate "Settings" %}

+

+ {% blocktranslate %} + In the table below the settings for this specific Boefje can be seen. + Set or change the value of the variables by editing the settings. + {% endblocktranslate %} +

+
+
+ {% translate "Edit Settings" %} +
+
+
+
+ + + + + + + + + + {% for setting in object_list %} - - - - - - - - {% for setting in object_list %} - - - - + + - + - + {% elif setting.secret %} + ••••••••••••• {% else %} - + {{ setting.value }} {% endif %} - - {% endfor %} - -
{% translate "Variable" %}{% translate "Value" %}{% translate "Required" %}
{% translate "Name" %}{% translate "Value" %}{% translate "Required" %}{% translate "Action" %}
{{ setting.name }} - {% if setting.value is None %} - {% translate "Unset" %} - {% elif setting.secret %} - ••••••••••••• - {% else %} - {{ setting.value }} - {% endif %} - - {% if setting.required %} - {% translate "Yes" %} - {% else %} - {% translate "No" %} - {% endif %} - {{ setting.name }} {% if setting.value is None %} - - {% translate "Add" %} - - {% translate "Edit" %} -
-
- {% csrf_token %} - -
-
+
+ {% if setting.required %} + {% translate "Yes" %} + {% else %} + {% translate "No" %} + {% endif %} +
-
+ {% endif %} diff --git a/rocky/katalogus/urls.py b/rocky/katalogus/urls.py index 7a254ce01da..90fea04b60d 100644 --- a/rocky/katalogus/urls.py +++ b/rocky/katalogus/urls.py @@ -1,5 +1,6 @@ -from django.urls import path +from django.urls import path, re_path +from katalogus.views.boefje_setup import BoefjeSetupView from katalogus.views.change_clearance_level import ChangeClearanceLevel from katalogus.views.katalogus import ( AboutPluginsView, @@ -32,7 +33,12 @@ name="confirm_clone_settings", ), path( - "plugins/boefjes//", + "plugins/boefjes/add/", + BoefjeSetupView.as_view(), + name="boefje_setup", + ), + re_path( + r"^plugins/boefjes/(?P(grid|table))/$", BoefjeListView.as_view(), name="boefjes_list", ), diff --git a/rocky/katalogus/views/boefje_setup.py b/rocky/katalogus/views/boefje_setup.py new file mode 100644 index 00000000000..18e608c0158 --- /dev/null +++ b/rocky/katalogus/views/boefje_setup.py @@ -0,0 +1,61 @@ +import uuid +from datetime import datetime + +from account.mixins import OrganizationPermissionRequiredMixin, OrganizationView +from django.shortcuts import redirect +from django.urls import reverse +from django.views.generic.edit import FormView +from tools.forms.boefje import BoefjeAddForm + +from katalogus.client import Boefje, get_katalogus +from octopoes.models.types import type_by_name + + +class BoefjeSetupView(OrganizationPermissionRequiredMixin, OrganizationView, FormView): + """View where the user can create a new boefje""" + + template_name = "boefje_setup.html" + form_class = BoefjeAddForm + permission_required = "tools.can_set_katalogus_settings" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + context["breadcrumbs"] = [ + {"url": reverse("katalogus", kwargs={"organization_code": self.organization.code}), "text": "KAT-alogus"}, + { + "url": reverse("boefje_setup", kwargs={"organization_code": self.organization.code}), + "text": "Boefje setup", + }, + ] + + return context + + def form_valid(self, form): + """If the form is valid, redirect to the supplied URL.""" + form_data = form.cleaned_data + input_object = {type_by_name(form_data["consumes"])} + arguments = form_data["oci_arguments"].split() + produces = form_data["produces"].replace(",", "").split() + boefje_id = str(uuid.uuid4()) + + boefje = Boefje( + id=boefje_id, + name=form_data["name"] or None, + created=str(datetime.now()), + description=form_data["description"] or None, + enabled=False, + type="boefje", + scan_level=form_data["scan_level"], + consumes=input_object, + produces=produces, + schema=form_data["schema"], + oci_image=form_data["oci_image"] or None, + oci_arguments=arguments, + ) + + get_katalogus(self.organization.code).create_plugin(boefje) + + return redirect( + reverse("boefje_detail", kwargs={"organization_code": self.organization.code, "plugin_id": boefje_id}) + ) diff --git a/rocky/katalogus/views/plugin_settings_list.py b/rocky/katalogus/views/plugin_settings_list.py index f27b16f1d43..90c66269254 100644 --- a/rocky/katalogus/views/plugin_settings_list.py +++ b/rocky/katalogus/views/plugin_settings_list.py @@ -24,7 +24,7 @@ def get_plugin_settings(self) -> list[dict[str, Any]]: return [] settings = self.katalogus_client.get_plugin_settings(plugin_id=self.plugin.id) - props = self.plugin_schema["properties"] + props = self.plugin_schema.get("properties", []) return [ { diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 856c78aeee1..3591cc84059 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -27,13 +27,13 @@ msgid "Important dates" msgstr "" #: account/forms/account_setup.py katalogus/templates/katalogus_settings.html -#: katalogus/templates/plugin_settings_list.html +#: katalogus/templates/plugin_container_image.html #: reports/report_types/dns_report/report.html #: reports/report_types/tls_report/report.html #: reports/templates/partials/export_report_settings.html #: reports/templates/report_overview/report_history_table.html #: reports/templates/report_overview/subreports_table.html -#: rocky/templates/organizations/organization_list.html +#: tools/forms/boefje.py rocky/templates/organizations/organization_list.html #: rocky/templates/organizations/organization_settings.html #: rocky/templates/partials/ooi_detail_related_object.html msgid "Name" @@ -647,8 +647,9 @@ msgstr "" #: katalogus/templates/about_plugins.html katalogus/templates/boefjes.html msgid "" -"Boefjes gather factual information, such as by calling an external scanning " -"tool like nmap or using a database like shodan." +"Boefjes are used to scan for objects. They detect vulnerabilities, security " +"issues, and give insight. Each boefje is a separate scan that can run on a " +"selection of objects." msgstr "" #: katalogus/templates/about_plugins.html katalogus/templates/normalizers.html @@ -665,21 +666,10 @@ msgstr "" #: katalogus/templates/boefje_detail.html #: katalogus/templates/partials/plugin_tile_modal.html +#: katalogus/templates/plugin_container_image.html msgid "Scan level" msgstr "" -#: katalogus/templates/boefje_detail.html -#: katalogus/templates/normalizer_detail.html -#: reports/report_types/aggregate_organisation_report/appendix.html -#: reports/report_types/dns_report/report.html -#: reports/report_types/findings_report/report.html -#: reports/report_types/vulnerability_report/report.html -#: reports/templates/summary/report_asset_overview.html -#: tools/forms/finding_type.py rocky/templates/oois/ooi_detail.html -#: rocky/templates/oois/ooi_detail_findings_list.html rocky/templates/scan.html -msgid "Description" -msgstr "" - #: katalogus/templates/boefje_detail.html #: katalogus/templates/normalizer_detail.html msgid "Consumes" @@ -709,10 +699,42 @@ msgstr "" msgid "%(plugin_name)s can produce the following output:" msgstr "" +#: katalogus/templates/boefje_detail.html +#, python-format +msgid "%(plugin_name)s doesn't produce any output mime types." +msgstr "" + +#: katalogus/templates/boefje_setup.html +msgid "Boefje setup" +msgstr "" + +#: katalogus/templates/boefje_setup.html +msgid "" +"\n" +" You can create a new Boefje. If you want more " +"information on this,\n" +" you can check out the documentation.\n" +" " +msgstr "" + +#: katalogus/templates/boefje_setup.html +msgid "Create variant" +msgstr "" + +#: katalogus/templates/boefje_setup.html +msgid "Discard variant" +msgstr "" + #: katalogus/templates/boefjes.html katalogus/templates/normalizers.html msgid "available" msgstr "" +#: katalogus/templates/boefjes.html +msgid "Add Boefje" +msgstr "" + #: katalogus/templates/change_clearance_level.html #: katalogus/templates/partials/objects_to_scan.html #: reports/templates/partials/report_setup_scan.html @@ -753,8 +775,8 @@ msgstr "" #: katalogus/templates/change_clearance_level.html onboarding/forms.py #: reports/templates/partials/report_ooi_list.html -#: reports/templates/summary/ooi_selection.html tools/forms/ooi.py -#: rocky/templates/oois/ooi_page_tabs.html +#: reports/templates/summary/ooi_selection.html tools/forms/boefje.py +#: tools/forms/ooi.py rocky/templates/oois/ooi_page_tabs.html #: rocky/templates/partials/explanations.html msgid "Clearance level" msgstr "" @@ -837,6 +859,17 @@ msgstr "" msgid "Value" msgstr "" +#: katalogus/templates/normalizer_detail.html +#: reports/report_types/aggregate_organisation_report/appendix.html +#: reports/report_types/dns_report/report.html +#: reports/report_types/findings_report/report.html +#: reports/report_types/vulnerability_report/report.html +#: reports/templates/summary/report_asset_overview.html tools/forms/boefje.py +#: tools/forms/finding_type.py rocky/templates/oois/ooi_detail.html +#: rocky/templates/oois/ooi_detail_findings_list.html rocky/templates/scan.html +msgid "Description" +msgstr "" + #: katalogus/templates/normalizer_detail.html #, python-format msgid "%(plugin_name)s is able to process the following mime types:" @@ -990,7 +1023,6 @@ msgid "Required settings" msgstr "" #: katalogus/templates/partials/plugin_settings_required.html -#: katalogus/templates/plugin_settings_list.html #: rocky/templates/findings/finding_add.html #: rocky/templates/partials/ooi_detail_related_object.html #: rocky/templates/partials/ooi_list_toolbar.html @@ -1064,6 +1096,98 @@ msgstr "" msgid "All" msgstr "" +#: katalogus/templates/plugin_container_image.html tools/forms/boefje.py +msgid "Container image" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "The container image for this Boefje is:" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "Variants" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "" +"\n" +" Boefje variants that use the same container image. For " +"more\n" +" information about Boefje variants you can read the " +"documentation.\n" +" " +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "Add variant" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "Overview of variants" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "Published by" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +#: reports/report_types/tls_report/report.html +#: reports/templates/partials/plugin_overview_table.html +#: rocky/templates/organizations/organization_member_list.html +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html +msgid "Status" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +#: reports/report_types/dns_report/report.html +#: reports/report_types/findings_report/report.html +#: reports/report_types/vulnerability_report/report.html +#: rocky/templates/crisis_room/crisis_room_findings_block.html +#: rocky/templates/findings/finding_list.html +#: rocky/templates/organizations/organization_crisis_room.html +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html +msgid "Close details" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +#: reports/report_types/dns_report/report.html +#: reports/report_types/findings_report/report.html +#: reports/report_types/vulnerability_report/report.html +#: rocky/templates/crisis_room/crisis_room_findings_block.html +#: rocky/templates/findings/finding_list.html +#: rocky/templates/organizations/organization_crisis_room.html +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html +msgid "Open details" +msgstr "" + +#: katalogus/templates/plugin_container_image.html tools/forms/boefje.py +msgid "Arguments" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "The following arguments are used for this Boefje variant." +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "no variants explanation" +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "This Boefje has no variants yet." +msgstr "" + +#: katalogus/templates/plugin_container_image.html +msgid "" +"You can make a variant and change the arguments and JSON Schema to customize " +"it to fit your needs." +msgstr "" + #: katalogus/templates/plugin_settings_add.html msgid "" "\n" @@ -1123,30 +1247,35 @@ msgid "" msgstr "" #: katalogus/templates/plugin_settings_delete.html -#: katalogus/templates/plugin_settings_list.html #: katalogus/views/plugin_settings_delete.py #: rocky/templates/admin/delete_confirmation.html rocky/views/ooi_delete.py msgid "Delete" msgstr "" #: katalogus/templates/plugin_settings_list.html -msgid " Details" +msgid "" +"\n" +" In the table below the settings for this specific " +"Boefje can be seen.\n" +" Set or change the value of the variables by editing " +"the settings.\n" +" " msgstr "" #: katalogus/templates/plugin_settings_list.html -msgid "Overview of settings" +msgid "Edit Settings" msgstr "" #: katalogus/templates/plugin_settings_list.html -msgid "Required" +msgid "Overview of settings" msgstr "" #: katalogus/templates/plugin_settings_list.html -msgid "Action" +msgid "Variable" msgstr "" #: katalogus/templates/plugin_settings_list.html -msgid "Unset" +msgid "Required" msgstr "" #: katalogus/templates/plugin_settings_list.html @@ -1159,13 +1288,6 @@ msgstr "" msgid "No" msgstr "" -#: katalogus/templates/plugin_settings_list.html -#: rocky/templates/organizations/organization_member_list.html -#: rocky/templates/organizations/organization_settings.html -#: rocky/views/ooi_edit.py rocky/views/organization_edit.py -msgid "Edit" -msgstr "" - #: katalogus/views/change_clearance_level.py msgid "Session has terminated, please select objects again." msgstr "" @@ -2731,30 +2853,6 @@ msgstr "" msgid "Details" msgstr "" -#: reports/report_types/dns_report/report.html -#: reports/report_types/findings_report/report.html -#: reports/report_types/vulnerability_report/report.html -#: rocky/templates/crisis_room/crisis_room_findings_block.html -#: rocky/templates/findings/finding_list.html -#: rocky/templates/organizations/organization_crisis_room.html -#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html -#: rocky/templates/tasks/ooi_detail_task_list.html -#: rocky/templates/tasks/plugin_detail_task_list.html -msgid "Close details" -msgstr "" - -#: reports/report_types/dns_report/report.html -#: reports/report_types/findings_report/report.html -#: reports/report_types/vulnerability_report/report.html -#: rocky/templates/crisis_room/crisis_room_findings_block.html -#: rocky/templates/findings/finding_list.html -#: rocky/templates/organizations/organization_crisis_room.html -#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html -#: rocky/templates/tasks/ooi_detail_task_list.html -#: rocky/templates/tasks/plugin_detail_task_list.html -msgid "Open details" -msgstr "" - #: reports/report_types/dns_report/report.html msgid "Findings information" msgstr "" @@ -3226,15 +3324,6 @@ msgstr "" msgid "Ciphers" msgstr "" -#: reports/report_types/tls_report/report.html -#: reports/templates/partials/plugin_overview_table.html -#: rocky/templates/organizations/organization_member_list.html -#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html -#: rocky/templates/tasks/ooi_detail_task_list.html -#: rocky/templates/tasks/plugin_detail_task_list.html -msgid "Status" -msgstr "" - #: reports/report_types/tls_report/report.html msgid "Protocol" msgstr "" @@ -4006,6 +4095,18 @@ msgstr "" msgid "The selected date is in the future. Please select a different date." msgstr "" +#: tools/forms/boefje.py +msgid "JSON Schema" +msgstr "" + +#: tools/forms/boefje.py +msgid "Input object type" +msgstr "" + +#: tools/forms/boefje.py +msgid "Output mime types" +msgstr "" + #: tools/forms/finding_type.py msgid "KAT-ID" msgstr "" @@ -4292,6 +4393,47 @@ msgid "" "the view to represent that moment in time." msgstr "" +#: tools/forms/settings.py +msgid "" +"

A description of the boefje explaining in short what it can do. This will " +"both be displayed inside the KAT-alogus and on the Boefje details page.

" +msgstr "" + +#: tools/forms/settings.py +msgid "" +"

If any other settings are needed for your Boefje, add these as a JSON " +"Schema, otherwise, leave the field empty or 'null'.

This JSON is " +"used as the basis for a form for the user. When the user enables this Boefje " +"they can get the option to give extra information. For example, it can " +"contain an API key that the script requires.

More information about " +"what the schema.json file looks like can be found here.

" +msgstr "" + +#: tools/forms/settings.py +msgid "" +"

Select the object type that your Boefje consumes.

This object type " +"triggers the Boefje to run. Whenever this OOI gets added, this Boefje will " +"run with that OOI.

" +msgstr "" + +#: tools/forms/settings.py +msgid "" +"

Add a set of mime types that are produced by this Boefje, separated by " +"commas. For example: 'text/html', 'image/jpeg' or 'boefje/" +"{boefje-id}'

These output mime types will be shown on the Boefje " +"detail page as information for other users.

" +msgstr "" + +#: tools/forms/settings.py +msgid "" +"

Select a clearance level for your Boefje. For more information about the " +"different clearance levels please check the documentation." +"

" +msgstr "" + #: tools/forms/settings.py msgid "Depth of the tree." msgstr "" @@ -5551,6 +5693,12 @@ msgstr "" msgid "Assigned clearance level" msgstr "" +#: rocky/templates/organizations/organization_member_list.html +#: rocky/templates/organizations/organization_settings.html +#: rocky/views/ooi_edit.py rocky/views/organization_edit.py +msgid "Edit" +msgstr "" + #: rocky/templates/organizations/organization_member_list.html msgid "Super user" msgstr "" diff --git a/rocky/rocky/templates/partials/form/field_input.html b/rocky/rocky/templates/partials/form/field_input.html index fcaad77193a..d2d09ec65e7 100644 --- a/rocky/rocky/templates/partials/form/field_input.html +++ b/rocky/rocky/templates/partials/form/field_input.html @@ -1,7 +1,8 @@ {% load i18n %}
- {{ field.label_tag }} +

{{ field.label_tag }}

+

{{ field.field.widget.attrs.description }}

{% if form_view != "vertical" %}
{% if field.field.required %} diff --git a/rocky/rocky/templates/tasks/normalizers.html b/rocky/rocky/templates/tasks/normalizers.html index 340dad142de..90353c04698 100644 --- a/rocky/rocky/templates/tasks/normalizers.html +++ b/rocky/rocky/templates/tasks/normalizers.html @@ -56,7 +56,11 @@

{% translate "Normalizers" %}

{{ task.created_at }} {{ task.modified_at }} - {{ task.data.raw_data.boefje_meta.boefje.id }} + {% if task.data.raw_data.boefje_meta.boefje.name %} + {{ task.data.raw_data.boefje_meta.boefje.name }} + {% else %} + {{ task.data.raw_data.boefje_meta.boefje.id }} + {% endif %} {{ task.data.raw_data.boefje_meta.input_ooi }} diff --git a/rocky/rocky/templates/tasks/plugin_detail_task_list.html b/rocky/rocky/templates/tasks/plugin_detail_task_list.html index 4e0778c4c76..80240a75031 100644 --- a/rocky/rocky/templates/tasks/plugin_detail_task_list.html +++ b/rocky/rocky/templates/tasks/plugin_detail_task_list.html @@ -3,11 +3,9 @@

{% translate "Tasks" %}

{% if not task_list %} -

{% translate "There are no tasks for" %} {{ plugin.name }}

- {% include "tasks/partials/task_filter.html" %} - +

{% translate "There are no tasks for" %} {{ plugin.name }}.

{% else %} -

{% translate "List of tasks for" %} {{ plugin.name }}

+

{% translate "List of tasks for" %} {{ plugin.name }}:

{% include "tasks/partials/task_filter.html" %} diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index e786260e9c8..ac260544261 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -1106,6 +1106,25 @@ def plugin_details(): ) +@pytest.fixture +def plugin_details_with_container(): + return parse_plugin( + { + "id": "test-boefje", + "type": "boefje", + "name": "TestBoefje", + "description": "Meows to the moon", + "scan_level": 1, + "consumes": ["Network"], + "produces": ["Network"], + "enabled": True, + "schema": {}, + "oci_image": "ghcr.io/test/image:123", + "oci_arguments": ["-test", "-arg"], + } + ) + + @pytest.fixture def plugin_schema(): return { diff --git a/rocky/tests/katalogus/test_katalogus_create_boefje.py b/rocky/tests/katalogus/test_katalogus_create_boefje.py new file mode 100644 index 00000000000..8229e375571 --- /dev/null +++ b/rocky/tests/katalogus/test_katalogus_create_boefje.py @@ -0,0 +1,21 @@ +from katalogus.views.boefje_setup import BoefjeSetupView +from pytest_django.asserts import assertContains + +from tests.conftest import setup_request + + +def test_boefje_setup(rf, superuser_member): + request = setup_request(rf.get("boefje_setup"), superuser_member.user) + response = BoefjeSetupView.as_view()(request, organization_code=superuser_member.organization.code) + + assert response.status_code == 200 + assertContains(response, "Boefje setup") + assertContains(response, "Container image") + assertContains(response, "Name") + assertContains(response, "Description") + assertContains(response, "Arguments") + assertContains(response, "JSON Schema") + assertContains(response, "Input object type") + assertContains(response, "Output mime types") + assertContains(response, "Clearance level") + assertContains(response, "Create variant") diff --git a/rocky/tests/katalogus/test_katalogus_plugin_detail.py b/rocky/tests/katalogus/test_katalogus_plugin_detail.py index 1585f32f436..ceeb6d030b4 100644 --- a/rocky/tests/katalogus/test_katalogus_plugin_detail.py +++ b/rocky/tests/katalogus/test_katalogus_plugin_detail.py @@ -1,5 +1,5 @@ from katalogus.views.plugin_detail import BoefjeDetailView -from pytest_django.asserts import assertContains +from pytest_django.asserts import assertContains, assertNotContains from tests.conftest import setup_request @@ -27,6 +27,35 @@ def test_plugin_detail_view( assertContains(response, "Object list") assertContains(response, "Consumes") assertContains(response, plugin_details.description) + assertNotContains(response, "Container image") + assertNotContains(response, "Variants") + + +def test_plugin_detail_view_with_container_image( + rf, + superuser_member, + mock_mixins_katalogus, + plugin_details_with_container, + mock_organization_view_octopoes, + mock_scheduler_client_task_list, +): + mock_mixins_katalogus().get_plugin.return_value = plugin_details_with_container + + request = setup_request(rf.get("boefje_detail"), superuser_member.user) + response = BoefjeDetailView.as_view()( + request, + organization_code=superuser_member.organization.code, + plugin_id="test-plugin", + ) + + assertContains(response, "TestBoefje") + assertContains(response, "Container image") + assertContains(response, "Variants") + assertContains(response, "Produces") + assertContains(response, "Tasks") + assertContains(response, "Object list") + assertContains(response, "Consumes") + assertContains(response, plugin_details_with_container.description) def test_plugin_detail_view_no_consumes( diff --git a/rocky/tools/forms/boefje.py b/rocky/tools/forms/boefje.py new file mode 100644 index 00000000000..e0cdb1ef269 --- /dev/null +++ b/rocky/tools/forms/boefje.py @@ -0,0 +1,67 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +from octopoes.models.types import ALL_TYPES +from tools.enums import SCAN_LEVEL +from tools.forms.base import BaseRockyForm +from tools.forms.settings import ( + BOEFJE_CONSUMES_HELP_TEXT, + BOEFJE_DESCRIPTION_HELP_TEXT, + BOEFJE_PRODUCES_HELP_TEXT, + BOEFJE_SCAN_LEVEL_HELP_TEXT, + BOEFJE_SCHEMA_HELP_TEXT, +) + +OOI_TYPE_CHOICES = sorted((ooi_type.get_object_type(), ooi_type.get_object_type()) for ooi_type in ALL_TYPES) + + +class BoefjeAddForm(BaseRockyForm): + oci_image = forms.CharField( + required=True, + label=_("Container image"), + widget=forms.TextInput( + attrs={ + "description": "The name of the Docker image. For example: ghcr.io/minvws/openkat/nmap", + "aria-describedby": "input-description", + } + ), + ) + name = forms.CharField( + required=True, + label=_("Name"), + ) + description = forms.CharField( + required=False, + label=_("Description"), + widget=forms.Textarea(attrs={"rows": 3}), + help_text=BOEFJE_DESCRIPTION_HELP_TEXT, + ) + oci_arguments = forms.CharField( + required=False, + label=_("Arguments"), + widget=forms.TextInput( + attrs={"description": "For example: -sTU --top-ports 1000", "aria-describedby": "input-description"} + ), + ) + schema = forms.JSONField( + required=False, + label=_("JSON Schema"), + help_text=BOEFJE_SCHEMA_HELP_TEXT, + ) + consumes = forms.CharField( + required=False, + label=_("Input object type"), + widget=forms.Select(choices=OOI_TYPE_CHOICES), + help_text=BOEFJE_CONSUMES_HELP_TEXT, + ) + produces = forms.CharField( + required=False, + label=_("Output mime types"), + help_text=BOEFJE_PRODUCES_HELP_TEXT, + ) + scan_level = forms.CharField( + required=False, + label=_("Clearance level"), + widget=forms.Select(choices=SCAN_LEVEL.choices), + help_text=BOEFJE_SCAN_LEVEL_HELP_TEXT, + ) diff --git a/rocky/tools/forms/settings.py b/rocky/tools/forms/settings.py index 3e107731ff7..2bf599a3e98 100644 --- a/rocky/tools/forms/settings.py +++ b/rocky/tools/forms/settings.py @@ -1,5 +1,6 @@ from typing import Any +from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ from tools.enums import SCAN_LEVEL @@ -58,6 +59,51 @@ "Select a datetime to change the view to represent that moment in time." ) +BOEFJE_DESCRIPTION_HELP_TEXT = mark_safe( + _( + "

A description of the boefje explaining in short what it can do. " + "This will both be displayed inside the KAT-alogus and on the Boefje details page.

" + ) +) + +BOEFJE_SCHEMA_HELP_TEXT = mark_safe( + _( + "

If any other settings are needed for your Boefje, add these as a JSON Schema, " + "otherwise, leave the field empty or 'null'.

" + "

This JSON is used as the basis for a form for the user. " + "When the user enables this Boefje they can get the option to give extra information. " + "For example, it can contain an API key that the script requires.

" + "

More information about what the schema.json file looks like can be found " + " " + "here.

" + ) +) + +BOEFJE_CONSUMES_HELP_TEXT = mark_safe( + _( + "

Select the object type that your Boefje consumes.

" + "

This object type triggers the Boefje to run. Whenever this OOI gets added, " + "this Boefje will run with that OOI.

" + ) +) + + +BOEFJE_PRODUCES_HELP_TEXT = mark_safe( + _( + "

Add a set of mime types that are produced by this Boefje, separated by commas. " + "For example: 'text/html', 'image/jpeg' or 'boefje/{boefje-id}'

" + "

These output mime types will be shown on the Boefje detail page as information for other users.

" + ) +) +BOEFJE_SCAN_LEVEL_HELP_TEXT = mark_safe( + _( + "

Select a clearance level for your Boefje. For more information about the different " + "clearance levels please check the " + " " + "documentation.

" + ) +) + DEPTH_DEFAULT = 9 DEPTH_MAX = 15 DEPTH_HELP_TEXT = _("Depth of the tree.") diff --git a/rocky/tools/management/commands/setup_dev_account.py b/rocky/tools/management/commands/setup_dev_account.py index ae401e8543b..ff0ea20f036 100644 --- a/rocky/tools/management/commands/setup_dev_account.py +++ b/rocky/tools/management/commands/setup_dev_account.py @@ -39,6 +39,7 @@ def handle(self, *args, **options): redteamer_permissions = [ "can_scan_organization", "can_enable_disable_boefje", + "can_add_boefje", "can_set_clearance_level", "can_delete_oois", "can_mute_findings", diff --git a/rocky/tools/migrations/0042_alter_organization_options.py b/rocky/tools/migrations/0042_alter_organization_options.py new file mode 100644 index 00000000000..6972b7a0ab7 --- /dev/null +++ b/rocky/tools/migrations/0042_alter_organization_options.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.8 on 2024-08-26 08:52 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("tools", "0001_squashed_0041_merge_20230731_1131"), + ] + + operations = [ + migrations.AlterModelOptions( + name="organization", + options={ + "permissions": ( + ("can_switch_organization", "Can switch organization"), + ("can_scan_organization", "Can scan organization"), + ("can_enable_disable_boefje", "Can enable or disable boefje"), + ("can_add_boefje", "Can add new or duplicated boefje"), + ("can_set_clearance_level", "Can set clearance level"), + ("can_delete_oois", "Can delete oois"), + ("can_mute_findings", "Can mute findings"), + ("can_view_katalogus_settings", "Can view KAT-alogus settings"), + ("can_set_katalogus_settings", "Can set KAT-alogus settings"), + ("can_recalculate_bits", "Can recalculate bits"), + ) + }, + ), + ] diff --git a/rocky/tools/migrations/0043_alter_organization_options.py b/rocky/tools/migrations/0043_alter_organization_options.py new file mode 100644 index 00000000000..2d662f68c64 --- /dev/null +++ b/rocky/tools/migrations/0043_alter_organization_options.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.8 on 2024-08-27 09:27 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("tools", "0042_alter_organization_options"), + ] + + operations = [ + migrations.AlterModelOptions( + name="organization", + options={ + "permissions": ( + ("can_switch_organization", "Can switch organization"), + ("can_scan_organization", "Can scan organization"), + ("can_enable_disable_boefje", "Can enable or disable boefje"), + ("can_add_boefje", "Can add new or duplicate boefjes"), + ("can_set_clearance_level", "Can set clearance level"), + ("can_delete_oois", "Can delete oois"), + ("can_mute_findings", "Can mute findings"), + ("can_view_katalogus_settings", "Can view KAT-alogus settings"), + ("can_set_katalogus_settings", "Can set KAT-alogus settings"), + ("can_recalculate_bits", "Can recalculate bits"), + ) + }, + ), + ] diff --git a/rocky/tools/models.py b/rocky/tools/models.py index 5e8cf119958..7cf68230ab8 100644 --- a/rocky/tools/models.py +++ b/rocky/tools/models.py @@ -101,6 +101,7 @@ class Meta: ("can_switch_organization", "Can switch organization"), ("can_scan_organization", "Can scan organization"), ("can_enable_disable_boefje", "Can enable or disable boefje"), + ("can_add_boefje", "Can add new or duplicate boefjes"), ("can_set_clearance_level", "Can set clearance level"), ("can_delete_oois", "Can delete oois"), ("can_mute_findings", "Can mute findings"), From de06d4a7741571f8dc1cdc4b59118e66c33a20af Mon Sep 17 00:00:00 2001 From: HeleenSG Date: Wed, 11 Sep 2024 11:02:59 +0200 Subject: [PATCH 18/28] feat: multi select dropdown (#3446) Co-authored-by: Roelof Korporaal Co-authored-by: Jan Klopper --- rocky/assets/css/components/destructive.scss | 41 +++++++++++ .../assets/css/components/dropdown-form.scss | 72 +++++++++++++++++++ rocky/assets/css/components/dropdown.scss | 4 ++ rocky/assets/css/components/form.scss | 21 ++++++ rocky/assets/css/main.scss | 2 + .../css/themes/soft/manon/checkbox.scss | 1 + rocky/assets/css/themes/soft/manon/form.scss | 1 + .../soft/manon/language-selector-list.scss | 2 +- rocky/assets/css/themes/soft/manon/link.scss | 4 ++ rocky/assets/css/themes/soft/manon/tags.scss | 1 + 10 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 rocky/assets/css/components/destructive.scss create mode 100644 rocky/assets/css/components/dropdown-form.scss diff --git a/rocky/assets/css/components/destructive.scss b/rocky/assets/css/components/destructive.scss new file mode 100644 index 00000000000..52a218ef2ff --- /dev/null +++ b/rocky/assets/css/components/destructive.scss @@ -0,0 +1,41 @@ +@mixin destructive-text-element { + &.destructive { + color: var(--color-alert-negative); + + &:hover { + color: var(--color-alert-negative-darker); + } + } +} + +@mixin destructive-button { + &.destructive { + color: var(--colors-white); + background-color: var(--color-alert-negative); + + &:hover { + background-color: var(--color-alert-negative-darker); + } + + &.ghost { + color: var(--color-alert-negative); + background-color: var(--color-white); + + &:hover { + color: var(--color-alert-negative-darker); + } + } + } +} + +a { + @include destructive-text-element; +} + +button, +a.button, +input[type="button"], +input[type="submit"], +input[type="reset"] { + @include destructive-button; +} diff --git a/rocky/assets/css/components/dropdown-form.scss b/rocky/assets/css/components/dropdown-form.scss new file mode 100644 index 00000000000..8d3b1e91b09 --- /dev/null +++ b/rocky/assets/css/components/dropdown-form.scss @@ -0,0 +1,72 @@ +@use "destructive"; + +.dropdown { + form { + $form-padding-sides: 1rem; + + background-color: transparent; + padding: 0.75rem $form-padding-sides 0 $form-padding-sides; + + .toolbar { + justify-content: space-between; + + a { + font-weight: bold; + text-decoration: none; + + @include destructive.destructive-text-element; + } + + button, + a.button, + input[type="button"], + input[type="submit"], + input[type="reset"] { + background-color: transparent; + color: var(--link-text-color); + border: 0; + padding: 0; + min-width: 0; + margin: 0; + min-height: 0; + border-radius: 0; + + &:hover { + color: var(--colors-blue-700); + } + + @include destructive.destructive-button; + + &.destructive { + color: var(--color-alert-negative); + background-color: transparent; + + &:hover { + color: var(--color-alert-negative-darker); + background-color: transparent; + } + } + } + } + + ul { + padding-left: 0; + + /* Negative margin added to allow li background color to full width of the form */ + margin-left: calc($form-padding-sides * -1); + margin-right: calc($form-padding-sides * -1); + + li { + min-height: var(--collapsing-element-list-item-min-height); + display: flex; + align-items: center; + padding-left: $form-padding-sides; + padding-right: $form-padding-sides; + + &:hover { + background-color: var(--colors-black-05); + } + } + } + } +} diff --git a/rocky/assets/css/components/dropdown.scss b/rocky/assets/css/components/dropdown.scss index e226bbdb21e..dd80fa95389 100644 --- a/rocky/assets/css/components/dropdown.scss +++ b/rocky/assets/css/components/dropdown.scss @@ -25,6 +25,9 @@ margin-block-start: 0; padding-inline-start: 0; gap: 0; + border-radius: var(--border-radius-s); + border: 1px solid var(--colors-grey-200); + background-color: var(--colors-white); &[aria-expanded="true"] { display: flex; @@ -37,6 +40,7 @@ background-color: var(--colors-white); list-style-type: disc; gap: 0; + border-radius: var(--border-radius-s); &:not(:last-child) { border-bottom: 1px solid var(--colors-grey-200); diff --git a/rocky/assets/css/components/form.scss b/rocky/assets/css/components/form.scss index d5a184cd9cb..cb741c63629 100644 --- a/rocky/assets/css/components/form.scss +++ b/rocky/assets/css/components/form.scss @@ -19,6 +19,27 @@ form { } } + .input-icon { + position: relative; + display: flex; + flex-direction: row; + align-items: center; + + input { + padding-right: 2.5rem; + + &:hover { + border-color: var(--colors-grey-700); + } + } + + .icon { + display: block; + margin-left: -1.25rem; + stroke: var(--colors-blue-600); + } + } + @media (min-width: $breakpoint-filter) { .filter-fields-direction { &.row { diff --git a/rocky/assets/css/main.scss b/rocky/assets/css/main.scss index f6171506258..541b3d8dd42 100644 --- a/rocky/assets/css/main.scss +++ b/rocky/assets/css/main.scss @@ -33,6 +33,8 @@ @import "components/cat-loader"; @import "components/cat-paw-loader"; @import "components/cytoscape"; +@import "components/destructive"; +@import "components/dropdown-form"; @import "components/dropdown"; @import "components/dropdown-list"; @import "components/filter"; diff --git a/rocky/assets/css/themes/soft/manon/checkbox.scss b/rocky/assets/css/themes/soft/manon/checkbox.scss index 7d58f541dc2..1e198264016 100644 --- a/rocky/assets/css/themes/soft/manon/checkbox.scss +++ b/rocky/assets/css/themes/soft/manon/checkbox.scss @@ -1,3 +1,4 @@ :root { --checkbox-accent-color: var(--branding-color-2, initial); + --form-fieldset-checkbox-margin: 0.75rem 0; } diff --git a/rocky/assets/css/themes/soft/manon/form.scss b/rocky/assets/css/themes/soft/manon/form.scss index cdfbe4649f3..b8f619912fe 100644 --- a/rocky/assets/css/themes/soft/manon/form.scss +++ b/rocky/assets/css/themes/soft/manon/form.scss @@ -12,6 +12,7 @@ --form-input-border-style: solid; --form-input-border-color: var(--colors-grey-200); --form-inline-gap: 1rem; + --form-input-border-radius: var(--border-radius-s); /* Help */ --form-help-button-icon-content: "\ec9d"; diff --git a/rocky/assets/css/themes/soft/manon/language-selector-list.scss b/rocky/assets/css/themes/soft/manon/language-selector-list.scss index 651c607f45b..4ad717484df 100644 --- a/rocky/assets/css/themes/soft/manon/language-selector-list.scss +++ b/rocky/assets/css/themes/soft/manon/language-selector-list.scss @@ -46,7 +46,7 @@ /* List item */ --language-selector-list-item-background-color: transparent; --language-selector-list-item-active-background-color: transparent; - --language-selector-list-item-hover-background-color: var(--colors-grey-100); + --language-selector-list-item-hover-background-color: var(--colors-black-05); /* List item link */ --language-selector-list-item-link-border-width: 0; diff --git a/rocky/assets/css/themes/soft/manon/link.scss b/rocky/assets/css/themes/soft/manon/link.scss index 30a8a69fef0..6238c19d40e 100644 --- a/rocky/assets/css/themes/soft/manon/link.scss +++ b/rocky/assets/css/themes/soft/manon/link.scss @@ -6,4 +6,8 @@ a { text-decoration-thickness: 0.045rem; + + &:hover { + color: var(--colors-blue-700); + } } diff --git a/rocky/assets/css/themes/soft/manon/tags.scss b/rocky/assets/css/themes/soft/manon/tags.scss index 78e97397ca2..a6b2bd95fa9 100644 --- a/rocky/assets/css/themes/soft/manon/tags.scss +++ b/rocky/assets/css/themes/soft/manon/tags.scss @@ -2,4 +2,5 @@ :root { --tag-font-size: var(--body-text-small-font-size); + --tag-font-weight: bold; } From 68eef2c4ae298361a0412a2538dfc6ba3bd6f1c7 Mon Sep 17 00:00:00 2001 From: noamblitz <43830693+noamblitz@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:47:49 +0200 Subject: [PATCH 19/28] Fix hanging upload of large files (#3489) Co-authored-by: Jeroen Dekkers Co-authored-by: Jan Klopper --- bytes/poetry.lock | 23 ++++++++++++++++++++++- bytes/pyproject.toml | 1 - bytes/requirements-dev.txt | 21 +++++++++++++++++++++ bytes/requirements.txt | 21 +++++++++++++++++++++ bytes/tests/client.py | 4 ++-- bytes/tests/integration/test_bytes_api.py | 17 +++++++++++++++++ 6 files changed, 83 insertions(+), 4 deletions(-) diff --git a/bytes/poetry.lock b/bytes/poetry.lock index f70bba87163..62b5b7ad32c 100644 --- a/bytes/poetry.lock +++ b/bytes/poetry.lock @@ -1474,30 +1474,51 @@ description = "Database Abstraction Library" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ + {file = "SQLAlchemy-1.4.51-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b"}, {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1"}, {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340"}, {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678"}, {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-win32.whl", hash = "sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-win_amd64.whl", hash = "sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d"}, {file = "SQLAlchemy-1.4.51-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f"}, {file = "SQLAlchemy-1.4.51-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-win32.whl", hash = "sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-win_amd64.whl", hash = "sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d"}, {file = "SQLAlchemy-1.4.51-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916"}, {file = "SQLAlchemy-1.4.51-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-win32.whl", hash = "sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-win_amd64.whl", hash = "sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0"}, {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd"}, {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b"}, {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c"}, {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-win32.whl", hash = "sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-win_amd64.whl", hash = "sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b"}, {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b"}, {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707"}, {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade"}, {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-win32.whl", hash = "sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-win_amd64.whl", hash = "sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298"}, {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12"}, {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c"}, {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3"}, {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-win32.whl", hash = "sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-win_amd64.whl", hash = "sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99"}, {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349"}, {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4"}, {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea"}, {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-win32.whl", hash = "sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-win_amd64.whl", hash = "sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba"}, {file = "SQLAlchemy-1.4.51.tar.gz", hash = "sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9"}, ] @@ -1714,4 +1735,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "b38550771cc3c4f564889d3c83e98d927e7b0db07f0cdbcb01b6c89a877fa2df" +content-hash = "58cec422be3f5e2e68f57975ffdcc243d64db0cb688abebd817fa9d0a7f3fbe1" diff --git a/bytes/pyproject.toml b/bytes/pyproject.toml index eb8534de8f3..e1bab9ef739 100644 --- a/bytes/pyproject.toml +++ b/bytes/pyproject.toml @@ -18,7 +18,6 @@ pynacl = "^1.5.0" rfc3161ng = "^2.1.3" sqlalchemy = "^1.4.48" uvicorn = "^0.29.0" -asgiref = "^3.8.1" # OpenTelemetry opentelemetry-sdk = "^1.26.0" diff --git a/bytes/requirements-dev.txt b/bytes/requirements-dev.txt index fa977408c9c..07b92536586 100644 --- a/bytes/requirements-dev.txt +++ b/bytes/requirements-dev.txt @@ -652,9 +652,12 @@ sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc sqlalchemy==1.4.51 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65 \ --hash=sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678 \ --hash=sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e \ --hash=sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea \ + --hash=sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0 \ + --hash=sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b \ --hash=sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb \ --hash=sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3 \ --hash=sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1 \ @@ -662,20 +665,38 @@ sqlalchemy==1.4.51 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b \ --hash=sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad \ --hash=sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707 \ + --hash=sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff \ + --hash=sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171 \ --hash=sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd \ + --hash=sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b \ + --hash=sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81 \ --hash=sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c \ + --hash=sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2 \ --hash=sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916 \ + --hash=sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee \ + --hash=sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298 \ --hash=sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f \ --hash=sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c \ --hash=sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340 \ + --hash=sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c \ + --hash=sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87 \ + --hash=sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba \ + --hash=sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d \ + --hash=sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1 \ --hash=sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072 \ + --hash=sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee \ + --hash=sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99 \ --hash=sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f \ --hash=sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894 \ + --hash=sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b \ + --hash=sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054 \ --hash=sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4 \ --hash=sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349 \ --hash=sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9 \ --hash=sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6 \ + --hash=sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d \ --hash=sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12 \ + --hash=sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3 \ --hash=sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade starlette==0.37.2 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee \ diff --git a/bytes/requirements.txt b/bytes/requirements.txt index dc6f031361e..b5c65385389 100644 --- a/bytes/requirements.txt +++ b/bytes/requirements.txt @@ -637,9 +637,12 @@ sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc sqlalchemy==1.4.51 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65 \ --hash=sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678 \ --hash=sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e \ --hash=sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea \ + --hash=sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0 \ + --hash=sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b \ --hash=sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb \ --hash=sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3 \ --hash=sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1 \ @@ -647,20 +650,38 @@ sqlalchemy==1.4.51 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b \ --hash=sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad \ --hash=sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707 \ + --hash=sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff \ + --hash=sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171 \ --hash=sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd \ + --hash=sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b \ + --hash=sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81 \ --hash=sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c \ + --hash=sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2 \ --hash=sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916 \ + --hash=sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee \ + --hash=sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298 \ --hash=sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f \ --hash=sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c \ --hash=sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340 \ + --hash=sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c \ + --hash=sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87 \ + --hash=sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba \ + --hash=sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d \ + --hash=sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1 \ --hash=sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072 \ + --hash=sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee \ + --hash=sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99 \ --hash=sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f \ --hash=sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894 \ + --hash=sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b \ + --hash=sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054 \ --hash=sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4 \ --hash=sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349 \ --hash=sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9 \ --hash=sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6 \ + --hash=sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d \ --hash=sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12 \ + --hash=sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3 \ --hash=sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade starlette==0.37.2 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee \ diff --git a/bytes/tests/client.py b/bytes/tests/client.py index 22405c75ffc..0fcdb95b8b5 100644 --- a/bytes/tests/client.py +++ b/bytes/tests/client.py @@ -6,7 +6,7 @@ from uuid import UUID import httpx -from httpx import HTTPError +from httpx import HTTPError, HTTPStatusError from bytes.api.models import BoefjeOutput from bytes.models import BoefjeMeta, NormalizerMeta @@ -23,7 +23,7 @@ def wrapper(self, *args, **kwargs): # type: ignore try: return function(self, *args, **kwargs) except HTTPError as error: - if error.response.status_code != 401: + if not isinstance(error, HTTPStatusError) or error.response.status_code != 401: raise self.login() diff --git a/bytes/tests/integration/test_bytes_api.py b/bytes/tests/integration/test_bytes_api.py index afc34ec75bd..0677bf3ce0f 100644 --- a/bytes/tests/integration/test_bytes_api.py +++ b/bytes/tests/integration/test_bytes_api.py @@ -236,6 +236,23 @@ def test_raw(bytes_api_client: BytesAPIClient, event_manager: RabbitMQEventManag assert str(boefje_meta.id) in body.decode() +def test_raw_big(bytes_api_client: BytesAPIClient, event_manager: RabbitMQEventManager) -> None: + boefje_meta = get_boefje_meta() + bytes_api_client.save_boefje_meta(boefje_meta) + + raw = b"test 123" * 100000 + raw_id = bytes_api_client.save_raw(boefje_meta.id, raw) + + retrieved_raw = bytes_api_client.get_raw(raw_id) + + assert retrieved_raw == raw + + method, properties, body = event_manager.connection.channel().basic_get("test__raw_file_received") + event_manager.connection.channel().basic_ack(method.delivery_tag) + + assert str(boefje_meta.id) in body.decode() + + def test_save_raw_with_one_mime_type(bytes_api_client: BytesAPIClient) -> None: boefje_meta = get_boefje_meta(meta_id=uuid.uuid4()) bytes_api_client.save_boefje_meta(boefje_meta) From b85e80ae3dadf8687b46fdb97a61f3c2f561646a Mon Sep 17 00:00:00 2001 From: noamblitz <43830693+noamblitz@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:20:32 +0200 Subject: [PATCH 20/28] Add timezone to valid time (#3429) Co-authored-by: Jan Klopper --- rocky/rocky/views/upload_raw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocky/rocky/views/upload_raw.py b/rocky/rocky/views/upload_raw.py index 5159b2ad9bc..7e286581b9d 100644 --- a/rocky/rocky/views/upload_raw.py +++ b/rocky/rocky/views/upload_raw.py @@ -62,7 +62,7 @@ def process_raw(self, form): raw_file = form.cleaned_data["raw_file"] mime_types = form.cleaned_data["mime_types"] input_ooi = form.cleaned_data["ooi"] - valid_time = form.cleaned_data["date"] + valid_time = form.cleaned_data["date"].replace(tzinfo=timezone.utc) try: get_bytes_client(self.organization.code).upload_raw( From 1c98a0c1b64650540b2d95e5828ff74375223a08 Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:28:35 +0200 Subject: [PATCH 21/28] Check if the task is still running according to the scheduler before changing the status (#3506) Co-authored-by: Jan Klopper --- boefjes/boefjes/app.py | 9 ++- boefjes/boefjes/docker_boefjes_runner.py | 1 - boefjes/tests/conftest.py | 13 +++-- .../scheduler/pop_response_boefje_2.json | 25 +++++++++ .../pop_response_boefje_dispatched.json | 25 +++++++++ boefjes/tests/test_app.py | 55 +++++++++++++------ 6 files changed, 104 insertions(+), 24 deletions(-) create mode 100644 boefjes/tests/examples/scheduler/pop_response_boefje_2.json create mode 100644 boefjes/tests/examples/scheduler/pop_response_boefje_dispatched.json diff --git a/boefjes/boefjes/app.py b/boefjes/boefjes/app.py index c12f1e15bc9..2b671f0f642 100644 --- a/boefjes/boefjes/app.py +++ b/boefjes/boefjes/app.py @@ -174,7 +174,7 @@ def _cleanup_pending_worker_task(self, worker: mp.Process) -> None: try: task = self.scheduler_client.get_task(handling_task_id) - if task.status is TaskStatus.DISPATCHED: + if task.status is TaskStatus.DISPATCHED or task.status is TaskStatus.RUNNING: try: self.scheduler_client.patch_task(task.id, TaskStatus.FAILED) logger.warning("Set status to failed in the scheduler for task[id=%s]", handling_task_id) @@ -244,6 +244,7 @@ def _start_working( handling_tasks[os.getpid()] = str(p_item.id) try: + scheduler_client.patch_task(p_item.id, TaskStatus.RUNNING) handler.handle(p_item.data) status = TaskStatus.COMPLETED except Exception: # noqa @@ -253,8 +254,10 @@ def _start_working( raise finally: try: - scheduler_client.patch_task(p_item.id, status) # Note: implicitly, we have p_item.id == task_id - logger.info("Set status to %s in the scheduler for task[id=%s]", status, p_item.data.id) + if scheduler_client.get_task(p_item.id).status == TaskStatus.RUNNING: + # The docker runner could have handled this already + scheduler_client.patch_task(p_item.id, status) # Note that implicitly, we have p_item.id == task_id + logger.info("Set status to %s in the scheduler for task[id=%s]", status, p_item.data.id) except HTTPError: logger.exception("Could not patch scheduler task to %s", status.value) diff --git a/boefjes/boefjes/docker_boefjes_runner.py b/boefjes/boefjes/docker_boefjes_runner.py index f28c64055ae..3efea0c48c9 100644 --- a/boefjes/boefjes/docker_boefjes_runner.py +++ b/boefjes/boefjes/docker_boefjes_runner.py @@ -36,7 +36,6 @@ def run(self) -> None: stderr_mime_types = boefjes.plugins.models._default_mime_types(self.boefje_meta.boefje) task_id = self.boefje_meta.id - self.scheduler_client.patch_task(task_id, TaskStatus.RUNNING) self.boefje_meta.started_at = datetime.now(timezone.utc) try: diff --git a/boefjes/tests/conftest.py b/boefjes/tests/conftest.py index 079185699ed..da8085d5f39 100644 --- a/boefjes/tests/conftest.py +++ b/boefjes/tests/conftest.py @@ -44,11 +44,13 @@ def __init__( log_path: Path, raise_on_empty_queue: Exception = KeyboardInterrupt, iterations_to_wait_for_exception: int = 0, - sleep_time: int = 0.1, + sleep_time: float = 0.1, ): self.queue_response = queue_response self.boefje_responses = boefje_responses self.normalizer_responses = normalizer_responses + + log_path.touch(exist_ok=True) self.log_path = log_path self.raise_on_empty_queue = raise_on_empty_queue self.iterations_to_wait_for_exception = iterations_to_wait_for_exception @@ -76,6 +78,7 @@ def pop_item(self, queue: str) -> Task | None: if WorkerManager.Queue.NORMALIZERS.value in queue: p_item = TypeAdapter(Task).validate_json(self.normalizer_responses.pop(0)) self._popped_items[str(p_item.id)] = p_item + self._tasks[str(p_item.id)] = self._task_from_id(p_item.id) return p_item except IndexError: raise self.raise_on_empty_queue @@ -128,9 +131,11 @@ def item_handler(tmp_path: Path): def manager(item_handler: MockHandler, tmp_path: Path) -> SchedulerWorkerManager: scheduler_client = MockSchedulerClient( queue_response=get_dummy_data("scheduler/queues_response.json"), - boefje_responses=( - 2 * [get_dummy_data("scheduler/pop_response_boefje.json")] + [get_dummy_data("scheduler/should_crash.json")] - ), + boefje_responses=[ + get_dummy_data("scheduler/pop_response_boefje.json"), + get_dummy_data("scheduler/pop_response_boefje_2.json"), + get_dummy_data("scheduler/should_crash.json"), + ], normalizer_responses=[get_dummy_data("scheduler/pop_response_normalizer.json")], log_path=tmp_path / "patch_task_log", ) diff --git a/boefjes/tests/examples/scheduler/pop_response_boefje_2.json b/boefjes/tests/examples/scheduler/pop_response_boefje_2.json new file mode 100644 index 00000000000..762be767cc9 --- /dev/null +++ b/boefjes/tests/examples/scheduler/pop_response_boefje_2.json @@ -0,0 +1,25 @@ +{ + "id": "70da7d4f-f41f-4940-901b-d98a92e9014c", + "priority": 1, + "scheduler_id": "boefje-_dev", + "schedule_id": null, + "status": "dispatched", + "type": "boefje", + "hash": "70da7d4f-f41f-4940-901b-d98a92e9014c", + "data": { + "id": "70da7d4f-f41f-4940-901b-d98a92e9014c", + "boefje": { + "id": "dns-records", + "version": null + }, + "input_ooi": "Hostname|internet|test.test", + "organization": "_dev", + "arguments": {}, + "started_at": null, + "runnable_hash": null, + "environment": null, + "ended_at": null + }, + "created_at": "2021-06-29T14:00:00", + "modified_at": "2021-06-29T14:00:00" +} diff --git a/boefjes/tests/examples/scheduler/pop_response_boefje_dispatched.json b/boefjes/tests/examples/scheduler/pop_response_boefje_dispatched.json new file mode 100644 index 00000000000..29e7d5dfb72 --- /dev/null +++ b/boefjes/tests/examples/scheduler/pop_response_boefje_dispatched.json @@ -0,0 +1,25 @@ +{ + "id": "70da7d4f-f41f-4940-901b-d98a92e9014b", + "priority": 1, + "scheduler_id": "boefje-_dev", + "schedule_id": null, + "status": "dispatched", + "type": "boefje", + "hash": "70da7d4f-f41f-4940-901b-d98a92e9014b", + "data": { + "id": "70da7d4f-f41f-4940-901b-d98a92e9014b", + "boefje": { + "id": "dns-records", + "version": null + }, + "input_ooi": "Hostname|internet|test.test", + "organization": "_dev", + "arguments": {}, + "started_at": null, + "runnable_hash": null, + "environment": null, + "ended_at": null + }, + "created_at": "2021-06-29T14:00:00", + "modified_at": "2021-06-29T14:00:00" +} diff --git a/boefjes/tests/test_app.py b/boefjes/tests/test_app.py index 05bc10a35d5..70e1523979f 100644 --- a/boefjes/tests/test_app.py +++ b/boefjes/tests/test_app.py @@ -22,10 +22,15 @@ def test_one_process(manager: SchedulerWorkerManager, item_handler: MockHandler) patched_tasks = manager.scheduler_client.get_all_patched_tasks() - assert len(patched_tasks) == 3 - assert patched_tasks[0] == ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed") - assert patched_tasks[1] == ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed") - assert patched_tasks[2] == ("9071c9fd-2b9f-440f-a524-ef1ca4824fd4", "failed") + assert len(patched_tasks) == 6 + assert set(patched_tasks) == { + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014c", "running"), + ("9071c9fd-2b9f-440f-a524-ef1ca4824fd4", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed"), + ("70da7d4f-f41f-4940-901b-d98a92e9014c", "completed"), + ("9071c9fd-2b9f-440f-a524-ef1ca4824fd4", "failed"), + } def test_two_processes(manager: SchedulerWorkerManager, item_handler: MockHandler) -> None: @@ -39,9 +44,14 @@ def test_two_processes(manager: SchedulerWorkerManager, item_handler: MockHandle assert len(items) == 2 patched_tasks = manager.scheduler_client.get_all_patched_tasks() - assert len(patched_tasks) == 3 - assert patched_tasks.count(("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed")) == 2 - assert patched_tasks.count(("9071c9fd-2b9f-440f-a524-ef1ca4824fd4", "failed")) == 1 + assert set(patched_tasks) == { + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014c", "running"), + ("9071c9fd-2b9f-440f-a524-ef1ca4824fd4", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed"), + ("70da7d4f-f41f-4940-901b-d98a92e9014c", "completed"), + ("9071c9fd-2b9f-440f-a524-ef1ca4824fd4", "failed"), + } def test_two_processes_exception(manager: SchedulerWorkerManager, item_handler: MockHandler, tmp_path) -> None: @@ -78,7 +88,7 @@ def test_two_processes_handler_exception(manager: SchedulerWorkerManager, item_h patched_tasks = manager.scheduler_client.get_all_patched_tasks() - assert len(patched_tasks) == 3 + assert len(patched_tasks) == 6 # Handler starts raising an Exception from the second call onward, # so we have 2 completed tasks and 4 failed tasks. assert patched_tasks.count(("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed")) == 1 @@ -89,9 +99,11 @@ def test_two_processes_cleanup_unfinished_tasks( manager: SchedulerWorkerManager, item_handler: MockHandler, tmp_path ) -> None: """ - Push 3 slow tasks to 2 workers, - then crash (from popping from an empty queue), - then clean up task on queue and the tasks being handled + We push 2 slow tasks to the Queue, which will be popped by 2 workers, emptying the Queue and stalling the 2 workers. + Because the Queue is now empty, the manager will get 2 new tasks from the scheduler to push to the queue. But only + one will be pushed because we do not have any tasks from the scheduler anymore (triggering a KeyboardInterrupt to + crash the main process). Then the manager should clean up the running tasks by setting the status of the running + tasks to failed and push any tasks still on the Queue back to the scheduler. """ manager.scheduler_client = MockSchedulerClient( @@ -112,10 +124,14 @@ def test_two_processes_cleanup_unfinished_tasks( assert len(items) == 0 patched_tasks = manager.scheduler_client.get_all_patched_tasks() - assert len(patched_tasks) == 1 + assert len(patched_tasks) == 3 # Task was running but main process crashed intentionally and cleaned it up - assert patched_tasks.count(("70da7d4f-f41f-4940-901b-d98a92e9014b", "failed")) == 1 + assert set(patched_tasks) == { + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "failed"), + } # Tasks (one with the same id) was still unhandled the queue and pushed back to the scheduler by the main process assert manager.scheduler_client._pushed_items["70da7d4f-f41f-4940-901b-d98a92e9014b"][0] == "boefje" @@ -140,6 +156,7 @@ def test_null(manager: SchedulerWorkerManager, tmp_path: Path, item_handler: Moc [get_dummy_data("scheduler/pop_response_normalizer.json")], tmp_path / "patch_task_log", iterations_to_wait_for_exception=2, + sleep_time=0.3, ) with pytest.raises(KeyboardInterrupt): @@ -149,9 +166,15 @@ def test_null(manager: SchedulerWorkerManager, tmp_path: Path, item_handler: Moc patched_tasks = manager.scheduler_client.get_all_patched_tasks() assert len(items) == 3 - assert len(patched_tasks) == 3 - assert patched_tasks[0] == ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed") - assert patched_tasks[2] == ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed") + assert len(patched_tasks) == 6 + assert set(patched_tasks) == { + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "running"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed"), + ("70da7d4f-f41f-4940-901b-d98a92e9014b", "completed"), + } def test_create_manager(): From ecdf1df6c628c555b9c520834b00ded6b8ceaa7f Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 12 Sep 2024 10:20:36 +0200 Subject: [PATCH 22/28] Exclude OOIs creation from the OOI add form by OOI-types (#3490) Co-authored-by: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> Co-authored-by: Jan Klopper --- rocky/rocky/views/ooi_add.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/rocky/rocky/views/ooi_add.py b/rocky/rocky/views/ooi_add.py index 1840c35ee06..12f451e5f99 100644 --- a/rocky/rocky/views/ooi_add.py +++ b/rocky/rocky/views/ooi_add.py @@ -8,12 +8,20 @@ from tools.view_helpers import existing_ooi_type from octopoes.models import OOI +from octopoes.models.ooi.monitoring import Incident +from octopoes.models.ooi.question import Question +from octopoes.models.ooi.reports import Report, ReportData +from octopoes.models.ooi.web import RESTAPI, ImageMetadata from octopoes.models.types import type_by_name from rocky.views.ooi_view import BaseOOIFormView +EXCLUDE_OOI_TYPES = [ + ooi_type.get_object_type() for ooi_type in [Question, RESTAPI, Incident, ImageMetadata, Report, ReportData] +] + def ooi_type_input_choices(): - ooi_types = OOI_TYPES_WITHOUT_FINDINGS + ooi_types = [ooi_type for ooi_type in OOI_TYPES_WITHOUT_FINDINGS if ooi_type not in EXCLUDE_OOI_TYPES] ooi_types.sort() return [{"value": ooi_type, "text": ooi_type} for ooi_type in ooi_types] @@ -66,9 +74,12 @@ def setup(self, request, *args, **kwargs): def get_ooi_class(self) -> type[OOI]: try: - return type_by_name(self.kwargs["ooi_type"]) + ooi_type = self.kwargs["ooi_type"] + if ooi_type in EXCLUDE_OOI_TYPES: + raise KeyError + return type_by_name(ooi_type) except KeyError: - raise Http404("OOI not found") + raise Http404("OOI-type not found") def get_form_kwargs(self): kwargs = super().get_form_kwargs() From 53d71a33594ce569b6f0d2dbdccc6bb6da0f85b4 Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:26:50 +0200 Subject: [PATCH 23/28] Use the right variable name in the template's if-statement (#3519) Signed-off-by: Donny Peeters Co-authored-by: Jan Klopper --- rocky/katalogus/templates/boefje_detail.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocky/katalogus/templates/boefje_detail.html b/rocky/katalogus/templates/boefje_detail.html index 04d6abcb960..7b59666d1c5 100644 --- a/rocky/katalogus/templates/boefje_detail.html +++ b/rocky/katalogus/templates/boefje_detail.html @@ -32,7 +32,7 @@

{{ plugin.name }}

- {% if perms.tools.can_view_katalogus_settings and object_list %} + {% if perms.tools.can_view_katalogus_settings and plugin_settings %}
{% include "plugin_settings_list.html" with object_list=plugin_settings plugin=plugin %} From 8ec43412faaceaa86105e67772b7e2473ac3bec8 Mon Sep 17 00:00:00 2001 From: Donny Peeters <46660228+Donnype@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:23:14 +0200 Subject: [PATCH 24/28] Add regex pattern check to PORTS setting of `nmap-ports` (#3516) Signed-off-by: Donny Peeters Co-authored-by: Jan Klopper --- .../plugins/kat_nmap_ports/schema.json | 1 + boefjes/tests/test_nmap.py | 86 +++++++++++++++---- 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/boefjes/boefjes/plugins/kat_nmap_ports/schema.json b/boefjes/boefjes/plugins/kat_nmap_ports/schema.json index 4b52a8af648..b078f26b0b8 100644 --- a/boefjes/boefjes/plugins/kat_nmap_ports/schema.json +++ b/boefjes/boefjes/plugins/kat_nmap_ports/schema.json @@ -6,6 +6,7 @@ "title": "PORTS", "maxLength": 2048, "type": "string", + "pattern": "^((6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)|(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)-(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d))$|^((6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)|(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)-(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d))(,((6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)|(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)-(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)))+$", "description": "Specify the ports that need to be scanned (nmap format). Single ports are comma separated, port ranges can be specified using the dash symbol. For example: 22,111,137,80-100 will scan ports 22, 111, 137 and the port range 80 up to 100." } }, diff --git a/boefjes/tests/test_nmap.py b/boefjes/tests/test_nmap.py index 9d0de572fdf..f936160f910 100644 --- a/boefjes/tests/test_nmap.py +++ b/boefjes/tests/test_nmap.py @@ -1,21 +1,77 @@ -from unittest import TestCase +import re + +import pytest +from jsonschema.exceptions import ValidationError +from jsonschema.validators import validate from boefjes.plugins.kat_nmap_tcp.normalize import run from octopoes.models.ooi.network import IPAddressV4, Network from tests.loading import get_dummy_data -class NmapTest(TestCase): - def test_normalizer(self): - input_ooi = IPAddressV4(network=Network(name="internet").reference, address="134.209.85.72") - output = list(run(input_ooi.serialize(), get_dummy_data("raw/nmap_mispoes.xml"))) - self.assertEqual(15, len(output)) - for i, out in enumerate(output[:-1]): - if out.object_type == "IPPort" and output[i + 1].object_type == "Service": - if out.port == 80: - self.assertEqual("http", output[i + 1].name) - elif out.port == 443: - self.assertEqual("https", output[i + 1].name) - else: - self.assertNotEqual("http", output[i + 1].name) - self.assertNotEqual("https", output[i + 1].name) +def test_normalizer(): + input_ooi = IPAddressV4(network=Network(name="internet").reference, address="134.209.85.72") + output = list(run(input_ooi.serialize(), get_dummy_data("raw/nmap_mispoes.xml"))) + assert len(output) == 15 + for i, out in enumerate(output[:-1]): + if out.object_type == "IPPort" and output[i + 1].object_type == "Service": + name = output[i + 1].name + if out.port == 80: + assert name == "http" + elif out.port == 443: + assert name == "https" + else: + assert name != "http" + assert name != "https" + + +def get_pattern(): + max_65535 = r"(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{0,4}|\d)" + max_65535_or_port_range = f"({max_65535}|{max_65535}-{max_65535})" + one_or_comma_separated = f"^{max_65535_or_port_range}$|^{max_65535_or_port_range}(,{max_65535_or_port_range})+$" + + return re.compile(one_or_comma_separated) + + +def test_single_port_pattern(local_repo): + schema = local_repo.schema("nmap-ports") + for single_port in ["1", "2", "20", "200", "2000", "20000", "65535"]: + assert get_pattern().search(single_port) is not None + validate(instance={"PORTS": single_port}, schema=schema) + + +def test_bad_single_port_pattern(local_repo): + schema = local_repo.schema("nmap-ports") + for bad_single_port in ["-1", "-2000", "65536", "222222"]: + assert get_pattern().search(bad_single_port) is None + with pytest.raises(ValidationError): + validate(instance={"PORTS": bad_single_port}, schema=schema) + + +def test_multi_ports_pattern(local_repo): + schema = local_repo.schema("nmap-ports") + for multi_port in ["1,2", "2,3,4", "2,3,4,5,6,7", "2,20,200,2000,65535"]: + assert get_pattern().search(multi_port) is not None + validate(instance={"PORTS": multi_port}, schema=schema) + + +def test_port_range_pattern(local_repo): + schema = local_repo.schema("nmap-ports") + for port_range in ["1-2", "2-20000", "65533-65535"]: + assert get_pattern().search(port_range) is not None + validate(instance={"PORTS": port_range}, schema=schema) + + +def test_combined(local_repo): + schema = local_repo.schema("nmap-ports") + for port_range in ["1,1-65000", "1,2,234,4300-5999,1"]: + assert get_pattern().search(port_range) is not None + validate(instance={"PORTS": port_range}, schema=schema) + + +def test_badly_combined(local_repo): + schema = local_repo.schema("nmap-ports") + for port_range in ["1,1-", "1-234-323"]: + assert get_pattern().search(port_range) is None + with pytest.raises(ValidationError): + validate(instance={"PORTS": port_range}, schema=schema) From 93b7faf39237816597c26a6ddc9c06ca25c03c7e Mon Sep 17 00:00:00 2001 From: Jeroen Dekkers Date: Thu, 12 Sep 2024 14:30:18 +0200 Subject: [PATCH 25/28] Update xtdb-http-multinode to the latest version (#3523) --- boefjes/.ci/docker-compose.yml | 2 +- docker-compose.release-example.yml | 2 +- octopoes/.ci/docker-compose.yml | 2 +- rocky/.ci/docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/boefjes/.ci/docker-compose.yml b/boefjes/.ci/docker-compose.yml index f3dc4ecc244..bf7ee7aeadd 100644 --- a/boefjes/.ci/docker-compose.yml +++ b/boefjes/.ci/docker-compose.yml @@ -85,7 +85,7 @@ services: - .ci/.env.test ci_xtdb: - image: "ghcr.io/dekkers/xtdb-http-multinode:v1.0.8" + image: "ghcr.io/dekkers/xtdb-http-multinode:v1.1.0" ci_octopoes_api_worker: build: diff --git a/docker-compose.release-example.yml b/docker-compose.release-example.yml index 9469c26bff7..c90b1bf7bdb 100644 --- a/docker-compose.release-example.yml +++ b/docker-compose.release-example.yml @@ -31,7 +31,7 @@ services: APPS: "ROCKY BYTES KATALOGUS SCHEDULER" crux: - image: "ghcr.io/dekkers/xtdb-http-multinode:v1.0.8" + image: "ghcr.io/dekkers/xtdb-http-multinode:v1.1.0" restart: on-failure ports: - "127.0.0.1:3000:3000" diff --git a/octopoes/.ci/docker-compose.yml b/octopoes/.ci/docker-compose.yml index 6b4d0205f6a..d4e33b42e13 100644 --- a/octopoes/.ci/docker-compose.yml +++ b/octopoes/.ci/docker-compose.yml @@ -25,7 +25,7 @@ services: - ".ci/ci.env" xtdb: - image: "ghcr.io/dekkers/xtdb-http-multinode:v1.0.8" + image: "ghcr.io/dekkers/xtdb-http-multinode:v1.1.0" ports: - "127.0.0.1:29002:3000" diff --git a/rocky/.ci/docker-compose.yml b/rocky/.ci/docker-compose.yml index 130d3e59f20..03940ae2236 100644 --- a/rocky/.ci/docker-compose.yml +++ b/rocky/.ci/docker-compose.yml @@ -52,7 +52,7 @@ services: - .env.test xtdb: - image: "ghcr.io/dekkers/xtdb-http-multinode:v1.0.8" + image: "ghcr.io/dekkers/xtdb-http-multinode:v1.1.0" rabbitmq: restart: on-failure From dc0c4dc6a01a774fb0fea5212d45750e644fba4e Mon Sep 17 00:00:00 2001 From: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:32:15 +0200 Subject: [PATCH 26/28] Updated findings in the findings database (#3427) Co-authored-by: Jan Klopper From 822b776904ea9c7358b925f383f971a78694387a Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Thu, 12 Sep 2024 17:26:25 +0200 Subject: [PATCH 27/28] remove unneeded column from filtered plugin table view (#3515) Co-authored-by: Ammar Co-authored-by: Rieven --- rocky/katalogus/templates/katalogus.html | 2 +- rocky/katalogus/templates/partials/plugins.html | 16 ++++++++++------ rocky/rocky/locale/django.pot | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/rocky/katalogus/templates/katalogus.html b/rocky/katalogus/templates/katalogus.html index 24107431705..3284a9491e3 100644 --- a/rocky/katalogus/templates/katalogus.html +++ b/rocky/katalogus/templates/katalogus.html @@ -32,7 +32,7 @@

{% translate "All plugins" %}

{% include "partials/katalogus_filter.html" with form=form %} {% include "partials/katalogus_toolbar.html" %} - {% include "partials/plugins.html" %} + {% include "partials/plugins.html" with active="all" %}
diff --git a/rocky/katalogus/templates/partials/plugins.html b/rocky/katalogus/templates/partials/plugins.html index 95524ce1d0b..495f8d431cb 100644 --- a/rocky/katalogus/templates/partials/plugins.html +++ b/rocky/katalogus/templates/partials/plugins.html @@ -8,7 +8,9 @@ - + {% if active == "all" %} + + {% endif %} @@ -21,11 +23,13 @@ {{ plugin.name }} - + {% if active == "all" %} + + {% endif %} diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 3591cc84059..553fcdeb553 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-09-05 15:36+0000\n" +"POT-Creation-Date: 2024-09-11 14:03+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" From ae790c32365482b19cd1917ce3e5d0abaef4f0ef Mon Sep 17 00:00:00 2001 From: originalsouth Date: Mon, 16 Sep 2024 10:17:02 +0200 Subject: [PATCH 28/28] Also delete self-affirming or self-infered objects (#3498) Co-authored-by: ammar92 --- octopoes/octopoes/core/service.py | 9 +- .../tests/integration/test_ooi_deletion.py | 98 ++++++++++++++++++- 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index cc7f64e00af..70c89f48f90 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -162,7 +162,14 @@ def get_ooi_tree( def _delete_ooi(self, reference: Reference, valid_time: datetime) -> None: referencing_origins = self.origin_repository.list_origins(valid_time, result=reference) - if not referencing_origins: + if not any( + origin + for origin in referencing_origins + if not ( + origin.origin_type == OriginType.AFFIRMATION + or (origin.origin_type == OriginType.INFERENCE and origin.source == reference) + ) + ): self.ooi_repository.delete(reference, valid_time) def save_origin( diff --git a/octopoes/tests/integration/test_ooi_deletion.py b/octopoes/tests/integration/test_ooi_deletion.py index ad52fbe1b98..86e40eda52b 100644 --- a/octopoes/tests/integration/test_ooi_deletion.py +++ b/octopoes/tests/integration/test_ooi_deletion.py @@ -7,7 +7,7 @@ import pytest -from octopoes.api.models import Declaration, Observation +from octopoes.api.models import Affirmation, Declaration, Observation from octopoes.connector.octopoes import OctopoesAPIConnector from octopoes.core.service import OctopoesService from octopoes.events.events import OOIDBEvent, OriginDBEvent @@ -429,3 +429,99 @@ def chain(source, results): assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 0 assert len(list(filter(lambda x: x.operation_type.value == "delete", event_manager.queue))) > 0 + + +def test_affirming_ooi_delete(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetime): + # Make an object A + network = Network(name="internet") + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) + + # Observe an object B "derived" by A + url = "mispo.es" + hostname = Hostname(network=network.reference, name=url) + hostname_origin = Observation( + method="", + source=network.reference, + source_method=None, + result=[hostname], + task_id=uuid.uuid4(), + valid_time=valid_time, + ) + octopoes_api_connector.save_observation(hostname_origin) + time.sleep(1) + assert octopoes_api_connector.list_objects({Hostname}, valid_time).count == 1 + + # Delete A and validate B is not present + octopoes_api_connector.delete(network.reference, valid_time) + time.sleep(1) + assert octopoes_api_connector.list_objects({Hostname}, valid_time).count == 0 + + # Re-observe an object B "derived" by A + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) + octopoes_api_connector.save_observation(hostname_origin) + time.sleep(1) + assert octopoes_api_connector.list_objects({Hostname}, valid_time).count == 1 + + # Affirm object B + octopoes_api_connector.save_affirmation( + Affirmation( + ooi=hostname, + source_method=None, + task_id=uuid.uuid4(), + valid_time=valid_time, + ) + ) + time.sleep(1) + + # Delete A and validate B is not present + octopoes_api_connector.delete(network.reference, valid_time) + time.sleep(1) + assert octopoes_api_connector.list_objects({Hostname}, valid_time).count == 0 + + +def test_delecration_ooi_delete(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetime): + # Make an object A + network = Network(name="internet") + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) + + # Observe an object B "derived" by A + url = "mispo.es" + hostname = Hostname(network=network.reference, name=url) + hostname_origin = Observation( + method="", + source=network.reference, + source_method=None, + result=[hostname], + task_id=uuid.uuid4(), + valid_time=valid_time, + ) + octopoes_api_connector.save_observation(hostname_origin) + time.sleep(1) + assert octopoes_api_connector.list_objects({Hostname}, valid_time).count == 1 + + # Delete A and validate B is not present + octopoes_api_connector.delete(network.reference, valid_time) + time.sleep(1) + assert octopoes_api_connector.list_objects({Hostname}, valid_time).count == 0 + + # Re-observe an object B "derived" by A + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) + octopoes_api_connector.save_observation(hostname_origin) + time.sleep(1) + assert octopoes_api_connector.list_objects({Hostname}, valid_time).count == 1 + + # Infer object B + octopoes_api_connector.save_declaration( + Declaration( + ooi=hostname, + source_method=None, + task_id=uuid.uuid4(), + valid_time=valid_time, + ) + ) + time.sleep(1) + + # Delete A and validate B is not present + octopoes_api_connector.delete(network.reference, valid_time) + time.sleep(1) + assert octopoes_api_connector.list_objects({Hostname}, valid_time).count == 1
{% translate "Plugin name" %}{% translate "Plugin type" %}{% translate "Plugin type" %}{% translate "Plugin description" %} {% translate "Actions" %}
-

- {{ plugin.type|title }} -

- 		+

+ {{ plugin.type|title }} +

+ 		{% if plugin.description %}{{ plugin.description }}{% endif %}