From 3c4b5c09fdd7debef5666c3e83b0eae5546dfc3c Mon Sep 17 00:00:00 2001 From: Madelon Dohmen <99282220+madelondohmen@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:32:32 +0100 Subject: [PATCH] Edit report recipe (#3690) Co-authored-by: Rieven Co-authored-by: JP Bruins Slot Co-authored-by: ammar92 Co-authored-by: Jan Klopper Co-authored-by: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> --- octopoes/octopoes/models/ooi/reports.py | 2 +- .../scheduled_reports_table.html | 36 +++---- rocky/reports/views/report_overview.py | 34 +++--- rocky/rocky/locale/django.pot | 6 +- rocky/rocky/scheduler.py | 15 +++ rocky/rocky/views/ooi_edit.py | 24 ++++- rocky/rocky/views/scheduler.py | 9 ++ rocky/tests/conftest.py | 101 +++++++++++++++++- rocky/tests/objects/test_objects_edit.py | 49 +++++++++ rocky/tools/forms/ooi_form.py | 11 +- 10 files changed, 241 insertions(+), 46 deletions(-) diff --git a/octopoes/octopoes/models/ooi/reports.py b/octopoes/octopoes/models/ooi/reports.py index 462dd0bb02d..e3af38fe23e 100644 --- a/octopoes/octopoes/models/ooi/reports.py +++ b/octopoes/octopoes/models/ooi/reports.py @@ -55,7 +55,7 @@ class ReportRecipe(OOI): recipe_id: UUID report_name_format: str - subreport_name_format: str + subreport_name_format: str | None = None input_recipe: dict[str, Any] # can contain a query which maintains a live set of OOIs or manually picked OOIs. report_types: list[str] diff --git a/rocky/reports/templates/report_overview/scheduled_reports_table.html b/rocky/reports/templates/report_overview/scheduled_reports_table.html index 5b549066f9d..ced610163e0 100644 --- a/rocky/reports/templates/report_overview/scheduled_reports_table.html +++ b/rocky/reports/templates/report_overview/scheduled_reports_table.html @@ -37,22 +37,18 @@ {{ schedule.deadline_at }} {{ schedule.cron }} - {% if schedule.reports %} - - - - {% else %} - - {% endif %} + + + - {% if schedule.reports %} - - + + + {% if schedule.reports %} @@ -83,9 +79,13 @@ {% endfor %}
- - - {% endif %} + {% endif %} +
+ {% translate "Edit report recipe" %} +
+ + {% endif %} {% endfor %} diff --git a/rocky/reports/views/report_overview.py b/rocky/reports/views/report_overview.py index 229f3efb317..46171a1ffa2 100644 --- a/rocky/reports/views/report_overview.py +++ b/rocky/reports/views/report_overview.py @@ -56,22 +56,24 @@ def get_queryset(self) -> list[dict[str, Any]]: recipes = [] if report_schedules: for schedule in report_schedules: - recipe_id = schedule["data"]["report_recipe_id"] - # TODO: This is a workaround to get the recipes and reports. - # We should create an endpoint for this in octopoes - recipe_ooi_tree = self.get_recipe_ooi_tree(recipe_id) - if recipe_ooi_tree is not None: - recipe_tree = recipe_ooi_tree.store.values() - recipe_ooi = [ooi for ooi in recipe_tree if isinstance(ooi, ReportRecipe)][0] - report_oois = [ooi for ooi in recipe_tree if isinstance(ooi, Report)] - recipes.append( - { - "recipe": recipe_ooi, - "cron": schedule["schedule"], - "deadline_at": datetime.fromisoformat(schedule["deadline_at"]), - "reports": report_oois, - } - ) + if schedule["data"]: + recipe_id = schedule["data"]["report_recipe_id"] + # TODO: This is a workaround to get the recipes and reports. + # We should create an endpoint for this in octopoes + recipe_ooi_tree = self.get_recipe_ooi_tree(recipe_id) + if recipe_ooi_tree is not None: + recipe_tree = recipe_ooi_tree.store.values() + recipe_ooi = next(ooi for ooi in recipe_tree if isinstance(ooi, ReportRecipe)) + report_oois = [ooi for ooi in recipe_tree if isinstance(ooi, Report)] + recipes.append( + { + "schedule_id": schedule["id"], + "recipe": recipe_ooi, + "cron": schedule["schedule"], + "deadline_at": datetime.fromisoformat(schedule["deadline_at"]), + "reports": report_oois, + } + ) return recipes diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 41c7d24c5d9..05725644468 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -9,7 +9,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-10-21 16:21+0000\n" +"POT-Creation-Date: 2024-10-28 12:21+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -4186,6 +4186,10 @@ msgstr "" msgid "Show report details" msgstr "" +#: reports/templates/report_overview/scheduled_reports_table.html +msgid "Edit report recipe" +msgstr "" + #: reports/templates/report_overview/scheduled_reports_table.html msgid "No scheduled reports have been generated yet." msgstr "" diff --git a/rocky/rocky/scheduler.py b/rocky/rocky/scheduler.py index e417b8fa696..d94f28cb189 100644 --- a/rocky/rocky/scheduler.py +++ b/rocky/rocky/scheduler.py @@ -287,6 +287,21 @@ def get_schedule_details(self, schedule_id: str) -> ScheduleResponse: except ConnectError: raise SchedulerConnectError() + def post_schedule_search(self, filters: dict[str, str]) -> PaginatedSchedulesResponse: + try: + res = self._client.post("/schedules/search", json=filters) + res.raise_for_status() + return PaginatedSchedulesResponse.model_validate_json(res.content) + except ConnectError: + raise SchedulerConnectError() + + def patch_schedule(self, schedule_id: str, params: dict[str, Any]) -> None: + try: + response = self._client.patch(f"/schedules/{schedule_id}", json=params) + response.raise_for_status() + except (HTTPStatusError, ConnectError): + raise SchedulerHTTPError() + def post_schedule(self, schedule: ScheduleRequest) -> ScheduleResponse: try: res = self._client.post("/schedules", json=schedule.model_dump(exclude_none=True)) diff --git a/rocky/rocky/views/ooi_edit.py b/rocky/rocky/views/ooi_edit.py index 33e4ee3dc31..8d0535cc7f4 100644 --- a/rocky/rocky/views/ooi_edit.py +++ b/rocky/rocky/views/ooi_edit.py @@ -1,13 +1,16 @@ +from datetime import datetime, timezone from enum import Enum from django.utils.translation import gettext_lazy as _ from tools.view_helpers import get_ooi_url from rocky.views.ooi_view import BaseOOIFormView +from rocky.views.scheduler import SchedulerView -class OOIEditView(BaseOOIFormView): +class OOIEditView(BaseOOIFormView, SchedulerView): template_name = "oois/ooi_edit.html" + task_type = "report" def setup(self, request, *args, **kwargs): super().setup(request, *args, **kwargs) @@ -36,6 +39,25 @@ def get_form_kwargs(self): return kwargs + def form_valid(self, form): + form_data = form.cleaned_data + report_recipe_id = form_data.get("recipe_id") + cron_expression = form_data.get("cron_expression") + + # If the cron_expression of the ReportRecipe is changed, the scheduler must also be updated + if report_recipe_id and cron_expression: + deadline_at = datetime.now(timezone.utc).isoformat() + filters = { + "filters": [ + {"column": "data", "field": "report_recipe_id", "operator": "eq", "value": report_recipe_id} + ] + } + + schedule_id = str(self.get_schedule_with_filters(filters).id) + self.edit_report_schedule(schedule_id, {"schedule": cron_expression, "deadline_at": deadline_at}) + + return super().form_valid(form) + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) diff --git a/rocky/rocky/views/scheduler.py b/rocky/rocky/views/scheduler.py index cdb47b5ae51..25081cc3be9 100644 --- a/rocky/rocky/views/scheduler.py +++ b/rocky/rocky/views/scheduler.py @@ -139,6 +139,9 @@ def create_report_schedule(self, report_recipe: ReportRecipe, deadline_at: str) except SchedulerError as error: return messages.error(self.request, error.message) + def edit_report_schedule(self, schedule_id: str, params): + self.scheduler_client.patch_schedule(schedule_id=schedule_id, params=params) + def get_report_schedules(self) -> list[dict[str, Any]]: try: return self.scheduler_client.get_scheduled_reports(scheduler_id=self.scheduler_id) @@ -180,6 +183,12 @@ def get_json_task_details(self) -> JsonResponse | None: except SchedulerError as error: return messages.error(self.request, error.message) + def get_schedule_with_filters(self, filters: dict[str, Any]) -> ScheduleResponse: + try: + return self.scheduler_client.post_schedule_search(filters).results[0] + except SchedulerError as error: + return messages.error(self.request, error.message) + def schedule_task(self, task: Task) -> None: if not self.indemnification_present: return self.indemnification_error() diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index dd3d3a0c68c..9edf54ac91f 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -32,7 +32,7 @@ from octopoes.models.ooi.dns.zone import Hostname from octopoes.models.ooi.findings import CVEFindingType, Finding, KATFindingType, RiskLevelSeverity from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, Protocol -from octopoes.models.ooi.reports import Report +from octopoes.models.ooi.reports import Report, ReportRecipe from octopoes.models.ooi.service import IPService, Service from octopoes.models.ooi.software import Software from octopoes.models.ooi.web import URL, SecurityTXT, Website @@ -1310,6 +1310,105 @@ def get_subreports() -> list[tuple[str, Report]]: ] +@pytest.fixture +def report_recipe(): + return ReportRecipe( + recipe_id="744d054e-9c70-4f18-ad27-122cfc1b7903", + report_name_format="Test Report Name Format", + subreport_name_format="Test Subreport Name Format", + input_recipe={"input_oois": ["Hostname|internet|mispo.es"]}, + report_types=["dns-report"], + cron_expression="0 0 * * *", + ) + + +@pytest.fixture +def get_report_schedules(): + report_schedule = [ + { + "id": "06a783b3-af62-429d-8b48-5934c8702366", + "scheduler_id": "report-madelon123", + "hash": "f429f60fda539f9017544758163a955e", + "data": { + "type": "report", + "organisation_id": "madelon123", + "report_recipe_id": "6a121a53-f795-4bfe-9aea-f410a96fca59", + }, + "enabled": "true", + "schedule": "0 * * * *", + "tasks": [ + { + "id": "21ce00b8-57c4-4d65-a53e-275b4e44da6f", + "scheduler_id": "report-madelon123", + "schedule_id": "06a783b3-af62-429d-8b48-5934c8702366", + "priority": 1729520184, + "status": "completed", + "type": "report", + "hash": "f429f60fda539f9017544758163a955e", + "data": { + "organisation_id": "madelon123", + "report_recipe_id": "6a121a53-f795-4bfe-9aea-f410a96fca59", + }, + "created_at": "2024-10-21T14:16:24.702759Z", + "modified_at": "2024-10-21T14:16:24.702761Z", + }, + { + "id": "f63c2224-a4b2-4161-9790-9b4d665a4683", + "scheduler_id": "report-madelon123", + "schedule_id": "06a783b3-af62-429d-8b48-5934c8702366", + "priority": 1729580020, + "status": "completed", + "type": "report", + "hash": "f429f60fda539f9017544758163a955e", + "data": { + "organisation_id": "madelon123", + "report_recipe_id": "6a121a53-f795-4bfe-9aea-f410a96fca59", + }, + "created_at": "2024-10-22T06:53:40.405185Z", + "modified_at": "2024-10-22T06:53:40.405192Z", + }, + ], + "deadline_at": "2024-10-22T12:00:00Z", + "created_at": "2024-10-21T14:14:00.359039Z", + "modified_at": "2024-10-21T14:14:00.359043Z", + }, + { + "id": "e9a00bc1-850a-401a-89d9-252d98823bb3", + "scheduler_id": "report-madelon123", + "hash": "02e67c8676cb8135681d52ca62a9fe5b", + "data": { + "type": "report", + "organisation_id": "madelon123", + "report_recipe_id": "31c79490-fb51-440d-9108-0c388276f655", + }, + "enabled": "true", + "schedule": "0 0 * * *", + "tasks": [ + { + "id": "f4f938e0-5f2d-4bcd-9b28-11831b7835e4", + "scheduler_id": "report-madelon123", + "schedule_id": "e9a00bc1-850a-401a-89d9-252d98823bb3", + "priority": 1729521145, + "status": "completed", + "type": "report", + "hash": "02e67c8676cb8135681d52ca62a9fe5b", + "data": { + "organisation_id": "madelon123", + "report_recipe_id": "31c79490-fb51-440d-9108-0c388276f655", + }, + "created_at": "2024-10-21T14:32:25.247525Z", + "modified_at": "2024-10-21T14:32:25.247527Z", + } + ], + "deadline_at": "2024-10-23T00:00:00Z", + "created_at": "2024-10-21T13:34:42.791561Z", + "modified_at": "2024-10-21T13:34:42.791563Z", + }, + ] + + return report_schedule + + def setup_request(request, user): request = SessionMiddleware(lambda r: r)(request) request.session[DEVICE_ID_SESSION_KEY] = user.staticdevice_set.get().persistent_id diff --git a/rocky/tests/objects/test_objects_edit.py b/rocky/tests/objects/test_objects_edit.py index 88c81245a0b..dab3f83a6c4 100644 --- a/rocky/tests/objects/test_objects_edit.py +++ b/rocky/tests/objects/test_objects_edit.py @@ -1,3 +1,6 @@ +import json + +from django.urls import reverse from pytest_django.asserts import assertContains from rocky.views.ooi_edit import OOIEditView @@ -13,3 +16,49 @@ def test_ooi_edit(rf, client_member, mock_organization_view_octopoes, network): assert response.status_code == 200 assertContains(response, "testnetwork") assertContains(response, "Save Network") + + +def test_ooi_edit_report_recipe_get(rf, client_member, mock_organization_view_octopoes, report_recipe): + mock_organization_view_octopoes().get.return_value = report_recipe + ooi_id = f"ReportRecipe|{report_recipe.recipe_id}" + + request = setup_request(rf.get("ooi_edit", {"ooi_id": ooi_id}), client_member.user) + response = OOIEditView.as_view()(request, organization_code=client_member.organization.code) + + assert response.status_code == 200 + assertContains(response, "Edit ReportRecipe: " + ooi_id) + + +def test_ooi_edit_report_recipe_post( + rf, client_member, mock_organization_view_octopoes, report_recipe, mocker, mock_scheduler +): + mock_organization_view_octopoes().get.return_value = report_recipe + mocker.patch("rocky.views.ooi_view.create_ooi") + ooi_id = f"ReportRecipe|{report_recipe.recipe_id}" + + request_url = ( + reverse("ooi_edit", kwargs={"organization_code": client_member.organization.code}) + f"?ooi_id={ooi_id}" + ) + + request = setup_request( + rf.post( + request_url, + { + "ooi_type": "ReportRecipe", + "user": client_member.user.email, + "recipe_id": report_recipe.recipe_id, + "report_name_format": report_recipe.report_name_format, + "subreport_name_format": report_recipe.subreport_name_format, + "input_recipe": json.dumps(report_recipe.input_recipe), + "report_types": json.dumps(report_recipe.report_types), + "cron_expression": report_recipe.cron_expression, + }, + ), + client_member.user, + ) + response = OOIEditView.as_view()(request, organization_code=client_member.organization.code) + + assert response.status_code == 302 + + response_url = "/en/{}/objects/detail/?ooi_id=ReportRecipe%7C{}" + assert response.url == response_url.format(client_member.organization.code, report_recipe.recipe_id) diff --git a/rocky/tools/forms/ooi_form.py b/rocky/tools/forms/ooi_form.py index 6b14b566f07..9914d3b2fca 100644 --- a/rocky/tools/forms/ooi_form.py +++ b/rocky/tools/forms/ooi_form.py @@ -2,11 +2,11 @@ from enum import Enum from inspect import isclass from ipaddress import IPv4Address, IPv6Address -from typing import Literal, Union, get_args, get_origin +from typing import Any, Literal, Union, get_args, get_origin from django import forms from django.utils.translation import gettext_lazy as _ -from pydantic import AnyUrl, JsonValue +from pydantic import AnyUrl from pydantic.fields import FieldInfo from octopoes.connector.octopoes import OctopoesAPIConnector @@ -70,12 +70,7 @@ def generate_form_fields(self, hidden_ooi_fields: dict[str, str] | None = None) fields[name] = generate_ip_field(field) elif annotation == AnyUrl: fields[name] = generate_url_field(field) - elif ( - annotation == dict - or annotation == dict[str, str] - or annotation == list[str] - or annotation == dict[str, JsonValue] - ): + elif annotation == dict or annotation == list[str] or annotation == dict[str, Any]: fields[name] = forms.JSONField(**default_attrs) elif annotation == int or (hasattr(annotation, "__args__") and int in annotation.__args__): fields[name] = forms.IntegerField(**default_attrs)