From d2148969de11b579f88ce08ce7ddcb23aa8c273d Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 26 Sep 2024 14:35:54 +0200 Subject: [PATCH] Add schdule reports --- rocky/reports/forms.py | 37 ++---- .../partials/export_report_settings.html | 23 ++-- rocky/reports/views/base.py | 40 ++++--- rocky/rocky/scheduler.py | 2 + rocky/rocky/views/scheduler.py | 110 ++++++++---------- 5 files changed, 96 insertions(+), 116 deletions(-) diff --git a/rocky/reports/forms.py b/rocky/reports/forms.py index c1b2df7d506..c0f6dafdc76 100644 --- a/rocky/reports/forms.py +++ b/rocky/reports/forms.py @@ -1,5 +1,4 @@ from datetime import datetime, timezone -from typing import Any from django import forms from django.utils.translation import gettext_lazy as _ @@ -33,7 +32,7 @@ def __init__(self, report_types: set[Report], *args, **kwargs): self.fields["report_type"].choices = report_types_choices -class ReportReferenceDateForm(BaseRockyForm): +class ReportScheduleStartDateChoiceForm(BaseRockyForm): choose_date = forms.ChoiceField( label="", required=False, @@ -41,21 +40,18 @@ class ReportReferenceDateForm(BaseRockyForm): choices=(("today", _("Today")), ("schedule", _("Different date"))), initial="today", ) + + +class ReportScheduleStartDateForm(BaseRockyForm): start_date = forms.DateField( label="", - widget=forms.HiddenInput(), + widget=DateInput(format="%Y-%m-%d"), initial=lambda: datetime.now(tz=timezone.utc).date(), - required=False, + required=True, ) - def clean(self) -> dict[str, Any]: - cleaned_data = super().clean() - if cleaned_data.get("choose_date") == "schedule": - self.fields["start_date"].widget = DateInput(format="%Y-%m-%d") - return cleaned_data - -class ReportRecurrenceForm(BaseRockyForm): +class ReportRecurrenceChoiceForm(BaseRockyForm): choose_recurrence = forms.ChoiceField( label="", required=False, @@ -63,10 +59,13 @@ class ReportRecurrenceForm(BaseRockyForm): choices=(("once", _("No, just once")), ("repeat", _("Yes, repeat"))), initial="once", ) + + +class ReportScheduleRecurrenceForm(BaseRockyForm): recurrence = forms.ChoiceField( label="", required=False, - widget=forms.Select, + widget=forms.Select(attrs={"form": "generate_report"}), choices=[ ("daily", _("Daily")), ("weekly", _("Weekly")), @@ -75,20 +74,6 @@ class ReportRecurrenceForm(BaseRockyForm): ], ) - def clean(self) -> dict[str, Any]: - cleaned_data = super().clean() - if cleaned_data.get("choose_recurrence") == "once": - self.fields["recurrence"].widget = forms.HiddenInput() - self.fields["recurrence"].choices = [("no_repeat", _("No Repeat"))] - return cleaned_data - - -class ReportScheduleForm(ReportReferenceDateForm, ReportRecurrenceForm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - for field in self.fields: - self.fields[field].widget = forms.HiddenInput() - class CustomReportScheduleForm(BaseRockyForm): start_date = forms.DateField( diff --git a/rocky/reports/templates/partials/export_report_settings.html b/rocky/reports/templates/partials/export_report_settings.html index efa2006f5c4..f9cd8999731 100644 --- a/rocky/reports/templates/partials/export_report_settings.html +++ b/rocky/reports/templates/partials/export_report_settings.html @@ -19,27 +19,23 @@

{% translate "Report schedule" %}

{% csrf_token %} {% include "forms/report_form_fields.html" %} -

{% translate "Reference date" %}

- {% include "partials/form/fieldset.html" with fields=report_reference_date_form %} -

{% translate "Recurrence" %}

- {% include "partials/form/fieldset.html" with fields=report_recurrence_form %} - - - {% if show_listed_report_names %} - {% include "partials/report_names_header.html" %} + {% include "partials/form/fieldset.html" with fields=report_schedule_form_recurrence_choice %} - {% endif %} - {% if is_scheduled_report %} - {% include "partials/report_names_header.html" %} + {% if is_scheduled_report %} + {% include "partials/form/fieldset.html" with fields=report_schedule_form_recurrence %} - {% endif %} + {% endif %} +
{% csrf_token %} {% include "forms/report_form_fields.html" %} - {% include "partials/form/fieldset.html" with fields=report_schedule_form %} + {% if show_listed_report_names %} + {% include "partials/report_names_header.html" %} {% include "partials/report_names_form.html" %} {% endif %} {% if is_scheduled_report %} + {% include "partials/report_names_header.html" %} {% include "partials/form/fieldset.html" with fields=report_parent_name_form %} {% if reports|length > 1 %} diff --git a/rocky/reports/views/base.py b/rocky/reports/views/base.py index 8230267b35b..a2541e42be4 100644 --- a/rocky/reports/views/base.py +++ b/rocky/reports/views/base.py @@ -24,6 +24,7 @@ from octopoes.models import OOI, Reference from octopoes.models.ooi.reports import Report as ReportOOI +from octopoes.models.ooi.reports import ReportRecipe from reports.forms import OOITypeMultiCheckboxForReportForm from reports.report_types.aggregate_organisation_report.report import AggregateOrganisationReport from reports.report_types.concatenated_report.report import ConcatenatedReport @@ -240,18 +241,12 @@ def get_plugin_data_for_saving(self) -> list[dict]: return plugin_data def show_report_names(self) -> bool: - date_choice = self.request.POST.get("choose_date", "today") recurrence_choice = self.request.POST.get("choose_recurrence", "once") - - return date_choice == "today" and recurrence_choice == "once" + return recurrence_choice == "once" def is_scheduled_report(self) -> bool: - date_choice = self.request.POST.get("choose_date", "schedule") - recurrence_choice = self.request.POST.get("choose_recurrence", "repeat") - - return (recurrence_choice in ["once", "repeat"] and date_choice == "schedule") or ( - date_choice == "today" and recurrence_choice == "repeat" - ) + recurrence_choice = self.request.POST.get("choose_recurrence", "") + return recurrence_choice == "repeat" def save_report_raw(self, data: dict) -> str: report_data_raw_id = self.bytes_client.upload_raw( @@ -301,6 +296,16 @@ def save_report_ooi( def get_observed_at(self): return self.observed_at if self.observed_at < datetime.now(timezone.utc) else datetime.now(timezone.utc) + def create_report_recipe(self, report_name_format: str, subreport_name_format: str, schedule: str): + return ReportRecipe( + recipe_id=uuid4(), + report_name_format=report_name_format, + subreport_name_format=subreport_name_format, + input_recipe={"input_oois": self.get_ooi_pks()}, + report_types=self.get_report_type_ids(), + cron_expression=schedule, + ) + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -562,8 +567,8 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["reports"] = self.get_report_names() - context["report_reference_date_form"] = self.get_report_reference_date_form() - context["report_recurrence_form"] = self.get_report_recurrence_form() + context["report_schedule_form_recurrence_choice"] = self.get_report_schedule_form_recurrence_choice() + context["report_schedule_form_recurrence"] = self.get_report_schedule_form_recurrence() context["report_parent_name_form"] = self.get_report_parent_name_form() context["report_child_name_form"] = self.get_report_child_name_form() @@ -571,8 +576,6 @@ def get_context_data(self, **kwargs): context["show_listed_report_names"] = self.show_listes_report_names context["is_scheduled_report"] = self.is_a_scheduled_report - context["report_schedule_form"] = self.get_report_schedule_form() - context["created_at"] = datetime.now() return context @@ -595,7 +598,16 @@ def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: + urlencode({"report_id": report_ooi.reference}) ) elif self.is_scheduled_report(): - self.schedule_report() + report_name_format = request.POST.get("parent_report_name", "") + subreport_name_format = request.POST.get("child_report_name", "") + + recurrence = request.POST.get("recurrence", "") + + schedule = self.convert_recurrence_to_cron_expressions(recurrence) + + report_recipe = self.create_report_recipe(report_name_format, subreport_name_format, schedule) + + self.create_report_schedule(report_recipe) return redirect(reverse("report_history", kwargs={"organization_code": self.organization.code})) diff --git a/rocky/rocky/scheduler.py b/rocky/rocky/scheduler.py index 23a9a971f17..f304bd4932e 100644 --- a/rocky/rocky/scheduler.py +++ b/rocky/rocky/scheduler.py @@ -285,6 +285,8 @@ def post_schedule(self, schedule: ScheduleRequest) -> ScheduleResponse: return ScheduleResponse.model_validate_json(res.content) except ValidationError: raise SchedulerValidationError(extra_message="Report schedule failed: ") + except HTTPStatusError: + raise SchedulerValidationError(extra_message="Report schedule failed: ") except ConnectError: raise SchedulerConnectError(extra_message="Report schedule failed: ") diff --git a/rocky/rocky/views/scheduler.py b/rocky/rocky/views/scheduler.py index 90f5af864ef..83449fee7e3 100644 --- a/rocky/rocky/views/scheduler.py +++ b/rocky/rocky/views/scheduler.py @@ -9,13 +9,15 @@ from reports.forms import ( ChildReportNameForm, ParentReportNameForm, - ReportRecurrenceForm, - ReportReferenceDateForm, - ReportScheduleForm, + ReportRecurrenceChoiceForm, + ReportScheduleRecurrenceForm, + ReportScheduleStartDateChoiceForm, + ReportScheduleStartDateForm, ) from tools.forms.scheduler import TaskFilterForm from octopoes.models import OOI +from octopoes.models.ooi.reports import ReportRecipe from rocky.scheduler import Boefje as SchedulerBoefje from rocky.scheduler import ( BoefjeTask, @@ -41,15 +43,17 @@ def get_date_time(date: str | None) -> datetime | None: class SchedulerView(OctopoesView): task_type: str + task_filter_form = TaskFilterForm - report_reference_date_form = ReportReferenceDateForm - report_recurrence_form = ReportRecurrenceForm + report_schedule_form_start_date_choice = ReportScheduleStartDateChoiceForm # today or different date + report_schedule_form_start_date = ReportScheduleStartDateForm # date widget - report_parent_name_form = ParentReportNameForm - report_child_name_form = ChildReportNameForm + report_schedule_form_recurrence_choice = ReportRecurrenceChoiceForm # once or repeat + report_schedule_form_recurrence = ReportScheduleRecurrenceForm # select interval (daily, weekly, etc..) - report_schedule_form = ReportScheduleForm + report_parent_name_form = ParentReportNameForm # parent name format + report_child_name_form = ChildReportNameForm # child name format def setup(self, request, *args, **kwargs): super().setup(request, *args, **kwargs) @@ -88,14 +92,17 @@ def get_task_list(self) -> LazyTaskList | list[Any]: messages.error(self.request, error.message) return [] - def get_report_schedule_form(self): - return self.report_schedule_form(self.request.POST) + def get_report_schedule_form_start_date_choice(self): + return self.report_schedule_form_start_date_choice(self.request.POST) + + def get_report_schedule_form_start_date(self): + return self.report_schedule_form_start_date(self.request.POST) - def get_report_reference_date_form(self): - return self.report_reference_date_form(self.request.POST) + def get_report_schedule_form_recurrence_choice(self): + return self.report_schedule_form_recurrence_choice(self.request.POST) - def get_report_recurrence_form(self): - return self.report_recurrence_form(self.request.POST) + def get_report_schedule_form_recurrence(self): + return self.report_schedule_form_recurrence(self.request.POST) def get_report_parent_name_form(self): return self.report_parent_name_form() @@ -103,52 +110,26 @@ def get_report_parent_name_form(self): def get_report_child_name_form(self): return self.report_child_name_form() - def get_report_schedule_form_data(self): - form_data = self.get_report_schedule_form().data - - return {k: v for k, v in form_data.items() if v} - def get_task_details(self, task_id: str) -> Task | None: try: return self.scheduler_client.get_task_details(task_id) except SchedulerError as error: return messages.error(self.request, error.message) - def create_report_schedule(self, start_date: str, recurrence: str) -> ScheduleResponse | None: + def create_report_schedule(self, report_recipe: ReportRecipe) -> ScheduleResponse | None: try: - schedule = self.convert_recurrence_to_cron_expressions(start_date, recurrence) schedule_request = ScheduleRequest( scheduler_id=self.scheduler_id, data=ReportTask( - organisation_id=self.organization.code, - report_recipe_id="", # TODO - ), - schedule=schedule, + organization=self.organization.code, + report_recipe_id=str(report_recipe.recipe_id), + ).model_dump(), + schedule=report_recipe.cron_expression, ) return self.scheduler_client.post_schedule(schedule=schedule_request) except SchedulerError as error: return messages.error(self.request, error.message) - def schedule_report(self) -> bool: - form_data = self.get_report_schedule_form_data() - start_date = None - recurrence = None - if form_data.get("choose_date") == "today": - start_date = datetime.now(tz=timezone.utc).date().strftime("%Y-%m-%d") - if form_data.get("choose_date") == "schedule": - start_date = form_data.get("start_date") - - if form_data.get("choose_recurrence") == "once": - recurrence = "no-repeat" - - if form_data.get("choose_recurrence") == "repeat": - recurrence = form_data.get("recurrence") - - if start_date is not None and recurrence is not None: - return False - - return False - def get_task_statistics(self) -> dict[Any, Any]: stats = {} try: @@ -274,29 +255,32 @@ def run_boefje_for_oois( except SchedulerError as error: messages.error(self.request, error.message) - def convert_recurrence_to_cron_expressions(self, start_date: str, recurrence: str) -> str: + def convert_recurrence_to_cron_expressions(self, recurrence: str) -> str: """ Because there is no time defined for the start date, we use midnight 00:00 for all expressions. """ - date: datetime = datetime.strptime(start_date, "%Y-%m-%d") - day = date.day - month = date.month - year = date.year + start_date = datetime.now(tz=timezone.utc).date() # for now, not set by user - weekday = date.strftime("%a").upper() # ex. THU - month_3L = date.strftime("%b").upper() # ex. AUG + if start_date and recurrence: + day = start_date.day + month = start_date.month + year = start_date.year - cron_expr = { - "no_repeat": f"0 0 0 {day} {month} ? {year}", # Run once on this date - "daily": "0 0 0 ? * * *", # Recurres every day at 00:00 - "weekly": f"0 0 0 ? * {weekday} *", # Recurres on every {weekday} at 00:00 - "yearly": f"0 0 0 {day} {month_3L} ? *", # Recurres every year on the {day} of the {month} at 00:00 - } + weekday = start_date.strftime("%a").upper() # ex. THU + month_3L = start_date.strftime("%b").upper() # ex. AUG - if 28 <= day <= 31: - cron_expr["monthly"] = "0 0 0 28-31 * * *" - else: - cron_expr["monthly"] = f"0 0 0 {day} * ? *" # Recurres on the exact {day} of the month at 00:00 + cron_expr = { + "no_repeat": f"0 0 0 {day} {month} ? {year}", # Run once on this date + "daily": "0 0 0 ? * * *", # Recurres every day at 00:00 + "weekly": f"0 0 0 ? * {weekday} *", # Recurres on every {weekday} at 00:00 + "yearly": f"0 0 0 {day} {month_3L} ? *", # Recurres every year on the {day} of the {month} at 00:00 + } + + if 28 <= day <= 31: + cron_expr["monthly"] = "0 0 0 28-31 * * *" + else: + cron_expr["monthly"] = f"0 0 0 {day} * ? *" # Recurres on the exact {day} of the month at 00:00 - return cron_expr.get(recurrence, "") + return cron_expr.get(recurrence, "") + return ""