From fa1935e470a1d8752a4e86ce916b58be4634b846 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 10 Oct 2024 14:43:09 +0200 Subject: [PATCH 01/15] rest api updates + tests --- src/hct_mis_api/api/endpoints/core/filters.py | 15 ++ src/hct_mis_api/api/endpoints/core/views.py | 22 +++ .../api/endpoints/lookups/__init__.py | 3 +- src/hct_mis_api/api/endpoints/lookups/base.py | 15 +- .../api/endpoints/program/filters.py | 28 ++++ .../api/endpoints/program/views.py | 21 +++ src/hct_mis_api/api/endpoints/serializers.py | 10 ++ src/hct_mis_api/api/urls.py | 3 +- src/hct_mis_api/config/fragments/drf.py | 3 +- tests/unit/api/test_lookups.py | 51 +++++++ tests/unit/api/test_program.py | 133 ++++++++++-------- 11 files changed, 241 insertions(+), 63 deletions(-) create mode 100644 src/hct_mis_api/api/endpoints/core/filters.py create mode 100644 src/hct_mis_api/api/endpoints/program/filters.py create mode 100644 tests/unit/api/test_lookups.py diff --git a/src/hct_mis_api/api/endpoints/core/filters.py b/src/hct_mis_api/api/endpoints/core/filters.py new file mode 100644 index 0000000000..8edf2657bd --- /dev/null +++ b/src/hct_mis_api/api/endpoints/core/filters.py @@ -0,0 +1,15 @@ +from django_filters import DateFromToRangeFilter +from django_filters.rest_framework import FilterSet + +from hct_mis_api.apps.core.models import BusinessArea + + +class BusinessAreaFilter(FilterSet): + updated_at = DateFromToRangeFilter() + + class Meta: + model = BusinessArea + fields = ( + "active", + "updated_at", + ) diff --git a/src/hct_mis_api/api/endpoints/core/views.py b/src/hct_mis_api/api/endpoints/core/views.py index 5a3abd7b79..ace581f6c9 100644 --- a/src/hct_mis_api/api/endpoints/core/views.py +++ b/src/hct_mis_api/api/endpoints/core/views.py @@ -1,10 +1,32 @@ +from typing import TYPE_CHECKING, Any + from rest_framework.generics import ListAPIView +from rest_framework.response import Response from hct_mis_api.api.endpoints.base import HOPEAPIView +from hct_mis_api.api.endpoints.core.filters import BusinessAreaFilter from hct_mis_api.api.endpoints.core.serializers import BusinessAreaSerializer from hct_mis_api.apps.core.models import BusinessArea +if TYPE_CHECKING: + from rest_framework.request import Request + class BusinessAreaListView(HOPEAPIView, ListAPIView): serializer_class = BusinessAreaSerializer queryset = BusinessArea.objects.all() + + def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response: + queryset = self.queryset + + filterset = BusinessAreaFilter(request.GET, queryset=queryset) + if filterset.is_valid(): + queryset = filterset.qs + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) diff --git a/src/hct_mis_api/api/endpoints/lookups/__init__.py b/src/hct_mis_api/api/endpoints/lookups/__init__.py index 5782862d81..f6a24e5bb6 100644 --- a/src/hct_mis_api/api/endpoints/lookups/__init__.py +++ b/src/hct_mis_api/api/endpoints/lookups/__init__.py @@ -1,10 +1,11 @@ from hct_mis_api.api.endpoints.lookups.area import AreaList, AreaTypeList # noqa: F401 from hct_mis_api.api.endpoints.lookups.base import ( # noqa: F401 - Country, + CountryAPIView, DataCollectingPolicy, DocumentType, MaritalStatus, ObservedDisability, + ProgramStatuses, Relationship, ResidenceStatus, Roles, diff --git a/src/hct_mis_api/api/endpoints/lookups/base.py b/src/hct_mis_api/api/endpoints/lookups/base.py index 737b463a67..18d4ae51e6 100644 --- a/src/hct_mis_api/api/endpoints/lookups/base.py +++ b/src/hct_mis_api/api/endpoints/lookups/base.py @@ -1,9 +1,10 @@ from typing import TYPE_CHECKING, Any, Optional -from django_countries import Countries from rest_framework.response import Response from hct_mis_api.api.endpoints.base import HOPEAPIView +from hct_mis_api.api.endpoints.serializers import CountrySerializer +from hct_mis_api.apps.geo.models import Country from hct_mis_api.apps.household.models import ( COLLECT_TYPES, IDENTIFICATION_TYPE_CHOICE, @@ -25,9 +26,12 @@ def get(self, request: "Request", format: Optional[Any] = None) -> Response: return Response(dict(IDENTIFICATION_TYPE_CHOICE)) -class Country(HOPEAPIView): +class CountryAPIView(HOPEAPIView): + serializer_class = CountrySerializer + def get(self, request: "Request", format: Optional[Any] = None) -> Response: - return Response(dict(Countries())) + serializer = self.serializer_class(Country.objects.all(), many=True) + return Response(serializer.data) class ResidenceStatus(HOPEAPIView): @@ -78,3 +82,8 @@ def get(self, request: "Request", format: Optional[Any] = None) -> Response: class ProgramScope(HOPEAPIView): def get(self, request: "Request", format: Optional[Any] = None) -> Response: return Response(dict(Program.SCOPE_CHOICE)) + + +class ProgramStatuses(HOPEAPIView): + def get(self, request: "Request", format: Optional[Any] = None) -> Response: + return Response(dict(Program.STATUS_CHOICE)) diff --git a/src/hct_mis_api/api/endpoints/program/filters.py b/src/hct_mis_api/api/endpoints/program/filters.py new file mode 100644 index 0000000000..e1ca22b5c7 --- /dev/null +++ b/src/hct_mis_api/api/endpoints/program/filters.py @@ -0,0 +1,28 @@ +from django.db.models.query import QuerySet + +from django_filters import BooleanFilter, CharFilter, DateFromToRangeFilter +from django_filters.rest_framework import FilterSet + +from hct_mis_api.apps.program.models import Program + + +class ProgramFilter(FilterSet): + business_area = CharFilter(field_name="business_area__slug") + active = BooleanFilter(method="is_active_filter") + updated_at = DateFromToRangeFilter() + + class Meta: + model = Program + fields = ( + "business_area", + "active", + "updated_at", + "status", + ) + + def is_active_filter(self, queryset: "QuerySet[Program]", name: str, value: bool) -> "QuerySet[Program]": + if value is True: + return queryset.filter(status=Program.ACTIVE) + elif value is False: + return queryset.exclude(status=Program.ACTIVE) + return queryset diff --git a/src/hct_mis_api/api/endpoints/program/views.py b/src/hct_mis_api/api/endpoints/program/views.py index 78e77da2f3..3402938df8 100644 --- a/src/hct_mis_api/api/endpoints/program/views.py +++ b/src/hct_mis_api/api/endpoints/program/views.py @@ -1,10 +1,31 @@ +from typing import TYPE_CHECKING, Any + from rest_framework.generics import ListAPIView +from rest_framework.response import Response from hct_mis_api.api.endpoints.base import HOPEAPIView +from hct_mis_api.api.endpoints.program.filters import ProgramFilter from hct_mis_api.api.endpoints.program.serializers import ProgramGlobalSerializer from hct_mis_api.apps.program.models import Program +if TYPE_CHECKING: + from rest_framework.request import Request + class ProgramGlobalListView(HOPEAPIView, ListAPIView): serializer_class = ProgramGlobalSerializer queryset = Program.objects.all() + + def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response: + queryset = self.queryset + filterset = ProgramFilter(request.GET, queryset=queryset) + if filterset.is_valid(): + queryset = filterset.qs + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.serializer_class(queryset, many=True) + return Response(serializer.data) diff --git a/src/hct_mis_api/api/endpoints/serializers.py b/src/hct_mis_api/api/endpoints/serializers.py index 0485a2e91e..163262a532 100644 --- a/src/hct_mis_api/api/endpoints/serializers.py +++ b/src/hct_mis_api/api/endpoints/serializers.py @@ -1,7 +1,17 @@ from django.db import models from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from hct_mis_api.apps.geo.models import Country + class RejectPolicy(models.TextChoices): STRICT = "STRICT", _("STRICT") LAX = "LAX", _("Lax") + + +class CountrySerializer(serializers.ModelSerializer): + class Meta: + model = Country + fields = ("name", "short_name", "iso_code2", "iso_code3", "iso_num") diff --git a/src/hct_mis_api/api/urls.py b/src/hct_mis_api/api/urls.py index 0871c71f28..86bdf21af0 100644 --- a/src/hct_mis_api/api/urls.py +++ b/src/hct_mis_api/api/urls.py @@ -22,7 +22,7 @@ path("areatypes/", endpoints.lookups.AreaTypeList().as_view(), name="areatype-list"), path("constance/", ConstanceSettingsAPIView().as_view(), name="constance-list"), path("lookups/document/", endpoints.lookups.DocumentType().as_view(), name="document-list"), - path("lookups/country/", endpoints.lookups.Country().as_view(), name="country-list"), + path("lookups/country/", endpoints.lookups.CountryAPIView().as_view(), name="country-list"), path("lookups/residencestatus/", endpoints.lookups.ResidenceStatus().as_view(), name="residencestatus-list"), path("lookups/maritalstatus/", endpoints.lookups.MaritalStatus().as_view(), name="maritalstatus-list"), path( @@ -36,6 +36,7 @@ ), path("lookups/role/", endpoints.lookups.Roles().as_view(), name="role-list"), path("lookups/sex/", endpoints.lookups.Sex().as_view(), name="sex-list"), + path("lookups/program-statuses/", endpoints.lookups.ProgramStatuses().as_view(), name="program-statuses-list"), path("business_areas/", endpoints.core.BusinessAreaListView.as_view(), name="business-area-list"), path("programs/", ProgramGlobalListView.as_view(), name="program-global-list"), path( diff --git a/src/hct_mis_api/config/fragments/drf.py b/src/hct_mis_api/config/fragments/drf.py index 11b5ee8354..543421b577 100644 --- a/src/hct_mis_api/config/fragments/drf.py +++ b/src/hct_mis_api/config/fragments/drf.py @@ -1,9 +1,10 @@ REST_FRAMEWORK = { "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", - "PAGE_SIZE": 10, + "PAGE_SIZE": 50, "TEST_REQUEST_DEFAULT_FORMAT": "json", "DEFAULT_AUTHENTICATION_CLASSES": [ "hct_mis_api.api.utils.CsrfExemptSessionAuthentication", ], "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", + "SECURE_SSL_REDIRECT": True, } diff --git a/tests/unit/api/test_lookups.py b/tests/unit/api/test_lookups.py new file mode 100644 index 0000000000..d6e1bc0270 --- /dev/null +++ b/tests/unit/api/test_lookups.py @@ -0,0 +1,51 @@ +from rest_framework import status +from rest_framework.reverse import reverse + +from hct_mis_api.apps.geo.fixtures import CountryFactory +from hct_mis_api.apps.program.models import Program +from tests.unit.api.base import HOPEApiTestCase + + +class APIProgramStatuesTests(HOPEApiTestCase): + databases = {"default"} + user_permissions = [] + + def test_get_program_statues(self) -> None: + url = reverse("api:program-statuses-list") + response = self.client.get(url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json(), dict(Program.STATUS_CHOICE)) + + +class APICountriesTests(HOPEApiTestCase): + databases = {"default"} + user_permissions = [] + + def test_get_countries(self) -> None: + country_afghanistan = CountryFactory() + country_poland = CountryFactory( + name="Poland", + short_name="Poland", + iso_code2="PL", + iso_code3="POL", + iso_num="0620", + ) + url = reverse("api:country-list") + response = self.client.get(url) + assert response.status_code == status.HTTP_200_OK + assert response.json() == [ + { + "name": country_afghanistan.name, + "short_name": country_afghanistan.short_name, + "iso_code2": country_afghanistan.iso_code2, + "iso_code3": country_afghanistan.iso_code3, + "iso_num": country_afghanistan.iso_num, + }, + { + "name": country_poland.name, + "short_name": country_poland.short_name, + "iso_code2": country_poland.iso_code2, + "iso_code3": country_poland.iso_code3, + "iso_num": country_poland.iso_num, + }, + ] diff --git a/tests/unit/api/test_program.py b/tests/unit/api/test_program.py index 1f006f50ba..021660149f 100644 --- a/tests/unit/api/test_program.py +++ b/tests/unit/api/test_program.py @@ -6,6 +6,7 @@ from hct_mis_api.api.models import APIToken, Grant from hct_mis_api.apps.account.fixtures import BusinessAreaFactory from hct_mis_api.apps.core.fixtures import DataCollectingTypeFactory +from hct_mis_api.apps.core.models import BusinessArea from hct_mis_api.apps.program.fixtures import ProgramFactory from hct_mis_api.apps.program.models import Program from tests.unit.api.base import HOPEApiTestCase @@ -156,39 +157,69 @@ class APIGlobalProgramTests(HOPEApiTestCase): def setUpTestData(cls) -> None: super().setUpTestData() cls.list_url = reverse("api:program-global-list") - - def test_list_program(self) -> None: - program1: Program = ProgramFactory( + cls.program1: Program = ProgramFactory( budget=10000, start_date="2022-01-12", end_date="2022-09-12", - business_area=self.business_area, + business_area=cls.business_area, population_goal=200, status=Program.ACTIVE, ) - program2: Program = ProgramFactory( + cls.program2: Program = ProgramFactory( budget=200, start_date="2022-01-10", end_date="2022-09-10", - business_area=self.business_area, + business_area=cls.business_area, population_goal=200, status=Program.DRAFT, ) # program from another BA - also listed as we do not filter by BA - business_area2 = BusinessAreaFactory(name="Ukraine") - program_from_another_ba = ProgramFactory( + cls.business_area2 = BusinessAreaFactory(name="Ukraine") + cls.program_from_another_ba = ProgramFactory( budget=200, start_date="2022-01-10", end_date="2022-09-10", - business_area=business_area2, + business_area=cls.business_area2, population_goal=400, status=Program.ACTIVE, ) - program1.refresh_from_db() - program2.refresh_from_db() - program_from_another_ba.refresh_from_db() + cls.program1.refresh_from_db() + cls.program2.refresh_from_db() + cls.program_from_another_ba.refresh_from_db() + + def expected_response(program: Program, business_area: BusinessArea) -> dict: + return { + "budget": str(program.budget), + "business_area_code": business_area.code, + "cash_plus": program.cash_plus, + "end_date": program.end_date.strftime("%Y-%m-%d"), + "frequency_of_payments": program.frequency_of_payments, + "id": str(program.id), + "name": program.name, + "population_goal": program.population_goal, + "programme_code": program.programme_code, + "scope": program.scope, + "sector": program.sector, + "status": program.status, + "start_date": program.start_date.strftime("%Y-%m-%d"), + } + + cls.program_from_another_ba_expected_response = expected_response( + cls.program_from_another_ba, + cls.business_area2, + ) + + cls.program1_expected_response = expected_response( + cls.program1, + cls.business_area, + ) + cls.program2_expected_response = expected_response( + cls.program2, + cls.business_area, + ) + def test_list_program(self) -> None: response = self.client.get(self.list_url) assert response.status_code == 403 @@ -197,56 +228,44 @@ def test_list_program(self) -> None: self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()["results"]), 3) self.assertIn( - { - "budget": str(program1.budget), - "business_area_code": self.business_area.code, - "cash_plus": program1.cash_plus, - "end_date": program1.end_date.strftime("%Y-%m-%d"), - "frequency_of_payments": program1.frequency_of_payments, - "id": str(program1.id), - "name": program1.name, - "population_goal": program1.population_goal, - "programme_code": program1.programme_code, - "scope": program1.scope, - "sector": program1.sector, - "status": program1.status, - "start_date": program1.start_date.strftime("%Y-%m-%d"), - }, + self.program1_expected_response, response.json()["results"], ) self.assertIn( - { - "budget": str(program2.budget), - "business_area_code": self.business_area.code, - "cash_plus": program2.cash_plus, - "end_date": program2.end_date.strftime("%Y-%m-%d"), - "frequency_of_payments": program2.frequency_of_payments, - "id": str(program2.id), - "name": program2.name, - "population_goal": program2.population_goal, - "programme_code": program2.programme_code, - "scope": program2.scope, - "sector": program2.sector, - "status": program2.status, - "start_date": program2.start_date.strftime("%Y-%m-%d"), - }, + self.program2_expected_response, response.json()["results"], ) self.assertIn( - { - "budget": str(program_from_another_ba.budget), - "business_area_code": business_area2.code, - "cash_plus": program_from_another_ba.cash_plus, - "end_date": program_from_another_ba.end_date.strftime("%Y-%m-%d"), - "frequency_of_payments": program_from_another_ba.frequency_of_payments, - "id": str(program_from_another_ba.id), - "name": program_from_another_ba.name, - "population_goal": program_from_another_ba.population_goal, - "programme_code": program_from_another_ba.programme_code, - "scope": program_from_another_ba.scope, - "sector": program_from_another_ba.sector, - "status": program_from_another_ba.status, - "start_date": program_from_another_ba.start_date.strftime("%Y-%m-%d"), - }, + self.program_from_another_ba_expected_response, + response.json()["results"], + ) + + def test_list_program_filter_business_area(self) -> None: + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(self.list_url, {"business_area": "afghanistan"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["results"]), 2) + self.assertNotIn( + self.program_from_another_ba_expected_response, + response.json()["results"], + ) + + def test_list_program_filter_active(self) -> None: + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(self.list_url, {"active": "true"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["results"]), 2) + self.assertNotIn( + self.program2_expected_response, + response.json()["results"], + ) + + def test_list_program_filter_status(self) -> None: + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(self.list_url, {"status": Program.DRAFT}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["results"]), 1) + self.assertIn( + self.program2_expected_response, response.json()["results"], ) From 824df8eedac76365c03a49cc07acd71b9e35a5a0 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Mon, 14 Oct 2024 20:50:19 +0200 Subject: [PATCH 02/15] apply updated_at filter to all endpoints --- src/hct_mis_api/api/endpoints/core/filters.py | 13 ++---- .../api/endpoints/program/filters.py | 8 ++-- src/hct_mis_api/api/endpoints/rdi/program.py | 18 +++++--- src/hct_mis_api/apps/core/api/__init__.py | 0 src/hct_mis_api/apps/core/api/filters.py | 6 +++ src/hct_mis_api/apps/geo/api/filters.py | 3 +- .../apps/periodic_data_update/api/views.py | 11 +++-- src/hct_mis_api/apps/program/api/filters.py | 3 +- .../apps/registration_data/api/filters.py | 3 +- src/hct_mis_api/apps/targeting/api/filters.py | 3 +- tests/unit/api/base.py | 13 ++++++ tests/unit/api/test_business_area.py | 17 +------ tests/unit/api/test_lookups.py | 9 ++-- tests/unit/api/test_program.py | 44 +++++++++++++------ 14 files changed, 91 insertions(+), 60 deletions(-) create mode 100644 src/hct_mis_api/apps/core/api/__init__.py create mode 100644 src/hct_mis_api/apps/core/api/filters.py diff --git a/src/hct_mis_api/api/endpoints/core/filters.py b/src/hct_mis_api/api/endpoints/core/filters.py index 8edf2657bd..eeb392e7f8 100644 --- a/src/hct_mis_api/api/endpoints/core/filters.py +++ b/src/hct_mis_api/api/endpoints/core/filters.py @@ -1,15 +1,8 @@ -from django_filters import DateFromToRangeFilter -from django_filters.rest_framework import FilterSet - +from hct_mis_api.apps.core.api.filters import UpdatedAtFilter from hct_mis_api.apps.core.models import BusinessArea -class BusinessAreaFilter(FilterSet): - updated_at = DateFromToRangeFilter() - +class BusinessAreaFilter(UpdatedAtFilter): class Meta: model = BusinessArea - fields = ( - "active", - "updated_at", - ) + fields = ("active",) diff --git a/src/hct_mis_api/api/endpoints/program/filters.py b/src/hct_mis_api/api/endpoints/program/filters.py index e1ca22b5c7..b00e330ca0 100644 --- a/src/hct_mis_api/api/endpoints/program/filters.py +++ b/src/hct_mis_api/api/endpoints/program/filters.py @@ -1,22 +1,20 @@ from django.db.models.query import QuerySet -from django_filters import BooleanFilter, CharFilter, DateFromToRangeFilter -from django_filters.rest_framework import FilterSet +from django_filters import BooleanFilter, CharFilter +from hct_mis_api.apps.core.api.filters import UpdatedAtFilter from hct_mis_api.apps.program.models import Program -class ProgramFilter(FilterSet): +class ProgramFilter(UpdatedAtFilter): business_area = CharFilter(field_name="business_area__slug") active = BooleanFilter(method="is_active_filter") - updated_at = DateFromToRangeFilter() class Meta: model = Program fields = ( "business_area", "active", - "updated_at", "status", ) diff --git a/src/hct_mis_api/api/endpoints/rdi/program.py b/src/hct_mis_api/api/endpoints/rdi/program.py index fe783b388e..b64cf61dc3 100644 --- a/src/hct_mis_api/api/endpoints/rdi/program.py +++ b/src/hct_mis_api/api/endpoints/rdi/program.py @@ -8,6 +8,7 @@ from hct_mis_api.api.endpoints.base import HOPEAPIBusinessAreaViewSet from hct_mis_api.api.models import Grant +from hct_mis_api.apps.core.api.filters import UpdatedAtFilter from hct_mis_api.apps.program.models import Program if TYPE_CHECKING: @@ -33,17 +34,17 @@ class Meta: class ProgramViewSet(CreateModelMixin, HOPEAPIBusinessAreaViewSet): - serializer = ProgramSerializer + serializer_class = ProgramSerializer model = Program permission = Grant.API_READ_ONLY - def perform_create(self, serializer: "BaseSerializer") -> None: - serializer.save(business_area=self.selected_business_area) + def perform_create(self, serializer_class: "BaseSerializer") -> None: + serializer_class.save(business_area=self.selected_business_area) @extend_schema(request=ProgramSerializer) def create(self, request: "Request", *args: Any, **kwargs: Any) -> Response: - self.selected_business_area # TODO does it work? It should be called - serializer = ProgramSerializer(data=request.data) + self.selected_business_area + serializer = self.serializer_class(data=request.data) serializer.is_valid(raise_exception=True) if Grant.API_PROGRAM_CREATE.name not in request.auth.grants: raise PermissionDenied() @@ -53,5 +54,10 @@ def create(self, request: "Request", *args: Any, **kwargs: Any) -> Response: def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response: queryset = self.model.objects.filter(business_area=self.selected_business_area) - serializer = self.serializer(queryset, many=True) + + filterset = UpdatedAtFilter(request.GET, queryset=queryset) + if filterset.is_valid(): + queryset = filterset.qs + + serializer = self.serializer_class(queryset, many=True) return Response(serializer.data) diff --git a/src/hct_mis_api/apps/core/api/__init__.py b/src/hct_mis_api/apps/core/api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/hct_mis_api/apps/core/api/filters.py b/src/hct_mis_api/apps/core/api/filters.py new file mode 100644 index 0000000000..640c16782e --- /dev/null +++ b/src/hct_mis_api/apps/core/api/filters.py @@ -0,0 +1,6 @@ +from django_filters import DateFromToRangeFilter +from django_filters.rest_framework import FilterSet + + +class UpdatedAtFilter(FilterSet): + updated_at = DateFromToRangeFilter() diff --git a/src/hct_mis_api/apps/geo/api/filters.py b/src/hct_mis_api/apps/geo/api/filters.py index cbe9f3d022..f1a2495546 100644 --- a/src/hct_mis_api/apps/geo/api/filters.py +++ b/src/hct_mis_api/apps/geo/api/filters.py @@ -1,9 +1,10 @@ from django_filters import rest_framework as filters +from hct_mis_api.apps.core.api.filters import UpdatedAtFilter from hct_mis_api.apps.geo.models import Area -class AreaFilter(filters.FilterSet): +class AreaFilter(UpdatedAtFilter): level = filters.NumberFilter( field_name="area_type__area_level", ) diff --git a/src/hct_mis_api/apps/periodic_data_update/api/views.py b/src/hct_mis_api/apps/periodic_data_update/api/views.py index 68f8757903..fbfabb7fa6 100644 --- a/src/hct_mis_api/apps/periodic_data_update/api/views.py +++ b/src/hct_mis_api/apps/periodic_data_update/api/views.py @@ -5,6 +5,7 @@ from django.db.models import QuerySet from constance import config +from django_filters.rest_framework import DjangoFilterBackend from rest_framework import mixins, status from rest_framework.decorators import action from rest_framework.exceptions import ValidationError @@ -22,6 +23,7 @@ PDUUploadPermission, PDUViewListAndDetailsPermission, ) +from hct_mis_api.apps.core.api.filters import UpdatedAtFilter from hct_mis_api.apps.core.api.mixins import ( ActionMixin, BusinessAreaProgramMixin, @@ -69,7 +71,8 @@ class PeriodicDataUpdateTemplateViewSet( "export": [PDUTemplateCreatePermission], "download": [PDUTemplateDownloadPermission], } - filter_backends = (OrderingFilter,) + filter_backends = (OrderingFilter, DjangoFilterBackend) + filterset_class = UpdatedAtFilter def get_queryset(self) -> QuerySet: business_area = self.get_business_area() @@ -132,7 +135,8 @@ class PeriodicDataUpdateUploadViewSet( "retrieve": [PDUViewListAndDetailsPermission], "upload": [PDUUploadPermission], } - filter_backends = (OrderingFilter,) + filter_backends = (OrderingFilter, DjangoFilterBackend) + filterset_class = UpdatedAtFilter def get_queryset(self) -> QuerySet: business_area = self.get_business_area() @@ -183,7 +187,8 @@ class PeriodicFieldViewSet( ): serializer_class = PeriodicFieldSerializer permission_classes = [PDUViewListAndDetailsPermission] - filter_backends = (OrderingFilter,) + filter_backends = (OrderingFilter, DjangoFilterBackend) + filterset_class = UpdatedAtFilter def get_queryset(self) -> QuerySet: program = self.get_program() diff --git a/src/hct_mis_api/apps/program/api/filters.py b/src/hct_mis_api/apps/program/api/filters.py index f0d6950b51..ac1e7f6073 100644 --- a/src/hct_mis_api/apps/program/api/filters.py +++ b/src/hct_mis_api/apps/program/api/filters.py @@ -7,11 +7,12 @@ from django_filters import rest_framework as filters +from hct_mis_api.apps.core.api.filters import UpdatedAtFilter from hct_mis_api.apps.core.utils import decode_id_string_required from hct_mis_api.apps.program.models import ProgramCycle -class ProgramCycleFilter(filters.FilterSet): +class ProgramCycleFilter(UpdatedAtFilter): search = filters.CharFilter(method="search_filter") status = filters.MultipleChoiceFilter( choices=ProgramCycle.STATUS_CHOICE, diff --git a/src/hct_mis_api/apps/registration_data/api/filters.py b/src/hct_mis_api/apps/registration_data/api/filters.py index 2dffd4d71b..52170229fc 100644 --- a/src/hct_mis_api/apps/registration_data/api/filters.py +++ b/src/hct_mis_api/apps/registration_data/api/filters.py @@ -1,9 +1,10 @@ from django_filters import rest_framework as filters +from hct_mis_api.apps.core.api.filters import UpdatedAtFilter from hct_mis_api.apps.registration_data.models import RegistrationDataImport -class RegistrationDataImportFilter(filters.FilterSet): +class RegistrationDataImportFilter(UpdatedAtFilter): name = filters.CharFilter( field_name="name", lookup_expr="startswith", diff --git a/src/hct_mis_api/apps/targeting/api/filters.py b/src/hct_mis_api/apps/targeting/api/filters.py index b91674f511..f349c8785a 100644 --- a/src/hct_mis_api/apps/targeting/api/filters.py +++ b/src/hct_mis_api/apps/targeting/api/filters.py @@ -1,9 +1,10 @@ from django_filters import rest_framework as filters +from hct_mis_api.apps.core.api.filters import UpdatedAtFilter from hct_mis_api.apps.targeting.models import TargetPopulation -class TargetPopulationFilter(filters.FilterSet): +class TargetPopulationFilter(UpdatedAtFilter): name = filters.CharFilter( field_name="name", lookup_expr="startswith", diff --git a/tests/unit/api/base.py b/tests/unit/api/base.py index f13b28ad72..f156e50bc0 100644 --- a/tests/unit/api/base.py +++ b/tests/unit/api/base.py @@ -1,3 +1,6 @@ +import contextlib +from typing import Iterator + from django.urls import reverse from rest_framework import status @@ -13,6 +16,16 @@ from tests.unit.api.factories import APITokenFactory +@contextlib.contextmanager +def token_grant_permission(token: APIToken, grant: Grant) -> Iterator: + old = token.grants + token.grants += [grant.name] + token.save() + yield + token.grants = old + token.save() + + class HOPEApiTestCase(APITestCase): databases = {"default"} user_permissions = [ diff --git a/tests/unit/api/test_business_area.py b/tests/unit/api/test_business_area.py index 5367be30b4..8f94b43b15 100644 --- a/tests/unit/api/test_business_area.py +++ b/tests/unit/api/test_business_area.py @@ -1,22 +1,9 @@ -import contextlib -from typing import Iterator - from rest_framework.reverse import reverse -from hct_mis_api.api.models import APIToken, Grant +from hct_mis_api.api.models import Grant from hct_mis_api.apps.account.fixtures import BusinessAreaFactory from hct_mis_api.apps.core.models import BusinessArea -from tests.unit.api.base import HOPEApiTestCase - - -@contextlib.contextmanager -def token_grant_permission(token: APIToken, grant: Grant) -> Iterator: - old = token.grants - token.grants += [grant.name] - token.save() - yield - token.grants = old - token.save() +from tests.unit.api.base import HOPEApiTestCase, token_grant_permission class APIBusinessAreaTests(HOPEApiTestCase): diff --git a/tests/unit/api/test_lookups.py b/tests/unit/api/test_lookups.py index d6e1bc0270..5a6c1f6768 100644 --- a/tests/unit/api/test_lookups.py +++ b/tests/unit/api/test_lookups.py @@ -1,9 +1,10 @@ from rest_framework import status from rest_framework.reverse import reverse +from hct_mis_api.api.models import Grant from hct_mis_api.apps.geo.fixtures import CountryFactory from hct_mis_api.apps.program.models import Program -from tests.unit.api.base import HOPEApiTestCase +from tests.unit.api.base import HOPEApiTestCase, token_grant_permission class APIProgramStatuesTests(HOPEApiTestCase): @@ -12,7 +13,8 @@ class APIProgramStatuesTests(HOPEApiTestCase): def test_get_program_statues(self) -> None: url = reverse("api:program-statuses-list") - response = self.client.get(url) + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json(), dict(Program.STATUS_CHOICE)) @@ -31,7 +33,8 @@ def test_get_countries(self) -> None: iso_num="0620", ) url = reverse("api:country-list") - response = self.client.get(url) + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(url) assert response.status_code == status.HTTP_200_OK assert response.json() == [ { diff --git a/tests/unit/api/test_program.py b/tests/unit/api/test_program.py index 021660149f..abd7ee99fc 100644 --- a/tests/unit/api/test_program.py +++ b/tests/unit/api/test_program.py @@ -1,25 +1,16 @@ -import contextlib -from typing import Iterator +from datetime import timedelta + +from django.utils import timezone from rest_framework.reverse import reverse -from hct_mis_api.api.models import APIToken, Grant +from hct_mis_api.api.models import Grant from hct_mis_api.apps.account.fixtures import BusinessAreaFactory from hct_mis_api.apps.core.fixtures import DataCollectingTypeFactory from hct_mis_api.apps.core.models import BusinessArea from hct_mis_api.apps.program.fixtures import ProgramFactory from hct_mis_api.apps.program.models import Program -from tests.unit.api.base import HOPEApiTestCase - - -@contextlib.contextmanager -def token_grant_permission(token: APIToken, grant: Grant) -> Iterator: - old = token.grants - token.grants += [grant.name] - token.save() - yield - token.grants = old - token.save() +from tests.unit.api.base import HOPEApiTestCase, token_grant_permission class APIProgramTests(HOPEApiTestCase): @@ -269,3 +260,28 @@ def test_list_program_filter_status(self) -> None: self.program2_expected_response, response.json()["results"], ) + + def test_list_program_filter_updated_at(self) -> None: + tomorrow = (timezone.now() + timedelta(days=1)).date() + tomorrow_str = tomorrow.strftime("%Y-%m-%d") + yesterday = (timezone.now() - timedelta(days=1)).date() + yesterday_str = yesterday.strftime("%Y-%m-%d") + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(self.list_url, {"updated_at_before": tomorrow_str}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["results"]), 3) + + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(self.list_url, {"updated_at_after": tomorrow_str}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["results"]), 0) + + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(self.list_url, {"updated_at_before": yesterday_str}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["results"]), 0) + + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(self.list_url, {"updated_at_after": yesterday_str}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["results"]), 3) From 1bbaf1e45ff5aa87caaff1aae56d50680e2cb8d4 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Mon, 14 Oct 2024 21:37:44 +0200 Subject: [PATCH 03/15] fix mypy --- src/hct_mis_api/apps/payment/services/payment_gateway.py | 6 +++--- .../apps/registration_datahub/apis/deduplication_engine.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/hct_mis_api/apps/payment/services/payment_gateway.py b/src/hct_mis_api/apps/payment/services/payment_gateway.py index 73fb54539c..05c8402441 100644 --- a/src/hct_mis_api/apps/payment/services/payment_gateway.py +++ b/src/hct_mis_api/apps/payment/services/payment_gateway.py @@ -278,8 +278,8 @@ class PaymentGatewayAPIException(Exception): class PaymentGatewayMissingAPICredentialsException(Exception): pass - API_EXCEPTION_CLASS = PaymentGatewayAPIException - API_MISSING_CREDENTIALS_EXCEPTION_CLASS = PaymentGatewayMissingAPICredentialsException + API_EXCEPTION_CLASS = PaymentGatewayAPIException # type: ignore + API_MISSING_CREDENTIALS_EXCEPTION_CLASS = PaymentGatewayMissingAPICredentialsException # type: ignore class Endpoints: CREATE_PAYMENT_INSTRUCTION = "payment_instructions/" @@ -307,7 +307,7 @@ def create_payment_instruction(self, data: dict) -> PaymentInstructionData: def change_payment_instruction_status(self, status: PaymentInstructionStatus, remote_id: str) -> str: if status.value not in [s.value for s in PaymentInstructionStatus]: - raise self.API_EXCEPTION_CLASS(f"Can't set invalid Payment Instruction status: {status}") + raise self.API_EXCEPTION_CLASS(f"Can't set invalid Payment Instruction status: {status}") # type: ignore action_endpoint_map = { PaymentInstructionStatus.ABORTED: self.Endpoints.ABORT_PAYMENT_INSTRUCTION_STATUS, diff --git a/src/hct_mis_api/apps/registration_datahub/apis/deduplication_engine.py b/src/hct_mis_api/apps/registration_datahub/apis/deduplication_engine.py index 8fb5df3078..269374a383 100644 --- a/src/hct_mis_api/apps/registration_datahub/apis/deduplication_engine.py +++ b/src/hct_mis_api/apps/registration_datahub/apis/deduplication_engine.py @@ -50,8 +50,8 @@ class DeduplicationEngineAPIException(Exception): class DeduplicationEngineMissingAPICredentialsException(Exception): pass - API_EXCEPTION_CLASS = DeduplicationEngineAPIException - API_MISSING_CREDENTIALS_EXCEPTION_CLASS = DeduplicationEngineMissingAPICredentialsException + API_EXCEPTION_CLASS = DeduplicationEngineAPIException # type: ignore + API_MISSING_CREDENTIALS_EXCEPTION_CLASS = DeduplicationEngineMissingAPICredentialsException # type: ignore class Endpoints: GET_DEDUPLICATION_SETS = "deduplication_sets/" # GET - List view From d3f16fd1fbfa70dcbcabb6243e4d06e693dcb2d1 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Tue, 15 Oct 2024 00:42:14 +0200 Subject: [PATCH 04/15] codecov adjustments --- codecov.yml | 12 ++++++++++++ src/hct_mis_api/api/endpoints/core/views.py | 8 ++------ src/hct_mis_api/api/endpoints/program/filters.py | 4 +--- src/hct_mis_api/api/endpoints/program/views.py | 8 ++------ tests/unit/api/test_program.py | 10 ++++++++++ 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/codecov.yml b/codecov.yml index 72f96c556e..58ffb82578 100644 --- a/codecov.yml +++ b/codecov.yml @@ -25,3 +25,15 @@ coverage: - "hct_mis_api/conftest.py" - "hct_mis_api/config/settings.py" - "hct_mis_api/apps/core/management/commands/*" + - "pragma: no cover" + - "pragma: no-cover" + - "def __repr__" + - "if self\.debug" + - "raise AssertionError" + - "raise NotImplementedError" + - "except ImportError" + - "#if 0:" + - "if __name__ == .__main__." + - "if TYPE_CHECKING" + - "^\\s*(import\\s.+|from\\s+.+import\\s+.+)" + - "logger.exception(e)" \ No newline at end of file diff --git a/src/hct_mis_api/api/endpoints/core/views.py b/src/hct_mis_api/api/endpoints/core/views.py index ace581f6c9..ad16b5ff97 100644 --- a/src/hct_mis_api/api/endpoints/core/views.py +++ b/src/hct_mis_api/api/endpoints/core/views.py @@ -24,9 +24,5 @@ def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response: queryset = filterset.qs page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) diff --git a/src/hct_mis_api/api/endpoints/program/filters.py b/src/hct_mis_api/api/endpoints/program/filters.py index b00e330ca0..c497b08ece 100644 --- a/src/hct_mis_api/api/endpoints/program/filters.py +++ b/src/hct_mis_api/api/endpoints/program/filters.py @@ -21,6 +21,4 @@ class Meta: def is_active_filter(self, queryset: "QuerySet[Program]", name: str, value: bool) -> "QuerySet[Program]": if value is True: return queryset.filter(status=Program.ACTIVE) - elif value is False: - return queryset.exclude(status=Program.ACTIVE) - return queryset + return queryset.exclude(status=Program.ACTIVE) diff --git a/src/hct_mis_api/api/endpoints/program/views.py b/src/hct_mis_api/api/endpoints/program/views.py index 3402938df8..0140de84f7 100644 --- a/src/hct_mis_api/api/endpoints/program/views.py +++ b/src/hct_mis_api/api/endpoints/program/views.py @@ -23,9 +23,5 @@ def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response: queryset = filterset.qs page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.serializer_class(queryset, many=True) - return Response(serializer.data) + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) diff --git a/tests/unit/api/test_program.py b/tests/unit/api/test_program.py index abd7ee99fc..64fd577740 100644 --- a/tests/unit/api/test_program.py +++ b/tests/unit/api/test_program.py @@ -251,6 +251,16 @@ def test_list_program_filter_active(self) -> None: response.json()["results"], ) + def test_list_program_filter_not_active(self) -> None: + with token_grant_permission(self.token, Grant.API_READ_ONLY): + response = self.client.get(self.list_url, {"active": "false"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["results"]), 1) + self.assertIn( + self.program2_expected_response, + response.json()["results"], + ) + def test_list_program_filter_status(self) -> None: with token_grant_permission(self.token, Grant.API_READ_ONLY): response = self.client.get(self.list_url, {"status": Program.DRAFT}) From 4c317c279dc031653427b0f2577706ead968b0e2 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Tue, 15 Oct 2024 12:04:20 +0200 Subject: [PATCH 05/15] codecov fixes --- codecov.yml | 2 -- tests/.coveragerc | 1 - 2 files changed, 3 deletions(-) diff --git a/codecov.yml b/codecov.yml index 58ffb82578..bf1bacd789 100644 --- a/codecov.yml +++ b/codecov.yml @@ -28,11 +28,9 @@ coverage: - "pragma: no cover" - "pragma: no-cover" - "def __repr__" - - "if self\.debug" - "raise AssertionError" - "raise NotImplementedError" - "except ImportError" - - "#if 0:" - "if __name__ == .__main__." - "if TYPE_CHECKING" - "^\\s*(import\\s.+|from\\s+.+import\\s+.+)" diff --git a/tests/.coveragerc b/tests/.coveragerc index 7942061b4e..8e5e354932 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -18,7 +18,6 @@ omit = hct_mis_api/conftest.py hct_mis_api/config/settings.py hct_mis_api/apps/core/management/commands/* - hct_mis_api/apps/household/forms.py [report] # Regexes for lines to exclude from consideration From 01e97f41eb4efc2b64d8706d95948242d11d23e5 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Tue, 15 Oct 2024 16:31:28 +0200 Subject: [PATCH 06/15] use ListAPIView for Country API, simplify program view --- src/hct_mis_api/api/endpoints/lookups/base.py | 9 ++++----- .../api/endpoints/program/views.py | 20 ++++--------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/hct_mis_api/api/endpoints/lookups/base.py b/src/hct_mis_api/api/endpoints/lookups/base.py index 18d4ae51e6..85fcd905e3 100644 --- a/src/hct_mis_api/api/endpoints/lookups/base.py +++ b/src/hct_mis_api/api/endpoints/lookups/base.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Any, Optional +from rest_framework.generics import ListAPIView from rest_framework.response import Response from hct_mis_api.api.endpoints.base import HOPEAPIView @@ -26,12 +27,10 @@ def get(self, request: "Request", format: Optional[Any] = None) -> Response: return Response(dict(IDENTIFICATION_TYPE_CHOICE)) -class CountryAPIView(HOPEAPIView): +class CountryAPIView(HOPEAPIView, ListAPIView): + queryset = Country.objects.all() serializer_class = CountrySerializer - - def get(self, request: "Request", format: Optional[Any] = None) -> Response: - serializer = self.serializer_class(Country.objects.all(), many=True) - return Response(serializer.data) + pagination_class = None class ResidenceStatus(HOPEAPIView): diff --git a/src/hct_mis_api/api/endpoints/program/views.py b/src/hct_mis_api/api/endpoints/program/views.py index 0140de84f7..cb8ee4626e 100644 --- a/src/hct_mis_api/api/endpoints/program/views.py +++ b/src/hct_mis_api/api/endpoints/program/views.py @@ -1,27 +1,15 @@ -from typing import TYPE_CHECKING, Any - +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework.filters import OrderingFilter from rest_framework.generics import ListAPIView -from rest_framework.response import Response from hct_mis_api.api.endpoints.base import HOPEAPIView from hct_mis_api.api.endpoints.program.filters import ProgramFilter from hct_mis_api.api.endpoints.program.serializers import ProgramGlobalSerializer from hct_mis_api.apps.program.models import Program -if TYPE_CHECKING: - from rest_framework.request import Request - class ProgramGlobalListView(HOPEAPIView, ListAPIView): serializer_class = ProgramGlobalSerializer queryset = Program.objects.all() - - def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response: - queryset = self.queryset - filterset = ProgramFilter(request.GET, queryset=queryset) - if filterset.is_valid(): - queryset = filterset.qs - - page = self.paginate_queryset(queryset) - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) + filter_backends = (OrderingFilter, DjangoFilterBackend) + filterset_class = ProgramFilter From 08667db919fcd2f6c91cb41b6351993568e0a40b Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Tue, 15 Oct 2024 20:31:34 +0200 Subject: [PATCH 07/15] The label should contain 'English(EN)' --- src/hct_mis_api/apps/core/fixtures.py | 4 ++-- .../apps/core/migrations/0088_migration.py | 19 +++++++++++++++++ src/hct_mis_api/apps/core/models.py | 9 +++++++- .../apps/periodic_data_update/test_models.py | 21 +++++++++++++++++++ 4 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 src/hct_mis_api/apps/core/migrations/0088_migration.py diff --git a/src/hct_mis_api/apps/core/fixtures.py b/src/hct_mis_api/apps/core/fixtures.py index 9035c9e502..c9fe19ad5a 100644 --- a/src/hct_mis_api/apps/core/fixtures.py +++ b/src/hct_mis_api/apps/core/fixtures.py @@ -1,3 +1,4 @@ +import json import random from typing import Any, List @@ -146,9 +147,8 @@ def program(self) -> Any: @classmethod def _create(cls, target_class: Any, *args: Any, **kwargs: Any) -> FlexibleAttribute: label = kwargs.pop("label", None) + kwargs["label"] = json.dumps({"English(EN)": label}) obj = super()._create(target_class, *args, **kwargs) - obj.label = {"English(EN)": label} - obj.save() return obj class Meta: diff --git a/src/hct_mis_api/apps/core/migrations/0088_migration.py b/src/hct_mis_api/apps/core/migrations/0088_migration.py new file mode 100644 index 0000000000..8a42e11705 --- /dev/null +++ b/src/hct_mis_api/apps/core/migrations/0088_migration.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-10-15 17:18 + +from django.db import migrations, models +import hct_mis_api.apps.core.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0087_migration'), + ] + + operations = [ + migrations.AlterField( + model_name='flexibleattribute', + name='label', + field=models.JSONField(default=dict, validators=[hct_mis_api.apps.core.models.label_contains_english_en_validator]), + ), + ] diff --git a/src/hct_mis_api/apps/core/models.py b/src/hct_mis_api/apps/core/models.py index 01a83a7e9c..dca60f1810 100644 --- a/src/hct_mis_api/apps/core/models.py +++ b/src/hct_mis_api/apps/core/models.py @@ -191,6 +191,11 @@ def get_sys_option(self, key: str, default: None = None) -> Any: return default +def label_contains_english_en_validator(data: dict) -> None: + if "English(EN)" not in data: + raise ValidationError('The "English(EN)" key is required in the label.') + + class FlexibleAttribute(SoftDeletableModel, NaturalKeyModel, TimeStampedUUIDModel): STRING = "STRING" IMAGE = "IMAGE" @@ -237,7 +242,7 @@ class FlexibleAttribute(SoftDeletableModel, NaturalKeyModel, TimeStampedUUIDMode null=True, related_name="flex_field", ) - label = JSONField(default=dict) + label = JSONField(default=dict, validators=[label_contains_english_en_validator]) hint = JSONField(default=dict) group = models.ForeignKey( "core.FlexibleAttributeGroup", on_delete=models.CASCADE, related_name="flex_attributes", null=True, blank=True @@ -264,6 +269,8 @@ def clean(self) -> None: ): raise ValidationError(f'Flex field with name "{self.name}" already exists inside a program.') + label_contains_english_en_validator(self.label) + def save(self, *args: Any, **kwargs: Any) -> None: self.clean() super().save(*args, **kwargs) diff --git a/tests/unit/apps/periodic_data_update/test_models.py b/tests/unit/apps/periodic_data_update/test_models.py index 72df36fb94..186fbda9f3 100644 --- a/tests/unit/apps/periodic_data_update/test_models.py +++ b/tests/unit/apps/periodic_data_update/test_models.py @@ -31,6 +31,7 @@ def setUp(self) -> None: name="flex_field_1", type=FlexibleAttribute.STRING, associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) self.pdu_field = FlexibleAttributeForPDUFactory( @@ -63,6 +64,7 @@ def test_unique_name_rules_for_flex_fields(self) -> None: name=self.flex_field.name, type=FlexibleAttribute.STRING, associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) self.assertIn( 'duplicate key value violates unique constraint "unique_name_without_program"', @@ -75,6 +77,7 @@ def test_unique_name_rules_for_flex_fields(self) -> None: name=self.pdu_field.name, type=FlexibleAttribute.STRING, associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) self.assertIn( f'Flex field with name "{self.pdu_field.name}" already exists inside a program.', @@ -90,3 +93,21 @@ def test_unique_name_rules_for_flex_fields(self) -> None: f'Flex field with name "{self.flex_field.name}" already exists without a program.', str(ve_context.exception), ) + + def test_flexible_attribute_label_without_english_en_key(self) -> None: + with self.assertRaisesMessage(ValidationError, 'The "English(EN)" key is required in the label.'): + FlexibleAttribute.objects.create( + name="flex_field_2", + type=FlexibleAttribute.STRING, + associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"other value": "value"}, + ) + flexible_attribute = FlexibleAttribute.objects.create( + name="flex_field_2", + type=FlexibleAttribute.STRING, + associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, + ) + with self.assertRaisesMessage(ValidationError, 'The "English(EN)" key is required in the label.'): + flexible_attribute.label = {"wrong": "value"} + flexible_attribute.save() From c4def4c2c4d3600eac3f35680f1f900617f0bdd9 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Tue, 15 Oct 2024 22:36:11 +0200 Subject: [PATCH 08/15] fix tests --- tests/unit/apps/core/test_schema.py | 2 +- tests/unit/apps/program/test_copy_program.py | 1 + tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py | 2 ++ .../apps/targeting/test_create_target_population_mutation.py | 1 + tests/unit/apps/targeting/test_individual_block_filters.py | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/apps/core/test_schema.py b/tests/unit/apps/core/test_schema.py index 8e0ff41188..4d4f31cca3 100644 --- a/tests/unit/apps/core/test_schema.py +++ b/tests/unit/apps/core/test_schema.py @@ -132,7 +132,7 @@ def setUpTestData(cls) -> None: # Create a non-PDU field FlexibleAttribute.objects.create( type=FlexibleAttribute.STRING, - label="Not PDU Field", + label={"English(EN)": "value", "Not PDU Field": ""}, associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, ) diff --git a/tests/unit/apps/program/test_copy_program.py b/tests/unit/apps/program/test_copy_program.py index 1055b9e3ed..857eb27fa7 100644 --- a/tests/unit/apps/program/test_copy_program.py +++ b/tests/unit/apps/program/test_copy_program.py @@ -144,6 +144,7 @@ def setUpTestData(cls) -> None: name="flex_field_1", type=FlexibleAttribute.STRING, associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) cls.individual.flex_fields = { pdu_field.name: { diff --git a/tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py b/tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py index 937de2ab72..55541e5b23 100644 --- a/tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py +++ b/tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py @@ -91,11 +91,13 @@ def setUpTestData(cls) -> None: type=FlexibleAttribute.INTEGER, name="muac_i_f", associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) FlexibleAttribute.objects.create( type=FlexibleAttribute.DECIMAL, name="jan_decimal_i_f", associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) content = Path( f"{settings.TESTS_ROOT}/apps/registration_datahub/test_file/new_reg_data_import.xlsx" diff --git a/tests/unit/apps/targeting/test_create_target_population_mutation.py b/tests/unit/apps/targeting/test_create_target_population_mutation.py index bb5a020a53..3128d59bfa 100644 --- a/tests/unit/apps/targeting/test_create_target_population_mutation.py +++ b/tests/unit/apps/targeting/test_create_target_population_mutation.py @@ -86,6 +86,7 @@ def setUpTestData(cls) -> None: name="flex_field_1", type=FlexibleAttribute.STRING, associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) pdu_data = PeriodicFieldDataFactory( subtype=PeriodicFieldData.DECIMAL, diff --git a/tests/unit/apps/targeting/test_individual_block_filters.py b/tests/unit/apps/targeting/test_individual_block_filters.py index 9aece156b3..c0083412d9 100644 --- a/tests/unit/apps/targeting/test_individual_block_filters.py +++ b/tests/unit/apps/targeting/test_individual_block_filters.py @@ -180,6 +180,7 @@ def test_filter_on_flex_field(self) -> None: name="flex_field_1", type=FlexibleAttribute.STRING, associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) query = Household.objects.all() flex_field_filter = TargetingIndividualBlockRuleFilter( From 4dc12acdd706c3cf9c7b036d61c193dabd371a19 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Wed, 16 Oct 2024 00:41:51 +0200 Subject: [PATCH 09/15] fix tests --- src/hct_mis_api/apps/core/fixtures.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hct_mis_api/apps/core/fixtures.py b/src/hct_mis_api/apps/core/fixtures.py index c9fe19ad5a..091353a82f 100644 --- a/src/hct_mis_api/apps/core/fixtures.py +++ b/src/hct_mis_api/apps/core/fixtures.py @@ -1,4 +1,3 @@ -import json import random from typing import Any, List @@ -147,7 +146,7 @@ def program(self) -> Any: @classmethod def _create(cls, target_class: Any, *args: Any, **kwargs: Any) -> FlexibleAttribute: label = kwargs.pop("label", None) - kwargs["label"] = json.dumps({"English(EN)": label}) + kwargs["label"] = {"English(EN)": label} obj = super()._create(target_class, *args, **kwargs) return obj From 7d0a765f925eba6f8ca41b2af88095aaa96343c9 Mon Sep 17 00:00:00 2001 From: marekbiczysko Date: Wed, 16 Oct 2024 00:48:54 +0200 Subject: [PATCH 10/15] 217950_Payment_Module_Getting_Delivery_Mechanism_error_when_rejecting_a_payment_plan_and_assigning_FSP --- .../VolumeByDeliveryMechanismSection.tsx | 4 ++-- .../VolumeByDeliveryMechanismSection.tsx | 4 ++-- .../pages/paymentmodule/EditFollowUpSetUpFspPage.tsx | 2 +- .../paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx | 2 +- .../pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/frontend/src/components/paymentmodule/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx b/src/frontend/src/components/paymentmodule/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx index b5a93c7b5f..45d964847e 100644 --- a/src/frontend/src/components/paymentmodule/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx +++ b/src/frontend/src/components/paymentmodule/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx @@ -69,8 +69,8 @@ export const VolumeByDeliveryMechanismSection: React.FC< color={getDeliveryMechanismColor(vdm.deliveryMechanism.name)} > diff --git a/src/frontend/src/components/paymentmodulepeople/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx b/src/frontend/src/components/paymentmodulepeople/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx index b5a93c7b5f..45d964847e 100644 --- a/src/frontend/src/components/paymentmodulepeople/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx +++ b/src/frontend/src/components/paymentmodulepeople/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx @@ -69,8 +69,8 @@ export const VolumeByDeliveryMechanismSection: React.FC< color={getDeliveryMechanismColor(vdm.deliveryMechanism.name)} > diff --git a/src/frontend/src/containers/pages/paymentmodule/EditFollowUpSetUpFspPage.tsx b/src/frontend/src/containers/pages/paymentmodule/EditFollowUpSetUpFspPage.tsx index 1392fe4259..644f111b80 100644 --- a/src/frontend/src/containers/pages/paymentmodule/EditFollowUpSetUpFspPage.tsx +++ b/src/frontend/src/containers/pages/paymentmodule/EditFollowUpSetUpFspPage.tsx @@ -29,7 +29,7 @@ export function EditFollowUpSetUpFspPage(): React.ReactElement { const mappedInitialDeliveryMechanisms = paymentPlanData.paymentPlan.deliveryMechanisms.map((el) => ({ - deliveryMechanism: el.name, + deliveryMechanism: el.code, fsp: el.fsp?.id || '', chosenConfiguration: el.chosenConfiguration || '', })); diff --git a/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx b/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx index e179c16557..2269a6a4e0 100644 --- a/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx +++ b/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx @@ -29,7 +29,7 @@ export const EditPeopleFollowUpSetUpFspPage = (): React.ReactElement => { const mappedInitialDeliveryMechanisms = paymentPlanData.paymentPlan.deliveryMechanisms.map((el) => ({ - deliveryMechanism: el.name, + deliveryMechanism: el.code, fsp: el.fsp?.id || '', chosenConfiguration: el.chosenConfiguration || '', })); diff --git a/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx b/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx index 59e380fcf2..ae9fcbe338 100644 --- a/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx +++ b/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx @@ -29,7 +29,7 @@ export const EditPeopleSetUpFspPage = (): React.ReactElement => { const mappedInitialDeliveryMechanisms = paymentPlanData.paymentPlan.deliveryMechanisms.map((el) => ({ - deliveryMechanism: el.name, + deliveryMechanism: el.code, fsp: el.fsp?.id || '', chosenConfiguration: el.chosenConfiguration || '', })); From 2d82056e5d370cd5b79e7144d9d20687591f7f97 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Wed, 16 Oct 2024 01:22:38 +0200 Subject: [PATCH 11/15] fix ImportExportPaymentPlanPaymentListTest --- .../payment/test_import_export_payment_plan_payment_list.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/apps/payment/test_import_export_payment_plan_payment_list.py b/tests/unit/apps/payment/test_import_export_payment_plan_payment_list.py index a4d739ae8d..79beaa3754 100644 --- a/tests/unit/apps/payment/test_import_export_payment_plan_payment_list.py +++ b/tests/unit/apps/payment/test_import_export_payment_plan_payment_list.py @@ -406,12 +406,14 @@ def test_payment_row_flex_fields(self) -> None: type=FlexibleAttribute.DECIMAL, name="flex_decimal_i_f", associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) decimal_flexible_attribute.save() date_flexible_attribute = FlexibleAttribute( type=FlexibleAttribute.DECIMAL, name="flex_date_i_f", associated_with=FlexibleAttribute.ASSOCIATED_WITH_HOUSEHOLD, + label={"English(EN)": "value"}, ) date_flexible_attribute.save() flex_fields = [ @@ -561,12 +563,14 @@ def test_flex_fields_admin_visibility(self) -> None: type=FlexibleAttribute.DECIMAL, name="flex_decimal_i_f", associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) decimal_flexible_attribute.save() date_flexible_attribute = FlexibleAttribute( type=FlexibleAttribute.DECIMAL, name="flex_date_i_f", associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) date_flexible_attribute.save() self.client.login(username="admin", password="password") @@ -590,6 +594,7 @@ def test_payment_row_get_flex_field_if_no_snapshot_data(self) -> None: type=FlexibleAttribute.DECIMAL, name="flex_decimal_i_f", associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL, + label={"English(EN)": "value"}, ) flex_field.save() fsp_xlsx_template = FinancialServiceProviderXlsxTemplateFactory(flex_fields=[flex_field.name]) From f1ce6f26eea0336320ea4cd2013fb6a9bc152cbe Mon Sep 17 00:00:00 2001 From: gabra4 <131235609+gabra4@users.noreply.github.com> Date: Wed, 16 Oct 2024 09:53:14 +0300 Subject: [PATCH 12/15] Update README.md --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c70e1fce29..4ea1602984 100644 --- a/README.md +++ b/README.md @@ -39,15 +39,17 @@ Following the Principles for Digital Development, the “reuse and improve” an ### Why -* Put cash recipients at the center with secure personal data processing -* Unified consolidated reporting for beneficiary data -* Increase accountability beneficiaries and donors -* Financial Inclusion of Beneficiaries -* To Enhance Traceability -* Simplify current process in EMOPS -* Ensuring due diligence in-cash transfer -* Grow the use of effective cash program for children in a risk informed manner -* Eliminate redundancies and dupes in how we do things +|#| HOPE Functions | Added Value | +|1|Registration of payees data | Quality Assurance for data collection, verification of eligibility | +|2|Deduplication of personal records|Mitigate the risk of duplicate records| +|3|Target payments|Ensure inclusion of relevant payees in payment plan| +|4|Entitlment Calculations|Ensure cash benefit entitlement rules are uphold| +|5|Payment Management|Approval, Authorization and Financial Release tracking| +|6|Reconciliation of Individual payment|Support liquidation or advance or reimbursement to Financial Service Provider| +|7|Payment Verification|Mitigate the risk of inaccurate FSP reports and fraud| +|8|Grievances Redressals and Payees Communications|Ensure payment quality and direct communication with payees| +|9|Roles base access|Uphold segragation of duties and data protection principles| +|10|Reporting|Access process and output indicators| ## Legal Humanitarian cash Operations and Programme Ecosystem From 38488d811b2e64d7da017215f2b3b0797584ec57 Mon Sep 17 00:00:00 2001 From: gabra4 <131235609+gabra4@users.noreply.github.com> Date: Wed, 16 Oct 2024 09:56:11 +0300 Subject: [PATCH 13/15] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4ea1602984..dc0ba2a73f 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Following the Principles for Digital Development, the “reuse and improve” an ### Why |#| HOPE Functions | Added Value | +|---|--------------------------|----------------------------------------------| |1|Registration of payees data | Quality Assurance for data collection, verification of eligibility | |2|Deduplication of personal records|Mitigate the risk of duplicate records| |3|Target payments|Ensure inclusion of relevant payees in payment plan| From d5a4955578f227f431a8761b8470ebcc4fb69401 Mon Sep 17 00:00:00 2001 From: Pavlo Mokiichuk Date: Wed, 16 Oct 2024 09:51:47 +0200 Subject: [PATCH 14/15] Script Clean/Remove pre GPF data (#4326) * add remove_migrated_data_is_original * upd test * add more info and hard delete * review --- src/hct_mis_api/apps/household/fixtures.py | 18 ++ .../remove_migrated_data_is_original.py | 27 +++ .../test_remove_migrated_data_is_original.py | 173 ++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py create mode 100644 tests/unit/one_time_scripts/test_remove_migrated_data_is_original.py diff --git a/src/hct_mis_api/apps/household/fixtures.py b/src/hct_mis_api/apps/household/fixtures.py index ceeaf16fb1..84751a28f8 100644 --- a/src/hct_mis_api/apps/household/fixtures.py +++ b/src/hct_mis_api/apps/household/fixtures.py @@ -35,6 +35,8 @@ PendingDocument, PendingHousehold, PendingIndividual, + PendingIndividualIdentity, + PendingIndividualRoleInHousehold, ) from hct_mis_api.apps.program.fixtures import ProgramFactory from hct_mis_api.apps.registration_data.fixtures import RegistrationDataImportFactory @@ -166,6 +168,15 @@ class Meta: number = factory.Faker("pystr", min_chars=None, max_chars=20) +class PendingIndividualIdentityFactory(DjangoModelFactory): + rdi_merge_status = MergeStatusModel.PENDING + + class Meta: + model = PendingIndividualIdentity + + number = factory.Faker("pystr", min_chars=None, max_chars=20) + + class IndividualRoleInHouseholdFactory(DjangoModelFactory): rdi_merge_status = MergeStatusModel.MERGED @@ -173,6 +184,13 @@ class Meta: model = IndividualRoleInHousehold +class PendingIndividualRoleInHouseholdFactory(DjangoModelFactory): + rdi_merge_status = MergeStatusModel.PENDING + + class Meta: + model = PendingIndividualRoleInHousehold + + class IndividualCollectionFactory(DjangoModelFactory): class Meta: model = IndividualCollection diff --git a/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py b/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py new file mode 100644 index 0000000000..0c222b4309 --- /dev/null +++ b/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py @@ -0,0 +1,27 @@ +from django.apps import apps + + +def remove_migrated_data_is_original() -> None: + all_models = apps.get_models() + + for model in all_models: + if hasattr(model, "is_original"): + if model.__name__ == "GrievanceTicket": + # 'GrievanceTicket' has no attribute 'all_objects' + queryset_all = model.default_for_migrations_fix.all() + queryset_is_original = queryset_all.filter(is_original=True) + elif model.__name__ in ["HouseholdSelection", "EntitlementCard", "Feedback", "Message"]: + queryset_all = model.original_and_repr_objects.all() + queryset_is_original = queryset_all.filter(is_original=True) + else: + queryset_all = model.all_objects.all() + queryset_is_original = queryset_all.filter(is_original=True) + + print( + f"*** {model.__name__} All objects: {queryset_all.count()}. " + f"Removing objects with 'is_original=True': {queryset_is_original.count()}" + ) + + count, _ = queryset_is_original.delete() + + print(f"Deleted {model.__name__} and related objects {count}.\n") diff --git a/tests/unit/one_time_scripts/test_remove_migrated_data_is_original.py b/tests/unit/one_time_scripts/test_remove_migrated_data_is_original.py new file mode 100644 index 0000000000..35a057688c --- /dev/null +++ b/tests/unit/one_time_scripts/test_remove_migrated_data_is_original.py @@ -0,0 +1,173 @@ +from django.test import TestCase + +from hct_mis_api.apps.account.fixtures import BusinessAreaFactory +from hct_mis_api.apps.accountability.fixtures import ( + CommunicationMessageFactory, + FeedbackFactory, +) +from hct_mis_api.apps.accountability.models import Feedback, Message +from hct_mis_api.apps.grievance.fixtures import ( + GrievanceTicketFactory, + TicketNeedsAdjudicationDetailsFactory, +) +from hct_mis_api.apps.grievance.models import ( + GrievanceTicket, + TicketNeedsAdjudicationDetails, +) +from hct_mis_api.apps.household.fixtures import ( + BankAccountInfoFactory, + DocumentFactory, + EntitlementCardFactory, + HouseholdFactory, + IndividualFactory, + IndividualIdentityFactory, + IndividualRoleInHouseholdFactory, + PendingBankAccountInfoFactory, + PendingDocumentFactory, + PendingHouseholdFactory, + PendingIndividualFactory, + PendingIndividualIdentityFactory, + PendingIndividualRoleInHouseholdFactory, +) +from hct_mis_api.apps.household.models import ( + ROLE_PRIMARY, + BankAccountInfo, + Document, + EntitlementCard, + Household, + Individual, + IndividualIdentity, + IndividualRoleInHousehold, + PendingBankAccountInfo, + PendingDocument, + PendingHousehold, + PendingIndividual, + PendingIndividualIdentity, + PendingIndividualRoleInHousehold, +) +from hct_mis_api.apps.targeting.fixtures import HouseholdSelectionFactory +from hct_mis_api.apps.targeting.models import HouseholdSelection +from hct_mis_api.one_time_scripts.remove_migrated_data_is_original import ( + remove_migrated_data_is_original, +) + + +class BaseMigrateDataTestCase(TestCase): + def setUp(self) -> None: + BusinessAreaFactory(name="Afghanistan") + + ind = IndividualFactory(household=None) + ind2 = IndividualFactory(is_original=True, household=None) + pending_ind = PendingIndividualFactory(household=None) + pending_ind2 = PendingIndividualFactory(is_original=True, household=None) + + hh = HouseholdFactory() + hh2 = HouseholdFactory(is_original=True) + pending_hh = PendingHouseholdFactory() + pending_hh2 = PendingHouseholdFactory(is_original=True) + + HouseholdSelectionFactory(household=hh) + HouseholdSelectionFactory(is_original=True, household=hh2) + + DocumentFactory(individual=ind) + DocumentFactory(is_original=True, individual=ind2) + PendingDocumentFactory(individual=pending_ind) + PendingDocumentFactory(is_original=True, individual=pending_ind2) + + IndividualIdentityFactory(individual=ind) + IndividualIdentityFactory(is_original=True, individual=ind) + PendingIndividualIdentityFactory(individual=pending_ind) + PendingIndividualIdentityFactory(is_original=True, individual=pending_ind) + + IndividualRoleInHouseholdFactory(household=hh, individual=ind, role=ROLE_PRIMARY) + IndividualRoleInHouseholdFactory(is_original=True, household=hh2, individual=ind2, role=ROLE_PRIMARY) + PendingIndividualRoleInHouseholdFactory(household=pending_hh, individual=pending_ind, role=ROLE_PRIMARY) + PendingIndividualRoleInHouseholdFactory( + is_original=True, household=pending_hh2, individual=pending_ind2, role=ROLE_PRIMARY + ) + + EntitlementCardFactory(household=hh) + EntitlementCardFactory(is_original=True, household=hh2) + + BankAccountInfoFactory(individual=ind) + BankAccountInfoFactory(is_original=True, individual=ind2) + PendingBankAccountInfoFactory(individual=pending_ind) + PendingBankAccountInfoFactory(is_original=True, individual=pending_ind2) + + CommunicationMessageFactory() + CommunicationMessageFactory(is_original=True) + FeedbackFactory() + FeedbackFactory(is_original=True) + + gr1 = GrievanceTicketFactory() + gr2 = GrievanceTicketFactory(is_original=True) + + TicketNeedsAdjudicationDetailsFactory( + golden_records_individual=ind, + ticket=gr1, + ) + TicketNeedsAdjudicationDetailsFactory( + golden_records_individual=ind2, + ticket=gr2, + ) + + def test_run_remove_migrated_data_is_original(self) -> None: + # check count before + self.assertEqual(Individual.all_objects.count(), 4) + self.assertEqual(PendingIndividual.all_objects.count(), 4) + self.assertEqual(Household.all_objects.count(), 4) + self.assertEqual(PendingHousehold.all_objects.count(), 4) + self.assertEqual(HouseholdSelection.original_and_repr_objects.count(), 2) + self.assertEqual(Document.all_objects.count(), 4) + self.assertEqual(PendingDocument.all_objects.count(), 4) + self.assertEqual(IndividualIdentity.all_objects.count(), 4) + self.assertEqual(PendingIndividualIdentity.all_objects.count(), 4) + self.assertEqual(IndividualRoleInHousehold.all_objects.count(), 4) + self.assertEqual(PendingIndividualRoleInHousehold.all_objects.count(), 4) + self.assertEqual(EntitlementCard.original_and_repr_objects.count(), 2) + self.assertEqual(BankAccountInfo.all_objects.count(), 4) + self.assertEqual(PendingBankAccountInfo.all_objects.count(), 4) + self.assertEqual(Message.original_and_repr_objects.count(), 2) + self.assertEqual(Feedback.original_and_repr_objects.count(), 2) + self.assertEqual(GrievanceTicket.default_for_migrations_fix.count(), 2) + self.assertEqual(TicketNeedsAdjudicationDetails.objects.count(), 2) + + remove_migrated_data_is_original() + + # check count after + self.assertEqual(Individual.all_objects.count(), 2) + self.assertEqual(PendingIndividual.all_objects.count(), 2) + self.assertEqual(Household.all_objects.count(), 2) + self.assertEqual(PendingHousehold.all_objects.count(), 2) + self.assertEqual(HouseholdSelection.original_and_repr_objects.count(), 1) + self.assertEqual(Document.all_objects.count(), 2) + self.assertEqual(PendingDocument.all_objects.count(), 2) + self.assertEqual(IndividualIdentity.all_objects.count(), 2) + self.assertEqual(PendingIndividualIdentity.all_objects.count(), 2) + self.assertEqual(IndividualRoleInHousehold.all_objects.count(), 2) + self.assertEqual(PendingIndividualRoleInHousehold.all_objects.count(), 2) + self.assertEqual(EntitlementCard.original_and_repr_objects.count(), 1) + self.assertEqual(BankAccountInfo.all_objects.count(), 2) + self.assertEqual(PendingBankAccountInfo.all_objects.count(), 2) + self.assertEqual(Message.original_and_repr_objects.count(), 1) + self.assertEqual(Feedback.original_and_repr_objects.count(), 1) + self.assertEqual(GrievanceTicket.default_for_migrations_fix.count(), 1) + self.assertEqual(TicketNeedsAdjudicationDetails.objects.count(), 1) + + self.assertEqual(Individual.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(PendingIndividual.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(Household.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(PendingHousehold.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(HouseholdSelection.original_and_repr_objects.filter(is_original=True).count(), 0) + self.assertEqual(Document.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(PendingDocument.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(IndividualIdentity.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(PendingIndividualIdentity.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(IndividualRoleInHousehold.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(PendingIndividualRoleInHousehold.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(EntitlementCard.original_and_repr_objects.filter(is_original=True).count(), 0) + self.assertEqual(BankAccountInfo.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(PendingBankAccountInfo.all_objects.filter(is_original=True).count(), 0) + self.assertEqual(Message.original_and_repr_objects.filter(is_original=True).count(), 0) + self.assertEqual(Feedback.original_and_repr_objects.filter(is_original=True).count(), 0) + self.assertEqual(GrievanceTicket.default_for_migrations_fix.filter(is_original=True).count(), 0) From b741d4493ed586b4e57025b80af360327d23c519 Mon Sep 17 00:00:00 2001 From: Pavlo Mokiichuk Date: Thu, 17 Oct 2024 08:59:27 +0200 Subject: [PATCH 15/15] Clean/Remove pre GPF data: Script Optimization (#4337) * script optimization * upd only() --- .../remove_migrated_data_is_original.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py b/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py index 0c222b4309..d42edec2c9 100644 --- a/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py +++ b/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py @@ -1,20 +1,21 @@ from django.apps import apps +from django.utils import timezone -def remove_migrated_data_is_original() -> None: +def remove_migrated_data_is_original(batch_size: int = 1000) -> None: + start_time = timezone.now() all_models = apps.get_models() for model in all_models: if hasattr(model, "is_original"): if model.__name__ == "GrievanceTicket": - # 'GrievanceTicket' has no attribute 'all_objects' - queryset_all = model.default_for_migrations_fix.all() + queryset_all = model.default_for_migrations_fix.all().only("is_original", "id") queryset_is_original = queryset_all.filter(is_original=True) elif model.__name__ in ["HouseholdSelection", "EntitlementCard", "Feedback", "Message"]: - queryset_all = model.original_and_repr_objects.all() + queryset_all = model.original_and_repr_objects.all().only("is_original", "id") queryset_is_original = queryset_all.filter(is_original=True) else: - queryset_all = model.all_objects.all() + queryset_all = model.all_objects.all().only("is_original", "id") queryset_is_original = queryset_all.filter(is_original=True) print( @@ -22,6 +23,14 @@ def remove_migrated_data_is_original() -> None: f"Removing objects with 'is_original=True': {queryset_is_original.count()}" ) - count, _ = queryset_is_original.delete() + deleted_count = 0 + ids_to_delete = list(queryset_is_original.values_list("id", flat=True).iterator(chunk_size=batch_size)) - print(f"Deleted {model.__name__} and related objects {count}.\n") + for i in range(0, len(ids_to_delete), batch_size): + batch_pks = ids_to_delete[i : i + batch_size] + count, _ = queryset_all.filter(pk__in=batch_pks).delete() + deleted_count += count + + print(f"Deleted {model.__name__} and related objects {deleted_count}.\n") + + print(f"Completed in {timezone.now() - start_time}\n", "*" * 55)