From fa1935e470a1d8752a4e86ce916b58be4634b846 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Thu, 10 Oct 2024 14:43:09 +0200
Subject: [PATCH 01/15] rest api updates + tests
---
src/hct_mis_api/api/endpoints/core/filters.py | 15 ++
src/hct_mis_api/api/endpoints/core/views.py | 22 +++
.../api/endpoints/lookups/__init__.py | 3 +-
src/hct_mis_api/api/endpoints/lookups/base.py | 15 +-
.../api/endpoints/program/filters.py | 28 ++++
.../api/endpoints/program/views.py | 21 +++
src/hct_mis_api/api/endpoints/serializers.py | 10 ++
src/hct_mis_api/api/urls.py | 3 +-
src/hct_mis_api/config/fragments/drf.py | 3 +-
tests/unit/api/test_lookups.py | 51 +++++++
tests/unit/api/test_program.py | 133 ++++++++++--------
11 files changed, 241 insertions(+), 63 deletions(-)
create mode 100644 src/hct_mis_api/api/endpoints/core/filters.py
create mode 100644 src/hct_mis_api/api/endpoints/program/filters.py
create mode 100644 tests/unit/api/test_lookups.py
diff --git a/src/hct_mis_api/api/endpoints/core/filters.py b/src/hct_mis_api/api/endpoints/core/filters.py
new file mode 100644
index 0000000000..8edf2657bd
--- /dev/null
+++ b/src/hct_mis_api/api/endpoints/core/filters.py
@@ -0,0 +1,15 @@
+from django_filters import DateFromToRangeFilter
+from django_filters.rest_framework import FilterSet
+
+from hct_mis_api.apps.core.models import BusinessArea
+
+
+class BusinessAreaFilter(FilterSet):
+ updated_at = DateFromToRangeFilter()
+
+ class Meta:
+ model = BusinessArea
+ fields = (
+ "active",
+ "updated_at",
+ )
diff --git a/src/hct_mis_api/api/endpoints/core/views.py b/src/hct_mis_api/api/endpoints/core/views.py
index 5a3abd7b79..ace581f6c9 100644
--- a/src/hct_mis_api/api/endpoints/core/views.py
+++ b/src/hct_mis_api/api/endpoints/core/views.py
@@ -1,10 +1,32 @@
+from typing import TYPE_CHECKING, Any
+
from rest_framework.generics import ListAPIView
+from rest_framework.response import Response
from hct_mis_api.api.endpoints.base import HOPEAPIView
+from hct_mis_api.api.endpoints.core.filters import BusinessAreaFilter
from hct_mis_api.api.endpoints.core.serializers import BusinessAreaSerializer
from hct_mis_api.apps.core.models import BusinessArea
+if TYPE_CHECKING:
+ from rest_framework.request import Request
+
class BusinessAreaListView(HOPEAPIView, ListAPIView):
serializer_class = BusinessAreaSerializer
queryset = BusinessArea.objects.all()
+
+ def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response:
+ queryset = self.queryset
+
+ filterset = BusinessAreaFilter(request.GET, queryset=queryset)
+ if filterset.is_valid():
+ queryset = filterset.qs
+
+ page = self.paginate_queryset(queryset)
+ if page is not None:
+ serializer = self.get_serializer(page, many=True)
+ return self.get_paginated_response(serializer.data)
+
+ serializer = self.get_serializer(queryset, many=True)
+ return Response(serializer.data)
diff --git a/src/hct_mis_api/api/endpoints/lookups/__init__.py b/src/hct_mis_api/api/endpoints/lookups/__init__.py
index 5782862d81..f6a24e5bb6 100644
--- a/src/hct_mis_api/api/endpoints/lookups/__init__.py
+++ b/src/hct_mis_api/api/endpoints/lookups/__init__.py
@@ -1,10 +1,11 @@
from hct_mis_api.api.endpoints.lookups.area import AreaList, AreaTypeList # noqa: F401
from hct_mis_api.api.endpoints.lookups.base import ( # noqa: F401
- Country,
+ CountryAPIView,
DataCollectingPolicy,
DocumentType,
MaritalStatus,
ObservedDisability,
+ ProgramStatuses,
Relationship,
ResidenceStatus,
Roles,
diff --git a/src/hct_mis_api/api/endpoints/lookups/base.py b/src/hct_mis_api/api/endpoints/lookups/base.py
index 737b463a67..18d4ae51e6 100644
--- a/src/hct_mis_api/api/endpoints/lookups/base.py
+++ b/src/hct_mis_api/api/endpoints/lookups/base.py
@@ -1,9 +1,10 @@
from typing import TYPE_CHECKING, Any, Optional
-from django_countries import Countries
from rest_framework.response import Response
from hct_mis_api.api.endpoints.base import HOPEAPIView
+from hct_mis_api.api.endpoints.serializers import CountrySerializer
+from hct_mis_api.apps.geo.models import Country
from hct_mis_api.apps.household.models import (
COLLECT_TYPES,
IDENTIFICATION_TYPE_CHOICE,
@@ -25,9 +26,12 @@ def get(self, request: "Request", format: Optional[Any] = None) -> Response:
return Response(dict(IDENTIFICATION_TYPE_CHOICE))
-class Country(HOPEAPIView):
+class CountryAPIView(HOPEAPIView):
+ serializer_class = CountrySerializer
+
def get(self, request: "Request", format: Optional[Any] = None) -> Response:
- return Response(dict(Countries()))
+ serializer = self.serializer_class(Country.objects.all(), many=True)
+ return Response(serializer.data)
class ResidenceStatus(HOPEAPIView):
@@ -78,3 +82,8 @@ def get(self, request: "Request", format: Optional[Any] = None) -> Response:
class ProgramScope(HOPEAPIView):
def get(self, request: "Request", format: Optional[Any] = None) -> Response:
return Response(dict(Program.SCOPE_CHOICE))
+
+
+class ProgramStatuses(HOPEAPIView):
+ def get(self, request: "Request", format: Optional[Any] = None) -> Response:
+ return Response(dict(Program.STATUS_CHOICE))
diff --git a/src/hct_mis_api/api/endpoints/program/filters.py b/src/hct_mis_api/api/endpoints/program/filters.py
new file mode 100644
index 0000000000..e1ca22b5c7
--- /dev/null
+++ b/src/hct_mis_api/api/endpoints/program/filters.py
@@ -0,0 +1,28 @@
+from django.db.models.query import QuerySet
+
+from django_filters import BooleanFilter, CharFilter, DateFromToRangeFilter
+from django_filters.rest_framework import FilterSet
+
+from hct_mis_api.apps.program.models import Program
+
+
+class ProgramFilter(FilterSet):
+ business_area = CharFilter(field_name="business_area__slug")
+ active = BooleanFilter(method="is_active_filter")
+ updated_at = DateFromToRangeFilter()
+
+ class Meta:
+ model = Program
+ fields = (
+ "business_area",
+ "active",
+ "updated_at",
+ "status",
+ )
+
+ def is_active_filter(self, queryset: "QuerySet[Program]", name: str, value: bool) -> "QuerySet[Program]":
+ if value is True:
+ return queryset.filter(status=Program.ACTIVE)
+ elif value is False:
+ return queryset.exclude(status=Program.ACTIVE)
+ return queryset
diff --git a/src/hct_mis_api/api/endpoints/program/views.py b/src/hct_mis_api/api/endpoints/program/views.py
index 78e77da2f3..3402938df8 100644
--- a/src/hct_mis_api/api/endpoints/program/views.py
+++ b/src/hct_mis_api/api/endpoints/program/views.py
@@ -1,10 +1,31 @@
+from typing import TYPE_CHECKING, Any
+
from rest_framework.generics import ListAPIView
+from rest_framework.response import Response
from hct_mis_api.api.endpoints.base import HOPEAPIView
+from hct_mis_api.api.endpoints.program.filters import ProgramFilter
from hct_mis_api.api.endpoints.program.serializers import ProgramGlobalSerializer
from hct_mis_api.apps.program.models import Program
+if TYPE_CHECKING:
+ from rest_framework.request import Request
+
class ProgramGlobalListView(HOPEAPIView, ListAPIView):
serializer_class = ProgramGlobalSerializer
queryset = Program.objects.all()
+
+ def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response:
+ queryset = self.queryset
+ filterset = ProgramFilter(request.GET, queryset=queryset)
+ if filterset.is_valid():
+ queryset = filterset.qs
+
+ page = self.paginate_queryset(queryset)
+ if page is not None:
+ serializer = self.get_serializer(page, many=True)
+ return self.get_paginated_response(serializer.data)
+
+ serializer = self.serializer_class(queryset, many=True)
+ return Response(serializer.data)
diff --git a/src/hct_mis_api/api/endpoints/serializers.py b/src/hct_mis_api/api/endpoints/serializers.py
index 0485a2e91e..163262a532 100644
--- a/src/hct_mis_api/api/endpoints/serializers.py
+++ b/src/hct_mis_api/api/endpoints/serializers.py
@@ -1,7 +1,17 @@
from django.db import models
from django.utils.translation import gettext_lazy as _
+from rest_framework import serializers
+
+from hct_mis_api.apps.geo.models import Country
+
class RejectPolicy(models.TextChoices):
STRICT = "STRICT", _("STRICT")
LAX = "LAX", _("Lax")
+
+
+class CountrySerializer(serializers.ModelSerializer):
+ class Meta:
+ model = Country
+ fields = ("name", "short_name", "iso_code2", "iso_code3", "iso_num")
diff --git a/src/hct_mis_api/api/urls.py b/src/hct_mis_api/api/urls.py
index 0871c71f28..86bdf21af0 100644
--- a/src/hct_mis_api/api/urls.py
+++ b/src/hct_mis_api/api/urls.py
@@ -22,7 +22,7 @@
path("areatypes/", endpoints.lookups.AreaTypeList().as_view(), name="areatype-list"),
path("constance/", ConstanceSettingsAPIView().as_view(), name="constance-list"),
path("lookups/document/", endpoints.lookups.DocumentType().as_view(), name="document-list"),
- path("lookups/country/", endpoints.lookups.Country().as_view(), name="country-list"),
+ path("lookups/country/", endpoints.lookups.CountryAPIView().as_view(), name="country-list"),
path("lookups/residencestatus/", endpoints.lookups.ResidenceStatus().as_view(), name="residencestatus-list"),
path("lookups/maritalstatus/", endpoints.lookups.MaritalStatus().as_view(), name="maritalstatus-list"),
path(
@@ -36,6 +36,7 @@
),
path("lookups/role/", endpoints.lookups.Roles().as_view(), name="role-list"),
path("lookups/sex/", endpoints.lookups.Sex().as_view(), name="sex-list"),
+ path("lookups/program-statuses/", endpoints.lookups.ProgramStatuses().as_view(), name="program-statuses-list"),
path("business_areas/", endpoints.core.BusinessAreaListView.as_view(), name="business-area-list"),
path("programs/", ProgramGlobalListView.as_view(), name="program-global-list"),
path(
diff --git a/src/hct_mis_api/config/fragments/drf.py b/src/hct_mis_api/config/fragments/drf.py
index 11b5ee8354..543421b577 100644
--- a/src/hct_mis_api/config/fragments/drf.py
+++ b/src/hct_mis_api/config/fragments/drf.py
@@ -1,9 +1,10 @@
REST_FRAMEWORK = {
"DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",
- "PAGE_SIZE": 10,
+ "PAGE_SIZE": 50,
"TEST_REQUEST_DEFAULT_FORMAT": "json",
"DEFAULT_AUTHENTICATION_CLASSES": [
"hct_mis_api.api.utils.CsrfExemptSessionAuthentication",
],
"DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
+ "SECURE_SSL_REDIRECT": True,
}
diff --git a/tests/unit/api/test_lookups.py b/tests/unit/api/test_lookups.py
new file mode 100644
index 0000000000..d6e1bc0270
--- /dev/null
+++ b/tests/unit/api/test_lookups.py
@@ -0,0 +1,51 @@
+from rest_framework import status
+from rest_framework.reverse import reverse
+
+from hct_mis_api.apps.geo.fixtures import CountryFactory
+from hct_mis_api.apps.program.models import Program
+from tests.unit.api.base import HOPEApiTestCase
+
+
+class APIProgramStatuesTests(HOPEApiTestCase):
+ databases = {"default"}
+ user_permissions = []
+
+ def test_get_program_statues(self) -> None:
+ url = reverse("api:program-statuses-list")
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ self.assertEqual(response.json(), dict(Program.STATUS_CHOICE))
+
+
+class APICountriesTests(HOPEApiTestCase):
+ databases = {"default"}
+ user_permissions = []
+
+ def test_get_countries(self) -> None:
+ country_afghanistan = CountryFactory()
+ country_poland = CountryFactory(
+ name="Poland",
+ short_name="Poland",
+ iso_code2="PL",
+ iso_code3="POL",
+ iso_num="0620",
+ )
+ url = reverse("api:country-list")
+ response = self.client.get(url)
+ assert response.status_code == status.HTTP_200_OK
+ assert response.json() == [
+ {
+ "name": country_afghanistan.name,
+ "short_name": country_afghanistan.short_name,
+ "iso_code2": country_afghanistan.iso_code2,
+ "iso_code3": country_afghanistan.iso_code3,
+ "iso_num": country_afghanistan.iso_num,
+ },
+ {
+ "name": country_poland.name,
+ "short_name": country_poland.short_name,
+ "iso_code2": country_poland.iso_code2,
+ "iso_code3": country_poland.iso_code3,
+ "iso_num": country_poland.iso_num,
+ },
+ ]
diff --git a/tests/unit/api/test_program.py b/tests/unit/api/test_program.py
index 1f006f50ba..021660149f 100644
--- a/tests/unit/api/test_program.py
+++ b/tests/unit/api/test_program.py
@@ -6,6 +6,7 @@
from hct_mis_api.api.models import APIToken, Grant
from hct_mis_api.apps.account.fixtures import BusinessAreaFactory
from hct_mis_api.apps.core.fixtures import DataCollectingTypeFactory
+from hct_mis_api.apps.core.models import BusinessArea
from hct_mis_api.apps.program.fixtures import ProgramFactory
from hct_mis_api.apps.program.models import Program
from tests.unit.api.base import HOPEApiTestCase
@@ -156,39 +157,69 @@ class APIGlobalProgramTests(HOPEApiTestCase):
def setUpTestData(cls) -> None:
super().setUpTestData()
cls.list_url = reverse("api:program-global-list")
-
- def test_list_program(self) -> None:
- program1: Program = ProgramFactory(
+ cls.program1: Program = ProgramFactory(
budget=10000,
start_date="2022-01-12",
end_date="2022-09-12",
- business_area=self.business_area,
+ business_area=cls.business_area,
population_goal=200,
status=Program.ACTIVE,
)
- program2: Program = ProgramFactory(
+ cls.program2: Program = ProgramFactory(
budget=200,
start_date="2022-01-10",
end_date="2022-09-10",
- business_area=self.business_area,
+ business_area=cls.business_area,
population_goal=200,
status=Program.DRAFT,
)
# program from another BA - also listed as we do not filter by BA
- business_area2 = BusinessAreaFactory(name="Ukraine")
- program_from_another_ba = ProgramFactory(
+ cls.business_area2 = BusinessAreaFactory(name="Ukraine")
+ cls.program_from_another_ba = ProgramFactory(
budget=200,
start_date="2022-01-10",
end_date="2022-09-10",
- business_area=business_area2,
+ business_area=cls.business_area2,
population_goal=400,
status=Program.ACTIVE,
)
- program1.refresh_from_db()
- program2.refresh_from_db()
- program_from_another_ba.refresh_from_db()
+ cls.program1.refresh_from_db()
+ cls.program2.refresh_from_db()
+ cls.program_from_another_ba.refresh_from_db()
+
+ def expected_response(program: Program, business_area: BusinessArea) -> dict:
+ return {
+ "budget": str(program.budget),
+ "business_area_code": business_area.code,
+ "cash_plus": program.cash_plus,
+ "end_date": program.end_date.strftime("%Y-%m-%d"),
+ "frequency_of_payments": program.frequency_of_payments,
+ "id": str(program.id),
+ "name": program.name,
+ "population_goal": program.population_goal,
+ "programme_code": program.programme_code,
+ "scope": program.scope,
+ "sector": program.sector,
+ "status": program.status,
+ "start_date": program.start_date.strftime("%Y-%m-%d"),
+ }
+
+ cls.program_from_another_ba_expected_response = expected_response(
+ cls.program_from_another_ba,
+ cls.business_area2,
+ )
+
+ cls.program1_expected_response = expected_response(
+ cls.program1,
+ cls.business_area,
+ )
+ cls.program2_expected_response = expected_response(
+ cls.program2,
+ cls.business_area,
+ )
+ def test_list_program(self) -> None:
response = self.client.get(self.list_url)
assert response.status_code == 403
@@ -197,56 +228,44 @@ def test_list_program(self) -> None:
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.json()["results"]), 3)
self.assertIn(
- {
- "budget": str(program1.budget),
- "business_area_code": self.business_area.code,
- "cash_plus": program1.cash_plus,
- "end_date": program1.end_date.strftime("%Y-%m-%d"),
- "frequency_of_payments": program1.frequency_of_payments,
- "id": str(program1.id),
- "name": program1.name,
- "population_goal": program1.population_goal,
- "programme_code": program1.programme_code,
- "scope": program1.scope,
- "sector": program1.sector,
- "status": program1.status,
- "start_date": program1.start_date.strftime("%Y-%m-%d"),
- },
+ self.program1_expected_response,
response.json()["results"],
)
self.assertIn(
- {
- "budget": str(program2.budget),
- "business_area_code": self.business_area.code,
- "cash_plus": program2.cash_plus,
- "end_date": program2.end_date.strftime("%Y-%m-%d"),
- "frequency_of_payments": program2.frequency_of_payments,
- "id": str(program2.id),
- "name": program2.name,
- "population_goal": program2.population_goal,
- "programme_code": program2.programme_code,
- "scope": program2.scope,
- "sector": program2.sector,
- "status": program2.status,
- "start_date": program2.start_date.strftime("%Y-%m-%d"),
- },
+ self.program2_expected_response,
response.json()["results"],
)
self.assertIn(
- {
- "budget": str(program_from_another_ba.budget),
- "business_area_code": business_area2.code,
- "cash_plus": program_from_another_ba.cash_plus,
- "end_date": program_from_another_ba.end_date.strftime("%Y-%m-%d"),
- "frequency_of_payments": program_from_another_ba.frequency_of_payments,
- "id": str(program_from_another_ba.id),
- "name": program_from_another_ba.name,
- "population_goal": program_from_another_ba.population_goal,
- "programme_code": program_from_another_ba.programme_code,
- "scope": program_from_another_ba.scope,
- "sector": program_from_another_ba.sector,
- "status": program_from_another_ba.status,
- "start_date": program_from_another_ba.start_date.strftime("%Y-%m-%d"),
- },
+ self.program_from_another_ba_expected_response,
+ response.json()["results"],
+ )
+
+ def test_list_program_filter_business_area(self) -> None:
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(self.list_url, {"business_area": "afghanistan"})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()["results"]), 2)
+ self.assertNotIn(
+ self.program_from_another_ba_expected_response,
+ response.json()["results"],
+ )
+
+ def test_list_program_filter_active(self) -> None:
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(self.list_url, {"active": "true"})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()["results"]), 2)
+ self.assertNotIn(
+ self.program2_expected_response,
+ response.json()["results"],
+ )
+
+ def test_list_program_filter_status(self) -> None:
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(self.list_url, {"status": Program.DRAFT})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()["results"]), 1)
+ self.assertIn(
+ self.program2_expected_response,
response.json()["results"],
)
From 824df8eedac76365c03a49cc07acd71b9e35a5a0 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Mon, 14 Oct 2024 20:50:19 +0200
Subject: [PATCH 02/15] apply updated_at filter to all endpoints
---
src/hct_mis_api/api/endpoints/core/filters.py | 13 ++----
.../api/endpoints/program/filters.py | 8 ++--
src/hct_mis_api/api/endpoints/rdi/program.py | 18 +++++---
src/hct_mis_api/apps/core/api/__init__.py | 0
src/hct_mis_api/apps/core/api/filters.py | 6 +++
src/hct_mis_api/apps/geo/api/filters.py | 3 +-
.../apps/periodic_data_update/api/views.py | 11 +++--
src/hct_mis_api/apps/program/api/filters.py | 3 +-
.../apps/registration_data/api/filters.py | 3 +-
src/hct_mis_api/apps/targeting/api/filters.py | 3 +-
tests/unit/api/base.py | 13 ++++++
tests/unit/api/test_business_area.py | 17 +------
tests/unit/api/test_lookups.py | 9 ++--
tests/unit/api/test_program.py | 44 +++++++++++++------
14 files changed, 91 insertions(+), 60 deletions(-)
create mode 100644 src/hct_mis_api/apps/core/api/__init__.py
create mode 100644 src/hct_mis_api/apps/core/api/filters.py
diff --git a/src/hct_mis_api/api/endpoints/core/filters.py b/src/hct_mis_api/api/endpoints/core/filters.py
index 8edf2657bd..eeb392e7f8 100644
--- a/src/hct_mis_api/api/endpoints/core/filters.py
+++ b/src/hct_mis_api/api/endpoints/core/filters.py
@@ -1,15 +1,8 @@
-from django_filters import DateFromToRangeFilter
-from django_filters.rest_framework import FilterSet
-
+from hct_mis_api.apps.core.api.filters import UpdatedAtFilter
from hct_mis_api.apps.core.models import BusinessArea
-class BusinessAreaFilter(FilterSet):
- updated_at = DateFromToRangeFilter()
-
+class BusinessAreaFilter(UpdatedAtFilter):
class Meta:
model = BusinessArea
- fields = (
- "active",
- "updated_at",
- )
+ fields = ("active",)
diff --git a/src/hct_mis_api/api/endpoints/program/filters.py b/src/hct_mis_api/api/endpoints/program/filters.py
index e1ca22b5c7..b00e330ca0 100644
--- a/src/hct_mis_api/api/endpoints/program/filters.py
+++ b/src/hct_mis_api/api/endpoints/program/filters.py
@@ -1,22 +1,20 @@
from django.db.models.query import QuerySet
-from django_filters import BooleanFilter, CharFilter, DateFromToRangeFilter
-from django_filters.rest_framework import FilterSet
+from django_filters import BooleanFilter, CharFilter
+from hct_mis_api.apps.core.api.filters import UpdatedAtFilter
from hct_mis_api.apps.program.models import Program
-class ProgramFilter(FilterSet):
+class ProgramFilter(UpdatedAtFilter):
business_area = CharFilter(field_name="business_area__slug")
active = BooleanFilter(method="is_active_filter")
- updated_at = DateFromToRangeFilter()
class Meta:
model = Program
fields = (
"business_area",
"active",
- "updated_at",
"status",
)
diff --git a/src/hct_mis_api/api/endpoints/rdi/program.py b/src/hct_mis_api/api/endpoints/rdi/program.py
index fe783b388e..b64cf61dc3 100644
--- a/src/hct_mis_api/api/endpoints/rdi/program.py
+++ b/src/hct_mis_api/api/endpoints/rdi/program.py
@@ -8,6 +8,7 @@
from hct_mis_api.api.endpoints.base import HOPEAPIBusinessAreaViewSet
from hct_mis_api.api.models import Grant
+from hct_mis_api.apps.core.api.filters import UpdatedAtFilter
from hct_mis_api.apps.program.models import Program
if TYPE_CHECKING:
@@ -33,17 +34,17 @@ class Meta:
class ProgramViewSet(CreateModelMixin, HOPEAPIBusinessAreaViewSet):
- serializer = ProgramSerializer
+ serializer_class = ProgramSerializer
model = Program
permission = Grant.API_READ_ONLY
- def perform_create(self, serializer: "BaseSerializer") -> None:
- serializer.save(business_area=self.selected_business_area)
+ def perform_create(self, serializer_class: "BaseSerializer") -> None:
+ serializer_class.save(business_area=self.selected_business_area)
@extend_schema(request=ProgramSerializer)
def create(self, request: "Request", *args: Any, **kwargs: Any) -> Response:
- self.selected_business_area # TODO does it work? It should be called
- serializer = ProgramSerializer(data=request.data)
+ self.selected_business_area
+ serializer = self.serializer_class(data=request.data)
serializer.is_valid(raise_exception=True)
if Grant.API_PROGRAM_CREATE.name not in request.auth.grants:
raise PermissionDenied()
@@ -53,5 +54,10 @@ def create(self, request: "Request", *args: Any, **kwargs: Any) -> Response:
def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response:
queryset = self.model.objects.filter(business_area=self.selected_business_area)
- serializer = self.serializer(queryset, many=True)
+
+ filterset = UpdatedAtFilter(request.GET, queryset=queryset)
+ if filterset.is_valid():
+ queryset = filterset.qs
+
+ serializer = self.serializer_class(queryset, many=True)
return Response(serializer.data)
diff --git a/src/hct_mis_api/apps/core/api/__init__.py b/src/hct_mis_api/apps/core/api/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/src/hct_mis_api/apps/core/api/filters.py b/src/hct_mis_api/apps/core/api/filters.py
new file mode 100644
index 0000000000..640c16782e
--- /dev/null
+++ b/src/hct_mis_api/apps/core/api/filters.py
@@ -0,0 +1,6 @@
+from django_filters import DateFromToRangeFilter
+from django_filters.rest_framework import FilterSet
+
+
+class UpdatedAtFilter(FilterSet):
+ updated_at = DateFromToRangeFilter()
diff --git a/src/hct_mis_api/apps/geo/api/filters.py b/src/hct_mis_api/apps/geo/api/filters.py
index cbe9f3d022..f1a2495546 100644
--- a/src/hct_mis_api/apps/geo/api/filters.py
+++ b/src/hct_mis_api/apps/geo/api/filters.py
@@ -1,9 +1,10 @@
from django_filters import rest_framework as filters
+from hct_mis_api.apps.core.api.filters import UpdatedAtFilter
from hct_mis_api.apps.geo.models import Area
-class AreaFilter(filters.FilterSet):
+class AreaFilter(UpdatedAtFilter):
level = filters.NumberFilter(
field_name="area_type__area_level",
)
diff --git a/src/hct_mis_api/apps/periodic_data_update/api/views.py b/src/hct_mis_api/apps/periodic_data_update/api/views.py
index 68f8757903..fbfabb7fa6 100644
--- a/src/hct_mis_api/apps/periodic_data_update/api/views.py
+++ b/src/hct_mis_api/apps/periodic_data_update/api/views.py
@@ -5,6 +5,7 @@
from django.db.models import QuerySet
from constance import config
+from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import mixins, status
from rest_framework.decorators import action
from rest_framework.exceptions import ValidationError
@@ -22,6 +23,7 @@
PDUUploadPermission,
PDUViewListAndDetailsPermission,
)
+from hct_mis_api.apps.core.api.filters import UpdatedAtFilter
from hct_mis_api.apps.core.api.mixins import (
ActionMixin,
BusinessAreaProgramMixin,
@@ -69,7 +71,8 @@ class PeriodicDataUpdateTemplateViewSet(
"export": [PDUTemplateCreatePermission],
"download": [PDUTemplateDownloadPermission],
}
- filter_backends = (OrderingFilter,)
+ filter_backends = (OrderingFilter, DjangoFilterBackend)
+ filterset_class = UpdatedAtFilter
def get_queryset(self) -> QuerySet:
business_area = self.get_business_area()
@@ -132,7 +135,8 @@ class PeriodicDataUpdateUploadViewSet(
"retrieve": [PDUViewListAndDetailsPermission],
"upload": [PDUUploadPermission],
}
- filter_backends = (OrderingFilter,)
+ filter_backends = (OrderingFilter, DjangoFilterBackend)
+ filterset_class = UpdatedAtFilter
def get_queryset(self) -> QuerySet:
business_area = self.get_business_area()
@@ -183,7 +187,8 @@ class PeriodicFieldViewSet(
):
serializer_class = PeriodicFieldSerializer
permission_classes = [PDUViewListAndDetailsPermission]
- filter_backends = (OrderingFilter,)
+ filter_backends = (OrderingFilter, DjangoFilterBackend)
+ filterset_class = UpdatedAtFilter
def get_queryset(self) -> QuerySet:
program = self.get_program()
diff --git a/src/hct_mis_api/apps/program/api/filters.py b/src/hct_mis_api/apps/program/api/filters.py
index f0d6950b51..ac1e7f6073 100644
--- a/src/hct_mis_api/apps/program/api/filters.py
+++ b/src/hct_mis_api/apps/program/api/filters.py
@@ -7,11 +7,12 @@
from django_filters import rest_framework as filters
+from hct_mis_api.apps.core.api.filters import UpdatedAtFilter
from hct_mis_api.apps.core.utils import decode_id_string_required
from hct_mis_api.apps.program.models import ProgramCycle
-class ProgramCycleFilter(filters.FilterSet):
+class ProgramCycleFilter(UpdatedAtFilter):
search = filters.CharFilter(method="search_filter")
status = filters.MultipleChoiceFilter(
choices=ProgramCycle.STATUS_CHOICE,
diff --git a/src/hct_mis_api/apps/registration_data/api/filters.py b/src/hct_mis_api/apps/registration_data/api/filters.py
index 2dffd4d71b..52170229fc 100644
--- a/src/hct_mis_api/apps/registration_data/api/filters.py
+++ b/src/hct_mis_api/apps/registration_data/api/filters.py
@@ -1,9 +1,10 @@
from django_filters import rest_framework as filters
+from hct_mis_api.apps.core.api.filters import UpdatedAtFilter
from hct_mis_api.apps.registration_data.models import RegistrationDataImport
-class RegistrationDataImportFilter(filters.FilterSet):
+class RegistrationDataImportFilter(UpdatedAtFilter):
name = filters.CharFilter(
field_name="name",
lookup_expr="startswith",
diff --git a/src/hct_mis_api/apps/targeting/api/filters.py b/src/hct_mis_api/apps/targeting/api/filters.py
index b91674f511..f349c8785a 100644
--- a/src/hct_mis_api/apps/targeting/api/filters.py
+++ b/src/hct_mis_api/apps/targeting/api/filters.py
@@ -1,9 +1,10 @@
from django_filters import rest_framework as filters
+from hct_mis_api.apps.core.api.filters import UpdatedAtFilter
from hct_mis_api.apps.targeting.models import TargetPopulation
-class TargetPopulationFilter(filters.FilterSet):
+class TargetPopulationFilter(UpdatedAtFilter):
name = filters.CharFilter(
field_name="name",
lookup_expr="startswith",
diff --git a/tests/unit/api/base.py b/tests/unit/api/base.py
index f13b28ad72..f156e50bc0 100644
--- a/tests/unit/api/base.py
+++ b/tests/unit/api/base.py
@@ -1,3 +1,6 @@
+import contextlib
+from typing import Iterator
+
from django.urls import reverse
from rest_framework import status
@@ -13,6 +16,16 @@
from tests.unit.api.factories import APITokenFactory
+@contextlib.contextmanager
+def token_grant_permission(token: APIToken, grant: Grant) -> Iterator:
+ old = token.grants
+ token.grants += [grant.name]
+ token.save()
+ yield
+ token.grants = old
+ token.save()
+
+
class HOPEApiTestCase(APITestCase):
databases = {"default"}
user_permissions = [
diff --git a/tests/unit/api/test_business_area.py b/tests/unit/api/test_business_area.py
index 5367be30b4..8f94b43b15 100644
--- a/tests/unit/api/test_business_area.py
+++ b/tests/unit/api/test_business_area.py
@@ -1,22 +1,9 @@
-import contextlib
-from typing import Iterator
-
from rest_framework.reverse import reverse
-from hct_mis_api.api.models import APIToken, Grant
+from hct_mis_api.api.models import Grant
from hct_mis_api.apps.account.fixtures import BusinessAreaFactory
from hct_mis_api.apps.core.models import BusinessArea
-from tests.unit.api.base import HOPEApiTestCase
-
-
-@contextlib.contextmanager
-def token_grant_permission(token: APIToken, grant: Grant) -> Iterator:
- old = token.grants
- token.grants += [grant.name]
- token.save()
- yield
- token.grants = old
- token.save()
+from tests.unit.api.base import HOPEApiTestCase, token_grant_permission
class APIBusinessAreaTests(HOPEApiTestCase):
diff --git a/tests/unit/api/test_lookups.py b/tests/unit/api/test_lookups.py
index d6e1bc0270..5a6c1f6768 100644
--- a/tests/unit/api/test_lookups.py
+++ b/tests/unit/api/test_lookups.py
@@ -1,9 +1,10 @@
from rest_framework import status
from rest_framework.reverse import reverse
+from hct_mis_api.api.models import Grant
from hct_mis_api.apps.geo.fixtures import CountryFactory
from hct_mis_api.apps.program.models import Program
-from tests.unit.api.base import HOPEApiTestCase
+from tests.unit.api.base import HOPEApiTestCase, token_grant_permission
class APIProgramStatuesTests(HOPEApiTestCase):
@@ -12,7 +13,8 @@ class APIProgramStatuesTests(HOPEApiTestCase):
def test_get_program_statues(self) -> None:
url = reverse("api:program-statuses-list")
- response = self.client.get(url)
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.json(), dict(Program.STATUS_CHOICE))
@@ -31,7 +33,8 @@ def test_get_countries(self) -> None:
iso_num="0620",
)
url = reverse("api:country-list")
- response = self.client.get(url)
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(url)
assert response.status_code == status.HTTP_200_OK
assert response.json() == [
{
diff --git a/tests/unit/api/test_program.py b/tests/unit/api/test_program.py
index 021660149f..abd7ee99fc 100644
--- a/tests/unit/api/test_program.py
+++ b/tests/unit/api/test_program.py
@@ -1,25 +1,16 @@
-import contextlib
-from typing import Iterator
+from datetime import timedelta
+
+from django.utils import timezone
from rest_framework.reverse import reverse
-from hct_mis_api.api.models import APIToken, Grant
+from hct_mis_api.api.models import Grant
from hct_mis_api.apps.account.fixtures import BusinessAreaFactory
from hct_mis_api.apps.core.fixtures import DataCollectingTypeFactory
from hct_mis_api.apps.core.models import BusinessArea
from hct_mis_api.apps.program.fixtures import ProgramFactory
from hct_mis_api.apps.program.models import Program
-from tests.unit.api.base import HOPEApiTestCase
-
-
-@contextlib.contextmanager
-def token_grant_permission(token: APIToken, grant: Grant) -> Iterator:
- old = token.grants
- token.grants += [grant.name]
- token.save()
- yield
- token.grants = old
- token.save()
+from tests.unit.api.base import HOPEApiTestCase, token_grant_permission
class APIProgramTests(HOPEApiTestCase):
@@ -269,3 +260,28 @@ def test_list_program_filter_status(self) -> None:
self.program2_expected_response,
response.json()["results"],
)
+
+ def test_list_program_filter_updated_at(self) -> None:
+ tomorrow = (timezone.now() + timedelta(days=1)).date()
+ tomorrow_str = tomorrow.strftime("%Y-%m-%d")
+ yesterday = (timezone.now() - timedelta(days=1)).date()
+ yesterday_str = yesterday.strftime("%Y-%m-%d")
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(self.list_url, {"updated_at_before": tomorrow_str})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()["results"]), 3)
+
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(self.list_url, {"updated_at_after": tomorrow_str})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()["results"]), 0)
+
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(self.list_url, {"updated_at_before": yesterday_str})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()["results"]), 0)
+
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(self.list_url, {"updated_at_after": yesterday_str})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()["results"]), 3)
From 1bbaf1e45ff5aa87caaff1aae56d50680e2cb8d4 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Mon, 14 Oct 2024 21:37:44 +0200
Subject: [PATCH 03/15] fix mypy
---
src/hct_mis_api/apps/payment/services/payment_gateway.py | 6 +++---
.../apps/registration_datahub/apis/deduplication_engine.py | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/src/hct_mis_api/apps/payment/services/payment_gateway.py b/src/hct_mis_api/apps/payment/services/payment_gateway.py
index 73fb54539c..05c8402441 100644
--- a/src/hct_mis_api/apps/payment/services/payment_gateway.py
+++ b/src/hct_mis_api/apps/payment/services/payment_gateway.py
@@ -278,8 +278,8 @@ class PaymentGatewayAPIException(Exception):
class PaymentGatewayMissingAPICredentialsException(Exception):
pass
- API_EXCEPTION_CLASS = PaymentGatewayAPIException
- API_MISSING_CREDENTIALS_EXCEPTION_CLASS = PaymentGatewayMissingAPICredentialsException
+ API_EXCEPTION_CLASS = PaymentGatewayAPIException # type: ignore
+ API_MISSING_CREDENTIALS_EXCEPTION_CLASS = PaymentGatewayMissingAPICredentialsException # type: ignore
class Endpoints:
CREATE_PAYMENT_INSTRUCTION = "payment_instructions/"
@@ -307,7 +307,7 @@ def create_payment_instruction(self, data: dict) -> PaymentInstructionData:
def change_payment_instruction_status(self, status: PaymentInstructionStatus, remote_id: str) -> str:
if status.value not in [s.value for s in PaymentInstructionStatus]:
- raise self.API_EXCEPTION_CLASS(f"Can't set invalid Payment Instruction status: {status}")
+ raise self.API_EXCEPTION_CLASS(f"Can't set invalid Payment Instruction status: {status}") # type: ignore
action_endpoint_map = {
PaymentInstructionStatus.ABORTED: self.Endpoints.ABORT_PAYMENT_INSTRUCTION_STATUS,
diff --git a/src/hct_mis_api/apps/registration_datahub/apis/deduplication_engine.py b/src/hct_mis_api/apps/registration_datahub/apis/deduplication_engine.py
index 8fb5df3078..269374a383 100644
--- a/src/hct_mis_api/apps/registration_datahub/apis/deduplication_engine.py
+++ b/src/hct_mis_api/apps/registration_datahub/apis/deduplication_engine.py
@@ -50,8 +50,8 @@ class DeduplicationEngineAPIException(Exception):
class DeduplicationEngineMissingAPICredentialsException(Exception):
pass
- API_EXCEPTION_CLASS = DeduplicationEngineAPIException
- API_MISSING_CREDENTIALS_EXCEPTION_CLASS = DeduplicationEngineMissingAPICredentialsException
+ API_EXCEPTION_CLASS = DeduplicationEngineAPIException # type: ignore
+ API_MISSING_CREDENTIALS_EXCEPTION_CLASS = DeduplicationEngineMissingAPICredentialsException # type: ignore
class Endpoints:
GET_DEDUPLICATION_SETS = "deduplication_sets/" # GET - List view
From d3f16fd1fbfa70dcbcabb6243e4d06e693dcb2d1 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Tue, 15 Oct 2024 00:42:14 +0200
Subject: [PATCH 04/15] codecov adjustments
---
codecov.yml | 12 ++++++++++++
src/hct_mis_api/api/endpoints/core/views.py | 8 ++------
src/hct_mis_api/api/endpoints/program/filters.py | 4 +---
src/hct_mis_api/api/endpoints/program/views.py | 8 ++------
tests/unit/api/test_program.py | 10 ++++++++++
5 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/codecov.yml b/codecov.yml
index 72f96c556e..58ffb82578 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -25,3 +25,15 @@ coverage:
- "hct_mis_api/conftest.py"
- "hct_mis_api/config/settings.py"
- "hct_mis_api/apps/core/management/commands/*"
+ - "pragma: no cover"
+ - "pragma: no-cover"
+ - "def __repr__"
+ - "if self\.debug"
+ - "raise AssertionError"
+ - "raise NotImplementedError"
+ - "except ImportError"
+ - "#if 0:"
+ - "if __name__ == .__main__."
+ - "if TYPE_CHECKING"
+ - "^\\s*(import\\s.+|from\\s+.+import\\s+.+)"
+ - "logger.exception(e)"
\ No newline at end of file
diff --git a/src/hct_mis_api/api/endpoints/core/views.py b/src/hct_mis_api/api/endpoints/core/views.py
index ace581f6c9..ad16b5ff97 100644
--- a/src/hct_mis_api/api/endpoints/core/views.py
+++ b/src/hct_mis_api/api/endpoints/core/views.py
@@ -24,9 +24,5 @@ def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response:
queryset = filterset.qs
page = self.paginate_queryset(queryset)
- if page is not None:
- serializer = self.get_serializer(page, many=True)
- return self.get_paginated_response(serializer.data)
-
- serializer = self.get_serializer(queryset, many=True)
- return Response(serializer.data)
+ serializer = self.get_serializer(page, many=True)
+ return self.get_paginated_response(serializer.data)
diff --git a/src/hct_mis_api/api/endpoints/program/filters.py b/src/hct_mis_api/api/endpoints/program/filters.py
index b00e330ca0..c497b08ece 100644
--- a/src/hct_mis_api/api/endpoints/program/filters.py
+++ b/src/hct_mis_api/api/endpoints/program/filters.py
@@ -21,6 +21,4 @@ class Meta:
def is_active_filter(self, queryset: "QuerySet[Program]", name: str, value: bool) -> "QuerySet[Program]":
if value is True:
return queryset.filter(status=Program.ACTIVE)
- elif value is False:
- return queryset.exclude(status=Program.ACTIVE)
- return queryset
+ return queryset.exclude(status=Program.ACTIVE)
diff --git a/src/hct_mis_api/api/endpoints/program/views.py b/src/hct_mis_api/api/endpoints/program/views.py
index 3402938df8..0140de84f7 100644
--- a/src/hct_mis_api/api/endpoints/program/views.py
+++ b/src/hct_mis_api/api/endpoints/program/views.py
@@ -23,9 +23,5 @@ def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response:
queryset = filterset.qs
page = self.paginate_queryset(queryset)
- if page is not None:
- serializer = self.get_serializer(page, many=True)
- return self.get_paginated_response(serializer.data)
-
- serializer = self.serializer_class(queryset, many=True)
- return Response(serializer.data)
+ serializer = self.get_serializer(page, many=True)
+ return self.get_paginated_response(serializer.data)
diff --git a/tests/unit/api/test_program.py b/tests/unit/api/test_program.py
index abd7ee99fc..64fd577740 100644
--- a/tests/unit/api/test_program.py
+++ b/tests/unit/api/test_program.py
@@ -251,6 +251,16 @@ def test_list_program_filter_active(self) -> None:
response.json()["results"],
)
+ def test_list_program_filter_not_active(self) -> None:
+ with token_grant_permission(self.token, Grant.API_READ_ONLY):
+ response = self.client.get(self.list_url, {"active": "false"})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()["results"]), 1)
+ self.assertIn(
+ self.program2_expected_response,
+ response.json()["results"],
+ )
+
def test_list_program_filter_status(self) -> None:
with token_grant_permission(self.token, Grant.API_READ_ONLY):
response = self.client.get(self.list_url, {"status": Program.DRAFT})
From 4c317c279dc031653427b0f2577706ead968b0e2 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Tue, 15 Oct 2024 12:04:20 +0200
Subject: [PATCH 05/15] codecov fixes
---
codecov.yml | 2 --
tests/.coveragerc | 1 -
2 files changed, 3 deletions(-)
diff --git a/codecov.yml b/codecov.yml
index 58ffb82578..bf1bacd789 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -28,11 +28,9 @@ coverage:
- "pragma: no cover"
- "pragma: no-cover"
- "def __repr__"
- - "if self\.debug"
- "raise AssertionError"
- "raise NotImplementedError"
- "except ImportError"
- - "#if 0:"
- "if __name__ == .__main__."
- "if TYPE_CHECKING"
- "^\\s*(import\\s.+|from\\s+.+import\\s+.+)"
diff --git a/tests/.coveragerc b/tests/.coveragerc
index 7942061b4e..8e5e354932 100644
--- a/tests/.coveragerc
+++ b/tests/.coveragerc
@@ -18,7 +18,6 @@ omit =
hct_mis_api/conftest.py
hct_mis_api/config/settings.py
hct_mis_api/apps/core/management/commands/*
- hct_mis_api/apps/household/forms.py
[report]
# Regexes for lines to exclude from consideration
From 01e97f41eb4efc2b64d8706d95948242d11d23e5 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Tue, 15 Oct 2024 16:31:28 +0200
Subject: [PATCH 06/15] use ListAPIView for Country API, simplify program view
---
src/hct_mis_api/api/endpoints/lookups/base.py | 9 ++++-----
.../api/endpoints/program/views.py | 20 ++++---------------
2 files changed, 8 insertions(+), 21 deletions(-)
diff --git a/src/hct_mis_api/api/endpoints/lookups/base.py b/src/hct_mis_api/api/endpoints/lookups/base.py
index 18d4ae51e6..85fcd905e3 100644
--- a/src/hct_mis_api/api/endpoints/lookups/base.py
+++ b/src/hct_mis_api/api/endpoints/lookups/base.py
@@ -1,5 +1,6 @@
from typing import TYPE_CHECKING, Any, Optional
+from rest_framework.generics import ListAPIView
from rest_framework.response import Response
from hct_mis_api.api.endpoints.base import HOPEAPIView
@@ -26,12 +27,10 @@ def get(self, request: "Request", format: Optional[Any] = None) -> Response:
return Response(dict(IDENTIFICATION_TYPE_CHOICE))
-class CountryAPIView(HOPEAPIView):
+class CountryAPIView(HOPEAPIView, ListAPIView):
+ queryset = Country.objects.all()
serializer_class = CountrySerializer
-
- def get(self, request: "Request", format: Optional[Any] = None) -> Response:
- serializer = self.serializer_class(Country.objects.all(), many=True)
- return Response(serializer.data)
+ pagination_class = None
class ResidenceStatus(HOPEAPIView):
diff --git a/src/hct_mis_api/api/endpoints/program/views.py b/src/hct_mis_api/api/endpoints/program/views.py
index 0140de84f7..cb8ee4626e 100644
--- a/src/hct_mis_api/api/endpoints/program/views.py
+++ b/src/hct_mis_api/api/endpoints/program/views.py
@@ -1,27 +1,15 @@
-from typing import TYPE_CHECKING, Any
-
+from django_filters.rest_framework import DjangoFilterBackend
+from rest_framework.filters import OrderingFilter
from rest_framework.generics import ListAPIView
-from rest_framework.response import Response
from hct_mis_api.api.endpoints.base import HOPEAPIView
from hct_mis_api.api.endpoints.program.filters import ProgramFilter
from hct_mis_api.api.endpoints.program.serializers import ProgramGlobalSerializer
from hct_mis_api.apps.program.models import Program
-if TYPE_CHECKING:
- from rest_framework.request import Request
-
class ProgramGlobalListView(HOPEAPIView, ListAPIView):
serializer_class = ProgramGlobalSerializer
queryset = Program.objects.all()
-
- def list(self, request: "Request", *args: Any, **kwargs: Any) -> Response:
- queryset = self.queryset
- filterset = ProgramFilter(request.GET, queryset=queryset)
- if filterset.is_valid():
- queryset = filterset.qs
-
- page = self.paginate_queryset(queryset)
- serializer = self.get_serializer(page, many=True)
- return self.get_paginated_response(serializer.data)
+ filter_backends = (OrderingFilter, DjangoFilterBackend)
+ filterset_class = ProgramFilter
From 08667db919fcd2f6c91cb41b6351993568e0a40b Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Tue, 15 Oct 2024 20:31:34 +0200
Subject: [PATCH 07/15] The label should contain 'English(EN)'
---
src/hct_mis_api/apps/core/fixtures.py | 4 ++--
.../apps/core/migrations/0088_migration.py | 19 +++++++++++++++++
src/hct_mis_api/apps/core/models.py | 9 +++++++-
.../apps/periodic_data_update/test_models.py | 21 +++++++++++++++++++
4 files changed, 50 insertions(+), 3 deletions(-)
create mode 100644 src/hct_mis_api/apps/core/migrations/0088_migration.py
diff --git a/src/hct_mis_api/apps/core/fixtures.py b/src/hct_mis_api/apps/core/fixtures.py
index 9035c9e502..c9fe19ad5a 100644
--- a/src/hct_mis_api/apps/core/fixtures.py
+++ b/src/hct_mis_api/apps/core/fixtures.py
@@ -1,3 +1,4 @@
+import json
import random
from typing import Any, List
@@ -146,9 +147,8 @@ def program(self) -> Any:
@classmethod
def _create(cls, target_class: Any, *args: Any, **kwargs: Any) -> FlexibleAttribute:
label = kwargs.pop("label", None)
+ kwargs["label"] = json.dumps({"English(EN)": label})
obj = super()._create(target_class, *args, **kwargs)
- obj.label = {"English(EN)": label}
- obj.save()
return obj
class Meta:
diff --git a/src/hct_mis_api/apps/core/migrations/0088_migration.py b/src/hct_mis_api/apps/core/migrations/0088_migration.py
new file mode 100644
index 0000000000..8a42e11705
--- /dev/null
+++ b/src/hct_mis_api/apps/core/migrations/0088_migration.py
@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-10-15 17:18
+
+from django.db import migrations, models
+import hct_mis_api.apps.core.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core', '0087_migration'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='flexibleattribute',
+ name='label',
+ field=models.JSONField(default=dict, validators=[hct_mis_api.apps.core.models.label_contains_english_en_validator]),
+ ),
+ ]
diff --git a/src/hct_mis_api/apps/core/models.py b/src/hct_mis_api/apps/core/models.py
index 01a83a7e9c..dca60f1810 100644
--- a/src/hct_mis_api/apps/core/models.py
+++ b/src/hct_mis_api/apps/core/models.py
@@ -191,6 +191,11 @@ def get_sys_option(self, key: str, default: None = None) -> Any:
return default
+def label_contains_english_en_validator(data: dict) -> None:
+ if "English(EN)" not in data:
+ raise ValidationError('The "English(EN)" key is required in the label.')
+
+
class FlexibleAttribute(SoftDeletableModel, NaturalKeyModel, TimeStampedUUIDModel):
STRING = "STRING"
IMAGE = "IMAGE"
@@ -237,7 +242,7 @@ class FlexibleAttribute(SoftDeletableModel, NaturalKeyModel, TimeStampedUUIDMode
null=True,
related_name="flex_field",
)
- label = JSONField(default=dict)
+ label = JSONField(default=dict, validators=[label_contains_english_en_validator])
hint = JSONField(default=dict)
group = models.ForeignKey(
"core.FlexibleAttributeGroup", on_delete=models.CASCADE, related_name="flex_attributes", null=True, blank=True
@@ -264,6 +269,8 @@ def clean(self) -> None:
):
raise ValidationError(f'Flex field with name "{self.name}" already exists inside a program.')
+ label_contains_english_en_validator(self.label)
+
def save(self, *args: Any, **kwargs: Any) -> None:
self.clean()
super().save(*args, **kwargs)
diff --git a/tests/unit/apps/periodic_data_update/test_models.py b/tests/unit/apps/periodic_data_update/test_models.py
index 72df36fb94..186fbda9f3 100644
--- a/tests/unit/apps/periodic_data_update/test_models.py
+++ b/tests/unit/apps/periodic_data_update/test_models.py
@@ -31,6 +31,7 @@ def setUp(self) -> None:
name="flex_field_1",
type=FlexibleAttribute.STRING,
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
self.pdu_field = FlexibleAttributeForPDUFactory(
@@ -63,6 +64,7 @@ def test_unique_name_rules_for_flex_fields(self) -> None:
name=self.flex_field.name,
type=FlexibleAttribute.STRING,
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
self.assertIn(
'duplicate key value violates unique constraint "unique_name_without_program"',
@@ -75,6 +77,7 @@ def test_unique_name_rules_for_flex_fields(self) -> None:
name=self.pdu_field.name,
type=FlexibleAttribute.STRING,
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
self.assertIn(
f'Flex field with name "{self.pdu_field.name}" already exists inside a program.',
@@ -90,3 +93,21 @@ def test_unique_name_rules_for_flex_fields(self) -> None:
f'Flex field with name "{self.flex_field.name}" already exists without a program.',
str(ve_context.exception),
)
+
+ def test_flexible_attribute_label_without_english_en_key(self) -> None:
+ with self.assertRaisesMessage(ValidationError, 'The "English(EN)" key is required in the label.'):
+ FlexibleAttribute.objects.create(
+ name="flex_field_2",
+ type=FlexibleAttribute.STRING,
+ associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"other value": "value"},
+ )
+ flexible_attribute = FlexibleAttribute.objects.create(
+ name="flex_field_2",
+ type=FlexibleAttribute.STRING,
+ associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
+ )
+ with self.assertRaisesMessage(ValidationError, 'The "English(EN)" key is required in the label.'):
+ flexible_attribute.label = {"wrong": "value"}
+ flexible_attribute.save()
From c4def4c2c4d3600eac3f35680f1f900617f0bdd9 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Tue, 15 Oct 2024 22:36:11 +0200
Subject: [PATCH 08/15] fix tests
---
tests/unit/apps/core/test_schema.py | 2 +-
tests/unit/apps/program/test_copy_program.py | 1 +
tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py | 2 ++
.../apps/targeting/test_create_target_population_mutation.py | 1 +
tests/unit/apps/targeting/test_individual_block_filters.py | 1 +
5 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/tests/unit/apps/core/test_schema.py b/tests/unit/apps/core/test_schema.py
index 8e0ff41188..4d4f31cca3 100644
--- a/tests/unit/apps/core/test_schema.py
+++ b/tests/unit/apps/core/test_schema.py
@@ -132,7 +132,7 @@ def setUpTestData(cls) -> None:
# Create a non-PDU field
FlexibleAttribute.objects.create(
type=FlexibleAttribute.STRING,
- label="Not PDU Field",
+ label={"English(EN)": "value", "Not PDU Field": ""},
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
)
diff --git a/tests/unit/apps/program/test_copy_program.py b/tests/unit/apps/program/test_copy_program.py
index 1055b9e3ed..857eb27fa7 100644
--- a/tests/unit/apps/program/test_copy_program.py
+++ b/tests/unit/apps/program/test_copy_program.py
@@ -144,6 +144,7 @@ def setUpTestData(cls) -> None:
name="flex_field_1",
type=FlexibleAttribute.STRING,
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
cls.individual.flex_fields = {
pdu_field.name: {
diff --git a/tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py b/tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py
index 937de2ab72..55541e5b23 100644
--- a/tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py
+++ b/tests/unit/apps/registration_datahub/test_rdi_xlsx_create.py
@@ -91,11 +91,13 @@ def setUpTestData(cls) -> None:
type=FlexibleAttribute.INTEGER,
name="muac_i_f",
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
FlexibleAttribute.objects.create(
type=FlexibleAttribute.DECIMAL,
name="jan_decimal_i_f",
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
content = Path(
f"{settings.TESTS_ROOT}/apps/registration_datahub/test_file/new_reg_data_import.xlsx"
diff --git a/tests/unit/apps/targeting/test_create_target_population_mutation.py b/tests/unit/apps/targeting/test_create_target_population_mutation.py
index bb5a020a53..3128d59bfa 100644
--- a/tests/unit/apps/targeting/test_create_target_population_mutation.py
+++ b/tests/unit/apps/targeting/test_create_target_population_mutation.py
@@ -86,6 +86,7 @@ def setUpTestData(cls) -> None:
name="flex_field_1",
type=FlexibleAttribute.STRING,
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
pdu_data = PeriodicFieldDataFactory(
subtype=PeriodicFieldData.DECIMAL,
diff --git a/tests/unit/apps/targeting/test_individual_block_filters.py b/tests/unit/apps/targeting/test_individual_block_filters.py
index 9aece156b3..c0083412d9 100644
--- a/tests/unit/apps/targeting/test_individual_block_filters.py
+++ b/tests/unit/apps/targeting/test_individual_block_filters.py
@@ -180,6 +180,7 @@ def test_filter_on_flex_field(self) -> None:
name="flex_field_1",
type=FlexibleAttribute.STRING,
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
query = Household.objects.all()
flex_field_filter = TargetingIndividualBlockRuleFilter(
From 4dc12acdd706c3cf9c7b036d61c193dabd371a19 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Wed, 16 Oct 2024 00:41:51 +0200
Subject: [PATCH 09/15] fix tests
---
src/hct_mis_api/apps/core/fixtures.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/hct_mis_api/apps/core/fixtures.py b/src/hct_mis_api/apps/core/fixtures.py
index c9fe19ad5a..091353a82f 100644
--- a/src/hct_mis_api/apps/core/fixtures.py
+++ b/src/hct_mis_api/apps/core/fixtures.py
@@ -1,4 +1,3 @@
-import json
import random
from typing import Any, List
@@ -147,7 +146,7 @@ def program(self) -> Any:
@classmethod
def _create(cls, target_class: Any, *args: Any, **kwargs: Any) -> FlexibleAttribute:
label = kwargs.pop("label", None)
- kwargs["label"] = json.dumps({"English(EN)": label})
+ kwargs["label"] = {"English(EN)": label}
obj = super()._create(target_class, *args, **kwargs)
return obj
From 7d0a765f925eba6f8ca41b2af88095aaa96343c9 Mon Sep 17 00:00:00 2001
From: marekbiczysko
Date: Wed, 16 Oct 2024 00:48:54 +0200
Subject: [PATCH 10/15]
217950_Payment_Module_Getting_Delivery_Mechanism_error_when_rejecting_a_payment_plan_and_assigning_FSP
---
.../VolumeByDeliveryMechanismSection.tsx | 4 ++--
.../VolumeByDeliveryMechanismSection.tsx | 4 ++--
.../pages/paymentmodule/EditFollowUpSetUpFspPage.tsx | 2 +-
.../paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx | 2 +-
.../pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx | 2 +-
5 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/src/frontend/src/components/paymentmodule/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx b/src/frontend/src/components/paymentmodule/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx
index b5a93c7b5f..45d964847e 100644
--- a/src/frontend/src/components/paymentmodule/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx
+++ b/src/frontend/src/components/paymentmodule/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx
@@ -69,8 +69,8 @@ export const VolumeByDeliveryMechanismSection: React.FC<
color={getDeliveryMechanismColor(vdm.deliveryMechanism.name)}
>
diff --git a/src/frontend/src/components/paymentmodulepeople/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx b/src/frontend/src/components/paymentmodulepeople/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx
index b5a93c7b5f..45d964847e 100644
--- a/src/frontend/src/components/paymentmodulepeople/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx
+++ b/src/frontend/src/components/paymentmodulepeople/PaymentPlanDetails/FspSection/VolumeByDeliveryMechanismSection/VolumeByDeliveryMechanismSection.tsx
@@ -69,8 +69,8 @@ export const VolumeByDeliveryMechanismSection: React.FC<
color={getDeliveryMechanismColor(vdm.deliveryMechanism.name)}
>
diff --git a/src/frontend/src/containers/pages/paymentmodule/EditFollowUpSetUpFspPage.tsx b/src/frontend/src/containers/pages/paymentmodule/EditFollowUpSetUpFspPage.tsx
index 1392fe4259..644f111b80 100644
--- a/src/frontend/src/containers/pages/paymentmodule/EditFollowUpSetUpFspPage.tsx
+++ b/src/frontend/src/containers/pages/paymentmodule/EditFollowUpSetUpFspPage.tsx
@@ -29,7 +29,7 @@ export function EditFollowUpSetUpFspPage(): React.ReactElement {
const mappedInitialDeliveryMechanisms =
paymentPlanData.paymentPlan.deliveryMechanisms.map((el) => ({
- deliveryMechanism: el.name,
+ deliveryMechanism: el.code,
fsp: el.fsp?.id || '',
chosenConfiguration: el.chosenConfiguration || '',
}));
diff --git a/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx b/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx
index e179c16557..2269a6a4e0 100644
--- a/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx
+++ b/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleFollowUpSetUpFspPage.tsx
@@ -29,7 +29,7 @@ export const EditPeopleFollowUpSetUpFspPage = (): React.ReactElement => {
const mappedInitialDeliveryMechanisms =
paymentPlanData.paymentPlan.deliveryMechanisms.map((el) => ({
- deliveryMechanism: el.name,
+ deliveryMechanism: el.code,
fsp: el.fsp?.id || '',
chosenConfiguration: el.chosenConfiguration || '',
}));
diff --git a/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx b/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx
index 59e380fcf2..ae9fcbe338 100644
--- a/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx
+++ b/src/frontend/src/containers/pages/paymentmodulepeople/EditPeopleSetUpFspPage.tsx
@@ -29,7 +29,7 @@ export const EditPeopleSetUpFspPage = (): React.ReactElement => {
const mappedInitialDeliveryMechanisms =
paymentPlanData.paymentPlan.deliveryMechanisms.map((el) => ({
- deliveryMechanism: el.name,
+ deliveryMechanism: el.code,
fsp: el.fsp?.id || '',
chosenConfiguration: el.chosenConfiguration || '',
}));
From 2d82056e5d370cd5b79e7144d9d20687591f7f97 Mon Sep 17 00:00:00 2001
From: Paulina Kujawa
Date: Wed, 16 Oct 2024 01:22:38 +0200
Subject: [PATCH 11/15] fix ImportExportPaymentPlanPaymentListTest
---
.../payment/test_import_export_payment_plan_payment_list.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/tests/unit/apps/payment/test_import_export_payment_plan_payment_list.py b/tests/unit/apps/payment/test_import_export_payment_plan_payment_list.py
index a4d739ae8d..79beaa3754 100644
--- a/tests/unit/apps/payment/test_import_export_payment_plan_payment_list.py
+++ b/tests/unit/apps/payment/test_import_export_payment_plan_payment_list.py
@@ -406,12 +406,14 @@ def test_payment_row_flex_fields(self) -> None:
type=FlexibleAttribute.DECIMAL,
name="flex_decimal_i_f",
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
decimal_flexible_attribute.save()
date_flexible_attribute = FlexibleAttribute(
type=FlexibleAttribute.DECIMAL,
name="flex_date_i_f",
associated_with=FlexibleAttribute.ASSOCIATED_WITH_HOUSEHOLD,
+ label={"English(EN)": "value"},
)
date_flexible_attribute.save()
flex_fields = [
@@ -561,12 +563,14 @@ def test_flex_fields_admin_visibility(self) -> None:
type=FlexibleAttribute.DECIMAL,
name="flex_decimal_i_f",
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
decimal_flexible_attribute.save()
date_flexible_attribute = FlexibleAttribute(
type=FlexibleAttribute.DECIMAL,
name="flex_date_i_f",
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
date_flexible_attribute.save()
self.client.login(username="admin", password="password")
@@ -590,6 +594,7 @@ def test_payment_row_get_flex_field_if_no_snapshot_data(self) -> None:
type=FlexibleAttribute.DECIMAL,
name="flex_decimal_i_f",
associated_with=FlexibleAttribute.ASSOCIATED_WITH_INDIVIDUAL,
+ label={"English(EN)": "value"},
)
flex_field.save()
fsp_xlsx_template = FinancialServiceProviderXlsxTemplateFactory(flex_fields=[flex_field.name])
From f1ce6f26eea0336320ea4cd2013fb6a9bc152cbe Mon Sep 17 00:00:00 2001
From: gabra4 <131235609+gabra4@users.noreply.github.com>
Date: Wed, 16 Oct 2024 09:53:14 +0300
Subject: [PATCH 12/15] Update README.md
---
README.md | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/README.md b/README.md
index c70e1fce29..4ea1602984 100644
--- a/README.md
+++ b/README.md
@@ -39,15 +39,17 @@ Following the Principles for Digital Development, the “reuse and improve” an
### Why
-* Put cash recipients at the center with secure personal data processing
-* Unified consolidated reporting for beneficiary data
-* Increase accountability beneficiaries and donors
-* Financial Inclusion of Beneficiaries
-* To Enhance Traceability
-* Simplify current process in EMOPS
-* Ensuring due diligence in-cash transfer
-* Grow the use of effective cash program for children in a risk informed manner
-* Eliminate redundancies and dupes in how we do things
+|#| HOPE Functions | Added Value |
+|1|Registration of payees data | Quality Assurance for data collection, verification of eligibility |
+|2|Deduplication of personal records|Mitigate the risk of duplicate records|
+|3|Target payments|Ensure inclusion of relevant payees in payment plan|
+|4|Entitlment Calculations|Ensure cash benefit entitlement rules are uphold|
+|5|Payment Management|Approval, Authorization and Financial Release tracking|
+|6|Reconciliation of Individual payment|Support liquidation or advance or reimbursement to Financial Service Provider|
+|7|Payment Verification|Mitigate the risk of inaccurate FSP reports and fraud|
+|8|Grievances Redressals and Payees Communications|Ensure payment quality and direct communication with payees|
+|9|Roles base access|Uphold segragation of duties and data protection principles|
+|10|Reporting|Access process and output indicators|
## Legal
Humanitarian cash Operations and Programme Ecosystem
From 38488d811b2e64d7da017215f2b3b0797584ec57 Mon Sep 17 00:00:00 2001
From: gabra4 <131235609+gabra4@users.noreply.github.com>
Date: Wed, 16 Oct 2024 09:56:11 +0300
Subject: [PATCH 13/15] Update README.md
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index 4ea1602984..dc0ba2a73f 100644
--- a/README.md
+++ b/README.md
@@ -40,6 +40,7 @@ Following the Principles for Digital Development, the “reuse and improve” an
### Why
|#| HOPE Functions | Added Value |
+|---|--------------------------|----------------------------------------------|
|1|Registration of payees data | Quality Assurance for data collection, verification of eligibility |
|2|Deduplication of personal records|Mitigate the risk of duplicate records|
|3|Target payments|Ensure inclusion of relevant payees in payment plan|
From d5a4955578f227f431a8761b8470ebcc4fb69401 Mon Sep 17 00:00:00 2001
From: Pavlo Mokiichuk
Date: Wed, 16 Oct 2024 09:51:47 +0200
Subject: [PATCH 14/15] Script Clean/Remove pre GPF data (#4326)
* add remove_migrated_data_is_original
* upd test
* add more info and hard delete
* review
---
src/hct_mis_api/apps/household/fixtures.py | 18 ++
.../remove_migrated_data_is_original.py | 27 +++
.../test_remove_migrated_data_is_original.py | 173 ++++++++++++++++++
3 files changed, 218 insertions(+)
create mode 100644 src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py
create mode 100644 tests/unit/one_time_scripts/test_remove_migrated_data_is_original.py
diff --git a/src/hct_mis_api/apps/household/fixtures.py b/src/hct_mis_api/apps/household/fixtures.py
index ceeaf16fb1..84751a28f8 100644
--- a/src/hct_mis_api/apps/household/fixtures.py
+++ b/src/hct_mis_api/apps/household/fixtures.py
@@ -35,6 +35,8 @@
PendingDocument,
PendingHousehold,
PendingIndividual,
+ PendingIndividualIdentity,
+ PendingIndividualRoleInHousehold,
)
from hct_mis_api.apps.program.fixtures import ProgramFactory
from hct_mis_api.apps.registration_data.fixtures import RegistrationDataImportFactory
@@ -166,6 +168,15 @@ class Meta:
number = factory.Faker("pystr", min_chars=None, max_chars=20)
+class PendingIndividualIdentityFactory(DjangoModelFactory):
+ rdi_merge_status = MergeStatusModel.PENDING
+
+ class Meta:
+ model = PendingIndividualIdentity
+
+ number = factory.Faker("pystr", min_chars=None, max_chars=20)
+
+
class IndividualRoleInHouseholdFactory(DjangoModelFactory):
rdi_merge_status = MergeStatusModel.MERGED
@@ -173,6 +184,13 @@ class Meta:
model = IndividualRoleInHousehold
+class PendingIndividualRoleInHouseholdFactory(DjangoModelFactory):
+ rdi_merge_status = MergeStatusModel.PENDING
+
+ class Meta:
+ model = PendingIndividualRoleInHousehold
+
+
class IndividualCollectionFactory(DjangoModelFactory):
class Meta:
model = IndividualCollection
diff --git a/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py b/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py
new file mode 100644
index 0000000000..0c222b4309
--- /dev/null
+++ b/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py
@@ -0,0 +1,27 @@
+from django.apps import apps
+
+
+def remove_migrated_data_is_original() -> None:
+ all_models = apps.get_models()
+
+ for model in all_models:
+ if hasattr(model, "is_original"):
+ if model.__name__ == "GrievanceTicket":
+ # 'GrievanceTicket' has no attribute 'all_objects'
+ queryset_all = model.default_for_migrations_fix.all()
+ queryset_is_original = queryset_all.filter(is_original=True)
+ elif model.__name__ in ["HouseholdSelection", "EntitlementCard", "Feedback", "Message"]:
+ queryset_all = model.original_and_repr_objects.all()
+ queryset_is_original = queryset_all.filter(is_original=True)
+ else:
+ queryset_all = model.all_objects.all()
+ queryset_is_original = queryset_all.filter(is_original=True)
+
+ print(
+ f"*** {model.__name__} All objects: {queryset_all.count()}. "
+ f"Removing objects with 'is_original=True': {queryset_is_original.count()}"
+ )
+
+ count, _ = queryset_is_original.delete()
+
+ print(f"Deleted {model.__name__} and related objects {count}.\n")
diff --git a/tests/unit/one_time_scripts/test_remove_migrated_data_is_original.py b/tests/unit/one_time_scripts/test_remove_migrated_data_is_original.py
new file mode 100644
index 0000000000..35a057688c
--- /dev/null
+++ b/tests/unit/one_time_scripts/test_remove_migrated_data_is_original.py
@@ -0,0 +1,173 @@
+from django.test import TestCase
+
+from hct_mis_api.apps.account.fixtures import BusinessAreaFactory
+from hct_mis_api.apps.accountability.fixtures import (
+ CommunicationMessageFactory,
+ FeedbackFactory,
+)
+from hct_mis_api.apps.accountability.models import Feedback, Message
+from hct_mis_api.apps.grievance.fixtures import (
+ GrievanceTicketFactory,
+ TicketNeedsAdjudicationDetailsFactory,
+)
+from hct_mis_api.apps.grievance.models import (
+ GrievanceTicket,
+ TicketNeedsAdjudicationDetails,
+)
+from hct_mis_api.apps.household.fixtures import (
+ BankAccountInfoFactory,
+ DocumentFactory,
+ EntitlementCardFactory,
+ HouseholdFactory,
+ IndividualFactory,
+ IndividualIdentityFactory,
+ IndividualRoleInHouseholdFactory,
+ PendingBankAccountInfoFactory,
+ PendingDocumentFactory,
+ PendingHouseholdFactory,
+ PendingIndividualFactory,
+ PendingIndividualIdentityFactory,
+ PendingIndividualRoleInHouseholdFactory,
+)
+from hct_mis_api.apps.household.models import (
+ ROLE_PRIMARY,
+ BankAccountInfo,
+ Document,
+ EntitlementCard,
+ Household,
+ Individual,
+ IndividualIdentity,
+ IndividualRoleInHousehold,
+ PendingBankAccountInfo,
+ PendingDocument,
+ PendingHousehold,
+ PendingIndividual,
+ PendingIndividualIdentity,
+ PendingIndividualRoleInHousehold,
+)
+from hct_mis_api.apps.targeting.fixtures import HouseholdSelectionFactory
+from hct_mis_api.apps.targeting.models import HouseholdSelection
+from hct_mis_api.one_time_scripts.remove_migrated_data_is_original import (
+ remove_migrated_data_is_original,
+)
+
+
+class BaseMigrateDataTestCase(TestCase):
+ def setUp(self) -> None:
+ BusinessAreaFactory(name="Afghanistan")
+
+ ind = IndividualFactory(household=None)
+ ind2 = IndividualFactory(is_original=True, household=None)
+ pending_ind = PendingIndividualFactory(household=None)
+ pending_ind2 = PendingIndividualFactory(is_original=True, household=None)
+
+ hh = HouseholdFactory()
+ hh2 = HouseholdFactory(is_original=True)
+ pending_hh = PendingHouseholdFactory()
+ pending_hh2 = PendingHouseholdFactory(is_original=True)
+
+ HouseholdSelectionFactory(household=hh)
+ HouseholdSelectionFactory(is_original=True, household=hh2)
+
+ DocumentFactory(individual=ind)
+ DocumentFactory(is_original=True, individual=ind2)
+ PendingDocumentFactory(individual=pending_ind)
+ PendingDocumentFactory(is_original=True, individual=pending_ind2)
+
+ IndividualIdentityFactory(individual=ind)
+ IndividualIdentityFactory(is_original=True, individual=ind)
+ PendingIndividualIdentityFactory(individual=pending_ind)
+ PendingIndividualIdentityFactory(is_original=True, individual=pending_ind)
+
+ IndividualRoleInHouseholdFactory(household=hh, individual=ind, role=ROLE_PRIMARY)
+ IndividualRoleInHouseholdFactory(is_original=True, household=hh2, individual=ind2, role=ROLE_PRIMARY)
+ PendingIndividualRoleInHouseholdFactory(household=pending_hh, individual=pending_ind, role=ROLE_PRIMARY)
+ PendingIndividualRoleInHouseholdFactory(
+ is_original=True, household=pending_hh2, individual=pending_ind2, role=ROLE_PRIMARY
+ )
+
+ EntitlementCardFactory(household=hh)
+ EntitlementCardFactory(is_original=True, household=hh2)
+
+ BankAccountInfoFactory(individual=ind)
+ BankAccountInfoFactory(is_original=True, individual=ind2)
+ PendingBankAccountInfoFactory(individual=pending_ind)
+ PendingBankAccountInfoFactory(is_original=True, individual=pending_ind2)
+
+ CommunicationMessageFactory()
+ CommunicationMessageFactory(is_original=True)
+ FeedbackFactory()
+ FeedbackFactory(is_original=True)
+
+ gr1 = GrievanceTicketFactory()
+ gr2 = GrievanceTicketFactory(is_original=True)
+
+ TicketNeedsAdjudicationDetailsFactory(
+ golden_records_individual=ind,
+ ticket=gr1,
+ )
+ TicketNeedsAdjudicationDetailsFactory(
+ golden_records_individual=ind2,
+ ticket=gr2,
+ )
+
+ def test_run_remove_migrated_data_is_original(self) -> None:
+ # check count before
+ self.assertEqual(Individual.all_objects.count(), 4)
+ self.assertEqual(PendingIndividual.all_objects.count(), 4)
+ self.assertEqual(Household.all_objects.count(), 4)
+ self.assertEqual(PendingHousehold.all_objects.count(), 4)
+ self.assertEqual(HouseholdSelection.original_and_repr_objects.count(), 2)
+ self.assertEqual(Document.all_objects.count(), 4)
+ self.assertEqual(PendingDocument.all_objects.count(), 4)
+ self.assertEqual(IndividualIdentity.all_objects.count(), 4)
+ self.assertEqual(PendingIndividualIdentity.all_objects.count(), 4)
+ self.assertEqual(IndividualRoleInHousehold.all_objects.count(), 4)
+ self.assertEqual(PendingIndividualRoleInHousehold.all_objects.count(), 4)
+ self.assertEqual(EntitlementCard.original_and_repr_objects.count(), 2)
+ self.assertEqual(BankAccountInfo.all_objects.count(), 4)
+ self.assertEqual(PendingBankAccountInfo.all_objects.count(), 4)
+ self.assertEqual(Message.original_and_repr_objects.count(), 2)
+ self.assertEqual(Feedback.original_and_repr_objects.count(), 2)
+ self.assertEqual(GrievanceTicket.default_for_migrations_fix.count(), 2)
+ self.assertEqual(TicketNeedsAdjudicationDetails.objects.count(), 2)
+
+ remove_migrated_data_is_original()
+
+ # check count after
+ self.assertEqual(Individual.all_objects.count(), 2)
+ self.assertEqual(PendingIndividual.all_objects.count(), 2)
+ self.assertEqual(Household.all_objects.count(), 2)
+ self.assertEqual(PendingHousehold.all_objects.count(), 2)
+ self.assertEqual(HouseholdSelection.original_and_repr_objects.count(), 1)
+ self.assertEqual(Document.all_objects.count(), 2)
+ self.assertEqual(PendingDocument.all_objects.count(), 2)
+ self.assertEqual(IndividualIdentity.all_objects.count(), 2)
+ self.assertEqual(PendingIndividualIdentity.all_objects.count(), 2)
+ self.assertEqual(IndividualRoleInHousehold.all_objects.count(), 2)
+ self.assertEqual(PendingIndividualRoleInHousehold.all_objects.count(), 2)
+ self.assertEqual(EntitlementCard.original_and_repr_objects.count(), 1)
+ self.assertEqual(BankAccountInfo.all_objects.count(), 2)
+ self.assertEqual(PendingBankAccountInfo.all_objects.count(), 2)
+ self.assertEqual(Message.original_and_repr_objects.count(), 1)
+ self.assertEqual(Feedback.original_and_repr_objects.count(), 1)
+ self.assertEqual(GrievanceTicket.default_for_migrations_fix.count(), 1)
+ self.assertEqual(TicketNeedsAdjudicationDetails.objects.count(), 1)
+
+ self.assertEqual(Individual.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(PendingIndividual.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(Household.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(PendingHousehold.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(HouseholdSelection.original_and_repr_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(Document.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(PendingDocument.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(IndividualIdentity.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(PendingIndividualIdentity.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(IndividualRoleInHousehold.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(PendingIndividualRoleInHousehold.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(EntitlementCard.original_and_repr_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(BankAccountInfo.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(PendingBankAccountInfo.all_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(Message.original_and_repr_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(Feedback.original_and_repr_objects.filter(is_original=True).count(), 0)
+ self.assertEqual(GrievanceTicket.default_for_migrations_fix.filter(is_original=True).count(), 0)
From b741d4493ed586b4e57025b80af360327d23c519 Mon Sep 17 00:00:00 2001
From: Pavlo Mokiichuk
Date: Thu, 17 Oct 2024 08:59:27 +0200
Subject: [PATCH 15/15] Clean/Remove pre GPF data: Script Optimization (#4337)
* script optimization
* upd only()
---
.../remove_migrated_data_is_original.py | 23 +++++++++++++------
1 file changed, 16 insertions(+), 7 deletions(-)
diff --git a/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py b/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py
index 0c222b4309..d42edec2c9 100644
--- a/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py
+++ b/src/hct_mis_api/one_time_scripts/remove_migrated_data_is_original.py
@@ -1,20 +1,21 @@
from django.apps import apps
+from django.utils import timezone
-def remove_migrated_data_is_original() -> None:
+def remove_migrated_data_is_original(batch_size: int = 1000) -> None:
+ start_time = timezone.now()
all_models = apps.get_models()
for model in all_models:
if hasattr(model, "is_original"):
if model.__name__ == "GrievanceTicket":
- # 'GrievanceTicket' has no attribute 'all_objects'
- queryset_all = model.default_for_migrations_fix.all()
+ queryset_all = model.default_for_migrations_fix.all().only("is_original", "id")
queryset_is_original = queryset_all.filter(is_original=True)
elif model.__name__ in ["HouseholdSelection", "EntitlementCard", "Feedback", "Message"]:
- queryset_all = model.original_and_repr_objects.all()
+ queryset_all = model.original_and_repr_objects.all().only("is_original", "id")
queryset_is_original = queryset_all.filter(is_original=True)
else:
- queryset_all = model.all_objects.all()
+ queryset_all = model.all_objects.all().only("is_original", "id")
queryset_is_original = queryset_all.filter(is_original=True)
print(
@@ -22,6 +23,14 @@ def remove_migrated_data_is_original() -> None:
f"Removing objects with 'is_original=True': {queryset_is_original.count()}"
)
- count, _ = queryset_is_original.delete()
+ deleted_count = 0
+ ids_to_delete = list(queryset_is_original.values_list("id", flat=True).iterator(chunk_size=batch_size))
- print(f"Deleted {model.__name__} and related objects {count}.\n")
+ for i in range(0, len(ids_to_delete), batch_size):
+ batch_pks = ids_to_delete[i : i + batch_size]
+ count, _ = queryset_all.filter(pk__in=batch_pks).delete()
+ deleted_count += count
+
+ print(f"Deleted {model.__name__} and related objects {deleted_count}.\n")
+
+ print(f"Completed in {timezone.now() - start_time}\n", "*" * 55)