From 8d5d6ddd00017efebdf614b110350b81f0a45cb7 Mon Sep 17 00:00:00 2001 From: Domenico DiNicola Date: Fri, 18 Oct 2024 17:41:38 +0200 Subject: [PATCH] refactoring --- compose.yml | 2 +- .../core/management/commands/inspect_hope.py | 4 +- src/hope_country_report/apps/core/storage.py | 10 ++++ .../apps/hope/models/_inspect.py | 41 +++++++--------- .../apps/power_query/processors.py | 5 +- .../apps/power_query/storage.py | 47 ------------------- src/hope_country_report/config/__init__.py | 34 +++++++------- .../config/fragments/storage.py | 16 ------- src/hope_country_report/config/settings.py | 9 ++-- tests/conftest.py | 5 +- 10 files changed, 58 insertions(+), 115 deletions(-) create mode 100644 src/hope_country_report/apps/core/storage.py delete mode 100644 src/hope_country_report/apps/power_query/storage.py delete mode 100644 src/hope_country_report/config/fragments/storage.py diff --git a/compose.yml b/compose.yml index 3b9367cc..8c3ad73f 100644 --- a/compose.yml +++ b/compose.yml @@ -28,7 +28,7 @@ x-django-env: &django-env - AZURE_TENANT_ID= - AZURE_CLIENT_KEY= - FLOWER_URL=http://flower:5555 - - STATIC_FILE_STORAGE=django.core.files.storage.FileSystemStorage + - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage services: backend: diff --git a/src/hope_country_report/apps/core/management/commands/inspect_hope.py b/src/hope_country_report/apps/core/management/commands/inspect_hope.py index c509755c..38de5f06 100644 --- a/src/hope_country_report/apps/core/management/commands/inspect_hope.py +++ b/src/hope_country_report/apps/core/management/commands/inspect_hope.py @@ -199,7 +199,7 @@ def strip_prefix(s): yield "from django.contrib.gis.db import models" yield "from hope_country_report.apps.hope.models._base import HopeModel" - yield "from hope_country_report.apps.power_query.storage import HopeStorage" + yield "from hope_country_report.apps.core.storage import get_hope_storage" basemodel = "HopeModel" known_models = [] @@ -326,7 +326,7 @@ def strip_prefix(s): field_desc += f", related_name='{_related_name}'" if field_type.startswith("ImageField("): - field_desc += "storage=HopeStorage()" + field_desc += "storage=get_hope_storage()()" if extra_params: if not field_desc.endswith("("): diff --git a/src/hope_country_report/apps/core/storage.py b/src/hope_country_report/apps/core/storage.py new file mode 100644 index 00000000..3216ce7f --- /dev/null +++ b/src/hope_country_report/apps/core/storage.py @@ -0,0 +1,10 @@ +import importlib + +from django.conf import settings + + +def get_hope_storage(): + options = settings.STORAGES["hope"]["OPTIONS"] + package_name, klass_name = settings.STORAGES["hope"]["BACKEND"].rsplit(".", 1) + module = importlib.import_module(package_name) + return getattr(module, klass_name)(**options) diff --git a/src/hope_country_report/apps/hope/models/_inspect.py b/src/hope_country_report/apps/hope/models/_inspect.py index ff2b8ace..9e9d8b4d 100644 --- a/src/hope_country_report/apps/hope/models/_inspect.py +++ b/src/hope_country_report/apps/hope/models/_inspect.py @@ -8,8 +8,8 @@ import django.contrib.postgres.fields from django.contrib.gis.db import models +from hope_country_report.apps.core.storage import get_hope_storage from hope_country_report.apps.hope.models._base import HopeModel -from hope_country_report.apps.power_query.storage import HopeStorage class BusinessArea(HopeModel): @@ -23,9 +23,9 @@ class BusinessArea(HopeModel): region_name = models.CharField(max_length=8, null=True) kobo_username = models.CharField(max_length=255, blank=True, null=True) slug = models.CharField(unique=True, max_length=250, null=True) + rapid_pro_payment_verification_token = models.CharField(max_length=40, blank=True, null=True) rapid_pro_host = models.CharField(max_length=200, blank=True, null=True) has_data_sharing_agreement = models.BooleanField(null=True) - rapid_pro_payment_verification_token = models.CharField(max_length=40, blank=True, null=True) is_split = models.BooleanField(null=True) parent = models.ForeignKey( "self", on_delete=models.DO_NOTHING, related_name="businessarea_parent", blank=True, null=True @@ -62,7 +62,6 @@ def __str__(self) -> str: class BusinessareaCountries(HopeModel): - id = models.BigAutoField(primary_key=True) businessarea = models.ForeignKey( BusinessArea, on_delete=models.DO_NOTHING, related_name="businessareacountries_businessarea", null=True ) @@ -745,6 +744,7 @@ class Ticketpaymentverificationdetails(HopeModel): ) approve_status = models.BooleanField(null=True) new_received_amount = models.DecimalField(max_digits=12, decimal_places=2, blank=True, null=True) + new_status = models.CharField(max_length=50, blank=True, null=True) payment_verification = models.ForeignKey( "Paymentverification", on_delete=models.DO_NOTHING, @@ -752,7 +752,6 @@ class Ticketpaymentverificationdetails(HopeModel): blank=True, null=True, ) - new_status = models.CharField(max_length=50, blank=True, null=True) old_received_amount = models.DecimalField(max_digits=12, decimal_places=2, blank=True, null=True) class Meta: @@ -764,7 +763,6 @@ class Tenant: class TicketpaymentverificationdetailsPaymentVerificaf7C9(HopeModel): - id = models.BigAutoField(primary_key=True) ticketpaymentverificationdetails = models.ForeignKey( Ticketpaymentverificationdetails, on_delete=models.DO_NOTHING, @@ -928,7 +926,7 @@ class Document(HopeModel): created_at = models.DateTimeField(null=True) updated_at = models.DateTimeField(null=True) document_number = models.CharField(max_length=255, null=True) - photo = models.ImageField(storage=HopeStorage(), null=True) + photo = models.ImageField(storage=get_hope_storage()(), null=True) individual = models.ForeignKey( "Individual", on_delete=models.DO_NOTHING, related_name="document_individual", null=True ) @@ -1061,7 +1059,7 @@ class Household(HopeModel): ) child_hoh = models.BooleanField(blank=True, null=True) consent_sharing = models.CharField(max_length=63, null=True) - consent_sign = models.ImageField(storage=HopeStorage(), null=True) + consent_sign = models.ImageField(storage=get_hope_storage()(), null=True) deviceid = models.CharField(max_length=250, null=True) fchild_hoh = models.BooleanField(blank=True, null=True) name_enumerator = models.CharField(max_length=250, null=True) @@ -1071,6 +1069,7 @@ class Household(HopeModel): village = models.CharField(max_length=250, null=True) consent = models.BooleanField(blank=True, null=True) is_removed = models.BooleanField(null=True) + collect_individual_data = models.CharField(max_length=250, null=True) currency = models.CharField(max_length=250, null=True) female_age_group_18_59_count = models.IntegerField(blank=True, null=True) female_age_group_18_59_disabled_count = models.IntegerField(blank=True, null=True) @@ -1081,12 +1080,11 @@ class Household(HopeModel): male_age_group_60_count = models.IntegerField(blank=True, null=True) male_age_group_60_disabled_count = models.IntegerField(blank=True, null=True) registration_method = models.CharField(max_length=250, null=True) - collect_individual_data = models.CharField(max_length=250, null=True) unhcr_id = models.CharField(max_length=250, null=True) version = models.BigIntegerField(null=True) - removed_date = models.DateTimeField(blank=True, null=True) withdrawn = models.BooleanField(null=True) withdrawn_date = models.DateTimeField(blank=True, null=True) + removed_date = models.DateTimeField(blank=True, null=True) user_fields = models.JSONField(null=True) country = models.ForeignKey( Country, on_delete=models.DO_NOTHING, related_name="household_country", blank=True, null=True @@ -1119,7 +1117,7 @@ class Household(HopeModel): Area, on_delete=models.DO_NOTHING, related_name="household_admin4", blank=True, null=True ) zip_code = models.CharField(max_length=12, blank=True, null=True) - registration_id = models.TextField(blank=True, null=True) # This field type is a guess. + registration_id = models.TextField(unique=True, blank=True, null=True) # This field type is a guess. copied_from = models.ForeignKey( "self", on_delete=models.DO_NOTHING, related_name="household_copied_from", blank=True, null=True ) @@ -1157,7 +1155,6 @@ class Tenant: class HouseholdPrograms(HopeModel): - id = models.BigAutoField(primary_key=True) household = models.ForeignKey( Household, on_delete=models.DO_NOTHING, related_name="householdprograms_household", null=True ) @@ -1190,7 +1187,7 @@ class Individual(HopeModel): created_at = models.DateTimeField(null=True) updated_at = models.DateTimeField(null=True) individual_id = models.CharField(max_length=255, null=True) - photo = models.ImageField(storage=HopeStorage(), null=True) + photo = models.ImageField(storage=get_hope_storage()(), null=True) full_name = models.TextField(null=True) # This field type is a guess. given_name = models.TextField(null=True) # This field type is a guess. middle_name = models.TextField(null=True) # This field type is a guess. @@ -1221,10 +1218,10 @@ class Individual(HopeModel): first_registration_date = models.DateField(null=True) last_registration_date = models.DateField(null=True) unicef_id = models.CharField(max_length=255, blank=True, null=True) + deduplication_golden_record_status = models.CharField(max_length=50, null=True) + deduplication_golden_record_results = models.JSONField(null=True) sanction_list_possible_match = models.BooleanField(null=True) pregnant = models.BooleanField(blank=True, null=True) - deduplication_golden_record_results = models.JSONField(null=True) - deduplication_golden_record_status = models.CharField(max_length=50, null=True) deduplication_batch_results = models.JSONField(null=True) deduplication_batch_status = models.CharField(max_length=50, null=True) imported_individual_id = models.UUIDField(blank=True, null=True) @@ -1242,16 +1239,16 @@ class Individual(HopeModel): ) is_removed = models.BooleanField(null=True) version = models.BigIntegerField(null=True) - duplicate_date = models.DateTimeField(blank=True, null=True) - removed_date = models.DateTimeField(blank=True, null=True) - withdrawn_date = models.DateTimeField(blank=True, null=True) duplicate = models.BooleanField(null=True) + duplicate_date = models.DateTimeField(blank=True, null=True) withdrawn = models.BooleanField(null=True) + withdrawn_date = models.DateTimeField(blank=True, null=True) + removed_date = models.DateTimeField(blank=True, null=True) sanction_list_confirmed_match = models.BooleanField(null=True) user_fields = models.JSONField(null=True) child_hoh = models.BooleanField(null=True) fchild_hoh = models.BooleanField(null=True) - disability_certificate_picture = models.ImageField(storage=HopeStorage(), blank=True, null=True) + disability_certificate_picture = models.ImageField(storage=get_hope_storage()(), blank=True, null=True) vector_column = models.TextField(blank=True, null=True) # This field type is a guess. phone_no_alternative_valid = models.BooleanField(blank=True, null=True) phone_no_valid = models.BooleanField(blank=True, null=True) @@ -2065,7 +2062,7 @@ class Program(HopeModel): name = models.TextField(null=True) # This field type is a guess. status = models.CharField(max_length=10, null=True) start_date = models.DateField(null=True) - end_date = models.DateField(blank=True, null=True) + end_date = models.DateField(null=True) description = models.CharField(max_length=255, null=True) budget = models.DecimalField(max_digits=11, decimal_places=2, null=True) frequency_of_payments = models.CharField(max_length=50, null=True) @@ -2105,7 +2102,6 @@ def __str__(self) -> str: class ProgramAdminAreas(HopeModel): - id = models.BigAutoField(primary_key=True) program = models.ForeignKey( Program, on_delete=models.DO_NOTHING, related_name="programadminareas_program", null=True ) @@ -2120,6 +2116,7 @@ class Tenant: class ProgramCycle(HopeModel): + is_removed = models.BooleanField(null=True) id = models.UUIDField(primary_key=True) created_at = models.DateTimeField(null=True) updated_at = models.DateTimeField(null=True) @@ -2500,9 +2497,7 @@ class TargetPoulation(HopeModel): blank=True, null=True, ) - program = models.ForeignKey( - Program, on_delete=models.DO_NOTHING, related_name="targetpoulation_program", blank=True, null=True - ) + program = models.ForeignKey(Program, on_delete=models.DO_NOTHING, related_name="targetpoulation_program", null=True) change_date = models.DateTimeField(blank=True, null=True) finalized_at = models.DateTimeField(blank=True, null=True) business_area = models.ForeignKey( diff --git a/src/hope_country_report/apps/power_query/processors.py b/src/hope_country_report/apps/power_query/processors.py index 4071ef1a..a4afd724 100644 --- a/src/hope_country_report/apps/power_query/processors.py +++ b/src/hope_country_report/apps/power_query/processors.py @@ -21,7 +21,6 @@ from strategy_field.registry import Registry from strategy_field.utils import fqn -from hope_country_report.apps.power_query.storage import HopeStorage from hope_country_report.apps.power_query.utils import ( convert_pdf_to_image_pdf, get_field_rect, @@ -30,6 +29,8 @@ to_dataset, ) +from ..core.storage import get_hope_storage + logger = logging.getLogger(__name__) if TYPE_CHECKING: from .models import Dataset, Formatter, ReportTemplate @@ -318,7 +319,7 @@ def is_special_language_field(self, field_name: str) -> Optional[str]: return None def load_image_from_blob_storage(self, image_path: str) -> BytesIO: - with HopeStorage().open(image_path, "rb") as img_file: + with get_hope_storage().open(image_path, "rb") as img_file: return BytesIO(img_file.read()) diff --git a/src/hope_country_report/apps/power_query/storage.py b/src/hope_country_report/apps/power_query/storage.py deleted file mode 100644 index e280665d..00000000 --- a/src/hope_country_report/apps/power_query/storage.py +++ /dev/null @@ -1,47 +0,0 @@ -import os - -from storages.backends.azure_storage import AzureStorage - - -class ReadOnlyStorageMixin: - def delete(self, name): - raise RuntimeError("This storage cannot delete files") - - def save(self, name, content, max_length=None): - raise RuntimeError("This storage cannot save files") - - def open(self, name, mode="rb"): - if "w" in mode: - raise RuntimeError("This storage cannot open files in write mode") - return super().open(name, mode="rb") - - def listdir(self, path=""): - return [] - - -class HCRAzureStorage(AzureStorage): - prefix = "" - - def get_default_settings(self): - base = super().get_default_settings() - for k, v in base.items(): - if value := os.getenv(f"{self.prefix}_AZURE_{k.upper()}", None): - base[k] = value - return base - - -class HopeStorage(ReadOnlyStorageMixin, HCRAzureStorage): - prefix = "HOPE" - - -class MediaStorage(HCRAzureStorage): - prefix = "MEDIA" - - def get_available_name(self, name: str, max_length: int | None = None) -> str: - if self.exists(name): - self.delete(name) - return name - - -class StaticStorage(HCRAzureStorage): - prefix = "STATIC" diff --git a/src/hope_country_report/config/__init__.py b/src/hope_country_report/config/__init__.py index 0c53e0fc..8330e261 100644 --- a/src/hope_country_report/config/__init__.py +++ b/src/hope_country_report/config/__init__.py @@ -61,9 +61,24 @@ class Group(Enum): "https://django-environ.readthedocs.io/en/latest/types.html#environ-env-db-url", ), "DEBUG": (bool, False, setting("debug")), - "DEFAULT_FILE_STORAGE": ( + "FILE_STORAGE_DEFAULT": ( str, - "hope_country_report.apps.power_query.storage.MediaStorage", + "django.core.files.storage.FileSystemStorage", + setting("storages"), + ), + "FILE_STORAGE_MEDIA": ( + str, + "django.core.files.storage.FileSystemStorage", + setting("storages"), + ), + "FILE_STORAGE_STATIC": ( + str, + "django.contrib.staticfiles.storage.StaticFilesStorage", + setting("storages"), + ), + "FILE_STORAGE_HOPE": ( + str, + "storages.backends.azure_storage.AzureStorage", setting("storages"), ), "EMAIL_BACKEND": (str, "anymail.backends.mailjet.EmailBackend", "Do not change in prod"), @@ -77,11 +92,6 @@ class Group(Enum): "HOPE_AZURE_ACCOUNT_KEY": (str, ""), "HOPE_AZURE_AZURE_CONTAINER": (str, ""), "HOPE_AZURE_SAS_TOKEN": (str, ""), - "HOPE_FILE_STORAGE": ( - str, - "hope_country_report.apps.power_query.storage.HopeStorage", - setting("storages"), - ), "HOST": (str, "http://localhost:8000"), "MAILJET_API_KEY": (str, NOT_SET), "MAILJET_SECRET_KEY": (str, NOT_SET), @@ -91,11 +101,6 @@ class Group(Enum): "MEDIA_AZURE_ACCOUNT_KEY": (str, ""), "MEDIA_AZURE_AZURE_CONTAINER": (str, ""), "MEDIA_AZURE_SAS_TOKEN": (str, ""), - "MEDIA_FILE_STORAGE": ( - str, - "hope_country_report.apps.power_query.storage.MediaStorage", - setting("storages"), - ), "MEDIA_ROOT": (str, "/tmp/media/", setting("media-root")), "MEDIA_URL": (str, "/media/", setting("media-url")), "POWER_QUERY_FLOWER_ADDRESS": (str, "http://localhost:5555", "Flower address"), @@ -117,11 +122,6 @@ class Group(Enum): True, "https://python-social-auth.readthedocs.io/en/latest/configuration/settings.html", ), - "STATIC_FILE_STORAGE": ( - str, - "django.core.files.storage.FileSystemStorage", - setting("storages"), - ), "STATIC_ROOT": (str, "/tmp/static/", setting("static-root")), "STATIC_URL": (str, "/static/", setting("static-url")), "TIME_ZONE": (str, "UTC", setting("std-setting-TIME_ZONE")), diff --git a/src/hope_country_report/config/fragments/storage.py b/src/hope_country_report/config/fragments/storage.py deleted file mode 100644 index 2487d199..00000000 --- a/src/hope_country_report/config/fragments/storage.py +++ /dev/null @@ -1,16 +0,0 @@ -from ..settings import env - -AZURE_OVERWRITE_FILES = True -AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME") -AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY") -AZURE_CONTAINER = env("AZURE_CONTAINER") - -HOPE_AZURE_ACCOUNT_NAME = env("HOPE_AZURE_ACCOUNT_NAME") -HOPE_AZURE_ACCOUNT_KEY = env("HOPE_AZURE_ACCOUNT_KEY") -HOPE_AZURE_AZURE_CONTAINER = env("HOPE_AZURE_AZURE_CONTAINER") -HOPE_AZURE_SAS_TOKEN = env("HOPE_AZURE_SAS_TOKEN") - -MEDIA_AZURE_ACCOUNT_NAME = env("MEDIA_AZURE_ACCOUNT_NAME") -MEDIA_AZURE_ACCOUNT_KEY = env("MEDIA_AZURE_ACCOUNT_KEY") -MEDIA_AZURE_AZURE_CONTAINER = env("MEDIA_AZURE_AZURE_CONTAINER") -MEDIA_AZURE_SAS_TOKEN = env("MEDIA_AZURE_SAS_TOKEN") diff --git a/src/hope_country_report/config/settings.py b/src/hope_country_report/config/settings.py index 9b63ffb2..69d477e2 100644 --- a/src/hope_country_report/config/settings.py +++ b/src/hope_country_report/config/settings.py @@ -33,10 +33,10 @@ MIGRATION_MODULES = {"hope": None} STORAGES = { - "default": env.storage("DEFAULT_FILE_STORAGE"), - "staticfiles": env.storage("STATIC_FILE_STORAGE"), - "media": env.storage("MEDIA_FILE_STORAGE"), - "hope": env.storage("HOPE_FILE_STORAGE"), + "default": env.storage("FILE_STORAGE_DEFAULT"), + "staticfiles": env.storage("FILE_STORAGE_DEFAULT"), + "media": env.storage("FILE_STORAGE_MEDIA"), + "hope": env.storage("FILE_STORAGE_HOPE"), } INSTALLED_APPS = [ "hope_country_report.web", @@ -269,7 +269,6 @@ from .fragments.silk import * # noqa from .fragments.smart_admin import * # noqa from .fragments.social_auth import * # noqa -from .fragments.storage import * # noqa from .fragments.taggit import * # noqa from .fragments.tailwind import * # noqa diff --git a/tests/conftest.py b/tests/conftest.py index bb314bee..e3c30f63 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,9 +80,10 @@ def pytest_configure(config): ADMINS="", ALLOWED_HOSTS="*", AUTHENTICATION_BACKENDS="", - DEFAULT_FILE_STORAGE="django.core.files.storage.FileSystemStorage", - STATIC_FILE_STORAGE="django.core.files.storage.FileSystemStorage", DJANGO_SETTINGS_MODULE="hope_country_report.config.settings", + FILE_STORAGE_DEFAULT="django.core.files.storage.FileSystemStorage", + FILE_STORAGE_MEDIA="", + FILE_STORAGE_HOPE="", CATCH_ALL_EMAIL="", CELERY_TASK_ALWAYS_EAGER="1", CSRF_COOKIE_SECURE="False",