Merge pull request #238 from unicef-drp/investigate-ephemeral-storage

Investigate ephemeral storage

django_project/core/admin.py

@@ -99,6 +99,11 @@ class SitePreferencesAdmin(admin.ModelAdmin):
             'fields': (
                 'login_help_text',
             )
+        }),
+        ('Logging', {
+            'fields': (
+                'ephemeral_paths',
+            )
         })
     )
     inlines = (SitePreferencesImageInline,)

django_project/core/middleware/__init__.py

@@ -1 +1,2 @@
-from .VersionMiddleware import VersionMiddleware
+from .version import VersionMiddleware
+from .ephemeral import EphemeralMiddleware

django_project/core/middleware/ephemeral.py

@@ -0,0 +1,42 @@
+import subprocess
+from sentry_sdk import capture_message, configure_scope
+
+from core.models import SitePreferences
+
+
+def du(paths):
+    """disk usage for each path"""
+    cmd = ['du', '-sc']
+    cmd.extend(paths)
+    bytes_arr = subprocess.check_output(cmd).splitlines()
+    return [b.decode('utf-8') for b in bytes_arr]
+
+
+class EphemeralMiddleware:
+    """Add Ephemeral information after API call."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        pref = SitePreferences.preferences()
+        if not pref.ephemeral_paths:
+            return response
+
+        resolver_match = getattr(request, 'resolver_match', None)
+        possible_versions = []
+        if resolver_match and resolver_match.namespace:
+            possible_versions = resolver_match.namespace.split(':')
+
+        # check for layer tiles path
+        is_layer_tiles = '/layer_tiles/' in request.path
+        if possible_versions or is_layer_tiles:
+            with configure_scope() as scope:
+                sizes = du(pref.ephemeral_paths)
+                for size in sizes:
+                    splits = size.split()
+                    if len(splits) == 2:
+                        scope.set_extra(splits[1], splits[0])
+                capture_message('Ephemeral Storage Event')
+        return response

django_project/core/migrations/0033_sitepreferences_ephemeral_paths.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.7 on 2024-10-07 11:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0032_sitepreferences_login_help_text'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sitepreferences',
+            name='ephemeral_paths',
+            field=models.JSONField(blank=True, default=list, help_text='List of path to monitor the size and send to sentry.', null=True),
+        ),
+    ]

django_project/core/models/preferences.py

@@ -283,6 +283,15 @@ class SitePreferences(SingletonModel):
         ),
     )
 
+    ephemeral_paths = models.JSONField(
+        default=list,
+        blank=True,
+        null=True,
+        help_text=(
+            'List of path to monitor the size and send to sentry.'
+        )
+    )
+
     class Meta:  # noqa: D106
         verbose_name_plural = "site preferences"
 

django_project/core/settings/base.py

@@ -71,7 +71,8 @@
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
-    'core.middleware.VersionMiddleware'
+    'core.middleware.VersionMiddleware',
+    'core.middleware.EphemeralMiddleware'
 ]
 
 ROOT_URLCONF = 'core.urls'