merge release/2.2

unicef · Sep 19, 2019 · e388de3 · e388de3
2 parents 510d2e0 + 742f286
commit e388de3
Show file tree

Hide file tree

Showing 27 changed files with 486 additions and 112 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.1
+current_version = 2.2
 commit = False
 tag = False
 allow_dirty = True

diff --git a/CHANGES b/CHANGES
@@ -1,3 +1,9 @@
+2.2
+---
+* fixes InterventionBudgetLoader
+* admin improvements
+* improves indexing
+
 2.1
 ---
 * Fixes TripLoader

diff --git a/docker/Makefile b/docker/Makefile
@@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?=
 DEVELOP?=1
 DOCKER_PASS?=
 DOCKER_USER?=
-TARGET?=2.1
+TARGET?=2.2
 BASE=2.0
 # below vars are used internally
 BUILD_OPTIONS?=--squash
@@ -18,7 +18,7 @@ DOCKERFILE?=Dockerfile
 RUN_OPTIONS?=
 PIPENV_ARGS?=
 PORTS=
-ABSOLUTE_BASE_URL?="http://192.168.66.66:8000"
+ABSOLUTE_BASE_URL?="http://192.268.66.66:8000"
 
 help:
 	@echo "dev                  build dev image (based on local code)"
@@ -71,12 +71,12 @@ build_old:
 	cd .. && docker run \
 	 		--rm \
 			-e ABSOLUTE_BASE_URL=${ABSOLUTE_BASE_URL} \
-			-e CACHE_URL=redis://192.168.66.66:6379/1 \
-			-e CACHE_URL_API=redis://192.168.66.66:6379/2 \
-			-e CACHE_URL_LOCK=redis://192.168.66.66:6379/3 \
-			-e CACHE_URL_TEMPLATE=redis://192.168.66.66:6379/4 \
-			-e CELERY_BROKER_URL=redis://192.168.66.66:6379/2 \
-			-e CELERY_RESULT_BACKEND=redis://192.168.66.66:6379/3 \
+			-e CACHE_URL=redis://192.268.66.66:6379/1 \
+			-e CACHE_URL_API=redis://192.268.66.66:6379/2 \
+			-e CACHE_URL_LOCK=redis://192.268.66.66:6379/3 \
+			-e CACHE_URL_TEMPLATE=redis://192.268.66.66:6379/4 \
+			-e CELERY_BROKER_URL=redis://192.268.66.66:6379/2 \
+			-e CELERY_RESULT_BACKEND=redis://192.268.66.66:6379/3 \
 			-e CSRF_COOKIE_SECURE=false \
 			-e DATABASE_URL=${DATABASE_URL} \
 			-e DATABASE_URL_ETOOLS=${DATABASE_URL_ETOOLS} \

diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py
@@ -1,7 +1,7 @@
 import warnings
 
 NAME = 'etools-datamart'
-VERSION = __version__ = '2.1'
+VERSION = __version__ = '2.2'
 __author__ = ''
 
 # UserWarning: The psycopg2 wheel package will be renamed from release 2.8;

diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py
@@ -292,8 +292,9 @@ class EtoolsUserAdmin(DataModelAdmin):
 
 @register(models.InterventionBudget)
 class InterventionBudgetAdmin(DataModelAdmin):
-    list_display = ('title', 'number',
-                    'budget_cso_contribution', 'budget_unicef_cash')
+    list_display = ('source_id', 'schema_name',
+                    'reference_number', 'agreement_reference_number')
+    search_fields = ('reference_number', 'agreement_reference_number')
 
 
 @register(models.Office)
@@ -317,7 +318,7 @@ class TripAdmin(DataModelAdmin):
     list_display = ('reference_number', 'traveler_name',
                     'partner_name', 'vendor_number', 'end_date',)
     list_filter = ('start_date', 'end_date')
-    search_fields = ('reference_number', )
+    search_fields = ('reference_number',)
 
 
 @register(models.Engagement)

diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py
@@ -17,7 +17,14 @@
 
 
 class EToolsLoaderOptions(BaseLoaderOptions):
-    pass
+    DEFAULT_KEY = lambda loader, record: dict(schema_name=loader.context['country'].schema_name,
+                                              source_id=record.pk)
+
+
+#     def __init__(self, base=None):
+#         super().__init__(base)
+#         self.key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name,
+#                                                source_id=record.pk)
 
 
 class EtoolsLoader(BaseLoader):
@@ -140,7 +147,6 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor
                             stdout.flush()
                         self.post_process_country()
                         if self.config.sync_deleted_records(self):
-                            cache.set("STATUS:%s" % self.etl_task.task, '[remove deleted]')
                             self.remove_deleted()
                     if stdout and verbosity > 0:
                         stdout.write("\n")
@@ -161,8 +167,9 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor
             raise
         else:
             self.on_end(None)
+            cache.set("STATUS:%s" % self.etl_task.task, "completed - %s" % self.results.processed)
         finally:
-            cache.delete("STATUS:%s" % self.etl_task.task)
+            cache.set("STATUS:%s" % self.etl_task.task, "error")
             if lock:  # pragma: no branch
                 try:
                     lock.release()
@@ -171,6 +178,10 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor
         return self.results
 
 
+class CommonSchemaLoaderOptions(BaseLoaderOptions):
+    DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk)
+
+
 class CommonSchemaLoader(EtoolsLoader):
     def get_mart_values(self, record=None):
         ret = {'seen': self.context['today']}

diff --git a/src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py b/src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py
@@ -0,0 +1,13 @@
+# Generated by Django 2.2.5 on 2019-09-19 10:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0093_auto_20190917_0603'),
+    ]
+
+    operations = [
+    ]
diff --git a/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py b/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py
@@ -0,0 +1,42 @@
+
+# Generated by Django 2.2.5 on 2019-09-19 10:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0094_RELEASE_2_1'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='attachment',
+            index=models.Index(fields=['source_id', 'schema_name'], name='data_attach_source__529f94_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='engagement',
+            index=models.Index(fields=['source_id', 'schema_name'], name='data_engage_source__85fc81_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='grant',
+            index=models.Index(fields=['source_id', 'schema_name'], name='data_grant_source__df511a_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='partnerstaffmember',
+            index=models.Index(fields=['source_id', 'schema_name'], name='data_partne_source__9d46b3_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='section',
+            index=models.Index(fields=['source_id', 'schema_name'], name='data_sectio_source__565f46_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='tpmactivity',
+            index=models.Index(fields=['source_id', 'schema_name'], name='data_tpmact_source__2a9531_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='tpmvisit',
+            index=models.Index(fields=['source_id', 'schema_name'], name='data_tpmvis_source__90b361_idx'),
+        ),
+    ]
diff --git a/src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py b/src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py
@@ -0,0 +1,138 @@
+# Generated by Django 2.2.5 on 2019-09-19 16:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0095_auto_20190919_1016'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='actionpoint',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='agreement',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='attachment',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='engagement',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='famindicator',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='fundsreservation',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='gatewaytype',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='grant',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='hact',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='hacthistory',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='intervention',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='interventionbudget',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='interventionbylocation',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='location',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='office',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='partner',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='partnerstaffmember',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='pdindicator',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='pmpindicators',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='reportindicator',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='section',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='travel',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='travelactivity',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='trip',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='userstats',
+            name='country_name',
+            field=models.CharField(max_length=100),
+        ),
+    ]
diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py
@@ -5,7 +5,8 @@
 
 from celery.local import class_property
 
-from etools_datamart.apps.data.loader import EtoolsLoader, EToolsLoaderOptions
+from etools_datamart.apps.data.loader import (CommonSchemaLoader, CommonSchemaLoaderOptions,
+                                              EtoolsLoader, EToolsLoaderOptions,)
 from etools_datamart.apps.etl.base import DataMartModelBase
 
 
@@ -40,12 +41,17 @@ def truncate(self, reset=True):
                                                                       restart))
 
 
-class EToolsDataMartModelBase(DataMartModelBase):
+class CommonDataMartModelModelBase(DataMartModelBase):
+    loader_option_class = CommonSchemaLoaderOptions
+    loader_class = CommonSchemaLoader
+
+
+class EToolsDataMartModelBase(CommonDataMartModelModelBase):
     loader_option_class = EToolsLoaderOptions
     loader_class = EtoolsLoader
 
 
-class CommonDataMartModel(models.Model, metaclass=DataMartModelBase):
+class CommonDataMartModel(models.Model, metaclass=CommonDataMartModelModelBase):
     source_id = models.IntegerField(blank=True, null=True, db_index=True)
     last_modify_date = models.DateTimeField(blank=True, auto_now=True)
     seen = models.DateTimeField(blank=True, null=True)
@@ -65,11 +71,14 @@ def linked_services(self):
 
 
 class EtoolsDataMartModel(CommonDataMartModel, metaclass=EToolsDataMartModelBase):
-    country_name = models.CharField(max_length=100, db_index=True)
+    country_name = models.CharField(max_length=100)
     schema_name = models.CharField(max_length=63, db_index=True)
     area_code = models.CharField(max_length=10, db_index=True)
 
     class Meta:
         abstract = True
+        indexes = [
+            models.Index(fields=['source_id', 'schema_name']),
+        ]
 
     objects = DataMartManager()
diff --git a/src/etools_datamart/apps/data/models/fam.py b/src/etools_datamart/apps/data/models/fam.py
@@ -69,6 +69,3 @@ class Options:
         source = AuditEngagement
         sync_deleted_records = lambda loader: False
         # mapping = dict(source_id='engagement_ptr_id')
-        # key = lambda loader, record: dict(country_name=loader.context['country'].name,
-        #                                   schema_name=loader.context['country'].schema_name,
-        #                                   source_id=record.engagement_ptr.id)
diff --git a/src/etools_datamart/apps/data/models/hact.py b/src/etools_datamart/apps/data/models/hact.py
@@ -70,6 +70,3 @@ class Options:
         source = HactAggregatehact
         sync_deleted_records = lambda loader: False
         truncate = False
-        # key = lambda loader, record: dict(country_name=loader.context['country'].name,
-        #                                   schema_name=loader.context['country'].schema_name,
-        #                                   year=loader.context['today'].year)
diff --git a/src/etools_datamart/apps/data/models/intervention.py b/src/etools_datamart/apps/data/models/intervention.py
@@ -193,7 +193,8 @@ def get_unicef_focal_points(self, original: PartnersIntervention, values: dict,
 
 
 class InterventionAbstract(models.Model):
-    agreement_reference_number = models.CharField(max_length=300, blank=True, null=True)
+    agreement_reference_number = models.CharField(max_length=300,
+                                                  blank=True, null=True)
     amendment_types = models.TextField(blank=True, null=True)
     attachment_types = models.TextField(blank=True, null=True)
     agreement_id = models.IntegerField(blank=True, null=True)