merge release/2.14

CHANGES

@@ -1,3 +1,10 @@
+2.14
+----
+* Add geonameid to locations endpoint
+* Add pd_url to pd indicator endpoint
+* Set budget values in process_country and not through mapping
+* Fix InterventionBudget loader queryset
+
 2.13
 ----
 * Add new office endpoint

docker/Dockerfile.alpine.base

@@ -11,9 +11,9 @@ ENV TEST ${TEST}
 
 
 RUN apk add --no-cache --virtual .build-deps \
-        --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \
-        --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \
-        --repository http://dl-3.alpinelinux.org/alpine/edge/community/ \
+        --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \
+        --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/testing \
+        --repository http://dl-3.alpinelinux.org/alpine/latest-stable/community/ \
         gcc \
         g++ \
         gdal-dev \

docker/Makefile

@@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?=
 DEVELOP?=1
 DOCKER_PASS?=
 DOCKER_USER?=
-TARGET?=2.13.1a
+TARGET?=2.14.1a
 PUSH_BASE?=1
 BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" )
 #BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" | echo "`xargs`xx" )
@@ -55,7 +55,7 @@ build-test:
 	TARGET=dev $(MAKE) .build test
 
 build:
-	$(MAKE) .build test
+	$(MAKE) .build
 
 
 .run:

src/etools_datamart/__init__.py

@@ -1,7 +1,7 @@
 import warnings
 
 NAME = 'etools-datamart'
-VERSION = __version__ = '2.13'
+VERSION = __version__ = '2.14'
 __author__ = ''
 
 # UserWarning: The psycopg2 wheel package will be renamed from release 2.11;

src/etools_datamart/api/endpoints/datamart/location.py

@@ -26,11 +26,18 @@ class Meta:
 
 
 class LocationSerializer(serializers.ModelSerializer):
+    geonameid = serializers.SerializerMethodField()
+
     class Meta:
         model = models.Location
         exclude = ('schema_name', 'tree_id', 'lft', 'rght', 'level', 'source_id',
                    'geom', 'point', 'latitude', 'longitude')
 
+    def get_geonameid(self, obj):
+        if obj.geoname:
+            return obj.geoname.geoname_id
+        return None
+
 
 class LocationViewSet(common.DataMartViewSet):
     serializer_class = LocationSerializer

src/etools_datamart/apps/mart/data/migrations/0114_auto_20200420_1755.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.9 on 2020-04-20 17:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0113_auto_20200408_1747'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='section',
+            name='name',
+            field=models.CharField(blank=True, max_length=128, null=True),
+        ),
+    ]

src/etools_datamart/apps/mart/data/migrations/0115_pdindicator_pd_url.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.9 on 2020-04-21 13:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0114_auto_20200420_1755'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pdindicator',
+            name='pd_url',
+            field=models.CharField(blank=True, max_length=254, null=True),
+        ),
+    ]

src/etools_datamart/apps/mart/data/migrations/0116_auto_20200417_1734.py

@@ -0,0 +1,37 @@
+# Generated by Django 2.2.11 on 2020-04-17 17:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0115_pdindicator_pd_url'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GeoName',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('toponym_name', models.CharField(max_length=254, null=True)),
+                ('name', models.CharField(max_length=254, null=True)),
+                ('lat', models.FloatField()),
+                ('lng', models.FloatField()),
+                ('geoname_id', models.CharField(max_length=50, null=True)),
+                ('country_code', models.CharField(max_length=20, null=True)),
+                ('country_name', models.CharField(max_length=150, null=True)),
+                ('fcl', models.CharField(max_length=50, null=True)),
+                ('fcode', models.CharField(max_length=50, null=True)),
+                ('distance', models.FloatField(null=True)),
+            ],
+            options={
+                'unique_together': {('lat', 'lng')},
+            },
+        ),
+        migrations.AddField(
+            model_name='location',
+            name='geoname',
+            field=models.ForeignKey(blank=True, null=True, on_delete=models.deletion.DO_NOTHING, to='data.GeoName'),
+        ),
+    ]

src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.9 on 2020-04-21 15:44
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0116_auto_20200417_1734'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='location',
+            name='geoname',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='data.GeoName'),
+        ),
+    ]

src/etools_datamart/apps/mart/data/models/location.py

@@ -1,7 +1,12 @@
+from xml.etree import ElementTree
+
+from django.conf import settings
 from django.contrib.gis.db import models as geomodels
 from django.contrib.gis.db.models.functions import Centroid
 from django.db import connection, models
 
+import requests
+
 from etools_datamart.apps.core.models import DataMartManager, DataMartQuerySet
 from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel
 from etools_datamart.apps.sources.etools.models import LocationsGatewaytype, LocationsLocation
@@ -39,12 +44,29 @@ def batch_update_centroid(self):
         with connection.cursor() as cursor:
             cursor.execute(sql)
 
+        # need to update geoname
+        for record in super().filter(
+                latitude__isnull=False,
+                longitude__isnull=False,
+        ).all():
+            geoname = GeoName.objects.get_or_add(
+                lat=record.latitude,
+                lng=record.longitude,
+            )
+            if record.geoname != geoname:
+                record.geoname = geoname
+                record.save()
+
     def update_centroid(self):
         clone = self._chain()
         for each in clone.annotate(cent=Centroid('geom')):
             each.point = each.cent
             each.latitude = each.point.y
             each.longitude = each.point.x
+            each.geoname = GeoName.objects.get_or_add(
+                lat=each.point.y,
+                lng=each.point.x,
+            )
             each.save()
 
 
@@ -69,6 +91,7 @@ class Location(EtoolsDataMartModel):
     point = geomodels.PointField(blank=True, null=True)
     gateway = models.ForeignKey(GatewayType, models.DO_NOTHING, blank=True, null=True)
     geom = geomodels.MultiPolygonField(blank=True, null=True)
+    geoname = models.ForeignKey("GeoName", models.DO_NOTHING, blank=True, null=True)
     level = models.IntegerField(db_index=True)
     lft = models.IntegerField()
     parent = models.ForeignKey('self', models.DO_NOTHING, blank=True, null=True)
@@ -90,13 +113,87 @@ class Options:
         source = LocationsLocation
         queryset = lambda: LocationsLocation.objects.order_by('-parent')
         last_modify_field = 'modified'
-        exclude_from_compare = ['latitude', 'longitude', 'point']
+        exclude_from_compare = ['latitude', 'longitude', 'point', 'geoname']
         # sync_deleted_records = False
         mapping = {'source_id': 'id',
                    # 'area_code': lambda loader, record: loader.context['country'].business_area_code,
                    'parent': '__self__',
-                   'gateway': GatewayType
+                   'gateway': GatewayType,
                    }
 
     def __str__(self):
         return self.name
+
+
+# store geo names pulled from http://api.geonames.org
+# Using lat and lng get geoname data with
+# http://api.geonames.org/findNearby?lat=47.3&lng=9&username=ntrncic
+class GeoNameManager(models.Manager):
+    def get_or_add(self, lat, lng):
+        # check if we have a matching lat, lng record
+        # if so, return that record
+        # otherwise create a new record based on results
+        # of request to geonames.org
+        if not lat or not lng:
+            return None
+        try:
+            geoname = self.get_queryset().get(lat=lat, lng=lng)
+        except GeoName.DoesNotExist:
+            payload = {
+                "lat": lat,
+                "lng": lng,
+                "username": settings.GEONAMES_USERNAME,
+            }
+            res = requests.get(
+                settings.GEONAMES_URL,
+                params=payload,
+                timeout=settings.REQUEST_TIMEOUT,
+            )
+            geoname = ElementTree.fromstring(res.content)[0]
+            mapping = [
+                ("toponym_name", "toponymName"),
+                ("name", "name"),
+                ("lat", "lat"),
+                ("lng", "lng"),
+                ("geoname_id", "geonameId"),
+                ("country_code", "countryCode"),
+                ("country_name", "countryName"),
+                ("fcl", "fcl"),
+                ("fcode", "fcode"),
+                ("distance", "distance"),
+            ]
+            data = {}
+            for k, f in mapping:
+                try:
+                    data[k] = geoname.find(f).text
+                except AttributeError:
+                    return None
+            lat = data.pop("lat")
+            lng = data.pop("lng")
+            geoname, __ = GeoName.objects.get_or_create(
+                lat=lat,
+                lng=lng,
+                defaults=data,
+            )
+        return geoname
+
+
+class GeoName(models.Model):
+    toponym_name = models.CharField(max_length=254, null=True)
+    name = models.CharField(max_length=254, null=True)
+    lat = models.FloatField()
+    lng = models.FloatField()
+    geoname_id = models.CharField(max_length=50, null=True)
+    country_code = models.CharField(max_length=20, null=True)
+    country_name = models.CharField(max_length=150, null=True)
+    fcl = models.CharField(max_length=50, null=True)
+    fcode = models.CharField(max_length=50, null=True)
+    distance = models.FloatField(null=True)
+
+    objects = GeoNameManager()
+
+    class Meta:
+        unique_together = ('lat', 'lng')
+
+    def __str__(self):
+        return f"{self.name} ({self.lat}, {self.lng})"

src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py

@@ -3,19 +3,24 @@
 from etools_datamart.apps.mart.data.models import Location
 from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel
 from etools_datamart.apps.mart.data.models.intervention import InterventionAbstract, InterventionLoader
-from etools_datamart.apps.mart.data.models.mixins import extend
 from etools_datamart.apps.sources.etools.models import (FundsFundsreservationheader, models,
                                                         PartnersIntervention, PartnersInterventionbudget,)
 
 
 class InterventionBudgetLoader(InterventionLoader):
+    def get_queryset(self):
+        return PartnersInterventionbudget.objects
+
     def process_country(self):
-        qs = PartnersInterventionbudget.objects.all()
-        for record in qs.all():
-            record.intervention.budget = record
+        for record in self.get_queryset().all():
             filters = self.config.key(self, record)
             values = self.get_values(record.intervention)
             values['source_id'] = record.id
+            values['budget_cso_contribution'] = record.partner_contribution_local
+            values['budget_unicef_cash'] = record.unicef_cash_local
+            values['budget_total'] = record.total_local
+            values['budget_currency'] = record.currency
+            values['budget_unicef_supply'] = record.in_kind_amount_local
             op = self.process_record(filters, values)
             self.increment_counter(op)
 
@@ -57,13 +62,4 @@ class Options(InterventionAbstract.Options):
         model = PartnersInterventionbudget
         depends = (Location,)
         key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name,
-                                          source_id=record.intervention.budget.pk)
-
-        mapping = extend(InterventionAbstract.Options.mapping,
-                         dict(
-                             budget_cso_contribution='budget.partner_contribution_local',
-                             budget_unicef_cash='budget.unicef_cash_local',
-                             budget_total='budget.total_local',
-                             budget_currency='budget.currency',
-                             budget_unicef_supply='budget.in_kind_amount_local',
-                         ))
+                                          source_id=record.pk)

src/etools_datamart/apps/mart/data/models/pd_indicator.py

@@ -1,3 +1,5 @@
+from django.shortcuts import reverse
+
 from etools_datamart.apps.mart.data.fields import SafeDecimal
 from etools_datamart.apps.mart.data.loader import EtoolsLoader
 from etools_datamart.apps.mart.data.models import Location
@@ -30,6 +32,12 @@ def process_country(self):
                     op = self.process_record(filters, values)
                     self.increment_counter(op)
 
+    def get_pd_url(self, record: ReportsAppliedindicator, values: dict, **kwargs):
+        return reverse(
+            "api:intervention-detail",
+            args=["latest", record.lower_result.result_link.intervention.pk],
+        )
+
 
 class PDIndicator(LocationMixin, EtoolsDataMartModel):
     context_code = models.CharField(max_length=50, blank=True, null=True)
@@ -73,6 +81,7 @@ class PDIndicator(LocationMixin, EtoolsDataMartModel):
     # from lower_result
     lower_result_name = models.CharField(max_length=500, blank=True, null=True)
     result_link_intervention = models.IntegerField(blank=True, null=True)
+    pd_url = models.CharField(max_length=254, blank=True, null=True)
 
     # from section
     section_name = models.CharField(max_length=45, blank=True, null=True)
@@ -126,6 +135,7 @@ class Options:
             section_name='section.name',
             lower_result_name='lower_result.name',
             result_link_intervention='lower_result.result_link.intervention.pk',
+            pd_url='-',
 
             disaggregation_name='disaggregation.name',
             disaggregation_active='disaggregation.active',

src/etools_datamart/apps/mart/data/models/report_sector.py

@@ -5,7 +5,7 @@
 
 
 class Section(EtoolsDataMartModel):
-    name = models.CharField(max_length=45, blank=True, null=True)
+    name = models.CharField(max_length=128, blank=True, null=True)
     description = models.CharField(max_length=256, blank=True, null=True)
     alternate_id = models.IntegerField(blank=True, null=True)
     alternate_name = models.CharField(max_length=255, blank=True, null=True)