Merge pull request #2454 from zimmerman-team/develop

Release

OIPA/api/activity/serializers.py

@@ -1160,11 +1160,7 @@ def update(self, instance, validated_data):
 class ActivitySectorSerializer(ModelSerializerNoValidation):
     id = serializers.HiddenField(default=None)
     sector = SectorSerializer(fields=('url', 'code', 'name'))
-    percentage = serializers.DecimalField(
-        max_digits=5,
-        decimal_places=2,
-        coerce_to_string=False
-    )
+    percentage = serializers.CharField()
     vocabulary = VocabularySerializer()
     vocabulary_uri = serializers.URLField()
 
@@ -1475,11 +1471,7 @@ class ActivityRecipientRegionSerializer(DynamicFieldsModelSerializer):
     region = BasicRegionSerializer(
         fields=('url', 'code', 'name'),
     )
-    percentage = serializers.DecimalField(
-        max_digits=5,
-        decimal_places=2,
-        coerce_to_string=False
-    )
+    percentage = serializers.CharField()
     vocabulary = VocabularySerializer()
     vocabulary_uri = serializers.URLField(required=False)
 
@@ -1601,11 +1593,7 @@ def update(self, instance, validated_data):
 class RecipientCountrySerializer(DynamicFieldsModelSerializer):
     id = serializers.HiddenField(default=None)
     country = CountrySerializer(fields=('url', 'code', 'name'))
-    percentage = serializers.DecimalField(
-        max_digits=5,
-        decimal_places=2,
-        coerce_to_string=False
-    )
+    percentage = serializers.CharField()
     activity = serializers.CharField(write_only=True)
     narrative = NarrativeSerializer(many=True, required=True,
                                     source='narratives')

OIPA/api/activity/tests/test_activities_csv_endpoints.py

@@ -108,7 +108,7 @@ def test_activities_csv_endpoint_data(self):
             self.assertEqual(row[default_csv_headers[1]], sector.sector.code
                              + ';')
             self.assertEqual(row[default_csv_headers[2]],
-                             str(format(sector.percentage, '.2f'))
+                             str(sector.percentage)
                              + ';')
             self.assertEqual(row[default_csv_headers[4]],
                              country.country.code + ';')

OIPA/api/activity/tests/test_serializers.py

@@ -331,7 +331,8 @@ def test_ActivitySectorSerializer(self):
             activity_sector,
             context={'request': self.request_dummy},
         )
-        assert serializer.data['percentage'] == activity_sector.percentage,\
+        assert serializer.data['percentage'] == str(
+            activity_sector.percentage),\
             """
             'activity_sector.percentage' should be serialized to a field
             called percentage
@@ -360,7 +361,8 @@ def test_ActivityRecipientRegionSerializer(self):
         serializer = serializers.ActivityRecipientRegionSerializer(
             recipient_region, context={'request': self.request_dummy})
 
-        assert serializer.data['percentage'] == recipient_region.percentage,\
+        assert serializer.data['percentage'] == str(
+            recipient_region.percentage),\
             """
             'recipient_region.percentage' should be serialized to a field
             called 'percentage'
@@ -405,7 +407,8 @@ def test_RecipientCountrySerializer(self):
             recipient_country,
             context={'request': self.request_dummy}
         )
-        assert serializer.data['percentage'] == recipient_country.percentage,\
+        assert serializer.data['percentage'] == str(
+            recipient_country.percentage),\
             """
             'recipient_country.percentage' should be serialized to a field
             called 'percentage'

OIPA/api/budget/views.py

@@ -1,5 +1,6 @@
 from django.conf import settings
-from django.db.models import Count, F, Sum
+from django.db.models import Count, ExpressionWrapper, F, FloatField, Sum
+from django.db.models.functions import Cast
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_page
 from django_filters.rest_framework import DjangoFilterBackend
@@ -66,10 +67,12 @@ def annotate_currency(query_params, groupings):
     additions = list(set(param_additions).union(grouping_additions))
 
     for percentage_field in additions:
-        percentage_expression = F(percentage_field) / 100.0
+        percentage_expression = Cast(percentage_field,
+                                     output_field=FloatField()) / 100.0
         annotation_components = annotation_components * percentage_expression
 
-    return Sum(annotation_components)
+    return ExpressionWrapper(Sum(annotation_components),
+                             output_field=FloatField())
 
 
 class BudgetAggregations(AggregationView):
@@ -155,20 +158,24 @@ class BudgetAggregations(AggregationView):
         ),
         GroupBy(
             query_param="recipient_region",
-            fields="activity__recipient_region",
+            fields="activity__recipient_region__code",
             renamed_fields="recipient_region",
             queryset=Region.objects.all(),
             serializer=RegionSerializer,
+            serializer_fk='code',
             serializer_fields=('url', 'code', 'name',),
             name_search_field="activity__recipient_region__name",
             renamed_name_search_field="recipient_region_name",
         ),
         GroupBy(
             query_param="sector",
-            fields="budgetsector__sector",
+            fields="budgetsector__sector__code",
             renamed_fields="sector",
             queryset=Sector.objects.all(),
             serializer=SectorSerializer,
+            # though code is not fk, it is used in searching sector code
+            # in Sector model.
+            serializer_fk='code',
             serializer_fields=('url', 'code', 'name'),
             name_search_field="budgetsector__sector__name",
             renamed_name_search_field="sector_name",

OIPA/api/transaction/views.py

@@ -1,5 +1,6 @@
 from django.conf import settings
-from django.db.models import Count, F, Q, Sum
+from django.db.models import Count, ExpressionWrapper, F, FloatField, Q, Sum
+from django.db.models.functions import Cast
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_page
 from django_filters.rest_framework import DjangoFilterBackend
@@ -276,10 +277,12 @@ def annotate_currency(query_params, groupings):
     additions = list(set(param_additions).union(grouping_additions))
 
     for percentage_field in additions:
-        percentage_expression = F(percentage_field) / 100.0
+        percentage_expression = Cast(percentage_field,
+                                     output_field=FloatField()) / 100.0
         annotation_components = annotation_components * percentage_expression
 
-    return Sum(annotation_components)
+    return ExpressionWrapper(Sum(annotation_components),
+                             output_field=FloatField())
 
 
 class TransactionAggregation(AggregationView):
@@ -452,20 +455,22 @@ class TransactionAggregation(AggregationView):
         ),
         GroupBy(
             query_param="recipient_region",
-            fields="transactionrecipientregion__region",
+            fields="transactionrecipientregion__region__code",
             renamed_fields="recipient_region",
             queryset=Region.objects.all(),
             serializer=RegionSerializer,
+            serializer_fk='code',
             serializer_fields=('url', 'code', 'name', 'location'),
             name_search_field="transactionrecipientregion__region__name",
             renamed_name_search_field="recipient_region_name",
         ),
         GroupBy(
             query_param="sector",
-            fields="transactionsector__sector",
+            fields="transactionsector__sector__code",
             renamed_fields="sector",
             queryset=Sector.objects.all(),
             serializer=SectorSerializer,
+            serializer_fk='code',
             serializer_fields=('url', 'code', 'name', 'location'),
             name_search_field="transactionsector__sector__name",
             renamed_name_search_field="sector_name",

OIPA/iati/migrations/0071_auto_20200904_0706.py

@@ -0,0 +1,53 @@
+# Generated by Django 2.0.13 on 2020-09-04 07:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('iati', '0070_auto_20200703_1100'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='activityrecipientcountry',
+            name='percentage',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='activityrecipientregion',
+            name='percentage',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='activitysector',
+            name='percentage',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='budgetitem',
+            name='percentage',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='budgetsector',
+            name='percentage',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='transactionrecipientcountry',
+            name='percentage',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='transactionrecipientregion',
+            name='percentage',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='transactionsector',
+            name='percentage',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

OIPA/iati/models.py

@@ -650,12 +650,7 @@ class ActivitySector(models.Model):
                                    null=True, blank=True,
                                    default=None, on_delete=models.CASCADE)
     vocabulary_uri = models.URLField(null=True, blank=True)
-    percentage = models.DecimalField(
-        max_digits=5,
-        decimal_places=2,
-        null=True,
-        blank=True,
-        default=None)
+    percentage = models.CharField(max_length=100, null=True, blank=True)
     narratives = GenericRelation(
         Narrative,
         content_type_field='related_content_type',
@@ -675,12 +670,7 @@ def get_activity(self):
 class ActivityRecipientCountry(models.Model):
     activity = models.ForeignKey(Activity, on_delete=models.CASCADE)
     country = models.ForeignKey(Country, on_delete=models.CASCADE)
-    percentage = models.DecimalField(
-        max_digits=5,
-        decimal_places=2,
-        null=True,
-        blank=True,
-        default=None)
+    percentage = models.CharField(max_length=100, null=True, blank=True)
     narratives = GenericRelation(
         Narrative,
         content_type_field='related_content_type',
@@ -730,12 +720,7 @@ class BudgetItem(models.Model):
     country_budget_item = models.ForeignKey(CountryBudgetItem,
                                             on_delete=models.CASCADE)
     code = models.ForeignKey(BudgetIdentifier, on_delete=models.CASCADE)
-    percentage = models.DecimalField(
-        max_digits=5,
-        decimal_places=2,
-        null=True,
-        blank=True,
-        default=None)
+    percentage = models.CharField(max_length=100, null=True, blank=True)
 
     def get_activity(self):
         return self.country_budget_item.activity
@@ -763,12 +748,7 @@ class ActivityRecipientRegion(models.Model):
         Narrative,
         content_type_field='related_content_type',
         object_id_field='related_object_id')
-    percentage = models.DecimalField(
-        max_digits=5,
-        decimal_places=2,
-        null=True,
-        blank=True,
-        default=None)
+    percentage = models.CharField(max_length=100, null=True, blank=True)
 
     def __unicode__(self,):
         return "name: %s" % self.region
@@ -1389,9 +1369,7 @@ class BudgetSector(models.Model):
         Sector,
         on_delete=models.CASCADE)
 
-    percentage = models.DecimalField(
-        max_digits=5,
-        decimal_places=2)
+    percentage = models.CharField(max_length=100, null=True, blank=True)
 
     def __unicode__(self, ):
         return "%s - %s" % (self.budget.id, self.sector.code)

OIPA/iati/parser/IATI_2_02.py

@@ -1411,7 +1411,7 @@ def iati_activities__iati_activity__sector(self, element):
                 None,
                 vocabulary.code)
 
-        sector = models.Sector.objects.filter(code=code,
+        sector = models.Sector.objects.filter(code=slugify(code),
                                               vocabulary=vocabulary).first()
 
         if not sector and vocabulary.code == '1':
@@ -1433,6 +1433,7 @@ def iati_activities__iati_activity__sector(self, element):
             if not sector:
                 sector = models.Sector()
                 sector.code = code
+                sector.vocabulary = vocabulary
                 sector.name = 'Vocabulary 99 or 98'
                 sector.description = 'The sector reported corresponds to a sector vocabulary maintained by the reporting organisation for this activity'  # NOQA: E501
                 sector.save()
@@ -2573,7 +2574,7 @@ def iati_activities__iati_activity__transaction__sector(self, element):
                 None,
                 element.attrib.get('vocabulary'))
 
-        sector = models.Sector.objects.filter(code=code,
+        sector = models.Sector.objects.filter(code=slugify(code),
                                               vocabulary=vocabulary).first()
 
         if not sector and vocabulary.code == '1':
@@ -2595,6 +2596,7 @@ def iati_activities__iati_activity__transaction__sector(self, element):
             if not sector:
                 sector = models.Sector()
                 sector.code = code
+                sector.vocabulary = vocabulary
                 sector.name = 'Vocabulary 99 or 98'
                 sector.description = 'The sector reported corresponds to a sector vocabulary maintained by the reporting organisation for this activity'  # NOQA: E501
                 sector.save()

OIPA/iati/parser/IATI_2_03.py

@@ -1460,7 +1460,7 @@ def iati_activities__iati_activity__sector(self, element):
                 None,
                 None)
 
-        sector = models.Sector.objects.filter(code=code,
+        sector = models.Sector.objects.filter(code=slugify(code),
                                               vocabulary=vocabulary).first()
 
         if not sector and vocabulary.code == '1':
@@ -1482,6 +1482,7 @@ def iati_activities__iati_activity__sector(self, element):
             if not sector:
                 sector = models.Sector()
                 sector.code = code
+                sector.vocabulary = vocabulary
                 sector.name = 'Vocabulary 99 or 98'
                 sector.description = 'The sector reported corresponds to a sector vocabulary maintained by the reporting organisation for this activity'  # NOQA: E501
                 sector.save()
@@ -2673,7 +2674,7 @@ def iati_activities__iati_activity__transaction__sector(self, element):
                 None,
                 element.attrib.get('vocabulary'))
 
-        sector = models.Sector.objects.filter(code=code,
+        sector = models.Sector.objects.filter(code=slugify(code),
                                               vocabulary=vocabulary).first()
 
         if not sector and vocabulary.code == '1':
@@ -2694,6 +2695,7 @@ def iati_activities__iati_activity__transaction__sector(self, element):
             if not sector:
                 sector = models.Sector()
                 sector.code = code
+                sector.vocabulary = vocabulary
                 sector.name = 'Vocabulary 99 or 98'
                 sector.description = 'The sector reported corresponds to a sector vocabulary maintained by the reporting organisation for this activity'  # NOQA: E501
                 sector.save()