merge release/1.5

CHANGES

@@ -1,3 +1,11 @@
+1.5
+---
+* add .xlsx format support
+* add ability to invalide all cache at startup (set `INVALIDATE_CACHE` env variable)
+* add ability to create default users at startup (set `AUTOCREATE_USERS` env variable)
+* enable Azure login without email
+* add partner.name to Intervention endpoint
+
 1.4.1
 -----
 * fixes dependencies

MANIFEST.in

@@ -22,6 +22,7 @@ exclude Makefile
 exclude manage.py
 exclude manage.py
 exclude tests
+exclude ROADMAP
 
 prune .circleci
 prune db

Pipfile

@@ -34,7 +34,6 @@ pyparsing = "*"
 raven = "*"
 sqlparse = "*"
 whitenoise = "*"
-django-monthfield = "*"
 django-model-utils = "*"
 python-social-auth = "*"
 social-auth-app-django = "*"
@@ -44,6 +43,7 @@ cryptography = "*"
 "django-rest-framework-social-oauth2" = "*"
 django-countries = "*"
 django-filter = "*"
+drf-renderer-xlsx = "*"
 
 [dev-packages]
 "flake8" = ">=3.6.0"
@@ -62,7 +62,6 @@ pytest-coverage = "*"
 pytest-django = "*"
 pytest-echo = "*"
 pytest-pythonpath = "*"
-#tox-pipenv = "==1.6.0"
 yapf = "*"
 vcrpy = "*"
 

ROADMAP

@@ -0,0 +1 @@
+Features

src/etools_datamart/__init__.py

@@ -1,3 +1,3 @@
 NAME = 'etools-datamart'
-VERSION = __version__ = '1.4.1'
+VERSION = __version__ = '1.5'
 __author__ = ''

src/etools_datamart/api/endpoints/common.py

@@ -7,6 +7,7 @@
 from django.db import connections
 from django.http import Http404
 from drf_querystringfilter.exceptions import QueryFilterException
+from drf_renderer_xlsx.renderers import XLSXRenderer
 from dynamic_serializer.core import DynamicSerializerMixin
 from rest_framework.exceptions import NotAuthenticated, PermissionDenied
 from rest_framework.filters import OrderingFilter
@@ -59,6 +60,7 @@ class APIReadOnlyModelViewSet(ReadOnlyModelViewSet):
     renderer_classes = [JSONRenderer,
                         APIBrowsableAPIRenderer,
                         CSVRenderer,
+                        XLSXRenderer,
                         ]
     filter_backends = [SystemFilterBackend,
                        DatamartQueryStringFilterBackend,

src/etools_datamart/api/filtering.py

@@ -15,63 +15,6 @@
           'oct', 'nov', 'dec']
 
 
-# class SchemaFilterBackend(BaseFilterBackend):
-#     @lru_cache(100)
-#     def get_schema_fields(self, view):
-#         return [coreapi.Field(
-#             name='country_name',
-#             required=False,
-#             location='query',
-#             schema=coreschema.String(
-#                 title='country_name',
-#                 description="""case insensitive, comma separated list of country names<br/>
-# {c}
-# """.format(c=", ".join([c.name for c in connections['etools'].get_tenants()]))
-#             )
-#         )]
-#
-#     def filter_queryset(self, request, queryset, view):
-#         value = request.GET.get('country_name', None)
-#         assert queryset.model._meta.app_label == 'etools'
-#         conn = connections['etools']
-#         if not value:
-#             if request.user.is_superuser:
-#                 conn.set_all_schemas()
-#             else:
-#                 allowed = get_etools_allowed_schemas(request.user)
-#                 if not allowed:
-#                     raise PermissionDenied("You don't have enabled schemas")
-#                 conn.set_schemas(get_etools_allowed_schemas(request.user))
-#         else:
-#             value = set(value.split(","))
-#             validate_schemas(*value)
-#             if not request.user.is_superuser:
-#                 user_schemas = get_etools_allowed_schemas(request.user)
-#                 if not user_schemas.issuperset(value):
-#                     raise NotAuthorizedSchema(",".join(sorted(value - user_schemas)))
-#             conn.set_schemas(value)
-#         return queryset
-# class CountryFilterBackend(SchemaFilterBackend):
-#     def filter_queryset(self, request, queryset, view):
-#         value = request.GET.get('country_name', None)
-#         assert queryset.model._meta.app_label != 'etools'
-#         if not value:
-#             if not request.user.is_superuser:
-#                 allowed = get_etools_allowed_schemas(request.user)
-#                 if not allowed:
-#                     raise PermissionDenied("You don't have enabled schemas")
-#                 queryset.filter(country_name__in=allowed)
-#         else:
-#             value = set(value.split(","))
-#             validate_schemas(*value)
-#             if not request.user.is_superuser:
-#                 user_schemas = get_etools_allowed_schemas(request.user)
-#                 if not user_schemas.issuperset(value):
-#                     raise NotAuthorizedSchema(",".join(sorted(value - user_schemas)))
-#             queryset.filter(country_name__in=value)
-#         return queryset
-
-
 class CountryNameProcessor:
     def process_country_name(self, efilters, eexclude, field, value, request,
                              op, param, negate, **payload):
@@ -99,16 +42,6 @@ def process_country_name(self, efilters, eexclude, field, value, request,
 
 class CountryNameProcessorTenantModel(CountryNameProcessor):
     pass
-    # def process_country_name(self, filters, exclude, field, value, request, **payload):
-    #     _f, _e = super().process_country_name({}, {}, field, value, request, **payload)
-    #     # assert queryset.model._meta.app_label == 'etools'
-    #     conn = connections['etools']
-    #     if 'country_name__iregex' in _f:
-    #         conn.set_schemas(_f['country_name__iregex'])
-    #     else:
-    #         conn.set_all_schemas()
-    #
-    #     return filters, exclude
 
 
 class MonthProcessor:

src/etools_datamart/apps/data/admin.py

@@ -123,6 +123,6 @@ class FAMIndicatorAdmin(DataModelAdmin):
 @register(models.UserStats)
 class UserStatsAdmin(DataModelAdmin):
     list_display = ('country_name', 'schema_name', 'month', 'total', 'unicef', 'logins', 'unicef_logins')
-    list_filter = (SchemaFilter, 'month',)
+    list_filter = (SchemaFilter, 'month')
     load_handler = load_user_report
     date_hierarchy = 'month'

src/etools_datamart/apps/data/migrations/0002_intervention_partner_name.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.3 on 2018-11-06 13:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='intervention',
+            name='partner_name',
+            field=models.CharField(max_length=200, null=True),
+        ),
+    ]

src/etools_datamart/apps/data/models/intervention.py

@@ -34,6 +34,7 @@ class Intervention(DataMartModel):
     unicef_signatory_last_name = models.CharField(max_length=30, null=True)
     unicef_signatory_email = models.CharField(max_length=254, null=True)
 
+    partner_name = models.CharField(max_length=200, null=True)
     partner_signatory_title = models.CharField(max_length=64, null=True)
     partner_signatory_first_name = models.CharField(max_length=64, null=True)
     partner_signatory_last_name = models.CharField(max_length=64, null=True)

src/etools_datamart/apps/etl/tasks/etl.py

@@ -104,7 +104,7 @@ def load_intervention():
                                         end_date=record.end,
                                         review_date_prc=record.review_date_prc,
                                         prc_review_document=record.prc_review_document,
-
+                                        partner_name=record.agreement.partner.name,
                                         agreement_id=record.agreement.pk,
                                         partner_authorized_officer_signatory_id=get_attr(record,
                                                                                          'partner_authorized_officer_signatory.pk'),

src/etools_datamart/apps/init/management/commands/init-setup.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import warnings
 
 from django.conf import settings
 from django.contrib.auth import get_user_model
@@ -11,6 +12,7 @@
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from humanize import naturaldelta
 from strategy_field.utils import fqn
+from unicef_rest_framework.models.acl import GroupAccessControl
 
 from etools_datamart.apps.etl.models import TaskLog
 from etools_datamart.celery import app
@@ -83,6 +85,7 @@ def handle(self, *args, **options):
                                                                               "is_staff": True,
                                                                               "password": make_password(pwd)})
         Group.objects.get_or_create(name='Guests')
+        all_access, __ = Group.objects.get_or_create(name='Endpoints all access')
 
         if created:  # pragma: no cover
             self.stdout.write(f"Created superuser `{admin}` with password `{pwd}`")
@@ -92,6 +95,35 @@ def handle(self, *args, **options):
         from unicef_rest_framework.models import Service
         created, deleted, total = Service.objects.load_services()
         self.stdout.write(f"{total} services found. {created} new. {deleted} deleted")
+        if os.environ.get('INVALIDATE_CACHE'):
+            Service.objects.invalidate_cache()
+
+        for service in Service.objects.all():
+            GroupAccessControl.objects.get_or_create(
+                group=all_access,
+                service=service,
+                serializers=['*'],
+                policy=GroupAccessControl.POLICY_ALLOW
+            )
+
+        if os.environ.get('AUTOCREATE_USERS'):
+            self.stdout.write("Found 'AUTOCREATE_USERS' environment variable")
+            self.stdout.write("Going to create new users")
+            try:
+                for entry in os.environ.get('AUTOCREATE_USERS').split(','):
+                    user, pwd = entry.split('|')
+                    User = get_user_model()
+                    u, created = User.objects.get_or_create(username=user)
+                    if created:
+                        self.stdout.write(f"Created user {u}")
+                        u.set_password(pwd)
+                        u.save()
+                        u.groups.add(all_access)
+                    else:
+                        self.stdout.write(f"User {u} already exists.")
+
+            except Exception as e:
+                warnings.warn(f"Unable to create default users. {e}")
 
         if options['tasks'] or _all or options['refresh']:
             midnight, __ = CrontabSchedule.objects.get_or_create(minute=0, hour=0)

src/etools_datamart/apps/web/templates/base.html

@@ -4,6 +4,7 @@
     <meta charset="UTF-8">
     <title>Title</title>
     {% block head %}
+        <link rel="icon" type="image/ico" href="{% static 'favicon.ico' %}">
         <link rel="stylesheet" type="text/css" href="{% static 'style.css' %}?_={% version %}">
     {% endblock head %}
 

src/etools_datamart/config/settings.py

@@ -362,7 +362,7 @@
 AZURE_URL_EXPIRATION_SECS = 10800
 AZURE_ACCESS_POLICY_EXPIRY = 10800  # length of time before signature expires in seconds
 AZURE_ACCESS_POLICY_PERMISSION = 'r'
-AZURE_TOKEN_URL = 'https://login.microsoftonline.com/unicef.org/oauth2/token'
+AZURE_TOKEN_URL = 'https://login.microsoftonline.com/saxix.onmicrosoft.com/oauth2/token'
 AZURE_GRAPH_API_BASE_URL = 'https://graph.microsoft.com'
 AZURE_GRAPH_API_VERSION = 'v1.0'
 AZURE_GRAPH_API_PAGE_SIZE = 300
@@ -389,22 +389,29 @@
 }
 
 # social auth
+# WARNINGS: UNICEF pipeline does not work if other provider
+# are added to UNICEF AD. Dio not change below settings
+#
 SOCIAL_AUTH_POSTGRES_JSONFIELD = True
-SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL = True
-SOCIAL_AUTH_PROTECTED_USER_FIELDS = ['email']
+SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL = False
+SOCIAL_AUTH_PROTECTED_USER_FIELDS = ['username']
 SOCIAL_AUTH_SANITIZE_REDIRECTS = False
 SOCIAL_AUTH_URL_NAMESPACE = 'social'
-SOCIAL_AUTH_WHITELISTED_DOMAINS = ['unicef.org', 'google.com']
+SOCIAL_AUTH_WHITELISTED_DOMAINS = ['unicef.org', ]
 SOCIAL_AUTH_PIPELINE = (
     'social_core.pipeline.social_auth.social_details',
-    'social_core.pipeline.social_auth.social_uid',
-    'social_core.pipeline.social_auth.social_user',
-    'social_core.pipeline.user.get_username',
-    'social_core.pipeline.user.create_user',
-    'social_core.pipeline.social_auth.associate_user',
-    'social_core.pipeline.social_auth.load_extra_data',
-    'social_core.pipeline.user.user_details',
-    'social_core.pipeline.social_auth.associate_by_email',
+    'unicef_security.azure.get_unicef_user',
+    # 'unicef_security.azure.social_uid',
+    # 'social_core.pipeline.social_auth.social_uid',
+    # 'social_core.pipeline.social_auth.social_user',
+    # 'social_core.pipeline.user.get_username',
+    # 'social_core.pipeline.user.create_user',
+    # 'unicef_security.azure.get_username',
+    # 'unicef_security.azure.create_user',
+    # 'social_core.pipeline.social_auth.associate_user',
+    # 'social_core.pipeline.social_auth.load_extra_data',
+    # 'social_core.pipeline.user.user_details',
+    # 'social_core.pipeline.social_auth.associate_by_email',
     'unicef_security.azure.default_group',
 )