merge release/1.1

.circleci/config.yml

@@ -37,7 +37,7 @@ jobs:
             export PATH=/home/circleci/.local/bin:$PATH
             export PYTHONHASHSEED=${RANDOM}
             pip install tox --user
-            tox -e py36-d20
+            tox -e py36-d20 deps
 
       - store_artifacts:
           path: ~build/coverage
@@ -153,6 +153,7 @@ jobs:
               else
                 echo "Skipped as '$CIRCLE_BRANCH' does not match '$RELEASE_MATCH' branch"
               fi
+
   dockerize:
     working_directory: ~/code
     docker:
@@ -170,13 +171,17 @@ jobs:
           POSTGRES_PASSWORD: postgres
 
     environment:
-      DOCKER_IMAGE: unicef/etools-datamart
+      DOCKER_IMAGE: unicef/datamart
     steps:
       - checkout
       - add_ssh_keys:
           fingerprints:
             - "70:aa:15:2a:d9:1c:74:f3:a1:f7:c4:cd:9b:d2:d1:f5"
       - setup_remote_docker
+      - run:
+          name: Dump Environment
+          command: |
+            env | sort
       - run:
           name: Build docker image
           command: |
@@ -188,12 +193,14 @@ jobs:
                     --build-arg GITHUB_TOKEN=${GITHUB_TOKEN} \
                     --build-arg DATAMART_VERSION=${TAG} \
                     -t ${DOCKER_IMAGE}:${TAG} \
-                    -f Dockerfile .
-            docker images
+                    -f docker/Dockerfile .
+            docker images "unicef/datamart*"
       - run:
           name: Test Backend docker image
           command: |
-              docker run -p 8000:8000 --rm \
+              echo "Testing ${DOCKER_IMAGE}:${TAG}"
+              docker run -p 8000:8000 \
+                      --rm \
                       -e DATABASE_URL=${DATABASE_URL} \
                       -e DEBUG=0 \
                       -e SECURE_SSL_REDIRECT=0 \
@@ -206,19 +213,10 @@ jobs:
       - deploy:
           name: Push Backend docker image
           command: |
+              echo "Pushing ${DOCKER_IMAGE}:${TAG} to Docker Hub"
               export TODAY=`date '+%d %B %Y at %H:%M'`
 
-              git config --global user.email "[email protected]"
-              git config --global user.name "CI"
-              echo "$TODAY Backend $TAG released" | cat - CHANGES > temp && mv temp CHANGES
-              git add CHANGES
-              git commit -m "Released Backend v$TAG"
-              git push
-              git tag -a -m "backend/$TAG" backend/$TAG
-              git push --tags
-
               docker login -u $DOCKER_USER -p $DOCKER_PASS
-
               docker tag ${DOCKER_IMAGE}:${TAG} ${DOCKER_IMAGE}:latest
               docker push ${DOCKER_IMAGE}:latest
               docker push ${DOCKER_IMAGE}:${TAG}

.gitignore

@@ -29,3 +29,4 @@ db/db1.bz2
 .venv
 docker/cache
 src/etools_datamart/apps/core/static/api-doc.css.map
+docker/superset.db

CHANGES

@@ -1,3 +1,7 @@
+1.1
+---
+* Fixes wrong import in middleware
+
 1.0
 ---
 * SystemFilters. Ability to force per user/service filters

docker-compose.yml

@@ -2,8 +2,7 @@ version: '2'
 
 services:
   datamart:
-  # to use this docker file with local code, set env var TAG=dev and uncomment
-  # build section
+  # to use this docker file with local code, set env var TAG=dev
     build:
       context: .
       dockerfile: ./docker/$DOCKERFILE
@@ -32,6 +31,10 @@ services:
       - etools
       - db
       - redis
+    links:
+      - db:db
+      - etools:etools
+      - redis:redis
 
   db:
     image: mdillon/postgis:9.5
@@ -48,10 +51,10 @@ services:
   redis:
     image: redis:alpine
 
-#  superset:
-#    image: amancevice/superset
-#    volumes:
-#      - $PWD/docker:/etc/superset/
+  superset:
+    image: amancevice/superset
+    volumes:
+      - $PWD/docker:/etc/superset/
 
   celery:
     image: unicef/datamart:$TAG

docker/Makefile

@@ -8,7 +8,7 @@ DOCKER_IMAGE_NAME=unicef/datamart
 DOCKERFILE?=Dockerfile
 PYPI_INDEX?=https://pypi.org/simple/
 RUN_OPTIONS?=
-TARGET?=0.4
+TARGET?=
 
 help:
 	@echo "dev                  build dev image (based on local code)"
@@ -75,10 +75,10 @@ dev-run:
 		CMD=stack $(MAKE) .run
 
 test:
-	CMD='django-admin check --deploy' $(MAKE) run
+	CMD='bash -c "django-admin check --deploy"' $(MAKE) .run
 
 scratch:
-	RUN_OPTIONS=-it CMD='/bin/bash' $(MAKE) run
+	RUN_OPTIONS=-it CMD='/bin/bash' $(MAKE) .run
 
 shell:
 	docker exec -it ${CONTAINER_NAME} /bin/bash

docker/entrypoint.sh

@@ -4,7 +4,8 @@ set -e
 mkdir -p /var/datamart/{static,log,conf,run,redis}
 rm -f /var/datamart/run/*
 
-if [ "$@" == "celery" ];then
+
+if [ "$1" == "celery" ];then
     export START_REDIS="false"
     export START_DATAMART="false"
     export START_CELERY="true"
@@ -15,7 +16,7 @@ if [ "$@" == "celery" ];then
     cd /var/datamart
 
     exec supervisord --nodaemon
-elif [ "$@" == "datamart" ];then
+elif [ "$1" == "datamart" ];then
 
     django-admin db-isready --wait --timeout 60
     django-admin check --deploy
@@ -27,7 +28,7 @@ elif [ "$@" == "datamart" ];then
     else
         gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi
     fi
-elif [ "$@" == "stack" ];then
+elif [ "$1" == "stack" ];then
     django-admin db-isready --wait --timeout 60
     django-admin check --deploy
     django-admin init-setup --all --verbosity 1

src/etools_datamart/__init__.py

@@ -1,3 +1,3 @@
 NAME = 'etools-datamart'
-VERSION = __version__ = '1.0'
+VERSION = __version__ = '1.1'
 __author__ = ''

src/etools_datamart/apps/init/apps.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+from django.apps import AppConfig
+
+
+class Config(AppConfig):
+    name = 'etools_datamart.apps.init'
+
+    def ready(self):
+        from . import checks  # noqa

src/etools_datamart/apps/init/checks.py

@@ -0,0 +1,42 @@
+import importlib
+import pkgutil
+
+from django.core.checks import Error, register
+
+
+def check_imports():
+    import etools_datamart as package
+    ""
+    for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
+        current_module = '{}.{}'.format(package.__name__, modname)
+        m = importlib.import_module(current_module)
+        if hasattr(m, '__path__'):
+            for _, sub_mod, __ in pkgutil.iter_modules(m.__path__):
+                sub_module = '{}.{}'.format(current_module, sub_mod)
+                sm = importlib.import_module(sub_module)
+                if hasattr(sm, '__path__'):
+                    for _, s_sub_mod, __ in pkgutil.iter_modules(sm.__path__):
+                        s_sub_mod = '{}.{}.{}'.format(current_module, sub_mod, s_sub_mod)
+                        try:
+                            importlib.import_module(s_sub_mod)
+                        except Exception as e:
+                            raise Exception(f"""Error importing '{s_sub_mod}'.
+    {e}
+    """)
+
+
+@register(deploy=True)
+def check_importable(app_configs, **kwargs):
+    errors = []
+    try:
+        check_imports()
+    except Exception as e:
+        errors.append(
+            Error(
+                str(e),
+                hint='A hint.',
+                obj=None,
+                id='datamart.E001',
+            )
+        )
+    return errors

src/etools_datamart/apps/init/management/commands/check-deps.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+from django.core.management.base import BaseCommand, CommandError
+
+from etools_datamart.apps.init.checks import check_imports
+
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Imports all modules to check for missed dependencies.'
+    requires_migrations_checks = False
+    requires_system_checks = False
+
+    def __init__(self, *args, **kwargs):
+        super(Command, self).__init__(*args, **kwargs)
+
+    def add_arguments(self, parser):
+        super(Command, self).add_arguments(parser)
+
+    def _get_cursor(self, conn):
+        conn.cursor()
+        return conn
+
+    def handle(self, *args, **options):
+        try:
+            check_imports()
+        except Exception as e:
+            raise CommandError(e)

src/etools_datamart/apps/tracking/middleware.py

@@ -4,7 +4,6 @@
 import json
 import logging
 
-import pytest
 from django.conf import settings
 from django.db.models import F
 from django.utils.timezone import now
@@ -17,8 +16,6 @@
 
 logger = logging.getLogger(__name__)
 
-pytestmarker = pytest.mark.django_db
-
 
 def log_request(**kwargs):
     log = APIRequestLog.objects.create(**kwargs)

src/etools_datamart/config/settings.py

@@ -217,7 +217,7 @@
 ]
 
 INSTALLED_APPS = [
-    'etools_datamart.apps.init',
+    'etools_datamart.apps.init.apps.Config',
     'etools_datamart.apps.multitenant',
 
     'constance',

tox.ini

@@ -52,3 +52,9 @@ commands =
             --cov-report=html \
             --cov-config=tests/.coveragerc \
             --cov=sir
+
+
+[testenv:deps]
+commands =
+    pipenv sync
+    pipenv run {toxinidir}/manage.py check --deploy