diff --git a/.circleci/config.yml b/.circleci/config.yml index fa5961748..dd7506602 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,7 @@ jobs: export PATH=/home/circleci/.local/bin:$PATH export PYTHONHASHSEED=${RANDOM} pip install tox --user - tox -e py36-d20 + tox -e py36-d20 deps - store_artifacts: path: ~build/coverage @@ -153,6 +153,7 @@ jobs: else echo "Skipped as '$CIRCLE_BRANCH' does not match '$RELEASE_MATCH' branch" fi + dockerize: working_directory: ~/code docker: @@ -170,13 +171,17 @@ jobs: POSTGRES_PASSWORD: postgres environment: - DOCKER_IMAGE: unicef/etools-datamart + DOCKER_IMAGE: unicef/datamart steps: - checkout - add_ssh_keys: fingerprints: - "70:aa:15:2a:d9:1c:74:f3:a1:f7:c4:cd:9b:d2:d1:f5" - setup_remote_docker + - run: + name: Dump Environment + command: | + env | sort - run: name: Build docker image command: | @@ -188,12 +193,14 @@ jobs: --build-arg GITHUB_TOKEN=${GITHUB_TOKEN} \ --build-arg DATAMART_VERSION=${TAG} \ -t ${DOCKER_IMAGE}:${TAG} \ - -f Dockerfile . - docker images + -f docker/Dockerfile . + docker images "unicef/datamart*" - run: name: Test Backend docker image command: | - docker run -p 8000:8000 --rm \ + echo "Testing ${DOCKER_IMAGE}:${TAG}" + docker run -p 8000:8000 \ + --rm \ -e DATABASE_URL=${DATABASE_URL} \ -e DEBUG=0 \ -e SECURE_SSL_REDIRECT=0 \ @@ -206,19 +213,10 @@ jobs: - deploy: name: Push Backend docker image command: | + echo "Pushing ${DOCKER_IMAGE}:${TAG} to Docker Hub" export TODAY=`date '+%d %B %Y at %H:%M'` - git config --global user.email "s.apostolico@gmail.com" - git config --global user.name "CI" - echo "$TODAY Backend $TAG released" | cat - CHANGES > temp && mv temp CHANGES - git add CHANGES - git commit -m "Released Backend v$TAG" - git push - git tag -a -m "backend/$TAG" backend/$TAG - git push --tags - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker tag ${DOCKER_IMAGE}:${TAG} ${DOCKER_IMAGE}:latest docker push ${DOCKER_IMAGE}:latest docker push ${DOCKER_IMAGE}:${TAG} diff --git a/.gitignore b/.gitignore index 21f10abef..81d5d6377 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ db/db1.bz2 .venv docker/cache src/etools_datamart/apps/core/static/api-doc.css.map +docker/superset.db diff --git a/CHANGES b/CHANGES index f432cc154..252f9cb7c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +1.1 +--- +* Fixes wrong import in middleware + 1.0 --- * SystemFilters. Ability to force per user/service filters diff --git a/docker-compose.yml b/docker-compose.yml index f6f4eabd2..c04fa33d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,7 @@ version: '2' services: datamart: - # to use this docker file with local code, set env var TAG=dev and uncomment - # build section + # to use this docker file with local code, set env var TAG=dev build: context: . dockerfile: ./docker/$DOCKERFILE @@ -32,6 +31,10 @@ services: - etools - db - redis + links: + - db:db + - etools:etools + - redis:redis db: image: mdillon/postgis:9.5 @@ -48,10 +51,10 @@ services: redis: image: redis:alpine -# superset: -# image: amancevice/superset -# volumes: -# - $PWD/docker:/etc/superset/ + superset: + image: amancevice/superset + volumes: + - $PWD/docker:/etc/superset/ celery: image: unicef/datamart:$TAG diff --git a/docker/Makefile b/docker/Makefile index 7d30df0d3..1303d0caa 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -8,7 +8,7 @@ DOCKER_IMAGE_NAME=unicef/datamart DOCKERFILE?=Dockerfile PYPI_INDEX?=https://pypi.org/simple/ RUN_OPTIONS?= -TARGET?=0.4 +TARGET?= help: @echo "dev build dev image (based on local code)" @@ -75,10 +75,10 @@ dev-run: CMD=stack $(MAKE) .run test: - CMD='django-admin check --deploy' $(MAKE) run + CMD='bash -c "django-admin check --deploy"' $(MAKE) .run scratch: - RUN_OPTIONS=-it CMD='/bin/bash' $(MAKE) run + RUN_OPTIONS=-it CMD='/bin/bash' $(MAKE) .run shell: docker exec -it ${CONTAINER_NAME} /bin/bash diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f6299b418..dfbdc6692 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -4,7 +4,8 @@ set -e mkdir -p /var/datamart/{static,log,conf,run,redis} rm -f /var/datamart/run/* -if [ "$@" == "celery" ];then + +if [ "$1" == "celery" ];then export START_REDIS="false" export START_DATAMART="false" export START_CELERY="true" @@ -15,7 +16,7 @@ if [ "$@" == "celery" ];then cd /var/datamart exec supervisord --nodaemon -elif [ "$@" == "datamart" ];then +elif [ "$1" == "datamart" ];then django-admin db-isready --wait --timeout 60 django-admin check --deploy @@ -27,7 +28,7 @@ elif [ "$@" == "datamart" ];then else gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi fi -elif [ "$@" == "stack" ];then +elif [ "$1" == "stack" ];then django-admin db-isready --wait --timeout 60 django-admin check --deploy django-admin init-setup --all --verbosity 1 diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index ccbc1c271..37d72c00f 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,3 +1,3 @@ NAME = 'etools-datamart' -VERSION = __version__ = '1.0' +VERSION = __version__ = '1.1' __author__ = '' diff --git a/src/etools_datamart/apps/init/apps.py b/src/etools_datamart/apps/init/apps.py new file mode 100644 index 000000000..8a71164c9 --- /dev/null +++ b/src/etools_datamart/apps/init/apps.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +from django.apps import AppConfig + + +class Config(AppConfig): + name = 'etools_datamart.apps.init' + + def ready(self): + from . import checks # noqa diff --git a/src/etools_datamart/apps/init/checks.py b/src/etools_datamart/apps/init/checks.py new file mode 100644 index 000000000..89f37d552 --- /dev/null +++ b/src/etools_datamart/apps/init/checks.py @@ -0,0 +1,42 @@ +import importlib +import pkgutil + +from django.core.checks import Error, register + + +def check_imports(): + import etools_datamart as package + "" + for importer, modname, ispkg in pkgutil.iter_modules(package.__path__): + current_module = '{}.{}'.format(package.__name__, modname) + m = importlib.import_module(current_module) + if hasattr(m, '__path__'): + for _, sub_mod, __ in pkgutil.iter_modules(m.__path__): + sub_module = '{}.{}'.format(current_module, sub_mod) + sm = importlib.import_module(sub_module) + if hasattr(sm, '__path__'): + for _, s_sub_mod, __ in pkgutil.iter_modules(sm.__path__): + s_sub_mod = '{}.{}.{}'.format(current_module, sub_mod, s_sub_mod) + try: + importlib.import_module(s_sub_mod) + except Exception as e: + raise Exception(f"""Error importing '{s_sub_mod}'. + {e} + """) + + +@register(deploy=True) +def check_importable(app_configs, **kwargs): + errors = [] + try: + check_imports() + except Exception as e: + errors.append( + Error( + str(e), + hint='A hint.', + obj=None, + id='datamart.E001', + ) + ) + return errors diff --git a/src/etools_datamart/apps/init/management/commands/check-deps.py b/src/etools_datamart/apps/init/management/commands/check-deps.py new file mode 100644 index 000000000..8e374fded --- /dev/null +++ b/src/etools_datamart/apps/init/management/commands/check-deps.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +from django.core.management.base import BaseCommand, CommandError + +from etools_datamart.apps.init.checks import check_imports + + +class Command(BaseCommand): + args = '' + help = 'Imports all modules to check for missed dependencies.' + requires_migrations_checks = False + requires_system_checks = False + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + + def add_arguments(self, parser): + super(Command, self).add_arguments(parser) + + def _get_cursor(self, conn): + conn.cursor() + return conn + + def handle(self, *args, **options): + try: + check_imports() + except Exception as e: + raise CommandError(e) diff --git a/src/etools_datamart/apps/tracking/middleware.py b/src/etools_datamart/apps/tracking/middleware.py index e6ba5144b..564cb74a2 100644 --- a/src/etools_datamart/apps/tracking/middleware.py +++ b/src/etools_datamart/apps/tracking/middleware.py @@ -4,7 +4,6 @@ import json import logging -import pytest from django.conf import settings from django.db.models import F from django.utils.timezone import now @@ -17,8 +16,6 @@ logger = logging.getLogger(__name__) -pytestmarker = pytest.mark.django_db - def log_request(**kwargs): log = APIRequestLog.objects.create(**kwargs) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 512f3c536..342cee88d 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -217,7 +217,7 @@ ] INSTALLED_APPS = [ - 'etools_datamart.apps.init', + 'etools_datamart.apps.init.apps.Config', 'etools_datamart.apps.multitenant', 'constance', diff --git a/tox.ini b/tox.ini index 09bf230f7..7da8087f8 100644 --- a/tox.ini +++ b/tox.ini @@ -52,3 +52,9 @@ commands = --cov-report=html \ --cov-config=tests/.coveragerc \ --cov=sir + + +[testenv:deps] +commands = + pipenv sync + pipenv run {toxinidir}/manage.py check --deploy