merge release/1.6

.circleci/config.yml

@@ -38,7 +38,13 @@ jobs:
             export PYTHONHASHSEED=${RANDOM}
             pip install tox --user
             tox -e py36-d21 deps
-
+      - run:
+          name: codecov
+          command: |
+            PATH=$HOME/.local/bin:$PATH
+            pip install --user codecov
+            coverage xml
+            ~/.local/bin/codecov --required -X search gcov pycov -f coverage.xml --flags $CIRCLE_JOB
       - store_artifacts:
           path: ~build/coverage
           destination: coverage
@@ -188,7 +194,7 @@ jobs:
             fi
             docker build  \
                     --build-arg GITHUB_TOKEN=${GITHUB_TOKEN} \
-                    --build-arg DATAMART_VERSION=${TAG} \
+                    --build-arg VERSION=${TAG} \
                     -t ${DOCKER_IMAGE}:${TAG} \
                     -f docker/Dockerfile .
             docker images "unicef/datamart*"

.dockerignore

@@ -1,7 +1,5 @@
-.git
-.tox
-.venv
-~build
-.pytest_cache
-.circleci
+.*
+~*
 db
+docs
+tests

CHANGES

@@ -1,3 +1,14 @@
+1.6
+---
+* add ability to invalidate cache directly from admin endpoint
+* add ability to queue task from task log
+* invalidate cache on dataset refresh
+* add RedisBoard to monitor cache entries
+* new HACT endpoint
+* add Microsoft style JSON output
+* add Microsoft style XML output
+
+
 1.5
 ---
 * add .xlsx format support
@@ -6,10 +17,12 @@
 * enable Azure login without email
 * add partner.name to Intervention endpoint
 
+
 1.4.1
 -----
 * fixes dependencies
 
+
 1.4
 ---
 * WARNINGS: migration reset

Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-"psycopg2-binary" = "*"
+psycopg2 = "*"
 admin-extra-urls = ">=2.1"
 celery = "*"
 coreapi = "*"
@@ -27,7 +27,6 @@ djangorestframework-jwt = "*"
 drf-dynamic-serializer = ">=1.2.0"
 drf-extensions = "*"
 drf-yasg = {version = "*", extras = ["validation"]}
-flower = "*"
 gunicorn = "*"
 humanize = "*"
 pyparsing = "*"
@@ -44,6 +43,9 @@ cryptography = "*"
 django-countries = "*"
 django-filter = "*"
 drf-renderer-xlsx = "*"
+django-redisboard = "*"
+djangorestframework-xml = "*"
+redis = "==2.10.6"
 
 [dev-packages]
 "flake8" = ">=3.6.0"

README.md

@@ -3,9 +3,9 @@ eTools DataMart
 
 [![CircleCI](https://circleci.com/gh/unicef/etools-datamart/tree/develop.svg?style=svg&circle-token=)](https://circleci.com/gh/unicef/etools-datamart/tree/develop)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/819135a936894e678066e895604fd24f)](https://www.codacy.com/app/UNICEF/etools-datamart?utm_source=github.com&utm_medium=referral&utm_content=unicef/etools-datamart&utm_campaign=Badge_Grade)
+[![codecov](https://codecov.io/gh/unicef/etools-datamart/branch/develop/graph/badge.svg)](https://codecov.io/gh/unicef/etools-datamart)
 [![](https://images.microbadger.com/badges/version/unicef/datamart.svg)](https://hub.docker.com/r/unicef/datamart/)
 
-
 UNICEF eTools API and Datamart
 
 

db/update_etools_schema.sh

@@ -16,6 +16,7 @@ export PGHOST=127.0.0.1
 export PGPORT=5432
 export DATABASE_NAME=etools
 export DATABASE_USER=postgres
+export DATABASE_PASS=
 export BASE_SCHEMA=kenya
 
 help (){
@@ -30,6 +31,11 @@ help (){
     echo "  -ni,--no-inspect    do not inspect schema"
     echo "  -ns,--no-summary    do not display summary infos"
     echo "  -nc,--no-clean      do not clean temporary files"
+    echo "  --host              database host"
+    echo "  --port              database port"
+    echo "  --db-name           database name"
+    echo "  --db-user           database username"
+    echo "  --db-pass           database password"
     echo "  -h,--help           this help screen"
     exit 1
 }
@@ -57,6 +63,31 @@ case $1 in
         shift
         shift
         ;;
+    --host)
+        shift
+        PGHOST=$1
+        shift
+        ;;
+    --port)
+        shift
+        PGPORT=$1
+        shift
+        ;;
+    --db-name)
+        shift
+        DATABASE_NAME=$1
+        shift
+        ;;
+    --db-user)
+        shift
+        DATABASE_USER=$1
+        shift
+        ;;
+    --db-pass)
+        shift
+        DATABASE_PASS=$1
+        shift
+        ;;
     -nr|--no-restore)
         RESTORE=0
         shift
@@ -101,6 +132,7 @@ echo "create tests data files    $DUMP"
 echo "move testing data files    $MOVE"
 echo "inspect db and update ORM  $INSPECT"
 echo "clean temporary files      $CLEAN"
+echo "Connection                 http://${DATABASE_USER}:${DATABASE_PASS}@${PGHOST}:${PGPORT}/${DATABASE_NAME}"
 echo
 echo "Running..."
 # 1 - restore from database dump

docker/Dockerfile

@@ -13,14 +13,29 @@ RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \
     && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \
     fi
 
-FROM python:3.6.4
+FROM python:3.6.7-alpine as base
 COPY --from=builder /code /code
-LABEL org.label-schema.name="" \
-      org.label-schema.description="" \
-      org.label-schema.url="" \
-      org.label-schema.vcs-url="https://github.com/unicef/etools-datamart" \
-      org.label-schema.version=$VERSION \
-      org.label-schema.schema-version="1.0"
+RUN apk add --no-cache --virtual .build-deps \
+    gcc \
+    libffi-dev \
+    linux-headers \
+    musl-dev \
+    postgresql-dev \
+    python3-dev
+
+
+RUN apk add --no-cache --virtual .deps \
+    bash
+
+FROM base
+
+LABEL org.label.name="eTools Datamart" \
+      org.label.maintainer="[email protected]" \
+      org.label.description="" \
+      org.label.url="https://datamart.unicef.io/" \
+      org.label.vcs-url="https://github.com/unicef/etools-datamart" \
+      org.label.version=$VERSION
+
 
 ARG BUILD_DATE
 ARG PIPENV_PYPI_MIRROR
@@ -30,6 +45,7 @@ ARG VERSION
 ENV VERSION ${VERSION}
 
 ENV PIPENV_PYPI_MIRROR ${PIPENV_PYPI_MIRROR}
+ENV PIPENV_ARGS ${PIPENV_ARGS}
 ENV HOME /root/
 ENV PIPSI_HOME=/usr/local/pipsi/environments
 ENV PIPSI_BIN_DIR=/usr/local/bin
@@ -63,20 +79,17 @@ ENV SENTRY_DSN ""
 ENV SESSION_COOKIE_HTTPONLY True
 ENV SESSION_COOKIE_SECURE True
 ENV STATIC_ROOT /tmp/static
-ENV SUPERVISOR_USER admin
-ENV SUPERVISOR_PWD ""
-ENV FLOWER_USER admin
-ENV FLOWER_PWD ""
-ENV X_FRAME_OPTIONS "DENY"
-ENV START_DATAMART "true"
-ENV START_REDIS "true"
-ENV START_CELERY "true"
-
-RUN apt-get update && apt-get install -y --force-yes \
-    curl \
-    supervisor \
-    redis-server \
-    gcc
+#ENV SUPERVISOR_USER admin
+#ENV SUPERVISOR_PWD ""
+#ENV FLOWER_USER admin
+#ENV FLOWER_PWD ""
+#ENV X_FRAME_OPTIONS "DENY"
+#ENV START_DATAMART "true"
+#ENV START_REDIS "true"
+#ENV START_CELERY "true"
+
+#RUN apt-get update && apt-get install -y --force-yes \
+#    gcc
 
 RUN mkdir -p \
     /var/datamart/{static,log,conf,run} \
@@ -86,25 +99,23 @@ WORKDIR /code
 
 RUN set -ex \
     ls -al /code \
-    && pipenv install --system --deploy --ignore-pipfile $PIPENV_ARGS \
-    && pip install . \
-    && rm -fr /code
+    && pipenv install --system --deploy --ignore-pipfile $PIPENV_ARGS
 
-RUN apt-get autoremove --yes --force-yes \
-    gcc \
-    && rm -fr /var/lib/apt/lists/* \
-    && rm -fr /var/cache/apt/*
+RUN pip install . \
+    && rm -fr /code
 
+#RUN apt-get autoremove --yes --force-yes \
+#    gcc \
+#    && rm -fr /var/lib/apt/lists/* \
+#    && rm -fr /var/cache/apt/*
 
-ADD docker/redis.conf /etc/redis.conf
-ADD docker/supervisord.conf /etc/supervisord.conf
+#RUN apk del .build-deps \
+#    && rm -rf /var/cache/apk/* \
+#    && rm -fr /root/.cache/
 
 WORKDIR /var/datamart
 
 EXPOSE 8000
-EXPOSE 15000
-EXPOSE 5555
-EXPOSE 6379
 
 ADD docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 ENTRYPOINT ["docker-entrypoint.sh"]