diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3978a0f7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git +.gitignore diff --git a/.travis.yml b/.travis.yml index cd21b01a..b1c16bc6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,32 +1,49 @@ +branches: + only: + - gh-pages + - /.*/ + language: python python: - 2.7 cache: directories: - - $HOME/.cache/pip - - $HOME/build/hotosm/osm-tasking-manager2/env + - "$HOME/.cache/pip" + - "$HOME/build/hotosm/osm-tasking-manager2/env" sudo: false before_install: -- deactivate + - deactivate install: -- virtualenv env -- env/bin/pip install --upgrade pip -- env/bin/pip install -r requirements.txt -- createuser www-data --no-superuser --no-createdb --no-createrole -U postgres -- createdb -O www-data osmtm_tests -U postgres -- psql -d osmtm_tests -c "CREATE EXTENSION postgis;" -U postgres -- env/bin/pip install flake8 + - virtualenv env + - env/bin/pip install --upgrade pip + - env/bin/pip install -r requirements.txt + - createuser www-data --no-superuser --no-createdb --no-createrole -U postgres + - createdb -O www-data osmtm_tests -U postgres + - psql -d osmtm_tests -c "CREATE EXTENSION postgis;" -U postgres + - env/bin/pip install flake8 script: -- echo [app:main] >> local.test.ini -- echo sqlalchemy.url = postgresql://www-data:www-data@localhost/osmtm_tests >> local.test.ini -- env/bin/nosetests -- env/bin/flake8 osmtm --exclude=osmtm/static + - echo [app:main] >> local.test.ini + - echo sqlalchemy.url = postgresql://www-data:www-data@localhost/osmtm_tests >> local.test.ini + - env/bin/nosetests + - env/bin/flake8 osmtm --exclude=osmtm/static after_success: -- env/bin/pip install coveralls -- env/bin/coveralls + - env/bin/pip install coveralls + - env/bin/coveralls + +before_deploy: + - 'curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"' + - 'unzip awscli-bundle.zip' + - './awscli-bundle/install -b ~/bin/aws' + - 'export PATH=~/bin:$PATH' + - 'rm awscli-bundle.zip' + - 'rm -rf awscli-bundle/' + +deploy: + provider: script + script: scripts/deploy.sh diff --git a/Dockerfile.app b/Dockerfile.app new file mode 100644 index 00000000..32bc6dc9 --- /dev/null +++ b/Dockerfile.app @@ -0,0 +1,39 @@ +FROM python:2.7.12 + +RUN apt-get update \ + && apt-get install -y \ + --no-install-recommends \ + libgeos-dev \ + software-properties-common \ + python-software-properties \ + wget \ + && apt-get clean \ + && apt-get autoremove \ + && rm -r /var/lib/apt/lists/* + +RUN add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main" +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN apt-get update \ + && apt-get install -y \ + --no-install-recommends \ + postgresql-client-common \ + postgresql-client-9.6 \ + && apt-get clean \ + && apt-get autoremove \ + && rm -r /var/lib/apt/lists/* + +ADD requirements.txt /opt/app/requirements.txt +ADD setup.py /opt/app/setup.py +ADD setup.cfg /opt/app/setup.cfg +ADD README.md /opt/app/README.md +ADD CHANGELOG.md /opt/app/CHANGELOG.md + +WORKDIR /opt/app + +RUN easy_install virtualenv +RUN virtualenv --no-site-packages env +RUN ./env/bin/pip install -r requirements.txt + +ADD . /opt/app + +CMD ./startup-tm.sh diff --git a/Dockerfile.db b/Dockerfile.db new file mode 100644 index 00000000..981c2c93 --- /dev/null +++ b/Dockerfile.db @@ -0,0 +1,19 @@ +FROM library/postgres + +RUN apt-get update \ + && apt-get install -y \ + --no-install-recommends \ + postgis \ + postgresql-9.6-postgis-2.3 \ + postgresql-contrib-9.6 \ + postgresql-9.6-postgis-scripts \ + pgtop \ + && apt-get clean \ + && apt-get autoremove \ + && rm -r /var/lib/apt/lists/* + +COPY docker-entrypoint-initdb.d /docker-entrypoint-initdb.d + +RUN dpkg-query -l postgis + +RUN ls -lAh /usr/share/postgresql/9.6/extension/postgis* diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 00000000..30d2aba5 --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,27 @@ +version: '2' + +services: + + app: + command: ["true"] + +# https://github.com/vincetse/docker-compose-zero-downtime-deployment + app_green: + container_name: app_green + ports: + - 3000:6543 + environment: + - VIRTUAL_PORT=3000 + - VIRTUAL_HOST=tasks.hotosm.org,tasks-green.hotosm.org + extends: + service: app + + app_blue: + container_name: app_blue + ports: + - 3001:6543 + environment: + - VIRTUAL_PORT=3001 + - VIRTUAL_HOST=tasks.hotosm.org,tasks-blue.hotosm.org + extends: + service: app diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml new file mode 100644 index 00000000..49e491db --- /dev/null +++ b/docker-compose.staging.yml @@ -0,0 +1,27 @@ +version: '2' + +services: + + app: + command: ["true"] + +# https://github.com/vincetse/docker-compose-zero-downtime-deployment + app_green: + container_name: app_green + ports: + - 3000:6543 + environment: + - VIRTUAL_PORT=3000 + - VIRTUAL_HOST=tasks-staging.hotosm.org,tasks-staging-green.hotosm.org + extends: + service: app + + app_blue: + container_name: app_blue + ports: + - 3001:6543 + environment: + - VIRTUAL_PORT=3001 + - VIRTUAL_HOST=tasks-staging.hotosm.org,tasks-staging-blue.hotosm.org + extends: + service: app diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..b47d3ce3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +version: '2' + +services: +# https://hub.docker.com/_/postgres/ +# https://store.docker.com/images/022689bf-dfd8-408f-9e1c-19acac32e57b + + db: + build: + context: ./ + dockerfile: Dockerfile.db + container_name: db + restart: always + volumes: + - postgres-db-volume:/data/postgres + - /tmp:/srv + cpu_shares: 8 + cpuset: 0-8 + networks: + - database_network + + app: + build: + context: ./ + dockerfile: Dockerfile.app + container_name: app + environment: + - VIRTUAL_HOST=tasks-staging.hotosm.org + restart: always + # https://docs.docker.com/compose/startup-order/ + command: ["./wait-for-postgres.sh", "db", "./startup-tm.sh"] + networks: + - database_network + - code_network + +# cpuset, such as `docker run –cpuset-cpus`, allows users to specify +# which CPUs to allow execution in. accepts a string that looks like +# 0 +# 0,1 +# 0-2 +# cpu_shares + +volumes: + postgres-db-volume: + +networks: + code_network: + driver: bridge + database_network: + driver: bridge diff --git a/docker-entrypoint-initdb.d/setup.sql b/docker-entrypoint-initdb.d/setup.sql new file mode 100644 index 00000000..f92b1924 --- /dev/null +++ b/docker-entrypoint-initdb.d/setup.sql @@ -0,0 +1,6 @@ +CREATE USER "www-data" WITH PASSWORD 'password'; +ALTER USER "www-data" WITH NOSUPERUSER NOCREATEROLE NOCREATEDB; +CREATE DATABASE osmtm OWNER "www-data" ENCODING 'UTF8' TEMPLATE template0; +\connect osmtm +CREATE EXTENSION postgis; +SELECT postgis_full_version(); diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 00000000..342fba89 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,99 @@ +#!/bin/bash -eux + +DEPLOY_USER="mozart" +HOST="62.210.100.219" +SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i mozart_rsa -o ForwardAgent=yes -o IPQoS=throughput -tt" +BASE_DIR="/srv/deploy/osm-tasking-manager2" + +# download mozarts key +aws s3 cp s3://hotosm-secure/keys/mozart_rsa.enc mozart_rsa.enc +# unencrypt mozarts key +openssl aes-256-cbc -K $encrypted_e101044e37e0_key -iv $encrypted_e101044e37e0_iv \ + -in mozart_rsa.enc -out mozart_rsa -d +# fix permissions +chmod 700 mozart_rsa + +# set environment variable based on branch or tag +if [ ! -z $TRAVIS_TAG ]; then + echo "Deploying $TRAVIS_TAG to prod" + URL_PREFIX="tasks" + ENV="production" +elif [ "$TRAVIS_BRANCH" == "develop" -o "$TRAVIS_PULL_REQUEST" != "false" ]; then + echo "Deploying develop branch to staging" + URL_PREFIX="tasks-staging" + ENV="staging" +else + echo "Not a tag or develop branch. Doing nothing" + exit 0 +fi + +# rsync files to server and respective environment path +rsync -arvz --progress . \ + --exclude='.git/' \ + -e "ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i mozart_rsa" \ +$DEPLOY_USER@$HOST:$BASE_DIR-$ENV + +# find the live environment +LIVE_COLOR=$(ssh $SSH_OPTS $DEPLOY_USER@$HOST \ + "sudo su -c 'cd $BASE_DIR-$ENV && docker-compose \ + -f docker-compose.yml \ + -f docker-compose.$ENV.yml \ + ps'" | awk '/app.*Up/ {print $1}') + +# set values for live color and deploy color +if [[ $LIVE_COLOR == *"blue"* ]]; then + deploy="green"; live="blue" +elif [[ $LIVE_COLOR == *"green"* ]]; then + deploy="blue"; live="green" +else + echo "app not running" +fi + +# echo build color +echo "building app_$deploy" + +# build new color +ssh $DEPLOY_USER@$HOST $SSH_OPTS \ + "sudo su -c 'cd $BASE_DIR-$ENV && docker-compose \ + -f docker-compose.yml \ + -f docker-compose.$ENV.yml \ + build app app_$deploy'" + +# echo deploy color +echo "deploying app_$deploy" + +# deploy new color +ssh $DEPLOY_USER@$HOST $SSH_OPTS \ + "sudo su -c 'cd $BASE_DIR-$ENV && docker-compose \ + -f docker-compose.yml \ + -f docker-compose.$ENV.yml \ + up -d \ + --force-recreate app_$deploy'" + +# wait a little in case the service takes a bit to start +sleep 5 + +# test new server alt url +until $(curl --output /dev/null --silent --head --fail http://$URL_PREFIX-$deploy.hotosm.org); do + printf '.' + sleep 2 +done + +# TODO +# add timeout +# add fail check to tear down new servers + +# stop old server +ssh $DEPLOY_USER@$HOST $SSH_OPTS \ + "sudo su -c 'cd $BASE_DIR-$ENV && docker-compose \ + -f docker-compose.yml \ + -f docker-compose.$ENV.yml \ + stop app_$live'" + +# test main site +until $(curl --output /dev/null --silent --head --fail http://$URL_PREFIX.hotosm.org); do + printf '.' + sleep 2 +done diff --git a/startup-tm.sh b/startup-tm.sh new file mode 100755 index 00000000..a69aff5d --- /dev/null +++ b/startup-tm.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# ./env/bin/initialize_osmtm_db +./env/bin/pserve --reload development.ini diff --git a/wait-for-postgres.sh b/wait-for-postgres.sh new file mode 100755 index 00000000..5a817d49 --- /dev/null +++ b/wait-for-postgres.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# wait-for-postgres.sh + +set -e + +host="$1" +shift +cmd="$@" + +until psql -h "$host" -U "postgres" -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - executing command" +exec $cmd