diff --git a/.dockerignore b/.dockerignore index 0fb97f2e8..66349c8a4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,23 +1,124 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class -build -dist -__pycache__ -.pytest_cache -.tox -*.egg-info -*.egg/ -*.pyc -*.swp +# C extensions +*.so +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +.idea/* + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ .coverage -.vscode +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook .ipynb_checkpoints -.devcontainer/ -*.rdb +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Visual Code +.vscode/ # terraform .terraform/ *.lock.hcl *.tfstate -*.tfstate.* \ No newline at end of file +*.tfstate.* + + +# local dev stuff +.devcontainer/ +*.ipynb +*.rdb +/protobuf* + +# Git +.git/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7de78900c..f0baf13eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,65 @@ -# FROM gcr.io/neuromancer-seung-import/pychunkedgraph:graph-tool_dracopy -FROM seunglab/pychunkedgraph:graph-tool_dracopy +ARG PYTHON_VERSION=3.10 +ARG BASE_IMAGE=tiangolo/uwsgi-nginx-flask:python${PYTHON_VERSION} + + +###################################################### +# Build Image - PCG dependencies +###################################################### +FROM ${BASE_IMAGE} AS pcg-build +ENV PATH="/root/miniconda3/bin:${PATH}" +ENV CONDA_ENV="pychunkedgraph" + +# Setup Miniconda +RUN apt-get update && apt-get install build-essential wget -y +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + +# Install PCG dependencies - especially graph-tool +# Note: uwsgi has trouble with pip and python3.10, so adding this with conda, too +COPY requirements.txt . +COPY requirements.yml . +RUN conda env create -n ${CONDA_ENV} -f requirements.yml + +# Shrink conda environment into portable non-conda env +RUN conda install conda-pack -c conda-forge +RUN conda-pack -n ${CONDA_ENV} -o /tmp/env.tar \ + && mkdir -p /app/venv \ + && cd /app/venv \ + && tar xf /tmp/env.tar \ + && rm /tmp/env.tar +RUN /app/venv/bin/conda-unpack + + +###################################################### +# Build Image - Bigtable Emulator (without Google SDK) +###################################################### +FROM golang:buster as bigtable-emulator-build +RUN mkdir -p /usr/src +WORKDIR /usr/src +ENV GOOGLE_CLOUD_GO_VERSION bigtable/v1.12.0 +RUN apt-get update && apt-get install git -y +RUN git clone --depth=1 --branch="$GOOGLE_CLOUD_GO_VERSION" https://github.com/googleapis/google-cloud-go.git . \ + && cd bigtable \ + && go install -v ./cmd/emulator + + +###################################################### +# Production Image +###################################################### +FROM ${BASE_IMAGE} +ENV VIRTUAL_ENV=/app/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +COPY --from=pcg-build /app/venv /app/venv +COPY --from=bigtable-emulator-build /go/bin/emulator /app/venv/bin/cbtemulator +COPY override/gcloud /app/venv/bin/gcloud COPY override/timeout.conf /etc/nginx/conf.d/timeout.conf COPY override/supervisord.conf /etc/supervisor/conf.d/supervisord.conf -COPY requirements.txt /app -RUN pip install pip==20.2 && pip install --no-cache-dir --upgrade -r requirements.txt COPY . /app + +RUN mkdir -p /home/nginx/.cloudvolume/secrets \ + && chown -R nginx /home/nginx \ + && usermod -d /home/nginx -s /bin/bash nginx diff --git a/override/gcloud b/override/gcloud new file mode 100755 index 000000000..882fbc73e --- /dev/null +++ b/override/gcloud @@ -0,0 +1,55 @@ +#!/app/venv/bin/python + +import argparse +import os +import subprocess +import re + +CONFIG_ENV = os.path.expanduser("~/.config/gcloud/emulators/bigtable/env.yaml") + +def env_init(args): + try: + with open(CONFIG_ENV, "r") as f: + hostport = re.findall(r"BIGTABLE_EMULATOR_HOST:\s+([:\w]+:\d+)", f.read())[0] + print(f"export BIGTABLE_EMULATOR={hostport}") + except Exception: + print(f"export BIGTABLE_EMULATOR=localhost:9000") + +def start(args): + os.makedirs(os.path.dirname(CONFIG_ENV), exist_ok=True) + with open(CONFIG_ENV, "w") as f: + f.write(f"---\nBIGTABLE_EMULATOR_HOST: {args.host_port}") + + host, port = args.host_port.rsplit(':', 1) + subprocess.Popen(["cbtemulator", "-host", host, "-port", port], start_new_session=True) + +def usage(args): + print("""This is not gcloud. Only supported commands are: + - gcloud beta emulators bigtable env-init + - gcloud beta emulators bigtable start [--host-port localhost:9000]""") + +if __name__ == '__main__': + parser_gcloud = argparse.ArgumentParser(prog='gcloud') + parser_gcloud.set_defaults(func=usage) + subparser_gcloud = parser_gcloud.add_subparsers() + + parser_beta = subparser_gcloud.add_parser('beta') + subparser_beta = parser_beta.add_subparsers() + + parser_emulators = subparser_beta.add_parser('emulators') + subparser_emulators = parser_emulators.add_subparsers() + + parser_bigtable = subparser_emulators.add_parser('bigtable') + subparser_bigtable = parser_bigtable.add_subparsers() + + parser_env_init = subparser_bigtable.add_parser('env-init') + parser_env_init.set_defaults(func=env_init) + + parser_start = subparser_bigtable.add_parser('start') + parser_start.add_argument('--host-port', default='localhost:9000') + parser_start.set_defaults(func=start) + + args = parser_gcloud.parse_args() + args.func(args) + + diff --git a/override/supervisord.conf b/override/supervisord.conf index 37c371e07..90960a56e 100644 --- a/override/supervisord.conf +++ b/override/supervisord.conf @@ -3,7 +3,7 @@ nodaemon=true [program:uwsgi] -command=/usr/local/bin/uwsgi --ini /etc/uwsgi/uwsgi.ini --die-on-term --need-app +command=%(ENV_VIRTUAL_ENV)s/bin/uwsgi --ini /etc/uwsgi/uwsgi.ini --die-on-term --need-app stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr diff --git a/uwsgi.ini b/uwsgi.ini index 291efa1dc..a9227822e 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -13,6 +13,10 @@ gid = nginx env = HOME=/home/nginx +# Python venv +if-env = VIRTUAL_ENV +virtualenv = %(_) +endif = ### Worker scaling # maximum number of workers