docker: buster,python=3.10,graph-tool=2.45,bigtable/v1.12.0

.dockerignore

@@ -1,23 +1,124 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
 
-build
-dist
-__pycache__
-.pytest_cache
-.tox
-*.egg-info
-*.egg/
-*.pyc
-*.swp
+# C extensions
+*.so
 
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+.idea/*
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
 .coverage
-.vscode
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
 .ipynb_checkpoints
 
-.devcontainer/
-*.rdb
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Visual Code
+.vscode/
 
 # terraform
 .terraform/
 *.lock.hcl
 *.tfstate
-*.tfstate.*
+*.tfstate.*
+
+
+# local dev stuff
+.devcontainer/
+*.ipynb
+*.rdb
+/protobuf*
+
+# Git
+.git/

Dockerfile

@@ -1,7 +1,65 @@
-# FROM gcr.io/neuromancer-seung-import/pychunkedgraph:graph-tool_dracopy
-FROM seunglab/pychunkedgraph:graph-tool_dracopy
+ARG PYTHON_VERSION=3.10
+ARG BASE_IMAGE=tiangolo/uwsgi-nginx-flask:python${PYTHON_VERSION}
+
+
+######################################################
+# Build Image - PCG dependencies
+######################################################
+FROM ${BASE_IMAGE} AS pcg-build
+ENV PATH="/root/miniconda3/bin:${PATH}"
+ENV CONDA_ENV="pychunkedgraph"
+
+# Setup Miniconda
+RUN apt-get update && apt-get install build-essential wget -y
+RUN wget \
+    https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && mkdir /root/.conda \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b \
+    && rm -f Miniconda3-latest-Linux-x86_64.sh
+
+# Install PCG dependencies - especially graph-tool
+# Note: uwsgi has trouble with pip and python3.10, so adding this with conda, too
+COPY requirements.txt .
+COPY requirements.yml .
+RUN conda env create -n ${CONDA_ENV} -f requirements.yml
+
+# Shrink conda environment into portable non-conda env
+RUN conda install conda-pack -c conda-forge
+RUN conda-pack -n ${CONDA_ENV} -o /tmp/env.tar \
+  && mkdir -p /app/venv \
+  && cd /app/venv \
+  && tar xf /tmp/env.tar \
+  && rm /tmp/env.tar
+RUN /app/venv/bin/conda-unpack
+
+
+######################################################
+# Build Image - Bigtable Emulator (without Google SDK)
+######################################################
+FROM golang:buster as bigtable-emulator-build
+RUN mkdir -p /usr/src
+WORKDIR /usr/src
+ENV GOOGLE_CLOUD_GO_VERSION bigtable/v1.12.0
+RUN apt-get update && apt-get install git -y
+RUN git clone --depth=1 --branch="$GOOGLE_CLOUD_GO_VERSION" https://github.com/googleapis/google-cloud-go.git . \
+  && cd bigtable \
+  && go install -v ./cmd/emulator
+
+
+######################################################
+# Production Image
+######################################################
+FROM ${BASE_IMAGE}
+ENV VIRTUAL_ENV=/app/venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+COPY --from=pcg-build /app/venv /app/venv
+COPY --from=bigtable-emulator-build /go/bin/emulator /app/venv/bin/cbtemulator
+COPY override/gcloud /app/venv/bin/gcloud
 COPY override/timeout.conf /etc/nginx/conf.d/timeout.conf
 COPY override/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-COPY requirements.txt /app
-RUN pip install pip==20.2 && pip install --no-cache-dir --upgrade -r requirements.txt
 COPY . /app
+
+RUN mkdir -p /home/nginx/.cloudvolume/secrets \
+  && chown -R nginx /home/nginx \
+  && usermod -d /home/nginx -s /bin/bash nginx

override/gcloud

@@ -0,0 +1,55 @@
+#!/app/venv/bin/python
+
+import argparse
+import os
+import subprocess
+import re
+
+CONFIG_ENV = os.path.expanduser("~/.config/gcloud/emulators/bigtable/env.yaml")
+
+def env_init(args):
+    try:
+        with open(CONFIG_ENV, "r") as f:
+            hostport = re.findall(r"BIGTABLE_EMULATOR_HOST:\s+([:\w]+:\d+)", f.read())[0]
+            print(f"export BIGTABLE_EMULATOR={hostport}")
+    except Exception:
+        print(f"export BIGTABLE_EMULATOR=localhost:9000")
+
+def start(args):
+    os.makedirs(os.path.dirname(CONFIG_ENV), exist_ok=True)
+    with open(CONFIG_ENV, "w") as f:
+        f.write(f"---\nBIGTABLE_EMULATOR_HOST: {args.host_port}")
+
+    host, port = args.host_port.rsplit(':', 1)
+    subprocess.Popen(["cbtemulator", "-host", host, "-port", port], start_new_session=True)
+
+def usage(args):
+    print("""This is not gcloud. Only supported commands are:
+    - gcloud beta emulators bigtable env-init
+    - gcloud beta emulators bigtable start [--host-port localhost:9000]""")
+
+if __name__ == '__main__':
+    parser_gcloud = argparse.ArgumentParser(prog='gcloud')
+    parser_gcloud.set_defaults(func=usage)
+    subparser_gcloud = parser_gcloud.add_subparsers()
+
+    parser_beta = subparser_gcloud.add_parser('beta')
+    subparser_beta = parser_beta.add_subparsers()
+
+    parser_emulators = subparser_beta.add_parser('emulators')
+    subparser_emulators = parser_emulators.add_subparsers()
+
+    parser_bigtable = subparser_emulators.add_parser('bigtable')
+    subparser_bigtable = parser_bigtable.add_subparsers()
+
+    parser_env_init = subparser_bigtable.add_parser('env-init')
+    parser_env_init.set_defaults(func=env_init)
+
+    parser_start = subparser_bigtable.add_parser('start')
+    parser_start.add_argument('--host-port', default='localhost:9000')
+    parser_start.set_defaults(func=start)
+
+    args = parser_gcloud.parse_args()
+    args.func(args)
+
+

override/supervisord.conf

@@ -3,7 +3,7 @@ nodaemon=true
 
 
 [program:uwsgi]
-command=/usr/local/bin/uwsgi --ini /etc/uwsgi/uwsgi.ini --die-on-term --need-app
+command=%(ENV_VIRTUAL_ENV)s/bin/uwsgi --ini /etc/uwsgi/uwsgi.ini --die-on-term --need-app
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr

uwsgi.ini

@@ -13,6 +13,10 @@ gid = nginx
 
 env = HOME=/home/nginx
 
+# Python venv
+if-env = VIRTUAL_ENV
+virtualenv = %(_)
+endif =
 
 ### Worker scaling
 # maximum number of workers