From 94ee4c435424054ca2a0e70e2a78533b3826f249 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Fri, 27 Oct 2023 15:37:22 -0700 Subject: [PATCH] Add a Dockerfile for butler server --- Dockerfile | 99 +++++++++++++++++++ server/requirements.txt | 1 + server/scripts/install-base-packages.sh | 64 ++++++++++++ server/scripts/install-dependency-packages.sh | 66 +++++++++++++ 4 files changed, 230 insertions(+) create mode 100644 Dockerfile create mode 100644 server/requirements.txt create mode 100755 server/scripts/install-base-packages.sh create mode 100755 server/scripts/install-dependency-packages.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..9c8918171e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,99 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This Dockerfile is based on the fastapi_safir_app template from +# lsst/templates +# +# This Dockerfile has four stages: +# +# base-image +# Updates the base Python image with security patches and common system +# packages. This image becomes the base of all other images. +# dependencies-image +# Installs third-party dependencies (requirements/main.txt) into a virtual +# environment. This virtual environment is ideal for copying across build +# stages. +# install-image +# Installs the app into the virtual environment. +# runtime-image +# - Copies the virtual environment into place. +# - Runs a non-root user. +# - Sets up the entrypoint and port. + +FROM python:3.11.1-slim-bullseye as base-image + +# Update system packages +COPY server/scripts/install-base-packages.sh . +RUN ./install-base-packages.sh && rm ./install-base-packages.sh + +FROM base-image AS dependencies-image + +# Install system packages only needed for building dependencies. +COPY server/scripts/install-dependency-packages.sh . +RUN ./install-dependency-packages.sh + +# Create a Python virtual environment +ENV VIRTUAL_ENV=/opt/venv +RUN python -m venv $VIRTUAL_ENV +# Make sure we use the virtualenv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Put the latest pip and setuptools in the virtualenv +RUN pip install --upgrade --no-cache-dir pip setuptools wheel + +# Install the app's Python runtime dependencies +COPY requirements.txt . +COPY server/requirements.txt server_requirements.txt +RUN pip install --quiet --no-cache-dir -r requirements.txt -r server_requirements.txt + +FROM dependencies-image AS install-image + +# Use the virtualenv +ENV PATH="/opt/venv/bin:$PATH" + +COPY . /workdir +WORKDIR /workdir +RUN pip install --no-cache-dir . + +FROM base-image AS runtime-image + +# Create a non-root user +RUN useradd --create-home appuser + +# Copy the virtualenv +COPY --from=install-image /opt/venv /opt/venv + +# Make sure we use the virtualenv +ENV PATH="/opt/venv/bin:$PATH" + +# Switch to the non-root user. +USER appuser + +# Expose the port. +EXPOSE 8080 + +# Run the application. +CMD ["uvicorn", "lsst.daf.butler.remote_butler.server:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/server/requirements.txt b/server/requirements.txt new file mode 100644 index 0000000000..f193b62170 --- /dev/null +++ b/server/requirements.txt @@ -0,0 +1 @@ +uvicorn diff --git a/server/scripts/install-base-packages.sh b/server/scripts/install-base-packages.sh new file mode 100755 index 0000000000..4048ad5b50 --- /dev/null +++ b/server/scripts/install-base-packages.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script is based on the fastapi_safir_app template from +# lsst/templates + +# This script updates packages in the base Docker image that's used by both the +# build and runtime images, and gives us a place to install additional +# system-level packages with apt-get. +# +# Based on the blog post: +# https://pythonspeed.com/articles/system-packages-docker/ + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Tell apt-get we're never going to be able to give manual +# feedback: +export DEBIAN_FRONTEND=noninteractive + +# Update the package listing, so we know what packages exist: +apt-get update + +# Install security updates: +apt-get -y upgrade + +# Example of installing a new package, without unnecessary packages: +#apt-get -y install --no-install-recommends git + +# Delete cached files we don't need anymore: +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/server/scripts/install-dependency-packages.sh b/server/scripts/install-dependency-packages.sh new file mode 100755 index 0000000000..7a00218481 --- /dev/null +++ b/server/scripts/install-dependency-packages.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script is based on the fastapi_safir_app template from +# lsst/templates + +# This script installs additional packages used by the dependency image but +# not needed by the runtime image, such as additional packages required to +# build Python dependencies. +# +# Since the base image wipes all the apt caches to clean up the image that +# will be reused by the runtime image, we unfortunately have to do another +# apt-get update here, which wastes some time and network. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Tell apt-get we're never going to be able to give manual +# feedback: +export DEBIAN_FRONTEND=noninteractive + +# Update the package listing, so we know what packages exist: +apt-get update + +# Install build-essential because sometimes Python dependencies need to build +# C modules, particularly when upgrading to newer Python versions. libffi-dev +# is sometimes needed to build cffi (a cryptography dependency). +# Git is needed because we still have some Python dependencies pointing +# directly at Git repos +apt-get -y install --no-install-recommends build-essential libffi-dev git + +# Delete cached files we don't need anymore: +apt-get clean +rm -rf /var/lib/apt/lists/*