From 335951d6aee06ac87789f611e457a6f9789f0325 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Mon, 5 Aug 2024 14:02:05 -0700 Subject: [PATCH] Add a way for Butler server to serve static files You can now set the environment variable DAF_BUTLER_SERVER_STATIC_FILES_PATH to point to a directory of static files to be served by Butler server from its `/configs/` HTTP path. There isn't currently a good place for the DirectButler configuration files used in the RSP lab containers to live. These need to be controlled by Phalanx so we can update them as part of the IDF deployment process, but Phalanx doesn't provide a convenient way of deploying static files. Much of this configuration is shared with Butler server or updated at the same time that Butler server is updated. So it is convenient for the moment to just serve these files from Butler server rather than standing up additional infrastructure to host them. --- .../butler/remote_butler/server/_config.py | 54 +++++++++++++++++++ .../butler/remote_butler/server/_server.py | 15 ++++++ python/lsst/daf/butler/tests/server_utils.py | 2 + tests/test_server.py | 12 +++++ 4 files changed, 83 insertions(+) create mode 100644 python/lsst/daf/butler/remote_butler/server/_config.py diff --git a/python/lsst/daf/butler/remote_butler/server/_config.py b/python/lsst/daf/butler/remote_butler/server/_config.py new file mode 100644 index 0000000000..16537eff09 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/server/_config.py @@ -0,0 +1,54 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import annotations + +import dataclasses +import os + + +@dataclasses.dataclass(frozen=True) +class ButlerServerConfig: + """Butler server configuration loaded from environment variables. + + Notes + ----- + Besides these variables, there is one critical environment variable. The + list of repositories to be hosted must be defined using + ``DAF_BUTLER_REPOSITORIES`` or ``DAF_BUTLER_REPOSITORY_INDEX`` -- see + `ButlerRepoIndex`. + """ + + static_files_path: str | None + """Absolute path to a directory of files that will be served to end-users + as static files from the `configs/` HTTP route. + """ + + +def load_config() -> ButlerServerConfig: + """Read the Butler server configuration from the environment.""" + return ButlerServerConfig(static_files_path=os.environ.get("DAF_BUTLER_SERVER_STATIC_FILES_PATH")) diff --git a/python/lsst/daf/butler/remote_butler/server/_server.py b/python/lsst/daf/butler/remote_butler/server/_server.py index d0547ee201..9c72f520d5 100644 --- a/python/lsst/daf/butler/remote_butler/server/_server.py +++ b/python/lsst/daf/butler/remote_butler/server/_server.py @@ -34,11 +34,13 @@ import safir.dependencies.logger from fastapi import FastAPI, Request, Response from fastapi.middleware.gzip import GZipMiddleware +from fastapi.staticfiles import StaticFiles from safir.logging import configure_logging, configure_uvicorn_logging from ..._exceptions import ButlerUserError from .._errors import serialize_butler_user_error from ..server_models import CLIENT_REQUEST_ID_HEADER_NAME, ERROR_STATUS_CODE, ErrorResponseModel +from ._config import load_config from .handlers._external import external_router from .handlers._external_query import query_router from .handlers._internal import internal_router @@ -49,6 +51,8 @@ def create_app() -> FastAPI: """Create a Butler server FastAPI application.""" + config = load_config() + app = FastAPI() app.add_middleware(GZipMiddleware, minimum_size=1000) @@ -67,6 +71,17 @@ def create_app() -> FastAPI: ) app.include_router(internal_router) + # If configured to do so, serve a directory of static files via HTTP. + # + # Until we are able to fully transition away from DirectButler for the RSP, + # we need a place to host DirectButler configuration files. Since this + # same configuration is needed for Butler server, it's easier to configure + # in Phalanx if we just host them from Butler server itself. + # This will also host the end-user repository index file for the RSP, for + # lack of a better place to put it. + if config.static_files_path: + app.mount(f"{default_api_path}/configs", StaticFiles(directory=config.static_files_path)) + # Any time an exception is returned by a handler, add a log message that # includes the username and request ID from the client. This will make it # easier to track down user-reported issues in the logs. diff --git a/python/lsst/daf/butler/tests/server_utils.py b/python/lsst/daf/butler/tests/server_utils.py index 5f4c37f9e3..c92db6a1f0 100644 --- a/python/lsst/daf/butler/tests/server_utils.py +++ b/python/lsst/daf/butler/tests/server_utils.py @@ -70,5 +70,7 @@ def _is_authenticated_endpoint(path: str) -> bool: return False if path.endswith("/butler.json"): return False + if path.startswith("/api/butler/configs"): + return False return True diff --git a/tests/test_server.py b/tests/test_server.py index e28ff2eb58..c3fb6c31a3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -26,6 +26,7 @@ # along with this program. If not, see . import os.path +import tempfile import unittest import uuid @@ -106,6 +107,17 @@ def test_health_check(self): self.assertEqual(response.status_code, 200) self.assertEqual(response.json()["name"], "butler") + def test_static_files(self): + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.path.join(tmpdir, "temp.txt"), "w") as fh: + fh.write("test data 123") + + with mock_env({"DAF_BUTLER_SERVER_STATIC_FILES_PATH": tmpdir}): + with create_test_server(TESTDIR) as server: + response = server.client.get("/api/butler/configs/temp.txt") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.text, "test data 123") + def test_dimension_universe(self): universe = self.butler.dimensions self.assertEqual(universe.namespace, "daf_butler")