Add Tracing and rework cli so that the controller command does not start a FastAPI app

.devcontainer/devcontainer.json

@@ -39,6 +39,8 @@
         // Make sure SELinux does not disable with access to host filesystems like tmp
         "--security-opt=label=disable"
     ],
+    // Remove this before pushing, only to allow local testing with editable library
+    "mounts": [],
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",
     // After the container is created, install the python project in editable form

.github/pages/make_switcher.py

@@ -52,7 +52,7 @@ def get_versions(ref: str, add: str | None) -> list[str]:
     return versions
 
 
-def write_json(path: Path, repository: str, versions: str):
+def write_json(path: Path, repository: str, versions: list[str]):
     org, repo_name = repository.split("/")
     struct = [
         {"version": version, "url": f"https://{org}.github.io/{repo_name}/{version}/"}

.vscode/launch.json

@@ -25,25 +25,35 @@
             "request": "launch",
             "justMyCode": false,
             "module": "blueapi",
-            "args": [
-                "serve"
-            ]
+            "env": {
+                "OTLP_EXPORT_ENABLED": "false"
+            },
+            "args": "-c ${input:config_path} serve"
         },
         {
             "name": "Blueapi Controller",
             "type": "debugpy",
             "request": "launch",
             "justMyCode": false,
             "module": "blueapi",
-            "args": "controller ${input:args}"
+            "env": {
+                "OTLP_EXPORT_ENABLED": "false"
+            },
+            "args": "-c ${input:config_path} controller ${input:args}"
         },
     ],
     "inputs": [
+        {
+            "id": "config_path",
+            "type": "promptString",
+            "description": "Path to configuration YAML file",
+            "default": ""
+        },
         {
             "id": "args",
             "type": "promptString",
             "description": "Arguments to pass to controller",
             "default": ""
         }
     ]
-}
+}

Dockerfile

@@ -12,6 +12,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 RUN python -m venv /venv
 ENV PATH=/venv/bin:$PATH
 
+ENV OTLP_EXPORT_ENABLED=false
+# enable opentelemetry support
+ENV OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf
+# Change this to point to Jaeger server before merging e.g. https://daq-services-jaeger
+ENV OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
+# Ensure that all Http headers are captured
+ENV OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST=".*"
+ENV OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE=".*"
+
 # The build stage installs the context into the venv
 FROM developer as build
 COPY . /context

dev-requirements.txt

@@ -9,8 +9,8 @@ annotated-types==0.7.0
 anyio==4.4.0
 appdirs==1.4.4
 asciitree==0.3.3
+asgiref==3.8.1
 asttokens==2.4.1
-async-timeout==4.0.3
 attrs==24.2.0
 babel==2.16.0
 beautifulsoup4==4.12.3
@@ -20,7 +20,6 @@ bluesky-kafka==0.10.0
 bluesky-live==0.0.8
 bluesky-stomp==0.1.2
 boltons==24.0.0
-bump-pydantic==0.8.0
 cachetools==5.5.0
 caproto==1.1.1
 certifi==2024.8.30
@@ -42,6 +41,7 @@ databroker==1.2.5
 dataclasses-json==0.6.7
 decorator==5.1.1
 deepmerge==2.0
+Deprecated==1.2.14
 distlib==0.3.8
 dls-bluesky-core==0.0.4
 dls-dodal==1.31.1
@@ -54,7 +54,6 @@ email_validator==2.2.0
 entrypoints==0.4
 epicscorelibs==7.0.7.99.0.2
 event-model==1.21.0
-exceptiongroup==1.2.2
 executing==2.1.0
 fastapi==0.114.2
 fastapi-cli==0.0.5
@@ -68,7 +67,9 @@ fsspec==2024.9.0
 funcy==2.0
 gitdb==4.0.11
 GitPython==3.1.43
+googleapis-common-protos==1.65.0
 graypy==2.1.0
+grpcio==1.66.2
 h11==0.14.0
 h5py==3.11.0
 HeapDict==1.0.1
@@ -81,7 +82,7 @@ identify==2.6.1
 idna==3.10
 imageio==2.35.1
 imagesize==1.4.1
-importlib_metadata==8.5.0
+importlib_metadata==8.4.0
 importlib_resources==6.4.5
 iniconfig==2.0.0
 intake==0.6.4
@@ -96,8 +97,6 @@ jsonschema-specifications==2023.12.1
 jupyterlab_widgets==3.0.13
 kiwisolver==1.4.7
 ldap3==2.9.1
-libcst==1.4.0
-livereload==2.7.0
 locket==1.0.0
 lz4==4.3.3
 markdown-it-py==3.0.0
@@ -122,7 +121,21 @@ nose2==0.15.1
 nslsii==0.10.3
 numcodecs==0.13.0
 numpy==1.26.4
+observability-utils==0.1.2
 opencv-python-headless==4.10.0.84
+opentelemetry-api==1.27.0
+opentelemetry-distro==0.48b0
+opentelemetry-exporter-otlp==1.27.0
+opentelemetry-exporter-otlp-proto-common==1.27.0
+opentelemetry-exporter-otlp-proto-grpc==1.27.0
+opentelemetry-exporter-otlp-proto-http==1.27.0
+opentelemetry-instrumentation==0.48b0
+opentelemetry-instrumentation-asgi==0.48b0
+opentelemetry-instrumentation-fastapi==0.48b0
+opentelemetry-proto==1.27.0
+opentelemetry-sdk==1.27.0
+opentelemetry-semantic-conventions==0.48b0
+opentelemetry-util-http==0.48b0
 ophyd==1.9.0
 ophyd-async==0.5.2
 orjson==3.10.7
@@ -147,6 +160,7 @@ ply==3.11
 pre-commit==3.8.0
 prettytable==3.11.0
 prompt-toolkit==3.0.36
+protobuf==4.25.5
 psutil==6.0.0
 ptyprocess==0.7.0
 pure_eval==0.2.3
@@ -173,7 +187,6 @@ python-dotenv==1.0.1
 python-multipart==0.0.9
 pytz==2024.2
 PyYAML==6.0.2
-pyyaml-include==2.1
 questionary==2.0.1
 redis==5.0.8
 redis-json-dict==0.2.0
@@ -217,9 +230,7 @@ suitcase-msgpack==0.3.0
 suitcase-utils==0.5.4
 super-state-machine==2.0.2
 tifffile==2024.8.30
-tomli==2.0.1
 toolz==0.12.1
-tornado==6.4.1
 tox==3.28.0
 tox-direct==0.4
 tqdm==4.66.5
@@ -244,6 +255,7 @@ websocket-client==1.8.0
 websockets==13.0.1
 widgetsnbextension==4.0.13
 workflows==2.27
+wrapt==1.16.0
 xarray==2024.9.0
 yarl==1.11.1
 zarr==2.18.3

docs/reference/openapi.yaml

@@ -179,6 +179,10 @@ components:
           default: true
           title: Is Pending
           type: boolean
+        request_id:
+          default: ''
+          title: Request Id
+          type: string
         task:
           title: Task
         task_id:

helm/blueapi/values.yaml

@@ -76,34 +76,58 @@ listener:
   resources: {}
 
 # Additional envVars to mount to the pod as a String
-extraEnvVars: []
+extraEnvVars: |
+  - name: OTLP_EXPORT_ENABLED
+    value: {{ .Values.tracing.otlp.export_enabled] }}
+  - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL
+    value: {{ .Values.tracing.otlp.protocol }}
+  - name: OTEL_EXPORTER_OTLP_ENDPOINT
+    value: "{{ .Values.tracing.otlp.host }}:{{ .Values.tracing.otlp.port }}"
+  - name: OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST
+    value: {{ .Values.tracing.http.request.headers }}
+  - name: OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE
+    value: {{ .Values.tracing.http.response.headers }}
 # - name: RABBITMQ_PASSWORD
 #   valueFrom:
 #     secretKeyRef:
 #       name: rabbitmq-password
 #       key: rabbitmq-password
 
+tracing:
+  otlp:
+    export_enabled: false
+    protocol: http/protobuf
+    host: https://daq-services-jaeger # replace with central instance
+    port: 4318
+  http:
+    request:
+      headers: ".*"
+    response:
+      headers: ".*"
+
 # Config for the worker goes here, will be mounted into a config file
 worker:
   api:
     host: 0.0.0.0 # Allow non-loopback traffic
     port: 8000
   env:
     sources:
-    - kind: deviceFunctions
-      module: blueapi.startup.example_devices
-    - kind: planFunctions
-      module: blueapi.startup.example_plans
-    - kind: planFunctions
-      module: dls_bluesky_core.plans
-    - kind: planFunctions
-      module: dls_bluesky_core.stubs
+      - kind: deviceFunctions
+        module: blueapi.startup.example_devices
+      - kind: planFunctions
+        module: blueapi.startup.example_plans
+      - kind: planFunctions
+        module: dls_bluesky_core.plans
+      - kind: planFunctions
+        module: dls_bluesky_core.stubs
   stomp:
     auth:
       username: guest
       password: guest
     host: rabbitmq
     port: 61613
+  tracing_exporter:
+    host: ""
 
 initContainer:
   scratch:

pyproject.toml

@@ -27,11 +27,14 @@ dependencies = [
     "fastapi>=0.112.0",
     "uvicorn",
     "requests",
-    "dls-bluesky-core",     #requires ophyd-async
+    "dls-bluesky-core",                              #requires ophyd-async
     "dls-dodal>=1.31.0",
-    "super-state-machine",  # See GH issue 553
+    "super-state-machine",                           # See GH issue 553
     "GitPython",
     "bluesky-stomp>=0.1.2",
+    "opentelemetry-distro>=0.48b0",
+    "opentelemetry-instrumentation-fastapi>=0.48b0",
+    "observability-utils>=0.1.2",
 ]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -79,7 +82,7 @@ write_to = "src/blueapi/_version.py"
 
 [tool.mypy]
 ignore_missing_imports = true # Ignore missing stubs in imported modules
-namespace_packages = false    # rely only on __init__ files to determine fully qualified module names.
+namespace_packages = true     # necessary for tracing sdk to work with mypy
 
 [tool.pytest.ini_options]
 # Run pytest with all our checkers, and don't spam us with massive tracebacks on error

src/blueapi/cli/cli.py

@@ -9,6 +9,9 @@
 from bluesky.callbacks.best_effort import BestEffortCallback
 from bluesky_stomp.messaging import MessageContext, StompClient
 from bluesky_stomp.models import Broker
+from observability_utils.tracing import setup_tracing
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.trace import get_tracer_provider
 from pydantic import ValidationError
 from requests.exceptions import ConnectionError
 
@@ -18,14 +21,7 @@
 from blueapi.client.event_bus import AnyEvent, BlueskyStreamingError, EventBusClient
 from blueapi.client.rest import BlueskyRemoteControlError
 from blueapi.config import ApplicationConfig, ConfigLoader
-from blueapi.core import DataEvent
-from blueapi.service.main import start
-from blueapi.service.openapi import (
-    DOCS_SCHEMA_LOCATION,
-    generate_schema,
-    print_schema_as_yaml,
-    write_schema_as_yaml,
-)
+from blueapi.core import OTLP_EXPORT_ENABLED, DataEvent
 from blueapi.worker import ProgressEvent, Task, WorkerEvent
 
 from .scratch import setup_scratch
@@ -70,6 +66,16 @@ def main(ctx: click.Context, config: Path | None | tuple[Path, ...]) -> None:
     help="[Development only] update the schema in the documentation",
 )
 def schema(output: Path | None = None, update: bool = False) -> None:
+    """Only import the service functions when starting the service or generating
+    the schema, not the controller as a new FastAPI app will be started each time.
+    """
+    from blueapi.service.openapi import (
+        DOCS_SCHEMA_LOCATION,
+        generate_schema,
+        print_schema_as_yaml,
+        write_schema_as_yaml,
+    )
+
     """Generate the schema for the REST API"""
     schema = generate_schema()
 
@@ -87,6 +93,17 @@ def start_application(obj: dict):
     """Run a worker that accepts plans to run"""
     config: ApplicationConfig = obj["config"]
 
+    """Only import the service functions when starting the service or generating
+    the schema, not the controller as a new FastAPI app will be started each time.
+    """
+    from blueapi.service.main import app, start
+
+    """
+    Set up basic automated instrumentation for the FastAPI app, creating the
+    observability context.
+    """
+    setup_tracing("BlueAPI", OTLP_EXPORT_ENABLED)
+    FastAPIInstrumentor().instrument_app(app, tracer_provider=get_tracer_provider())
     start(config)
 
 
@@ -101,6 +118,7 @@ def start_application(obj: dict):
 def controller(ctx: click.Context, output: str) -> None:
     """Client utility for controlling and introspecting the worker"""
 
+    setup_tracing("BlueAPICLI", OTLP_EXPORT_ENABLED)
     if ctx.invoked_subcommand is None:
         print("Please invoke subcommand!")
         return