New signal typing

docs/explanations/decisions/0008-signal-types.md

@@ -0,0 +1,48 @@
+# 8. Settle on Signal Types
+Date: 2024-10-18
+
+## Status
+
+Accepted
+
+## Context
+
+At present, soft Signals allow any sort of datatype, while CA, PVA, Tango restrict these to what the control system allows. This means that some soft signals when `describe()` is called on them will give `dtype=object` which is not understood by downstream tools. It also means that load/save will not necessarily understand how to serialize results. Finally we now require `dtype_numpy` for tiled, so arbitrary object types are not suitable even if they are serializable. We should restrict the datatypes allowed in Signals to objects that are serializable and are sensible to add support for in downstream tools.
+
+## Decision
+
+We will allow the following:
+- Primitives:
+    - `bool`
+    - `int`
+    - `float`
+    - `str`
+- Enums:
+    - `StrictEnum` subclass which will be checked to have the same members as the CS
+    - `SubsetEnum` subclass which will be checked to be a subset of the CS members
+- 1D arrays:
+    - `Array1D[np.bool_]`
+    - `Array1D[np.int8]`
+    - `Array1D[np.uint8]`
+    - `Array1D[np.int16]`
+    - `Array1D[np.uint16]`
+    - `Array1D[np.int32]`
+    - `Array1D[np.uint32]`
+    - `Array1D[np.int64]`
+    - `Array1D[np.uint64]`
+    - `Array1D[np.float32]`
+    - `Array1D[np.float64]`
+    - `Sequence[str]`
+    - `Sequence[MyEnum]` where `MyEnum` is a subclass of `StrictEnum` or `SubsetEnum`
+- Specific structures:
+    - `np.ndarray` to represent arrays where dimensionality and dtype can change and must be read from CS
+    - `Table` subclass (which is a pydantic `BaseModel`) where all members are 1D arrays
+
+## Consequences
+
+Clients will be expected to understand:
+- Python primitives (with Enums serializing as strings)
+- Numpy arrays
+- Pydantic BaseModels
+
+All of the above have sensible `dtype_numpy` fields, but `Table` will give a structured row-wise `dtype_numpy`, while the data will be serialized in a column-wise fashion.

docs/how-to/make-a-simple-device.rst

@@ -1,6 +1,6 @@
 .. note::
 
-   Ophyd async is included on a provisional basis until the v1.0 release and 
+   Ophyd async is included on a provisional basis until the v1.0 release and
    may change API on minor release numbers before then
 
 Make a Simple Device
@@ -31,7 +31,7 @@ its Python type, which could be:
 - A primitive (`str`, `int`, `float`)
 - An array (`numpy.typing.NDArray` ie. ``numpy.typing.NDArray[numpy.uint16]`` or ``Sequence[str]``)
 - An enum (`enum.Enum`) which **must** also extend `str`
-    - `str` and ``EnumClass(str, Enum)`` are the only valid ``datatype`` for an enumerated signal.
+    - `str` and ``EnumClass(StrictEnum)`` are the only valid ``datatype`` for an enumerated signal.
 
 The rest of the arguments are PV connection information, in this case the PV suffix.
 
@@ -45,7 +45,7 @@ Finally `super().__init__() <StandardReadable>` is called with:
   without renaming
 
 All signals passed into this init method will be monitored between ``stage()``
-and ``unstage()`` and their cached values returned on ``read()`` and 
+and ``unstage()`` and their cached values returned on ``read()`` and
 ``read_configuration()`` for perfomance.
 
 Movable
@@ -64,7 +64,7 @@ informing watchers of the progress. When it gets to the requested value it
 completes. This co-routine is wrapped in a timeout handler, and passed to an
 `AsyncStatus` which will start executing it as soon as the Run Engine adds a
 callback to it. The ``stop()`` method then pokes a PV if the move needs to be
-interrupted. 
+interrupted.
 
 Assembly
 --------

pyproject.toml

@@ -13,12 +13,12 @@ classifiers = [
 description = "Asynchronous Bluesky hardware abstraction code, compatible with control systems like EPICS and Tango"
 dependencies = [
     "networkx>=2.0",
-    "numpy<2.0.0",
+    "numpy",
     "packaging",
     "pint",
-    "bluesky>=1.13.0a3",
-    "event_model",
-    "p4p",
+    "bluesky @ git+https://github.com/bluesky/bluesky@main",
+    "event-model @ git+https://github.com/bluesky/event-model@main",
+    "p4p>=4.2.0a3",
     "pyyaml",
     "colorlog",
     "pydantic>=2.0",
@@ -39,10 +39,6 @@ dev = [
     "ophyd_async[sim]",
     "ophyd_async[ca]",
     "ophyd_async[tango]",
-    "black",
-    "flake8",
-    "flake8-isort",
-    "Flake8-pyproject",
     "inflection",
     "ipython",
     "ipywidgets",
@@ -150,15 +146,17 @@ commands =
 src = ["src", "tests", "system_tests"]
 line-length = 88
 lint.select = [
-    "B",   # flake8-bugbear - https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
-    "C4",  # flake8-comprehensions - https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4
-    "E",   # pycodestyle errors - https://docs.astral.sh/ruff/rules/#error-e
-    "F",   # pyflakes rules - https://docs.astral.sh/ruff/rules/#pyflakes-f
-    "W",   # pycodestyle warnings - https://docs.astral.sh/ruff/rules/#warning-w
-    "I",   # isort - https://docs.astral.sh/ruff/rules/#isort-i
-    "UP",  # pyupgrade - https://docs.astral.sh/ruff/rules/#pyupgrade-up
-    "SLF", # self - https://docs.astral.sh/ruff/settings/#lintflake8-self
+    "B",       # flake8-bugbear - https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
+    "C4",      # flake8-comprehensions - https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4
+    "E",       # pycodestyle errors - https://docs.astral.sh/ruff/rules/#error-e
+    "F",       # pyflakes rules - https://docs.astral.sh/ruff/rules/#pyflakes-f
+    "W",       # pycodestyle warnings - https://docs.astral.sh/ruff/rules/#warning-w
+    "I",       # isort - https://docs.astral.sh/ruff/rules/#isort-i
+    "UP",      # pyupgrade - https://docs.astral.sh/ruff/rules/#pyupgrade-up
+    "SLF",     # self - https://docs.astral.sh/ruff/settings/#lintflake8-self
+    "PLC2701", # private import - https://docs.astral.sh/ruff/rules/import-private-name/
 ]
+lint.preview = true # so that preview mode PLC2701 is enabled
 
 [tool.ruff.lint.per-file-ignores]
 # By default, private member access is allowed in tests

src/ophyd_async/core/__init__.py

@@ -5,7 +5,8 @@
     StandardDetector,
     TriggerInfo,
 )
-from ._device import Device, DeviceCollector, DeviceVector
+from ._device import Device, DeviceCollector, DeviceConnector, DeviceVector
+from ._device_filler import DeviceFiller
 from ._device_save_loader import (
     all_at_once,
     get_signal_values,
@@ -62,9 +63,11 @@
     wait_for_value,
 )
 from ._signal_backend import (
-    RuntimeSubsetEnum,
+    Array1D,
     SignalBackend,
-    SubsetEnum,
+    SignalDatatype,
+    SignalDatatypeT,
+    make_datakey,
 )
 from ._soft_signal_backend import SignalMetadata, SoftSignalBackend
 from ._status import AsyncStatus, WatchableAsyncStatus, completed_status
@@ -73,14 +76,16 @@
     CALCULATE_TIMEOUT,
     DEFAULT_TIMEOUT,
     CalculatableTimeout,
+    Callback,
     NotConnected,
-    ReadingValueCallback,
+    StrictEnum,
+    SubsetEnum,
     T,
     WatcherUpdate,
     get_dtype,
+    get_enum_cls,
     get_unique,
     in_micros,
-    is_pydantic_model,
     wait_for_connection,
 )
 
@@ -91,8 +96,10 @@
     "StandardDetector",
     "TriggerInfo",
     "Device",
+    "DeviceConnector",
     "DeviceCollector",
     "DeviceVector",
+    "DeviceFiller",
     "all_at_once",
     "get_signal_values",
     "load_device",
@@ -146,25 +153,29 @@
     "soft_signal_r_and_setter",
     "soft_signal_rw",
     "wait_for_value",
-    "RuntimeSubsetEnum",
+    "Array1D",
     "SignalBackend",
+    "make_datakey",
+    "StrictEnum",
     "SubsetEnum",
+    "SignalDatatype",
+    "SignalDatatypeT",
     "SignalMetadata",
     "SoftSignalBackend",
     "AsyncStatus",
     "WatchableAsyncStatus",
     "DEFAULT_TIMEOUT",
     "CalculatableTimeout",
+    "Callback",
     "CALCULATE_TIMEOUT",
     "NotConnected",
-    "ReadingValueCallback",
     "Table",
     "T",
     "WatcherUpdate",
     "get_dtype",
+    "get_enum_cls",
     "get_unique",
     "in_micros",
-    "is_pydantic_model",
     "wait_for_connection",
     "completed_status",
 ]

src/ophyd_async/core/_detector.py

@@ -4,11 +4,7 @@
 import time
 from abc import ABC, abstractmethod
 from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator, Sequence
-from enum import Enum
 from functools import cached_property
-from typing import (
-    Generic,
-)
 
 from bluesky.protocols import (
     Collectable,
@@ -23,14 +19,14 @@
 from event_model import DataKey
 from pydantic import BaseModel, Field, NonNegativeInt, computed_field
 
-from ._device import Device
+from ._device import Device, DeviceConnector
 from ._protocol import AsyncConfigurable, AsyncReadable
 from ._signal import SignalR
 from ._status import AsyncStatus, WatchableAsyncStatus
-from ._utils import DEFAULT_TIMEOUT, T, WatcherUpdate, merge_gathered_dicts
+from ._utils import DEFAULT_TIMEOUT, StrictEnum, WatcherUpdate, merge_gathered_dicts
 
 
-class DetectorTrigger(str, Enum):
+class DetectorTrigger(StrictEnum):
     """Type of mechanism for triggering a detector to take frames"""
 
     #: Detector generates internal trigger for given rate
@@ -172,7 +168,6 @@ class StandardDetector(
     Flyable,
     Collectable,
     WritesStreamAssets,
-    Generic[T],
 ):
     """
     Useful detector base class for step and fly scanning detectors.
@@ -185,6 +180,7 @@ def __init__(
         writer: DetectorWriter,
         config_sigs: Sequence[SignalR] = (),
         name: str = "",
+        connector: DeviceConnector | None = None,
     ) -> None:
         """
         Constructor
@@ -213,8 +209,7 @@ def __init__(
         self._completable_frames: int = 0
         self._number_of_triggers_iter: Iterator[int] | None = None
         self._initial_frame: int = 0
-
-        super().__init__(name)
+        super().__init__(name, connector=connector)
 
     @property
     def controller(self) -> DetectorController: