diff --git a/doc/changes/DM-38687.removal.md b/doc/changes/DM-38687.removal.md new file mode 100644 index 0000000000..e42ac64d40 --- /dev/null +++ b/doc/changes/DM-38687.removal.md @@ -0,0 +1,3 @@ +Removed various already-deprecated factory methods for `DimensionPacker` objects and their support code, as well as the concrete `ObservationDimensionPacker`. + +While `daf_butler` still defines the `DimensionPacker` abstract interface, all construction logic has moved to downstream packages. diff --git a/python/lsst/daf/butler/__init__.py b/python/lsst/daf/butler/__init__.py index 743766dd55..7ac88bef8d 100644 --- a/python/lsst/daf/butler/__init__.py +++ b/python/lsst/daf/butler/__init__.py @@ -53,7 +53,7 @@ from ._file_descriptor import * from ._formatter import * -# Do not import 'instrument' or 'json' at all by default. +# Do not import 'json' at all by default. from ._limited_butler import * from ._location import * from ._named import * diff --git a/python/lsst/daf/butler/dimensions/_config.py b/python/lsst/daf/butler/dimensions/_config.py index b9333fc6a6..1b562163ed 100644 --- a/python/lsst/daf/butler/dimensions/_config.py +++ b/python/lsst/daf/butler/dimensions/_config.py @@ -29,7 +29,6 @@ __all__ = ("DimensionConfig",) -import warnings from collections.abc import Iterator, Mapping, Sequence from typing import Any @@ -45,7 +44,6 @@ ) from ._elements import KeyColumnSpec, MetadataColumnSpec from ._governor import GovernorDimensionConstructionVisitor -from ._packer import DimensionPackerConstructionVisitor from ._skypix import SkyPixConstructionVisitor from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor @@ -77,8 +75,7 @@ class DimensionConfig(ConfigSubset): - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, with dictionary values that each define a `StandardTopologicalFamily`. - - packers: a nested dictionary whose entries define factories for a - `DimensionPacker` instance. + - packers: ignored. See the documentation for the linked classes above for more information on the configuration syntax. @@ -267,31 +264,6 @@ def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: members=members, ) - # TODO: remove this method and callers on DM-38687. - # Note that the corresponding entries in the dimensions config should - # not be removed at that time, because that's formally a schema migration. - def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]: - """Process the 'packers' section of the configuration. - - Yields construction visitors for each `DimensionPackerFactory`. - - Yields - ------ - visitor : `DimensionConstructionVisitor` - Object that adds a `DinmensionPackerFactory` to an - under-construction `DimensionUniverse`. - """ - with warnings.catch_warnings(): - # Don't warn when deprecated code calls other deprecated code. - warnings.simplefilter("ignore", FutureWarning) - for name, subconfig in self["packers"].items(): - yield DimensionPackerConstructionVisitor( - name=name, - clsName=subconfig["cls"], - fixed=subconfig["fixed"], - dimensions=subconfig["dimensions"], - ) - def makeBuilder(self) -> DimensionConstructionBuilder: """Construct a `DinmensionConstructionBuilder`. @@ -313,5 +285,4 @@ def makeBuilder(self) -> DimensionConstructionBuilder: builder.update(self._extractSkyPixVisitors()) builder.update(self._extractElementVisitors()) builder.update(self._extractTopologyVisitors()) - builder.update(self._extractPackerVisitors()) return builder diff --git a/python/lsst/daf/butler/dimensions/_coordinate.py b/python/lsst/daf/butler/dimensions/_coordinate.py index 6b1935b51a..c1ad2c436b 100644 --- a/python/lsst/daf/butler/dimensions/_coordinate.py +++ b/python/lsst/daf/butler/dimensions/_coordinate.py @@ -45,7 +45,7 @@ import warnings from abc import abstractmethod from collections.abc import Iterable, Iterator, Mapping, Set -from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast, overload +from typing import TYPE_CHECKING, Any, ClassVar, cast from deprecated.sphinx import deprecated from lsst.daf.butler._compat import _BaseModelCompat @@ -874,50 +874,6 @@ def timespan(self) -> Timespan | None: else: return Timespan.intersection(*timespans) - @overload - def pack(self, name: str, *, returnMaxBits: Literal[True]) -> tuple[int, int]: - ... - - @overload - def pack(self, name: str, *, returnMaxBits: Literal[False]) -> int: - ... - - # TODO: Remove this method and its overloads above on DM-38687. - @deprecated( - "Deprecated in favor of configurable dimension packers. Will be removed after v26.", - version="v26", - category=FutureWarning, - ) - def pack(self, name: str, *, returnMaxBits: bool = False) -> tuple[int, int] | int: - """Pack this data ID into an integer. - - Parameters - ---------- - name : `str` - Name of the `DimensionPacker` algorithm (as defined in the - dimension configuration). - returnMaxBits : `bool`, optional - If `True` (`False` is default), return the maximum number of - nonzero bits in the returned integer across all data IDs. - - Returns - ------- - packed : `int` - Integer ID. This ID is unique only across data IDs that have - the same values for the packer's "fixed" dimensions. - maxBits : `int`, optional - Maximum number of nonzero bits in ``packed``. Not returned unless - ``returnMaxBits`` is `True`. - - Notes - ----- - Accessing this attribute if `hasRecords` returns `False` is a logic - error that may or may not raise an exception, depending on the - implementation and whether assertions are enabled. - """ - assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." - return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) - def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate: """Convert this class to a simple python type. diff --git a/python/lsst/daf/butler/dimensions/_packer.py b/python/lsst/daf/butler/dimensions/_packer.py index 84a0f637f7..4c4cc32c62 100644 --- a/python/lsst/daf/butler/dimensions/_packer.py +++ b/python/lsst/daf/butler/dimensions/_packer.py @@ -29,17 +29,11 @@ __all__ = ("DimensionPacker",) -import warnings from abc import ABCMeta, abstractmethod -from collections.abc import Iterable, Set from typing import TYPE_CHECKING, Any -from deprecated.sphinx import deprecated -from lsst.utils import doImportType - from ._coordinate import DataCoordinate, DataId from ._graph import DimensionGraph, DimensionGroup -from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. from ._universe import DimensionUniverse @@ -186,122 +180,3 @@ def unpack(self, packedId: int) -> DataCoordinate: The packed ID values are only unique and reversible with these dimensions held fixed. """ - - -# TODO: Remove this class on DM-38687. -@deprecated( - "Deprecated in favor of configurable dimension packers. Will be removed after v26.", - version="v26", - category=FutureWarning, -) -class DimensionPackerFactory: - """A factory class for `DimensionPacker` instances. - - Can be constructed from configuration. - - This class is primarily intended for internal use by `DimensionUniverse`. - - Parameters - ---------- - clsName : `str` - Fully-qualified name of the packer class this factory constructs. - fixed : `~collections.abc.Set` [ `str` ] - Names of dimensions whose values must be provided to the packer when it - is constructed. This will be expanded lazily into a `DimensionGroup` - prior to `DimensionPacker` construction. - dimensions : `~collections.abc.Set` [ `str` ] - Names of dimensions whose values are passed to `DimensionPacker.pack`. - This will be expanded lazily into a `DimensionGroup` prior to - `DimensionPacker` construction. - """ - - def __init__( - self, - clsName: str, - fixed: Set[str], - dimensions: Set[str], - ): - # We defer turning these into DimensionGroup objects until first use - # because __init__ is called before a DimensionUniverse exists, and - # DimensionGroup instances can only be constructed afterwards. - self._fixed: Set[str] | DimensionGroup = fixed - self._dimensions: Set[str] | DimensionGroup = dimensions - self._clsName = clsName - self._cls: type[DimensionPacker] | None = None - - def __call__(self, universe: DimensionUniverse, fixed: DataCoordinate) -> DimensionPacker: - """Construct a `DimensionPacker` instance for the given fixed data ID. - - Parameters - ---------- - universe : `DimensionUniverse` - The dimension universe for which this packer should be created. - fixed : `DataCoordinate` - Data ID that provides values for the "fixed" dimensions of the - packer. Must be expanded with all metadata known to the - `Registry`. ``fixed.hasRecords()`` must return `True`. - """ - # Construct DimensionGroup instances if necessary on first use. - # See related comment in __init__. - self._fixed = universe.conform(self._fixed) - self._dimensions = universe.conform(self._dimensions) - assert fixed.graph.issuperset(self._fixed) - if self._cls is None: - packer_class = doImportType(self._clsName) - assert not isinstance( - packer_class, DimensionPacker - ), f"Packer class {self._clsName} must be a DimensionPacker." - self._cls = packer_class - return self._cls(fixed, self._dimensions) - - -# TODO: Remove this class on DM-38687. -@deprecated( - "Deprecated in favor of configurable dimension packers. Will be removed after v26.", - version="v26", - category=FutureWarning, -) -class DimensionPackerConstructionVisitor(DimensionConstructionVisitor): - """Builder visitor for a single `DimensionPacker`. - - A single `DimensionPackerConstructionVisitor` should be added to a - `DimensionConstructionBuilder` for each `DimensionPackerFactory` that - should be added to a universe. - - Parameters - ---------- - name : `str` - Name used to identify this configuration of the packer in a - `DimensionUniverse`. - clsName : `str` - Fully-qualified name of a `DimensionPacker` subclass. - fixed : `~collections.abc.Iterable` [ `str` ] - Names of dimensions whose values must be provided to the packer when it - is constructed. This will be expanded lazily into a `DimensionGroup` - prior to `DimensionPacker` construction. - dimensions : `~collections.abc.Iterable` [ `str` ] - Names of dimensions whose values are passed to `DimensionPacker.pack`. - This will be expanded lazily into a `DimensionGroup` prior to - `DimensionPacker` construction. - """ - - def __init__(self, name: str, clsName: str, fixed: Iterable[str], dimensions: Iterable[str]): - super().__init__(name) - self._fixed = set(fixed) - self._dimensions = set(dimensions) - self._clsName = clsName - - def hasDependenciesIn(self, others: Set[str]) -> bool: - # Docstring inherited from DimensionConstructionVisitor. - return False - - def visit(self, builder: DimensionConstructionBuilder) -> None: - # Docstring inherited from DimensionConstructionVisitor. - with warnings.catch_warnings(): - # Don't warn when deprecated code calls other deprecated code. - warnings.simplefilter("ignore", FutureWarning) - builder.packers[self.name] = DimensionPackerFactory( - clsName=self._clsName, - fixed=self._fixed, - dimensions=self._dimensions, - ) diff --git a/python/lsst/daf/butler/dimensions/_universe.py b/python/lsst/daf/butler/dimensions/_universe.py index f71533ba3d..d447c83617 100644 --- a/python/lsst/daf/butler/dimensions/_universe.py +++ b/python/lsst/daf/butler/dimensions/_universe.py @@ -51,8 +51,6 @@ from ._skypix import SkyPixDimension, SkyPixSystem if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. - from ._coordinate import DataCoordinate - from ._packer import DimensionPacker, DimensionPackerFactory from .construction import DimensionConstructionBuilder @@ -148,7 +146,6 @@ def __new__( self._dimensions = builder.dimensions self._elements = builder.elements self._topology = builder.topology - self._packers = builder.packers self.dimensionConfig = builder.config commonSkyPix = self._dimensions[builder.commonSkyPixName] assert isinstance(commonSkyPix, SkyPixDimension) @@ -558,31 +555,6 @@ def sorted(self, elements: Iterable[Any], *, reverse: bool = False) -> list[Any] result.reverse() return result - # TODO: Remove this method on DM-38687. - @deprecated( - "Deprecated in favor of configurable dimension packers. Will be removed after v26.", - version="v26", - category=FutureWarning, - ) - def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: - """Make a dimension packer. - - Constructs a `DimensionPacker` that can pack data ID dictionaries - into unique integers. - - Parameters - ---------- - name : `str` - Name of the packer, matching a key in the "packers" section of the - dimension configuration. - dataId : `DataCoordinate` - Fully-expanded data ID that identifies the at least the "fixed" - dimensions of the packer (i.e. those that are assumed/given, - setting the space over which packed integer IDs are unique). - ``dataId.hasRecords()`` must return `True`. - """ - return self._packers[name](self, dataId) - def getEncodeLength(self) -> int: """Return encoded size of graph. @@ -674,8 +646,6 @@ def __deepcopy__(self, memo: dict) -> DimensionUniverse: _elementIndices: dict[str, int] - _packers: dict[str, DimensionPackerFactory] - _populates: defaultdict[str, NamedValueSet[DimensionElement]] _version: int diff --git a/python/lsst/daf/butler/dimensions/construction.py b/python/lsst/daf/butler/dimensions/construction.py index c7c12d8ce8..d11d7a2f47 100644 --- a/python/lsst/daf/butler/dimensions/construction.py +++ b/python/lsst/daf/butler/dimensions/construction.py @@ -37,7 +37,6 @@ if TYPE_CHECKING: from ._config import DimensionConfig from ._elements import Dimension, DimensionElement - from ._packer import DimensionPackerFactory class DimensionConstructionVisitor(ABC): @@ -137,7 +136,6 @@ def __init__( self.dimensions = NamedValueSet() self.elements = NamedValueSet() self.topology = {space: NamedValueSet() for space in TopologicalSpace.__members__.values()} - self.packers = {} self.version = version self.namespace = namespace self.config = config @@ -222,8 +220,3 @@ def finish(self) -> None: """Dictionary containing all `TopologicalFamily` objects (`dict` [ `TopologicalSpace`, `NamedValueSet` [ `TopologicalFamily` ] ] ). """ - - packers: dict[str, DimensionPackerFactory] - """Dictionary containing all `DimensionPackerFactory` objects - (`dict` [ `str`, `DimensionPackerFactory` ] ). - """ diff --git a/python/lsst/daf/butler/instrument.py b/python/lsst/daf/butler/instrument.py deleted file mode 100644 index 4e33bf7e26..0000000000 --- a/python/lsst/daf/butler/instrument.py +++ /dev/null @@ -1,92 +0,0 @@ -# This file is part of daf_butler. -# -# Developed for the LSST Data Management System. -# This product includes software developed by the LSST Project -# (http://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This software is dual licensed under the GNU General Public License and also -# under a 3-clause BSD license. Recipients may choose which of these licenses -# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, -# respectively. If you choose the GPL option then the following text applies -# (but note that there is still no warranty even if you opt for BSD instead): -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -__all__ = ["ObservationDimensionPacker"] - -# TODO: Remove this entire module on DM-38687. - -from deprecated.sphinx import deprecated -from lsst.daf.butler import DataCoordinate, DimensionGraph, DimensionGroup, DimensionPacker - - -# TODO: remove on DM-38687. -@deprecated( - "Deprecated in favor of configurable dimension packers. Will be removed after v26.", - version="v26", - category=FutureWarning, -) -class ObservationDimensionPacker(DimensionPacker): - """A `DimensionPacker` for visit+detector or exposure+detector, given an - instrument. - - Parameters - ---------- - fixed : `DataCoordinate` - Expanded data ID for the dimensions whose values must remain fixed - (to these values) in all calls to `pack`, and are used in the results - of calls to `unpack`. Subclasses may ignore particular dimensions, and - are permitted to require that ``fixed.hasRecords()`` return `True`. - dimensions : `DimensionGroup` or `DimensionGraph` - The dimensions of data IDs packed by this instance. - """ - - def __init__(self, fixed: DataCoordinate, dimensions: DimensionGraph | DimensionGroup): - super().__init__(fixed, dimensions) - self._instrumentName = fixed["instrument"] - record = fixed.records["instrument"] - assert record is not None - if self._dimensions.required.names == {"instrument", "visit", "detector"}: - self._observationName = "visit" - obsMax = record.visit_max - elif dimensions.required.names == {"instrument", "exposure", "detector"}: - self._observationName = "exposure" - obsMax = record.exposure_max - else: - raise ValueError(f"Invalid dimensions for ObservationDimensionPacker: {dimensions.required}") - self._detectorMax = record.detector_max - self._maxBits = (obsMax * self._detectorMax).bit_length() - - @property - def maxBits(self) -> int: - # Docstring inherited from DimensionPacker.maxBits - return self._maxBits - - def _pack(self, dataId: DataCoordinate) -> int: - # Docstring inherited from DimensionPacker._pack - return dataId["detector"] + self._detectorMax * dataId[self._observationName] - - def unpack(self, packedId: int) -> DataCoordinate: - # Docstring inherited from DimensionPacker.unpack - observation, detector = divmod(packedId, self._detectorMax) - return DataCoordinate.standardize( - { - "instrument": self._instrumentName, - "detector": detector, - self._observationName: observation, - }, - dimensions=self._dimensions, - )