diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd900c70..096530f41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,3 +18,7 @@ Write the date in place of the "Unreleased" in the case a new version is release ### Changed ### Fixed + +### Other + + * Updated the pydantic version in the pyproject.toml. Now the allowed versions are >2.0.0 - <3.0.0 . diff --git a/pyproject.toml b/pyproject.toml index 8d82c11d0..efe78dd90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,8 @@ all = [ "prometheus_client", "psutil", "pyarrow", - "pydantic >=1.8.2,<2", + "pydantic >=2, <3", + "pydantic-settings >=2, <3", "python-dateutil", "python-jose[cryptography]", "python-multipart", @@ -208,7 +209,8 @@ minimal-server = [ "parquet", "psutil", "prometheus_client", - "pydantic >=1.8.2,<2", + "pydantic >=2, <3", + "pydantic-settings >=2, <3", "python-dateutil", "python-jose[cryptography]", "python-multipart", @@ -259,7 +261,8 @@ server = [ "prometheus_client", "psutil", "pyarrow", - "pydantic >=1.8.2,<2", + "pydantic >=2, <3", + "pydantic-settings >=2, <3", "python-dateutil", "python-jose[cryptography]", "python-multipart", diff --git a/tiled/adapters/zarr.py b/tiled/adapters/zarr.py index 3ecd436bb..40834a67d 100644 --- a/tiled/adapters/zarr.py +++ b/tiled/adapters/zarr.py @@ -40,6 +40,7 @@ def init_storage(cls, data_uri, structure): directory = path_from_uri(data_uri) directory.mkdir(parents=True, exist_ok=True) storage = zarr.storage.DirectoryStore(str(directory)) + zarr.storage.init_array( storage, shape=shape, diff --git a/tiled/catalog/adapter.py b/tiled/catalog/adapter.py index c7f0ff39c..062df4940 100644 --- a/tiled/catalog/adapter.py +++ b/tiled/catalog/adapter.py @@ -577,6 +577,7 @@ async def create_node( key = key or self.context.key_maker() data_sources = data_sources or [] + node = orm.Node( key=key, ancestors=self.segments, diff --git a/tiled/client/base.py b/tiled/client/base.py index 80a1550d7..6ea5432e3 100644 --- a/tiled/client/base.py +++ b/tiled/client/base.py @@ -110,6 +110,7 @@ def __init__( self._include_data_sources = include_data_sources attributes = self.item["attributes"] structure_family = attributes["structure_family"] + if structure is not None: # Allow the caller to optionally hand us a structure that is already # parsed from a dict into a structure dataclass. @@ -119,6 +120,7 @@ def __init__( else: structure_type = STRUCTURE_TYPES[attributes["structure_family"]] self._structure = structure_type.from_json(attributes["structure"]) + super().__init__() def structure(self): @@ -215,6 +217,7 @@ def data_sources(self): client or pass the optional parameter `include_data_sources=True` to `from_uri(...)` or similar.""" ) + return self.include_data_sources().item["attributes"].get("data_sources") def include_data_sources(self): diff --git a/tiled/client/container.py b/tiled/client/container.py index 4ee5da122..ff077caf7 100644 --- a/tiled/client/container.py +++ b/tiled/client/container.py @@ -602,6 +602,7 @@ def new( if isinstance(spec, str): spec = Spec(spec) normalized_specs.append(asdict(spec)) + item = { "attributes": { "metadata": metadata, @@ -627,6 +628,7 @@ def new( content=safe_json_dump(body), ) ).json() + if structure_family == StructureFamily.container: structure = {"contents": None, "count": None} else: diff --git a/tiled/client/utils.py b/tiled/client/utils.py index a29a3395c..5cd294042 100644 --- a/tiled/client/utils.py +++ b/tiled/client/utils.py @@ -129,6 +129,7 @@ def client_for_item( class_ = structure_clients[structure_family] except KeyError: raise UnknownStructureFamily(structure_family) from None + return class_( context=context, item=item, diff --git a/tiled/server/authentication.py b/tiled/server/authentication.py index 6877711ce..78d5328c2 100644 --- a/tiled/server/authentication.py +++ b/tiled/server/authentication.py @@ -27,6 +27,7 @@ from fastapi.security.api_key import APIKeyBase, APIKeyCookie, APIKeyQuery from fastapi.security.utils import get_authorization_scheme_param from fastapi.templating import Jinja2Templates +from pydantic_settings import BaseSettings from sqlalchemy.future import select from sqlalchemy.orm import selectinload from sqlalchemy.sql import func @@ -45,7 +46,7 @@ warnings.simplefilter("ignore") from jose import ExpiredSignatureError, JWTError, jwt -from pydantic import BaseModel, BaseSettings +from pydantic import BaseModel from ..authn_database import orm from ..authn_database.connection_pool import get_database_session diff --git a/tiled/server/core.py b/tiled/server/core.py index a4b8818a5..813c3f820 100644 --- a/tiled/server/core.py +++ b/tiled/server/core.py @@ -433,6 +433,7 @@ async def construct_resource( attributes["specs"] = specs if (entry is not None) and entry.structure_family == StructureFamily.container: attributes["structure_family"] = StructureFamily.container + if schemas.EntryFields.structure in fields: if ( ((max_depth is None) or (depth < max_depth)) @@ -497,6 +498,7 @@ async def construct_resource( "id": id_, "attributes": schemas.NodeAttributes(**attributes), } + if not omit_links: d["links"] = links_for_node( entry.structure_family, @@ -529,6 +531,7 @@ async def construct_resource( attributes["structure_family"] = entry.structure_family if schemas.EntryFields.structure in fields: attributes["structure"] = structure + else: # We only have entry names, not structure_family, so ResourceLinksT = schemas.SelfLinkOnly diff --git a/tiled/server/dependencies.py b/tiled/server/dependencies.py index 59773f73d..ccca20785 100644 --- a/tiled/server/dependencies.py +++ b/tiled/server/dependencies.py @@ -1,7 +1,7 @@ from functools import lru_cache from typing import Optional -import pydantic +import pydantic_settings from fastapi import Depends, HTTPException, Query, Request, Security from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND @@ -54,7 +54,7 @@ async def inner( path: str, request: Request, principal: str = Depends(get_current_principal), - root_tree: pydantic.BaseSettings = Depends(get_root_tree), + root_tree: pydantic_settings.BaseSettings = Depends(get_root_tree), session_state: dict = Depends(get_session_state), ): """ diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index 81617f433..b469055a9 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -74,7 +74,7 @@ def from_json(cls, structure): class Field(BaseModel): name: str dtype: Union[BuiltinDtype, "StructDtype"] - shape: Optional[Tuple[int, ...]] + shape: Optional[Tuple[int, ...]] = None @classmethod def from_numpy_descr(cls, field): @@ -163,6 +163,9 @@ class ArrayStructure(BaseModel): dims: Optional[Tuple[str, ...]] = None # None or tuple of names like ("x", "y") resizable: Union[bool, Tuple[bool, ...]] = False + class Config: + extra = "forbid" + @classmethod def from_json(cls, structure): if "fields" in structure["data_type"]: diff --git a/tiled/server/pydantic_awkward.py b/tiled/server/pydantic_awkward.py index a4a3d22f0..eda918427 100644 --- a/tiled/server/pydantic_awkward.py +++ b/tiled/server/pydantic_awkward.py @@ -5,6 +5,9 @@ class AwkwardStructure(pydantic.BaseModel): length: int form: dict + class Config: + extra = "forbid" + @classmethod def from_json(cls, structure): return cls(**structure) diff --git a/tiled/server/pydantic_sparse.py b/tiled/server/pydantic_sparse.py index e12ff8268..38d6056ae 100644 --- a/tiled/server/pydantic_sparse.py +++ b/tiled/server/pydantic_sparse.py @@ -12,6 +12,9 @@ class COOStructure(pydantic.BaseModel): resizable: Union[bool, Tuple[bool, ...]] = False layout: SparseLayout = SparseLayout.COO + class Config: + extra = "forbid" + # This may be extended to a Union of structures if more are added. SparseStructure = COOStructure diff --git a/tiled/server/pydantic_table.py b/tiled/server/pydantic_table.py index c9e99f7cf..a8fcba968 100644 --- a/tiled/server/pydantic_table.py +++ b/tiled/server/pydantic_table.py @@ -21,6 +21,9 @@ class TableStructure(BaseModel): columns: List[str] resizable: Union[bool, Tuple[bool, ...]] = False + class Config: + extra = "forbid" + @classmethod def from_dask_dataframe(cls, ddf): import dask.dataframe.utils diff --git a/tiled/server/router.py b/tiled/server/router.py index 034d85145..cf6eb5b88 100644 --- a/tiled/server/router.py +++ b/tiled/server/router.py @@ -10,7 +10,7 @@ import anyio from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request, Security from jmespath.exceptions import JMESPathError -from pydantic import BaseSettings +from pydantic_settings import BaseSettings from starlette.responses import FileResponse from starlette.status import ( HTTP_200_OK, @@ -344,6 +344,7 @@ async def metadata( detail=f"Malformed 'select_metadata' parameter raised JMESPathError: {err}", ) meta = {"root_path": request.scope.get("root_path") or "/"} if root_path else {} + return json_or_msgpack( request, schemas.Response(data=resource, meta=meta).dict(), @@ -1191,6 +1192,7 @@ async def _create_node( } if metadata_modified: response_data["metadata"] = metadata + return json_or_msgpack(request, response_data) diff --git a/tiled/server/schemas.py b/tiled/server/schemas.py index 39eb85fce..270078ee2 100644 --- a/tiled/server/schemas.py +++ b/tiled/server/schemas.py @@ -5,10 +5,9 @@ from datetime import datetime from typing import Any, Dict, Generic, List, Optional, TypeVar, Union -import pydantic -import pydantic.dataclasses -import pydantic.errors import pydantic.generics +from pydantic import Field, StringConstraints +from typing_extensions import Annotated from ..structures.core import StructureFamily from ..structures.data_source import Management @@ -29,11 +28,11 @@ class Error(pydantic.BaseModel): class Response(pydantic.generics.GenericModel, Generic[DataT, LinksT, MetaT]): data: Optional[DataT] - error: Optional[Error] - links: Optional[LinksT] - meta: Optional[MetaT] + error: Optional[Error] = None + links: Optional[LinksT] = None + meta: Optional[MetaT] = None - @pydantic.validator("error", always=True) + @pydantic.field_validator("error") def check_consistency(cls, v, values): if v is not None and values["data"] is not None: raise ValueError("must not provide both data and error") @@ -62,9 +61,12 @@ class EntryFields(str, enum.Enum): class NodeStructure(pydantic.BaseModel): - contents: Optional[Dict[str, Resource[NodeAttributes, ResourceLinksT, EmptyDict]]] + contents: Optional[Dict[str, Any]] count: int + class Config: + extra = "forbid" + class SortingDirection(int, enum.Enum): ASCENDING = 1 @@ -76,20 +78,20 @@ class SortingItem(pydantic.BaseModel): direction: SortingDirection -class Spec(pydantic.BaseModel, extra=pydantic.Extra.forbid, frozen=True): - name: pydantic.constr(max_length=255) - version: Optional[pydantic.constr(max_length=255)] +class Spec(pydantic.BaseModel, extra="forbid", frozen=True): + name: Annotated[str, StringConstraints(max_length=255)] + version: Optional[Annotated[str, StringConstraints(max_length=255)]] = None # Wait for fix https://github.com/pydantic/pydantic/issues/3957 -# Specs = pydantic.conlist(Spec, max_items=20, unique_items=True) -Specs = pydantic.conlist(Spec, max_items=20) +# Specs = pydantic.conlist(Spec, max_length=20, unique_items=True) +Specs = Annotated[List[Spec], Field(max_length=20)] class Asset(pydantic.BaseModel): data_uri: str is_directory: bool - parameter: Optional[str] + parameter: Optional[str] = None num: Optional[int] = None id: Optional[int] = None @@ -132,13 +134,13 @@ def from_orm(cls, orm): class DataSource(pydantic.BaseModel): id: Optional[int] = None - structure_family: StructureFamily + structure_family: Optional[StructureFamily] = None structure: Optional[ Union[ ArrayStructure, AwkwardStructure, - NodeStructure, SparseStructure, + NodeStructure, TableStructure, ] ] = None @@ -147,6 +149,9 @@ class DataSource(pydantic.BaseModel): assets: List[Asset] = [] management: Management = Management.writable + class Config: + extra = "forbid" + @classmethod def from_orm(cls, orm): return cls( @@ -162,20 +167,24 @@ def from_orm(cls, orm): class NodeAttributes(pydantic.BaseModel): ancestors: List[str] - structure_family: Optional[StructureFamily] - specs: Optional[Specs] - metadata: Optional[Dict] # free-form, user-specified dict + structure_family: Optional[StructureFamily] = None + specs: Optional[Specs] = None + metadata: Optional[Dict] = None # free-form, user-specified dict structure: Optional[ Union[ ArrayStructure, AwkwardStructure, - NodeStructure, SparseStructure, + NodeStructure, TableStructure, ] - ] - sorting: Optional[List[SortingItem]] - data_sources: Optional[List[DataSource]] + ] = None + + sorting: Optional[List[SortingItem]] = None + data_sources: Optional[List[DataSource]] = None + + class Config: + extra = "forbid" AttributesT = TypeVar("AttributesT") @@ -240,8 +249,8 @@ class Resource( "A JSON API Resource" id: Union[str, uuid.UUID] attributes: AttributesT - links: Optional[ResourceLinksT] - meta: Optional[ResourceMetaT] + links: Optional[ResourceLinksT] = None + meta: Optional[ResourceMetaT] = None class AccessAndRefreshTokens(pydantic.BaseModel): @@ -270,7 +279,7 @@ class AboutAuthenticationProvider(pydantic.BaseModel): provider: str mode: AuthenticationMode links: Dict[str, str] - confirmation_message: Optional[str] + confirmation_message: Optional[str] = None class AboutAuthenticationLinks(pydantic.BaseModel): @@ -284,7 +293,7 @@ class AboutAuthenticationLinks(pydantic.BaseModel): class AboutAuthentication(pydantic.BaseModel): required: bool providers: List[AboutAuthenticationProvider] - links: Optional[AboutAuthenticationLinks] + links: Optional[AboutAuthenticationLinks] = None class About(pydantic.BaseModel): @@ -303,22 +312,25 @@ class PrincipalType(str, enum.Enum): service = "service" -class Identity(pydantic.BaseModel, orm_mode=True): - id: pydantic.constr(max_length=255) - provider: pydantic.constr(max_length=255) - latest_login: Optional[datetime] +class Identity(pydantic.BaseModel): + model_config = pydantic.ConfigDict(from_attributes=True) + id: Annotated[str, StringConstraints(max_length=255)] + provider: Annotated[str, StringConstraints(max_length=255)] + latest_login: Optional[datetime] = None -class Role(pydantic.BaseModel, orm_mode=True): +class Role(pydantic.BaseModel): + model_config = pydantic.ConfigDict(from_attributes=True) name: str scopes: List[str] # principals -class APIKey(pydantic.BaseModel, orm_mode=True): - first_eight: pydantic.constr(min_length=8, max_length=8) - expiration_time: Optional[datetime] - note: Optional[pydantic.constr(max_length=255)] +class APIKey(pydantic.BaseModel): + model_config = pydantic.ConfigDict(from_attributes=True) + first_eight: Annotated[str, StringConstraints(min_length=8, max_length=8)] + expiration_time: Optional[datetime] = None + note: Optional[Annotated[str, StringConstraints(max_length=255)]] = None scopes: List[str] latest_activity: Optional[datetime] = None @@ -338,7 +350,7 @@ def from_orm(cls, orm, secret): ) -class Session(pydantic.BaseModel, orm_mode=True): +class Session(pydantic.BaseModel): """ This related to refresh tokens, which have a session uuid ("sid") claim. @@ -349,15 +361,17 @@ class Session(pydantic.BaseModel, orm_mode=True): # The id field (primary key) is intentionally not exposed to the application. # It is left as an internal database concern. + model_config = pydantic.ConfigDict(from_attributes=True) uuid: uuid.UUID expiration_time: datetime revoked: bool -class Principal(pydantic.BaseModel, orm_mode=True): +class Principal(pydantic.BaseModel): "Represents a User or Service" # The id field (primary key) is intentionally not exposed to the application. # It is left as an internal database concern. + model_config = pydantic.ConfigDict(from_attributes=True) uuid: uuid.UUID type: PrincipalType identities: List[Identity] = [] @@ -379,7 +393,7 @@ class APIKeyRequestParams(pydantic.BaseModel): # try to use the instantly-expiring API key! expires_in: Optional[int] = pydantic.Field(..., example=600) # seconds scopes: Optional[List[str]] = pydantic.Field(..., example=["inherit"]) - note: Optional[str] + note: Optional[str] = None class PostMetadataRequest(pydantic.BaseModel): @@ -391,13 +405,13 @@ class PostMetadataRequest(pydantic.BaseModel): # Wait for fix https://github.com/pydantic/pydantic/issues/3957 # to do this with `unique_items` parameters to `pydantic.constr`. - @pydantic.validator("specs", always=True) + @pydantic.field_validator("specs") def specs_uniqueness_validator(cls, v): if v is None: return None for i, value in enumerate(v, start=1): if value in v[i:]: - raise pydantic.errors.ListUniqueItemsError() + raise ValueError return v @@ -416,35 +430,35 @@ class PutMetadataResponse(pydantic.BaseModel, Generic[ResourceLinksT]): id: str links: Union[ArrayLinks, DataFrameLinks, SparseLinks] # May be None if not altered - metadata: Optional[Dict] - data_sources: Optional[List[DataSource]] + metadata: Optional[Dict] = None + data_sources: Optional[List[DataSource]] = None class DistinctValueInfo(pydantic.BaseModel): - value: Any - count: Optional[int] + value: Any = None + count: Optional[int] = None class GetDistinctResponse(pydantic.BaseModel): - metadata: Optional[Dict[str, List[DistinctValueInfo]]] - structure_families: Optional[List[DistinctValueInfo]] - specs: Optional[List[DistinctValueInfo]] + metadata: Optional[Dict[str, List[DistinctValueInfo]]] = None + structure_families: Optional[List[DistinctValueInfo]] = None + specs: Optional[List[DistinctValueInfo]] = None class PutMetadataRequest(pydantic.BaseModel): # These fields are optional because None means "no changes; do not update". - metadata: Optional[Dict] - specs: Optional[Specs] + metadata: Optional[Dict] = None + specs: Optional[Specs] = None # Wait for fix https://github.com/pydantic/pydantic/issues/3957 # to do this with `unique_items` parameters to `pydantic.constr`. - @pydantic.validator("specs", always=True) + @pydantic.field_validator("specs") def specs_uniqueness_validator(cls, v): if v is None: return None for i, value in enumerate(v, start=1): if value in v[i:]: - raise pydantic.errors.ListUniqueItemsError() + raise ValueError return v diff --git a/tiled/server/settings.py b/tiled/server/settings.py index 426aaa986..071aba070 100644 --- a/tiled/server/settings.py +++ b/tiled/server/settings.py @@ -5,7 +5,7 @@ from functools import lru_cache from typing import Any, List, Optional -from pydantic import BaseSettings +from pydantic_settings import BaseSettings DatabaseSettings = collections.namedtuple( "DatabaseSettings", "uri pool_size pool_pre_ping max_overflow" @@ -20,14 +20,18 @@ class Settings(BaseSettings): allow_origins: List[str] = [ item for item in os.getenv("TILED_ALLOW_ORIGINS", "").split() if item ] - object_cache_available_bytes = float( + object_cache_available_bytes: float = float( os.getenv("TILED_OBJECT_CACHE_AVAILABLE_BYTES", "0.15") ) - object_cache_log_level = os.getenv("TILED_OBJECT_CACHE_LOG_LEVEL", "INFO") + object_cache_log_level: str = os.getenv("TILED_OBJECT_CACHE_LOG_LEVEL", "INFO") authenticator: Any = None # These 'single user' settings are only applicable if authenticator is None. - single_user_api_key = os.getenv("TILED_SINGLE_USER_API_KEY", secrets.token_hex(32)) - single_user_api_key_generated = not ("TILED_SINGLE_USER_API_KEY" in os.environ) + single_user_api_key: str = os.getenv( + "TILED_SINGLE_USER_API_KEY", secrets.token_hex(32) + ) + single_user_api_key_generated: bool = not ( + "TILED_SINGLE_USER_API_KEY" in os.environ + ) # The TILED_SERVER_SECRET_KEYS may be a single key or a ;-separated list of # keys to support key rotation. The first key will be used for encryption. Each # key will be tried in turn for decryption. @@ -48,10 +52,12 @@ class Settings(BaseSettings): # Put a fairly low limit on the maximum size of one chunk, keeping in mind # that data should generally be chunked. When we implement async responses, # we can raise this global limit. - response_bytesize_limit = int( + response_bytesize_limit: int = int( os.getenv("TILED_RESPONSE_BYTESIZE_LIMIT", 300_000_000) ) # 300 MB - reject_undeclared_specs = bool(int(os.getenv("TILED_REJECT_UNDECLARED_SPECS", 0))) + reject_undeclared_specs: bool = bool( + int(os.getenv("TILED_REJECT_UNDECLARED_SPECS", 0)) + ) database_uri: Optional[str] = os.getenv("TILED_DATABASE_URI") database_init_if_not_exists: bool = int( os.getenv("TILED_DATABASE_INIT_IF_NOT_EXISTS", False)