Update to pydantic v2 (#695)

* pydantic version upgrade

* bump-pydantic changes

* some pydantic changes: note there is bug

* some solutions to pydantic errors. Note: not sure they are the best ones

* some cleaning of the files

* responded to comments

* included a changelog entry

---------

Co-authored-by: Seher Karakuzu <[email protected]>

CHANGELOG.md

@@ -18,3 +18,7 @@ Write the date in place of the "Unreleased" in the case a new version is release
 ### Changed
 
 ### Fixed
+
+### Other
+
+ * Updated the pydantic version in the pyproject.toml. Now the allowed versions are >2.0.0 - <3.0.0 .

pyproject.toml

@@ -80,7 +80,8 @@ all = [
     "prometheus_client",
     "psutil",
     "pyarrow",
-    "pydantic >=1.8.2,<2",
+    "pydantic >=2, <3",
+    "pydantic-settings >=2, <3",
     "python-dateutil",
     "python-jose[cryptography]",
     "python-multipart",
@@ -208,7 +209,8 @@ minimal-server = [
     "parquet",
     "psutil",
     "prometheus_client",
-    "pydantic >=1.8.2,<2",
+    "pydantic >=2, <3",
+    "pydantic-settings >=2, <3",
     "python-dateutil",
     "python-jose[cryptography]",
     "python-multipart",
@@ -259,7 +261,8 @@ server = [
     "prometheus_client",
     "psutil",
     "pyarrow",
-    "pydantic >=1.8.2,<2",
+    "pydantic >=2, <3",
+    "pydantic-settings >=2, <3",
     "python-dateutil",
     "python-jose[cryptography]",
     "python-multipart",

tiled/adapters/zarr.py

@@ -40,6 +40,7 @@ def init_storage(cls, data_uri, structure):
         directory = path_from_uri(data_uri)
         directory.mkdir(parents=True, exist_ok=True)
         storage = zarr.storage.DirectoryStore(str(directory))
+
         zarr.storage.init_array(
             storage,
             shape=shape,

tiled/catalog/adapter.py

@@ -577,6 +577,7 @@ async def create_node(
 
         key = key or self.context.key_maker()
         data_sources = data_sources or []
+
         node = orm.Node(
             key=key,
             ancestors=self.segments,

tiled/client/base.py

@@ -110,6 +110,7 @@ def __init__(
         self._include_data_sources = include_data_sources
         attributes = self.item["attributes"]
         structure_family = attributes["structure_family"]
+
         if structure is not None:
             # Allow the caller to optionally hand us a structure that is already
             # parsed from a dict into a structure dataclass.
@@ -119,6 +120,7 @@ def __init__(
         else:
             structure_type = STRUCTURE_TYPES[attributes["structure_family"]]
             self._structure = structure_type.from_json(attributes["structure"])
+
         super().__init__()
 
     def structure(self):
@@ -215,6 +217,7 @@ def data_sources(self):
 client or pass the optional parameter `include_data_sources=True` to
 `from_uri(...)` or similar."""
             )
+
         return self.include_data_sources().item["attributes"].get("data_sources")
 
     def include_data_sources(self):

tiled/client/container.py

@@ -602,6 +602,7 @@ def new(
             if isinstance(spec, str):
                 spec = Spec(spec)
             normalized_specs.append(asdict(spec))
+
         item = {
             "attributes": {
                 "metadata": metadata,
@@ -627,6 +628,7 @@ def new(
                 content=safe_json_dump(body),
             )
         ).json()
+
         if structure_family == StructureFamily.container:
             structure = {"contents": None, "count": None}
         else:

tiled/client/utils.py

@@ -129,6 +129,7 @@ def client_for_item(
             class_ = structure_clients[structure_family]
         except KeyError:
             raise UnknownStructureFamily(structure_family) from None
+
     return class_(
         context=context,
         item=item,

tiled/server/authentication.py

@@ -27,6 +27,7 @@
 from fastapi.security.api_key import APIKeyBase, APIKeyCookie, APIKeyQuery
 from fastapi.security.utils import get_authorization_scheme_param
 from fastapi.templating import Jinja2Templates
+from pydantic_settings import BaseSettings
 from sqlalchemy.future import select
 from sqlalchemy.orm import selectinload
 from sqlalchemy.sql import func
@@ -45,7 +46,7 @@
     warnings.simplefilter("ignore")
     from jose import ExpiredSignatureError, JWTError, jwt
 
-from pydantic import BaseModel, BaseSettings
+from pydantic import BaseModel
 
 from ..authn_database import orm
 from ..authn_database.connection_pool import get_database_session

tiled/server/core.py

@@ -433,6 +433,7 @@ async def construct_resource(
         attributes["specs"] = specs
     if (entry is not None) and entry.structure_family == StructureFamily.container:
         attributes["structure_family"] = StructureFamily.container
+
         if schemas.EntryFields.structure in fields:
             if (
                 ((max_depth is None) or (depth < max_depth))
@@ -497,6 +498,7 @@ async def construct_resource(
             "id": id_,
             "attributes": schemas.NodeAttributes(**attributes),
         }
+
         if not omit_links:
             d["links"] = links_for_node(
                 entry.structure_family,
@@ -529,6 +531,7 @@ async def construct_resource(
                 attributes["structure_family"] = entry.structure_family
             if schemas.EntryFields.structure in fields:
                 attributes["structure"] = structure
+
         else:
             # We only have entry names, not structure_family, so
             ResourceLinksT = schemas.SelfLinkOnly

tiled/server/dependencies.py

@@ -1,7 +1,7 @@
 from functools import lru_cache
 from typing import Optional
 
-import pydantic
+import pydantic_settings
 from fastapi import Depends, HTTPException, Query, Request, Security
 from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND
 
@@ -54,7 +54,7 @@ async def inner(
         path: str,
         request: Request,
         principal: str = Depends(get_current_principal),
-        root_tree: pydantic.BaseSettings = Depends(get_root_tree),
+        root_tree: pydantic_settings.BaseSettings = Depends(get_root_tree),
         session_state: dict = Depends(get_session_state),
     ):
         """

tiled/server/pydantic_array.py

@@ -74,7 +74,7 @@ def from_json(cls, structure):
 class Field(BaseModel):
     name: str
     dtype: Union[BuiltinDtype, "StructDtype"]
-    shape: Optional[Tuple[int, ...]]
+    shape: Optional[Tuple[int, ...]] = None
 
     @classmethod
     def from_numpy_descr(cls, field):
@@ -163,6 +163,9 @@ class ArrayStructure(BaseModel):
     dims: Optional[Tuple[str, ...]] = None  # None or tuple of names like ("x", "y")
     resizable: Union[bool, Tuple[bool, ...]] = False
 
+    class Config:
+        extra = "forbid"
+
     @classmethod
     def from_json(cls, structure):
         if "fields" in structure["data_type"]:

tiled/server/pydantic_awkward.py

@@ -5,6 +5,9 @@ class AwkwardStructure(pydantic.BaseModel):
     length: int
     form: dict
 
+    class Config:
+        extra = "forbid"
+
     @classmethod
     def from_json(cls, structure):
         return cls(**structure)

tiled/server/pydantic_sparse.py

@@ -12,6 +12,9 @@ class COOStructure(pydantic.BaseModel):
     resizable: Union[bool, Tuple[bool, ...]] = False
     layout: SparseLayout = SparseLayout.COO
 
+    class Config:
+        extra = "forbid"
+
 
 # This may be extended to a Union of structures if more are added.
 SparseStructure = COOStructure

tiled/server/pydantic_table.py

@@ -21,6 +21,9 @@ class TableStructure(BaseModel):
     columns: List[str]
     resizable: Union[bool, Tuple[bool, ...]] = False
 
+    class Config:
+        extra = "forbid"
+
     @classmethod
     def from_dask_dataframe(cls, ddf):
         import dask.dataframe.utils

tiled/server/router.py

@@ -10,7 +10,7 @@
 import anyio
 from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request, Security
 from jmespath.exceptions import JMESPathError
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings
 from starlette.responses import FileResponse
 from starlette.status import (
     HTTP_200_OK,
@@ -344,6 +344,7 @@ async def metadata(
             detail=f"Malformed 'select_metadata' parameter raised JMESPathError: {err}",
         )
     meta = {"root_path": request.scope.get("root_path") or "/"} if root_path else {}
+
     return json_or_msgpack(
         request,
         schemas.Response(data=resource, meta=meta).dict(),
@@ -1191,6 +1192,7 @@ async def _create_node(
     }
     if metadata_modified:
         response_data["metadata"] = metadata
+
     return json_or_msgpack(request, response_data)