Allow to track removed collections in collection-meta.yaml (#173)

* Allow information on removed collections.

* Add comments for all attributes.

* Fix spelling.

Co-authored-by: Maxwell G <[email protected]>

* Consolidate removal.major_version and removed_version as removal.version.

* Adjust error message, add test.

* Avoid unnecessary comparsion.

---------

Co-authored-by: Maxwell G <[email protected]>

changelogs/fragments/173-schema-removal.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "Allow information on removed collections in collection metadata schema (https://github.com/ansible-community/antsibull-core/pull/173)."

src/antsibull_core/collection_meta.py

@@ -13,15 +13,19 @@
 
 import os
 import typing as t
+from collections.abc import Collection
 
 import pydantic as p
 from antsibull_fileutils.yaml import load_yaml_file
 
 from .pydantic import forbid_extras, get_formatted_error_messages
 from .schemas.collection_meta import (
+    BaseRemovalInformation,
     CollectionMetadata,
     CollectionsMetadata,
     RemovalInformation,
+    RemovedCollectionMetadata,
+    RemovedRemovalInformation,
 )
 
 if t.TYPE_CHECKING:
@@ -42,6 +46,12 @@ def __init__(self, *, all_collections: list[str], major_release: int):
         self.all_collections = all_collections
         self.major_release = major_release
 
+    def _validate_removal_base(
+        self, collection: str, removal: BaseRemovalInformation, prefix: str
+    ) -> None:
+        if removal.reason == "renamed" and removal.new_name == collection:
+            self.errors.append(f"{prefix} new_name: Must not be the collection's name")
+
     def _validate_removal(
         self, collection: str, removal: RemovalInformation, prefix: str
     ) -> None:
@@ -63,8 +73,7 @@ def _validate_removal(
                 f" must not be larger than the current major version {self.major_release}"
             )
 
-        if removal.reason == "renamed" and removal.new_name == collection:
-            self.errors.append(f"{prefix} new_name: Must not be the collection's name")
+        self._validate_removal_base(collection, removal, prefix)
 
     def _validate_collection(
         self, collection: str, meta: CollectionMetadata, prefix: str
@@ -75,18 +84,51 @@ def _validate_collection(
         if meta.removal:
             self._validate_removal(collection, meta.removal, f"{prefix} removal ->")
 
-    def validate(self, data: CollectionsMetadata) -> None:
+    def _validate_removal_for_removed(
+        self, collection: str, removal: RemovedRemovalInformation, prefix: str
+    ) -> None:
+        if removal.version.major != self.major_release:
+            self.errors.append(
+                f"{prefix} version: Major version of removal version {removal.version} must"
+                f" be current major version {self.major_release}"
+            )
+
+        if (
+            removal.announce_version is not None
+            and removal.announce_version.major >= self.major_release
+        ):
+            self.errors.append(
+                f"{prefix} announce_version: Major version of {removal.announce_version}"
+                f" must be less than the current major version {self.major_release}"
+            )
+
+        self._validate_removal_base(collection, removal, prefix)
+
+    def _validate_removed_collection(
+        self, collection: str, meta: RemovedCollectionMetadata, prefix: str
+    ) -> None:
+        if meta.repository is None:
+            self.errors.append(f"{prefix} repository: Required field not provided")
+
+        self._validate_removal_for_removed(
+            collection, meta.removal, f"{prefix} removal ->"
+        )
+
+    def _validate_order(self, collection: Collection, what: str) -> None:
+        # Check order
+        sorted_list = sorted(collection)
+        raw_list = list(collection)
+        for raw_entry, sorted_entry in zip(raw_list, sorted_list):
+            if raw_entry != sorted_entry:
+                self.errors.append(
+                    f"{what} must be sorted; "
+                    f"{sorted_entry!r} must come before {raw_entry}"
+                )
+                break
+
+    def _validate_collections(self, data: CollectionsMetadata) -> None:
         # Check order
-        sorted_list = sorted(data.collections)
-        raw_list = list(data.collections)
-        if raw_list != sorted_list:
-            for raw_entry, sorted_entry in zip(raw_list, sorted_list):
-                if raw_entry != sorted_entry:
-                    self.errors.append(
-                        "The collection list must be sorted; "
-                        f"{sorted_entry!r} must come before {raw_entry}"
-                    )
-                    break
+        self._validate_order(data.collections, "The collection list")
 
         # Validate collection data
         remaining_collections = set(self.all_collections)
@@ -105,6 +147,24 @@ def validate(self, data: CollectionsMetadata) -> None:
         for collection in sorted(remaining_collections):
             self.errors.append(f"collections: No metadata present for {collection}")
 
+    def _validate_removed_collections(self, data: CollectionsMetadata) -> None:
+        # Check order
+        self._validate_order(data.removed_collections, "The removed collection list")
+
+        # Validate removed collection data
+        for collection, removed_meta in data.removed_collections.items():
+            if collection in self.all_collections:
+                self.errors.append(
+                    f"removed_collections -> {collection}: Collection in ansible.in"
+                )
+            self._validate_removed_collection(
+                collection, removed_meta, f"removed_collections -> {collection} ->"
+            )
+
+    def validate(self, data: CollectionsMetadata) -> None:
+        self._validate_collections(data)
+        self._validate_removed_collections(data)
+
 
 def lint_collection_meta(
     *, collection_meta_path: StrPath, major_release: int, all_collections: list[str]

src/antsibull_core/schemas/collection_meta.py

@@ -44,14 +44,14 @@ def _convert_pypi_version(v: t.Any) -> t.Any:
 PydanticPypiVersion = Annotated[PypiVer, BeforeValidator(_convert_pypi_version)]
 
 
-class RemovalInformation(p.BaseModel):
+class BaseRemovalInformation(p.BaseModel):
     """
-    Stores metadata on when and why a collection will get removed.
+    Stores metadata on why a collection was/will get removed.
     """
 
     model_config = p.ConfigDict(arbitrary_types_allowed=True)
 
-    major_version: t.Union[int, t.Literal["TBD"]]
+    # The reason because of which the collection will be removed.
     reason: t.Literal[
         "deprecated",
         "considered-unmaintained",
@@ -60,9 +60,19 @@ class RemovalInformation(p.BaseModel):
         "other",
     ]
     reason_text: t.Optional[str] = None
+
+    # The Ansible version in which the announcement was made. This is needed
+    # for changelog generation.
     announce_version: t.Optional[PydanticPypiVersion] = None
+
+    # In case reason=renamed, the new name of the collection.
     new_name: t.Optional[str] = None
+
+    # The link to the discussion of the removal.
     discussion: t.Optional[p.HttpUrl] = None
+
+    # In case reason=renamed, the major Ansible release in which the collection's
+    # contents have been replaced by deprecated redirects.
     redirect_replacement_major_version: t.Optional[int] = None
 
     @p.model_validator(mode="after")  # pyre-ignore[56]
@@ -80,60 +90,136 @@ def _check_reason_text(self) -> Self:
                 )
         return self
 
+    @p.model_validator(mode="after")  # pyre-ignore[56]
+    def _check_renamed_base(self) -> Self:
+        if self.reason == "renamed":
+            if self.new_name is None:
+                raise ValueError("new_name must be provided if reason is 'renamed'")
+        else:
+            if self.new_name is not None:
+                raise ValueError(
+                    "new_name must not be provided if reason is not 'renamed'"
+                )
+            if self.redirect_replacement_major_version is not None:
+                raise ValueError(
+                    "redirect_replacement_major_version must not be provided"
+                    " if reason is not 'renamed'"
+                )
+        return self
+
+
+class RemovalInformation(BaseRemovalInformation):
+    """
+    Stores metadata on when and why a collection will get removed.
+    """
+
+    # The Ansible major version from which the collection will be removed.
+    major_version: t.Union[int, t.Literal["TBD"]]
+
+    @p.model_validator(mode="after")  # pyre-ignore[56]
+    def _check_renamed(self) -> Self:
+        if self.reason == "renamed":
+            if (
+                self.redirect_replacement_major_version is not None
+                and self.major_version != "TBD"
+                and self.redirect_replacement_major_version
+                >= self.major_version  # pyre-ignore[58]
+            ):
+                raise ValueError(
+                    "redirect_replacement_major_version must be smaller than major_version"
+                )
+        else:
+            if self.major_version == "TBD":
+                raise ValueError(
+                    "major_version must not be TBD if reason is not 'renamed'"
+                )
+        return self
+
+
+class RemovedRemovalInformation(BaseRemovalInformation):
+    """
+    Stores metadata on when and why a collection was removed.
+    """
+
+    # The exact version from which the collection has been removed.
+    # This is needed for changelog generation.
+    version: PydanticPypiVersion
+
     @p.model_validator(mode="after")  # pyre-ignore[56]
     def _check_reason_is_renamed(self) -> Self:
         if self.reason != "renamed":
             return self
-        if self.new_name is None:
-            raise ValueError("new_name must be provided if reason is 'renamed'")
         if (
             self.redirect_replacement_major_version is not None
-            and self.major_version != "TBD"
-            and self.redirect_replacement_major_version
-            >= self.major_version  # pyre-ignore[58]
+            and self.redirect_replacement_major_version >= self.version.major
         ):
             raise ValueError(
-                "redirect_replacement_major_version must be smaller than major_version"
-            )
-        return self
-
-    @p.model_validator(mode="after")  # pyre-ignore[56]
-    def _check_reason_is_not_renamed(self) -> Self:
-        if self.reason == "renamed":
-            return self
-        if self.new_name is not None:
-            raise ValueError("new_name must not be provided if reason is not 'renamed'")
-        if self.redirect_replacement_major_version is not None:
-            raise ValueError(
-                "redirect_replacement_major_version must not be provided if reason is not 'renamed'"
+                "redirect_replacement_major_version must be smaller than"
+                " version's major version"
             )
-        if self.major_version == "TBD":
-            raise ValueError("major_version must not be TBD if reason is not 'renamed'")
         return self
 
 
-class CollectionMetadata(p.BaseModel):
+class BaseCollectionMetadata(p.BaseModel):
     """
     Stores metadata about one collection.
     """
 
+    # If the collection does not use changelogs/changelog.yaml, it can provide
+    # a URL where the collection's changelog can be found.
     changelog_url: t.Optional[str] = p.Field(alias="changelog-url", default=None)
+
+    # In case the collection is not located in the root of its repository, the
+    # subdirectory in which the collection appears.
     collection_directory: t.Optional[str] = p.Field(
         alias="collection-directory", default=None
     )
+
+    # The collection's repository.
     repository: t.Optional[str] = None
+
+    # A regular expression to match the collection's version from a tag in the repository.
     tag_version_regex: t.Optional[str] = None
+
+    # A list of maintainers. These should be usernames for the repository's
+    # hosting environment.
     maintainers: list[str] = []
+
+
+class CollectionMetadata(BaseCollectionMetadata):
+    """
+    Stores metadata about one collection.
+    """
+
+    model_config = p.ConfigDict(arbitrary_types_allowed=True)
+
+    # Optional information that the collection will be removed from
+    # a future Ansible release.
     removal: t.Optional[RemovalInformation] = None
 
 
+class RemovedCollectionMetadata(BaseCollectionMetadata):
+    """
+    Stores metadata about a removed collection.
+    """
+
+    model_config = p.ConfigDict(arbitrary_types_allowed=True)
+
+    # Information why the collection has been removed
+    removal: RemovedRemovalInformation
+
+
 class CollectionsMetadata(p.BaseModel):
     """
     Stores metadata about a set of collections.
     """
 
+    # Metadata on the collections included in Ansible.
     collections: dict[str, CollectionMetadata]
 
+    # Metadata on the collections removed from this major version of Ansible.
+    removed_collections: dict[str, RemovedCollectionMetadata] = {}
+
     @staticmethod
     def load_from(deps_dir: StrPath | None) -> CollectionsMetadata:
         if deps_dir is None: