Add collection-meta.yaml linter (#617)

* Add collection-meta.yaml linter.

* Allow reason=other.

* Use StrPath; fix collection missing error reporting.

* Replace some manual validation with model validators.

* Forgot to reformat.

* Fix message.

Co-authored-by: Maxwell G <[email protected]>

* Improve typing and validator naming, fix lint condition.

* Remove nested if.

* Improve PyPI version validation.

Co-authored-by: Maxwell G <[email protected]>

* Quote value.

Co-authored-by: Maxwell G <[email protected]>

* Add major_ prefix for major version field names.

* Re-do PypiVer parsing.

* Apparently I undid more things...

* Add new reason 'guidelines-violation'.

---------

Co-authored-by: Maxwell G <[email protected]>

.github/workflows/antsibull-build.yml

@@ -29,14 +29,14 @@ jobs:
             options: '-e antsibull_ansible_version=8.99.0'
             python: '3.9'
             antsibull_changelog_ref: 0.24.0
-            antsibull_core_ref: stable-2
+            antsibull_core_ref: main
             antsibull_docutils_ref: 1.0.0  # isn't used by antsibull-changelog 0.24.0
             antsibull_fileutils_ref: main
           - name: Ansible 9
             options: '-e antsibull_ansible_version=9.99.0'
             python: '3.11'
             antsibull_changelog_ref: main
-            antsibull_core_ref: stable-2
+            antsibull_core_ref: main
             antsibull_docutils_ref: main
             antsibull_fileutils_ref: main
           - name: Ansible 10

changelogs/fragments/617-collection-meta-linter.yml

@@ -0,0 +1,4 @@
+minor_changes:
+  - "Add subcommand ``lint-collection-meta`` for linting ``collection-meta.yaml`` files in ``ansible-build-data`` (https://github.com/ansible-community/antsibull/pull/617)."
+breaking_changes:
+  - "Antsibull now depends on antsibull-core >= 3.0.0 and pydantic >= 2.0.0 (https://github.com/ansible-community/antsibull/pull/617)."

pyproject.toml

@@ -27,7 +27,7 @@ classifiers = [
 requires-python = ">=3.9"
 dependencies = [
     "antsibull-changelog >= 0.24.0",
-    "antsibull-core >= 2.0.0, < 4.0.0",
+    "antsibull-core >= 3.0.0, < 4.0.0",
     "antsibull-fileutils >= 1.0.0, < 2.0.0",
     "asyncio-pool",
     "build",
@@ -38,8 +38,7 @@ dependencies = [
     "aiohttp >= 3.0.0",
     "twiggy",
     "pyyaml",
-    # We rely on deprecated features to maintain compat btw. pydantic v1 and v2
-    "pydantic < 3",
+    "pydantic >= 2, < 3",
     # pydantic already pulls it in, but we use it for TypedDict
     "typing_extensions",
 ]

src/antsibull/changelog.py

@@ -506,7 +506,7 @@ def get_changelog(
         PypiVer(ansible_ancestor_version_str) if ansible_ancestor_version_str else None
     )
 
-    collection_metadata = CollectionsMetadata(deps_dir)
+    collection_metadata = CollectionsMetadata.load_from(deps_dir)
 
     if deps_dir is not None:
         for path in glob.glob(os.path.join(deps_dir, "*.deps"), recursive=False):

src/antsibull/cli/antsibull_build.py

@@ -54,6 +54,7 @@
     rebuild_single_command,
 )
 from ..build_changelog import build_changelog  # noqa: E402
+from ..collection_meta_lint import lint_collection_meta  # noqa: E402
 from ..constants import MINIMUM_ANSIBLE_VERSION, SANITY_TESTS_DEFAULT  # noqa: E402
 from ..dep_closure import validate_dependencies_command  # noqa: E402
 from ..from_source import verify_upstream_command  # noqa: E402
@@ -84,6 +85,7 @@
     "sanity-tests": sanity_tests_command,
     "announcements": announcements_command,
     "send-announcements": send_announcements_command,
+    "lint-collection-meta": lint_collection_meta,
 }
 DISABLE_VERIFY_UPSTREAMS_IGNORES_SENTINEL = "NONE"
 DEFAULT_ANNOUNCEMENTS_DIR = Path("build/announce")
@@ -107,7 +109,11 @@ def _normalize_build_options(args: argparse.Namespace) -> None:
     ):
         return
 
-    if args.ansible_version < MINIMUM_ANSIBLE_VERSION:
+    if (
+        (args.ansible_version < MINIMUM_ANSIBLE_VERSION)
+        if args.command != "lint-collection-meta"
+        else (args.ansible_major_version < MINIMUM_ANSIBLE_VERSION.major)
+    ):
         raise InvalidArgumentError(
             f"Ansible < {MINIMUM_ANSIBLE_VERSION} is not supported"
             " by this antsibull version."
@@ -136,8 +142,8 @@ def _normalize_build_write_data_options(args: argparse.Namespace) -> None:
         )
 
 
-def _normalize_new_release_options(args: argparse.Namespace) -> None:
-    if args.command != "new-ansible":
+def _normalize_pieces_file_options(args: argparse.Namespace) -> None:
+    if args.command not in ("new-ansible", "lint-collection-meta"):
         return
 
     if args.pieces_file is None:
@@ -151,6 +157,11 @@ def _normalize_new_release_options(args: argparse.Namespace) -> None:
             " per line"
         )
 
+
+def _normalize_new_release_options(args: argparse.Namespace) -> None:
+    if args.command != "new-ansible":
+        return
+
     compat_version_part = f"{args.ansible_version.major}"
 
     if args.build_file is None:
@@ -769,6 +780,26 @@ def parse_args(program_name: str, args: list[str]) -> argparse.Namespace:
         dest="send_actions",
     )
 
+    lint_collection_meta_parser = subparsers.add_parser(
+        "lint-collection-meta",
+        description="Lint the collection-meta.yaml file.",
+    )
+    lint_collection_meta_parser.add_argument(
+        "ansible_major_version",
+        type=int,
+        help="The X major version of Ansible that this will be for",
+    )
+    lint_collection_meta_parser.add_argument(
+        "--data-dir", default=".", help="Directory to read .build and .deps files from"
+    )
+    lint_collection_meta_parser.add_argument(
+        "--pieces-file",
+        default=None,
+        help="File containing a list of collections to include.  This is"
+        " considered to be relative to --data-dir.  The default is"
+        f" {DEFAULT_PIECES_FILE}",
+    )
+
     # This must come after all parser setup
     if HAS_ARGCOMPLETE:
         argcomplete.autocomplete(parser)
@@ -780,6 +811,7 @@ def parse_args(program_name: str, args: list[str]) -> argparse.Namespace:
     _normalize_commands(parsed_args)
     _normalize_build_options(parsed_args)
     _normalize_build_write_data_options(parsed_args)
+    _normalize_pieces_file_options(parsed_args)
     _normalize_new_release_options(parsed_args)
     _normalize_release_build_options(parsed_args)
     _normalize_validate_tags_options(parsed_args)

src/antsibull/collection_meta.py

@@ -13,50 +13,143 @@
 
 import os
 import typing as t
-from collections.abc import Mapping
 
+import pydantic as p
 from antsibull_fileutils.yaml import load_yaml_file
+from packaging.version import Version as PypiVer
+from pydantic.functional_validators import BeforeValidator
+from typing_extensions import Annotated, Self
 
+if t.TYPE_CHECKING:
+    from _typeshed import StrPath
 
-class CollectionMetadata:
+
+def _convert_pypi_version(v: t.Any) -> t.Any:
+    if isinstance(v, str):
+        if not v:
+            raise ValueError(f"must be a non-trivial string, got {v!r}")
+        version = PypiVer(v)
+    elif isinstance(v, PypiVer):
+        version = v
+    else:
+        raise ValueError(f"must be a string or PypiVer object, got {v!r}")
+
+    if len(version.release) != 3:
+        raise ValueError(
+            f"must be a version with three release numbers (e.g. 1.2.3, 2.3.4a1), got {v!r}"
+        )
+    return version
+
+
+PydanticPypiVersion = Annotated[PypiVer, BeforeValidator(_convert_pypi_version)]
+
+
+class RemovalInformation(p.BaseModel):
+    """
+    Stores metadata on when and why a collection will get removed.
+    """
+
+    model_config = p.ConfigDict(extra="ignore", arbitrary_types_allowed=True)
+
+    major_version: t.Union[int, t.Literal["TBD"]]
+    reason: t.Literal[
+        "deprecated",
+        "considered-unmaintained",
+        "renamed",
+        "guidelines-violation",
+        "other",
+    ]
+    reason_text: t.Optional[str] = None
+    announce_version: t.Optional[PydanticPypiVersion] = None
+    new_name: t.Optional[str] = None
+    discussion: t.Optional[p.HttpUrl] = None
+    redirect_replacement_major_version: t.Optional[int] = None
+
+    @p.model_validator(mode="after")
+    def _check_reason_text(self) -> Self:
+        reasons_with_text = ("other", "guidelines-violation")
+        if self.reason in reasons_with_text:
+            if self.reason_text is None:
+                reasons = ", ".join(f"'{reason}'" for reason in reasons_with_text)
+                raise ValueError(f"reason_text must be provided if reason is {reasons}")
+        else:
+            if self.reason_text is not None:
+                reasons = ", ".join(f"'{reason}'" for reason in reasons_with_text)
+                raise ValueError(
+                    f"reason_text must not be provided if reason is not {reasons}"
+                )
+        return self
+
+    @p.model_validator(mode="after")
+    def _check_reason_is_renamed(self) -> Self:
+        if self.reason != "renamed":
+            return self
+        if self.new_name is None:
+            raise ValueError("new_name must be provided if reason is 'renamed'")
+        if (
+            self.redirect_replacement_major_version is not None
+            and self.major_version != "TBD"
+            and self.redirect_replacement_major_version >= self.major_version
+        ):
+            raise ValueError(
+                "redirect_replacement_major_version must be smaller than major_version"
+            )
+        return self
+
+    @p.model_validator(mode="after")
+    def _check_reason_is_not_renamed(self) -> Self:
+        if self.reason == "renamed":
+            return self
+        if self.new_name is not None:
+            raise ValueError("new_name must not be provided if reason is not 'renamed'")
+        if self.redirect_replacement_major_version is not None:
+            raise ValueError(
+                "redirect_replacement_major_version must not be provided if reason is not 'renamed'"
+            )
+        if self.major_version == "TBD":
+            raise ValueError("major_version must not be TBD if reason is not 'renamed'")
+        return self
+
+
+class CollectionMetadata(p.BaseModel):
     """
     Stores metadata about one collection.
     """
 
-    changelog_url: str | None
-    collection_directory: str | None
-    repository: str | None
-    tag_version_regex: str | None
+    model_config = p.ConfigDict(extra="ignore")
 
-    def __init__(self, source: Mapping[str, t.Any] | None = None):
-        if source is None:
-            source = {}
-        self.changelog_url = source.get("changelog-url")
-        self.collection_directory = source.get("collection-directory")
-        self.repository = source.get("repository")
-        self.tag_version_regex = source.get("tag_version_regex")
+    changelog_url: t.Optional[str] = p.Field(alias="changelog-url", default=None)
+    collection_directory: t.Optional[str] = p.Field(
+        alias="collection-directory", default=None
+    )
+    repository: t.Optional[str] = None
+    tag_version_regex: t.Optional[str] = None
+    maintainers: list[str] = []
+    removal: t.Optional[RemovalInformation] = None
 
 
-class CollectionsMetadata:
+class CollectionsMetadata(p.BaseModel):
     """
     Stores metadata about a set of collections.
     """
 
-    data: dict[str, CollectionMetadata]
+    model_config = p.ConfigDict(extra="ignore")
+
+    collections: dict[str, CollectionMetadata]
 
-    def __init__(self, deps_dir: str | None):
-        self.data = {}
-        if deps_dir is not None:
-            collection_meta_path = os.path.join(deps_dir, "collection-meta.yaml")
-            if os.path.exists(collection_meta_path):
-                data = load_yaml_file(collection_meta_path)
-                if data and "collections" in data:
-                    for collection_name, collection_data in data["collections"].items():
-                        self.data[collection_name] = CollectionMetadata(collection_data)
+    @staticmethod
+    def load_from(deps_dir: StrPath | None) -> CollectionsMetadata:
+        if deps_dir is None:
+            return CollectionsMetadata(collections={})
+        collection_meta_path = os.path.join(deps_dir, "collection-meta.yaml")
+        if not os.path.exists(collection_meta_path):
+            return CollectionsMetadata(collections={})
+        data = load_yaml_file(collection_meta_path)
+        return CollectionsMetadata.parse_obj(data)
 
     def get_meta(self, collection_name: str) -> CollectionMetadata:
-        result = self.data.get(collection_name)
+        result = self.collections.get(collection_name)
         if result is None:
             result = CollectionMetadata()
-            self.data[collection_name] = result
+            self.collections[collection_name] = result
         return result