Add antsibull-fileutils dependency (#166)

* Add antsibull-fileutils dependency.

* Make pyre aware of antsibull-fileutils.

* Use and reexport yaml from antsibull-fileutils.

* Patch noxfile.py from antsibull and antsibull-docs to add antsibull-fileutils from local git.

* Use hashing from antsibull-fileutils.

* Use io from antsibull-fileutils.

.github/workflows/antsibull-docs.yml

@@ -37,6 +37,12 @@ jobs:
           ref: main
           path: antsibull-docs-parser
 
+      - name: Check out dependent project antsibull-fileutils
+        uses: actions/checkout@v4
+        with:
+          repository: ansible-community/antsibull-fileutils
+          path: antsibull-fileutils
+
       # nb: this is the first version of antsibull-docs that declares support
       # for antsibull-core v3.
       - name: Check out antsibull-docs main
@@ -46,6 +52,11 @@ jobs:
           ref: 2.9.0
           path: antsibull-docs
 
+      - name: Patch antsibull-docs noxfile.py
+        run: |
+          sed -i noxfile.py -e 's/args = ("antsibull-core", "antsibull-docs-parser")/args = ("antsibull-core", "antsibull-docs-parser", "antsibull-fileutils")/g'
+        working-directory: antsibull-docs
+
       - name: Set up Python 3.12
         id: python
         uses: actions/setup-python@v5
@@ -59,7 +70,7 @@ jobs:
           pipx install --python "${PYTHON}" nox
           python -m venv venv
           . ./venv/bin/activate
-          python -m pip install . ../antsibull-core
+          python -m pip install . ../antsibull-core ../antsibull-docs-parser ../antsibull-fileutils
         working-directory: antsibull-docs
 
       - name: Run type checkers

.github/workflows/antsibull.yml

@@ -30,6 +30,12 @@ jobs:
         with:
           path: antsibull-core
 
+      - name: Check out dependent project antsibull-fileutils
+        uses: actions/checkout@v4
+        with:
+          repository: ansible-community/antsibull-fileutils
+          path: antsibull-fileutils
+
       # antsibull 0.61.0 depends on antsibull-changelog >= 0.24.0 as well, so install 0.24.0 of that
       - name: Check out antsibull-changelog 0.24.0
         uses: actions/checkout@v4
@@ -47,6 +53,11 @@ jobs:
           ref: 0.61.0
           path: antsibull
 
+      - name: Patch antsibull noxfile.py
+        run: |
+          sed -i noxfile.py -e 's/args = ("antsibull-core", "antsibull-changelog")/args = ("antsibull-core", "antsibull-changelog", "antsibull-fileutils")/g'
+        working-directory: antsibull
+
       - name: Patch antsibull pyproject.toml
         run: |
           sed -e 's/pyre-check/pyre-check <= 0.9.20,/' -i pyproject.toml
@@ -71,7 +82,7 @@ jobs:
           pipx install --python "${PYTHON}" nox
           python -m venv venv
           . ./venv/bin/activate
-          python -m pip install . ../antsibull-core ../antsibull-changelog
+          python -m pip install . ../antsibull-core ../antsibull-changelog ../antsibull-fileutils
           ansible-galaxy collection install community.general
         working-directory: antsibull
 

.github/workflows/nox.yml

@@ -47,6 +47,11 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: antsibull-core
+      - name: Check out dependent project antsibull-fileutils
+        uses: actions/checkout@v4
+        with:
+          repository: ansible-community/antsibull-fileutils
+          path: antsibull-fileutils
       - name: Install extra packages
         if: "matrix.packages != ''"
         run: |

README.md

@@ -31,6 +31,23 @@ Install and run `nox` to run all tests. That's it for simple contributions!
 `nox` will create virtual environments in `.nox` inside the checked out project
 and install the requirements needed to run the tests there.
 
+---
+
+antsibull-core depends on the sister antsibull-fileutils project.
+By default, `nox` will install a development version of this project from Github.
+If you're hacking on antsibull-fileutils alongside antsibull-core,
+nox will automatically install this project from `../antsibull-fileutils`
+when running tests if this path exists.
+You can change this behavior through the `OTHER_ANTSIBULL_MODE` env var:
+
+- `OTHER_ANTSIBULL_MODE=auto` — the default behavior described above
+- `OTHER_ANTSIBULL_MODE=local` — install the project from `../antsibull-fileutils`.
+  Fail if this path doesn't exist.
+- `OTHER_ANTSIBULL_MODE=git` — install the project from the Github main branch
+- `OTHER_ANTSIBULL_MODE=pypi` — install the latest version from PyPI
+
+---
+
 To run specific tests:
 
 1. `nox -e test` to only run unit tests;

changelogs/fragments/166-antsibull-fileutils.yml

@@ -0,0 +1,4 @@
+minor_changes:
+  - Antsibull-core now depends on the new project antsibull-fileutils. Some code has been moved to that library;
+    that code is re-imported to avoid breaking changes for users of antsibull-core
+    (https://github.com/ansible-community/antsibull-core/pull/166).

noxfile.py

@@ -10,6 +10,7 @@
 
 import nox
 
+DEFAULT_MODE = os.environ.get("OTHER_ANTSIBULL_MODE", "auto")
 IN_CI = "GITHUB_ACTIONS" in os.environ
 ALLOW_EDITABLE = os.environ.get("ALLOW_EDITABLE", str(not IN_CI)).lower() in (
     "1",
@@ -33,11 +34,45 @@ def install(session: nox.Session, *args, editable=False, **kwargs):
     session.install(*args, "-U", **kwargs)
 
 
+def other_antsibull(
+    mode: str | None = None,
+) -> list[str | Path]:
+    if mode is None:
+        mode = DEFAULT_MODE
+    to_install: list[str | Path] = []
+    args = ("antsibull-fileutils",)
+    for project in args:
+        path = Path("../", project)
+        path_exists = path.is_dir()
+        if mode == "auto":
+            if path_exists:
+                mode = "local"
+            else:
+                mode = "git"
+        if mode == "local":
+            if not path_exists:
+                raise ValueError(f"Cannot install {project}! {path} does not exist!")
+            if ALLOW_EDITABLE:
+                to_install.append("-e")
+            to_install.append(path)
+        elif mode == "git":
+            to_install.append(
+                f"{project} @ "
+                f"https://github.com/ansible-community/{project}/archive/main.tar.gz"
+            )
+        elif mode == "pypi":
+            to_install.append(project)
+        else:
+            raise ValueError(f"install_other_antsibull: invalid argument mode={mode!r}")
+    return to_install
+
+
 @nox.session(python=["3.9", "3.10", "3.11", "3.12"])
 def test(session: nox.Session):
     install(
         session,
         ".[test, coverage]",
+        *other_antsibull(),
         editable=True,
     )
     covfile = Path(session.create_tmp(), ".coverage")
@@ -58,7 +93,7 @@ def test(session: nox.Session):
 
 @nox.session
 def coverage(session: nox.Session):
-    install(session, ".[coverage]", editable=True)
+    install(session, ".[coverage]", *other_antsibull(), editable=True)
     combined = map(str, Path().glob(".nox/*/tmp/.coverage"))
     # Combine the results into a single .coverage file in the root
     session.run("coverage", "combine", "--keep", *combined)
@@ -77,7 +112,7 @@ def lint(session: nox.Session):
 
 @nox.session
 def formatters(session: nox.Session):
-    install(session, ".[formatters]")
+    install(session, ".[formatters]", *other_antsibull())
     posargs = list(session.posargs)
     if IN_CI:
         posargs.append("--check")
@@ -87,7 +122,7 @@ def formatters(session: nox.Session):
 
 @nox.session
 def codeqa(session: nox.Session):
-    install(session, ".[codeqa]", editable=True)
+    install(session, ".[codeqa]", *other_antsibull(), editable=True)
     session.run("flake8", "src", *session.posargs)
     session.run("pylint", "--rcfile", ".pylintrc.automated", "src/antsibull_core")
     session.run("reuse", "lint")
@@ -97,9 +132,15 @@ def codeqa(session: nox.Session):
 @nox.session
 def typing(session: nox.Session):
     # pyre does not work when we don't install ourself in editable mode 🙄.
-    install(session, "-e", ".[typing]")
+    others = other_antsibull()
+    install(session, "-e", ".[typing]", *others)
     session.run("mypy", "src/antsibull_core")
 
+    additional_libraries = []
+    for path in others:
+        if isinstance(path, Path):
+            additional_libraries.extend(("--search-path", str(path / "src")))
+
     purelib = (
         session.run(
             "python",
@@ -128,6 +169,7 @@ def typing(session: nox.Session):
         platlib,
         "--search-path",
         "stubs/",
+        *additional_libraries,
     )
 
 

pyproject.toml

@@ -28,6 +28,7 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
+    "antsibull-fileutils >= 1.0.0, < 2.0.0",
     "aiofiles",
     "aiohttp >= 3.3.0",
     "build",
@@ -36,7 +37,6 @@ dependencies = [
     "perky",
     # pydantic v2 is a major rewrite
     "pydantic ~= 2.0",
-    "PyYAML",
     "semantic_version",
     # 0.5.0 introduces dict_config
     "twiggy >= 0.5.0",
@@ -91,7 +91,6 @@ typing = [
     # https://github.com/facebook/pyre-check/issues/398
     "pyre-check >= 0.9.17",
     "types-aiofiles",
-    "types-PyYAML",
     "typing-extensions",
 ]
 dev = [

src/antsibull_core/ansible_core.py

@@ -16,14 +16,14 @@
 from urllib.parse import urljoin
 
 import aiofiles
+from antsibull_fileutils.hashing import verify_a_hash
+from antsibull_fileutils.io import copy_file
 from packaging.version import Version as PypiVer
 
 from . import app_context
 from .logging import log
 from .subprocess_util import async_log_run
-from .utils.hashing import verify_a_hash
 from .utils.http import retry_get
-from .utils.io import copy_file
 
 if t.TYPE_CHECKING:
     import aiohttp.client
@@ -145,7 +145,13 @@ async def retrieve(  # noqa C901
                 )
                 if os.path.isfile(cached_path):
                     tar_path = os.path.join(download_dir, tar_filename)
-                    await copy_file(cached_path, tar_path, check_content=False)
+                    await copy_file(
+                        cached_path,
+                        tar_path,
+                        check_content=False,
+                        file_check_content=lib_ctx.file_check_content,
+                        chunksize=lib_ctx.chunksize,
+                    )
                     return tar_path
 
         release_info = await self.get_release_info(package_name)
@@ -168,8 +174,16 @@ async def retrieve(  # noqa C901
         if lib_ctx.ansible_core_cache and "sha256" in digests:
             cached_path = os.path.join(lib_ctx.ansible_core_cache, tar_filename)
             if os.path.isfile(cached_path):
-                if await verify_a_hash(cached_path, digests):
-                    await copy_file(cached_path, tar_path, check_content=False)
+                if await verify_a_hash(
+                    cached_path, digests, chunksize=lib_ctx.chunksize
+                ):
+                    await copy_file(
+                        cached_path,
+                        tar_path,
+                        check_content=False,
+                        file_check_content=lib_ctx.file_check_content,
+                        chunksize=lib_ctx.chunksize,
+                    )
                     return tar_path
 
         async with retry_get(self.aio_session, pypi_url) as response:
@@ -179,7 +193,13 @@ async def retrieve(  # noqa C901
 
         if lib_ctx.ansible_core_cache:
             cached_path = os.path.join(lib_ctx.ansible_core_cache, tar_filename)
-            await copy_file(tar_path, cached_path, check_content=False)
+            await copy_file(
+                tar_path,
+                cached_path,
+                check_content=False,
+                file_check_content=lib_ctx.file_check_content,
+                chunksize=lib_ctx.chunksize,
+            )
 
         return tar_path
 

src/antsibull_core/galaxy.py

@@ -15,11 +15,11 @@
 
 import aiofiles
 import semantic_version as semver
+from antsibull_fileutils.hashing import verify_hash
+from antsibull_fileutils.io import copy_file
 
 from . import app_context
-from .utils.hashing import verify_hash
 from .utils.http import retry_get
-from .utils.io import copy_file
 
 # The type checker can handle finding aiohttp.client but flake8 cannot :-(
 if t.TYPE_CHECKING:
@@ -377,11 +377,18 @@ async def download(
         namespace, name = collection.split(".", 1)
         filename = f"{namespace}-{name}-{version}.tar.gz"
         download_filename = os.path.join(self.download_dir, filename)
+        lib_ctx = app_context.lib_ctx.get()
 
         if self.collection_cache and self.trust_collection_cache:
             cached_copy = os.path.join(self.collection_cache, filename)
             if os.path.isfile(cached_copy):
-                await copy_file(cached_copy, download_filename, check_content=False)
+                await copy_file(
+                    cached_copy,
+                    download_filename,
+                    check_content=False,
+                    file_check_content=lib_ctx.file_check_content,
+                    chunksize=lib_ctx.chunksize,
+                )
                 return download_filename
 
         release_info = await self.get_release_info(f"{namespace}/{name}", version)
@@ -392,8 +399,17 @@ async def download(
         if self.collection_cache:
             cached_copy = os.path.join(self.collection_cache, filename)
             if os.path.isfile(cached_copy):
-                if await verify_hash(cached_copy, sha256sum):
-                    await copy_file(cached_copy, download_filename, check_content=False)
+                lib_ctx = app_context.lib_ctx.get()
+                if await verify_hash(
+                    cached_copy, sha256sum, chunksize=lib_ctx.chunksize
+                ):
+                    await copy_file(
+                        cached_copy,
+                        download_filename,
+                        check_content=False,
+                        file_check_content=lib_ctx.file_check_content,
+                        chunksize=lib_ctx.chunksize,
+                    )
                     return download_filename
 
         async with retry_get(
@@ -408,7 +424,9 @@ async def download(
                     await f.write(chunk)
 
         # Verify the download
-        if not await verify_hash(download_filename, sha256sum):
+        if not await verify_hash(
+            download_filename, sha256sum, chunksize=lib_ctx.chunksize
+        ):
             raise DownloadFailure(
                 f"{release_url} failed to download correctly."
                 f" Expected checksum: {sha256sum}"
@@ -417,7 +435,13 @@ async def download(
         # Copy downloaded collection into cache
         if self.collection_cache:
             cached_copy = os.path.join(self.collection_cache, filename)
-            await copy_file(download_filename, cached_copy, check_content=False)
+            await copy_file(
+                download_filename,
+                cached_copy,
+                check_content=False,
+                file_check_content=lib_ctx.file_check_content,
+                chunksize=lib_ctx.chunksize,
+            )
 
         return download_filename