diff --git a/.github/workflows/antsibull-docs.yml b/.github/workflows/antsibull-docs.yml index 88ee47f..437ed38 100644 --- a/.github/workflows/antsibull-docs.yml +++ b/.github/workflows/antsibull-docs.yml @@ -37,6 +37,12 @@ jobs: ref: main path: antsibull-docs-parser + - name: Check out dependent project antsibull-fileutils + uses: actions/checkout@v4 + with: + repository: ansible-community/antsibull-fileutils + path: antsibull-fileutils + # nb: this is the first version of antsibull-docs that declares support # for antsibull-core v3. - name: Check out antsibull-docs main @@ -46,6 +52,11 @@ jobs: ref: 2.9.0 path: antsibull-docs + - name: Patch antsibull-docs noxfile.py + run: | + sed -i noxfile.py -e 's/args = ("antsibull-core", "antsibull-docs-parser")/args = ("antsibull-core", "antsibull-docs-parser", "antsibull-fileutils")/g' + working-directory: antsibull-docs + - name: Set up Python 3.12 id: python uses: actions/setup-python@v5 @@ -59,7 +70,7 @@ jobs: pipx install --python "${PYTHON}" nox python -m venv venv . ./venv/bin/activate - python -m pip install . ../antsibull-core + python -m pip install . ../antsibull-core ../antsibull-docs-parser ../antsibull-fileutils working-directory: antsibull-docs - name: Run type checkers diff --git a/.github/workflows/antsibull.yml b/.github/workflows/antsibull.yml index 0e5c45a..fc774e2 100644 --- a/.github/workflows/antsibull.yml +++ b/.github/workflows/antsibull.yml @@ -30,6 +30,12 @@ jobs: with: path: antsibull-core + - name: Check out dependent project antsibull-fileutils + uses: actions/checkout@v4 + with: + repository: ansible-community/antsibull-fileutils + path: antsibull-fileutils + # antsibull 0.61.0 depends on antsibull-changelog >= 0.24.0 as well, so install 0.24.0 of that - name: Check out antsibull-changelog 0.24.0 uses: actions/checkout@v4 @@ -47,6 +53,11 @@ jobs: ref: 0.61.0 path: antsibull + - name: Patch antsibull noxfile.py + run: | + sed -i noxfile.py -e 's/args = ("antsibull-core", "antsibull-changelog")/args = ("antsibull-core", "antsibull-changelog", "antsibull-fileutils")/g' + working-directory: antsibull + - name: Patch antsibull pyproject.toml run: | sed -e 's/pyre-check/pyre-check <= 0.9.20,/' -i pyproject.toml @@ -71,7 +82,7 @@ jobs: pipx install --python "${PYTHON}" nox python -m venv venv . ./venv/bin/activate - python -m pip install . ../antsibull-core ../antsibull-changelog + python -m pip install . ../antsibull-core ../antsibull-changelog ../antsibull-fileutils ansible-galaxy collection install community.general working-directory: antsibull diff --git a/.github/workflows/nox.yml b/.github/workflows/nox.yml index fe818eb..d7923c8 100644 --- a/.github/workflows/nox.yml +++ b/.github/workflows/nox.yml @@ -47,6 +47,11 @@ jobs: uses: actions/checkout@v4 with: path: antsibull-core + - name: Check out dependent project antsibull-fileutils + uses: actions/checkout@v4 + with: + repository: ansible-community/antsibull-fileutils + path: antsibull-fileutils - name: Install extra packages if: "matrix.packages != ''" run: | diff --git a/README.md b/README.md index 0142866..5197631 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,23 @@ Install and run `nox` to run all tests. That's it for simple contributions! `nox` will create virtual environments in `.nox` inside the checked out project and install the requirements needed to run the tests there. +--- + +antsibull-core depends on the sister antsibull-fileutils project. +By default, `nox` will install a development version of this project from Github. +If you're hacking on antsibull-fileutils alongside antsibull-core, +nox will automatically install this project from `../antsibull-fileutils` +when running tests if this path exists. +You can change this behavior through the `OTHER_ANTSIBULL_MODE` env var: + +- `OTHER_ANTSIBULL_MODE=auto` — the default behavior described above +- `OTHER_ANTSIBULL_MODE=local` — install the project from `../antsibull-fileutils`. + Fail if this path doesn't exist. +- `OTHER_ANTSIBULL_MODE=git` — install the project from the Github main branch +- `OTHER_ANTSIBULL_MODE=pypi` — install the latest version from PyPI + +--- + To run specific tests: 1. `nox -e test` to only run unit tests; diff --git a/changelogs/fragments/166-antsibull-fileutils.yml b/changelogs/fragments/166-antsibull-fileutils.yml new file mode 100644 index 0000000..634c480 --- /dev/null +++ b/changelogs/fragments/166-antsibull-fileutils.yml @@ -0,0 +1,4 @@ +minor_changes: + - Antsibull-core now depends on the new project antsibull-fileutils. Some code has been moved to that library; + that code is re-imported to avoid breaking changes for users of antsibull-core + (https://github.com/ansible-community/antsibull-core/pull/166). diff --git a/noxfile.py b/noxfile.py index 94aa6cc..6ed26c1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -10,6 +10,7 @@ import nox +DEFAULT_MODE = os.environ.get("OTHER_ANTSIBULL_MODE", "auto") IN_CI = "GITHUB_ACTIONS" in os.environ ALLOW_EDITABLE = os.environ.get("ALLOW_EDITABLE", str(not IN_CI)).lower() in ( "1", @@ -33,11 +34,45 @@ def install(session: nox.Session, *args, editable=False, **kwargs): session.install(*args, "-U", **kwargs) +def other_antsibull( + mode: str | None = None, +) -> list[str | Path]: + if mode is None: + mode = DEFAULT_MODE + to_install: list[str | Path] = [] + args = ("antsibull-fileutils",) + for project in args: + path = Path("../", project) + path_exists = path.is_dir() + if mode == "auto": + if path_exists: + mode = "local" + else: + mode = "git" + if mode == "local": + if not path_exists: + raise ValueError(f"Cannot install {project}! {path} does not exist!") + if ALLOW_EDITABLE: + to_install.append("-e") + to_install.append(path) + elif mode == "git": + to_install.append( + f"{project} @ " + f"https://github.com/ansible-community/{project}/archive/main.tar.gz" + ) + elif mode == "pypi": + to_install.append(project) + else: + raise ValueError(f"install_other_antsibull: invalid argument mode={mode!r}") + return to_install + + @nox.session(python=["3.9", "3.10", "3.11", "3.12"]) def test(session: nox.Session): install( session, ".[test, coverage]", + *other_antsibull(), editable=True, ) covfile = Path(session.create_tmp(), ".coverage") @@ -58,7 +93,7 @@ def test(session: nox.Session): @nox.session def coverage(session: nox.Session): - install(session, ".[coverage]", editable=True) + install(session, ".[coverage]", *other_antsibull(), editable=True) combined = map(str, Path().glob(".nox/*/tmp/.coverage")) # Combine the results into a single .coverage file in the root session.run("coverage", "combine", "--keep", *combined) @@ -77,7 +112,7 @@ def lint(session: nox.Session): @nox.session def formatters(session: nox.Session): - install(session, ".[formatters]") + install(session, ".[formatters]", *other_antsibull()) posargs = list(session.posargs) if IN_CI: posargs.append("--check") @@ -87,7 +122,7 @@ def formatters(session: nox.Session): @nox.session def codeqa(session: nox.Session): - install(session, ".[codeqa]", editable=True) + install(session, ".[codeqa]", *other_antsibull(), editable=True) session.run("flake8", "src", *session.posargs) session.run("pylint", "--rcfile", ".pylintrc.automated", "src/antsibull_core") session.run("reuse", "lint") @@ -97,9 +132,15 @@ def codeqa(session: nox.Session): @nox.session def typing(session: nox.Session): # pyre does not work when we don't install ourself in editable mode 🙄. - install(session, "-e", ".[typing]") + others = other_antsibull() + install(session, "-e", ".[typing]", *others) session.run("mypy", "src/antsibull_core") + additional_libraries = [] + for path in others: + if isinstance(path, Path): + additional_libraries.extend(("--search-path", str(path / "src"))) + purelib = ( session.run( "python", @@ -128,6 +169,7 @@ def typing(session: nox.Session): platlib, "--search-path", "stubs/", + *additional_libraries, ) diff --git a/pyproject.toml b/pyproject.toml index 1b03dea..44d3652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ + "antsibull-fileutils >= 1.0.0, < 2.0.0", "aiofiles", "aiohttp >= 3.3.0", "build", @@ -36,7 +37,6 @@ dependencies = [ "perky", # pydantic v2 is a major rewrite "pydantic ~= 2.0", - "PyYAML", "semantic_version", # 0.5.0 introduces dict_config "twiggy >= 0.5.0", @@ -91,7 +91,6 @@ typing = [ # https://github.com/facebook/pyre-check/issues/398 "pyre-check >= 0.9.17", "types-aiofiles", - "types-PyYAML", "typing-extensions", ] dev = [ diff --git a/src/antsibull_core/ansible_core.py b/src/antsibull_core/ansible_core.py index fc26d32..c693a8b 100644 --- a/src/antsibull_core/ansible_core.py +++ b/src/antsibull_core/ansible_core.py @@ -16,14 +16,14 @@ from urllib.parse import urljoin import aiofiles +from antsibull_fileutils.hashing import verify_a_hash +from antsibull_fileutils.io import copy_file from packaging.version import Version as PypiVer from . import app_context from .logging import log from .subprocess_util import async_log_run -from .utils.hashing import verify_a_hash from .utils.http import retry_get -from .utils.io import copy_file if t.TYPE_CHECKING: import aiohttp.client @@ -145,7 +145,13 @@ async def retrieve( # noqa C901 ) if os.path.isfile(cached_path): tar_path = os.path.join(download_dir, tar_filename) - await copy_file(cached_path, tar_path, check_content=False) + await copy_file( + cached_path, + tar_path, + check_content=False, + file_check_content=lib_ctx.file_check_content, + chunksize=lib_ctx.chunksize, + ) return tar_path release_info = await self.get_release_info(package_name) @@ -168,8 +174,16 @@ async def retrieve( # noqa C901 if lib_ctx.ansible_core_cache and "sha256" in digests: cached_path = os.path.join(lib_ctx.ansible_core_cache, tar_filename) if os.path.isfile(cached_path): - if await verify_a_hash(cached_path, digests): - await copy_file(cached_path, tar_path, check_content=False) + if await verify_a_hash( + cached_path, digests, chunksize=lib_ctx.chunksize + ): + await copy_file( + cached_path, + tar_path, + check_content=False, + file_check_content=lib_ctx.file_check_content, + chunksize=lib_ctx.chunksize, + ) return tar_path async with retry_get(self.aio_session, pypi_url) as response: @@ -179,7 +193,13 @@ async def retrieve( # noqa C901 if lib_ctx.ansible_core_cache: cached_path = os.path.join(lib_ctx.ansible_core_cache, tar_filename) - await copy_file(tar_path, cached_path, check_content=False) + await copy_file( + tar_path, + cached_path, + check_content=False, + file_check_content=lib_ctx.file_check_content, + chunksize=lib_ctx.chunksize, + ) return tar_path diff --git a/src/antsibull_core/galaxy.py b/src/antsibull_core/galaxy.py index c12cbdc..ee604bd 100644 --- a/src/antsibull_core/galaxy.py +++ b/src/antsibull_core/galaxy.py @@ -15,11 +15,11 @@ import aiofiles import semantic_version as semver +from antsibull_fileutils.hashing import verify_hash +from antsibull_fileutils.io import copy_file from . import app_context -from .utils.hashing import verify_hash from .utils.http import retry_get -from .utils.io import copy_file # The type checker can handle finding aiohttp.client but flake8 cannot :-( if t.TYPE_CHECKING: @@ -377,11 +377,18 @@ async def download( namespace, name = collection.split(".", 1) filename = f"{namespace}-{name}-{version}.tar.gz" download_filename = os.path.join(self.download_dir, filename) + lib_ctx = app_context.lib_ctx.get() if self.collection_cache and self.trust_collection_cache: cached_copy = os.path.join(self.collection_cache, filename) if os.path.isfile(cached_copy): - await copy_file(cached_copy, download_filename, check_content=False) + await copy_file( + cached_copy, + download_filename, + check_content=False, + file_check_content=lib_ctx.file_check_content, + chunksize=lib_ctx.chunksize, + ) return download_filename release_info = await self.get_release_info(f"{namespace}/{name}", version) @@ -392,8 +399,17 @@ async def download( if self.collection_cache: cached_copy = os.path.join(self.collection_cache, filename) if os.path.isfile(cached_copy): - if await verify_hash(cached_copy, sha256sum): - await copy_file(cached_copy, download_filename, check_content=False) + lib_ctx = app_context.lib_ctx.get() + if await verify_hash( + cached_copy, sha256sum, chunksize=lib_ctx.chunksize + ): + await copy_file( + cached_copy, + download_filename, + check_content=False, + file_check_content=lib_ctx.file_check_content, + chunksize=lib_ctx.chunksize, + ) return download_filename async with retry_get( @@ -408,7 +424,9 @@ async def download( await f.write(chunk) # Verify the download - if not await verify_hash(download_filename, sha256sum): + if not await verify_hash( + download_filename, sha256sum, chunksize=lib_ctx.chunksize + ): raise DownloadFailure( f"{release_url} failed to download correctly." f" Expected checksum: {sha256sum}" @@ -417,7 +435,13 @@ async def download( # Copy downloaded collection into cache if self.collection_cache: cached_copy = os.path.join(self.collection_cache, filename) - await copy_file(download_filename, cached_copy, check_content=False) + await copy_file( + download_filename, + cached_copy, + check_content=False, + file_check_content=lib_ctx.file_check_content, + chunksize=lib_ctx.chunksize, + ) return download_filename diff --git a/src/antsibull_core/utils/hashing.py b/src/antsibull_core/utils/hashing.py index 9a41443..c44771c 100644 --- a/src/antsibull_core/utils/hashing.py +++ b/src/antsibull_core/utils/hashing.py @@ -7,12 +7,11 @@ from __future__ import annotations -import dataclasses -import hashlib import typing as t from collections.abc import Mapping -import aiofiles +from antsibull_fileutils.hashing import verify_a_hash as _verify_a_hash +from antsibull_fileutils.hashing import verify_hash as _verify_hash from .. import app_context @@ -20,20 +19,6 @@ from _typeshed import StrOrBytesPath -@dataclasses.dataclass(frozen=True) -class _AlgorithmData: - name: str - algorithm: str - kwargs: dict[str, t.Any] - - -_PREFERRED_HASHES: tuple[_AlgorithmData, ...] = ( - # https://pypi.org/help/#verify-hashes, https://github.com/pypi/warehouse/issues/9628 - _AlgorithmData(name="sha256", algorithm="sha256", kwargs={}), - _AlgorithmData(name="blake2b_256", algorithm="blake2b", kwargs={"digest_size": 32}), -) - - async def verify_hash( filename: StrOrBytesPath, hash_digest: str, @@ -50,15 +35,14 @@ async def verify_hash( :kwarg algorithm_kwargs: Parameters to provide to the hash algorithm's constructor. :returns: True if the hash matches, otherwise False. """ - hasher = getattr(hashlib, algorithm)(**(algorithm_kwargs or {})) - async with aiofiles.open(filename, "rb") as f: - ctx = app_context.lib_ctx.get() - while chunk := await f.read(ctx.chunksize): - hasher.update(chunk) - if hasher.hexdigest() != hash_digest: - return False - - return True + ctx = app_context.lib_ctx.get() + return await _verify_hash( + filename, + hash_digest, + algorithm=algorithm, + algorithm_kwargs=algorithm_kwargs, + chunksize=ctx.chunksize, + ) async def verify_a_hash( @@ -72,12 +56,5 @@ async def verify_a_hash( :arg hash_digest: A mapping of hash types to digests. :returns: True if the hash matches, otherwise False. """ - for algorithm_data in _PREFERRED_HASHES: - if algorithm_data.name in hash_digests: - return await verify_hash( - filename, - hash_digests[algorithm_data.name], - algorithm=algorithm_data.algorithm, - algorithm_kwargs=algorithm_data.kwargs, - ) - return False + ctx = app_context.lib_ctx.get() + return await _verify_a_hash(filename, hash_digests, chunksize=ctx.chunksize) diff --git a/src/antsibull_core/utils/io.py b/src/antsibull_core/utils/io.py index ac7cf48..3ae0d68 100644 --- a/src/antsibull_core/utils/io.py +++ b/src/antsibull_core/utils/io.py @@ -7,11 +7,11 @@ from __future__ import annotations -import os -import os.path import typing as t -import aiofiles +from antsibull_fileutils.io import copy_file as _copy_file +from antsibull_fileutils.io import read_file as _read_file +from antsibull_fileutils.io import write_file as _write_file from .. import app_context from ..logging import log @@ -34,83 +34,20 @@ async def copy_file( destination file exists, first check whether source and destination are potentially equal before actually copying, """ - flog = mlog.fields(func="copy_file") - flog.debug("Enter") - lib_ctx = app_context.lib_ctx.get() - if check_content and lib_ctx.file_check_content > 0: - # Check whether the destination file exists and has the same content as the source file, - # in which case we won't overwrite the destination file - try: - stat_d = os.stat(dest_path) - if stat_d.st_size <= lib_ctx.file_check_content: - stat_s = os.stat(source_path) - if stat_d.st_size == stat_s.st_size: - # Read both files and compare - async with aiofiles.open(source_path, "rb") as f_in: - content_to_copy = await f_in.read() - async with aiofiles.open(dest_path, "rb") as f_in: - existing_content = await f_in.read() - if content_to_copy == existing_content: - flog.debug("Skipping copy, since files are identical") - return - # Since we already read the contents of the file to copy, simply write it to - # the destination instead of reading it again - async with aiofiles.open(dest_path, "wb") as f_out: - await f_out.write(content_to_copy) - return - except FileNotFoundError: - # Destination (or source) file does not exist - pass - - async with aiofiles.open(source_path, "rb") as f_in: - async with aiofiles.open(dest_path, "wb") as f_out: - while chunk := await f_in.read(lib_ctx.chunksize): - await f_out.write(chunk) - - flog.debug("Leave") + await _copy_file( + source_path, + dest_path, + check_content=check_content, + file_check_content=lib_ctx.file_check_content, + chunksize=lib_ctx.chunksize, + ) async def write_file(filename: StrOrBytesPath, content: str) -> None: - flog = mlog.fields(func="write_file") - flog.debug("Enter") - - content_bytes = content.encode("utf-8") - lib_ctx = app_context.lib_ctx.get() - if ( - lib_ctx.file_check_content > 0 - and len(content_bytes) <= lib_ctx.file_check_content - ): - # Check whether the destination file exists and has the same content as the one we want to - # write, in which case we won't overwrite the file - try: - stat = os.stat(filename) - if stat.st_size == len(content_bytes): - # Read file and compare - async with aiofiles.open(filename, "rb") as f: - existing_content = await f.read() - if existing_content == content_bytes: - flog.debug( - "Skipping write, since file already contains the exact content" - ) - return - except FileNotFoundError: - # Destination file does not exist - pass - - async with aiofiles.open(filename, "wb") as f: - await f.write(content_bytes) - - flog.debug("Leave") + await _write_file(filename, content, file_check_content=lib_ctx.file_check_content) async def read_file(filename: StrOrBytesPath, encoding: str = "utf-8") -> str: - flog = mlog.fields(func="read_file") - flog.debug("Enter") - - async with aiofiles.open(filename, "r", encoding=encoding) as f: - content = await f.read() - - flog.debug("Leave") - return content + return await _read_file(filename, encoding=encoding) diff --git a/src/antsibull_core/yaml.py b/src/antsibull_core/yaml.py index 6017900..694b870 100644 --- a/src/antsibull_core/yaml.py +++ b/src/antsibull_core/yaml.py @@ -10,53 +10,8 @@ from __future__ import annotations -import typing as t - -import yaml - -_SafeLoader: t.Any -_SafeDumper: t.Any -try: - # use C version if possible for speedup - from yaml import CSafeDumper as _SafeDumper - from yaml import CSafeLoader as _SafeLoader -except ImportError: - from yaml import SafeDumper as _SafeDumper - from yaml import SafeLoader as _SafeLoader - -if t.TYPE_CHECKING: - from _typeshed import StrOrBytesPath, SupportsWrite - - -def load_yaml_bytes(data: bytes) -> t.Any: - """ - Load and parse YAML from given bytes. - """ - return yaml.load(data, Loader=_SafeLoader) - - -def load_yaml_file(path: StrOrBytesPath) -> t.Any: - """ - Load and parse YAML file ``path``. - """ - with open(path, "rb") as stream: - return yaml.load(stream, Loader=_SafeLoader) - - -def store_yaml_file(path: StrOrBytesPath, content: t.Any) -> None: - """ - Store ``content`` as YAML file under ``path``. - """ - with open(path, "wb") as stream: - store_yaml_stream(stream, content) - - -def store_yaml_stream(stream: SupportsWrite, content: t.Any) -> None: - """ - Dump ``content`` as YAML to an IO ``stream``. - """ - dumper = _SafeDumper - dumper.ignore_aliases = lambda *args: True - yaml.dump( - content, stream, default_flow_style=False, encoding="utf-8", Dumper=dumper - ) +# pylint: disable=unused-import +from antsibull_fileutils.yaml import load_yaml_bytes # noqa: F401 +from antsibull_fileutils.yaml import load_yaml_file # noqa: F401 +from antsibull_fileutils.yaml import store_yaml_file # noqa: F401 +from antsibull_fileutils.yaml import store_yaml_stream # noqa: F401 diff --git a/tests/units/test_utils_hashing.py b/tests/units/test_utils_hashing.py deleted file mode 100644 index ab8ebc1..0000000 --- a/tests/units/test_utils_hashing.py +++ /dev/null @@ -1,103 +0,0 @@ -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: Ansible Project - -from __future__ import annotations - -import pytest - -from antsibull_core.utils.hashing import verify_a_hash, verify_hash - -HASH_TESTS = [ - ( - b"", - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "sha256", - {}, - True, - ), - ( - b"", - "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", - "sha256", - {}, - False, - ), -] - - -@pytest.mark.parametrize( - "content, hash, algorithm, algorithm_kwargs, expected_match", HASH_TESTS -) -@pytest.mark.asyncio -async def test_verify_hash( - content: bytes, - hash: bytes, - algorithm: str, - algorithm_kwargs: dict | None, - expected_match: bool, - tmp_path, -): - filename = tmp_path / "file" - with open(filename, "wb") as f: - f.write(content) - result = await verify_hash( - filename, hash, algorithm=algorithm, algorithm_kwargs=algorithm_kwargs - ) - assert result is expected_match - - -HASH_DICT_TESTS = [ - ( - b"foo", - { - "sha256": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - }, - True, - ), - ( - b"bar", - { - "sha256": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - }, - False, - ), - ( - b"foo", - { - "blake2b_256": "b8fe9f7f6255a6fa08f668ab632a8d081ad87983c77cd274e48ce450f0b349fd", - }, - True, - ), - ( - b"bar", - { - "blake2b_256": "b8fe9f7f6255a6fa08f668ab632a8d081ad87983c77cd274e48ce450f0b349fd", - }, - False, - ), - ( - b"", - {}, - False, - ), - ( - b"", - { - "foo": "bar", - }, - False, - ), -] - - -@pytest.mark.parametrize("content, hash_digests, expected_match", HASH_DICT_TESTS) -@pytest.mark.asyncio -async def test_verify_a_hash( - content: bytes, hash_digests: dict[str, str], expected_match: bool, tmp_path -): - filename = tmp_path / "file" - with open(filename, "wb") as f: - f.write(content) - result = await verify_a_hash(filename, hash_digests) - assert result is expected_match