Skip to content

Commit

Permalink
Use io from antsibull-fileutils.
Browse files Browse the repository at this point in the history
  • Loading branch information
felixfontein committed Sep 9, 2024
1 parent c5cb850 commit 13e34d5
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 83 deletions.
26 changes: 22 additions & 4 deletions src/antsibull_core/ansible_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@

import aiofiles
from antsibull_fileutils.hashing import verify_a_hash
from antsibull_fileutils.io import copy_file
from packaging.version import Version as PypiVer

from . import app_context
from .logging import log
from .subprocess_util import async_log_run
from .utils.http import retry_get
from .utils.io import copy_file

if t.TYPE_CHECKING:
import aiohttp.client
Expand Down Expand Up @@ -145,7 +145,13 @@ async def retrieve( # noqa C901
)
if os.path.isfile(cached_path):
tar_path = os.path.join(download_dir, tar_filename)
await copy_file(cached_path, tar_path, check_content=False)
await copy_file(
cached_path,
tar_path,
check_content=False,
file_check_content=lib_ctx.file_check_content,
chunksize=lib_ctx.chunksize,
)
return tar_path

release_info = await self.get_release_info(package_name)
Expand All @@ -171,7 +177,13 @@ async def retrieve( # noqa C901
if await verify_a_hash(
cached_path, digests, chunksize=lib_ctx.chunksize
):
await copy_file(cached_path, tar_path, check_content=False)
await copy_file(
cached_path,
tar_path,
check_content=False,
file_check_content=lib_ctx.file_check_content,
chunksize=lib_ctx.chunksize,
)
return tar_path

async with retry_get(self.aio_session, pypi_url) as response:
Expand All @@ -181,7 +193,13 @@ async def retrieve( # noqa C901

if lib_ctx.ansible_core_cache:
cached_path = os.path.join(lib_ctx.ansible_core_cache, tar_filename)
await copy_file(tar_path, cached_path, check_content=False)
await copy_file(
tar_path,
cached_path,
check_content=False,
file_check_content=lib_ctx.file_check_content,
chunksize=lib_ctx.chunksize,
)

return tar_path

Expand Down
27 changes: 23 additions & 4 deletions src/antsibull_core/galaxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
import aiofiles
import semantic_version as semver
from antsibull_fileutils.hashing import verify_hash
from antsibull_fileutils.io import copy_file

from . import app_context
from .utils.http import retry_get
from .utils.io import copy_file

# The type checker can handle finding aiohttp.client but flake8 cannot :-(
if t.TYPE_CHECKING:
Expand Down Expand Up @@ -377,11 +377,18 @@ async def download(
namespace, name = collection.split(".", 1)
filename = f"{namespace}-{name}-{version}.tar.gz"
download_filename = os.path.join(self.download_dir, filename)
lib_ctx = app_context.lib_ctx.get()

if self.collection_cache and self.trust_collection_cache:
cached_copy = os.path.join(self.collection_cache, filename)
if os.path.isfile(cached_copy):
await copy_file(cached_copy, download_filename, check_content=False)
await copy_file(
cached_copy,
download_filename,
check_content=False,
file_check_content=lib_ctx.file_check_content,
chunksize=lib_ctx.chunksize,
)
return download_filename

release_info = await self.get_release_info(f"{namespace}/{name}", version)
Expand All @@ -396,7 +403,13 @@ async def download(
if await verify_hash(
cached_copy, sha256sum, chunksize=lib_ctx.chunksize
):
await copy_file(cached_copy, download_filename, check_content=False)
await copy_file(
cached_copy,
download_filename,
check_content=False,
file_check_content=lib_ctx.file_check_content,
chunksize=lib_ctx.chunksize,
)
return download_filename

async with retry_get(
Expand All @@ -422,7 +435,13 @@ async def download(
# Copy downloaded collection into cache
if self.collection_cache:
cached_copy = os.path.join(self.collection_cache, filename)
await copy_file(download_filename, cached_copy, check_content=False)
await copy_file(
download_filename,
cached_copy,
check_content=False,
file_check_content=lib_ctx.file_check_content,
chunksize=lib_ctx.chunksize,
)

return download_filename

Expand Down
87 changes: 12 additions & 75 deletions src/antsibull_core/utils/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@

from __future__ import annotations

import os
import os.path
import typing as t

import aiofiles
from antsibull_fileutils.io import copy_file as _copy_file
from antsibull_fileutils.io import read_file as _read_file
from antsibull_fileutils.io import write_file as _write_file

from .. import app_context
from ..logging import log
Expand All @@ -34,83 +34,20 @@ async def copy_file(
destination file exists, first check whether source and destination are potentially equal
before actually copying,
"""
flog = mlog.fields(func="copy_file")
flog.debug("Enter")

lib_ctx = app_context.lib_ctx.get()
if check_content and lib_ctx.file_check_content > 0:
# Check whether the destination file exists and has the same content as the source file,
# in which case we won't overwrite the destination file
try:
stat_d = os.stat(dest_path)
if stat_d.st_size <= lib_ctx.file_check_content:
stat_s = os.stat(source_path)
if stat_d.st_size == stat_s.st_size:
# Read both files and compare
async with aiofiles.open(source_path, "rb") as f_in:
content_to_copy = await f_in.read()
async with aiofiles.open(dest_path, "rb") as f_in:
existing_content = await f_in.read()
if content_to_copy == existing_content:
flog.debug("Skipping copy, since files are identical")
return
# Since we already read the contents of the file to copy, simply write it to
# the destination instead of reading it again
async with aiofiles.open(dest_path, "wb") as f_out:
await f_out.write(content_to_copy)
return
except FileNotFoundError:
# Destination (or source) file does not exist
pass

async with aiofiles.open(source_path, "rb") as f_in:
async with aiofiles.open(dest_path, "wb") as f_out:
while chunk := await f_in.read(lib_ctx.chunksize):
await f_out.write(chunk)

flog.debug("Leave")
await _copy_file(
source_path,
dest_path,
check_content=check_content,
file_check_content=lib_ctx.file_check_content,
chunksize=lib_ctx.chunksize,
)


async def write_file(filename: StrOrBytesPath, content: str) -> None:
flog = mlog.fields(func="write_file")
flog.debug("Enter")

content_bytes = content.encode("utf-8")

lib_ctx = app_context.lib_ctx.get()
if (
lib_ctx.file_check_content > 0
and len(content_bytes) <= lib_ctx.file_check_content
):
# Check whether the destination file exists and has the same content as the one we want to
# write, in which case we won't overwrite the file
try:
stat = os.stat(filename)
if stat.st_size == len(content_bytes):
# Read file and compare
async with aiofiles.open(filename, "rb") as f:
existing_content = await f.read()
if existing_content == content_bytes:
flog.debug(
"Skipping write, since file already contains the exact content"
)
return
except FileNotFoundError:
# Destination file does not exist
pass

async with aiofiles.open(filename, "wb") as f:
await f.write(content_bytes)

flog.debug("Leave")
await _write_file(filename, content, file_check_content=lib_ctx.file_check_content)


async def read_file(filename: StrOrBytesPath, encoding: str = "utf-8") -> str:
flog = mlog.fields(func="read_file")
flog.debug("Enter")

async with aiofiles.open(filename, "r", encoding=encoding) as f:
content = await f.read()

flog.debug("Leave")
return content
return await _read_file(filename, encoding=encoding)

0 comments on commit 13e34d5

Please sign in to comment.