From 82234463937a1f8bfbe879c5b887a3976fb21f07 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 22 Sep 2023 11:47:42 -0400 Subject: [PATCH] feat(base): cache pip directory (#394) Co-authored-by: Alex Lowe Co-authored-by: Sheng Yu --- craft_providers/base.py | 55 ++++++++++++ craft_providers/bases/__init__.py | 2 +- craft_providers/bases/almalinux.py | 5 ++ craft_providers/bases/centos.py | 5 ++ craft_providers/bases/ubuntu.py | 4 + craft_providers/lxd/launcher.py | 8 +- craft_providers/lxd/lxd_provider.py | 4 +- .../multipass/multipass_provider.py | 4 +- tests/integration/lxd/conftest.py | 28 ++++++ tests/integration/lxd/test_launcher.py | 16 ++-- tests/integration/lxd/test_lxc.py | 90 ++++++++++--------- tests/integration/lxd/test_lxd_provider.py | 30 +++++-- .../multipass/test_multipass_provider.py | 30 +++++-- tests/unit/bases/test_almalinux.py | 17 +++- tests/unit/bases/test_centos_7.py | 16 +++- tests/unit/bases/test_ubuntu_buildd.py | 43 ++++++++- tests/unit/lxd/test_launcher.py | 56 ++++++++++-- tests/unit/test_base.py | 59 ++++++++++++ 18 files changed, 386 insertions(+), 86 deletions(-) diff --git a/craft_providers/base.py b/craft_providers/base.py index 4cee969a..2d11fd81 100644 --- a/craft_providers/base.py +++ b/craft_providers/base.py @@ -79,6 +79,10 @@ class Base(ABC): extend this tag, not overwrite it, e.g.: compatibility_tag = f"{appname}-{Base.compatibility_tag}.{apprevision}" to ensure base compatibility levels are maintained. + + :param cache_path: (Optional) Path to the shared cache directory. If this is + provided, shared cache directories will be mounted as appropriate. Some + directories depend on the base implementation. """ _environment: Dict[str, Optional[str]] @@ -91,6 +95,7 @@ class Base(ABC): _timeout_simple: Optional[float] = TIMEOUT_SIMPLE _timeout_complex: Optional[float] = TIMEOUT_COMPLEX _timeout_unpredictable: Optional[float] = TIMEOUT_UNPREDICTABLE + _cache_path: Optional[pathlib.Path] = None alias: Enum compatibility_tag: str = "base-v2" @@ -105,6 +110,7 @@ def __init__( snaps: Optional[List] = None, packages: Optional[List[str]] = None, use_default_packages: bool = True, + cache_path: Optional[pathlib.Path] = None, ) -> None: pass @@ -786,6 +792,51 @@ def _post_setup_network(self, executor: Executor) -> None: """ return + def _mount_shared_cache_dirs(self, executor: Executor) -> None: + """Mount shared cache directories for this base. + + e.g. + pip cache (normally $HOME/.cache/pip) + + This will only be run if caching is enabled for this instance. + + This step should usually be extended, but may be overridden if common + cache directories are in unusual locations. + """ + if self._cache_path is None: + logger.debug("No cache path set, not mounting cache directories.") + return + + # Get the real path with additional tags. + host_base_cache_path = self._cache_path.resolve().joinpath( + self.compatibility_tag, str(self.alias) + ) + + try: + host_base_cache_path.mkdir(parents=True, exist_ok=True) + except OSError as error: + raise BaseConfigurationError( + brief=f"Failed to create host cache directory: {host_base_cache_path}" + ) from error + + guest_cache_proc = executor.execute_run( + ["bash", "-c", "echo -n ${XDG_CACHE_HOME:-${HOME}/.cache}"], + capture_output=True, + text=True, + ) + guest_base_cache_path = pathlib.Path(guest_cache_proc.stdout) + + # PIP cache + host_pip_cache_path = host_base_cache_path / "pip" + host_pip_cache_path.mkdir(parents=True, exist_ok=True) + + guest_pip_cache_path = guest_base_cache_path / "pip" + executor.execute_run( + ["mkdir", "-p", guest_pip_cache_path.as_posix()], + ) + + executor.mount(host_source=host_pip_cache_path, target=guest_pip_cache_path) + def _pre_setup_packages(self, executor: Executor) -> None: """Do anything before setting up the packages. @@ -964,6 +1015,8 @@ def setup( self._update_compatibility_tag(executor=executor) + self._mount_shared_cache_dirs(executor=executor) + self._pre_setup_os(executor=executor) self._setup_os(executor=executor) self._post_setup_os(executor=executor) @@ -1029,6 +1082,8 @@ def warmup( self._image_check(executor=executor) self._post_image_check(executor=executor) + self._mount_shared_cache_dirs(executor=executor) + self._setup_wait_for_system_ready(executor=executor) self._setup_wait_for_network(executor=executor) diff --git a/craft_providers/bases/__init__.py b/craft_providers/bases/__init__.py index ec8e78f6..ce2a99d3 100644 --- a/craft_providers/bases/__init__.py +++ b/craft_providers/bases/__init__.py @@ -22,8 +22,8 @@ from typing import Dict, NamedTuple, Tuple, Type, Union from craft_providers.errors import BaseCompatibilityError, BaseConfigurationError +from craft_providers.base import Base -from ..base import Base from . import almalinux, centos from . import ubuntu from . import ubuntu as buildd diff --git a/craft_providers/bases/almalinux.py b/craft_providers/bases/almalinux.py index 50600ffe..20932517 100644 --- a/craft_providers/bases/almalinux.py +++ b/craft_providers/bases/almalinux.py @@ -18,6 +18,7 @@ """Almalinux image(s).""" import enum import logging +import pathlib import subprocess from typing import Dict, List, Optional @@ -63,6 +64,8 @@ class AlmaLinuxBase(Base): :param snaps: Optional list of snaps to install on the base image. :param packages: Optional list of system packages to install on the base image. :param use_default_packages: Optional bool to enable/disable default packages. + :param cache_path: (Optional) Path to the shared cache directory. If this is + provided, shared cache directories will be mounted as appropriate. """ compatibility_tag: str = f"almalinux-{Base.compatibility_tag}" @@ -77,7 +80,9 @@ def __init__( snaps: Optional[List[Snap]] = None, packages: Optional[List[str]] = None, use_default_packages: bool = True, + cache_path: Optional[pathlib.Path] = None, ) -> None: + self._cache_path = cache_path self.alias: AlmaLinuxBaseAlias = alias if environment is None: diff --git a/craft_providers/bases/centos.py b/craft_providers/bases/centos.py index df6d6281..2d52d076 100644 --- a/craft_providers/bases/centos.py +++ b/craft_providers/bases/centos.py @@ -18,6 +18,7 @@ """CentOS image(s).""" import enum import logging +import pathlib import subprocess from typing import Dict, List, Optional @@ -63,6 +64,8 @@ class CentOSBase(Base): :param snaps: Optional list of snaps to install on the base image. :param packages: Optional list of system packages to install on the base image. :param use_default_packages: Optional bool to enable/disable default packages. + :param cache_path: Optional path to the shared cache directory. If this is + provided, shared cache directories will be mounted as appropriate. """ compatibility_tag: str = f"centos-{Base.compatibility_tag}" @@ -77,7 +80,9 @@ def __init__( snaps: Optional[List[Snap]] = None, packages: Optional[List[str]] = None, use_default_packages: bool = True, + cache_path: Optional[pathlib.Path] = None, ) -> None: + self._cache_dir = cache_path self.alias: CentOSBaseAlias = alias if environment is None: diff --git a/craft_providers/bases/ubuntu.py b/craft_providers/bases/ubuntu.py index de73cd2f..ad77131b 100644 --- a/craft_providers/bases/ubuntu.py +++ b/craft_providers/bases/ubuntu.py @@ -71,6 +71,8 @@ class BuilddBase(Base): :param snaps: Optional list of snaps to install on the base image. :param packages: Optional list of system packages to install on the base image. :param use_default_packages: Optional bool to enable/disable default packages. + :param cache_path: Optional path to the shared cache directory. If this is + provided, shared cache directories will be mounted as appropriate. """ compatibility_tag: str = f"buildd-{Base.compatibility_tag}" @@ -85,8 +87,10 @@ def __init__( snaps: Optional[List[Snap]] = None, packages: Optional[List[str]] = None, use_default_packages: bool = True, + cache_path: Optional[pathlib.Path] = None, ) -> None: self.alias: BuilddBaseAlias = alias + self._cache_path = cache_path if environment is None: self._environment = self.default_command_environment() diff --git a/craft_providers/lxd/launcher.py b/craft_providers/lxd/launcher.py index ffe1ae62..5eac7f5d 100644 --- a/craft_providers/lxd/launcher.py +++ b/craft_providers/lxd/launcher.py @@ -130,6 +130,8 @@ def _create_instance( image=image_name, image_remote=image_remote, ephemeral=False, # base instance should not ephemeral + map_user_uid=map_user_uid, + uid=uid, ) base_instance_status = base_instance.config_get("user.craft_providers.status") @@ -178,7 +180,11 @@ def _create_instance( image_remote, ) instance.launch( - image=image_name, image_remote=image_remote, ephemeral=ephemeral + image=image_name, + image_remote=image_remote, + ephemeral=ephemeral, + map_user_uid=map_user_uid, + uid=uid, ) instance_status = instance.config_get("user.craft_providers.status") diff --git a/craft_providers/lxd/lxd_provider.py b/craft_providers/lxd/lxd_provider.py index ed3c3398..ac6b862e 100644 --- a/craft_providers/lxd/lxd_provider.py +++ b/craft_providers/lxd/lxd_provider.py @@ -20,7 +20,7 @@ import logging import pathlib from datetime import timedelta -from typing import Generator, Optional +from typing import Iterator, Optional from craft_providers import Executor, Provider from craft_providers.base import Base @@ -110,7 +110,7 @@ def launched_environment( build_base: Optional[str] = None, instance_name: str, allow_unstable: bool = False, - ) -> Generator[Executor, None, None]: + ) -> Iterator[Executor]: """Configure and launch environment for specified base. When this method loses context, all directories are unmounted and the diff --git a/craft_providers/multipass/multipass_provider.py b/craft_providers/multipass/multipass_provider.py index af340825..149114c7 100644 --- a/craft_providers/multipass/multipass_provider.py +++ b/craft_providers/multipass/multipass_provider.py @@ -21,7 +21,7 @@ import pathlib from dataclasses import dataclass from enum import Enum -from typing import Dict, Generator, Optional +from typing import Dict, Iterator, Optional from craft_providers import Base, Executor, Provider, base from craft_providers.bases import ubuntu @@ -187,7 +187,7 @@ def launched_environment( build_base: Optional[str] = None, instance_name: str, allow_unstable: bool = False, - ) -> Generator[Executor, None, None]: + ) -> Iterator[Executor]: """Configure and launch environment for specified base. When this method loses context, all directories are unmounted and the diff --git a/tests/integration/lxd/conftest.py b/tests/integration/lxd/conftest.py index 2ebf5cf1..94ecb66a 100644 --- a/tests/integration/lxd/conftest.py +++ b/tests/integration/lxd/conftest.py @@ -28,6 +28,7 @@ from craft_providers.lxd import LXC from craft_providers.lxd import project as lxc_project from craft_providers.lxd.lxd_instance import LXDInstance +from craft_providers.lxd.lxd_provider import LXDProvider @pytest.fixture(autouse=True, scope="module") @@ -143,3 +144,30 @@ def project(lxc, project_name): yield project_name lxc_project.purge(lxc=lxc, project=project_name) + + +@pytest.fixture(scope="session") +def session_project(installed_lxd): + lxc = LXC() + project_name = "craft-providers-test-session" + lxc_project.create_with_default_profile(lxc=lxc, project=project_name) + + projects = lxc.project_list() + assert project_name in projects + + instances = lxc.list(project=project_name) + assert instances == [] + + expected_cfg = lxc.profile_show(profile="default", project="default") + expected_cfg["used_by"] = [] + + assert lxc.profile_show(profile="default", project=project_name) == expected_cfg + + yield project_name + + lxc_project.purge(lxc=lxc, project=project_name) + + +@pytest.fixture(scope="session") +def session_provider(session_project): + return LXDProvider(lxd_project=session_project) diff --git a/tests/integration/lxd/test_launcher.py b/tests/integration/lxd/test_launcher.py index 59a1d37f..7c693d68 100644 --- a/tests/integration/lxd/test_launcher.py +++ b/tests/integration/lxd/test_launcher.py @@ -380,10 +380,10 @@ def test_launch_create_project(base_configuration, instance_name, project_name): def test_launch_with_project_and_use_base_instance( - base_configuration, get_base_instance, instance_name, project + base_configuration, get_base_instance, instance_name, session_project ): """With a LXD project specified, launch an instance and use base instances.""" - base_instance = get_base_instance(project=project) + base_instance = get_base_instance(project=session_project) # launch an instance from an image and create a base instance instance = lxd.launch( @@ -392,7 +392,7 @@ def test_launch_with_project_and_use_base_instance( image_name="22.04", image_remote="ubuntu", use_base_instance=True, - project=project, + project=session_project, remote="local", ) @@ -413,7 +413,7 @@ def test_launch_with_project_and_use_base_instance( image_name="22.04", image_remote="ubuntu", use_base_instance=True, - project=project, + project=session_project, remote="local", ) @@ -427,7 +427,7 @@ def test_launch_with_project_and_use_base_instance( image_name="22.04", image_remote="ubuntu", use_base_instance=True, - project=project, + project=session_project, remote="local", ) @@ -441,10 +441,10 @@ def test_launch_with_project_and_use_base_instance( def test_launch_with_project_and_use_base_instance_parallel( - base_configuration, get_base_instance, instance_name, project + base_configuration, get_base_instance, instance_name, session_project ): """Launch 5 instances at same time and use the same base instances.""" - base_instance = get_base_instance(project=project) + base_instance = get_base_instance(project=session_project) instances = [] @@ -461,7 +461,7 @@ def run(self): image_name="22.04", image_remote="ubuntu", use_base_instance=True, - project=project, + project=session_project, remote="local", ) diff --git a/tests/integration/lxd/test_lxc.py b/tests/integration/lxd/test_lxc.py index 34f68d5a..060d2ea0 100644 --- a/tests/integration/lxd/test_lxc.py +++ b/tests/integration/lxd/test_lxc.py @@ -25,46 +25,50 @@ @pytest.fixture() -def instance(instance_name, project): - with conftest.tmp_instance(name=instance_name, project=project) as tmp_instance: +def instance(instance_name, session_project): + with conftest.tmp_instance( + name=instance_name, project=session_project + ) as tmp_instance: yield tmp_instance -def test_exec(instance, lxc, project): +def test_exec(instance, lxc, session_project): proc = lxc.exec( instance_name=instance, command=["echo", "this is a test"], - project=project, + project=session_project, capture_output=True, ) assert proc.stdout == b"this is a test\n" -def test_config_get_and_set(instance, instance_name, lxc, project): +def test_config_get_and_set(instance, instance_name, lxc, session_project): """Set and get config key/value pairs.""" lxc.config_set( instance_name=instance, key="user.test-key", # `user` namespace is for arbitrary config values value="test-value", - project=project, + project=session_project, ) - value = lxc.config_get(instance_name=instance, key="user.test-key", project=project) + value = lxc.config_get( + instance_name=instance, key="user.test-key", project=session_project + ) assert value == "test-value" -def test_config_get_non_existent_key(instance, instance_name, lxc, project): +def test_config_get_non_existent_key(instance, instance_name, lxc, session_project): """Get a non-existent key and confirm the value is an empty string.""" value = lxc.config_get( - instance_name=instance, key="non-existant-key", project=project + instance_name=instance, key="non-existant-key", project=session_project ) assert not value -def test_copy(instance, instance_name, lxc, project): +def test_copy(instance, instance_name, lxc, session_project): """Test `copy()` with default arguments.""" destination_instance_name = instance_name + "-destination" @@ -72,23 +76,23 @@ def test_copy(instance, instance_name, lxc, project): lxc.copy( source_instance_name=instance, destination_instance_name=destination_instance_name, - project=project, + project=session_project, ) - instances = lxc.list_names(project=project) + instances = lxc.list_names(project=session_project) # verify both instances exist assert instances == [instance, destination_instance_name] -def test_copy_error(instance, instance_name, lxc, project): +def test_copy_error(instance, instance_name, lxc, session_project): """Raise a LXDError when the copy command fails.""" # the source and destination cannot be same, so LXC will fail to copy with pytest.raises(LXDError) as raised: lxc.copy( source_instance_name=instance, destination_instance_name=instance, - project=project, + project=session_project, ) assert raised.value == LXDError( @@ -97,7 +101,7 @@ def test_copy_error(instance, instance_name, lxc, project): f"{instance_name}'." ), details=( - f"* Command that failed: 'lxc --project {project} copy local:" + f"* Command that failed: 'lxc --project {session_project} copy local:" f"{instance_name} local:{instance_name}'\n" "* Command exit code: 1\n" "* Command standard error output: b'Error: Failed creating instance " @@ -107,54 +111,54 @@ def test_copy_error(instance, instance_name, lxc, project): ) -def test_delete(instance, lxc, project): +def test_delete(instance, lxc, session_project): with pytest.raises(LXDError): - lxc.delete(instance_name=instance, force=False, project=project) + lxc.delete(instance_name=instance, force=False, project=session_project) - instances = lxc.list_names(project=project) - assert instances == [instance] + instances = lxc.list_names(project=session_project) + assert instance in instances -def test_delete_force(instance, lxc, project): - lxc.delete(instance_name=instance, force=True, project=project) +def test_delete_force(instance, lxc, session_project): + lxc.delete(instance_name=instance, force=True, project=session_project) - instances = lxc.list_names(project=project) - assert instances == [] + instances = lxc.list_names(project=session_project) + assert instance not in instances -def test_image_copy(lxc, project): +def test_image_copy(lxc, session_project): lxc.image_copy( image="22.04", image_remote="ubuntu", alias="test-2204", - project=project, + project=session_project, ) - images = lxc.image_list(project=project) + images = lxc.image_list(project=session_project) assert len(images) == 1 -def test_image_delete(lxc, project): +def test_image_delete(lxc, session_project): lxc.image_copy( image="22.04", image_remote="ubuntu", alias="test-2204", - project=project, + project=session_project, ) - lxc.image_delete(image="test-2204", project=project) + lxc.image_delete(image="test-2204", project=session_project) - images = lxc.image_list(project=project) + images = lxc.image_list(project=session_project) assert images == [] -def test_file_push(instance, lxc, project, tmp_path): +def test_file_push(instance, lxc, session_project, tmp_path): test_file = tmp_path / "test.txt" test_file.write_text("this is a test") lxc.file_push( instance_name=instance, - project=project, + project=session_project, source=test_file, destination=pathlib.PurePosixPath("/tmp/foo"), ) @@ -162,7 +166,7 @@ def test_file_push(instance, lxc, project, tmp_path): proc = lxc.exec( command=["cat", "/tmp/foo"], instance_name=instance, - project=project, + project=session_project, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, @@ -171,21 +175,21 @@ def test_file_push(instance, lxc, project, tmp_path): assert proc.stdout == b"this is a test" -def test_file_pull(instance, lxc, project, tmp_path): +def test_file_pull(instance, lxc, session_project, tmp_path): out_path = tmp_path / "out.txt" test_file = tmp_path / "test.txt" test_file.write_text("this is a test") lxc.file_push( instance_name=instance, - project=project, + project=session_project, source=test_file, destination=pathlib.PurePosixPath("/tmp/foo"), ) lxc.file_pull( instance_name=instance, - project=project, + project=session_project, source=pathlib.PurePosixPath("/tmp/foo"), destination=out_path, ) @@ -193,7 +197,7 @@ def test_file_pull(instance, lxc, project, tmp_path): assert out_path.read_text() == "this is a test" -def test_disk_add_remove(instance, lxc, tmp_path, project): +def test_disk_add_remove(instance, lxc, tmp_path, session_project): mount_target = pathlib.PurePosixPath("/mnt") # Make sure permissions allow read from inside guest without mappings. @@ -207,7 +211,7 @@ def test_disk_add_remove(instance, lxc, tmp_path, project): source=tmp_path, path=mount_target, device="test_mount", - project=project, + project=session_project, ) proc = lxc.exec( @@ -215,7 +219,7 @@ def test_disk_add_remove(instance, lxc, tmp_path, project): instance_name=instance, capture_output=True, check=True, - project=project, + project=session_project, ) assert proc.stdout == b"this is a test" @@ -223,20 +227,20 @@ def test_disk_add_remove(instance, lxc, tmp_path, project): lxc.config_device_remove( instance_name=instance, device="test_mount", - project=project, + project=session_project, ) with pytest.raises(subprocess.CalledProcessError): lxc.exec( command=["test", "-f", "/mnt/test.txt"], - project=project, + project=session_project, instance_name=instance, check=True, ) -def test_info(instance, lxc, project): +def test_info(instance, lxc, session_project): """Test `info()` method works as expected.""" - data = lxc.info(instance_name=instance, project=project) + data = lxc.info(instance_name=instance, project=session_project) assert data["Name"] == instance diff --git a/tests/integration/lxd/test_lxd_provider.py b/tests/integration/lxd/test_lxd_provider.py index d92804a3..db2a6520 100644 --- a/tests/integration/lxd/test_lxd_provider.py +++ b/tests/integration/lxd/test_lxd_provider.py @@ -17,7 +17,7 @@ # import pytest -from craft_providers.bases import get_base_from_alias, ubuntu +from craft_providers.bases import almalinux, get_base_from_alias, ubuntu from craft_providers.lxd import LXDProvider, is_installed @@ -43,21 +43,37 @@ def test_create_environment(installed_lxd, instance_name): @pytest.mark.parametrize( "alias", - set(ubuntu.BuilddBaseAlias) - {ubuntu.BuilddBaseAlias.XENIAL}, + set(ubuntu.BuilddBaseAlias) - {ubuntu.BuilddBaseAlias.XENIAL} + | {almalinux.AlmaLinuxBaseAlias.NINE}, ) -def test_launched_environment(alias, installed_lxd, instance_name, tmp_path): - provider = LXDProvider() +def test_launched_environment( + alias, installed_lxd, instance_name, tmp_path, session_provider +): + cache_path = tmp_path / "cache" + project_path = tmp_path / "project" + cache_path.mkdir() + project_path.mkdir() + + base_configuration = get_base_from_alias(alias)(alias=alias, cache_path=cache_path) - base_configuration = get_base_from_alias(alias)(alias=alias) - with provider.launched_environment( + with session_provider.launched_environment( project_name="test-project", - project_path=tmp_path, + project_path=project_path, base_configuration=base_configuration, instance_name=instance_name, allow_unstable=True, ) as test_instance: assert test_instance.exists() is True assert test_instance.is_running() is True + test_instance.execute_run(["touch", "/root/.cache/pip/test-pip-cache"]) + + assert ( + cache_path + / base_configuration.compatibility_tag + / str(base_configuration.alias) + / "pip" + / "test-pip-cache" + ).exists() assert test_instance.exists() is True assert test_instance.is_running() is False diff --git a/tests/integration/multipass/test_multipass_provider.py b/tests/integration/multipass/test_multipass_provider.py index ad3ad2d5..8dfa2103 100644 --- a/tests/integration/multipass/test_multipass_provider.py +++ b/tests/integration/multipass/test_multipass_provider.py @@ -44,20 +44,27 @@ def test_create_environment(installed_multipass, instance_name): assert test_instance.exists() is False -@pytest.mark.parametrize("alias", set(BuilddBaseAlias) - {BuilddBaseAlias.XENIAL}) +ALIASES = list(BuilddBaseAlias) +ALIASES.remove(BuilddBaseAlias.XENIAL) + + +@pytest.mark.parametrize("alias", ALIASES) def test_launched_environment(alias, installed_multipass, instance_name, tmp_path): """Verify `launched_environment()` creates and starts an instance then stops the instance when the method loses context.""" if sys.platform == "darwin" and alias == BuilddBaseAlias.DEVEL: pytest.skip(reason="snapcraft:devel is not working on MacOS (LP #2007419)") + project_path = tmp_path / "project" + cache_path = tmp_path / "cache" + provider = MultipassProvider() - base_configuration = BuilddBase(alias=alias) + base_configuration = BuilddBase(alias=alias, cache_path=cache_path) with provider.launched_environment( project_name="test-multipass-project", - project_path=tmp_path, + project_path=project_path, base_configuration=base_configuration, instance_name=instance_name, allow_unstable=True, @@ -65,5 +72,18 @@ def test_launched_environment(alias, installed_multipass, instance_name, tmp_pat assert test_instance.exists() is True assert test_instance.is_running() is True - assert test_instance.exists() is True - assert test_instance.is_running() is False + test_instance.execute_run(["touch", "/root/.cache/pip/test-pip-cache"]) + + assert ( + cache_path + / base_configuration.compatibility_tag + / str(base_configuration.alias) + / "pip" + / "test-pip-cache" + ).exists() + + try: + assert test_instance.exists() is True + assert test_instance.is_running() is False + finally: + test_instance.delete() diff --git a/tests/unit/bases/test_almalinux.py b/tests/unit/bases/test_almalinux.py index 83b60b65..6bdf43db 100644 --- a/tests/unit/bases/test_almalinux.py +++ b/tests/unit/bases/test_almalinux.py @@ -14,8 +14,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # - - +import pathlib import subprocess from pathlib import Path from textwrap import dedent @@ -1161,7 +1160,10 @@ def test_ensuresetup_completed_not_setup(status, fake_executor, mock_load): }, ], ) -def test_warmup_overall(environment, fake_process, fake_executor, mock_load, mocker): +@pytest.mark.parametrize("cache_path", [None, pathlib.Path("/tmp")]) +def test_warmup_overall( + environment, fake_process, fake_executor, mock_load, mocker, cache_path +): mock_load.return_value = InstanceConfiguration( compatibility_tag="almalinux-base-v2", setup=True ) @@ -1170,7 +1172,10 @@ def test_warmup_overall(environment, fake_process, fake_executor, mock_load, moc if environment is None: environment = almalinux.AlmaLinuxBase.default_command_environment() - base_config = almalinux.AlmaLinuxBase(alias=alias, environment=environment) + base_config = almalinux.AlmaLinuxBase( + alias=alias, + environment=environment, + ) fake_process.register_subprocess( [*DEFAULT_FAKE_CMD, "cat", "/etc/os-release"], @@ -1186,6 +1191,10 @@ def test_warmup_overall(environment, fake_process, fake_executor, mock_load, moc fake_process.register_subprocess( [*DEFAULT_FAKE_CMD, "systemctl", "is-system-running"], stdout="degraded" ) + fake_process.register_subprocess( + [*DEFAULT_FAKE_CMD, "bash", "-c", "echo -n ${XDG_CACHE_HOME:-${HOME}/.cache}"], + stdout="/root/.cache", + ) fake_process.register_subprocess( [*DEFAULT_FAKE_CMD, "getent", "hosts", "snapcraft.io"] ) diff --git a/tests/unit/bases/test_centos_7.py b/tests/unit/bases/test_centos_7.py index 3f5358de..b7603abc 100644 --- a/tests/unit/bases/test_centos_7.py +++ b/tests/unit/bases/test_centos_7.py @@ -14,8 +14,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # - - +import pathlib import subprocess from pathlib import Path from textwrap import dedent @@ -1108,7 +1107,10 @@ def test_ensure_setup_completed_not_setup(status, fake_executor, mock_load): }, ], ) -def test_warmup_overall(environment, fake_process, fake_executor, mock_load, mocker): +@pytest.mark.parametrize("cache_path", [None, pathlib.Path("/tmp")]) +def test_warmup_overall( + environment, fake_process, fake_executor, mock_load, mocker, cache_path +): mock_load.return_value = InstanceConfiguration( compatibility_tag="centos-base-v2", setup=True ) @@ -1133,6 +1135,10 @@ def test_warmup_overall(environment, fake_process, fake_executor, mock_load, moc fake_process.register_subprocess( [*DEFAULT_FAKE_CMD, "systemctl", "is-system-running"], stdout="degraded" ) + fake_process.register_subprocess( + [*DEFAULT_FAKE_CMD, "bash", "-c", "echo -n ${XDG_CACHE_HOME:-${HOME}/.cache}"], + stdout="/root/.cache", + ) fake_process.register_subprocess( [*DEFAULT_FAKE_CMD, "getent", "hosts", "snapcraft.io"] ) @@ -1283,6 +1289,10 @@ def test_warmup_never_network(fake_process, fake_executor, mock_load): base_config._timeout_simple = 0.01 base_config._retry_wait = 0.02 + fake_process.register_subprocess( + [*DEFAULT_FAKE_CMD, "bash", "-c", "echo -n ${XDG_CACHE_HOME:-${HOME}/.cache}"], + stdout="/root/.cache", + ) fake_process.register_subprocess( [*DEFAULT_FAKE_CMD, "cat", "/etc/os-release"], stdout=dedent( diff --git a/tests/unit/bases/test_ubuntu_buildd.py b/tests/unit/bases/test_ubuntu_buildd.py index a1ebbb04..6ca36345 100644 --- a/tests/unit/bases/test_ubuntu_buildd.py +++ b/tests/unit/bases/test_ubuntu_buildd.py @@ -14,9 +14,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # - - +import pathlib import subprocess +import sys from pathlib import Path from textwrap import dedent from unittest.mock import ANY, call, patch @@ -686,6 +686,40 @@ def test_ensure_image_version_compatible_failure(fake_executor, monkeypatch): ) +@pytest.mark.parametrize("alias", list(ubuntu.BuilddBaseAlias)) +@pytest.mark.parametrize("cache_path", [pathlib.Path("/tmp/fake-cache-dir")]) +def test_mount_cache_dirs(fake_process, fake_executor, cache_path, alias): + """Test mounting of cache directories with a cache directory set.""" + base = ubuntu.BuilddBase(alias=alias, cache_path=cache_path) + host_cache_dir = cache_path / base.compatibility_tag / str(base.alias) + user_cache_dir = pathlib.Path("/root/.cache") + fake_process.register( + [*DEFAULT_FAKE_CMD, "bash", "-c", "echo -n ${XDG_CACHE_HOME:-${HOME}/.cache}"], + stdout=str(user_cache_dir), + ) + fake_process.register( + [*DEFAULT_FAKE_CMD, "mkdir", "-p", "/root/.cache/pip"], + ) + + base._mount_shared_cache_dirs(fake_executor) + + if sys.platform == "win32": + expected_mounts = [ + { + "host_source": pathlib.WindowsPath("D:") / host_cache_dir / "pip", + "target": user_cache_dir / "pip", + } + ] + else: + expected_mounts = [ + { + "host_source": host_cache_dir / "pip", + "target": user_cache_dir / "pip", + }, + ] + assert fake_executor.records_of_mount == expected_mounts + + def test_get_os_release(fake_process, fake_executor): """`_get_os_release` should parse data from `/etc/os-release` to a dict.""" base_config = ubuntu.BuilddBase(alias=ubuntu.BuilddBaseAlias.JAMMY) @@ -1507,7 +1541,10 @@ def test_ensure_setup_completed_not_setup(status, fake_executor, mock_load): }, ], ) -def test_warmup_overall(environment, fake_process, fake_executor, mock_load, mocker): +@pytest.mark.parametrize("cache_path", [None, pathlib.Path("/tmp")]) +def test_warmup_overall( + environment, fake_process, fake_executor, mock_load, mocker, cache_path +): mock_load.return_value = InstanceConfiguration( compatibility_tag="buildd-base-v2", setup=True ) diff --git a/tests/unit/lxd/test_launcher.py b/tests/unit/lxd/test_launcher.py index 2f2a9100..6cf5b880 100644 --- a/tests/unit/lxd/test_launcher.py +++ b/tests/unit/lxd/test_launcher.py @@ -153,7 +153,13 @@ def test_launch_no_base_instance( ] assert fake_instance.mock_calls == [ call.exists(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.launch( + image="image-name", + image_remote="image-remote", + ephemeral=False, + map_user_uid=False, + uid=None, + ), call.config_get("user.craft_providers.status"), call.config_set("user.craft_providers.status", "PREPARING"), call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), @@ -479,7 +485,13 @@ def test_launch_existing_base_instance_invalid( call.exists(), call.delete(), call.__bool__(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.launch( + image="image-name", + image_remote="image-remote", + ephemeral=False, + map_user_uid=False, + uid=None, + ), call.config_get("user.craft_providers.status"), call.config_set("user.craft_providers.status", "PREPARING"), call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), @@ -543,7 +555,13 @@ def test_launch_all_opts( ] assert fake_instance.mock_calls == [ call.exists(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=True), + call.launch( + image="image-name", + image_remote="image-remote", + ephemeral=True, + map_user_uid=True, + uid=1234, + ), call.config_get("user.craft_providers.status"), call.config_set("user.craft_providers.status", "PREPARING"), call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), @@ -643,7 +661,13 @@ def test_launch_create_project( ] assert fake_instance.mock_calls == [ call.exists(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.launch( + image="image-name", + image_remote="image-remote", + ephemeral=False, + map_user_uid=False, + uid=None, + ), call.config_get("user.craft_providers.status"), call.config_set("user.craft_providers.status", "PREPARING"), call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), @@ -779,7 +803,13 @@ def test_launch_with_existing_instance_incompatible_with_auto_clean( call.is_running(), call.start(), call.delete(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.launch( + image="image-name", + image_remote="image-remote", + ephemeral=False, + map_user_uid=False, + uid=None, + ), call.config_get("user.craft_providers.status"), call.config_set("user.craft_providers.status", "PREPARING"), call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), @@ -869,7 +899,13 @@ def test_launch_with_existing_ephemeral_instance( assert fake_instance.mock_calls == [ call.exists(), call.delete(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=True), + call.launch( + image="image-name", + image_remote="image-remote", + ephemeral=True, + map_user_uid=False, + uid=None, + ), call.config_get("user.craft_providers.status"), call.config_set("user.craft_providers.status", "PREPARING"), call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), @@ -1055,7 +1091,13 @@ def test_use_snapshots_deprecated( assert fake_base_instance.mock_calls == [ call.exists(), call.__bool__(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.launch( + image="image-name", + image_remote="image-remote", + ephemeral=False, + map_user_uid=False, + uid=None, + ), call.config_get("user.craft_providers.status"), call.config_set("user.craft_providers.status", "PREPARING"), call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), diff --git a/tests/unit/test_base.py b/tests/unit/test_base.py index a6efd16f..2eef8013 100644 --- a/tests/unit/test_base.py +++ b/tests/unit/test_base.py @@ -16,7 +16,9 @@ # """Tests for abstract Base's implementations.""" import enum +import pathlib import subprocess +import sys from unittest import mock import pytest @@ -138,6 +140,63 @@ def test_wait_for_network_timeout(fake_base, fake_executor, fake_process, callba fake_base._setup_wait_for_system_ready(fake_executor) +@pytest.mark.parametrize("cache_dir", [pathlib.Path("/tmp/fake-cache-dir")]) +def test_mount_shared_cache_dirs(fake_process, fake_base, fake_executor, cache_dir): + """Test mounting of cache directories with a cache directory set.""" + fake_base._cache_path = cache_dir + user_cache_dir = pathlib.Path("/root/.cache") + + fake_process.register( + [*DEFAULT_FAKE_CMD, "bash", "-c", "echo -n ${XDG_CACHE_HOME:-${HOME}/.cache}"], + stdout=str(user_cache_dir), + ) + + fake_process.register( + [*DEFAULT_FAKE_CMD, "mkdir", "-p", "/root/.cache/pip"], + ) + + fake_base._mount_shared_cache_dirs(fake_executor) + + if sys.platform == "win32": + expected = { + "host_source": pathlib.WindowsPath("d:") + / cache_dir + / "base-v2" + / "FakeBaseAlias.TREBLE" + / "pip", + "target": user_cache_dir / "pip", + } + else: + expected = { + "host_source": cache_dir / "base-v2" / "FakeBaseAlias.TREBLE" / "pip", + "target": user_cache_dir / "pip", + } + assert fake_executor.records_of_mount == [expected] + + +@pytest.mark.parametrize("cache_dir", [pathlib.Path("/tmp/fake-cache-dir")]) +def test_mount_shared_cache_dirs_mkdir_failed( + fake_process, fake_base, fake_executor, cache_dir, mocker +): + """Test mounting of cache directories with a cache directory set, but mkdir failed.""" + fake_base._cache_path = cache_dir + user_cache_dir = pathlib.Path("/root/.cache") + + fake_process.register( + [*DEFAULT_FAKE_CMD, "bash", "-c", "echo -n ${XDG_CACHE_HOME:-${HOME}/.cache}"], + stdout=str(user_cache_dir), + ) + + fake_process.register( + [*DEFAULT_FAKE_CMD, "mkdir", "-p", "/root/.cache/pip"], + ) + + mocker.patch("pathlib.Path.mkdir", side_effect=OSError) + + with pytest.raises(BaseConfigurationError): + fake_base._mount_shared_cache_dirs(fake_executor) + + @pytest.mark.parametrize( ("process_outputs", "expected"), [