diff --git a/craft_providers/lxd/launcher.py b/craft_providers/lxd/launcher.py index 7409ea55..8a927324 100644 --- a/craft_providers/lxd/launcher.py +++ b/craft_providers/lxd/launcher.py @@ -22,7 +22,9 @@ import re import subprocess import sys -from datetime import datetime, timedelta +import threading +import time +from datetime import datetime, timedelta, timezone from typing import Optional from craft_providers import Base, ProviderError, bases @@ -31,11 +33,54 @@ from .errors import LXDError from .lxc import LXC from .lxd_instance import LXDInstance +from .lxd_instance_status import ProviderInstanceStatus from .project import create_with_default_profile logger = logging.getLogger(__name__) +class InstanceTimer(threading.Thread): + """Timer for update instance that still alive.""" + + __active: bool = True + __interval: float + __instance: LXDInstance + + def __init__(self, instance: LXDInstance, interval: int = 3) -> None: + """Initialize the timer. + + :param instance: LXD instance to update. + :param interval: Interval in seconds to update the instance timer. + """ + self.__instance = instance + self.__interval = interval + super().__init__(daemon=True) + + def run(self) -> None: + """Run the timer.""" + while self.__active: + now = datetime.now(timezone.utc).isoformat() + try: + self.__instance.config_set("user.craft_providers.timer", now) + logger.debug("Set instance timer to %r", now) + except LXDError: + # Error in timer update is not critical + logging.exception("Error updating instance timer") + time.sleep(self.__interval) + + try: + self.__instance.config_set("user.craft_providers.timer", "DONE") + logger.debug("Set instance timer to 'DONE'") + except LXDError: + # Error in timer update is not critical + logging.exception("Error updating instance timer") + logger.debug("Instance timer update stopped.") + + def stop(self) -> None: + """Stop the timer.""" + self.__active = False + + def _create_instance( *, instance: LXDInstance, @@ -71,49 +116,107 @@ def _create_instance( :param lxc: LXC client. """ logger.info("Creating new instance from remote") - logger.debug( - "Creating new instance from image %r from remote %r.", image_name, image_remote - ) - instance.launch(image=image_name, image_remote=image_remote, ephemeral=ephemeral) - base_configuration.setup(executor=instance) - _set_timezone(instance, project, remote, lxc) - - # return early if base instances and user id mapping are not specified - if not base_instance and not map_user_uid: - return + # Lockable base instance creation. Only one caller can create the base instance. + # Other callers will will get LXDError and wait until the base instance is created. + if base_instance: + logger.info("Creating new base instance from remote") + logger.debug( + "Creating new base instance from image %r from remote %r", + image_name, + image_remote, + ) + base_instance.launch( + image=image_name, + image_remote=image_remote, + ephemeral=False, # base instance should not ephemeral + ) + base_instance_status = base_instance.config_get("user.craft_providers.status") - # the instance needs to be stopped before copying or updating the id map - instance.stop() + # Skip the base configuration if the instance is already configured. + if base_instance_status != ProviderInstanceStatus.FINISHED.value: + logger.debug("Setting up base instance %r", base_instance.instance_name) + base_instance.config_set( + "user.craft_providers.status", ProviderInstanceStatus.PREPARING.value + ) + config_timer = InstanceTimer(base_instance) + config_timer.start() + base_configuration.setup(executor=base_instance) + _set_timezone( + base_instance, + base_instance.project, + base_instance.remote, + base_instance.lxc, + ) + base_instance.config_set( + "user.craft_providers.status", ProviderInstanceStatus.FINISHED.value + ) + # set the full instance name as image description + lxc.config_set( + instance_name=base_instance.instance_name, + key="image.description", + value=base_instance.name, + project=project, + remote=remote, + ) + config_timer.stop() + base_instance.stop() - if base_instance: - logger.info("Creating new base instance from instance") + # Copy the base instance to the instance. + logger.info("Creating new instance from base instance") logger.debug( - "Creating new base instance %r from instance.", base_instance.instance_name + "Creating new instance %r from base instance %r", + instance.instance_name, + base_instance.instance_name, ) lxc.copy( - source_remote=remote, - source_instance_name=instance.instance_name, + source_remote=base_instance.remote, + source_instance_name=base_instance.instance_name, destination_remote=remote, - destination_instance_name=base_instance.instance_name, + destination_instance_name=instance.instance_name, project=project, ) - - # set the full instance name as image description - lxc.config_set( - instance_name=base_instance.instance_name, - key="image.description", - value=base_instance.name, - project=project, - remote=remote, + _set_timezone(instance, project, remote, lxc) + else: + logger.debug( + "Creating new instance from image %r from remote %r", + image_name, + image_remote, ) + instance.launch( + image=image_name, image_remote=image_remote, ephemeral=ephemeral + ) + instance_status = instance.config_get("user.craft_providers.status") + + # Skip the base configuration if the instance is already configured. + if instance_status != ProviderInstanceStatus.FINISHED.value: + logger.info("Setting up instance") + logger.debug("Setting up instance %r", instance.instance_name) + instance.config_set( + "user.craft_providers.status", ProviderInstanceStatus.PREPARING.value + ) + config_timer = InstanceTimer(instance) + config_timer.start() + base_configuration.setup(executor=instance) + _set_timezone(instance, project, remote, lxc) + instance.config_set( + "user.craft_providers.status", ProviderInstanceStatus.FINISHED.value + ) + config_timer.stop() + if not ephemeral: + # stop ephemeral instances will delete them immediately + instance.stop() # after creating the base instance, the id map can be set if map_user_uid: _set_id_map(instance=instance, lxc=lxc, project=project, remote=remote, uid=uid) # now restart and wait for the instance to be ready - instance.start() + if ephemeral: + # ephemeral instances can only be restarted + instance.restart() + else: + instance.start() base_configuration.wait_until_ready(executor=instance) diff --git a/craft_providers/lxd/lxc.py b/craft_providers/lxd/lxc.py index ff307778..19ed03d8 100644 --- a/craft_providers/lxd/lxc.py +++ b/craft_providers/lxd/lxc.py @@ -16,16 +16,22 @@ # """LXC wrapper.""" +import contextlib import enum import logging import pathlib import shlex import subprocess +import threading +import time +from collections import deque +from datetime import datetime, timezone from typing import Any, Callable, Dict, List, Optional import yaml from craft_providers import errors +from craft_providers.lxd.lxd_instance_status import ProviderInstanceStatus from .errors import LXDError @@ -58,6 +64,7 @@ def __init__( lxc_path: pathlib.Path = pathlib.Path("lxc"), ) -> None: self.lxc_path = lxc_path + self.lxc_lock = threading.Lock() def _run_lxc( self, @@ -88,11 +95,12 @@ def _run_lxc( logger.debug("Executing on host: %s", shlex.join(lxc_cmd)) - # for subprocess, input takes priority over stdin - if "input" in kwargs: - return subprocess.run(lxc_cmd, check=check, **kwargs) + with self.lxc_lock: + # for subprocess, input takes priority over stdin + if "input" in kwargs: + return subprocess.run(lxc_cmd, check=check, **kwargs) - return subprocess.run(lxc_cmd, check=check, stdin=stdin.value, **kwargs) + return subprocess.run(lxc_cmd, check=check, stdin=stdin.value, **kwargs) def config_device_add_disk( self, @@ -541,7 +549,7 @@ def info( ), ) from error - def launch( + def launch( # noqa: PLR0912 self, *, instance_name: str, @@ -564,6 +572,17 @@ def launch( :raises LXDError: on unexpected error. """ + _default_instance_metadata: Dict[str, str] = { + "user.craft_providers.status": ProviderInstanceStatus.STARTING.value, + "user.craft_providers.timer": datetime.now(timezone.utc).isoformat(), + } + retry_count: int = 0 + if config_keys: + config_keys = config_keys.copy() + config_keys.update(_default_instance_metadata) + else: + config_keys = _default_instance_metadata + command = ["launch", f"{image_remote}:{image}", f"{remote}:{instance_name}"] if ephemeral: @@ -573,18 +592,65 @@ def launch( for config_key in [f"{k}={v}" for k, v in config_keys.items()]: command.extend(["--config", config_key]) - try: - self._run_lxc( - command, - capture_output=True, - stdin=StdinType.INTERACTIVE, - project=project, + # The total times of retrying to launch the same instance per craft-providers. + # If parallel lxc failed, the bad instance will be deleted by the lock holder + # or any other craft-providers, and the lock will be released. + # However, the new instance lock could be held by any craft-providers. + # This is used to avoid lock holder dead and all others are blocked. + while retry_count < 3: + try: + # Try to launch instance + self._run_lxc( + command, + capture_output=True, + stdin=StdinType.INTERACTIVE, + project=project, + ) + except subprocess.CalledProcessError as error: + logger.debug( + "Failed to launch instance %s, retrying %s.", + instance_name, + retry_count, + ) + logger.debug(str(error)) + # Ignore first 3 failed attempts + if retry_count >= 2: + raise LXDError( + brief=f"Failed to launch instance {instance_name!r}.", + details=errors.details_from_called_process_error(error), + ) from error + else: + # Success launching instance, we hold the instance lock + logger.debug("Successfully launched instance %s.", instance_name) + return + + # Maybe race condition, check if the instance is preparing by others. + logger.debug( + "Failed to launch instance %s, checking status.", instance_name ) - except subprocess.CalledProcessError as error: - raise LXDError( - brief=f"Failed to launch instance {instance_name!r}.", - details=errors.details_from_called_process_error(error), - ) from error + try: + self.check_instance_status( + instance_name=instance_name, project=project, remote=remote + ) + except LXDError: + # Something went wrong. Delete the instance. + with contextlib.suppress(LXDError): + # Ignore errors that someone else already deleted the instance + logger.debug("Deleting instance %s due to error.", instance_name) + self.delete( + instance_name=instance_name, + project=project, + remote=remote, + force=True, + ) + + # Sleep for 10 seconds to avoid other delete new instance + time.sleep(10) + else: + # Someone else succeeded creating the instance, just return + return + + retry_count += 1 def image_copy( self, @@ -961,6 +1027,27 @@ def start( details=errors.details_from_called_process_error(error), ) from error + def restart( + self, *, instance_name: str, project: str = "default", remote: str = "local" + ) -> None: + """Restart container. + + :param instance_name: Name of instance to restart. + :param project: Name of LXD project. + :param remote: Name of LXD remote. + + :raises LXDError: on unexpected error. + """ + command = ["restart", f"{remote}:{instance_name}"] + + try: + self._run_lxc(command, capture_output=True, project=project) + except subprocess.CalledProcessError as error: + raise LXDError( + brief=f"Failed to restart {instance_name!r}.", + details=errors.details_from_called_process_error(error), + ) from error + def stop( self, *, @@ -995,3 +1082,76 @@ def stop( brief=f"Failed to stop {instance_name!r}.", details=errors.details_from_called_process_error(error), ) from error + + def check_instance_status( + self, + *, + instance_name: str, + project: str = "default", + remote: str = "local", + ) -> None: + """Check build status of instance. + + The possible status are: + - None: Either the instance is downloading or old that this is not set. + - STARTING: Instance is starting, the creation is successful. If it also STOPPED, + then there could be a boot issue. + - PREPARING: Instance is preparing, the boot is successful. If it also STOPPED, + then the craft-providers or craft-app configuration / installation + was interrupted or failed. + - FINISHED: Instance is ready, all configuration and installation is successful. + When it also STOPPED, then the instance is ready to be copied. + """ + instance_status: Optional[str] = None + instance_info: Dict[str, Any] = {"Status": ""} + start_time = time.time() + + # 20 * 3 seconds = 1 minute no change in timer + timer_queue: deque = deque([-2, -1], maxlen=20) + + # Retry unless the timer queue is all the same + while len(set(timer_queue)) > 1: + try: + # Get instance info + instance_info = self.info( + instance_name=instance_name, project=project, remote=remote + ) + logger.debug("Instance info: %s", instance_info) + + # Get build status + instance_status = self.config_get( + instance_name=instance_name, + key="user.craft_providers.status", + project=project, + remote=remote, + ) + logger.debug("Instance status: %s", instance_status) + + timer = self.config_get( + instance_name=instance_name, + key="user.craft_providers.timer", + project=project, + remote=remote, + ) + timer_queue.append(timer) + logger.debug("Timer: %s", timer) + except LXDError: + # Keep retrying since the instance might not be ready yet + # Max retry time is 10 minutes + if time.time() - start_time > 600: + logger.debug("Instance %s max waiting time reached.", instance_name) + raise + time.sleep(3) + continue + + if ( + instance_status == ProviderInstanceStatus.FINISHED.value + and instance_info["Status"] == "STOPPED" + ): + logger.debug("Instance %s is ready.", instance_name) + return + + time.sleep(3) + + # No timer change for 1 minute and the instance is still not ready. + raise LXDError("Instance setup failed. Check LXD logs for more details.") diff --git a/craft_providers/lxd/lxd_instance.py b/craft_providers/lxd/lxd_instance.py index 83377dfe..b55abe17 100644 --- a/craft_providers/lxd/lxd_instance.py +++ b/craft_providers/lxd/lxd_instance.py @@ -534,6 +534,15 @@ def start(self) -> None: instance_name=self.instance_name, project=self.project, remote=self.remote ) + def restart(self) -> None: + """Restart instance. + + :raises LXDError: on unexpected error. + """ + self.lxc.restart( + instance_name=self.instance_name, project=self.project, remote=self.remote + ) + def stop(self) -> None: """Stop instance. @@ -593,3 +602,37 @@ def unmount_all(self) -> None: project=self.project, remote=self.remote, ) + + def config_get(self, key: str) -> str: + """Get instance configuration value. + + :param key: Configuration key to get. + + :returns: Configuration value. + + :raises LXDError: On unexpected error. + """ + return self.lxc.config_get( + instance_name=self.instance_name, + key=key, + project=self.project, + remote=self.remote, + ) + + def config_set(self, key: str, value: str) -> None: + """Set instance configuration value. + + :param key: Configuration key to set. + :param value: Configuration key to the value. + + :returns: None. + + :raises LXDError: On unexpected error. + """ + self.lxc.config_set( + instance_name=self.instance_name, + key=key, + value=value, + project=self.project, + remote=self.remote, + ) diff --git a/craft_providers/lxd/lxd_instance_status.py b/craft_providers/lxd/lxd_instance_status.py new file mode 100644 index 00000000..369c6e7b --- /dev/null +++ b/craft_providers/lxd/lxd_instance_status.py @@ -0,0 +1,28 @@ +# +# Copyright 2023 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +"""LXD Instance Status.""" + +from enum import Enum + + +class ProviderInstanceStatus(Enum): + """Status of the instance.""" + + PREPARING = "PREPARING" + FINISHED = "FINISHED" + STARTING = "STARTING" diff --git a/tests/integration/lxd/test_launcher.py b/tests/integration/lxd/test_launcher.py index d73cd451..59a1d37f 100644 --- a/tests/integration/lxd/test_launcher.py +++ b/tests/integration/lxd/test_launcher.py @@ -22,6 +22,7 @@ import subprocess import sys from datetime import datetime, timedelta +from threading import Thread import pytest from craft_providers import lxd @@ -267,7 +268,7 @@ def test_launch_use_base_instance( .rstrip("\n") ) - assert instance_hostname == base_configuration._hostname + assert instance_hostname == instance.instance_name def test_launch_create_base_instance_with_correct_image_description( @@ -439,6 +440,63 @@ def test_launch_with_project_and_use_base_instance( base_instance.delete() +def test_launch_with_project_and_use_base_instance_parallel( + base_configuration, get_base_instance, instance_name, project +): + """Launch 5 instances at same time and use the same base instances.""" + base_instance = get_base_instance(project=project) + + instances = [] + + class InstanceThread(Thread): + def __init__(self, instance_name): + super().__init__() + self.instance_name = instance_name + self.instance = None + + def run(self): + self.instance = lxd.launch( + name=self.instance_name, + base_configuration=base_configuration, + image_name="22.04", + image_remote="ubuntu", + use_base_instance=True, + project=project, + remote="local", + ) + + # launch 5 instances from an image and create a base instance + for i in range(0, 5): + instance_thread = InstanceThread(f"{instance_name}-{i}") + instances.append(instance_thread) + + for instance_thread in instances: + instance_thread.start() + + for instance_thread in instances: + instance_thread.join() + + try: + assert base_instance.exists() + assert not base_instance.is_running() + + for instance_thread in instances: + assert instance_thread.instance is not None + assert instance_thread.instance.exists() + assert instance_thread.instance.is_running() + + finally: + for instance_thread in instances: + if ( + instance_thread.instance is not None + and instance_thread.instance.exists() + ): + instance_thread.instance.delete() + + if base_instance.exists(): + base_instance.delete() + + def test_launch_ephemeral(base_configuration, instance_name): """Launch an ephemeral instance and verify it is deleted after it is stopped.""" diff --git a/tests/unit/lxd/test_launcher.py b/tests/unit/lxd/test_launcher.py index 8668085a..085570e8 100644 --- a/tests/unit/lxd/test_launcher.py +++ b/tests/unit/lxd/test_launcher.py @@ -23,6 +23,7 @@ import pytest from craft_providers import Base, ProviderError, bases, lxd +from craft_providers.lxd import LXDError from freezegun import freeze_time from logassert import Exact # type: ignore @@ -50,7 +51,9 @@ def mock_platform(mocker): @pytest.fixture() def mock_timezone(fake_process): fake_process.register_subprocess( - ["timedatectl", "show", "-p", "Timezone", "--value"], stdout="fake/timezone" + ["timedatectl", "show", "-p", "Timezone", "--value"], + stdout="fake/timezone", + occurrences=10, ) @@ -70,7 +73,7 @@ def fake_instance(): @pytest.fixture() -def fake_base_instance(): +def fake_base_instance(fake_process): """Returns a fake base LXD Instance""" base_instance = MagicMock() # the name has an invalid character to ensure the instance_name will be different, @@ -81,6 +84,19 @@ def fake_base_instance(): base_instance.remote = "test-remote" base_instance.exists.return_value = False base_instance.is_running.return_value = False + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "get", + "user.craft_providers.status", + "test-remote:test-base-instance-e14661a426076717fa04", + ], + stdout="fake/timezone", + occurrences=10, + ) return base_instance @@ -113,14 +129,16 @@ def test_launch_no_base_instance( mock_timezone, ): """Create an instance from an image and do not save a copy as the base instance.""" - lxd.launch( - name=fake_instance.name, - base_configuration=mock_base_configuration, - image_name="image-name", - image_remote="image-remote", - use_base_instance=False, - lxc=mock_lxc, - ) + fake_instance.config_get.return_value = "STARTING" + with freeze_time("2023-01-01"): + lxd.launch( + name=fake_instance.name, + base_configuration=mock_base_configuration, + image_name="image-name", + image_remote="image-remote", + use_base_instance=False, + lxc=mock_lxc, + ) assert mock_lxc.mock_calls == [ call.project_list("local"), @@ -143,10 +161,17 @@ def test_launch_no_base_instance( assert fake_instance.mock_calls == [ call.exists(), call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.config_get("user.craft_providers.status"), + call.config_set("user.craft_providers.status", "PREPARING"), + call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), + call.config_set("user.craft_providers.status", "FINISHED"), + call.stop(), + call.start(), ] assert mock_base_configuration.mock_calls == [ call.get_command_environment(), call.setup(executor=fake_instance), + call.wait_until_ready(executor=fake_instance), ] @@ -160,7 +185,7 @@ def test_launch_use_base_instance( mock_platform, mock_timezone, ): - """Launch an instance from an image and save a copy as the base instance.""" + """Launch a base instance from an image and copy to the new instance.""" lxd.launch( name=fake_instance.name, base_configuration=mock_base_configuration, @@ -175,23 +200,23 @@ def test_launch_use_base_instance( assert mock_lxc.mock_calls == [ call.project_list("test-remote"), call.config_set( - instance_name="test-instance-fa2d407652a1c51f6019", - key="environment.TZ", - value="fake/timezone", + instance_name="test-base-instance-e14661a426076717fa04", + key="image.description", + value="test-base-instance-$", project="test-project", remote="test-remote", ), call.copy( source_remote="test-remote", - source_instance_name=fake_instance.instance_name, + source_instance_name=fake_base_instance.instance_name, destination_remote="test-remote", - destination_instance_name=fake_base_instance.instance_name, + destination_instance_name=fake_instance.instance_name, project="test-project", ), call.config_set( - instance_name="test-base-instance-e14661a426076717fa04", - key="image.description", - value="test-base-instance-$", + instance_name="test-instance-fa2d407652a1c51f6019", + key="environment.TZ", + value="fake/timezone", project="test-project", remote="test-remote", ), @@ -212,19 +237,44 @@ def test_launch_use_base_instance( ] assert fake_instance.mock_calls == [ call.exists(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=False), - call.stop(), call.start(), ] fake_base_instance.exists.assert_called_once() assert mock_base_configuration.mock_calls == [ call.get_command_environment(), call.get_command_environment(), - call.setup(executor=fake_instance), + call.setup(executor=fake_base_instance), call.wait_until_ready(executor=fake_instance), ] +def test_launch_use_base_instance_failed_lxc( + fake_instance, + fake_base_instance, + mock_base_configuration, + mock_is_valid, + mock_lxc, + mock_lxd_instance, + mock_platform, + mock_timezone, +): + """Launch a base instance from an image, but lxc commands fail.""" + mock_lxc.config_set.side_effect = [ + LXDError("test1"), + ] + with pytest.raises(LXDError): + lxd.launch( + name=fake_instance.name, + base_configuration=mock_base_configuration, + image_name="image-name", + image_remote="image-remote", + use_base_instance=True, + project="test-project", + remote="test-remote", + lxc=mock_lxc, + ) + + @pytest.mark.parametrize(("map_user_uid", "uid"), [(True, 1234), (False, None)]) def test_launch_use_existing_base_instance( fake_instance, @@ -362,38 +412,40 @@ def test_launch_existing_base_instance_invalid( """If the existing base instance is invalid, delete it and create a new instance.""" fake_base_instance.exists.return_value = True mock_is_valid.return_value = False + fake_base_instance.config_get.return_value = "STARTING" - lxd.launch( - name=fake_instance.name, - base_configuration=mock_base_configuration, - image_name="image-name", - image_remote="image-remote", - use_base_instance=True, - project="test-project", - remote="test-remote", - lxc=mock_lxc, - ) + with freeze_time("2023-01-01"): + lxd.launch( + name=fake_instance.name, + base_configuration=mock_base_configuration, + image_name="image-name", + image_remote="image-remote", + use_base_instance=True, + project="test-project", + remote="test-remote", + lxc=mock_lxc, + ) assert mock_lxc.mock_calls == [ call.project_list("test-remote"), call.config_set( - instance_name="test-instance-fa2d407652a1c51f6019", - key="environment.TZ", - value="fake/timezone", + instance_name="test-base-instance-e14661a426076717fa04", + key="image.description", + value="test-base-instance-$", project="test-project", remote="test-remote", ), call.copy( source_remote="test-remote", - source_instance_name=fake_instance.instance_name, + source_instance_name=fake_base_instance.instance_name, destination_remote="test-remote", - destination_instance_name=fake_base_instance.instance_name, + destination_instance_name=fake_instance.instance_name, project="test-project", ), call.config_set( - instance_name="test-base-instance-e14661a426076717fa04", - key="image.description", - value="test-base-instance-$", + instance_name="test-instance-fa2d407652a1c51f6019", + key="environment.TZ", + value="fake/timezone", project="test-project", remote="test-remote", ), @@ -412,22 +464,29 @@ def test_launch_existing_base_instance_invalid( default_command_environment={"foo": "bar"}, ), ] - assert fake_instance.mock_calls == [ - call.exists(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=False), - call.stop(), - call.start(), - ] + assert fake_instance.mock_calls == [call.exists(), call.start()] assert fake_base_instance.mock_calls == [ call.exists(), call.delete(), call.__bool__(), - call.__bool__(), + call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.config_get("user.craft_providers.status"), + call.config_set("user.craft_providers.status", "PREPARING"), + call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), + call.lxc.config_set( + instance_name="test-base-instance-e14661a426076717fa04", + key="environment.TZ", + value="fake/timezone", + project="test-project", + remote="test-remote", + ), + call.config_set("user.craft_providers.status", "FINISHED"), + call.stop(), ] assert mock_base_configuration.mock_calls == [ call.get_command_environment(), call.get_command_environment(), - call.setup(executor=fake_instance), + call.setup(executor=fake_base_instance), call.wait_until_ready(executor=fake_instance), ] @@ -441,20 +500,22 @@ def test_launch_all_opts( mock_platform, ): """Parse all parameters.""" - lxd.launch( - name=fake_instance.name, - base_configuration=mock_base_configuration, - image_name="image-name", - image_remote="image-remote", - auto_clean=True, - auto_create_project=True, - ephemeral=True, - map_user_uid=True, - uid=1234, - project="test-project", - remote="test-remote", - lxc=mock_lxc, - ) + fake_instance.config_get.return_value = "STARTING" + with freeze_time("2023-01-01"): + lxd.launch( + name=fake_instance.name, + base_configuration=mock_base_configuration, + image_name="image-name", + image_remote="image-remote", + auto_clean=True, + auto_create_project=True, + ephemeral=True, + map_user_uid=True, + uid=1234, + project="test-project", + remote="test-remote", + lxc=mock_lxc, + ) assert mock_lxc.mock_calls == [ call.project_list("test-remote"), @@ -484,8 +545,11 @@ def test_launch_all_opts( assert fake_instance.mock_calls == [ call.exists(), call.launch(image="image-name", image_remote="image-remote", ephemeral=True), - call.stop(), - call.start(), + call.config_get("user.craft_providers.status"), + call.config_set("user.craft_providers.status", "PREPARING"), + call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), + call.config_set("user.craft_providers.status", "FINISHED"), + call.restart(), ] assert mock_base_configuration.mock_calls == [ call.get_command_environment(), @@ -526,16 +590,18 @@ def test_launch_create_project( mock_timezone, ): """Create a project if it does not exist and auto_create_project is true.""" - lxd.launch( - name=fake_instance.name, - base_configuration=mock_base_configuration, - image_name="image-name", - image_remote="image-remote", - auto_create_project=True, - project="project-to-create", - remote="test-remote", - lxc=mock_lxc, - ) + fake_instance.config_get.return_value = "STARTING" + with freeze_time("2023-01-01"): + lxd.launch( + name=fake_instance.name, + base_configuration=mock_base_configuration, + image_name="image-name", + image_remote="image-remote", + auto_create_project=True, + project="project-to-create", + remote="test-remote", + lxc=mock_lxc, + ) assert mock_lxc.mock_calls == [ call.project_list("test-remote"), @@ -566,10 +632,17 @@ def test_launch_create_project( assert fake_instance.mock_calls == [ call.exists(), call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.config_get("user.craft_providers.status"), + call.config_set("user.craft_providers.status", "PREPARING"), + call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), + call.config_set("user.craft_providers.status", "FINISHED"), + call.stop(), + call.start(), ] assert mock_base_configuration.mock_calls == [ call.get_command_environment(), call.setup(executor=fake_instance), + call.wait_until_ready(executor=fake_instance), ] @@ -654,20 +727,22 @@ def test_launch_with_existing_instance_incompatible_with_auto_clean( """If instance is incompatible and auto_clean is true, launch a new instance.""" fake_instance.exists.return_value = True fake_instance.is_running.return_value = False + fake_instance.config_get.return_value = "STARTING" mock_base_configuration.warmup.side_effect = [ bases.BaseCompatibilityError(reason="foo"), None, ] - lxd.launch( - name=fake_instance.name, - base_configuration=mock_base_configuration, - image_name="image-name", - image_remote="image-remote", - auto_clean=True, - lxc=mock_lxc, - ) + with freeze_time("2023-01-01"): + lxd.launch( + name=fake_instance.name, + base_configuration=mock_base_configuration, + image_name="image-name", + image_remote="image-remote", + auto_clean=True, + lxc=mock_lxc, + ) assert mock_lxc.mock_calls == [ call.project_list("local"), @@ -693,11 +768,18 @@ def test_launch_with_existing_instance_incompatible_with_auto_clean( call.start(), call.delete(), call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.config_get("user.craft_providers.status"), + call.config_set("user.craft_providers.status", "PREPARING"), + call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), + call.config_set("user.craft_providers.status", "FINISHED"), + call.stop(), + call.start(), ] assert mock_base_configuration.mock_calls == [ call.get_command_environment(), call.warmup(executor=fake_instance), call.setup(executor=fake_instance), + call.wait_until_ready(executor=fake_instance), ] @@ -741,16 +823,18 @@ def test_launch_with_existing_ephemeral_instance( ): """Delete and recreate existing ephemeral instances.""" fake_instance.exists.return_value = True + fake_instance.config_get.return_value = "STARTING" - lxd.launch( - name=fake_instance.name, - base_configuration=mock_base_configuration, - image_name="image-name", - image_remote="image-remote", - ephemeral=True, - use_base_instance=False, - lxc=mock_lxc, - ) + with freeze_time("2023-01-01"): + lxd.launch( + name=fake_instance.name, + base_configuration=mock_base_configuration, + image_name="image-name", + image_remote="image-remote", + ephemeral=True, + use_base_instance=False, + lxc=mock_lxc, + ) assert mock_lxc.mock_calls == [ call.project_list("local"), @@ -774,10 +858,16 @@ def test_launch_with_existing_ephemeral_instance( call.exists(), call.delete(), call.launch(image="image-name", image_remote="image-remote", ephemeral=True), + call.config_get("user.craft_providers.status"), + call.config_set("user.craft_providers.status", "PREPARING"), + call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), + call.config_set("user.craft_providers.status", "FINISHED"), + call.restart(), ] assert mock_base_configuration.mock_calls == [ call.get_command_environment(), call.setup(executor=fake_instance), + call.wait_until_ready(executor=fake_instance), ] @@ -886,16 +976,19 @@ def test_use_snapshots_deprecated( mock_timezone, ): """Log deprecation warning for `use_snapshots` and continue to launch.""" - lxd.launch( - name=fake_instance.name, - base_configuration=mock_base_configuration, - image_name="image-name", - image_remote="image-remote", - use_snapshots=True, - project="test-project", - remote="test-remote", - lxc=mock_lxc, - ) + fake_base_instance.config_get.return_value = "STARTING" + + with freeze_time("2023-01-01"): + lxd.launch( + name=fake_instance.name, + base_configuration=mock_base_configuration, + image_name="image-name", + image_remote="image-remote", + use_snapshots=True, + project="test-project", + remote="test-remote", + lxc=mock_lxc, + ) assert ( Exact( @@ -908,23 +1001,23 @@ def test_use_snapshots_deprecated( assert mock_lxc.mock_calls == [ call.project_list("test-remote"), call.config_set( - instance_name="test-instance-fa2d407652a1c51f6019", - key="environment.TZ", - value="fake/timezone", + instance_name="test-base-instance-e14661a426076717fa04", + key="image.description", + value="test-base-instance-$", project="test-project", remote="test-remote", ), call.copy( source_remote="test-remote", - source_instance_name=fake_instance.instance_name, + source_instance_name=fake_base_instance.instance_name, destination_remote="test-remote", - destination_instance_name=fake_base_instance.instance_name, + destination_instance_name=fake_instance.instance_name, project="test-project", ), call.config_set( - instance_name="test-base-instance-e14661a426076717fa04", - key="image.description", - value="test-base-instance-$", + instance_name="test-instance-fa2d407652a1c51f6019", + key="environment.TZ", + value="fake/timezone", project="test-project", remote="test-remote", ), @@ -945,19 +1038,29 @@ def test_use_snapshots_deprecated( ] assert fake_instance.mock_calls == [ call.exists(), - call.launch(image="image-name", image_remote="image-remote", ephemeral=False), - call.stop(), call.start(), ] assert fake_base_instance.mock_calls == [ call.exists(), call.__bool__(), - call.__bool__(), + call.launch(image="image-name", image_remote="image-remote", ephemeral=False), + call.config_get("user.craft_providers.status"), + call.config_set("user.craft_providers.status", "PREPARING"), + call.config_set("user.craft_providers.timer", "2023-01-01T00:00:00+00:00"), + call.lxc.config_set( + instance_name="test-base-instance-e14661a426076717fa04", + key="environment.TZ", + value="fake/timezone", + project="test-project", + remote="test-remote", + ), + call.config_set("user.craft_providers.status", "FINISHED"), + call.stop(), ] assert mock_base_configuration.mock_calls == [ call.get_command_environment(), call.get_command_environment(), - call.setup(executor=fake_instance), + call.setup(executor=fake_base_instance), call.wait_until_ready(executor=fake_instance), ] @@ -1204,3 +1307,15 @@ def test_timezone_host_error( "Not setting instance's timezone because host timezone could not " "be determined: \\* Command that failed: 'timedatectl show -p Timezone --value'" ) in logs.debug + + +def test_timer_error_ignore(fake_instance, fake_process, mock_lxc, mocker): + """LXC timer should ignore errors.""" + mocker.patch("time.sleep") + + fake_instance.config_set.side_effect = LXDError("test error") + timer = lxd.launcher.InstanceTimer(fake_instance) + timer.start() + timer.stop() + + assert fake_instance.config_set.call_count > 0 diff --git a/tests/unit/lxd/test_lxc.py b/tests/unit/lxd/test_lxc.py index 03436241..a2b2a977 100644 --- a/tests/unit/lxd/test_lxc.py +++ b/tests/unit/lxd/test_lxc.py @@ -16,10 +16,14 @@ # import pathlib import subprocess +from textwrap import dedent +from unittest.mock import call import pytest from craft_providers import errors from craft_providers.lxd import LXC, LXDError, lxc +from craft_providers.lxd.lxc import StdinType +from freezegun import freeze_time def test_lxc_run_default(mocker, tmp_path): @@ -916,20 +920,154 @@ def test_launch(fake_process): "launch", "test-image-remote:test-image", "test-remote:test-instance", + "--config", + "user.craft_providers.status=STARTING", + "--config", + "user.craft_providers.timer=2023-01-01T00:00:00+00:00", ] ) - LXC().launch( - instance_name="test-instance", - image="test-image", - image_remote="test-image-remote", - project="test-project", - remote="test-remote", + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "set", + "test-remote:test-instance", + "user.craft_providers.status", + "PREPARING", + ] ) + with freeze_time("2023-01-01"): + LXC().launch( + instance_name="test-instance", + image="test-image", + image_remote="test-image-remote", + project="test-project", + remote="test-remote", + ) + assert len(fake_process.calls) == 1 +def test_launch_failed_retry_check(fake_process, mocker): + """Test that we use check_instance_status if launch fails.""" + mock_launch = mocker.patch("craft_providers.lxd.lxc.LXC._run_lxc") + mock_launch.side_effect = [ + subprocess.CalledProcessError(1, ["lxc", "fail", " test"]), + ] + mock_check = mocker.patch("craft_providers.lxd.lxc.LXC.check_instance_status") + mocker.patch("time.sleep") + + with freeze_time("2023-01-01"): + LXC().launch( + instance_name="test-instance", + image="test-image", + image_remote="test-image-remote", + project="test-project", + remote="test-remote", + ) + + assert mock_launch.mock_calls == [ + call( + [ + "launch", + "test-image-remote:test-image", + "test-remote:test-instance", + "--config", + "user.craft_providers.status=STARTING", + "--config", + "user.craft_providers.timer=2023-01-01T00:00:00+00:00", + ], + capture_output=True, + stdin=StdinType.INTERACTIVE, + project="test-project", + ), + ] + + assert mock_check.mock_calls == [ + call( + instance_name="test-instance", project="test-project", remote="test-remote" + ) + ] + + +def test_launch_failed_retry_failed(fake_process, mocker): + """Test that we retry launching an instance if it fails, but failed more than 3 times.""" + mock_launch = mocker.patch("craft_providers.lxd.lxc.LXC._run_lxc") + mock_launch.side_effect = subprocess.CalledProcessError(1, ["lxc", "fail", " test"]) + mock_check = mocker.patch("craft_providers.lxd.lxc.LXC.check_instance_status") + mock_check.side_effect = LXDError("test") + mocker.patch("time.sleep") + + with pytest.raises(LXDError): + with freeze_time("2023-01-01"): + LXC().launch( + instance_name="test-instance", + image="test-image", + image_remote="test-image-remote", + project="test-project", + remote="test-remote", + ) + + assert mock_launch.mock_calls == [ + call( + [ + "launch", + "test-image-remote:test-image", + "test-remote:test-instance", + "--config", + "user.craft_providers.status=STARTING", + "--config", + "user.craft_providers.timer=2023-01-01T00:00:00+00:00", + ], + capture_output=True, + stdin=StdinType.INTERACTIVE, + project="test-project", + ), + call( + ["delete", "test-remote:test-instance", "--force"], + capture_output=True, + project="test-project", + ), + call( + [ + "launch", + "test-image-remote:test-image", + "test-remote:test-instance", + "--config", + "user.craft_providers.status=STARTING", + "--config", + "user.craft_providers.timer=2023-01-01T00:00:00+00:00", + ], + capture_output=True, + stdin=StdinType.INTERACTIVE, + project="test-project", + ), + call( + ["delete", "test-remote:test-instance", "--force"], + capture_output=True, + project="test-project", + ), + call( + [ + "launch", + "test-image-remote:test-image", + "test-remote:test-instance", + "--config", + "user.craft_providers.status=STARTING", + "--config", + "user.craft_providers.timer=2023-01-01T00:00:00+00:00", + ], + capture_output=True, + stdin=StdinType.INTERACTIVE, + project="test-project", + ), + ] + + def test_launch_all_opts(fake_process): fake_process.register_subprocess( [ @@ -944,23 +1082,41 @@ def test_launch_all_opts(fake_process): "test-key=test-value", "--config", "test-key2=test-value2", + "--config", + "user.craft_providers.status=STARTING", + "--config", + "user.craft_providers.timer=2023-01-01T00:00:00+00:00", ] ) - LXC().launch( - instance_name="test-instance", - image="test-image", - image_remote="test-image-remote", - project="test-project", - remote="test-remote", - config_keys={"test-key": "test-value", "test-key2": "test-value2"}, - ephemeral=True, + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "set", + "test-remote:test-instance", + "user.craft_providers.status", + "PREPARING", + ] ) + with freeze_time("2023-01-01"): + LXC().launch( + instance_name="test-instance", + image="test-image", + image_remote="test-image-remote", + project="test-project", + remote="test-remote", + config_keys={"test-key": "test-value", "test-key2": "test-value2"}, + ephemeral=True, + ) + assert len(fake_process.calls) == 1 -def test_launch_error(fake_process): +def test_launch_error(fake_process, mocker): fake_process.register_subprocess( [ "lxc", @@ -969,24 +1125,375 @@ def test_launch_error(fake_process): "launch", "test-image-remote:test-image", "test-remote:test-instance", + "--config", + "user.craft_providers.status=STARTING", + "--config", + "user.craft_providers.timer=2023-01-01T00:00:00+00:00", ], returncode=1, + occurrences=4, + ) + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "delete", + "test-remote:test-instance", + "--force", + ], + returncode=0, + occurrences=4, ) + mocker.patch( + "craft_providers.lxd.lxc.LXC.check_instance_status" + ).side_effect = LXDError("Failed to get instance status.") + + mocker.patch("time.sleep") + with pytest.raises(LXDError) as exc_info: - LXC().launch( + with freeze_time("2023-01-01"): + LXC().launch( + instance_name="test-instance", + image="test-image", + image_remote="test-image-remote", + project="test-project", + remote="test-remote", + ) + + assert exc_info.value == LXDError( + brief="Failed to launch instance 'test-instance'.", + details="* Command that failed: 'lxc --project test-project launch test-image-remote:test-image test-remote:test-instance --config user.craft_providers.status=STARTING --config user.craft_providers.timer=2023-01-01T00:00:00+00:00'\n* Command exit code: 1", + resolution=None, + ) + + +def test_check_instance_status(fake_process): + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "info", + "test-remote:test-instance", + ], + returncode=0, + stdout=dedent( + """\ + Name: test-instance + Status: STOPPED + Type: container + Architecture: x86_64 + Created: 2023/08/02 14:04 EDT + Last Used: 2023/08/08 09:44 EDT + """ + ), + ) + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "get", + "test-remote:test-instance", + "user.craft_providers.status", + ], + returncode=0, + stdout="FINISHED", + ) + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "get", + "test-remote:test-instance", + "user.craft_providers.timer", + ], + returncode=0, + stdout="2023-01-01T00:00:00+00:00", + ) + + LXC().check_instance_status( + instance_name="test-instance", project="test-project", remote="test-remote" + ) + + +def test_check_instance_status_retry(fake_process, mocker): + time_time = mocker.patch("time.time") + time_time.side_effect = [0, 10] + mocker.patch("time.sleep") + mock_instance = mocker.patch("craft_providers.lxd.lxc.LXC.info") + mock_instance.side_effect = [ + {"Status": "STOPPED"}, + {"Status": "STOPPED"}, + ] + mock_instance_config = mocker.patch("craft_providers.lxd.lxc.LXC.config_get") + mock_instance_config.side_effect = [ + "STARTING", + "10", + "FINISHED", + "20", + ] + + LXC().check_instance_status( + instance_name="test-instance", project="test-project", remote="test-remote" + ) + + +def test_check_instance_status_boot_failed(fake_process, mocker): + time_time = mocker.patch("time.time") + time_time.return_value = 0 + mocker.patch("time.sleep") + mock_instance = mocker.patch("craft_providers.lxd.lxc.LXC.info") + mock_instance.return_value = {"Status": "STOPPED"} + mock_instance_config = mocker.patch("craft_providers.lxd.lxc.LXC.config_get") + mock_instance_config.side_effect = [ + "STARTING", + "2023-01-01T00:00:00+00:00", + ] * 20 + + with pytest.raises(LXDError): + LXC().check_instance_status( + instance_name="test-instance", project="test-project", remote="test-remote" + ) + + +def test_check_instance_status_wait(fake_process, mocker): + time_time = mocker.patch("time.time") + time_time.return_value = 0 + mocker.patch("time.sleep") + mock_instance = mocker.patch("craft_providers.lxd.lxc.LXC.info") + mock_instance.side_effect = [ + {"Status": "STOPPED"}, # STARTING + {"Status": "RUNNING"}, # STARTING + {"Status": "RUNNING"}, # PREPARING + {"Status": "RUNNING"}, # PREPARING + {"Status": "RUNNING"}, # FINISHED + {"Status": "STOPPED"}, # FINISHED + ] + mock_instance_config = mocker.patch("craft_providers.lxd.lxc.LXC.config_get") + mock_instance_config.side_effect = [ + "STARTING", + "2023-01-01T00:00:00+00:00", + "STARTING", + "2023-01-01T00:00:05+00:00", + "PREPARING", + "2023-01-01T00:00:10+00:00", + "PREPARING", + "2023-01-01T00:00:15+00:00", + "FINISHED", + "2023-01-01T00:00:20+00:00", + "FINISHED", + "2023-01-01T00:00:30+00:00", + ] + + LXC().check_instance_status( + instance_name="test-instance", project="test-project", remote="test-remote" + ) + assert mock_instance.call_count == 6 + assert mock_instance_config.call_count == 12 + + +def test_check_instance_status_lxd_error(fake_process, mocker): + time_time = mocker.patch("time.time") + time_time.side_effect = [0, 1000, 2000] + mocker.patch("time.sleep") + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "info", + "test-remote:test-instance", + ], + returncode=0, + stdout=dedent( + """\ + Name: test-instance + Status: STOPPED + Type: container + Architecture: x86_64 + Created: 2023/08/02 14:04 EDT + Last Used: 2023/08/08 09:44 EDT + """ + ), + occurrences=1000, + ) + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "get", + "test-remote:test-instance", + "user.craft_providers.status", + ], + returncode=1, + stdout="", + occurrences=1000, + ) + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "get", + "test-remote:test-instance", + "user.craft_providers.timer", + ], + returncode=1, + stdout="", + occurrences=1000, + ) + + with pytest.raises(LXDError) as exc_info: + LXC().check_instance_status( + instance_name="test-instance", project="test-project", remote="test-remote" + ) + + assert exc_info.value == LXDError( + brief="Failed to get value for config key 'user.craft_providers.status' for instance 'test-instance'.", + details="* Command that failed: 'lxc --project test-project config get test-remote:test-instance user.craft_providers.status'\n* Command exit code: 1", + resolution=None, + ) + + +def test_check_instance_status_lxd_error_retry(fake_process, mocker): + time_time = mocker.patch("time.time") + time_time.side_effect = [0, 100, 200, 300, 400, 500, 600, 700, 800, 900] + mocker.patch("time.sleep") + mock_instance = mocker.patch("craft_providers.lxd.lxc.LXC.info") + mock_instance.side_effect = [ + LXDError( + brief="Failed to get instance info.", + details="* Command that failed: 'lxc --project test-project info test-remote:test-instance'\n* Command exit code: 1", + resolution=None, + ), + {"Status": "STOPPED"}, + {"Status": "RUNNING"}, + {"Status": "RUNNING"}, + LXDError( + brief="Failed to get instance info.", + details="* Command that failed: 'lxc --project test-project info test-remote:test-instance'\n* Command exit code: 1", + resolution=None, + ), + {"Status": "RUNNING"}, + {"Status": "RUNNING"}, + {"Status": "RUNNING"}, + {"Status": "STOPPED"}, + ] + mock_instance_config = mocker.patch("craft_providers.lxd.lxc.LXC.config_get") + mock_instance_config.side_effect = [ + LXDError( + brief="Failed to get instance info.", + details="* Command that failed: 'lxc --project test-project info test-remote:test-instance'\n* Command exit code: 1", + resolution=None, + ), + LXDError( + brief="Failed to get instance info.", + details="* Command that failed: 'lxc --project test-project info test-remote:test-instance'\n* Command exit code: 1", + resolution=None, + ), + "STARTING", + "2023-01-01T00:00:00+00:00", + "STARTING", + "2023-01-01T00:00:10+00:00", + "PREPARING", + "2023-01-01T00:00:20+00:00", + LXDError( + brief="Failed to get instance info.", + details="* Command that failed: 'lxc --project test-project info test-remote:test-instance'\n* Command exit code: 1", + resolution=None, + ), + "FINISHED", + "2023-01-01T00:00:40+00:00", + "FINISHED", + "2023-01-01T00:00:50+00:00", + ] + + LXC().check_instance_status( + instance_name="test-instance", project="test-project", remote="test-remote" + ) + + +def test_check_instance_status_error_timeout(fake_process, mocker): + time_time = mocker.patch("time.time") + time_time.side_effect = [0, 1000, 2000] + mocker.patch("time.sleep") + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "info", + "test-remote:test-instance", + ], + returncode=0, + stdout=dedent( + """\ + Name: test-instance + Status: STOPPED + Type: container + Architecture: x86_64 + Created: 2023/08/02 14:04 EDT + Last Used: 2023/08/08 09:44 EDT + """ + ), + occurrences=1000, + ) + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "get", + "test-remote:test-instance", + "user.craft_providers.status", + ], + returncode=0, + stdout="STARTING", + occurrences=1000, + ) + + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "config", + "get", + "test-remote:test-instance", + "user.craft_providers.timer", + ], + returncode=0, + stdout="2023-01-01T00:00:00+00:00", + occurrences=1000, + ) + + with pytest.raises(LXDError) as exc_info: + LXC().check_instance_status( instance_name="test-instance", - image="test-image", - image_remote="test-image-remote", project="test-project", remote="test-remote", ) assert exc_info.value == LXDError( - brief="Failed to launch instance 'test-instance'.", - details=errors.details_from_called_process_error( - exc_info.value.__cause__ # type: ignore - ), + brief="Instance setup failed. Check LXD logs for more details.", ) @@ -1921,6 +2428,53 @@ def test_start_error(fake_process): ) +def test_restart(fake_process): + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "restart", + "test-remote:test-instance", + ], + ) + + LXC().restart( + instance_name="test-instance", + project="test-project", + remote="test-remote", + ) + + assert len(fake_process.calls) == 1 + + +def test_restart_error(fake_process): + fake_process.register_subprocess( + [ + "lxc", + "--project", + "test-project", + "restart", + "test-remote:test-instance", + ], + returncode=1, + ) + + with pytest.raises(LXDError) as exc_info: + LXC().restart( + instance_name="test-instance", + project="test-project", + remote="test-remote", + ) + + assert exc_info.value == LXDError( + brief="Failed to restart 'test-instance'.", + details=errors.details_from_called_process_error( + exc_info.value.__cause__ # type: ignore + ), + ) + + def test_stop(fake_process): fake_process.register_subprocess( [ diff --git a/tests/unit/lxd/test_lxd_instance.py b/tests/unit/lxd/test_lxd_instance.py index 00f8ea18..750e6250 100644 --- a/tests/unit/lxd/test_lxd_instance.py +++ b/tests/unit/lxd/test_lxd_instance.py @@ -24,6 +24,7 @@ import sys import tempfile from unittest import mock +from unittest.mock import call import pytest from craft_providers import errors @@ -108,6 +109,33 @@ def instance(mock_lxc): return LXDInstance(name=_TEST_INSTANCE["name"], lxc=mock_lxc) +def test_config_get(mock_lxc, instance): + instance.config_get(key="test-key") + + assert mock_lxc.mock_calls == [ + call.config_get( + instance_name="test-instance-fa2d407652a1c51f6019", + key="test-key", + project="default", + remote="local", + ) + ] + + +def test_config_set(mock_lxc, instance): + instance.config_set(key="test-key", value="test-value") + + assert mock_lxc.mock_calls == [ + call.config_set( + instance_name="test-instance-fa2d407652a1c51f6019", + key="test-key", + value="test-value", + project="default", + remote="local", + ) + ] + + def test_push_file_io( mock_lxc, mock_named_temporary_file, @@ -783,6 +811,18 @@ def test_start(mock_lxc, instance): ] +def test_restart(mock_lxc, instance): + instance.restart() + + assert mock_lxc.mock_calls == [ + call.restart( + instance_name="test-instance-fa2d407652a1c51f6019", + project="default", + remote="local", + ) + ] + + def test_stop(mock_lxc, instance): instance.stop()