feat(base): cache pip directory (#394)

Co-authored-by: Alex Lowe <[email protected]>
Co-authored-by: Sheng Yu <[email protected]>

craft_providers/base.py

@@ -79,6 +79,10 @@ class Base(ABC):
         extend this tag, not overwrite it, e.g.: compatibility_tag =
         f"{appname}-{Base.compatibility_tag}.{apprevision}" to ensure base
         compatibility levels are maintained.
+
+    :param cache_path: (Optional) Path to the shared cache directory. If this is
+        provided, shared cache directories will be mounted as appropriate. Some
+        directories depend on the base implementation.
     """
 
     _environment: Dict[str, Optional[str]]
@@ -91,6 +95,7 @@ class Base(ABC):
     _timeout_simple: Optional[float] = TIMEOUT_SIMPLE
     _timeout_complex: Optional[float] = TIMEOUT_COMPLEX
     _timeout_unpredictable: Optional[float] = TIMEOUT_UNPREDICTABLE
+    _cache_path: Optional[pathlib.Path] = None
     alias: Enum
     compatibility_tag: str = "base-v2"
 
@@ -105,6 +110,7 @@ def __init__(
         snaps: Optional[List] = None,
         packages: Optional[List[str]] = None,
         use_default_packages: bool = True,
+        cache_path: Optional[pathlib.Path] = None,
     ) -> None:
         pass
 
@@ -786,6 +792,51 @@ def _post_setup_network(self, executor: Executor) -> None:
         """
         return
 
+    def _mount_shared_cache_dirs(self, executor: Executor) -> None:
+        """Mount shared cache directories for this base.
+
+        e.g.
+            pip cache (normally $HOME/.cache/pip)
+
+        This will only be run if caching is enabled for this instance.
+
+        This step should usually be extended, but may be overridden if common
+        cache directories are in unusual locations.
+        """
+        if self._cache_path is None:
+            logger.debug("No cache path set, not mounting cache directories.")
+            return
+
+        # Get the real path with additional tags.
+        host_base_cache_path = self._cache_path.resolve().joinpath(
+            self.compatibility_tag, str(self.alias)
+        )
+
+        try:
+            host_base_cache_path.mkdir(parents=True, exist_ok=True)
+        except OSError as error:
+            raise BaseConfigurationError(
+                brief=f"Failed to create host cache directory: {host_base_cache_path}"
+            ) from error
+
+        guest_cache_proc = executor.execute_run(
+            ["bash", "-c", "echo -n ${XDG_CACHE_HOME:-${HOME}/.cache}"],
+            capture_output=True,
+            text=True,
+        )
+        guest_base_cache_path = pathlib.Path(guest_cache_proc.stdout)
+
+        # PIP cache
+        host_pip_cache_path = host_base_cache_path / "pip"
+        host_pip_cache_path.mkdir(parents=True, exist_ok=True)
+
+        guest_pip_cache_path = guest_base_cache_path / "pip"
+        executor.execute_run(
+            ["mkdir", "-p", guest_pip_cache_path.as_posix()],
+        )
+
+        executor.mount(host_source=host_pip_cache_path, target=guest_pip_cache_path)
+
     def _pre_setup_packages(self, executor: Executor) -> None:
         """Do anything before setting up the packages.
 
@@ -964,6 +1015,8 @@ def setup(
 
         self._update_compatibility_tag(executor=executor)
 
+        self._mount_shared_cache_dirs(executor=executor)
+
         self._pre_setup_os(executor=executor)
         self._setup_os(executor=executor)
         self._post_setup_os(executor=executor)
@@ -1029,6 +1082,8 @@ def warmup(
         self._image_check(executor=executor)
         self._post_image_check(executor=executor)
 
+        self._mount_shared_cache_dirs(executor=executor)
+
         self._setup_wait_for_system_ready(executor=executor)
         self._setup_wait_for_network(executor=executor)
 

craft_providers/bases/__init__.py

@@ -22,8 +22,8 @@
 from typing import Dict, NamedTuple, Tuple, Type, Union
 
 from craft_providers.errors import BaseCompatibilityError, BaseConfigurationError
+from craft_providers.base import Base
 
-from ..base import Base
 from . import almalinux, centos
 from . import ubuntu
 from . import ubuntu as buildd

craft_providers/bases/almalinux.py

@@ -18,6 +18,7 @@
 """Almalinux image(s)."""
 import enum
 import logging
+import pathlib
 import subprocess
 from typing import Dict, List, Optional
 
@@ -63,6 +64,8 @@ class AlmaLinuxBase(Base):
     :param snaps: Optional list of snaps to install on the base image.
     :param packages: Optional list of system packages to install on the base image.
     :param use_default_packages: Optional bool to enable/disable default packages.
+    :param cache_path: (Optional) Path to the shared cache directory. If this is
+        provided, shared cache directories will be mounted as appropriate.
     """
 
     compatibility_tag: str = f"almalinux-{Base.compatibility_tag}"
@@ -77,7 +80,9 @@ def __init__(
         snaps: Optional[List[Snap]] = None,
         packages: Optional[List[str]] = None,
         use_default_packages: bool = True,
+        cache_path: Optional[pathlib.Path] = None,
     ) -> None:
+        self._cache_path = cache_path
         self.alias: AlmaLinuxBaseAlias = alias
 
         if environment is None:

craft_providers/bases/centos.py

@@ -18,6 +18,7 @@
 """CentOS image(s)."""
 import enum
 import logging
+import pathlib
 import subprocess
 from typing import Dict, List, Optional
 
@@ -63,6 +64,8 @@ class CentOSBase(Base):
     :param snaps: Optional list of snaps to install on the base image.
     :param packages: Optional list of system packages to install on the base image.
     :param use_default_packages: Optional bool to enable/disable default packages.
+    :param cache_path: Optional path to the shared cache directory. If this is
+        provided, shared cache directories will be mounted as appropriate.
     """
 
     compatibility_tag: str = f"centos-{Base.compatibility_tag}"
@@ -77,7 +80,9 @@ def __init__(
         snaps: Optional[List[Snap]] = None,
         packages: Optional[List[str]] = None,
         use_default_packages: bool = True,
+        cache_path: Optional[pathlib.Path] = None,
     ) -> None:
+        self._cache_dir = cache_path
         self.alias: CentOSBaseAlias = alias
 
         if environment is None:

craft_providers/bases/ubuntu.py

@@ -71,6 +71,8 @@ class BuilddBase(Base):
     :param snaps: Optional list of snaps to install on the base image.
     :param packages: Optional list of system packages to install on the base image.
     :param use_default_packages: Optional bool to enable/disable default packages.
+    :param cache_path: Optional path to the shared cache directory. If this is
+        provided, shared cache directories will be mounted as appropriate.
     """
 
     compatibility_tag: str = f"buildd-{Base.compatibility_tag}"
@@ -85,8 +87,10 @@ def __init__(
         snaps: Optional[List[Snap]] = None,
         packages: Optional[List[str]] = None,
         use_default_packages: bool = True,
+        cache_path: Optional[pathlib.Path] = None,
     ) -> None:
         self.alias: BuilddBaseAlias = alias
+        self._cache_path = cache_path
 
         if environment is None:
             self._environment = self.default_command_environment()

craft_providers/lxd/launcher.py

@@ -130,6 +130,8 @@ def _create_instance(
             image=image_name,
             image_remote=image_remote,
             ephemeral=False,  # base instance should not ephemeral
+            map_user_uid=map_user_uid,
+            uid=uid,
         )
         base_instance_status = base_instance.config_get("user.craft_providers.status")
 
@@ -178,7 +180,11 @@ def _create_instance(
             image_remote,
         )
         instance.launch(
-            image=image_name, image_remote=image_remote, ephemeral=ephemeral
+            image=image_name,
+            image_remote=image_remote,
+            ephemeral=ephemeral,
+            map_user_uid=map_user_uid,
+            uid=uid,
         )
         instance_status = instance.config_get("user.craft_providers.status")
 

craft_providers/lxd/lxd_provider.py

@@ -20,7 +20,7 @@
 import logging
 import pathlib
 from datetime import timedelta
-from typing import Generator, Optional
+from typing import Iterator, Optional
 
 from craft_providers import Executor, Provider
 from craft_providers.base import Base
@@ -110,7 +110,7 @@ def launched_environment(
         build_base: Optional[str] = None,
         instance_name: str,
         allow_unstable: bool = False,
-    ) -> Generator[Executor, None, None]:
+    ) -> Iterator[Executor]:
         """Configure and launch environment for specified base.
 
         When this method loses context, all directories are unmounted and the

craft_providers/multipass/multipass_provider.py

@@ -21,7 +21,7 @@
 import pathlib
 from dataclasses import dataclass
 from enum import Enum
-from typing import Dict, Generator, Optional
+from typing import Dict, Iterator, Optional
 
 from craft_providers import Base, Executor, Provider, base
 from craft_providers.bases import ubuntu
@@ -187,7 +187,7 @@ def launched_environment(
         build_base: Optional[str] = None,
         instance_name: str,
         allow_unstable: bool = False,
-    ) -> Generator[Executor, None, None]:
+    ) -> Iterator[Executor]:
         """Configure and launch environment for specified base.
 
         When this method loses context, all directories are unmounted and the

tests/integration/lxd/conftest.py

@@ -28,6 +28,7 @@
 from craft_providers.lxd import LXC
 from craft_providers.lxd import project as lxc_project
 from craft_providers.lxd.lxd_instance import LXDInstance
+from craft_providers.lxd.lxd_provider import LXDProvider
 
 
 @pytest.fixture(autouse=True, scope="module")
@@ -143,3 +144,30 @@ def project(lxc, project_name):
     yield project_name
 
     lxc_project.purge(lxc=lxc, project=project_name)
+
+
+@pytest.fixture(scope="session")
+def session_project(installed_lxd):
+    lxc = LXC()
+    project_name = "craft-providers-test-session"
+    lxc_project.create_with_default_profile(lxc=lxc, project=project_name)
+
+    projects = lxc.project_list()
+    assert project_name in projects
+
+    instances = lxc.list(project=project_name)
+    assert instances == []
+
+    expected_cfg = lxc.profile_show(profile="default", project="default")
+    expected_cfg["used_by"] = []
+
+    assert lxc.profile_show(profile="default", project=project_name) == expected_cfg
+
+    yield project_name
+
+    lxc_project.purge(lxc=lxc, project=project_name)
+
+
+@pytest.fixture(scope="session")
+def session_provider(session_project):
+    return LXDProvider(lxd_project=session_project)

tests/integration/lxd/test_launcher.py

@@ -380,10 +380,10 @@ def test_launch_create_project(base_configuration, instance_name, project_name):
 
 
 def test_launch_with_project_and_use_base_instance(
-    base_configuration, get_base_instance, instance_name, project
+    base_configuration, get_base_instance, instance_name, session_project
 ):
     """With a LXD project specified, launch an instance and use base instances."""
-    base_instance = get_base_instance(project=project)
+    base_instance = get_base_instance(project=session_project)
 
     # launch an instance from an image and create a base instance
     instance = lxd.launch(
@@ -392,7 +392,7 @@ def test_launch_with_project_and_use_base_instance(
         image_name="22.04",
         image_remote="ubuntu",
         use_base_instance=True,
-        project=project,
+        project=session_project,
         remote="local",
     )
 
@@ -413,7 +413,7 @@ def test_launch_with_project_and_use_base_instance(
             image_name="22.04",
             image_remote="ubuntu",
             use_base_instance=True,
-            project=project,
+            project=session_project,
             remote="local",
         )
 
@@ -427,7 +427,7 @@ def test_launch_with_project_and_use_base_instance(
             image_name="22.04",
             image_remote="ubuntu",
             use_base_instance=True,
-            project=project,
+            project=session_project,
             remote="local",
         )
 
@@ -441,10 +441,10 @@ def test_launch_with_project_and_use_base_instance(
 
 
 def test_launch_with_project_and_use_base_instance_parallel(
-    base_configuration, get_base_instance, instance_name, project
+    base_configuration, get_base_instance, instance_name, session_project
 ):
     """Launch 5 instances at same time and use the same base instances."""
-    base_instance = get_base_instance(project=project)
+    base_instance = get_base_instance(project=session_project)
 
     instances = []
 
@@ -461,7 +461,7 @@ def run(self):
                 image_name="22.04",
                 image_remote="ubuntu",
                 use_base_instance=True,
-                project=project,
+                project=session_project,
                 remote="local",
             )