states: make stage package tracking optional (#343)

Stage package tracking is used Snapcraft to add a list of primed stage
packages to the manifest file when `--enable-manifest` is used. This is
implemented using extended file attributes in Linux.

Making stage package tracking optional brings in a number of advantages:
it allows allows other applications such as Rockcraft and Charmcraft to
run faster and consume less resources, and also allows work dirs on tmpfs
when package tracking is disabled (e.g. when configuring the default lxd
profile for application projects to mount `/tmp` and `/root` as tmpfs).

Signed-off-by: Claudio Matsuoka <[email protected]>

craft_parts/executor/executor.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright 2021 Canonical Ltd.
+# Copyright 2021-2023 Canonical Ltd.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -48,6 +48,7 @@ class Executor:
 
     :param part_list: The list of parts to process.
     :param project_info: Information about this project.
+    :param track_stage_packages: Add primed stage packages to the prime state.
     :param extra_build_packages: Additional packages to install on the host system.
     :param extra_build_snaps: Additional snaps to install on the host system.
     :param ignore_patterns: File patterns to ignore when pulling local sources.
@@ -60,6 +61,7 @@ def __init__(
         project_info: ProjectInfo,
         extra_build_packages: Optional[List[str]] = None,
         extra_build_snaps: Optional[List[str]] = None,
+        track_stage_packages: bool = False,
         ignore_patterns: Optional[List[str]] = None,
         base_layer_dir: Optional[Path] = None,
         base_layer_hash: Optional[LayerHash] = None,
@@ -68,6 +70,7 @@ def __init__(
         self._project_info = project_info
         self._extra_build_packages = extra_build_packages
         self._extra_build_snaps = extra_build_snaps
+        self._track_stage_packages = track_stage_packages
         self._base_layer_hash = base_layer_hash
         self._handler: Dict[str, PartHandler] = {}
         self._ignore_patterns = ignore_patterns
@@ -201,6 +204,7 @@ def _create_part_handler(
             part,
             part_info=PartInfo(self._project_info, part),
             part_list=self._part_list,
+            track_stage_packages=self._track_stage_packages,
             overlay_manager=self._overlay_manager,
             ignore_patterns=self._ignore_patterns,
             base_layer_hash=self._base_layer_hash,

craft_parts/executor/part_handler.py

@@ -86,13 +86,15 @@ def __init__(
         *,
         part_info: PartInfo,
         part_list: List[Part],
+        track_stage_packages: bool = False,
         overlay_manager: OverlayManager,
         ignore_patterns: Optional[List[str]] = None,
         base_layer_hash: Optional[LayerHash] = None,
     ):
         self._part = part
         self._part_info = part_info
         self._part_list = part_list
+        self._track_stage_packages = track_stage_packages
         self._overlay_manager = overlay_manager
         self._base_layer_hash = base_layer_hash
         self._app_environment: Dict[str, str] = {}
@@ -410,7 +412,11 @@ def _run_prime(
 
         self._migrate_overlay_files_to_prime()
 
-        if self._part.spec.stage_packages and is_deb_based():
+        if (
+            self._part.spec.stage_packages
+            and self._track_stage_packages
+            and is_deb_based()
+        ):
             primed_stage_packages = _get_primed_stage_packages(
                 contents.files, prime_dir=self._part.prime_dir
             )
@@ -938,6 +944,7 @@ def _unpack_stage_packages(self) -> None:
             stage_packages_path=self._part.part_packages_dir,
             install_path=Path(self._part.part_install_dir),
             stage_packages=pulled_packages,
+            track_stage_packages=self._track_stage_packages,
         )
 
     def _unpack_stage_snaps(self) -> None:

craft_parts/lifecycle_manager.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright 2021 Canonical Ltd.
+# Copyright 2021-2023 Canonical Ltd.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -64,6 +64,7 @@ class LifecycleManager:
     :param ignore_local_sources: A list of local source patterns to ignore.
     :param extra_build_packages: A list of additional build packages to install.
     :param extra_build_snaps: A list of additional build snaps to install.
+    :param track_stage_packages: Add primed stage packages to the prime state.
     :param base_layer_dir: The path to the overlay base layer, if using overlays.
     :param base_layer_hash: The validation hash of the overlay base image, if using
         overlays. The validation hash should be constant for a given image, and should
@@ -90,6 +91,7 @@ def __init__(
         ignore_local_sources: Optional[List[str]] = None,
         extra_build_packages: Optional[List[str]] = None,
         extra_build_snaps: Optional[List[str]] = None,
+        track_stage_packages: bool = False,
         base_layer_dir: Optional[Path] = None,
         base_layer_hash: Optional[bytes] = None,
         project_vars_part_name: Optional[str] = None,
@@ -168,6 +170,7 @@ def __init__(
             ignore_patterns=ignore_local_sources,
             extra_build_packages=extra_build_packages,
             extra_build_snaps=extra_build_snaps,
+            track_stage_packages=track_stage_packages,
             base_layer_dir=base_layer_dir,
             base_layer_hash=layer_hash,
         )

craft_parts/packages/base.py

@@ -161,6 +161,7 @@ def unpack_stage_packages(
         stage_packages_path: Path,
         install_path: Path,
         stage_packages: Optional[List[str]] = None,
+        track_stage_packages: bool = False,
     ) -> None:
         """Unpack stage packages.
 
@@ -236,6 +237,7 @@ def unpack_stage_packages(
         stage_packages_path: Path,
         install_path: Path,
         stage_packages: Optional[List[str]] = None,
+        track_stage_packages: bool = False,
     ) -> None:
         """Unpack stage packages to install_path."""
 

craft_parts/packages/deb.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright 2015-2021 Canonical Ltd.
+# Copyright 2015-2023 Canonical Ltd.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -651,6 +651,7 @@ def unpack_stage_packages(
         stage_packages_path: pathlib.Path,
         install_path: pathlib.Path,
         stage_packages: Optional[List[str]] = None,
+        track_stage_packages: bool = False,
     ) -> None:
         """Unpack stage packages to install_path."""
         if stage_packages is None:
@@ -661,7 +662,9 @@ def unpack_stage_packages(
             )
         else:
             cls._unpack_stage_debs(
-                stage_packages_path=stage_packages_path, install_path=install_path
+                stage_packages_path=stage_packages_path,
+                install_path=install_path,
+                track_stage_packages=track_stage_packages,
             )
 
     @classmethod
@@ -670,6 +673,7 @@ def _unpack_stage_debs(
         *,
         stage_packages_path: pathlib.Path,
         install_path: pathlib.Path,
+        track_stage_packages: bool,
     ) -> None:
         pkg_path = None
 
@@ -679,9 +683,12 @@ def _unpack_stage_debs(
             ) as extract_dir:
                 # Extract deb package.
                 deb_utils.extract_deb(pkg_path, Path(extract_dir), logger.debug)
+
                 # Mark source of files.
-                marked_name = cls._extract_deb_name_version(pkg_path)
-                mark_origin_stage_package(extract_dir, marked_name)
+                if track_stage_packages:
+                    marked_name = cls._extract_deb_name_version(pkg_path)
+                    mark_origin_stage_package(extract_dir, marked_name)
+
                 # Stage files to install_dir.
                 file_utils.link_or_copy_tree(extract_dir, install_path.as_posix())
 

tests/unit/executor/test_part_handler.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright 2021 Canonical Ltd.
+# Copyright 2021-2023 Canonical Ltd.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -238,14 +238,26 @@ def test_run_stage(self, mocker):
             overlay_hash="d12e3f53ba91f94656abc940abb50b12b209d246",
         )
 
-    def test_run_prime(self, mocker):
+    def test_run_prime(self, new_dir, mocker):
         mocker.patch(
             "craft_parts.executor.step_handler.StepHandler._builtin_prime",
             return_value=StepContents({"file"}, {"dir"}),
         )
         mocker.patch("os.getxattr", new=lambda x, y: b"pkg")
 
-        state = self._handler._run_prime(
+        info = ProjectInfo(application_name="test", cache_dir=new_dir)
+        ovmgr = OverlayManager(
+            project_info=info, part_list=[self._part], base_layer_dir=Path("/base")
+        )
+        handler = PartHandler(
+            self._part,
+            track_stage_packages=True,
+            part_info=self._part_info,
+            part_list=[self._part],
+            overlay_manager=ovmgr,
+        )
+
+        state = handler._run_prime(
             StepInfo(self._part_info, Step.PRIME), stdout=None, stderr=None
         )
         assert state == states.PrimeState(
@@ -256,6 +268,24 @@ def test_run_prime(self, mocker):
             primed_stage_packages={"pkg"},
         )
 
+    def test_run_prime_dont_track_packages(self, mocker):
+        mocker.patch(
+            "craft_parts.executor.step_handler.StepHandler._builtin_prime",
+            return_value=StepContents({"file"}, {"dir"}),
+        )
+        mocker.patch("os.getxattr", new=lambda x, y: b"pkg")
+
+        state = self._handler._run_prime(
+            StepInfo(self._part_info, Step.PRIME), stdout=None, stderr=None
+        )
+        assert state == states.PrimeState(
+            part_properties=self._part.spec.marshal(),
+            project_options=self._part_info.project_options,
+            files={"file"},
+            directories={"dir"},
+            primed_stage_packages=set(),
+        )
+
     @pytest.mark.parametrize(
         "step,scriptlet",
         [

tests/unit/test_lifecycle_manager.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright 2021 Canonical Ltd.
+# Copyright 2021-2023 Canonical Ltd.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -158,6 +158,7 @@ def test_executor_creation(self, new_dir, mocker):
                 ignore_patterns=["ign1", "ign2"],
                 extra_build_packages=["pkg1", "pkg2"],
                 extra_build_snaps=["snap1", "snap2"],
+                track_stage_packages=False,
                 base_layer_dir=None,
                 base_layer_hash=None,
             )