diff --git a/craft_parts/dirs.py b/craft_parts/dirs.py index 329fc5d0d..d4b9b3b0f 100644 --- a/craft_parts/dirs.py +++ b/craft_parts/dirs.py @@ -28,7 +28,7 @@ class ProjectDirs: """ def __init__(self, *, work_dir: Union[Path, str] = "."): - self._work_dir = Path(work_dir).absolute() + self._work_dir = Path(work_dir).expanduser().resolve() @property def work_dir(self) -> Path: diff --git a/craft_parts/executor/executor.py b/craft_parts/executor/executor.py index 914fcef0a..66200c6f5 100644 --- a/craft_parts/executor/executor.py +++ b/craft_parts/executor/executor.py @@ -44,12 +44,14 @@ def __init__( project_info: ProjectInfo, extra_build_packages: List[str] = None, extra_build_snaps: List[str] = None, + ignore_patterns: List[str] = None, ): self._part_list = part_list self._project_info = project_info self._extra_build_packages = extra_build_packages self._extra_build_snaps = extra_build_snaps self._handler: Dict[str, PartHandler] = {} + self._ignore_patterns = ignore_patterns def prologue(self) -> None: """Prepare the execution environment. @@ -139,6 +141,7 @@ def _create_part_handler(self, part: Part) -> PartHandler: part, part_info=PartInfo(self._project_info, part), part_list=self._part_list, + ignore_patterns=self._ignore_patterns, ) self._handler[part.name] = handler diff --git a/craft_parts/executor/part_handler.py b/craft_parts/executor/part_handler.py index 48ebb7a3e..4c09eb554 100644 --- a/craft_parts/executor/part_handler.py +++ b/craft_parts/executor/part_handler.py @@ -55,6 +55,7 @@ def __init__( *, part_info: PartInfo, part_list: List[Part], + ignore_patterns: Optional[List[str]] = None, ): self._part = part self._part_info = part_info @@ -71,6 +72,7 @@ def __init__( cache_dir=part_info.cache_dir, part=part, project_dirs=part_info.dirs, + ignore_patterns=ignore_patterns, ) self.build_packages = _get_build_packages(part=self._part, plugin=self._plugin) diff --git a/craft_parts/infos.py b/craft_parts/infos.py index e1ba419b5..50d983528 100644 --- a/craft_parts/infos.py +++ b/craft_parts/infos.py @@ -61,7 +61,7 @@ def __init__( project_dirs = ProjectDirs() self._application_name = application_name - self._cache_dir = Path(cache_dir).absolute() + self._cache_dir = Path(cache_dir).expanduser().resolve() self._set_machine(arch) self._base = base # TODO: infer base if not specified self._parallel_build_count = parallel_build_count diff --git a/craft_parts/lifecycle_manager.py b/craft_parts/lifecycle_manager.py index ed77fd173..462844a8f 100644 --- a/craft_parts/lifecycle_manager.py +++ b/craft_parts/lifecycle_manager.py @@ -18,7 +18,7 @@ import re from pathlib import Path -from typing import Any, Dict, List, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Union from pydantic import ValidationError @@ -55,6 +55,8 @@ class LifecycleManager: to the system where Craft Parts is being executed. :param parallel_build_count: The maximum number of concurrent jobs to be used to build each part of this project. + :param ignore_local_sources: A list of local source patterns to ignore. + :param extra_build_packages: A list of additional build packages to install. :param custom_args: Any additional arguments that will be passed directly to :ref:`callbacks`. """ @@ -69,7 +71,8 @@ def __init__( arch: str = "", base: str = "", parallel_build_count: int = 1, - extra_build_packages: List[str] = None, + ignore_local_sources: Optional[List[str]] = None, + extra_build_packages: Optional[List[str]] = None, **custom_args, # custom passthrough args ): if not re.match("^[A-Za-z][0-9A-Za-z_]*$", application_name): @@ -105,10 +108,12 @@ def __init__( self._sequencer = sequencer.Sequencer( part_list=self._part_list, project_info=project_info, + ignore_outdated=ignore_local_sources, ) self._executor = executor.Executor( part_list=self._part_list, project_info=project_info, + ignore_patterns=ignore_local_sources, extra_build_packages=extra_build_packages, ) self._project_info = project_info diff --git a/craft_parts/sequencer.py b/craft_parts/sequencer.py index dfb6f1051..33dbd9b95 100644 --- a/craft_parts/sequencer.py +++ b/craft_parts/sequencer.py @@ -34,12 +34,24 @@ class Sequencer: :param part_list: The list of parts to process. :param project_info: Information about this project. + :param ignore_outdated: A list of file patterns to ignore when testing for + outdated files. """ - def __init__(self, *, part_list: List[Part], project_info: ProjectInfo): + def __init__( + self, + *, + part_list: List[Part], + project_info: ProjectInfo, + ignore_outdated: Optional[List[str]] = None, + ): self._part_list = sort_parts(part_list) self._project_info = project_info - self._sm = StateManager(project_info=project_info, part_list=part_list) + self._sm = StateManager( + project_info=project_info, + part_list=part_list, + ignore_outdated=ignore_outdated, + ) self._actions: List[Action] = [] def plan(self, target_step: Step, part_names: Sequence[str] = None) -> List[Action]: diff --git a/craft_parts/sources/base.py b/craft_parts/sources/base.py index 26ed042d7..7e66ea864 100644 --- a/craft_parts/sources/base.py +++ b/craft_parts/sources/base.py @@ -56,10 +56,14 @@ def __init__( source_checksum: Optional[str] = None, command: Optional[str] = None, project_dirs: Optional[ProjectDirs] = None, + ignore_patterns: Optional[List[str]] = None, ): if not project_dirs: project_dirs = ProjectDirs() + if not ignore_patterns: + ignore_patterns = [] + self.source = str(source) self.part_src_dir = str(part_src_dir) self._cache_dir = cache_dir @@ -72,6 +76,7 @@ def __init__( self.command = command self._dirs = project_dirs self._checked = False + self._ignore_patterns = ignore_patterns # pylint: enable=too-many-arguments @@ -126,6 +131,7 @@ def __init__( source_checksum: Optional[str] = None, command: Optional[str] = None, project_dirs: Optional[ProjectDirs] = None, + ignore_patterns: Optional[List[str]] = None, ): super().__init__( source, @@ -138,6 +144,7 @@ def __init__( source_checksum=source_checksum, command=command, project_dirs=project_dirs, + ignore_patterns=ignore_patterns, ) self._file = Path() diff --git a/craft_parts/sources/local_source.py b/craft_parts/sources/local_source.py index 57fe92738..6230f2d5a 100644 --- a/craft_parts/sources/local_source.py +++ b/craft_parts/sources/local_source.py @@ -16,15 +16,20 @@ """The local source handler and helpers.""" +import contextlib import functools import glob +import logging import os +from pathlib import Path from typing import List, Optional from craft_parts.utils import file_utils from .base import SourceHandler +logger = logging.getLogger(__name__) + # TODO: change file operations to use pathlib @@ -36,14 +41,22 @@ def __init__(self, *args, copy_function=file_utils.link_or_copy, **kwargs): self.source_abspath = os.path.abspath(self.source) self.copy_function = copy_function - ignore_patterns = [ - self._dirs.parts_dir.name, - self._dirs.stage_dir.name, - self._dirs.prime_dir.name, - "*.snap", # FIXME: this should be specified by the application - ] + if self._dirs.work_dir.resolve() == Path(self.source_abspath): + # ignore parts/stage/dir if source dir matches workdir + self._ignore_patterns.append(self._dirs.parts_dir.name) + self._ignore_patterns.append(self._dirs.stage_dir.name) + self._ignore_patterns.append(self._dirs.prime_dir.name) + else: + # otherwise check if work_dir inside source dir + with contextlib.suppress(ValueError): + rel_work_dir = self._dirs.work_dir.relative_to(self.source_abspath) + # deep workdirs will be cut at the first component + self._ignore_patterns.append(rel_work_dir.parts[0]) + + logger.debug("ignore patterns: %r", self._ignore_patterns) + self._ignore = functools.partial( - _ignore, self.source_abspath, os.getcwd(), ignore_patterns + _ignore, self.source_abspath, os.getcwd(), self._ignore_patterns ) self._updated_files = set() self._updated_directories = set() @@ -67,6 +80,11 @@ def check_if_outdated( :return: Whether the sources are outdated. """ + if not ignore_files: + ignore_files = [] + + ignore_files.extend(self._ignore_patterns) + try: target_mtime = os.lstat(target).st_mtime except FileNotFoundError: diff --git a/craft_parts/sources/sources.py b/craft_parts/sources/sources.py index 2542818f0..478629612 100644 --- a/craft_parts/sources/sources.py +++ b/craft_parts/sources/sources.py @@ -74,7 +74,7 @@ import os import re from pathlib import Path -from typing import TYPE_CHECKING, Dict, Optional, Type +from typing import TYPE_CHECKING, Dict, List, Optional, Type from craft_parts.dirs import ProjectDirs @@ -101,6 +101,7 @@ def get_source_handler( cache_dir: Path, part: "Part", project_dirs: ProjectDirs, + ignore_patterns: Optional[List[str]] = None, ) -> Optional[SourceHandler]: """Return the appropriate handler for the given source. @@ -124,6 +125,7 @@ def get_source_handler( source_depth=part.spec.source_depth, source_commit=part.spec.source_commit, project_dirs=project_dirs, + ignore_patterns=ignore_patterns, ) return source_handler diff --git a/craft_parts/sources/tar_source.py b/craft_parts/sources/tar_source.py index cd286c910..ec163ef74 100644 --- a/craft_parts/sources/tar_source.py +++ b/craft_parts/sources/tar_source.py @@ -22,7 +22,7 @@ import tarfile import tempfile from pathlib import Path -from typing import Optional +from typing import List, Optional from craft_parts.dirs import ProjectDirs @@ -46,6 +46,7 @@ def __init__( source_depth: Optional[int] = None, source_checksum: Optional[str] = None, project_dirs: Optional[ProjectDirs] = None, + ignore_patterns: Optional[List[str]] = None, ): super().__init__( source, @@ -57,6 +58,7 @@ def __init__( source_depth=source_depth, source_checksum=source_checksum, project_dirs=project_dirs, + ignore_patterns=ignore_patterns, ) if source_tag: raise errors.InvalidSourceOption(source_type="tar", option="source-tag") diff --git a/craft_parts/state_manager/state_manager.py b/craft_parts/state_manager/state_manager.py index 8978a14e0..50097b81a 100644 --- a/craft_parts/state_manager/state_manager.py +++ b/craft_parts/state_manager/state_manager.py @@ -167,12 +167,21 @@ class StateManager: :param project_info: The project information. :param part_list: A list of this project's parts. + :param ignore_outdated: A list of file patterns to ignore when testing for + outdated files. """ - def __init__(self, *, project_info: ProjectInfo, part_list: List[Part]): + def __init__( + self, + *, + project_info: ProjectInfo, + part_list: List[Part], + ignore_outdated: Optional[List[str]] = None + ): self._state_db = _StateDB() self._project_info = project_info self._part_list = part_list + self._ignore_outdated = ignore_outdated self._source_handler_cache: Dict[str, Optional[SourceHandler]] = {} part_step_list = _sort_steps_by_state_timestamp(part_list) @@ -277,6 +286,7 @@ def check_if_outdated(self, part: Part, step: Step) -> Optional[OutdatedReport]: cache_dir=self._project_info.cache_dir, part=part, project_dirs=self._project_info.dirs, + ignore_patterns=self._ignore_outdated, ) self._source_handler_cache[part.name] = source_handler diff --git a/pyproject.toml b/pyproject.toml index c9b9ee7c7..35637b98f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ min-similarity-lines=13 max-line-length = "88" max-attributes = 15 max-args= 6 -max-locals = 16 +max-locals = 18 [tool.pylint.MASTER] extension-pkg-whitelist = [ diff --git a/tests/unit/sources/test_local_source.py b/tests/unit/sources/test_local_source.py index 8c91271e7..bbf2515a9 100644 --- a/tests/unit/sources/test_local_source.py +++ b/tests/unit/sources/test_local_source.py @@ -20,6 +20,7 @@ import pytest from craft_parts import errors +from craft_parts.dirs import ProjectDirs from craft_parts.sources import sources from craft_parts.sources.local_source import LocalSource @@ -100,44 +101,126 @@ def test_pulling_twice_with_existing_source_dir_recreates_hardlinks(self, new_di def test_pull_ignores_own_work_data(self, new_dir): # Make the snapcraft-specific directories + os.makedirs("parts/foo/src") + os.makedirs("stage") + os.makedirs("prime") + os.makedirs("other") + + # Create an application-specific file + open("foo.znap", "w").close() + + # Now make some real files + os.makedirs("dir") + open(os.path.join("dir", "file"), "w").close() + + local = LocalSource( + ".", "parts/foo/src", cache_dir=new_dir, ignore_patterns=["*.znap"] + ) + local.pull() + + # Verify that the work directories got filtered out + assert os.path.isdir(os.path.join("parts", "foo", "src", "parts")) is False + assert os.path.isdir(os.path.join("parts", "foo", "src", "stage")) is False + assert os.path.isdir(os.path.join("parts", "foo", "src", "prime")) is False + assert os.path.isdir(os.path.join("parts", "foo", "src", "other")) + assert os.path.isfile(os.path.join("parts", "foo", "src", "foo.znap")) is False + + # Verify that the real stuff made it in. + assert os.path.isdir(os.path.join("parts", "foo", "src", "dir")) + assert os.stat(os.path.join("parts", "foo", "src", "dir", "file")).st_nlink > 1 + + def test_pull_ignores_own_work_data_work_dir(self, new_dir): + # Make the snapcraft-specific directories + os.makedirs(os.path.join("src", "work_dir")) os.makedirs(os.path.join("src", "parts")) os.makedirs(os.path.join("src", "stage")) os.makedirs(os.path.join("src", "prime")) - os.makedirs(os.path.join("src", ".snapcraft")) - os.makedirs(os.path.join("src", "snap")) + os.makedirs(os.path.join("src", "other")) + open(os.path.join("src", "foo.znap"), "w").close() - # Make the snapcraft.yaml (and hidden one) and a built snap - open(os.path.join("src", "snapcraft.yaml"), "w").close() - open(os.path.join("src", ".snapcraft.yaml"), "w").close() - open(os.path.join("src", "foo.snap"), "w").close() + os.mkdir("destination") - # Make the global state cache - open(os.path.join("src", ".snapcraft", "state"), "w").close() + dirs = ProjectDirs(work_dir="src/work_dir") + local = LocalSource( + "src", + "destination", + cache_dir=new_dir, + project_dirs=dirs, + ignore_patterns=["*.znap"], + ) + local.pull() - # Now make some real files - os.makedirs(os.path.join("src", "dir")) - open(os.path.join("src", "dir", "file"), "w").close() + # Verify that the work directories got filtered out + assert os.path.isdir(os.path.join("destination", "work_dir")) is False + assert os.path.isdir(os.path.join("destination", "foo.znap")) is False + assert os.path.isdir(os.path.join("destination", "other")) + + # These are now allowed since we have set work_dir + assert os.path.isdir(os.path.join("destination", "parts")) + assert os.path.isdir(os.path.join("destination", "stage")) + assert os.path.isdir(os.path.join("destination", "prime")) + + def test_pull_ignores_own_work_data_deep_work_dir(self, new_dir): + # Make the snapcraft-specific directories + os.makedirs(os.path.join("src", "some/deep/work_dir")) + os.makedirs(os.path.join("src", "parts")) + os.makedirs(os.path.join("src", "stage")) + os.makedirs(os.path.join("src", "prime")) + os.makedirs(os.path.join("src", "other")) + os.makedirs(os.path.join("src", "work_dir")) + open(os.path.join("src", "foo.znap"), "w").close() os.mkdir("destination") - local = LocalSource("src", "destination", cache_dir=new_dir) + dirs = ProjectDirs(work_dir="src/some/deep/work_dir") + local = LocalSource( + "src", + "destination", + cache_dir=new_dir, + project_dirs=dirs, + ignore_patterns=["*.znap"], + ) local.pull() - # Verify that the snapcraft-specific stuff got filtered out - assert os.path.isdir(os.path.join("destination", "parts")) is False - assert os.path.isdir(os.path.join("destination", "stage")) is False - assert os.path.isdir(os.path.join("destination", "prime")) is False + # Verify that the work directories got filtered out + assert os.path.isdir(os.path.join("destination", "some/deep/work_dir")) is False + assert os.path.isdir(os.path.join("destination", "foo.znap")) is False + assert os.path.isdir(os.path.join("destination", "other")) - assert os.path.isdir(os.path.join("destination", "snap")) - assert os.path.isfile(os.path.join("destination", ".snapcraft.yaml")) - assert os.path.isfile(os.path.join("destination", "snapcraft.yaml")) + # These are now allowed since we have set work_dir + assert os.path.isdir(os.path.join("destination", "parts")) + assert os.path.isdir(os.path.join("destination", "stage")) + assert os.path.isdir(os.path.join("destination", "prime")) - assert os.path.isfile(os.path.join("destination", "foo.snap")) is False + # This has the same name but it's not the real work dir + assert os.path.isdir(os.path.join("destination", "work_dir")) - # Verify that the real stuff made it in. - assert os.path.islink("destination") is False - assert os.path.islink(os.path.join("destination", "dir")) is False - assert os.stat(os.path.join("destination", "dir", "file")).st_nlink > 1 + def test_pull_work_dir_outside(self, new_dir): + # Make the snapcraft-specific directories + os.makedirs(os.path.join("src", "work_dir")) + os.makedirs(os.path.join("src", "parts")) + os.makedirs(os.path.join("src", "stage")) + os.makedirs(os.path.join("src", "prime")) + os.makedirs(os.path.join("src", "other")) + + os.mkdir("destination") + + dirs = ProjectDirs(work_dir="/work_dir") + local = LocalSource( + "src", + "destination", + cache_dir=new_dir, + project_dirs=dirs, + ignore_patterns=["*.znap"], + ) + local.pull() + + # These are all allowed since work_dir is located outside + assert os.path.isdir(os.path.join("destination", "work_dir")) + assert os.path.isdir(os.path.join("destination", "other")) + assert os.path.isdir(os.path.join("destination", "parts")) + assert os.path.isdir(os.path.join("destination", "stage")) + assert os.path.isdir(os.path.join("destination", "prime")) def test_pull_keeps_symlinks(self, new_dir): # Create a source containing a directory, a file and symlinks to both. @@ -164,23 +247,32 @@ def test_has_source_handler_entry(self): class TestLocalUpdate: """Verify that the local source can detect changes and update.""" - def test_file_modified(self, new_dir): + @pytest.mark.parametrize( + "name,ignored", + [ + ("file", False), + ("file.ignore", True), + ], + ) + def test_file_modified(self, new_dir, name, ignored): source = "source" destination = "destination" os.mkdir(source) os.mkdir(destination) - with open(os.path.join(source, "file"), "w") as f: + with open(os.path.join(source, name), "w") as f: f.write("1") # Now make a reference file with a timestamp later than the file was # created. We'll ensure this by setting it ourselves - shutil.copy2(os.path.join(source, "file"), "reference") + shutil.copy2(os.path.join(source, name), "reference") access_time = os.stat("reference").st_atime modify_time = os.stat("reference").st_mtime os.utime("reference", (access_time, modify_time + 1)) - local = LocalSource(source, destination, cache_dir=new_dir) + local = LocalSource( + source, destination, cache_dir=new_dir, ignore_patterns=["*.ignore"] + ) local.pull() # Update check on non-existent files should return False @@ -189,25 +281,31 @@ def test_file_modified(self, new_dir): # Expect no updates to be available assert local.check_if_outdated("reference") is False - with open(os.path.join(destination, "file")) as f: - assert f.read() == "1" + if ignored: + assert os.path.exists(os.path.join(destination, name)) is False + else: + with open(os.path.join(destination, name)) as f: + assert f.read() == "1" # Now update the file in source, and make sure it has a timestamp # later than our reference (this whole test happens too fast) - with open(os.path.join(source, "file"), "w") as f: + with open(os.path.join(source, name), "w") as f: f.write("2") access_time = os.stat("reference").st_atime modify_time = os.stat("reference").st_mtime - os.utime(os.path.join(source, "file"), (access_time, modify_time + 1)) + os.utime(os.path.join(source, name), (access_time, modify_time + 1)) # Expect update to be available - assert local.check_if_outdated("reference") + assert local.check_if_outdated("reference") is not ignored local.update() - with open(os.path.join(destination, "file")) as f: - assert f.read() == "2" + if ignored: + assert os.path.exists(os.path.join(destination, name)) is False + else: + with open(os.path.join(destination, name)) as f: + assert f.read() == "2" def test_file_added(self, new_dir): source = "source" diff --git a/tests/unit/state_manager/test_state_manager.py b/tests/unit/state_manager/test_state_manager.py index 71164cf67..1cd9c1aed 100644 --- a/tests/unit/state_manager/test_state_manager.py +++ b/tests/unit/state_manager/test_state_manager.py @@ -412,6 +412,23 @@ def test_source_outdated(self, new_dir): for step in list(Step): assert sm.check_if_outdated(p1, step) is None + def test_source_outdated_ignored(self, new_dir): + info = ProjectInfo(application_name="test", cache_dir=new_dir) + p1 = Part("p1", {"source": "subdir"}) # source is local + + # p1 pull ran + s1 = states.StageState() + s1.write(Path("parts/p1/state/pull")) + + Path("subdir").mkdir() + os_utils.TimedWriter.write_text(Path("subdir/foo"), "content") + + sm = StateManager(project_info=info, part_list=[p1], ignore_outdated=["foo*"]) + + for step in list(Step): + report = sm.check_if_outdated(p1, step) + assert report is None + class TestStepDirty: """Verify dirty step checks.""" diff --git a/tests/unit/test_dirs.py b/tests/unit/test_dirs.py index 518ac7ec3..2b0ff2bc2 100644 --- a/tests/unit/test_dirs.py +++ b/tests/unit/test_dirs.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +from pathlib import Path + from craft_parts.dirs import ProjectDirs @@ -31,3 +33,8 @@ def test_dirs_work_dir(new_dir): assert dirs.parts_dir == new_dir / "foobar/parts" assert dirs.stage_dir == new_dir / "foobar/stage" assert dirs.prime_dir == new_dir / "foobar/prime" + + +def test_dirs_work_dir_resolving(): + dirs = ProjectDirs(work_dir="~/x/../y/.") + assert dirs.work_dir == Path.home() / "y" diff --git a/tests/unit/test_infos.py b/tests/unit/test_infos.py index aa42de876..0affe50df 100644 --- a/tests/unit/test_infos.py +++ b/tests/unit/test_infos.py @@ -119,6 +119,11 @@ def test_project_info_default(): assert info.parallel_build_count == 1 +def test_project_info_cache_dir_resolving(): + info = ProjectInfo(application_name="test", cache_dir=Path("~/x/../y/.")) + assert info.cache_dir == Path.home() / "y" + + def test_invalid_arch(): with pytest.raises(errors.InvalidArchitecture) as raised: ProjectInfo(application_name="test", cache_dir=Path(), arch="invalid") diff --git a/tests/unit/test_lifecycle_manager.py b/tests/unit/test_lifecycle_manager.py index 79ac17f4d..86dd88057 100644 --- a/tests/unit/test_lifecycle_manager.py +++ b/tests/unit/test_lifecycle_manager.py @@ -93,6 +93,7 @@ def test_part_initialization(self, new_dir, mocker): self._data, application_name="test_manager", cache_dir=new_dir, + ignore_local_sources=["foo.*"], ) assert len(lf._part_list) == 1 @@ -105,6 +106,7 @@ def test_part_initialization(self, new_dir, mocker): mock_seq.assert_called_once_with( part_list=lf._part_list, project_info=lf.project_info, + ignore_outdated=["foo.*"], )