Merge pull request #96 from cmatsuoka/fix/local-source-filters

many: fix source ignore patterns (CRAFT-375)

craft_parts/dirs.py

@@ -28,7 +28,7 @@ class ProjectDirs:
     """
 
     def __init__(self, *, work_dir: Union[Path, str] = "."):
-        self._work_dir = Path(work_dir).absolute()
+        self._work_dir = Path(work_dir).expanduser().resolve()
 
     @property
     def work_dir(self) -> Path:

craft_parts/executor/executor.py

@@ -44,12 +44,14 @@ def __init__(
         project_info: ProjectInfo,
         extra_build_packages: List[str] = None,
         extra_build_snaps: List[str] = None,
+        ignore_patterns: List[str] = None,
     ):
         self._part_list = part_list
         self._project_info = project_info
         self._extra_build_packages = extra_build_packages
         self._extra_build_snaps = extra_build_snaps
         self._handler: Dict[str, PartHandler] = {}
+        self._ignore_patterns = ignore_patterns
 
     def prologue(self) -> None:
         """Prepare the execution environment.
@@ -139,6 +141,7 @@ def _create_part_handler(self, part: Part) -> PartHandler:
             part,
             part_info=PartInfo(self._project_info, part),
             part_list=self._part_list,
+            ignore_patterns=self._ignore_patterns,
         )
         self._handler[part.name] = handler
 

craft_parts/executor/part_handler.py

@@ -55,6 +55,7 @@ def __init__(
         *,
         part_info: PartInfo,
         part_list: List[Part],
+        ignore_patterns: Optional[List[str]] = None,
     ):
         self._part = part
         self._part_info = part_info
@@ -71,6 +72,7 @@ def __init__(
             cache_dir=part_info.cache_dir,
             part=part,
             project_dirs=part_info.dirs,
+            ignore_patterns=ignore_patterns,
         )
 
         self.build_packages = _get_build_packages(part=self._part, plugin=self._plugin)

craft_parts/infos.py

@@ -61,7 +61,7 @@ def __init__(
             project_dirs = ProjectDirs()
 
         self._application_name = application_name
-        self._cache_dir = Path(cache_dir).absolute()
+        self._cache_dir = Path(cache_dir).expanduser().resolve()
         self._set_machine(arch)
         self._base = base  # TODO: infer base if not specified
         self._parallel_build_count = parallel_build_count

craft_parts/lifecycle_manager.py

@@ -18,7 +18,7 @@
 
 import re
 from pathlib import Path
-from typing import Any, Dict, List, Sequence, Union
+from typing import Any, Dict, List, Optional, Sequence, Union
 
 from pydantic import ValidationError
 
@@ -55,6 +55,8 @@ class LifecycleManager:
         to the system where Craft Parts is being executed.
     :param parallel_build_count: The maximum number of concurrent jobs to be
         used to build each part of this project.
+    :param ignore_local_sources: A list of local source patterns to ignore.
+    :param extra_build_packages: A list of additional build packages to install.
     :param custom_args: Any additional arguments that will be passed directly
         to :ref:`callbacks<callbacks>`.
     """
@@ -69,7 +71,8 @@ def __init__(
         arch: str = "",
         base: str = "",
         parallel_build_count: int = 1,
-        extra_build_packages: List[str] = None,
+        ignore_local_sources: Optional[List[str]] = None,
+        extra_build_packages: Optional[List[str]] = None,
         **custom_args,  # custom passthrough args
     ):
         if not re.match("^[A-Za-z][0-9A-Za-z_]*$", application_name):
@@ -105,10 +108,12 @@ def __init__(
         self._sequencer = sequencer.Sequencer(
             part_list=self._part_list,
             project_info=project_info,
+            ignore_outdated=ignore_local_sources,
         )
         self._executor = executor.Executor(
             part_list=self._part_list,
             project_info=project_info,
+            ignore_patterns=ignore_local_sources,
             extra_build_packages=extra_build_packages,
         )
         self._project_info = project_info

craft_parts/sequencer.py

@@ -34,12 +34,24 @@ class Sequencer:
 
     :param part_list: The list of parts to process.
     :param project_info: Information about this project.
+    :param ignore_outdated: A list of file patterns to ignore when testing for
+        outdated files.
     """
 
-    def __init__(self, *, part_list: List[Part], project_info: ProjectInfo):
+    def __init__(
+        self,
+        *,
+        part_list: List[Part],
+        project_info: ProjectInfo,
+        ignore_outdated: Optional[List[str]] = None,
+    ):
         self._part_list = sort_parts(part_list)
         self._project_info = project_info
-        self._sm = StateManager(project_info=project_info, part_list=part_list)
+        self._sm = StateManager(
+            project_info=project_info,
+            part_list=part_list,
+            ignore_outdated=ignore_outdated,
+        )
         self._actions: List[Action] = []
 
     def plan(self, target_step: Step, part_names: Sequence[str] = None) -> List[Action]:

craft_parts/sources/base.py

@@ -56,10 +56,14 @@ def __init__(
         source_checksum: Optional[str] = None,
         command: Optional[str] = None,
         project_dirs: Optional[ProjectDirs] = None,
+        ignore_patterns: Optional[List[str]] = None,
     ):
         if not project_dirs:
             project_dirs = ProjectDirs()
 
+        if not ignore_patterns:
+            ignore_patterns = []
+
         self.source = str(source)
         self.part_src_dir = str(part_src_dir)
         self._cache_dir = cache_dir
@@ -72,6 +76,7 @@ def __init__(
         self.command = command
         self._dirs = project_dirs
         self._checked = False
+        self._ignore_patterns = ignore_patterns
 
     # pylint: enable=too-many-arguments
 
@@ -126,6 +131,7 @@ def __init__(
         source_checksum: Optional[str] = None,
         command: Optional[str] = None,
         project_dirs: Optional[ProjectDirs] = None,
+        ignore_patterns: Optional[List[str]] = None,
     ):
         super().__init__(
             source,
@@ -138,6 +144,7 @@ def __init__(
             source_checksum=source_checksum,
             command=command,
             project_dirs=project_dirs,
+            ignore_patterns=ignore_patterns,
         )
         self._file = Path()
 

craft_parts/sources/local_source.py

@@ -16,15 +16,20 @@
 
 """The local source handler and helpers."""
 
+import contextlib
 import functools
 import glob
+import logging
 import os
+from pathlib import Path
 from typing import List, Optional
 
 from craft_parts.utils import file_utils
 
 from .base import SourceHandler
 
+logger = logging.getLogger(__name__)
+
 # TODO: change file operations to use pathlib
 
 
@@ -36,14 +41,22 @@ def __init__(self, *args, copy_function=file_utils.link_or_copy, **kwargs):
         self.source_abspath = os.path.abspath(self.source)
         self.copy_function = copy_function
 
-        ignore_patterns = [
-            self._dirs.parts_dir.name,
-            self._dirs.stage_dir.name,
-            self._dirs.prime_dir.name,
-            "*.snap",  # FIXME: this should be specified by the application
-        ]
+        if self._dirs.work_dir.resolve() == Path(self.source_abspath):
+            # ignore parts/stage/dir if source dir matches workdir
+            self._ignore_patterns.append(self._dirs.parts_dir.name)
+            self._ignore_patterns.append(self._dirs.stage_dir.name)
+            self._ignore_patterns.append(self._dirs.prime_dir.name)
+        else:
+            # otherwise check if work_dir inside source dir
+            with contextlib.suppress(ValueError):
+                rel_work_dir = self._dirs.work_dir.relative_to(self.source_abspath)
+                # deep workdirs will be cut at the first component
+                self._ignore_patterns.append(rel_work_dir.parts[0])
+
+        logger.debug("ignore patterns: %r", self._ignore_patterns)
+
         self._ignore = functools.partial(
-            _ignore, self.source_abspath, os.getcwd(), ignore_patterns
+            _ignore, self.source_abspath, os.getcwd(), self._ignore_patterns
         )
         self._updated_files = set()
         self._updated_directories = set()
@@ -67,6 +80,11 @@ def check_if_outdated(
 
         :return: Whether the sources are outdated.
         """
+        if not ignore_files:
+            ignore_files = []
+
+        ignore_files.extend(self._ignore_patterns)
+
         try:
             target_mtime = os.lstat(target).st_mtime
         except FileNotFoundError:

craft_parts/sources/sources.py

@@ -74,7 +74,7 @@
 import os
 import re
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, Optional, Type
+from typing import TYPE_CHECKING, Dict, List, Optional, Type
 
 from craft_parts.dirs import ProjectDirs
 
@@ -101,6 +101,7 @@ def get_source_handler(
     cache_dir: Path,
     part: "Part",
     project_dirs: ProjectDirs,
+    ignore_patterns: Optional[List[str]] = None,
 ) -> Optional[SourceHandler]:
     """Return the appropriate handler for the given source.
 
@@ -124,6 +125,7 @@ def get_source_handler(
             source_depth=part.spec.source_depth,
             source_commit=part.spec.source_commit,
             project_dirs=project_dirs,
+            ignore_patterns=ignore_patterns,
         )
 
     return source_handler

craft_parts/sources/tar_source.py

@@ -22,7 +22,7 @@
 import tarfile
 import tempfile
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 from craft_parts.dirs import ProjectDirs
 
@@ -46,6 +46,7 @@ def __init__(
         source_depth: Optional[int] = None,
         source_checksum: Optional[str] = None,
         project_dirs: Optional[ProjectDirs] = None,
+        ignore_patterns: Optional[List[str]] = None,
     ):
         super().__init__(
             source,
@@ -57,6 +58,7 @@ def __init__(
             source_depth=source_depth,
             source_checksum=source_checksum,
             project_dirs=project_dirs,
+            ignore_patterns=ignore_patterns,
         )
         if source_tag:
             raise errors.InvalidSourceOption(source_type="tar", option="source-tag")

craft_parts/state_manager/state_manager.py

@@ -167,12 +167,21 @@ class StateManager:
 
     :param project_info: The project information.
     :param part_list: A list of this project's parts.
+    :param ignore_outdated: A list of file patterns to ignore when testing for
+        outdated files.
     """
 
-    def __init__(self, *, project_info: ProjectInfo, part_list: List[Part]):
+    def __init__(
+        self,
+        *,
+        project_info: ProjectInfo,
+        part_list: List[Part],
+        ignore_outdated: Optional[List[str]] = None
+    ):
         self._state_db = _StateDB()
         self._project_info = project_info
         self._part_list = part_list
+        self._ignore_outdated = ignore_outdated
         self._source_handler_cache: Dict[str, Optional[SourceHandler]] = {}
 
         part_step_list = _sort_steps_by_state_timestamp(part_list)
@@ -277,6 +286,7 @@ def check_if_outdated(self, part: Part, step: Step) -> Optional[OutdatedReport]:
                     cache_dir=self._project_info.cache_dir,
                     part=part,
                     project_dirs=self._project_info.dirs,
+                    ignore_patterns=self._ignore_outdated,
                 )
                 self._source_handler_cache[part.name] = source_handler
 

pyproject.toml

@@ -16,7 +16,7 @@ min-similarity-lines=13
 max-line-length = "88"
 max-attributes = 15
 max-args= 6
-max-locals = 16
+max-locals = 18
 
 [tool.pylint.MASTER]
 extension-pkg-whitelist = [