feat: implement parameters to allow use of an extra ignore file

In supplement to the support for the .dockerignore and .containerignore
files, these two new parameters (extra-ignore-file and ingore-file-strategy)
allow to modify how the ignore list is managed.

This allows, for example in the case of BinderHub, the administrator to
have a default set of files or folders that get ignored if the repository
does not contain such any ignore file.

The following strategies are available:
- ours
- theirs
- merge

The first forces the use of the file passed in parameters
The second uses the file from the repository if it exists
The last puts both together

repo2docker/__main__.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import sys
+from pathlib import Path
 
 from . import __version__
 from .app import Repo2Docker
@@ -282,6 +283,22 @@ def get_argparser():
         help=Repo2Docker.engine.help,
     )
 
+    argparser.add_argument(
+        "--extra-ignore-file",
+        dest="extra_ignore_file",
+        type=Path,
+        help=Repo2Docker.extra_ignore_file.help,
+    )
+
+    argparser.add_argument(
+        "--ignore-file-strategy",
+        dest="ignore_file_strategy",
+        type=str,
+        choices=Repo2Docker.ignore_file_strategy.values,
+        default=Repo2Docker.ignore_file_strategy.default_value,
+        help=Repo2Docker.ignore_file_strategy.help,
+    )
+
     return argparser
 
 
@@ -464,6 +481,15 @@ def make_r2d(argv=None):
     if args.target_repo_dir:
         r2d.target_repo_dir = args.target_repo_dir
 
+    if args.extra_ignore_file is not None:
+        if not args.extra_ignore_file.exists():
+            print(f"The ignore file {args.extra_ignore_file} does not exist")
+            sys.exit(1)
+        r2d.extra_ignore_file = str(args.extra_ignore_file.resolve())
+
+    if args.ignore_file_strategy is not None:
+        r2d.ignore_file_strategy = args.ignore_file_strategy
+
     return r2d
 
 

repo2docker/app.py

@@ -21,13 +21,14 @@
 import entrypoints
 import escapism
 from pythonjsonlogger import jsonlogger
-from traitlets import Any, Bool, Dict, Int, List, Unicode, default, observe
+from traitlets import Any, Bool, Dict, Enum, Int, List, Unicode, default, observe
 from traitlets.config import Application
 
 from . import __version__, contentproviders
 from .buildpacks import (
     CondaBuildPack,
     DockerBuildPack,
+    ExcludesStrategy,
     JuliaProjectTomlBuildPack,
     JuliaRequireBuildPack,
     LegacyBinderDockerBuildPack,
@@ -462,6 +463,32 @@ def _dry_run_changed(self, change):
         """,
     )
 
+    extra_ignore_file = Unicode(
+        "",
+        config=True,
+        help="""
+        Path to an additional .dockerignore or .containerignore file to be applied
+        when building an image.
+
+        Depending on the strategy selected the content of the file will replace,
+        be merged or be ignored.
+        """,
+    )
+
+    ignore_file_strategy = Enum(
+        ExcludesStrategy.values(),
+        config=True,
+        default_value=ExcludesStrategy.theirs,
+        help="""
+        Strategy to use if an extra ignore file is passed:
+        - merge means that the content of the extra ignore file will be merged
+          with the ignore file contained in the repository (if any)
+        - ours means that the extra ignore file content will be used in any case
+        - theirs means that if there is an ignore file in the repository, the
+          extra ignore file will not be used.
+        """,
+    )
+
     def get_engine(self):
         """Return an instance of the container engine.
 
@@ -860,6 +887,8 @@ def build(self):
                         self.cache_from,
                         self.extra_build_kwargs,
                         platform=self.platform,
+                        extra_ignore_file=self.extra_ignore_file,
+                        ignore_file_strategy=self.ignore_file_strategy,
                     ):
                         if docker_client.string_output:
                             self.log.info(l, extra=dict(phase=R2dState.BUILDING))

repo2docker/buildpacks/__init__.py

@@ -1,4 +1,4 @@
-from .base import BaseImage, BuildPack
+from .base import BaseImage, BuildPack, ExcludesStrategy
 from .conda import CondaBuildPack
 from .docker import DockerBuildPack
 from .julia import JuliaProjectTomlBuildPack, JuliaRequireBuildPack

repo2docker/buildpacks/base.py

@@ -7,6 +7,7 @@
 import sys
 import tarfile
 import textwrap
+from enum import StrEnum, auto
 from functools import lru_cache
 
 import escapism
@@ -205,6 +206,16 @@
 DEFAULT_NB_UID = 1000
 
 
+class ExcludesStrategy(StrEnum):
+    theirs = auto()
+    ours = auto()
+    merge = auto()
+
+    @classmethod
+    def values(cls):
+        return [item.value for item in cls]
+
+
 class BuildPack:
     """
     A composable BuildPack.
@@ -582,6 +593,8 @@ def build(
         cache_from,
         extra_build_kwargs,
         platform=None,
+        extra_ignore_file=None,
+        ignore_file_strategy=ExcludesStrategy.theirs,
     ):
         tarf = io.BytesIO()
         tar = tarfile.open(fileobj=tarf, mode="w")
@@ -609,23 +622,34 @@ def _filter_tar(tarinfo):
         for fname in ("repo2docker-entrypoint", "python3-login"):
             tar.add(os.path.join(HERE, fname), fname, filter=_filter_tar)
 
-        exclude = []
+        def _read_excludes(filepath):
+            with open(filepath) as ignore_file:
+                cleaned_lines = [
+                    line.strip() for line in ignore_file.read().splitlines()
+                ]
+                return [line for line in cleaned_lines if line != "" and line[0] != "#"]
+
+        extra_excludes = []
+        if extra_ignore_file:
+            extra_excludes = _read_excludes(extra_ignore_file)
 
+        excludes = []
         for ignore_file_name in [".dockerignore", ".containerignore"]:
             if os.path.exists(ignore_file_name):
-                with open(ignore_file_name) as ignore_file:
-                    cleaned_lines = [
-                        line.strip() for line in ignore_file.read().splitlines()
-                    ]
-                    exclude.extend(
-                        [
-                            line
-                            for line in cleaned_lines
-                            if line != "" and line[0] != "#"
-                        ]
-                    )
-
-        files_to_add = exclude_paths(".", exclude)
+                excludes.extend(_read_excludes(ignore_file_name))
+
+        if extra_ignore_file is not None:
+            if ignore_file_strategy == ExcludesStrategy.ours:
+                excludes = extra_excludes
+            elif ignore_file_strategy == ExcludesStrategy.merge:
+                excludes.extend(extra_excludes)
+            else:
+                # ignore means that if an ignore file exist, its content is used
+                # otherwise, the extra exclude
+                if not excludes:
+                    excludes = extra_excludes
+
+        files_to_add = exclude_paths(".", excludes)
 
         if files_to_add:
             for item in files_to_add:

tests/conda/ignore-file

@@ -0,0 +1,2 @@
+# Docker compatible ignore file
+from-extra-ignore

tests/conda/py311-extra-ignore-file-merge/.dockerignore

@@ -0,0 +1 @@
+from-dockerignore

tests/conda/py311-extra-ignore-file-merge/environment.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - python=3.11

tests/conda/py311-extra-ignore-file-merge/from-dockerignore

@@ -0,0 +1 @@
+Must be ignored from .dockerignore file

tests/conda/py311-extra-ignore-file-merge/from-extra-ignore

@@ -0,0 +1 @@
+Must be ignored from extra ignore file

tests/conda/py311-extra-ignore-file-merge/test-extra-args.yaml

@@ -0,0 +1,5 @@
+# This file is respected by repo2docker's test suite, but not repo2docker
+# itself. It is used solely to help us test repo2docker's command line flags.
+#
+- --extra-ignore-file=ignore-file
+- --ignore-file-strategy=merge

tests/conda/py311-extra-ignore-file-merge/verify

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import pathlib
+
+assert not pathlib.Path("from-dockerignore").exists()
+assert not pathlib.Path("from-extra-ignore").exists()

tests/conda/py311-extra-ignore-file-ours/.dockerignore

@@ -0,0 +1 @@
+from-dockerignore

tests/conda/py311-extra-ignore-file-ours/environment.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - python=3.11

tests/conda/py311-extra-ignore-file-ours/from-dockerignore

@@ -0,0 +1 @@
+Must not be ignored because of ours strategy and extra ignore file does not contain it.

tests/conda/py311-extra-ignore-file-ours/from-extra-ignore

@@ -0,0 +1 @@
+Must be ignored

tests/conda/py311-extra-ignore-file-ours/test-extra-args.yaml

@@ -0,0 +1,5 @@
+# This file is respected by repo2docker's test suite, but not repo2docker
+# itself. It is used solely to help us test repo2docker's command line flags.
+#
+- --extra-ignore-file=ignore-file
+- --ignore-file-strategy=ours

tests/conda/py311-extra-ignore-file-ours/verify

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import pathlib
+
+assert pathlib.Path("from-dockerignore").exists()
+assert not pathlib.Path("from-extra-ignore").exists()

tests/conda/py311-extra-ignore-file-theirs-without-file/environment.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - python=3.11

tests/conda/py311-extra-ignore-file-theirs-without-file/from-dockerignore

@@ -0,0 +1 @@
+No docker ignore so should still appear

tests/conda/py311-extra-ignore-file-theirs-without-file/from-extra-ignore

@@ -0,0 +1 @@
+Must be ignored because of extra ignore file

tests/conda/py311-extra-ignore-file-theirs-without-file/test-extra-args.yaml

@@ -0,0 +1,5 @@
+# This file is respected by repo2docker's test suite, but not repo2docker
+# itself. It is used solely to help us test repo2docker's command line flags.
+#
+- --extra-ignore-file=ignore-file
+- --ignore-file-strategy=theirs

tests/conda/py311-extra-ignore-file-theirs-without-file/verify

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import pathlib
+
+assert pathlib.Path("from-dockerignore").exists()
+assert not pathlib.Path("from-extra-ignore").exists()

tests/conda/py311-extra-ignore-file-theirs/.dockerignore

@@ -0,0 +1 @@
+from-dockerignore

tests/conda/py311-extra-ignore-file-theirs/environment.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - python=3.11

tests/conda/py311-extra-ignore-file-theirs/from-dockerignore

@@ -0,0 +1 @@
+Must be ignored from .dockerignore file

tests/conda/py311-extra-ignore-file-theirs/from-extra-ignore

@@ -0,0 +1 @@
+Shall be present due to strategy being theirs and this file does not appear in .dockerignore

tests/conda/py311-extra-ignore-file-theirs/test-extra-args.yaml

@@ -0,0 +1,5 @@
+# This file is respected by repo2docker's test suite, but not repo2docker
+# itself. It is used solely to help us test repo2docker's command line flags.
+#
+- --extra-ignore-file=ignore-file
+- --ignore-file-strategy=theirs

tests/conda/py311-extra-ignore-file-theirs/verify

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import pathlib
+
+assert not pathlib.Path("from-dockerignore").exists()
+assert pathlib.Path("from-extra-ignore").exists()

tests/unit/test_args.py

@@ -128,3 +128,8 @@ def test_config_priority(tmp_path, trait, arg, default):
     assert getattr(r2d, trait) == "config"
     r2d = make_r2d(["--config", config_file, arg, "cli", "."])
     assert getattr(r2d, trait) == "cli"
+
+
+def test_non_existing_exclude_file():
+    with pytest.raises(SystemExit):
+        make_r2d(["--extra-ignore-file", "does-not-exist"])

tests/unit/test_argumentvalidation.py

@@ -247,3 +247,28 @@ def test_docker_no_build_success(temp_cwd):
     args_list = ["--no-build", "--no-run"]
 
     assert validate_arguments(builddir, args_list, disable_dockerd=True)
+
+
+@pytest.mark.parametrize(
+    "strategy, is_valid",
+    [
+        ("theirs", True),
+        ("ours", True),
+        ("merge", True),
+        ("invalid", False),
+    ],
+)
+def test_ignore_file_strategy(temp_cwd, strategy, is_valid):
+    """ """
+
+    args_list = ["--no-build", "--no-run", "--ignore-file-strategy", strategy]
+
+    assert (
+        validate_arguments(
+            builddir,
+            args_list,
+            "--ignore-file-strategy: invalid choice: 'invalid' (choose from 'theirs', 'ours', 'merge')",
+            disable_dockerd=True,
+        )
+        == is_valid
+    )