Merge branch 'mr/pmderodat/diff_context_size' into 'master'

DiffTestDriver: handle diff context size customization

See merge request it/e3-testsuite!17

NEWS.md

@@ -1,6 +1,7 @@
 26.0 (Not released yet)
 =======================
 
+* `DiffTestDriver`: handle diff context size customization.
 * Add a XUnit results importer for report indexes.
 * `AdaCoreLegacyDriver`: avoid CRLF line endings in test scripts.
 * `AdaCoreLegacyDriver`: fix handling of non-ASCII test scripts.

src/e3/testsuite/driver/diff.py

@@ -226,6 +226,11 @@ def diff_ignore_white_chars(self) -> bool:
         """
         return False
 
+    @property
+    def diff_context_size(self) -> int:
+        """Positive number of context lines to include in diff computations."""
+        return self.test_env.get("diff_context_size", 1)
+
     def set_up(self) -> None:
         super().set_up()
 
@@ -239,6 +244,7 @@ def compute_diff(
         actual: AnyStr,
         failure_message: str = "unexpected output",
         ignore_white_chars: Optional[bool] = None,
+        context_size: Optional[int] = None,
         truncate_logs_threshold: Optional[int] = None,
     ) -> List[str]:
         """Compute the diff between expected and actual outputs.
@@ -254,6 +260,8 @@ def compute_diff(
         :param ignore_white_chars: Whether to ignore whitespaces during the
             diff computation. If left to None, use
             ``self.diff_ignore_white_chars``.
+        :param context_size: Positive number of context lines to include in
+            diff computations. If left to None, use ``self.diff_context_size``.
         :param truncate_logs_threshold: Threshold to truncate the diff message
             in ``self.result.log``. See ``e3.testsuite.result.truncated``'s
             ``line_count`` argument. If left to None, use the testsuite's
@@ -262,6 +270,9 @@ def compute_diff(
         if ignore_white_chars is None:
             ignore_white_chars = self.diff_ignore_white_chars
 
+        if context_size is None:
+            context_size = self.diff_context_size
+
         if truncate_logs_threshold is None:
             truncate_logs_threshold = self.testsuite_options.truncate_logs
 
@@ -297,7 +308,10 @@ def compute_diff(
         # include the diff in the test log and return the given failure
         # message.
         d = diff(
-            expected_lines, actual_lines, ignore_white_chars=ignore_white_chars
+            expected_lines,
+            actual_lines,
+            ignore_white_chars=ignore_white_chars,
+            context=context_size,
         )
         if not d:
             return []

tests/tests/test_diff.py

@@ -9,6 +9,7 @@
 import e3.testsuite.driver.adacore as adacore
 import e3.testsuite.driver.diff as diff
 from e3.testsuite.result import TestStatus as Status
+from e3.testsuite.testcase_finder import ParsedTest
 
 from .utils import create_testsuite, extract_results, run_testsuite
 
@@ -75,6 +76,124 @@ class Mysuite(Suite):
     }
 
 
+def test_diff_context_size():
+    """Check handling for the diff_context_size setting."""
+    baseline = "".join(f"{i}\n" for i in range(100))
+    actual = "".join(f"{i}\n" for i in range(100) if i != 20)
+
+    def create_test(name, driver, test_env):
+        return ParsedTest(name, driver, test_env, ".", None)
+
+    tests = []
+
+    # Test default behavior and overriding it with the test environment
+
+    class MyDriver(diff.DiffTestDriver):
+        def run(self):
+            pass
+
+        def compute_failures(self):
+            return self.compute_diff("output", baseline, actual)
+
+    tests += [
+        create_test("default", MyDriver, {}),
+        create_test("test_env", MyDriver, {"diff_context_size": 2}),
+    ]
+
+    # Test overriding through the diff_context_size property: it should
+    # short-circuit the test environment.
+
+    class MyDriver2(MyDriver):
+        @property
+        def diff_context_size(self):
+            return 2
+
+    tests += [
+        create_test("property", MyDriver2, {}),
+        create_test("property_test_env", MyDriver2, {"diff_context_size": 5}),
+    ]
+
+    # Test direct calls to DiffTestDriver.compute_diff: non-None context_size
+    # argument should override the rest.
+
+    class MyDriver3(MyDriver):
+        @property
+        def diff_context_size(self):
+            return 2
+
+        def compute_failures(self):
+            return self.compute_diff(
+                "output", baseline, actual, context_size=5
+            )
+
+    tests += [
+        create_test("method", MyDriver3, {}),
+        create_test("method_test_env", MyDriver3, {"diff_context_size": 10}),
+    ]
+
+    class MySuite(Suite):
+        def get_test_list(self, sublist):
+            return tests
+
+    suite = run_testsuite(MySuite, args=["-j1"], expect_failure=True)
+    assert extract_results(suite) == {
+        "default": Status.FAIL,
+        "test_env": Status.FAIL,
+        "property": Status.FAIL,
+        "property_test_env": Status.FAIL,
+        "method": Status.FAIL,
+        "method_test_env": Status.FAIL,
+    }
+
+    def check_diff(test_name, expected):
+        assert suite.report_index.entries[test_name].load().diff == expected
+
+    diff_context_1 = (
+        "Diff failure: unexpected output\n"
+        "--- expected\n"
+        "+++ output\n"
+        "@@ -20,3 +20,2 @@\n"
+        " 19\n"
+        "-20\n"
+        " 21\n"
+    )
+    diff_context_2 = (
+        "Diff failure: unexpected output\n"
+        "--- expected\n"
+        "+++ output\n"
+        "@@ -19,5 +19,4 @@\n"
+        " 18\n"
+        " 19\n"
+        "-20\n"
+        " 21\n"
+        " 22\n"
+    )
+    diff_context_5 = (
+        "Diff failure: unexpected output\n"
+        "--- expected\n"
+        "+++ output\n"
+        "@@ -16,11 +16,10 @@\n"
+        " 15\n"
+        " 16\n"
+        " 17\n"
+        " 18\n"
+        " 19\n"
+        "-20\n"
+        " 21\n"
+        " 22\n"
+        " 23\n"
+        " 24\n"
+        " 25\n"
+    )
+
+    check_diff("default", diff_context_1)
+    check_diff("test_env", diff_context_2)
+    check_diff("property", diff_context_2)
+    check_diff("property_test_env", diff_context_2)
+    check_diff("method", diff_context_5)
+    check_diff("method_test_env", diff_context_5)
+
+
 def test_regexp_fullmatch():
     """Check that DiffTestDriver does a full match over regexp baselines."""
     # The "abcd" regexp baseline should not be able to match "abcde" (it used