Merge branch 'mr/pmderodat/missing-baseline' into 'master'

DiffTestDriver: tolerate missing baseline files when rewriting baselines

See merge request it/e3-testsuite!22

NEWS.md

@@ -1,6 +1,8 @@
 27.0 (Not released yet)
 =======================
 
+* `DiffTestDriver`: tolerate missing baseline files when rewriting baselines.
+
 26.0 (2023-01-19)
 =================
 

src/e3/testsuite/driver/diff.py

@@ -142,6 +142,17 @@ def refine(self, output: AnyStr) -> AnyStr:
 class DiffTestDriver(ClassicTestDriver):
     """Test driver to compute test output against a baseline."""
 
+    @property
+    def rewrite_baseline(self) -> bool:
+        """Return whether this driver should rewrite its baseline."""
+        # If a failure is expected for this test, consider that the actual
+        # output is wrong, and so do not rewrite the baseline.
+        return (
+            self.baseline_file is not None
+            and not self.test_control.xfail
+            and getattr(self.env, "rewrite_baselines", False)
+        )
+
     @property
     def baseline_file(self) -> Tuple[str, bool]:
         """Return the test output baseline file.
@@ -168,19 +179,25 @@ def baseline(self) -> Tuple[Optional[str], Union[str, bytes], bool]:
             filename is used to rewrite test output: leave it to None if
             rewriting does not make sense.
         """
+        encoding = self.default_encoding
+        is_binary = encoding == "binary"
         filename, is_regexp = self.baseline_file
         filename = self.test_dir(filename)
         baseline: Union[str, bytes]
 
+        # If the baseline should be rewritten, tolerate a missing baseline file
+        # and start from an empty baseline: we will create the baseline once
+        # the test has run.
+        if self.rewrite_baseline and not os.path.isfile(filename):
+            return (filename, b"" if is_binary else "", is_regexp)
+
         try:
-            if self.default_encoding == "binary":
-                with open(filename, "rb") as text_f:
-                    baseline = text_f.read()
-            else:
-                with open(
-                    filename, "r", encoding=self.default_encoding
-                ) as bin_f:
+            if is_binary:
+                with open(filename, "rb") as bin_f:
                     baseline = bin_f.read()
+            else:
+                with open(filename, "r", encoding=encoding) as text_f:
+                    baseline = text_f.read()
         except Exception as exc:
             raise TestAbortWithError(
                 "cannot read baseline file ({}: {})".format(
@@ -332,13 +349,9 @@ def compute_diff(
                 color = ""
             diff_lines.append(color + line + self.Style.RESET_ALL)
 
-        # If requested and the failure is not expected, rewrite the test
-        # baseline with the new one.
-        if (
-            baseline_file is not None
-            and not self.test_control.xfail
-            and getattr(self.env, "rewrite_baselines", False)
-        ):
+        # If requested, rewrite the test baseline with the actual output
+        if self.rewrite_baseline:
+            assert baseline_file is not None
             if isinstance(refined_actual, str):
                 with open(
                     baseline_file, "w", encoding=self.default_encoding

tests/tests/diff-rewriting-tests/missing-baseline/test.yaml

@@ -0,0 +1,2 @@
+driver: diff-script-driver
+process_args: [-phelloo, -pworld, -p!]

tests/tests/test_diff.py

@@ -241,11 +241,13 @@ def set_up(self):
                 self.env.test_environ = dict(os.environ)
 
         def check_test_out(test, expected_lines, encoding="utf-8"):
-            with open(
-                os.path.join(tests_copy, test, "test.out"), encoding=encoding
-            ) as f:
-                lines = [line.rstrip() for line in f]
-            assert lines == expected_lines
+            filename = os.path.join(tests_copy, test, "test.out")
+            if expected_lines is None:
+                assert not os.path.isfile(filename)
+            else:
+                with open(filename, encoding=encoding) as f:
+                    lines = [line.rstrip() for line in f]
+                assert lines == expected_lines
 
         # Make sure we have the expected baselines before running the testsuite
         check_test_out("adacore", ["legacy"])
@@ -254,6 +256,7 @@ def check_test_out(test, expected_lines, encoding="utf-8"):
         check_test_out("xfail", ["hello", "world"])
         check_test_out("iso-8859-1", ["héllo"], encoding="utf-8")
         check_test_out("bad-utf-8", ["héllo"], encoding="utf-8")
+        check_test_out("missing-baseline", None)
 
         # Run the testsuite in rewrite mode
         suite = run_testsuite(Mysuite, args=["-rE"], expect_failure=True)
@@ -264,6 +267,7 @@ def check_test_out(test, expected_lines, encoding="utf-8"):
             "xfail": Status.XFAIL,
             "iso-8859-1": Status.FAIL,
             "bad-utf-8": Status.ERROR,
+            "missing-baseline": Status.FAIL,
         }
 
         # Check that non-regexp baselines were updated, except when a failure
@@ -274,6 +278,7 @@ def check_test_out(test, expected_lines, encoding="utf-8"):
         check_test_out("xfail", ["hello", "world"])
         check_test_out("iso-8859-1", ["héllo"], encoding="iso-8859-1")
         check_test_out("bad-utf-8", ["héllo"], encoding="utf-8")
+        check_test_out("missing-baseline", ["helloo", "world", "!"])
 
 
 class TestDoubleDiff: