From 8318222355d60f938ae045e6ca807984e2f1c7ab Mon Sep 17 00:00:00 2001 From: Pierre-Marie de Rodat Date: Tue, 23 Jan 2024 22:11:56 +0100 Subject: [PATCH] DiffTestDriver: tolerate missing baseline files when rewriting baselines --- NEWS.md | 2 + src/e3/testsuite/driver/diff.py | 41 ++++++++++++------- .../missing-baseline/test.yaml | 2 + tests/tests/test_diff.py | 15 ++++--- 4 files changed, 41 insertions(+), 19 deletions(-) create mode 100644 tests/tests/diff-rewriting-tests/missing-baseline/test.yaml diff --git a/NEWS.md b/NEWS.md index 9ad3e82..b0e22a7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,8 @@ 27.0 (Not released yet) ======================= +* `DiffTestDriver`: tolerate missing baseline files when rewriting baselines. + 26.0 (2023-01-19) ================= diff --git a/src/e3/testsuite/driver/diff.py b/src/e3/testsuite/driver/diff.py index 1411352..9e590e2 100644 --- a/src/e3/testsuite/driver/diff.py +++ b/src/e3/testsuite/driver/diff.py @@ -142,6 +142,17 @@ def refine(self, output: AnyStr) -> AnyStr: class DiffTestDriver(ClassicTestDriver): """Test driver to compute test output against a baseline.""" + @property + def rewrite_baseline(self) -> bool: + """Return whether this driver should rewrite its baseline.""" + # If a failure is expected for this test, consider that the actual + # output is wrong, and so do not rewrite the baseline. + return ( + self.baseline_file is not None + and not self.test_control.xfail + and getattr(self.env, "rewrite_baselines", False) + ) + @property def baseline_file(self) -> Tuple[str, bool]: """Return the test output baseline file. @@ -168,19 +179,25 @@ def baseline(self) -> Tuple[Optional[str], Union[str, bytes], bool]: filename is used to rewrite test output: leave it to None if rewriting does not make sense. """ + encoding = self.default_encoding + is_binary = encoding == "binary" filename, is_regexp = self.baseline_file filename = self.test_dir(filename) baseline: Union[str, bytes] + # If the baseline should be rewritten, tolerate a missing baseline file + # and start from an empty baseline: we will create the baseline once + # the test has run. + if self.rewrite_baseline and not os.path.isfile(filename): + return (filename, b"" if is_binary else "", is_regexp) + try: - if self.default_encoding == "binary": - with open(filename, "rb") as text_f: - baseline = text_f.read() - else: - with open( - filename, "r", encoding=self.default_encoding - ) as bin_f: + if is_binary: + with open(filename, "rb") as bin_f: baseline = bin_f.read() + else: + with open(filename, "r", encoding=encoding) as text_f: + baseline = text_f.read() except Exception as exc: raise TestAbortWithError( "cannot read baseline file ({}: {})".format( @@ -332,13 +349,9 @@ def compute_diff( color = "" diff_lines.append(color + line + self.Style.RESET_ALL) - # If requested and the failure is not expected, rewrite the test - # baseline with the new one. - if ( - baseline_file is not None - and not self.test_control.xfail - and getattr(self.env, "rewrite_baselines", False) - ): + # If requested, rewrite the test baseline with the actual output + if self.rewrite_baseline: + assert baseline_file is not None if isinstance(refined_actual, str): with open( baseline_file, "w", encoding=self.default_encoding diff --git a/tests/tests/diff-rewriting-tests/missing-baseline/test.yaml b/tests/tests/diff-rewriting-tests/missing-baseline/test.yaml new file mode 100644 index 0000000..1201dad --- /dev/null +++ b/tests/tests/diff-rewriting-tests/missing-baseline/test.yaml @@ -0,0 +1,2 @@ +driver: diff-script-driver +process_args: [-phelloo, -pworld, -p!] diff --git a/tests/tests/test_diff.py b/tests/tests/test_diff.py index 85a4c8e..2551ec2 100644 --- a/tests/tests/test_diff.py +++ b/tests/tests/test_diff.py @@ -241,11 +241,13 @@ def set_up(self): self.env.test_environ = dict(os.environ) def check_test_out(test, expected_lines, encoding="utf-8"): - with open( - os.path.join(tests_copy, test, "test.out"), encoding=encoding - ) as f: - lines = [line.rstrip() for line in f] - assert lines == expected_lines + filename = os.path.join(tests_copy, test, "test.out") + if expected_lines is None: + assert not os.path.isfile(filename) + else: + with open(filename, encoding=encoding) as f: + lines = [line.rstrip() for line in f] + assert lines == expected_lines # Make sure we have the expected baselines before running the testsuite check_test_out("adacore", ["legacy"]) @@ -254,6 +256,7 @@ def check_test_out(test, expected_lines, encoding="utf-8"): check_test_out("xfail", ["hello", "world"]) check_test_out("iso-8859-1", ["héllo"], encoding="utf-8") check_test_out("bad-utf-8", ["héllo"], encoding="utf-8") + check_test_out("missing-baseline", None) # Run the testsuite in rewrite mode suite = run_testsuite(Mysuite, args=["-rE"], expect_failure=True) @@ -264,6 +267,7 @@ def check_test_out(test, expected_lines, encoding="utf-8"): "xfail": Status.XFAIL, "iso-8859-1": Status.FAIL, "bad-utf-8": Status.ERROR, + "missing-baseline": Status.FAIL, } # Check that non-regexp baselines were updated, except when a failure @@ -274,6 +278,7 @@ def check_test_out(test, expected_lines, encoding="utf-8"): check_test_out("xfail", ["hello", "world"]) check_test_out("iso-8859-1", ["héllo"], encoding="iso-8859-1") check_test_out("bad-utf-8", ["héllo"], encoding="utf-8") + check_test_out("missing-baseline", ["helloo", "world", "!"]) class TestDoubleDiff: