From 6db2f6e766555e3aed382921bdd593c2171190fb Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Wed, 15 May 2024 04:13:53 +0200 Subject: [PATCH] SymQEMU tests: on test error, print differences of generated test case (#60) * add colored hexdump diff of different test cases generated * add new package dependencies to docker --- Dockerfile | 5 ++++- tests/symqemu/test.py | 24 +++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 31cbef2332..2859665a16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,10 @@ RUN apt update && apt install -y \ z3 \ libz3-dev \ libz3-dev \ - libzstd-dev + libzstd-dev \ + colordiff \ + xxd \ + wdiff RUN pip install --user meson diff --git a/tests/symqemu/test.py b/tests/symqemu/test.py index 7731c8e470..c310452919 100644 --- a/tests/symqemu/test.py +++ b/tests/symqemu/test.py @@ -2,6 +2,8 @@ import pathlib import shutil import unittest +import tempfile +import subprocess import util @@ -16,13 +18,33 @@ def tearDown(self): shutil.rmtree(self.SYMQEMU_OUTPUT_DIR) def run_symqemu_and_assert_correct_result(self, binary_name): + symqemu_ref_output_dir = util.BINARIES_DIR / binary_name / 'expected_outputs' util.run_symqemu_on_test_binary(binary_name=binary_name, generated_test_cases_output_dir=self.SYMQEMU_OUTPUT_DIR) # `filecmp.dircmp` does a "shallow" comparison, but this is not a problem here because # the timestamps should always be different, so the actual content of the files will be compared. # See https://docs.python.org/3/library/filecmp.html#filecmp.dircmp - expected_vs_actual_output_comparison = filecmp.dircmp(self.SYMQEMU_OUTPUT_DIR, util.BINARIES_DIR / binary_name / 'expected_outputs') + expected_vs_actual_output_comparison = filecmp.dircmp(self.SYMQEMU_OUTPUT_DIR, symqemu_ref_output_dir) + + for diff_file in expected_vs_actual_output_comparison.diff_files: + ref_file = symqemu_ref_output_dir / diff_file + gen_file = self.SYMQEMU_OUTPUT_DIR / diff_file + + tmp_ref = tempfile.NamedTemporaryFile("w+") + subprocess.run(["xxd", f"{ref_file}"], stdout=tmp_ref, check=True) + + tmp_gen = tempfile.NamedTemporaryFile("w+") + subprocess.run(["xxd", f"{gen_file}"], stdout=tmp_gen, check=True) + + wdiff = subprocess.run(["wdiff", f"{tmp_ref.name}", f"{tmp_gen.name}"], capture_output=True) + colordiff = subprocess.run(["colordiff"], input=wdiff.stdout, capture_output=True) + + print(f"===== Diff found in {diff_file} ======") + print(f"{colordiff.stdout.decode('utf-8').strip()}") + print(f"=================================") + print() + self.assertEqual(expected_vs_actual_output_comparison.diff_files, []) self.assertEqual(expected_vs_actual_output_comparison.left_only, []) self.assertEqual(expected_vs_actual_output_comparison.right_only, [])