diff --git a/suite/auto-sync/.gitignore b/suite/auto-sync/.gitignore index 820a3356c8..67f50764a3 100644 --- a/suite/auto-sync/.gitignore +++ b/suite/auto-sync/.gitignore @@ -4,6 +4,8 @@ vendor/llvm_root src/auto-sync/config.json src/autosync/cpptranslator/Tests/Differ/test_saved_patches.json src/autosync.egg-info +src/autosync/Tests/MCUpdaterTests/ARCH/Output +src/autosync/Tests/MCUpdaterTests/Disassembler/ARCH/Output src/autosync/lit_config/test_dir_* src/autosync/lit_config/.lit_test_times.txt src/autosync/Tests/MCUpdaterTests/test_output diff --git a/suite/auto-sync/src/autosync/MCUpdater.py b/suite/auto-sync/src/autosync/MCUpdater.py index f86b8e549b..6c2e2a5460 100755 --- a/suite/auto-sync/src/autosync/MCUpdater.py +++ b/suite/auto-sync/src/autosync/MCUpdater.py @@ -60,6 +60,7 @@ def exec(self) -> sp.CompletedProcess: else: self.cmd = re.sub(r"llvm-mc", f"llvm-mc -mattr={self.mattr}", self.cmd) + log.debug(f"Run: {self.cmd}") result = sp.run(self.cmd.split(" "), input=content, capture_output=True) return result @@ -141,6 +142,7 @@ def init_tests(self, unified_test_cases: bool): if mc_output.stderr and not mc_output.stdout: # We can still continue. We just ignore the failed cases. log.debug(f"llvm-mc cmd stderr: {mc_output.stderr}") + log.debug(f"llvm-mc result: {mc_output}") text_section = 0 # Counts the .text sections asm_pat = f"(?P.+)" enc_pat = r"(\[?(?P((0x[a-fA-F0-9]{1,2}[, ]{0,2}))+)[^, ]?\]?)" @@ -318,7 +320,7 @@ def build_test_files(self, mc_cmds: list[LLVM_MC_Command]) -> list[TestFile]: test_files = list() n_all = len(mc_cmds) for i, mcc in enumerate(mc_cmds): - print(f"{i}/{n_all} {mcc.file.name}", flush=True, end="\r") + print(f"{i + 1}/{n_all} {mcc.file.name}", flush=True, end="\r") test_files.append( TestFile( self.arch, @@ -357,25 +359,34 @@ def run_llvm_lit(self, paths: list[Path]) -> list[LLVM_MC_Command]: def extract_llvm_mc_cmds(self, cmds: str) -> list[LLVM_MC_Command]: log.debug("Parsing llvm-mc commands") # Get only the RUN lines which have a show-encoding set. - matches = filter( - lambda l: ( - l if re.search(r"^RUN.+(show-encoding|disassemble)[^|]+", l) else None - ), - cmds.splitlines(), + cmd_lines = cmds.splitlines() + log.debug(f"NO FILTER: {cmd_lines}") + matches = list( + filter( + lambda l: ( + l + if re.search(r"^RUN.+(show-encoding|disassemble)[^|]+", l) + else None + ), + cmd_lines, + ) ) + log.debug(f"FILTER RUN: {' '.join(matches)}") # Don't add tests which are allowed to fail - matches = filter( - lambda m: None if re.search(r"not\s+llvm-mc", m) else m, matches + matches = list( + filter(lambda m: None if re.search(r"not\s+llvm-mc", m) else m, matches) ) + log.debug(f"FILTER not llvm-mc: {' '.join(matches)}") # Skip object file tests - matches = filter( - lambda m: None if re.search(r"filetype=obj", m) else m, matches + matches = list( + filter(lambda m: None if re.search(r"filetype=obj", m) else m, matches) ) + log.debug(f"FILTER filetype=obj-mc: {' '.join(matches)}") # Skip any relocation related tests. matches = filter(lambda m: None if re.search(r"reloc", m) else m, matches) # Remove 'RUN: at ...' prefix matches = map(lambda m: re.sub(r"^RUN: at line \d+: ", "", m), matches) - # Remove redirections + # Remove redirection matches = map(lambda m: re.sub(r"\d>&\d", "", m), matches) # Remove unused arguments matches = map(lambda m: re.sub(r"-o\s?-", "", m), matches) @@ -411,6 +422,7 @@ def gen_all(self): self.check_prerequisites(test_paths) log.info("Generate MC regression tests") llvm_mc_cmds = self.run_llvm_lit(test_paths) + log.info(f"Got {len(llvm_mc_cmds)} llvm-mc commands to run") self.test_files = self.build_test_files(llvm_mc_cmds) for slink in self.symbolic_links: log.debug(f"Unlink {slink}") diff --git a/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/ARCH/Output/test_b.txt.script b/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/ARCH/Output/test_b.txt.script deleted file mode 100644 index b7cb5f8b43..0000000000 --- a/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/ARCH/Output/test_b.txt.script +++ /dev/null @@ -1 +0,0 @@ -set -o pipefail;set -x;{ { set +x; } 2>/dev/null && echo 'RUN: at line 2': 'llvm-mc --disassemble -triple=arm64 < /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_1/test_b.txt | FileCheck /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_1/test_b.txt' >&2 && { set -x; } 2>/dev/null && { llvm-mc --disassemble -triple=arm64 < /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_1/test_b.txt | FileCheck /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_1/test_b.txt; }; } diff --git a/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/ARCH/Output/test_c.txt.script b/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/ARCH/Output/test_c.txt.script deleted file mode 100644 index 9987cd4e13..0000000000 --- a/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/ARCH/Output/test_c.txt.script +++ /dev/null @@ -1 +0,0 @@ -set -o pipefail;set -x;{ { set +x; } 2>/dev/null && echo 'RUN: at line 2': 'llvm-mc --disassemble -triple=arm64 --show-encoding < /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_1/test_c.txt | FileCheck /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_1/test_c.txt' >&2 && { set -x; } 2>/dev/null && { llvm-mc --disassemble -triple=arm64 --show-encoding < /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_1/test_c.txt | FileCheck /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_1/test_c.txt; }; } diff --git a/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/Disassembler/ARCH/Output/test_a.txt.script b/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/Disassembler/ARCH/Output/test_a.txt.script deleted file mode 100644 index 73499af947..0000000000 --- a/suite/auto-sync/src/autosync/Tests/MCUpdaterTests/Disassembler/ARCH/Output/test_a.txt.script +++ /dev/null @@ -1,2 +0,0 @@ -set -o pipefail;set -x;{ { set +x; } 2>/dev/null && echo 'RUN: at line 1': 'llvm-mc -triple=aarch64 -mattr=+v8a,+fp-armv8 -disassemble < /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_0/test_a.txt | FileCheck /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_0/test_a.txt' >&2 && { set -x; } 2>/dev/null && { llvm-mc -triple=aarch64 -mattr=+v8a,+fp-armv8 -disassemble < /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_0/test_a.txt | FileCheck /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_0/test_a.txt; }; } && -{ { set +x; } 2>/dev/null && echo 'RUN: at line 2': 'llvm-mc -triple=arm64 -mattr=+v8.2a -disassemble < /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_0/test_a.txt | FileCheck /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_0/test_a.txt --check-prefix=CHECK-V82' >&2 && { set -x; } 2>/dev/null && { llvm-mc -triple=arm64 -mattr=+v8.2a -disassemble < /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_0/test_a.txt | FileCheck /home/user/repos/capstone/suite/auto-sync/src/autosync/lit_config/test_dir_ARCH_0/test_a.txt --check-prefix=CHECK-V82; }; } diff --git a/suite/auto-sync/src/autosync/Tests/test_mcupdater.py b/suite/auto-sync/src/autosync/Tests/test_mcupdater.py index 90176dec31..2074967354 100644 --- a/suite/auto-sync/src/autosync/Tests/test_mcupdater.py +++ b/suite/auto-sync/src/autosync/Tests/test_mcupdater.py @@ -6,7 +6,6 @@ import sys import unittest from pathlib import Path -from threading import Lock from autosync.Helper import get_path, test_only_overwrite_path_var from autosync.MCUpdater import MCUpdater @@ -21,117 +20,112 @@ def setUpClass(cls): format="%(levelname)-5s - %(message)s", force=True, ) - cls.mutex = Lock() - def test_unified_test_cases(self): - with self.mutex: - out_dir = Path( - get_path("{MCUPDATER_TEST_OUT_DIR}") - .joinpath("merged") - .joinpath("unified") - ) - if not out_dir.exists(): - out_dir.mkdir(parents=True) - for file in out_dir.iterdir(): - logging.debug(f"Delete old file: {file}") - os.remove(file) - test_only_overwrite_path_var( - "{MCUPDATER_OUT_DIR}", - out_dir, - ) - self.updater = MCUpdater( - "ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], True - ) - self.updater.gen_all() - self.assertTrue( - self.compare_files(out_dir, ["test_a.txt.yaml", "test_b.txt.yaml"]) - ) + def test_test_case_gen(self): + """ + To enforce sequential execution of the tests, we execute them in here. + And don't make them a separated test. + """ + self.assertTrue(self.unified_test_cases(), "Failed: unified_test_cases") + self.assertTrue(self.separated_test_cases(), "Failed: separated_test_cases") + self.assertTrue( + self.multi_mode_unified_test_cases(), + "Failed: multi_mode_unified_test_cases", + ) + self.assertTrue( + self.multi_mode_separated_test_cases(), + "Failed: multi_mode_separated_test_cases", + ) + + def unified_test_cases(self): + out_dir = Path( + get_path("{MCUPDATER_TEST_OUT_DIR}").joinpath("merged").joinpath("unified") + ) + if not out_dir.exists(): + out_dir.mkdir(parents=True) + for file in out_dir.iterdir(): + logging.debug(f"Delete old file: {file}") + os.remove(file) + test_only_overwrite_path_var( + "{MCUPDATER_OUT_DIR}", + out_dir, + ) + self.updater = MCUpdater("ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], True) + self.updater.gen_all() + return self.compare_files(out_dir, ["test_a.txt.yaml", "test_b.txt.yaml"]) - def test_separated_test_cases(self): - with self.mutex: - out_dir = Path( - get_path("{MCUPDATER_TEST_OUT_DIR}") - .joinpath("merged") - .joinpath("separated") - ) - if not out_dir.exists(): - out_dir.mkdir(parents=True) - for file in out_dir.iterdir(): - logging.debug(f"Delete old file: {file}") - os.remove(file) - test_only_overwrite_path_var( - "{MCUPDATER_OUT_DIR}", - out_dir, - ) - self.updater = MCUpdater( - "ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], False - ) - self.updater.gen_all() - self.assertTrue( - self.compare_files(out_dir, ["test_a.txt.yaml", "test_b.txt.yaml"]) - ) + def separated_test_cases(self): + out_dir = Path( + get_path("{MCUPDATER_TEST_OUT_DIR}") + .joinpath("merged") + .joinpath("separated") + ) + if not out_dir.exists(): + out_dir.mkdir(parents=True) + for file in out_dir.iterdir(): + logging.debug(f"Delete old file: {file}") + os.remove(file) + test_only_overwrite_path_var( + "{MCUPDATER_OUT_DIR}", + out_dir, + ) + self.updater = MCUpdater( + "ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], False + ) + self.updater.gen_all() + return self.compare_files(out_dir, ["test_a.txt.yaml", "test_b.txt.yaml"]) - def test_multi_mode_unified_test_cases(self): - with self.mutex: - out_dir = Path( - get_path("{MCUPDATER_TEST_OUT_DIR}") - .joinpath("multi") - .joinpath("unified") - ) - if not out_dir.exists(): - out_dir.mkdir(parents=True) - for file in out_dir.iterdir(): - logging.debug(f"Delete old file: {file}") - os.remove(file) - test_only_overwrite_path_var( - "{MCUPDATER_OUT_DIR}", - out_dir, - ) - self.updater = MCUpdater( - "ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], True, multi_mode=True - ) - self.updater.gen_all() - self.assertTrue( - self.compare_files( - out_dir, - [ - "test_a_aarch64_v8a__fp_armv8.txt.yaml", - "test_a_arm64_v8.2a.txt.yaml", - "test_b_arm64.txt.yaml", - ], - ) - ) + def multi_mode_unified_test_cases(self): + out_dir = Path( + get_path("{MCUPDATER_TEST_OUT_DIR}").joinpath("multi").joinpath("unified") + ) + if not out_dir.exists(): + out_dir.mkdir(parents=True) + for file in out_dir.iterdir(): + logging.debug(f"Delete old file: {file}") + os.remove(file) + test_only_overwrite_path_var( + "{MCUPDATER_OUT_DIR}", + out_dir, + ) + self.updater = MCUpdater( + "ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], True, multi_mode=True + ) + self.updater.gen_all() + return self.compare_files( + out_dir, + [ + "test_a_aarch64_v8a__fp_armv8.txt.yaml", + "test_a_arm64_v8.2a.txt.yaml", + "test_b_arm64.txt.yaml", + ], + ) - def test_multi_mode_separated_test_cases(self): - with self.mutex: - out_dir = Path( - get_path("{MCUPDATER_TEST_OUT_DIR}") - .joinpath("multi") - .joinpath("separated") - ) - if not out_dir.exists(): - out_dir.mkdir(parents=True) - for file in out_dir.iterdir(): - logging.debug(f"Delete old file: {file}") - os.remove(file) - test_only_overwrite_path_var( - "{MCUPDATER_OUT_DIR}", - out_dir, - ) - self.updater = MCUpdater( - "ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], False, multi_mode=True - ) - self.updater.gen_all() - self.assertTrue( - self.compare_files( - out_dir, - [ - "test_a_aarch64_v8a__fp_armv8.txt.yaml", - "test_a_arm64_v8.2a.txt.yaml", - "test_b_arm64.txt.yaml", - ], - ) - ) + def multi_mode_separated_test_cases(self): + out_dir = Path( + get_path("{MCUPDATER_TEST_OUT_DIR}").joinpath("multi").joinpath("separated") + ) + if not out_dir.exists(): + out_dir.mkdir(parents=True) + for file in out_dir.iterdir(): + logging.debug(f"Delete old file: {file}") + os.remove(file) + test_only_overwrite_path_var( + "{MCUPDATER_OUT_DIR}", + out_dir, + ) + self.updater = MCUpdater( + "ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], False, multi_mode=True + ) + self.updater.gen_all() + return self.compare_files( + out_dir, + [ + "test_a_aarch64_v8a__fp_armv8.txt.yaml", + "test_a_arm64_v8.2a.txt.yaml", + "test_b_arm64.txt.yaml", + ], + ) def compare_files(self, out_dir: Path, filenames: list[str]) -> bool: if not out_dir.is_dir(): @@ -153,6 +147,7 @@ def compare_files(self, out_dir: Path, filenames: list[str]) -> bool: logging.error(f"{efile} does not exist") return False with open(efile) as f: + logging.debug(f"Read {efile}") expected = f.read() afile = out_dir.joinpath(file) @@ -160,9 +155,12 @@ def compare_files(self, out_dir: Path, filenames: list[str]) -> bool: logging.error(f"{afile} does not exist") return False with open(afile) as f: + logging.debug(f"Read {afile}") actual = f.read() if expected != actual: logging.error("Files mismatch") + print(f"Expected: {efile}") + print(f"Actual: {afile}\n") print(f"Expected:\n\n{expected}\n") print(f"Actual:\n\n{actual}\n") return False