Skip to content

Commit

Permalink
Remove any possibility to run tests in parallel.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Rot127
Rot127 committed Aug 26, 2024
1 parent 7c4c3ea commit 3c70410
Show file tree
Hide file tree
Showing 6 changed files with 131 additions and 123 deletions.
2 changes: 2 additions & 0 deletions suite/auto-sync/.gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ vendor/llvm_root
src/auto-sync/config.json
src/autosync/cpptranslator/Tests/Differ/test_saved_patches.json
src/autosync.egg-info
src/autosync/Tests/MCUpdaterTests/ARCH/Output
src/autosync/Tests/MCUpdaterTests/Disassembler/ARCH/Output
src/autosync/lit_config/test_dir_*
src/autosync/lit_config/.lit_test_times.txt
src/autosync/Tests/MCUpdaterTests/test_output
34 changes: 23 additions & 11 deletions suite/auto-sync/src/autosync/MCUpdater.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def exec(self) -> sp.CompletedProcess:
else:
self.cmd = re.sub(r"llvm-mc", f"llvm-mc -mattr={self.mattr}", self.cmd)

log.debug(f"Run: {self.cmd}")
result = sp.run(self.cmd.split(" "), input=content, capture_output=True)
return result

Expand Down Expand Up @@ -141,6 +142,7 @@ def init_tests(self, unified_test_cases: bool):
if mc_output.stderr and not mc_output.stdout:
# We can still continue. We just ignore the failed cases.
log.debug(f"llvm-mc cmd stderr: {mc_output.stderr}")
log.debug(f"llvm-mc result: {mc_output}")
text_section = 0 # Counts the .text sections
asm_pat = f"(?P<asm_text>.+)"
enc_pat = r"(\[?(?P<enc_bytes>((0x[a-fA-F0-9]{1,2}[, ]{0,2}))+)[^, ]?\]?)"
Expand Down Expand Up @@ -318,7 +320,7 @@ def build_test_files(self, mc_cmds: list[LLVM_MC_Command]) -> list[TestFile]:
test_files = list()
n_all = len(mc_cmds)
for i, mcc in enumerate(mc_cmds):
print(f"{i}/{n_all} {mcc.file.name}", flush=True, end="\r")
print(f"{i + 1}/{n_all} {mcc.file.name}", flush=True, end="\r")
test_files.append(
TestFile(
self.arch,
Expand Down Expand Up @@ -357,25 +359,34 @@ def run_llvm_lit(self, paths: list[Path]) -> list[LLVM_MC_Command]:
def extract_llvm_mc_cmds(self, cmds: str) -> list[LLVM_MC_Command]:
log.debug("Parsing llvm-mc commands")
# Get only the RUN lines which have a show-encoding set.
matches = filter(
lambda l: (
l if re.search(r"^RUN.+(show-encoding|disassemble)[^|]+", l) else None
),
cmds.splitlines(),
cmd_lines = cmds.splitlines()
log.debug(f"NO FILTER: {cmd_lines}")
matches = list(
filter(
lambda l: (
l
if re.search(r"^RUN.+(show-encoding|disassemble)[^|]+", l)
else None
),
cmd_lines,
)
)
log.debug(f"FILTER RUN: {' '.join(matches)}")
# Don't add tests which are allowed to fail
matches = filter(
lambda m: None if re.search(r"not\s+llvm-mc", m) else m, matches
matches = list(
filter(lambda m: None if re.search(r"not\s+llvm-mc", m) else m, matches)
)
log.debug(f"FILTER not llvm-mc: {' '.join(matches)}")
# Skip object file tests
matches = filter(
lambda m: None if re.search(r"filetype=obj", m) else m, matches
matches = list(
filter(lambda m: None if re.search(r"filetype=obj", m) else m, matches)
)
log.debug(f"FILTER filetype=obj-mc: {' '.join(matches)}")
# Skip any relocation related tests.
matches = filter(lambda m: None if re.search(r"reloc", m) else m, matches)
# Remove 'RUN: at ...' prefix
matches = map(lambda m: re.sub(r"^RUN: at line \d+: ", "", m), matches)
# Remove redirections
# Remove redirection
matches = map(lambda m: re.sub(r"\d>&\d", "", m), matches)
# Remove unused arguments
matches = map(lambda m: re.sub(r"-o\s?-", "", m), matches)
Expand Down Expand Up @@ -411,6 +422,7 @@ def gen_all(self):
self.check_prerequisites(test_paths)
log.info("Generate MC regression tests")
llvm_mc_cmds = self.run_llvm_lit(test_paths)
log.info(f"Got {len(llvm_mc_cmds)} llvm-mc commands to run")
self.test_files = self.build_test_files(llvm_mc_cmds)
for slink in self.symbolic_links:
log.debug(f"Unlink {slink}")
Expand Down
1 change: 0 additions & 1 deletion suite/auto-sync/src/autosync/Tests/MCUpdaterTests/ARCH/Output/test_b.txt.script
View file

This file was deleted.

1 change: 0 additions & 1 deletion suite/auto-sync/src/autosync/Tests/MCUpdaterTests/ARCH/Output/test_c.txt.script
View file

This file was deleted.

2 changes: 0 additions & 2 deletions suite/auto-sync/src/autosync/Tests/MCUpdaterTests/Disassembler/ARCH/Output/test_a.txt.script
View file

This file was deleted.

214 changes: 106 additions & 108 deletions suite/auto-sync/src/autosync/Tests/test_mcupdater.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import sys
import unittest
from pathlib import Path
from threading import Lock

from autosync.Helper import get_path, test_only_overwrite_path_var
from autosync.MCUpdater import MCUpdater
Expand All @@ -21,117 +20,112 @@ def setUpClass(cls):
format="%(levelname)-5s - %(message)s",
force=True,
)
cls.mutex = Lock()

def test_unified_test_cases(self):
with self.mutex:
out_dir = Path(
get_path("{MCUPDATER_TEST_OUT_DIR}")
.joinpath("merged")
.joinpath("unified")
)
if not out_dir.exists():
out_dir.mkdir(parents=True)
for file in out_dir.iterdir():
logging.debug(f"Delete old file: {file}")
os.remove(file)
test_only_overwrite_path_var(
"{MCUPDATER_OUT_DIR}",
out_dir,
)
self.updater = MCUpdater(
"ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], True
)
self.updater.gen_all()
self.assertTrue(
self.compare_files(out_dir, ["test_a.txt.yaml", "test_b.txt.yaml"])
)
def test_test_case_gen(self):
"""
To enforce sequential execution of the tests, we execute them in here.
And don't make them a separated test.
"""
self.assertTrue(self.unified_test_cases(), "Failed: unified_test_cases")
self.assertTrue(self.separated_test_cases(), "Failed: separated_test_cases")
self.assertTrue(
self.multi_mode_unified_test_cases(),
"Failed: multi_mode_unified_test_cases",
)
self.assertTrue(
self.multi_mode_separated_test_cases(),
"Failed: multi_mode_separated_test_cases",
)

def unified_test_cases(self):
out_dir = Path(
get_path("{MCUPDATER_TEST_OUT_DIR}").joinpath("merged").joinpath("unified")
)
if not out_dir.exists():
out_dir.mkdir(parents=True)
for file in out_dir.iterdir():
logging.debug(f"Delete old file: {file}")
os.remove(file)
test_only_overwrite_path_var(
"{MCUPDATER_OUT_DIR}",
out_dir,
)
self.updater = MCUpdater("ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], True)
self.updater.gen_all()
return self.compare_files(out_dir, ["test_a.txt.yaml", "test_b.txt.yaml"])

def test_separated_test_cases(self):
with self.mutex:
out_dir = Path(
get_path("{MCUPDATER_TEST_OUT_DIR}")
.joinpath("merged")
.joinpath("separated")
)
if not out_dir.exists():
out_dir.mkdir(parents=True)
for file in out_dir.iterdir():
logging.debug(f"Delete old file: {file}")
os.remove(file)
test_only_overwrite_path_var(
"{MCUPDATER_OUT_DIR}",
out_dir,
)
self.updater = MCUpdater(
"ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], False
)
self.updater.gen_all()
self.assertTrue(
self.compare_files(out_dir, ["test_a.txt.yaml", "test_b.txt.yaml"])
)
def separated_test_cases(self):
out_dir = Path(
get_path("{MCUPDATER_TEST_OUT_DIR}")
.joinpath("merged")
.joinpath("separated")
)
if not out_dir.exists():
out_dir.mkdir(parents=True)
for file in out_dir.iterdir():
logging.debug(f"Delete old file: {file}")
os.remove(file)
test_only_overwrite_path_var(
"{MCUPDATER_OUT_DIR}",
out_dir,
)
self.updater = MCUpdater(
"ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], False
)
self.updater.gen_all()
return self.compare_files(out_dir, ["test_a.txt.yaml", "test_b.txt.yaml"])

def test_multi_mode_unified_test_cases(self):
with self.mutex:
out_dir = Path(
get_path("{MCUPDATER_TEST_OUT_DIR}")
.joinpath("multi")
.joinpath("unified")
)
if not out_dir.exists():
out_dir.mkdir(parents=True)
for file in out_dir.iterdir():
logging.debug(f"Delete old file: {file}")
os.remove(file)
test_only_overwrite_path_var(
"{MCUPDATER_OUT_DIR}",
out_dir,
)
self.updater = MCUpdater(
"ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], True, multi_mode=True
)
self.updater.gen_all()
self.assertTrue(
self.compare_files(
out_dir,
[
"test_a_aarch64_v8a__fp_armv8.txt.yaml",
"test_a_arm64_v8.2a.txt.yaml",
"test_b_arm64.txt.yaml",
],
)
)
def multi_mode_unified_test_cases(self):
out_dir = Path(
get_path("{MCUPDATER_TEST_OUT_DIR}").joinpath("multi").joinpath("unified")
)
if not out_dir.exists():
out_dir.mkdir(parents=True)
for file in out_dir.iterdir():
logging.debug(f"Delete old file: {file}")
os.remove(file)
test_only_overwrite_path_var(
"{MCUPDATER_OUT_DIR}",
out_dir,
)
self.updater = MCUpdater(
"ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], True, multi_mode=True
)
self.updater.gen_all()
return self.compare_files(
out_dir,
[
"test_a_aarch64_v8a__fp_armv8.txt.yaml",
"test_a_arm64_v8.2a.txt.yaml",
"test_b_arm64.txt.yaml",
],
)

def test_multi_mode_separated_test_cases(self):
with self.mutex:
out_dir = Path(
get_path("{MCUPDATER_TEST_OUT_DIR}")
.joinpath("multi")
.joinpath("separated")
)
if not out_dir.exists():
out_dir.mkdir(parents=True)
for file in out_dir.iterdir():
logging.debug(f"Delete old file: {file}")
os.remove(file)
test_only_overwrite_path_var(
"{MCUPDATER_OUT_DIR}",
out_dir,
)
self.updater = MCUpdater(
"ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], False, multi_mode=True
)
self.updater.gen_all()
self.assertTrue(
self.compare_files(
out_dir,
[
"test_a_aarch64_v8a__fp_armv8.txt.yaml",
"test_a_arm64_v8.2a.txt.yaml",
"test_b_arm64.txt.yaml",
],
)
)
def multi_mode_separated_test_cases(self):
out_dir = Path(
get_path("{MCUPDATER_TEST_OUT_DIR}").joinpath("multi").joinpath("separated")
)
if not out_dir.exists():
out_dir.mkdir(parents=True)
for file in out_dir.iterdir():
logging.debug(f"Delete old file: {file}")
os.remove(file)
test_only_overwrite_path_var(
"{MCUPDATER_OUT_DIR}",
out_dir,
)
self.updater = MCUpdater(
"ARCH", get_path("{MCUPDATER_TEST_DIR}"), [], [], False, multi_mode=True
)
self.updater.gen_all()
return self.compare_files(
out_dir,
[
"test_a_aarch64_v8a__fp_armv8.txt.yaml",
"test_a_arm64_v8.2a.txt.yaml",
"test_b_arm64.txt.yaml",
],
)

def compare_files(self, out_dir: Path, filenames: list[str]) -> bool:
if not out_dir.is_dir():
Expand All @@ -153,16 +147,20 @@ def compare_files(self, out_dir: Path, filenames: list[str]) -> bool:
logging.error(f"{efile} does not exist")
return False
with open(efile) as f:
logging.debug(f"Read {efile}")
expected = f.read()

afile = out_dir.joinpath(file)
if not afile.exists():
logging.error(f"{afile} does not exist")
return False
with open(afile) as f:
logging.debug(f"Read {afile}")
actual = f.read()
if expected != actual:
logging.error("Files mismatch")
print(f"Expected: {efile}")
print(f"Actual: {afile}\n")
print(f"Expected:\n\n{expected}\n")
print(f"Actual:\n\n{actual}\n")
return False
Expand Down

0 comments on commit 3c70410

Please sign in to comment.