From 95ce0f04163d827e050c85d20688f3435f85e5fc Mon Sep 17 00:00:00 2001 From: Dave Parker Date: Wed, 10 Jan 2024 22:43:25 +0000 Subject: [PATCH] Switch --print-failures in prism-auto, enabled by default for "make tests". Sometimes, CI testing generates failures that are not reproducible locally. --- prism/Makefile | 8 ++++---- prism/etc/scripts/prism-auto | 10 ++++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/prism/Makefile b/prism/Makefile index bc5d325a..d7027c4c 100644 --- a/prism/Makefile +++ b/prism/Makefile @@ -521,7 +521,7 @@ testlpsolve: # Optionally, extra arguments for prism-auto are picked up via variable TESTS_ARGS tests: testslocal @if [ -d ../prism-tests ]; then \ - etc/scripts/prism-auto -t -m ../prism-tests -p bin/prism --nailgun --ngprism bin/ngprism $(TESTS_ARGS); \ + etc/scripts/prism-auto -t -m ../prism-tests -p bin/prism --print-failures --nailgun --ngprism bin/ngprism $(TESTS_ARGS); \ else \ echo "Skipping tests"; \ fi @@ -529,13 +529,13 @@ tests: testslocal # Just display the command to run the test suite on this version of PRISM # Optionally, extra arguments for prism-auto are picked up via variable TESTS_ARGS testsecho: - @echo etc/scripts/prism-auto -t -m ../prism-tests -p bin/prism --nailgun --ngprism bin/ngprism $(TESTS_ARGS) + @echo etc/scripts/prism-auto -t -m ../prism-tests -p bin/prism --print-failures --nailgun --ngprism bin/ngprism $(TESTS_ARGS) # Run local tests (in ./tests) # Optionally, extra arguments for prism-auto are picked up via variable TESTS_ARGS testslocal: @if [ -d tests ]; then \ - etc/scripts/prism-auto -t -m tests -p bin/prism --nailgun --ngprism bin/ngprism $(TESTS_ARGS); \ + etc/scripts/prism-auto -t -m tests -p bin/prism --print-failures --nailgun --ngprism bin/ngprism $(TESTS_ARGS); \ else \ echo "Skipping local tests"; \ fi @@ -548,7 +548,7 @@ testslocal: testsfull: etc/scripts/prism-auto -t -m ../prism-tests \ --skip-export-runs --skip-duplicate-runs --test-all -a ../prism-tests/all-engines.args --timeout 1m \ - -p bin/prism --nailgun $(TESTS_ARGS); + -p bin/prism --print-failures --nailgun $(TESTS_ARGS); ########################## # Building distributions # diff --git a/prism/etc/scripts/prism-auto b/prism/etc/scripts/prism-auto index 2fb84ff5..a4830832 100755 --- a/prism/etc/scripts/prism-auto +++ b/prism/etc/scripts/prism-auto @@ -679,8 +679,13 @@ def runPrism(args, bmArgs, dir=""): for line in open(logFile, 'r').readlines(): if re.match('Error:', line): printTestResult(line) - print("To see log file, run:") - print("edit " + logFile) + if (options.printFailures): + print("Failing test log file:") + for line in open(logFile, 'r').readlines(): + print(line, end="") + else: + print("To see log file, run:") + print("edit " + logFile) prismArgsPrint = list(prismArgs) prismArgsPrint[0] = options.prismExec print("To re-run failing test:") @@ -1162,6 +1167,7 @@ parser.add_option("--skip-duplicate-runs", action="store_true", dest="skipDuplic parser.add_option("--timeout", dest="timeout", metavar="N", default=None, help='Timeout for each PRISM run (examples values for N: 5 / 5s for seconds, 5m / 5h for minutes / hours)') parser.add_option("--dd-warnings", action="store_true", dest="ddWarnings", default=False, help="Print the DD reference count warnings") parser.add_option("--colour", dest="colourEnabled", metavar="X", type="choice", choices=["yes","no","auto"], default="auto", help="Whether to colour test results: yes, no, auto (yes iff in terminal mode) [default=auto]") +parser.add_option("--print-failures", action="store_true", dest="printFailures", default=False, help="Display logs for failing tests") (options, args) = parser.parse_args() if len(args) < 1: parser.print_help()