diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index 315145b78c51b18..a88fe7534452773 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -717,8 +717,13 @@ def add_parse_arguments(parser = None): "--verbose", action="count", default=0, - help="Emit debugging information, call multiple times to increase " - "verbosity.") + help="Call multiple times to increase verbosity.") + + parser.add_argument( + "-d", + "--debug", + action="store_true", + help="Emit debugging information.") parser.add_argument("-W", "--disable-warnings-as-errors", action="store_true", help="Do not treat warning conditions as errors.") diff --git a/scripts/pylib/twister/twisterlib/twister_main.py b/scripts/pylib/twister/twisterlib/twister_main.py index d2699ad49867e1f..b1975ba71b547fa 100644 --- a/scripts/pylib/twister/twisterlib/twister_main.py +++ b/scripts/pylib/twister/twisterlib/twister_main.py @@ -26,7 +26,7 @@ logger.setLevel(logging.DEBUG) -def setup_logging(outdir, log_file, verbose, timestamps): +def setup_logging(outdir, log_file, debug, timestamps): # create file handler which logs even debug messages if log_file: fh = logging.FileHandler(log_file) @@ -38,7 +38,7 @@ def setup_logging(outdir, log_file, verbose, timestamps): # create console handler with a higher log level ch = logging.StreamHandler() - if verbose > 1: + if debug: ch.setLevel(logging.DEBUG) else: ch.setLevel(logging.INFO) @@ -108,7 +108,7 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace): fp.write(previous_results) VERBOSE = options.verbose - setup_logging(options.outdir, options.log_file, VERBOSE, options.timestamps) + setup_logging(options.outdir, options.log_file, options.debug, options.timestamps) env = TwisterEnv(options, default_options) env.discover() diff --git a/scripts/tests/twister_blackbox/test_coverage.py b/scripts/tests/twister_blackbox/test_coverage.py index 0d5e035ecd218f1..5bc50974adc714b 100644 --- a/scripts/tests/twister_blackbox/test_coverage.py +++ b/scripts/tests/twister_blackbox/test_coverage.py @@ -13,6 +13,7 @@ import sys import json +# pylint: disable=duplicate-code from conftest import TEST_DATA, ZEPHYR_BASE, testsuite_filename_mock, clear_log_in_test from twisterlib.testplan import TestPlan @@ -189,7 +190,7 @@ def test_coverage(self, capfd, test_path, test_platforms, out_path, file_name): ) def test_enable_coverage(self, capfd, test_path, test_platforms, out_path, expected): args = ['-i','--outdir', out_path, '-T', test_path] + \ - ['--enable-coverage', '-vv'] + \ + ['--enable-coverage', '-vv', '-d'] + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms ) for val in pair] diff --git a/scripts/tests/twister_blackbox/test_disable.py b/scripts/tests/twister_blackbox/test_disable.py index 448c6f0eabf0219..cd82205b062344b 100644 --- a/scripts/tests/twister_blackbox/test_disable.py +++ b/scripts/tests/twister_blackbox/test_disable.py @@ -76,7 +76,7 @@ def teardown_class(cls): def test_disable_suite_name_check(self, capfd, out_path, test_path, test_platforms, flag, expected, expected_none): args = ['-i', '--outdir', out_path, '-T', test_path] + \ [flag] + \ - ['-vv'] + \ + ['-vv', '-d'] + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms ) for val in pair] diff --git a/scripts/tests/twister_blackbox/test_filter.py b/scripts/tests/twister_blackbox/test_filter.py index 60f685509b16470..cfcc8cbe136ccf6 100644 --- a/scripts/tests/twister_blackbox/test_filter.py +++ b/scripts/tests/twister_blackbox/test_filter.py @@ -191,7 +191,7 @@ def test_enable_slow_only(self, out_path): def test_arch(self, capfd, out_path, arch, expected): path = os.path.join(TEST_DATA, 'tests', 'no_filter') test_platforms = ['qemu_x86', 'hsdk', 'intel_adl_crb', 'it8xxx2_evb'] - args = ['--outdir', out_path, '-T', path, '-vv'] + \ + args = ['--outdir', out_path, '-T', path, '-vv', '-d'] + \ ['--arch', arch] + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms @@ -224,7 +224,7 @@ def test_arch(self, capfd, out_path, arch, expected): def test_vendor(self, capfd, out_path, vendor, expected): path = os.path.join(TEST_DATA, 'tests', 'no_filter') test_platforms = ['qemu_x86', 'hsdk', 'intel_adl_crb', 'it8xxx2_evb'] - args = ['--outdir', out_path, '-T', path, '-vv'] + \ + args = ['--outdir', out_path, '-T', path, '-vv', '-d'] + \ ['--vendor', vendor] + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms diff --git a/scripts/tests/twister_blackbox/test_output.py b/scripts/tests/twister_blackbox/test_output.py index 61584ca4349ef10..9bc35307d4151f1 100644 --- a/scripts/tests/twister_blackbox/test_output.py +++ b/scripts/tests/twister_blackbox/test_output.py @@ -14,7 +14,6 @@ import sys import json -# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test from twisterlib.testplan import TestPlan @@ -22,10 +21,14 @@ @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) class TestOutput: TESTDATA_1 = [ - ( - os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic') - ), -] + ([]), + (['-d']), + (['-v']), + (['-v', '-d']), + (['-vv']), + (['-vv', '-d']), + ] + @classmethod def setup_class(cls): apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister') @@ -149,38 +152,15 @@ def _get_matches(self, err, regex_line): matches = [] return matches - @pytest.mark.parametrize( - 'test_path', - TESTDATA_1, - ids=[ - 'single_v', - ] - ) - def test_single_v(self, capfd, out_path, test_path): - args = ['--outdir', out_path, '-T', test_path, '-v'] - - with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ - pytest.raises(SystemExit) as sys_exit: - self.loader.exec_module(self.twister_module) - - out, err = capfd.readouterr() - sys.stdout.write(out) - sys.stderr.write(err) - regex_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)'] - matches = self._get_matches(err, regex_line) - print(matches) - assert str(sys_exit.value) == '0' - assert len(matches) > 0 @pytest.mark.parametrize( - 'test_path', + 'flags', TESTDATA_1, - ids=[ - 'double_v', - ] + ids=['not verbose', 'not verbose + debug', 'v', 'v + debug', 'vv', 'vv + debug'] ) - def test_double_v(self, capfd, out_path, test_path): - args = ['--outdir', out_path, '-T', test_path, '-vv'] + def test_output_levels(self, capfd, out_path, flags): + test_path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic') + args = ['--outdir', out_path, '-T', test_path, *flags] with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ pytest.raises(SystemExit) as sys_exit: @@ -189,12 +169,26 @@ def test_double_v(self, capfd, out_path, test_path): out, err = capfd.readouterr() sys.stdout.write(out) sys.stderr.write(err) - regex_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)'] - matches = self._get_matches(err, regex_line) - booting_zephyr_regex = re.compile(r'^DEBUG\s+-\s+([^*]+)\*\*\*\s+Booting\s+Zephyr\s+OS\s+build.*$', re.MULTILINE) - info_debug_line_regex = r'^\s*(INFO|DEBUG)' assert str(sys_exit.value) == '0' - assert re.search(booting_zephyr_regex, err) is not None - assert re.search(info_debug_line_regex, err) is not None - assert len(matches) > 0 + + regex_debug_line = r'^\s*DEBUG' + debug_matches = re.search(regex_debug_line, err, re.MULTILINE) + if '-d' in flags: + assert debug_matches is not None + else: + assert debug_matches is None + + # Summary requires verbosity > 1 + if '-vv' in flags: + assert 'Total test suites: ' in out + else: + assert 'Total test suites: ' not in out + + # Brief summary shows up only on verbosity 0 - instance-by-instance otherwise + regex_info_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)'] + info_matches = self._get_matches(err, regex_info_line) + if not any(f in flags for f in ['-v', '-vv']): + assert not info_matches + else: + assert info_matches diff --git a/scripts/tests/twister_blackbox/test_quarantine.py b/scripts/tests/twister_blackbox/test_quarantine.py index 1d04d6fbb7140b9..27a84cd6845adbf 100644 --- a/scripts/tests/twister_blackbox/test_quarantine.py +++ b/scripts/tests/twister_blackbox/test_quarantine.py @@ -14,6 +14,7 @@ import sys import json +# pylint: disable=duplicate-code from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -75,7 +76,7 @@ def test_quarantine_verify(self, out_path): def test_quarantine_list(self, capfd, out_path, test_path, test_platforms, quarantine_directory): args = ['--outdir', out_path, '-T', test_path] +\ ['--quarantine-list', quarantine_directory] + \ - ['-vv'] + \ + ['-vv', '-d'] + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms ) for val in pair]