scripts: twister: decouple debug and verbosity

Currently, debug logging in the console and verbosity
are tightly coupled - verbosity of level 2 and higher
enables logging at the debug level.

This change introduces a separate Twister flag
responsible for controlling the debug logging,
while leaving the rest of verbosity unchanged.

This allows for controlling the verbosity on
both logging levels, according to one's needs.

Signed-off-by: Lukasz Mrugala <[email protected]>

scripts/pylib/twister/twisterlib/environment.py

@@ -717,8 +717,13 @@ def add_parse_arguments(parser = None):
         "--verbose",
         action="count",
         default=0,
-        help="Emit debugging information, call multiple times to increase "
-             "verbosity.")
+        help="Call multiple times to increase verbosity.")
+
+    parser.add_argument(
+        "-d",
+        "--debug",
+        action="store_true",
+        help="Emit debugging information.")
 
     parser.add_argument("-W", "--disable-warnings-as-errors", action="store_true",
                         help="Do not treat warning conditions as errors.")

scripts/pylib/twister/twisterlib/twister_main.py

@@ -26,7 +26,7 @@
 logger.setLevel(logging.DEBUG)
 
 
-def setup_logging(outdir, log_file, verbose, timestamps):
+def setup_logging(outdir, log_file, debug, timestamps):
     # create file handler which logs even debug messages
     if log_file:
         fh = logging.FileHandler(log_file)
@@ -38,7 +38,7 @@ def setup_logging(outdir, log_file, verbose, timestamps):
     # create console handler with a higher log level
     ch = logging.StreamHandler()
 
-    if verbose > 1:
+    if debug:
         ch.setLevel(logging.DEBUG)
     else:
         ch.setLevel(logging.INFO)
@@ -108,7 +108,7 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace):
             fp.write(previous_results)
 
     VERBOSE = options.verbose
-    setup_logging(options.outdir, options.log_file, VERBOSE, options.timestamps)
+    setup_logging(options.outdir, options.log_file, options.debug, options.timestamps)
 
     env = TwisterEnv(options, default_options)
     env.discover()

scripts/tests/twister_blackbox/test_coverage.py

@@ -189,7 +189,7 @@ def test_coverage(self, capfd, test_path, test_platforms, out_path, file_name):
     )
     def test_enable_coverage(self, capfd, test_path, test_platforms, out_path, expected):
         args = ['-i','--outdir', out_path, '-T', test_path] + \
-               ['--enable-coverage', '-vv'] + \
+               ['--enable-coverage', '-vv', '-d'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

scripts/tests/twister_blackbox/test_disable.py

@@ -76,7 +76,7 @@ def teardown_class(cls):
     def test_disable_suite_name_check(self, capfd, out_path, test_path, test_platforms, flag, expected, expected_none):
         args = ['-i', '--outdir', out_path, '-T', test_path] + \
                [flag] + \
-               ['-vv'] + \
+               ['-vv', '-d'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

scripts/tests/twister_blackbox/test_filter.py

@@ -191,7 +191,7 @@ def test_enable_slow_only(self, out_path):
     def test_arch(self, capfd, out_path, arch, expected):
         path = os.path.join(TEST_DATA, 'tests', 'no_filter')
         test_platforms = ['qemu_x86', 'hsdk', 'intel_adl_crb', 'it8xxx2_evb']
-        args = ['--outdir', out_path, '-T', path, '-vv'] + \
+        args = ['--outdir', out_path, '-T', path, '-vv', '-d'] + \
                ['--arch', arch] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
@@ -224,7 +224,7 @@ def test_arch(self, capfd, out_path, arch, expected):
     def test_vendor(self, capfd, out_path, vendor, expected):
         path = os.path.join(TEST_DATA, 'tests', 'no_filter')
         test_platforms = ['qemu_x86', 'hsdk', 'intel_adl_crb', 'it8xxx2_evb']
-        args = ['--outdir', out_path, '-T', path, '-vv'] + \
+        args = ['--outdir', out_path, '-T', path, '-vv', '-d'] + \
                ['--vendor', vendor] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms

scripts/tests/twister_blackbox/test_output.py

@@ -14,18 +14,21 @@
 import sys
 import json
 
-# pylint: disable=no-name-in-module
 from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test
 from twisterlib.testplan import TestPlan
 
 
 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
 class TestOutput:
     TESTDATA_1 = [
-    (
-        os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic')
-    ),
-]
+        ([]),
+        (['-d']),
+        (['-v']),
+        (['-v', '-d']),
+        (['-vv']),
+        (['-vv', '-d']),
+    ]
+
     @classmethod
     def setup_class(cls):
         apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
@@ -149,38 +152,15 @@ def _get_matches(self, err, regex_line):
                     matches = []
         return matches
 
-    @pytest.mark.parametrize(
-        'test_path',
-        TESTDATA_1,
-        ids=[
-            'single_v',
-        ]
-    )
-    def test_single_v(self, capfd, out_path, test_path):
-        args = ['--outdir', out_path, '-T', test_path, '-v']
-
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-            pytest.raises(SystemExit) as sys_exit:
-            self.loader.exec_module(self.twister_module)
-
-        out, err = capfd.readouterr()
-        sys.stdout.write(out)
-        sys.stderr.write(err)
-        regex_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)']
-        matches = self._get_matches(err, regex_line)
-        print(matches)
-        assert str(sys_exit.value) == '0'
-        assert len(matches) > 0
 
     @pytest.mark.parametrize(
-        'test_path',
+        'flags',
         TESTDATA_1,
-        ids=[
-            'double_v',
-        ]
+        ids=['not verbose', 'not verbose + debug', 'v', 'v + debug', 'vv', 'vv + debug']
     )
-    def test_double_v(self, capfd, out_path, test_path):
-        args = ['--outdir', out_path, '-T', test_path, '-vv']
+    def test_output_levels(self, capfd, out_path, flags):
+        test_path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic')
+        args = ['--outdir', out_path, '-T', test_path, *flags]
 
         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
             pytest.raises(SystemExit) as sys_exit:
@@ -189,12 +169,26 @@ def test_double_v(self, capfd, out_path, test_path):
         out, err = capfd.readouterr()
         sys.stdout.write(out)
         sys.stderr.write(err)
-        regex_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)']
-        matches = self._get_matches(err, regex_line)
-        booting_zephyr_regex = re.compile(r'^DEBUG\s+-\s+([^*]+)\*\*\*\s+Booting\s+Zephyr\s+OS\s+build.*$', re.MULTILINE)
-        info_debug_line_regex = r'^\s*(INFO|DEBUG)'
 
         assert str(sys_exit.value) == '0'
-        assert re.search(booting_zephyr_regex, err) is not None
-        assert re.search(info_debug_line_regex, err) is not None
-        assert len(matches) > 0
+
+        regex_debug_line = r'^\s*DEBUG'
+        debug_matches = re.search(regex_debug_line, err, re.MULTILINE)
+        if '-d' in flags:
+            assert debug_matches is not None
+        else:
+            assert debug_matches is None
+
+        # Summary requires verbosity > 1
+        if '-vv' in flags:
+            assert 'Total test suites: ' in out
+        else:
+            assert 'Total test suites: ' not in out
+
+        # Brief summary shows up only on verbosity 0 - instance-by-instance otherwise
+        regex_info_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)']
+        info_matches = self._get_matches(err, regex_info_line)
+        if not any(f in flags for f in ['-v', '-vv']):
+            assert not info_matches
+        else:
+            assert info_matches

scripts/tests/twister_blackbox/test_quarantine.py

@@ -14,6 +14,7 @@
 import sys
 import json
 
+# pylint: disable=duplicate-code
 from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
 from twisterlib.testplan import TestPlan
 
@@ -75,7 +76,7 @@ def test_quarantine_verify(self, out_path):
     def test_quarantine_list(self, capfd, out_path, test_path, test_platforms, quarantine_directory):
         args = ['--outdir', out_path, '-T', test_path] +\
                ['--quarantine-list', quarantine_directory] + \
-               ['-vv'] + \
+               ['-vv', '-d'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]