[nrf fromtree] twister: pytest: Improve reporting failed pytest scena…

…rios

When pytest scenario fails, then 'handler.log' is printed.
Changed to print 'twister_harness.log' that is a log from
pytest execution. That file tells much more when test fails.

Signed-off-by: Grzegorz Chwierut <[email protected]>
(cherry picked from commit e466b7a)

scripts/pylib/pytest-twister-harness/src/twister_harness/device/hardware_adapter.py

@@ -116,7 +116,6 @@ def _flash_and_run(self) -> None:
                 stdout_decoded = stdout.decode(errors='ignore')
                 with open(self.device_log_path, 'a+') as log_file:
                     log_file.write(stdout_decoded)
-                logger.debug(f'Flashing output:\n{stdout_decoded}')
             if self.device_config.post_flash_script:
                 self._run_custom_script(self.device_config.post_flash_script, self.base_timeout)
             if process is not None and process.returncode == 0:

scripts/pylib/twister/twisterlib/harness.py

@@ -400,8 +400,10 @@ def _parse_report_file(self, report):
         if elem_ts := root.find('testsuite'):
             if elem_ts.get('failures') != '0':
                 self.state = 'failed'
+                self.instance.reason = f"{elem_ts.get('failures')}/{elem_ts.get('tests')} pytest scenario(s) failed"
             elif elem_ts.get('errors') != '0':
                 self.state = 'error'
+                self.instance.reason = 'Error during pytest execution'
             elif elem_ts.get('skipped') == elem_ts.get('tests'):
                 self.state = 'skipped'
             else:

scripts/pylib/twister/twisterlib/reports.py

@@ -246,6 +246,7 @@ def json_report(self, filename, version="NA"):
         for instance in self.instances.values():
             suite = {}
             handler_log = os.path.join(instance.build_dir, "handler.log")
+            pytest_log = os.path.join(instance.build_dir, "twister_harness.log")
             build_log = os.path.join(instance.build_dir, "build.log")
             device_log = os.path.join(instance.build_dir, "device.log")
 
@@ -284,7 +285,9 @@ def json_report(self, filename, version="NA"):
                 suite['status'] = instance.status
                 suite["reason"] = instance.reason
                 # FIXME
-                if os.path.exists(handler_log):
+                if os.path.exists(pytest_log):
+                    suite["log"] = self.process_log(pytest_log)
+                elif os.path.exists(handler_log):
                     suite["log"] = self.process_log(handler_log)
                 elif os.path.exists(device_log):
                     suite["log"] = self.process_log(device_log)

scripts/pylib/twister/twisterlib/runner.py

@@ -40,6 +40,9 @@
 
 from twisterlib.log_helper import log_command
 from twisterlib.testinstance import TestInstance
+from twisterlib.environment import TwisterEnv
+from twisterlib.testsuite import TestSuite
+from twisterlib.platform import Platform
 from twisterlib.testplan import change_skip_to_error_if_integration
 from twisterlib.harness import HarnessImporter, Pytest
 
@@ -220,7 +223,7 @@ class CMake:
     config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
     dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
 
-    def __init__(self, testsuite, platform, source_dir, build_dir, jobserver):
+    def __init__(self, testsuite: TestSuite, platform: Platform, source_dir, build_dir, jobserver):
 
         self.cwd = None
         self.capture_output = True
@@ -414,7 +417,7 @@ def run_cmake(self, args="", filter_stages=[]):
 
 class FilterBuilder(CMake):
 
-    def __init__(self, testsuite, platform, source_dir, build_dir, jobserver):
+    def __init__(self, testsuite: TestSuite, platform: Platform, source_dir, build_dir, jobserver):
         super().__init__(testsuite, platform, source_dir, build_dir, jobserver)
 
         self.log = "config-twister.log"
@@ -517,7 +520,7 @@ def parse_generated(self, filter_stages=[]):
 
 class ProjectBuilder(FilterBuilder):
 
-    def __init__(self, instance, env, jobserver, **kwargs):
+    def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs):
         super().__init__(instance.testsuite, instance.platform, instance.testsuite.source_dir, instance.build_dir, jobserver)
 
         self.log = "build.log"
@@ -527,8 +530,7 @@ def __init__(self, instance, env, jobserver, **kwargs):
         self.env = env
         self.duts = None
 
-    @staticmethod
-    def log_info(filename, inline_logs):
+    def log_info(self, filename, inline_logs, log_testcases=False):
         filename = os.path.abspath(os.path.realpath(filename))
         if inline_logs:
             logger.info("{:-^100}".format(filename))
@@ -542,6 +544,17 @@ def log_info(filename, inline_logs):
             logger.error(data)
 
             logger.info("{:-^100}".format(filename))
+
+            if log_testcases:
+                for tc in self.instance.testcases:
+                    if not tc.reason:
+                        continue
+                    logger.info(
+                        f"\n{str(tc.name).center(100, '_')}\n"
+                        f"{tc.reason}\n"
+                        f"{100*'_'}\n"
+                        f"{tc.output}"
+                    )
         else:
             logger.error("see: " + Fore.YELLOW + filename + Fore.RESET)
 
@@ -551,9 +564,12 @@ def log_info_file(self, inline_logs):
         b_log = "{}/build.log".format(build_dir)
         v_log = "{}/valgrind.log".format(build_dir)
         d_log = "{}/device.log".format(build_dir)
+        pytest_log = "{}/twister_harness.log".format(build_dir)
 
         if os.path.exists(v_log) and "Valgrind" in self.instance.reason:
             self.log_info("{}".format(v_log), inline_logs)
+        elif os.path.exists(pytest_log) and os.path.getsize(pytest_log) > 0:
+            self.log_info("{}".format(pytest_log), inline_logs, log_testcases=True)
         elif os.path.exists(h_log) and os.path.getsize(h_log) > 0:
             self.log_info("{}".format(h_log), inline_logs)
         elif os.path.exists(d_log) and os.path.getsize(d_log) > 0:

scripts/tests/twister/pytest_integration/test_harness_pytest.py

@@ -188,6 +188,8 @@ def test_err():
         assert tc.status == "failed"
         assert tc.output
         assert tc.reason
+    assert testinstance.reason
+    assert '2/2' in testinstance.reason
 
 
 def test_if_report_with_skip(pytester, testinstance: TestInstance):