twister: DeviceHandler count test failures on DUTs

Twister DeviceHandler - add test failure counter for how many
test instances have been failed on each DUT (Device Under Test)
when it executes the current test plan.
Output DUT falure counter summary at the end of Twister run.

Signed-off-by: Dmitrii Golovanov <[email protected]>

scripts/pylib/twister/twisterlib/handlers.py

@@ -473,7 +473,7 @@ def device_is_available(self, instance):
                 d.counter_increment()
                 avail = True
                 logger.debug(f"Retain DUT:{d.platform}, Id:{d.id}, "
-                             f"counter:{d.counter}")
+                             f"counter:{d.counter}, failures:{d.failures}")
             d.lock.release()
             if avail:
                 return d
@@ -484,8 +484,10 @@ def device_is_available(self, instance):
         return None
 
     def make_dut_available(self, dut):
+        if self.instance.status in ["error", "failed"]:
+            dut.failures_increment()
         logger.debug(f"Release DUT:{dut.platform}, Id:{dut.id}, "
-                     f"counter:{dut.counter}")
+                     f"counter:{dut.counter}, failures:{dut.failures}")
         dut.available = 1
 
     @staticmethod

scripts/pylib/twister/twisterlib/hardwaremap.py

@@ -58,6 +58,7 @@ def __init__(self,
         self.serial_pty = serial_pty
         self._counter = Value("i", 0)
         self._available = Value("i", 1)
+        self._failures = Value("i", 0)
         self.connected = connected
         self.pre_script = pre_script
         self.id = id
@@ -100,9 +101,23 @@ def counter_increment(self, value=1):
         with self._counter.get_lock():
             self._counter.value += value
 
+    @property
+    def failures(self):
+        with self._failures.get_lock():
+            return self._failures.value
+
+    @failures.setter
+    def failures(self, value):
+        with self._failures.get_lock():
+            self._failures.value = value
+
+    def failures_increment(self, value=1):
+        with self._failures.get_lock():
+            self._failures.value += value
+
     def to_dict(self):
         d = {}
-        exclude = ['_available', '_counter', 'match']
+        exclude = ['_available', '_counter', '_failures', 'match']
         v = vars(self)
         for k in v.keys():
             if k not in exclude and v[k]:
@@ -208,10 +223,10 @@ def discover(self):
     def summary(self, selected_platforms):
         print("\nHardware distribution summary:\n")
         table = []
-        header = ['Board', 'ID', 'Counter']
+        header = ['Board', 'ID', 'Counter', 'Failures']
         for d in self.duts:
             if d.connected and d.platform in selected_platforms:
-                row = [d.platform, d.id, d.counter]
+                row = [d.platform, d.id, d.counter, d.failures]
                 table.append(row)
         print(tabulate(table, headers=header, tablefmt="github"))
 

scripts/tests/twister/test_handlers.py

@@ -1186,6 +1186,7 @@ def mock_serial(*args, **kwargs):
 
     dut = DUT()
     dut.available = 0
+    dut.failures = 0
 
     hardware_baud = 14400
     flash_timeout = 60
@@ -1200,11 +1201,13 @@ def mock_serial(*args, **kwargs):
 
     if expected_result:
         assert result is not None
+        assert dut.failures == 0
 
     if expected_exception:
         assert handler.instance.status == 'failed'
         assert handler.instance.reason == 'Serial Device Error'
         assert dut.available == 1
+        assert dut.failures == 1
         missing_mock.assert_called_once_with('blocked', 'Serial Device Error')
 
     if terminate_ser_pty_process:

scripts/tests/twister/test_hardwaremap.py

@@ -213,10 +213,10 @@ def test_hardwaremap_summary(capfd, mocked_hm):
     expected = """
 Hardware distribution summary:
 
-| Board   |   ID |   Counter |
-|---------|------|-----------|
-| p1      |    1 |         0 |
-| p7      |    7 |         0 |
+| Board   |   ID |   Counter |   Failures |
+|---------|------|-----------|------------|
+| p1      |    1 |         0 |          0 |
+| p7      |    7 |         0 |          0 |
 """
 
     out, err = capfd.readouterr()