twister: DeviceHandler improve DUT selection

Improve DUT selection at DeviceHandler: for each DUT it counts how many test instances have been failed on it during the current twister execution, so the next available DUT will be chosen ordering the eligible DUTs by less failures occured so far. The new selection mechanism should increase chances to retry failed tests on different DUTs, for instance to resolve ploblems when some DUTs have connectivity or HW issues slowing down test plan execution, or even block the execution when only one test suite runs whereas the same first DUT candidate in the list is not working and others were not chosen. Signed-off-by: Dmitrii Golovanov <[email protected]>
zephyrproject-rtos · Aug 1, 2024 · 89f455e · 89f455e
1 parent 248508f
commit 89f455e
Show file tree

Hide file tree

Showing 2 changed files with 79 additions and 6 deletions.
diff --git a/scripts/pylib/twister/twisterlib/handlers.py b/scripts/pylib/twister/twisterlib/handlers.py
@@ -458,14 +458,20 @@ def monitor_serial(self, ser, halt_event, harness):
     def device_is_available(self, instance):
         device = instance.platform.name
         fixture = instance.testsuite.harness_config.get("fixture")
-        dut_found = False
+        duts_found = []
 
         for d in self.duts:
             if fixture and fixture not in map(lambda f: f.split(sep=':')[0], d.fixtures):
                 continue
             if d.platform != device or (d.serial is None and d.serial_pty is None):
                 continue
-            dut_found = True
+            duts_found.append(d)
+
+        if not duts_found:
+            raise TwisterException(f"No device to serve as {device} platform.")
+
+        # Select an available DUT with less failures
+        for d in sorted(duts_found, key=lambda _dut: _dut.failures):
             d.lock.acquire()
             avail = False
             if d.available:
@@ -478,9 +484,6 @@ def device_is_available(self, instance):
             if avail:
                 return d
 
-        if not dut_found:
-            raise TwisterException(f"No device to serve as {device} platform.")
-
         return None
 
     def make_dut_available(self, dut):

diff --git a/scripts/tests/twister/test_handlers.py b/scripts/tests/twister/test_handlers.py
@@ -735,13 +735,15 @@ def test_devicehandler_monitor_serial(
                 fixtures=[],
                 platform='dummy_platform',
                 available=1,
+                failures=0,
                 counter_increment=mock.Mock(),
                 counter=0
             ),
             mock.Mock(
                 fixtures=['dummy fixture'],
                 platform='another_platform',
                 available=1,
+                failures=0,
                 counter_increment=mock.Mock(),
                 counter=0
             ),
@@ -751,6 +753,7 @@ def test_devicehandler_monitor_serial(
                 serial_pty=None,
                 serial=None,
                 available=1,
+                failures=0,
                 counter_increment=mock.Mock(),
                 counter=0
             ),
@@ -759,12 +762,73 @@ def test_devicehandler_monitor_serial(
                 platform='dummy_platform',
                 serial_pty=mock.Mock(),
                 available=1,
+                failures=0,
+                counter_increment=mock.Mock(),
+                counter=0
+            ),
+            mock.Mock(
+                fixtures=['dummy fixture'],
+                platform='dummy_platform',
+                serial_pty=mock.Mock(),
+                available=1,
+                failures=0,
                 counter_increment=mock.Mock(),
                 counter=0
             )
         ],
         3
     ),
+    (
+        'dummy_platform',
+        'dummy fixture',
+        [
+            mock.Mock(
+                fixtures=[],
+                platform='dummy_platform',
+                available=1,
+                failures=0,
+                counter_increment=mock.Mock(),
+                counter=0
+            ),
+            mock.Mock(
+                fixtures=['dummy fixture'],
+                platform='another_platform',
+                available=1,
+                failures=0,
+                counter_increment=mock.Mock(),
+                counter=0
+            ),
+            mock.Mock(
+                fixtures=['dummy fixture'],
+                platform='dummy_platform',
+                serial_pty=None,
+                serial=None,
+                available=1,
+                failures=0,
+                counter_increment=mock.Mock(),
+                counter=0
+            ),
+            mock.Mock(
+                fixtures=['dummy fixture'],
+                platform='dummy_platform',
+                serial_pty=mock.Mock(),
+                available=1,
+                failures=1,
+                counter_increment=mock.Mock(),
+                counter=0
+            ),
+            mock.Mock(
+                fixtures=['dummy fixture'],
+                platform='dummy_platform',
+                serial_pty=mock.Mock(),
+                available=1,
+                failures=0,
+                counter_increment=mock.Mock(),
+                counter=0
+            )
+        ],
+        4
+    ),
     (
         'dummy_platform',
         'dummy fixture',
@@ -780,27 +844,31 @@ def test_devicehandler_monitor_serial(
                 platform='dummy_platform',
                 serial_pty=mock.Mock(),
                 counter_increment=mock.Mock(),
+                failures=0,
                 available=0
             ),
             mock.Mock(
                 fixtures=['another fixture'],
                 platform='dummy_platform',
                 serial_pty=mock.Mock(),
                 counter_increment=mock.Mock(),
+                failures=0,
                 available=0
             ),
             mock.Mock(
                 fixtures=['dummy fixture'],
                 platform='dummy_platform',
                 serial=mock.Mock(),
                 counter_increment=mock.Mock(),
+                failures=0,
                 available=0
             ),
             mock.Mock(
                 fixtures=['another fixture'],
                 platform='dummy_platform',
                 serial=mock.Mock(),
                 counter_increment=mock.Mock(),
+                failures=0,
                 available=0
             )
         ],
@@ -811,7 +879,9 @@ def test_devicehandler_monitor_serial(
 @pytest.mark.parametrize(
     'platform_name, fixture, duts, expected',
     TESTDATA_10,
-    ids=['one good dut', 'exception - no duts', 'no available duts']
+    ids=['two good duts, select the first one',
+         'two duts, the first was failed once, select the second not failed',
+         'exception - no duts', 'no available duts']
 )
 def test_devicehandler_device_is_available(
     mocked_instance,