Finally sort out the flow

trafficlight/__init__.py

@@ -27,6 +27,7 @@
     adapter_shutdown,
     loop_check_for_new_tests,
     loop_cleanup_unresponsive_adapters,
+    loop_check_all_tests_done,
 )
 from trafficlight.internals.testsuite import TestSuite
 from trafficlight.store import add_testsuite, get_testsuites
@@ -113,6 +114,7 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart:
     async def startup() -> None:
         app.add_background_task(loop_cleanup_unresponsive_adapters)
         app.add_background_task(loop_check_for_new_tests)
+        app.add_background_task(loop_check_all_tests_done)
         if kiwi.kiwi_client:
             await kiwi.kiwi_client.start_run()
 
@@ -124,14 +126,26 @@ async def shutdown() -> None:
             await kiwi.kiwi_client.end_run()
         await adapter_shutdown()
 
-        print("Results:\n\n")
+        print("Results:\n")
+        exit_code = 0
+        total_tests = 0
+        successful_tests = 0
         for testsuite in get_testsuites():
             print(
-                f"{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful"
+                f"\n{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful"
             )
             for testcase in testsuite.test_cases:
                 print(f"  {testcase.client_types}: {testcase.state}")
+                total_tests += 1
+                if testcase.state != "success":
+                    exit_code = 1
+                else:
+                    successful_tests = successful_tests + 1
                 if testcase.state != "success" and testcase.state != "waiting":
-                    print(f"{testcase.exceptions}")
+                    for exception in testcase.exceptions:
+                        print(exception)
+
+        print(f"\nOverall: {successful_tests}/{total_tests} succeeded")
+        os._exit(exit_code)
 
     return app

trafficlight/http/adapter.py

@@ -34,6 +34,7 @@
     get_adapters,
     get_tests,
     remove_adapter,
+    get_testsuites,
 )
 
 IDLE_ADAPTER_UNRESPONSIVE_DELAY = timedelta(minutes=1)
@@ -125,9 +126,36 @@ async def interrupt_tasks() -> None:
         task.cancel()
 
 
+def should_finish_tests() -> bool:
+    for testsuite in get_testsuites():
+        for testcase in testsuite.test_cases:
+            if testcase.state not in ("failed", "error", "success"):
+                logger.info(f"Not exiting because of {testcase}")
+                return False
+    return True
+
+
+async def loop_check_all_tests_done() -> None:
+    while not stop_background_tasks:
+        logging.debug("Running check for test completion")
+        if should_finish_tests():
+            # do not await because shutdown() awaits all background tasks (inc this one) to shut down first.
+            asyncio.create_task(current_app.shutdown())
+
+        sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30))
+        try:
+            sleeping_tasks.add(sleep_task)
+            await sleep_task
+        except asyncio.CancelledError:
+            pass  # we don't mind this task being cancelled.
+        finally:
+            sleeping_tasks.remove(sleep_task)
+    logging.info("Termination task shutting down")
+
+
 async def loop_cleanup_unresponsive_adapters() -> None:
     while not stop_background_tasks:
-        logging.info("Running sweep for idle adapters")
+        logging.debug("Running sweep for idle adapters")
         await cleanup_unresponsive_adapters()
 
         sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30))
@@ -138,12 +166,12 @@ async def loop_cleanup_unresponsive_adapters() -> None:
             pass  # we don't mind this task being cancelled.
         finally:
             sleeping_tasks.remove(sleep_task)
-    logging.info("Finished sweep task")
+    logging.info("Sweep task shutting down")
 
 
 async def loop_check_for_new_tests() -> None:
     while not stop_background_tasks:
-        logging.info("Running sweep for new tests")
+        logging.debug("Running sweep for new tests")
         await check_for_new_tests()
         sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30))
         try:
@@ -153,7 +181,7 @@ async def loop_check_for_new_tests() -> None:
             pass  # we don't mind this task being cancelled.
         finally:
             sleeping_tasks.remove(sleep_task)
-    logging.info("Finished new test task")
+    logging.info("New test task shutting down")
 
 
 async def adapter_shutdown() -> None:

trafficlight/internals/client.py

@@ -290,6 +290,10 @@ async def get_lobby_data(self) -> LobbyData:
         snapshot_file = self.test_case.files[self.name + "_" + data["snapshot"]]
         invite_url = response["data"]["invite_url"]
         page_url = response["data"]["page_url"]
+        # Strip trailing & on page URLs until https://github.com/vector-im/element-call/issues/1639 is resolved
+        if page_url[-1] == "&":
+            page_url = page_url[1:-1]
+
         call_name = response["data"]["call_name"]
         lobby_data = LobbyData(
             video_muted=False,

trafficlight/tests/video/handle_invite_base.py

@@ -24,6 +24,9 @@ async def _run_test(
 
         await joiner.join_by_url(creator_lobby_data.invite_url)
 
+        # For now; wait a little so lobby data settles, because page dynamically updates the page_url
+        await asyncio.sleep(10)
+
         joiner_lobby_data = await joiner.get_lobby_data()
 
         with soft_assertions():