fix wait for idle on slow network

cou/steps/plan.py

@@ -77,7 +77,9 @@ async def generate_plan(analysis_result: Analysis, backup_database: bool) -> Upg
         PreUpgradeStep(
             description="Verify that all OpenStack applications are in idle state",
             parallel=False,
-            coro=analysis_result.model.wait_for_idle(timeout=11, idle_period=10),
+            coro=analysis_result.model.wait_for_idle(
+                timeout=11, idle_period=10, raise_on_blocked=True
+            ),
         )
     )
     if backup_database:

cou/utils/juju_utils.py

@@ -24,7 +24,7 @@
 from juju.client._definitions import FullStatus
 from juju.client.connector import NoConnectionException
 from juju.client.jujudata import FileJujuData
-from juju.errors import JujuError
+from juju.errors import JujuAppError, JujuError, JujuUnitError
 from juju.model import Model
 from juju.unit import Unit
 from macaroonbakery.httpbakery import BakeryException
@@ -450,6 +450,7 @@ async def wait_for_idle(
         timeout: int,
         idle_period: int = DEFAULT_MODEL_IDLE_PERIOD,
         apps: Optional[list[str]] = None,
+        raise_on_blocked: bool = False,
     ) -> None:
         """Wait for applications to reach an idle state.
 
@@ -464,6 +465,9 @@ async def wait_for_idle(
         :type idle_period: int
         :param apps: Applications to wait, defaults to None
         :type apps: Optional[list[str]], optional
+        :param raise_on_blocked: If any unit or app going into "blocked" status immediately raises
+                                 WaitForApplicationsTimeout.
+        :type raise_on_blocked: bool
         """
 
         @retry(timeout=timeout, no_retry_exceptions=(WaitForApplicationsTimeout,))
@@ -472,13 +476,21 @@ async def _wait_for_idle() -> None:
             # NOTE(rgildein): Defining wrapper so we can use retry with proper timeout
             model = await self._get_model()
             try:
-                await model.wait_for_idle(apps=apps, timeout=timeout, idle_period=idle_period)
-            except asyncio.exceptions.TimeoutError as error:
+                await model.wait_for_idle(
+                    apps=apps,
+                    timeout=timeout,
+                    idle_period=idle_period,
+                    raise_on_blocked=raise_on_blocked,
+                )
+            except (asyncio.exceptions.TimeoutError, JujuAppError, JujuUnitError) as error:
                 # NOTE(rgildein): Catching TimeoutError raised as exception when wait_for_idle
                 # reached timeout. Also adding two spaces to make it more user friendly.
                 # example:
                 # Timed out waiting for model:
-                #   nova-compute/0 [executing] maintenance: Installing packages
+                #   rabbitmq-server/0 [idle] active: Unit is ready
+                #   neutron-api/0 [idle] active: Unit is ready
+                #   glance/0 [idle] active: Unit is ready
+                #   cinder/0 [idle] active: Unit is ready
                 msg = str(error).replace("\n", "\n  ", 1)
                 raise WaitForApplicationsTimeout(msg) from error
 

tests/unit/steps/test_steps_plan.py

@@ -149,7 +149,9 @@ async def test_generate_plan(apps, model):
         PreUpgradeStep(
             description="Verify that all OpenStack applications are in idle state",
             parallel=False,
-            coro=analysis_result.model.wait_for_idle(timeout=11, idle_period=10),
+            coro=analysis_result.model.wait_for_idle(
+                timeout=11, idle_period=10, raise_on_blocked=True
+            ),
         )
     )
     expected_plan.add_step(

tests/unit/utils/test_juju_utils.py

@@ -482,7 +482,10 @@ async def test_coumodel_wait_for_idle(mock_get_supported_apps, mocked_model):
     await model.wait_for_idle(timeout)
 
     mocked_model.wait_for_idle.assert_awaited_once_with(
-        apps=["app1", "app2"], timeout=timeout, idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD
+        apps=["app1", "app2"],
+        timeout=timeout,
+        idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD,
+        raise_on_blocked=False,
     )
     mock_get_supported_apps.assert_awaited_once_with()
 
@@ -497,7 +500,10 @@ async def test_coumodel_wait_for_idle_apps(mock_get_supported_apps, mocked_model
     await model.wait_for_idle(timeout, apps=["app1"])
 
     mocked_model.wait_for_idle.assert_awaited_once_with(
-        apps=["app1"], timeout=timeout, idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD
+        apps=["app1"],
+        timeout=timeout,
+        idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD,
+        raise_on_blocked=False,
     )
     mock_get_supported_apps.assert_not_awaited()
 
@@ -515,6 +521,9 @@ async def test_coumodel_wait_for_idle_timeout(mock_get_supported_apps, mocked_mo
         await model.wait_for_idle(timeout, apps=exp_apps)
 
     mocked_model.wait_for_idle.assert_awaited_once_with(
-        apps=exp_apps, timeout=timeout, idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD
+        apps=exp_apps,
+        timeout=timeout,
+        idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD,
+        raise_on_blocked=False,
     )
     mock_get_supported_apps.assert_not_awaited()