fix wait for idle on slow network

cou/steps/plan.py

@@ -46,6 +46,7 @@
 from cou.steps import PreUpgradeStep, UpgradePlan
 from cou.steps.analyze import Analysis
 from cou.steps.backup import backup
+from cou.utils.juju_utils import DEFAULT_TIMEOUT
 from cou.utils.openstack import LTS_TO_OS_RELEASE, OpenStackRelease
 
 logger = logging.getLogger(__name__)
@@ -77,7 +78,16 @@ async def generate_plan(analysis_result: Analysis, backup_database: bool) -> Upg
         PreUpgradeStep(
             description="Verify that all OpenStack applications are in idle state",
             parallel=False,
-            coro=analysis_result.model.wait_for_idle(timeout=11, idle_period=10),
+            coro=analysis_result.model.wait_for_idle(
+                # NOTE (rgildein): We need to DEFAULT_TIMEOUT so it's possible to change if
+                # a network is too slow, this could cause an issue.
+                # Since the default timeout is 10 seconds, we need to add 1 second to it to make
+                # sure that timeout is greater than idle_period. At the same time, we are using max
+                # function to ensure lower timeout used it here is always at least 11.
+                timeout=max(DEFAULT_TIMEOUT + 1, 11),
+                idle_period=10,
+                raise_on_blocked=True,
+            ),
         )
     )
     if backup_database:

cou/utils/juju_utils.py

@@ -24,7 +24,7 @@
 from juju.client._definitions import FullStatus
 from juju.client.connector import NoConnectionException
 from juju.client.jujudata import FileJujuData
-from juju.errors import JujuError
+from juju.errors import JujuAppError, JujuError, JujuUnitError
 from juju.model import Model
 from juju.unit import Unit
 from macaroonbakery.httpbakery import BakeryException
@@ -450,6 +450,7 @@ async def wait_for_idle(
         timeout: int,
         idle_period: int = DEFAULT_MODEL_IDLE_PERIOD,
         apps: Optional[list[str]] = None,
+        raise_on_blocked: bool = False,
     ) -> None:
         """Wait for applications to reach an idle state.
 
@@ -464,6 +465,9 @@ async def wait_for_idle(
         :type idle_period: int
         :param apps: Applications to wait, defaults to None
         :type apps: Optional[list[str]], optional
+        :param raise_on_blocked: If any unit or app going into "blocked" status immediately raises
+                                 WaitForApplicationsTimeout.
+        :type raise_on_blocked: bool
         """
 
         @retry(timeout=timeout, no_retry_exceptions=(WaitForApplicationsTimeout,))
@@ -472,14 +476,22 @@ async def _wait_for_idle() -> None:
             # NOTE(rgildein): Defining wrapper so we can use retry with proper timeout
             model = await self._get_model()
             try:
-                await model.wait_for_idle(apps=apps, timeout=timeout, idle_period=idle_period)
-            except asyncio.exceptions.TimeoutError as error:
+                await model.wait_for_idle(
+                    apps=apps,
+                    timeout=timeout,
+                    idle_period=idle_period,
+                    raise_on_blocked=raise_on_blocked,
+                )
+            except (asyncio.exceptions.TimeoutError, JujuAppError, JujuUnitError) as error:
                 # NOTE(rgildein): Catching TimeoutError raised as exception when wait_for_idle
                 # reached timeout. Also adding two spaces to make it more user friendly.
                 # example:
                 # Timed out waiting for model:
-                #   nova-compute/0 [executing] maintenance: Installing packages
-                msg = str(error).replace("\n", "\n  ")
+                #   rabbitmq-server/0 [idle] active: Unit is ready
+                #   neutron-api/0 [idle] active: Unit is ready
+                #   glance/0 [idle] active: Unit is ready
+                #   cinder/0 [idle] active: Unit is ready
+                msg = str(error).replace("\n", "\n  ", 1)
                 raise WaitForApplicationsTimeout(msg) from error
 
         if apps is None:

tests/unit/steps/test_steps_plan.py

@@ -149,7 +149,9 @@ async def test_generate_plan(apps, model):
         PreUpgradeStep(
             description="Verify that all OpenStack applications are in idle state",
             parallel=False,
-            coro=analysis_result.model.wait_for_idle(timeout=11, idle_period=10),
+            coro=analysis_result.model.wait_for_idle(
+                timeout=11, idle_period=10, raise_on_blocked=True
+            ),
         )
     )
     expected_plan.add_step(

tests/unit/utils/test_juju_utils.py

@@ -482,7 +482,10 @@ async def test_coumodel_wait_for_idle(mock_get_supported_apps, mocked_model):
     await model.wait_for_idle(timeout)
 
     mocked_model.wait_for_idle.assert_awaited_once_with(
-        apps=["app1", "app2"], timeout=timeout, idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD
+        apps=["app1", "app2"],
+        timeout=timeout,
+        idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD,
+        raise_on_blocked=False,
     )
     mock_get_supported_apps.assert_awaited_once_with()
 
@@ -497,7 +500,10 @@ async def test_coumodel_wait_for_idle_apps(mock_get_supported_apps, mocked_model
     await model.wait_for_idle(timeout, apps=["app1"])
 
     mocked_model.wait_for_idle.assert_awaited_once_with(
-        apps=["app1"], timeout=timeout, idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD
+        apps=["app1"],
+        timeout=timeout,
+        idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD,
+        raise_on_blocked=False,
     )
     mock_get_supported_apps.assert_not_awaited()
 
@@ -515,6 +521,9 @@ async def test_coumodel_wait_for_idle_timeout(mock_get_supported_apps, mocked_mo
         await model.wait_for_idle(timeout, apps=exp_apps)
 
     mocked_model.wait_for_idle.assert_awaited_once_with(
-        apps=exp_apps, timeout=timeout, idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD
+        apps=exp_apps,
+        timeout=timeout,
+        idle_period=juju_utils.DEFAULT_MODEL_IDLE_PERIOD,
+        raise_on_blocked=False,
     )
     mock_get_supported_apps.assert_not_awaited()