Add archive old nova data pre-upgrade step (canonical#433)

Add an option to archive old nova data before upgrades.
This should improve performance of upgrading nova components,
as well as pave the way for purging (deleting) old data once archived.

Ref.

https://charmhub.io/nova-cloud-controller/actions#archive-data
https://docs.openstack.org/project-deploy-guide/charm-deployment-guide/wallaby/upgrade-openstack.html#archive-old-database-data
https://docs.openstack.org/nova/rocky/cli/nova-manage.html (see archive_deleted_rows )

cou/commands.py

@@ -93,6 +93,21 @@ def __call__(
         setattr(namespace, self.dest, cli_input)
 
 
+def batch_size_arg(value: str) -> int:
+    """Type converter for argparse.
+
+    :param value: input arg value to validate and convert
+    :type value: str
+    :return: the input value converted to an int
+    :rtype: int
+    :raises argparse.ArgumentTypeError: if integer is an invalid batch size
+    """
+    batch_size = int(value)
+    if batch_size <= 0:
+        raise argparse.ArgumentTypeError("batch size must be greater than 0")
+    return batch_size
+
+
 def get_subcommand_common_opts_parser() -> argparse.ArgumentParser:
     """Create a shared parser for options specific to subcommands.
 
@@ -124,6 +139,20 @@ def get_subcommand_common_opts_parser() -> argparse.ArgumentParser:
         action=argparse.BooleanOptionalAction,
         default=argparse.SUPPRESS,
     )
+    subcommand_common_opts_parser.add_argument(
+        "--archive",
+        help="Archive old database data (nova) before cloud upgrade.\n"
+        "Default to enabling archive.",
+        action=argparse.BooleanOptionalAction,
+        default=argparse.SUPPRESS,
+    )
+    subcommand_common_opts_parser.add_argument(
+        "--archive-batch-size",
+        help="Batch size for nova old database data archiving.\n"
+        "Decrease the batch size if performance issues are detected.\n(default: 1000)",
+        type=batch_size_arg,
+        default=1000,
+    )
     subcommand_common_opts_parser.add_argument(
         "--force",
         action="store_true",
@@ -382,6 +411,8 @@ class CLIargs:
     command: str
     verbosity: int = 0
     backup: bool = True
+    archive: bool = True
+    archive_batch_size: int = 1000
     quiet: bool = False
     force: bool = False
     auto_approve: bool = False

cou/steps/nova_cloud_controller.py

@@ -0,0 +1,89 @@
+# Copyright 2024 Canonical Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Functions for prereq steps relating to nova."""
+import logging
+
+from cou.exceptions import ApplicationNotFound, COUException, UnitNotFound
+from cou.utils.juju_utils import Model
+
+logger = logging.getLogger(__name__)
+
+
+async def archive(model: Model, *, batch_size: int) -> None:
+    """Archive data on a nova-cloud-controller unit.
+
+    The archive-data action only runs a single batch,
+    so we run it in a loop until completion.
+    See also:
+    https://charmhub.io/nova-cloud-controller/actions#archive-data
+    https://docs.openstack.org/project-deploy-guide/charm-deployment-guide/wallaby/upgrade-openstack.html#archive-old-database-data
+    :param model: juju model to work with
+    :type model: Model
+    :param batch_size: batch-size to pass to the archive-data action
+        (default is 1000; decrease if performance issues seen)
+    :type batch_size: int
+    :raises COUException: if action returned unexpected output
+    """  # noqa: E501 line too long
+    unit_name: str = await _get_nova_cloud_controller_unit_name(model)
+    # The archive-data action only archives a single batch,
+    # so we must run it in a loop until everything is archived.
+    while True:
+        logger.debug("Running action 'archive-data' on %s", unit_name)
+        # https://charmhub.io/nova-cloud-controller/actions#archive-data
+        action = await model.run_action(
+            unit_name=unit_name,
+            action_name="archive-data",
+            raise_on_failure=True,
+            action_params={"batch-size": batch_size},
+        )
+        logger.info("action output: %s", action.data)
+        output = action.data["results"].get("archive-deleted-rows")
+        if output is None:
+            raise COUException(
+                "Expected to find output in action results.'archive-deleted-rows', "
+                "but it was not present."
+            )
+        # The command will contain this string if there is nothing left to archive.
+        # This means we don't need to run the command any more.
+        if "Nothing was archived" in output:
+            logger.debug("Archiving complete.")
+            break
+        logger.debug("Potentially more data to archive...")
+
+
+async def _get_nova_cloud_controller_unit_name(model: Model) -> str:
+    """Get nova-cloud-controller application's first unit's name.
+
+    Assumes only a single nova-cloud-controller application is deployed.
+
+    :param model: juju model to work with
+    :type model: Model
+    :return: unit name
+    :rtype: str
+    :raises UnitNotFound: When cannot find a valid unit for 'nova-cloud-controller'
+    :raises ApplicationNotFound: When cannot find a 'nova-cloud-controller' application
+    """
+    status = await model.get_status()
+    for app_name, app_config in status.applications.items():
+        charm_name = await model.get_charm_name(app_name)
+        if charm_name == "nova-cloud-controller":
+            units = list(app_config.units.keys())
+            if units:
+                return units[0]
+            raise UnitNotFound(
+                f"Cannot find unit for 'nova-cloud-controller' in model '{model.name}'."
+            )
+
+    raise ApplicationNotFound(f"Cannot find 'nova-cloud-controller' in model '{model.name}'.")

cou/steps/plan.py

@@ -51,6 +51,7 @@
 from cou.steps.analyze import Analysis
 from cou.steps.backup import backup
 from cou.steps.hypervisor import HypervisorUpgradePlanner
+from cou.steps.nova_cloud_controller import archive
 from cou.utils.app_utils import set_require_osd_release_option
 from cou.utils.juju_utils import DEFAULT_TIMEOUT, Machine, Unit
 from cou.utils.nova_compute import get_empty_hypervisors
@@ -386,6 +387,16 @@ def _get_pre_upgrade_steps(analysis_result: Analysis, args: CLIargs) -> list[Pre
             )
         )
 
+    # Add a pre-upgrade step to archive old database data.
+    # This is a performance optimisation.
+    if args.archive:
+        steps.append(
+            PreUpgradeStep(
+                description="Archive old database data on nova-cloud-controller",
+                coro=archive(analysis_result.model, batch_size=args.archive_batch_size),
+            )
+        )
+
     return steps
 
 

docs/how-to/archive-old-data.rst

@@ -0,0 +1,62 @@
+==========================================
+Archive old data
+==========================================
+
+By default, **COU** plans for and runs an archive step
+before proceeding to actual upgrade steps.
+This can be turned off with the `--no-archive` flag.
+
+This archive step is a performance optimisation,
+moving data for soft deleted nova instances into a shadow table.
+
+The archiving is run in batches.
+The default batch size is 1000.
+On some clouds, it may be desirable to reduce the batch size to reduce database load.
+The batch size can be configured with `--archive-batch-size N`, where `N` is a positive integer.
+
+Usage examples
+--------------
+
+With a custom batch size:
+
+.. terminal::
+    :input: cou plan --archive-batch-size 200
+
+    Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'
+    Connected to 'test-model' ✔
+    Analyzing cloud... ✔
+    Generating upgrade plan... ✔
+    Upgrade cloud from 'ussuri' to 'victoria'
+        Verify that all OpenStack applications are in idle state
+        Back up MySQL databases
+        Archive old database data on nova-cloud-controller
+        Control Plane principal(s) upgrade plan
+        ...
+
+Disabling the archive step:
+
+.. terminal::
+    :input: cou plan --no-archive
+
+    Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'
+    Connected to 'test-model' ✔
+    Analyzing cloud... ✔
+    Generating upgrade plan... ✔
+    Upgrade cloud from 'ussuri' to 'victoria'
+        Verify that all OpenStack applications are in idle state
+        Back up MySQL databases
+        Control Plane principal(s) upgrade plan
+        ...
+
+More information
+----------------
+
+- `nova-cloud-controller charm actions`_
+- `nova-manage reference`_ - see `archive_deleted_rows` subcommand
+- OpenStack upgrade guide information on `archiving old database data`_
+
+
+.. LINKS
+.. _nova-cloud-controller charm actions: https://charmhub.io/nova-cloud-controller/actions
+.. _nova-manage reference: https://docs.openstack.org/nova/rocky/cli/nova-manage.html
+.. _archiving old database data: https://docs.openstack.org/project-deploy-guide/charm-deployment-guide/wallaby/upgrade-openstack.html#archive-old-database-data

docs/how-to/index.rst

@@ -15,4 +15,5 @@ are possible with **COU**.
    interruption
    configure-connection
    no-backup
+   archive-old-data
    verbosity

docs/how-to/no-backup.rst

@@ -20,6 +20,7 @@ Plan:
     Upgrade cloud from 'ussuri' to 'victoria'
         Verify that all OpenStack applications are in idle state
         # note that there's no backup step planned
+        Archive old database data on nova-cloud-controller
         Control Plane principal(s) upgrade plan
         Upgrade plan for 'rabbitmq-server' to 'victoria'
             Upgrade software packages of 'rabbitmq-server' from the current APT repositories

docs/how-to/plan-upgrade.rst

@@ -29,6 +29,7 @@ Output example
     Upgrade cloud from 'ussuri' to 'victoria'
         Verify that all OpenStack applications are in idle state
         Back up MySQL databases
+        Archive old database data on nova-cloud-controller
         Control Plane principal(s) upgrade plan
             Upgrade plan for 'keystone' to 'victoria'
                 Upgrade software packages of 'keystone' from the current APT repositories

docs/how-to/upgrade-cloud.rst

@@ -122,6 +122,7 @@ Usage example
     Upgrade cloud from 'ussuri' to 'victoria'
         Verify that all OpenStack applications are in idle state
         Back up MySQL databases
+        Archive old database data on nova-cloud-controller
         Control Plane principal(s) upgrade plan
         Upgrade plan for 'rabbitmq-server' to 'victoria'
             Upgrade software packages of 'rabbitmq-server' from the current APT repositories

tests/functional/tests/smoke.py

@@ -136,6 +136,7 @@ def generate_expected_plan(self, backup: bool = True) -> str:
             "Upgrade cloud from 'ussuri' to 'victoria'\n"
             "\tVerify that all OpenStack applications are in idle state\n"
             f"{backup_plan}"
+            "\tArchive old database data on nova-cloud-controller\n"
             "\tControl Plane principal(s) upgrade plan\n"
             "\t\tUpgrade plan for 'designate-bind' to 'victoria'\n"
             "\t\t\tUpgrade software packages of 'designate-bind' "
@@ -194,7 +195,14 @@ def test_upgrade(self) -> None:
             "Upgrade plan for 'mysql-innodb-cluster' to 'victoria'",
         ]
         result_before_upgrade = self.cou(
-            ["upgrade", "--model", self.model_name, "--no-backup", "--auto-approve"]
+            [
+                "upgrade",
+                "--model",
+                self.model_name,
+                "--no-backup",
+                "--no-archive",
+                "--auto-approve",
+            ]
         ).stdout
         for expected_msg in expected_msgs_before_upgrade:
             with self.subTest(expected_msg):
@@ -205,7 +213,9 @@ def test_upgrade(self) -> None:
             "Upgrade plan for 'designate-bind' to 'wallaby'",
             "Upgrade plan for 'mysql-innodb-cluster' to 'wallaby'",
         ]
-        result_after_upgrade = self.cou(["plan", "--model", self.model_name, "--no-backup"]).stdout
+        result_after_upgrade = self.cou(
+            ["plan", "--model", self.model_name, "--no-backup", "--no-archive"]
+        ).stdout
         for expected_msg in expected_msg_after_upgrade:
             with self.subTest(expected_msg):
                 self.assertIn(expected_msg, result_after_upgrade)

tests/mocked_plans/sample_plans/018346c5-f95c-46df-a34e-9a78bdec0018.yaml

@@ -2,6 +2,7 @@ plan: |
     Upgrade cloud from 'ussuri' to 'victoria'
         Verify that all OpenStack applications are in idle state
         Back up MySQL databases
+        Archive old database data on nova-cloud-controller
         Control Plane principal(s) upgrade plan
             Upgrade plan for 'rabbitmq-server' to 'victoria'
                 Upgrade software packages of 'rabbitmq-server' from the current APT repositories

tests/mocked_plans/sample_plans/9eb9af6a-b919-4cf9-8f2f-9df16a1556be.yaml

@@ -2,6 +2,7 @@ plan: |
     Upgrade cloud from 'ussuri' to 'victoria'
         Verify that all OpenStack applications are in idle state
         Back up MySQL databases
+        Archive old database data on nova-cloud-controller
         Control Plane principal(s) upgrade plan
             Upgrade plan for 'rabbitmq-server' to 'victoria'
                 Upgrade software packages of 'rabbitmq-server' from the current APT repositories

tests/mocked_plans/sample_plans/base.yaml

@@ -2,6 +2,7 @@ plan: |
     Upgrade cloud from 'ussuri' to 'victoria'
         Verify that all OpenStack applications are in idle state
         Back up MySQL databases
+        Archive old database data on nova-cloud-controller
         Control Plane principal(s) upgrade plan
             Upgrade plan for 'keystone' to 'victoria'
                 Upgrade software packages of 'keystone' from the current APT repositories