From c46531169dca36d38ec5a6037c28650be9bdc6ed Mon Sep 17 00:00:00 2001 From: marcoppenheimer <51744472+marcoppenheimer@users.noreply.github.com> Date: Thu, 29 Jun 2023 18:30:04 +0100 Subject: [PATCH] chore: core structure of base DataUpgrade lib and handlers for pre upgrade-charm (#64) --- lib/charms/data_platform_libs/v0/upgrade.py | 532 ++++++++++++++++++++ pyproject.toml | 37 ++ tests/unit/test_upgrade.py | 269 ++++++++++ 3 files changed, 838 insertions(+) create mode 100644 lib/charms/data_platform_libs/v0/upgrade.py create mode 100644 tests/unit/test_upgrade.py diff --git a/lib/charms/data_platform_libs/v0/upgrade.py b/lib/charms/data_platform_libs/v0/upgrade.py new file mode 100644 index 00000000..f83ebba5 --- /dev/null +++ b/lib/charms/data_platform_libs/v0/upgrade.py @@ -0,0 +1,532 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Handler for `upgrade` relation events for in-place upgrades on VMs.""" + +import json +import logging +from abc import ABC, abstractmethod +from typing import Optional, Union + +from ops.charm import ( + ActionEvent, + CharmBase, + RelationChangedEvent, + RelationCreatedEvent, + UpgradeCharmEvent, +) +from ops.framework import EventBase, Object +from ops.model import Relation +from pydantic import BaseModel, root_validator, validator + +# The unique Charmhub library identifier, never change it +LIBID = "156258aefb79435a93d933409a8c8684" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 1 + +logger = logging.getLogger(__name__) + +# --- DEPENDENCY RESOLUTION FUNCTIONS --- + + +def build_complete_sem_ver(version: str) -> list[int]: + """Builds complete major.minor.patch version from version string. + + Returns: + List of major.minor.patch version integers + """ + versions = [int(ver) if ver != "*" else 0 for ver in str(version).split(".")] + + # padding with 0s until complete major.minor.patch + return (versions + 3 * [0])[:3] + + +def verify_caret_requirements(version: str, requirement: str) -> bool: + """Verifies version requirements using carats. + + Args: + version: the version currently in use + requirement: the requirement version + + Returns: + True if `version` meets defined `requirement`. Otherwise False + """ + if not requirement.startswith("^"): + return True + + requirement = requirement[1:] + + sem_version = build_complete_sem_ver(version) + sem_requirement = build_complete_sem_ver(requirement) + + # caret uses first non-zero character, not enough to just count '. + max_version_index = requirement.count(".") + for i, semver in enumerate(sem_requirement): + if semver != 0: + max_version_index = i + break + + for i in range(3): + # version higher than first non-zero + if (i < max_version_index) and (sem_version[i] > sem_requirement[i]): + return False + + # version either higher or lower than first non-zero + if (i == max_version_index) and (sem_version[i] != sem_requirement[i]): + return False + + # valid + if (i > max_version_index) and (sem_version[i] > sem_requirement[i]): + return True + + return False + + +def verify_tilde_requirements(version: str, requirement: str) -> bool: + """Verifies version requirements using tildes. + + Args: + version: the version currently in use + requirement: the requirement version + + Returns: + True if `version` meets defined `requirement`. Otherwise False + """ + if not requirement.startswith("~"): + return True + + requirement = requirement[1:] + + sem_version = build_complete_sem_ver(version) + sem_requirement = build_complete_sem_ver(requirement) + + max_version_index = min(1, requirement.count(".")) + + for i in range(3): + # version higher before requirement level + if (i < max_version_index) and (sem_version[i] > sem_requirement[i]): + return False + + # version either higher or lower at requirement level + if (i == max_version_index) and (sem_version[i] != sem_requirement[i]): + return False + + # version lower after requirement level + if (i > max_version_index) and (sem_version[i] < sem_requirement[i]): + return False + + # must be valid + return True + + +def verify_wildcard_requirements(version: str, requirement: str) -> bool: + """Verifies version requirements using wildcards. + + Args: + version: the version currently in use + requirement: the requirement version + + Returns: + True if `version` meets defined `requirement`. Otherwise False + """ + if "*" not in requirement: + return True + + sem_version = build_complete_sem_ver(version) + sem_requirement = build_complete_sem_ver(requirement) + + max_version_index = requirement.count(".") + + for i in range(3): + # version not the same before wildcard + if (i < max_version_index) and (sem_version[i] != sem_requirement[i]): + return False + + # version not higher after wildcard + if (i == max_version_index) and (sem_version[i] < sem_requirement[i]): + return False + + # must be valid + return True + + +def verify_inequality_requirements(version: str, requirement: str) -> bool: + """Verifies version requirements using inequalities. + + Args: + version: the version currently in use + requirement: the requirement version + + Returns: + True if `version` meets defined `requirement`. Otherwise False + """ + if not any([char for char in [">", ">="] if requirement.startswith(char)]): + return True + + raw_requirement = requirement.replace(">", "").replace("=", "") + + sem_version = build_complete_sem_ver(version) + sem_requirement = build_complete_sem_ver(raw_requirement) + + max_version_index = raw_requirement.count(".") or 0 + + for i in range(3): + # valid at same requirement level + if ( + (i == max_version_index) + and ("=" in requirement) + and (sem_version[i] == sem_requirement[i]) + ): + return True + + # version not increased at any point + if sem_version[i] < sem_requirement[i]: + return False + + # valid + if sem_version[i] > sem_requirement[i]: + return True + + # must not be valid + return False + + +def verify_requirements(version: str, requirement: str) -> bool: + """Verifies a specified version against defined requirements. + + Supports caret (^), tilde (~), wildcard (*) and greater-than inequalities (>, >=) + + Args: + version: the version currently in use + requirement: the requirement version + + Returns: + True if `version` meets defined `requirement`. Otherwise False + """ + if not all( + [ + verify_inequality_requirements(version=version, requirement=requirement), + verify_caret_requirements(version=version, requirement=requirement), + verify_tilde_requirements(version=version, requirement=requirement), + verify_wildcard_requirements(version=version, requirement=requirement), + ] + ): + return False + + return True + + +# --- DEPENDENCY MODEL TYPES --- + + +class DependencyModel(BaseModel): + """Manager for a single dependency. + + To be used as part of another model representing a collection of arbitrary dependencies. + + Example Usage: + ```python + class KafkaDependenciesModel(BaseModel): + kafka_charm: DependencyModel + kafka_service: DependencyModel + + deps = { + "kafka_charm": { + "dependencies": {"zookeeper": ">5"}, + "name": "kafka", + "upgrade_supported": ">5", + "version": "10", + }, + "kafka_service": { + "dependencies": {"zookeeper": "^3.6"}, + "name": "kafka", + "upgrade_supported": "~3.3", + "version": "3.3.2", + }, + } + + # loading dict in to model + model = KafkaDependenciesModel(**deps) + + # exporting back validated deps + print(model.dict()) + ``` + """ + + dependencies: dict[str, str] + name: str + upgrade_supported: str + version: str + + @validator("dependencies", "upgrade_supported", each_item=True) + @classmethod + def dependencies_validator(cls, value): + """Validates values with dependencies for multiple special characters.""" + chars = ["~", "^", ">", "*"] + if (count := sum([value.count(char) for char in chars])) != 1: + raise ValueError( + f"Value uses greater than 1 special character (^ ~ > *). Found {count}." + ) + + return value + + @root_validator(skip_on_failure=True) + @classmethod + def version_upgrade_supported_validator(cls, values) -> dict[str, Union[dict[str, str], str]]: + """Validates specified `version` meets `upgrade_supported` requirement.""" + if not verify_requirements( + version=values.get("version"), requirement=values.get("upgrade_supported") + ): + raise ValueError( + f"upgrade_supported value {values.get('upgrade_supported')} greater than version value {values.get('version')} for {values.get('name')}." + ) + + return values + + # TODO: implement when comparing two dependency models for upgradability + def is_compatible(self, requirement: "DependencyModel"): + """Used for comparing two instances of `DependencyModel` for upgradability. + + Args: + requirement: the specific requiement to compare this dependency against + + Returns: + True if version is compatible with requirement. Otherwise False + """ + raise NotImplementedError + + +# --- CUSTOM EXCEPTIONS --- + + +class UpgradeError(Exception): + """Base class for upgrade related exceptions in the module.""" + + def __init__(self, message: str, cause: Optional[str], resolution: Optional[str]): + super().__init__(message) + self.message = message + self.cause = cause or "" + self.resolution = resolution or "" + + def __repr__(self): + """Representation of the UpgradeError class.""" + return f"{type(self).__module__}.{type(self).__name__} - {str(vars(self))}" + + def __str__(self): + """String representation of the UpgradeError class.""" + return repr(self) + + +class ClusterNotReadyError(UpgradeError): + """Exception flagging that the cluster is not ready to start upgrading. + + Args: + `message`: string message to be logged out + `cause`: short human-readable description of the cause of the error + `resolution`: short human-readable instructions for manual error resolution (optional) + """ + + def __init__(self, message: str, cause: str, resolution: Optional[str] = None): + super().__init__(message, cause=cause, resolution=resolution) + + +class UpgradeFailedError(UpgradeError): + """Exception flagging that something in the upgrade process failed, and should be aborted. + + Args: + `message`: string message to be logged out + `cause`: short human-readable description of the cause of the error + `resolution`: short human-readable instructions for manual solutions to the error + """ + + def __init__(self, message: str, cause: str, resolution: str): + super().__init__(message, cause=cause, resolution=resolution) + + +# --- CUSTOM EVENTS --- + + +class UpgradeGrantedEvent(EventBase): + """Used to tell units that they can process an upgrade.""" + + +# --- EVENT HANDLER --- + + +class DataUpgrade(Object, ABC): + """Manages `upgrade` relation operators for in-place upgrades.""" + + def __init__( + self, + charm: CharmBase, + dependency_model: BaseModel, + relation_name: str = "upgrade", + ): + super().__init__(charm, relation_name) + self.charm = charm + self.dependency_model = dependency_model + self.relation_name = relation_name + + # events + self.framework.observe( + self.charm.on[relation_name].relation_created, self._on_upgrade_created + ) + self.framework.observe( + self.charm.on[relation_name].relation_changed, self._on_upgrade_changed + ) + self.framework.observe(self.charm.on.upgrade_charm, self._on_upgrade_charm) + + # actions + self.framework.observe( + getattr(self.on, "pre_upgrade_check_action"), self._on_pre_upgrade_check_action + ) + + @property + def peer_relation(self) -> Optional[Relation]: + """The upgrade peer relation.""" + return self.charm.model.get_relation(self.relation_name) + + @property + def state(self) -> Optional[str]: + """The unit state from the upgrade peer relation.""" + if not self.peer_relation: + return None + + return self.peer_relation.data[self.charm.unit].get("state", None) + + @property + def stored_dependencies(self) -> Optional[BaseModel]: + """The application dependencies from the upgrade peer relation.""" + if not self.peer_relation: + return None + + if not (deps := self.peer_relation.data[self.charm.app].get("dependencies", "")): + return None + + return type(self.dependency_model)(**json.loads(deps)) + + @property + def upgrade_stack(self) -> Optional[list[int]]: + """The upgrade stack from the upgrade peer relation.""" + if not self.peer_relation: + return None + + return ( + json.loads(self.peer_relation.data[self.charm.app].get("upgrade-stack", "[]")) or None + ) + + @property + def upgrading_units(self) -> Optional[tuple[str, str]]: + """Check whether any peer units are currently upgrading. + + Returns: + Tuple of upgrading unit name and state. + """ + if not self.peer_relation: + return None + + for unit in set([self.charm.unit] + list(self.peer_relation.units)): + if (current_state := self.peer_relation.data[unit].get("state")) in { + "ready", + "upgrading", + "completed", + "failed", + }: + return (unit.name, current_state) + + return None + + @abstractmethod + def pre_upgrade_check(self) -> None: + """Runs necessary checks validating the cluster is in a healthy state to upgrade. + + Raises: + ClusterNotReadyError: if cluster is not ready to upgrade + """ + pass + + @abstractmethod + def build_upgrade_stack(self) -> list[int]: + """Builds ordered list of all application unit.ids to upgrade in. + + Returns: + List of integeter unit.ids, ordered by upgrade order + e.g [5, 2, 4, 1, 3] + """ + pass + + def _on_upgrade_created(self, _: RelationCreatedEvent) -> None: + """Handler for `upgrade-relation-created` events.""" + if not self.charm.unit.is_leader() or not self.peer_relation: + return + + # persisting dependencies to the relation data + self.peer_relation.data[self.charm.app].update( + {"dependencies": json.dumps(self.dependency_model.dict())} + ) + + def _on_pre_upgrade_check_action(self, event: ActionEvent): + """Handler for `pre-upgrade-check-action` events.""" + if not self.peer_relation: + event.fail(message="Could not find upgrade relation.") + return + + if not self.charm.unit.is_leader(): + event.fail(message="Action must be ran on the Juju leader.") + return + + # checking if upgrade in progress + if unit := self.upgrading_units: + event.fail(f"{unit[0]} is in {unit[1]} state and is currently upgrading.") + return + + try: + logger.info("Running pre-upgrade-check...") + self.pre_upgrade_check() + + logger.info("Building upgrade stack...") + upgrade_stack = self.build_upgrade_stack() + logger.info(f"Built upgrade stack of {upgrade_stack}") + except ClusterNotReadyError as e: + logger.error(e) + event.fail(message=e.message) + return + except Exception as e: + logger.error(e) + event.fail(message="Unknown error found.") + return + + logger.info("Setting upgrade_stack to peer relation data...") + self.peer_relation.data[self.charm.app].update( + {"upgrade-stack": json.dumps(upgrade_stack)} + ) + + logger.info("Marking units as waiting for upgrade-charm event...") + for unit in set([self.charm.unit] + list(self.peer_relation.units)): + self.peer_relation.data[unit].update({"state": "waiting"}) + + # TODO - implement + def _on_upgrade_changed(self, event: RelationChangedEvent): + """Handler for `upgrade-relation-changed` events.""" + raise NotImplementedError + + # TODO - implement + def _on_upgrade_charm(self, event: UpgradeCharmEvent): + """Handler for `upgrade-charm` events.""" + raise NotImplementedError diff --git a/pyproject.toml b/pyproject.toml index 4e9d36cf..cd45b707 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,3 +39,40 @@ docstring-convention = "google" copyright-check = "True" copyright-author = "Canonical Ltd." copyright-regexp = "Copyright\\s\\d{4}([-,]\\d{4})*\\s+%(author)s" + +[tool.ruff] +line-length = 99 +select = ["E", "W", "F", "C", "N", "D", "I001"] +extend-ignore = [ + "D203", + "D204", + "D213", + "D215", + "D400", + "D401", + "D404", + "D406", + "D407", + "D408", + "D409", + "D413", +] +ignore = ["E501", "D107"] +extend-exclude = ["__pycache__", "*.egg_info"] +per-file-ignores = {"tests/*" = ["D100","D101","D102","D103","D104", "E999"], "src/literals.py" = ["D101"]} +target-version="py310" +src = ["src", "tests"] + +[tool.ruff.mccabe] +max-complexity = 10 + +[tool.pyright] +include = ["src"] +extraPaths = ["./lib"] +pythonVersion = "3.10" +pythonPlatform = "All" +typeCheckingMode = "basic" +reportIncompatibleMethodOverride = false +reportImportCycles = false +reportMissingModuleSource = true +stubPath = "" diff --git a/tests/unit/test_upgrade.py b/tests/unit/test_upgrade.py new file mode 100644 index 00000000..85c4d646 --- /dev/null +++ b/tests/unit/test_upgrade.py @@ -0,0 +1,269 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +import pytest +from charms.data_platform_libs.v0.upgrade import ( + BaseModel, + DependencyModel, + build_complete_sem_ver, + verify_caret_requirements, + verify_inequality_requirements, + verify_tilde_requirements, + verify_wildcard_requirements, +) +from pydantic import ValidationError + + +class GandalfModel(BaseModel): + gandalf_the_white: DependencyModel + + +@pytest.mark.parametrize( + "version,output", + [ + ("0.0.24.0.4", [0, 0, 24]), + ("3.5.3", [3, 5, 3]), + ("0.3", [0, 3, 0]), + ("1.2", [1, 2, 0]), + ("3.5.*", [3, 5, 0]), + ("0.*", [0, 0, 0]), + ("1.*", [1, 0, 0]), + ("1.2.*", [1, 2, 0]), + ("*", [0, 0, 0]), + (1, [1, 0, 0]), + ], +) +def test_build_complete_sem_ver(version, output): + assert build_complete_sem_ver(version) == output + + +@pytest.mark.parametrize( + "requirement,version,output", + [ + ("~1.2.3", "1.2.2", True), + ("1.2.3", "1.2.2", True), + ("^1.2.3", "1.2.2", False), + ("^1.2.3", "1.3", True), + ("^1.2.3", "2.2.5", False), + ("^1.2", "1.2.2", True), + ("^1.2.3", "1.2", False), + ("^1.2.3", "2.2.5", False), + ("^1", "1.2.2", True), + ("^1", "1.6", True), + ("^1", "1.7.9", True), + ("^1", "0.6", False), + ("^1", "2", False), + ("^1", "2.3", False), + ("^0.2.3", "0.2.2", False), + ("^0.2.3", "0.2.5", True), + ("^0.2.3", "1.2.5", False), + ("^0.2.3", "0.3.6", False), + ("^0.0.3", "0.0.4", False), + ("^0.0.3", "0.0.2", False), + ("^0.0.3", "0.0", False), + ("^0.0.3", "0.3.6", False), + ("^0.0", "0.0.3", True), + ("^0.0", "0.1.0", False), + ("^0", "0.1.0", True), + ("^0", "0.3.6", True), + ("^0", "1.0.0", False), + ], +) +def test_verify_caret_requirements(requirement, version, output): + assert verify_caret_requirements(version=version, requirement=requirement) == output + + +@pytest.mark.parametrize( + "requirement,version,output", + [ + ("^1.2.3", "1.2.2", True), + ("1.2.3", "1.2.2", True), + ("~1.2.3", "1.2.2", False), + ("~1.2.3", "1.3.2", False), + ("~1.2.3", "1.3.5", False), + ("~1.2.3", "1.2.5", True), + ("~1.2.3", "1.2", False), + ("~1.2", "1.2", True), + ("~1.2", "1.6", False), + ("~1.2", "1.2.4", True), + ("~1.2", "1.1", False), + ("~1.2", "1.0.5", False), + ("~0.2", "0.2", True), + ("~0.2", "0.2.3", True), + ("~0.2", "0.3", False), + ("~1", "0.3", False), + ("~1", "1.3", True), + ("~1", "0.0.9", False), + ("~1", "0.9.9", False), + ("~1", "1.9.9", True), + ("~1", "1.7", True), + ("~1", "1", True), + ("~0", "1", False), + ("~0", "0.1", True), + ("~0", "0.5.9", True), + ], +) +def test_verify_tilde_requirements(requirement, version, output): + assert verify_tilde_requirements(version=version, requirement=requirement) == output + + +@pytest.mark.parametrize( + "requirement,version,output", + [ + ("~1", "1", True), + ("^0", "1", True), + ("0", "0.1", True), + ("*", "1.5.6", True), + ("*", "0.0.1", True), + ("*", "0.2.0", True), + ("*", "1.0.0", True), + ("1.*", "1.0.0", True), + ("1.*", "2.0.0", False), + ("1.*", "0.6.2", False), + ("1.2.*", "0.6.2", False), + ("1.2.*", "1.6.2", False), + ("1.2.*", "1.2.2", True), + ("1.2.*", "1.2.0", True), + ("1.2.*", "1.1.6", False), + ("1.2.*", "1.1.0", False), + ("0.2.*", "1.1.0", False), + ("0.2.*", "0.1.0", False), + ("0.2.*", "0.2.9", True), + ("0.2.*", "0.6.0", False), + ], +) +def test_verify_wildcard_requirements(requirement, version, output): + assert verify_wildcard_requirements(version=version, requirement=requirement) == output + + +@pytest.mark.parametrize( + "requirement,version,output", + [ + ("~1", "1", True), + ("^0", "1", True), + ("0", "0.1", True), + (">1", "1.8", True), + (">1", "8.8.0", True), + (">0", "8.8.0", True), + (">0", "0.0", False), + (">0", "0.0.0", False), + (">0", "0.0.1", True), + (">1.0", "1.0.0", False), + (">1.0", "1.0", False), + (">1.0", "1.5.6", True), + (">1.0", "2.0", True), + (">1.0", "0.0.4", False), + (">1.6", "1.3", False), + (">1.6", "1.3.8", False), + (">1.6", "1.35.8", True), + (">1.6.3", "1.7.8", True), + (">1.22.3", "1.7.8", False), + (">0.22.3", "1.7.8", True), + (">=1.0", "1.0.0", True), + (">=1.0", "1.0", True), + (">=0.2", "0.2", True), + (">=0.2.7", "0.2.7", True), + (">=1.0", "1.5.6", True), + (">=1", "1", True), + (">=1", "1.0", True), + (">=1", "1.0.0", True), + (">=1", "1.0.6", True), + (">=1", "0.0", False), + (">=1", "0.0.1", False), + (">=1.0", "2.0", True), + (">=1.0", "0.0.4", False), + (">=1.6", "1.3", False), + (">=1.6", "1.3.8", False), + (">=1.6", "1.35.8", True), + (">=1.6.3", "1.7.8", True), + (">=1.22.3", "1.7.8", False), + (">=0.22.3", "1.7.8", True), + ], +) +def test_verify_inequality_requirements(requirement, version, output): + assert verify_inequality_requirements(version=version, requirement=requirement) == output + + +def test_dependency_model_raises_for_incompatible_version(): + deps = { + "gandalf_the_white": { + "dependencies": {"gandalf_the_grey": ">5"}, + "name": "gandalf", + "upgrade_supported": ">5", + "version": "4", + }, + } + + with pytest.raises(ValidationError): + GandalfModel(**deps) + + +@pytest.mark.parametrize("value", ["saruman", "1.3", ""]) +def test_dependency_model_raises_for_bad_dependency(value): + deps = { + "gandalf_the_white": { + "dependencies": {"gandalf_the_grey": value}, + "name": "gandalf", + "upgrade_supported": ">6", + "version": "7", + }, + } + + with pytest.raises(ValidationError): + GandalfModel(**deps) + + +@pytest.mark.parametrize("value", ["balrog", "1.3", ""]) +def test_dependency_model_raises_for_bad_nested_dependency(value): + deps = { + "gandalf_the_white": { + "dependencies": {"gandalf_the_grey": "~1.0", "durin": value}, + "name": "gandalf", + "upgrade_supported": ">6", + "version": "7", + }, + } + + with pytest.raises(ValidationError): + GandalfModel(**deps) + + +@pytest.mark.parametrize("value", ["saruman", "1.3", ""]) +def test_dependency_model_raises_for_bad_upgrade_supported(value): + deps = { + "gandalf_the_white": { + "dependencies": {"gandalf_the_grey": ">5"}, + "name": "gandalf", + "upgrade_supported": value, + "version": "7", + }, + } + + with pytest.raises(ValidationError): + GandalfModel(**deps) + + +def test_dependency_model_succeeds(): + deps = { + "gandalf_the_white": { + "dependencies": {"gandalf_the_grey": ">5"}, + "name": "gandalf", + "upgrade_supported": ">1.2", + "version": "7", + }, + } + + GandalfModel(**deps) + + +def test_dependency_model_succeeds_nested(): + deps = { + "gandalf_the_white": { + "dependencies": {"gandalf_the_grey": "~1.0", "durin": "^1.2.5"}, + "name": "gandalf", + "upgrade_supported": ">1.2", + "version": "7", + }, + } + + GandalfModel(**deps)