diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 844244d4..498ea6ae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,6 +76,11 @@ jobs: {juju-snap-channel: "3.1/stable", juju-bootstrap-option: "3.1.6", libjuju-version: "3.2.2"}} + - {tox-environments: integration-secrets, + juju-version: + {juju-snap-channel: "3.1/stable", + juju-bootstrap-option: "3.1.6", + libjuju-version: "3.2.2"}} name: ${{ matrix.tox-environments }} Juju ${{ matrix.juju-version.juju-snap-channel}} -- libjuju ${{ matrix.juju-version.libjuju-version }} needs: - lint diff --git a/lib/charms/data_platform_libs/v0/data_secrets.py b/lib/charms/data_platform_libs/v0/data_secrets.py new file mode 100644 index 00000000..254b9af3 --- /dev/null +++ b/lib/charms/data_platform_libs/v0/data_secrets.py @@ -0,0 +1,143 @@ +"""Secrets related helper classes/functions.""" +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +from typing import Dict, Literal, Optional + +from ops import Secret, SecretInfo +from ops.charm import CharmBase +from ops.model import SecretNotFoundError + +# The unique Charmhub library identifier, never change it +LIBID = "d77fb3d01aba41ed88e837d0beab6be5" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 1 + + +APP_SCOPE = "app" +UNIT_SCOPE = "unit" +Scopes = Literal["app", "unit"] + + +class DataSecretsError(Exception): + """A secret that we want to create already exists.""" + + +class SecretAlreadyExistsError(DataSecretsError): + """A secret that we want to create already exists.""" + + +def generate_secret_label(charm: CharmBase, scope: Scopes) -> str: + """Generate unique group_mappings for secrets within a relation context. + + Defined as a standalone function, as the choice on secret labels definition belongs to the + Application Logic. To be kept separate from classes below, which are simply to provide a + (smart) abstraction layer above Juju Secrets. + """ + members = [charm.app.name, scope] + return f"{'.'.join(members)}" + + +# Secret cache + + +class CachedSecret: + """Abstraction layer above direct Juju access with caching. + + The data structure is precisely re-using/simulating Juju Secrets behavior, while + also making sure not to fetch a secret multiple times within the same event scope. + """ + + def __init__(self, charm: CharmBase, label: str, secret_uri: Optional[str] = None): + self._secret_meta = None + self._secret_content = {} + self._secret_uri = secret_uri + self.label = label + self.charm = charm + + def add_secret(self, content: Dict[str, str], scope: Scopes) -> Secret: + """Create a new secret.""" + if self._secret_uri: + raise SecretAlreadyExistsError( + "Secret is already defined with uri %s", self._secret_uri + ) + + if scope == APP_SCOPE: + secret = self.charm.app.add_secret(content, label=self.label) + else: + secret = self.charm.unit.add_secret(content, label=self.label) + self._secret_uri = secret.id + self._secret_meta = secret + return self._secret_meta + + @property + def meta(self) -> Optional[Secret]: + """Getting cached secret meta-information.""" + if self._secret_meta: + return self._secret_meta + + if not (self._secret_uri or self.label): + return + + try: + self._secret_meta = self.charm.model.get_secret(label=self.label) + except SecretNotFoundError: + if self._secret_uri: + self._secret_meta = self.charm.model.get_secret( + id=self._secret_uri, label=self.label + ) + return self._secret_meta + + def get_content(self) -> Dict[str, str]: + """Getting cached secret content.""" + if not self._secret_content: + if self.meta: + self._secret_content = self.meta.get_content() + return self._secret_content + + def set_content(self, content: Dict[str, str]) -> None: + """Setting cached secret content.""" + if self.meta: + self.meta.set_content(content) + self._secret_content = content + + def get_info(self) -> Optional[SecretInfo]: + """Wrapper function for get the corresponding call on the Secret object if any.""" + if self.meta: + return self.meta.get_info() + + +class SecretCache: + """A data structure storing CachedSecret objects.""" + + def __init__(self, charm): + self.charm = charm + self._secrets: Dict[str, CachedSecret] = {} + + def get(self, label: str, uri: Optional[str] = None) -> Optional[CachedSecret]: + """Getting a secret from Juju Secret store or cache.""" + if not self._secrets.get(label): + secret = CachedSecret(self.charm, label, uri) + + # Checking if the secret exists, otherwise we don't register it in the cache + if secret.meta: + self._secrets[label] = secret + return self._secrets.get(label) + + def add(self, label: str, content: Dict[str, str], scope: Scopes) -> CachedSecret: + """Adding a secret to Juju Secret.""" + if self._secrets.get(label): + raise SecretAlreadyExistsError(f"Secret {label} already exists") + + secret = CachedSecret(self.charm, label) + secret.add_secret(content, scope) + self._secrets[label] = secret + return self._secrets[label] + + +# END: Secret cache diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index dade673f..33ce39cf 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -54,6 +54,14 @@ def copy_s3_library_into_charm(ops_test: OpsTest): shutil.copyfile(library_path, install_path_requirer) +@pytest.fixture(scope="module", autouse=True) +def copy_data_secrets_library_into_charm(ops_test: OpsTest): + """Copy the data_interfaces library to the different charm folder.""" + library_path = "lib/charms/data_platform_libs/v0/data_secrets.py" + install_path = "tests/integration/secrets-charm/" + library_path + shutil.copyfile(library_path, install_path) + + @pytest.fixture(scope="module") async def application_charm(ops_test: OpsTest): """Build the application charm.""" @@ -107,6 +115,14 @@ async def opensearch_charm(ops_test: OpsTest): return charm +@pytest.fixture(scope="module") +async def secrets_charm(ops_test: OpsTest): + """Build the secrets charm.""" + charm_path = "tests/integration/secrets-charm" + charm = await ops_test.build_charm(charm_path) + return charm + + @pytest.fixture(autouse=True) async def without_errors(ops_test: OpsTest, request): """This fixture is to list all those errors that mustn't occur during execution.""" diff --git a/tests/integration/secrets-charm/actions.yaml b/tests/integration/secrets-charm/actions.yaml new file mode 100644 index 00000000..c1b0d1a2 --- /dev/null +++ b/tests/integration/secrets-charm/actions.yaml @@ -0,0 +1,26 @@ +add-secret: + description: Add a new secret, with label and content specified + label: + type: str + description: Unique part of the identifier of the secret + content: + type: dict + description: Content to be added as secret + scope: + type: str + description: Scope of the secret + +set-secret: + description: Set the content of the secret known by the label specified + content: + type: dict + description: Content to be set on the secret + label: + type: str + description: Identifier of the secret + +get-secret: + description: Retrieve a secret by label from the Juju Secret Store + label: + type: str + description: Identifier of the identifier of the secret diff --git a/tests/integration/secrets-charm/charmcraft.yaml b/tests/integration/secrets-charm/charmcraft.yaml new file mode 100644 index 00000000..e52b3d2b --- /dev/null +++ b/tests/integration/secrets-charm/charmcraft.yaml @@ -0,0 +1,11 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. + +type: charm +bases: + - build-on: + - name: "ubuntu" + channel: "22.04" + run-on: + - name: "ubuntu" + channel: "22.04" diff --git a/tests/integration/secrets-charm/lib/charms/data_platform_libs/v0/data_secrets.py b/tests/integration/secrets-charm/lib/charms/data_platform_libs/v0/data_secrets.py new file mode 100644 index 00000000..254b9af3 --- /dev/null +++ b/tests/integration/secrets-charm/lib/charms/data_platform_libs/v0/data_secrets.py @@ -0,0 +1,143 @@ +"""Secrets related helper classes/functions.""" +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +from typing import Dict, Literal, Optional + +from ops import Secret, SecretInfo +from ops.charm import CharmBase +from ops.model import SecretNotFoundError + +# The unique Charmhub library identifier, never change it +LIBID = "d77fb3d01aba41ed88e837d0beab6be5" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 1 + + +APP_SCOPE = "app" +UNIT_SCOPE = "unit" +Scopes = Literal["app", "unit"] + + +class DataSecretsError(Exception): + """A secret that we want to create already exists.""" + + +class SecretAlreadyExistsError(DataSecretsError): + """A secret that we want to create already exists.""" + + +def generate_secret_label(charm: CharmBase, scope: Scopes) -> str: + """Generate unique group_mappings for secrets within a relation context. + + Defined as a standalone function, as the choice on secret labels definition belongs to the + Application Logic. To be kept separate from classes below, which are simply to provide a + (smart) abstraction layer above Juju Secrets. + """ + members = [charm.app.name, scope] + return f"{'.'.join(members)}" + + +# Secret cache + + +class CachedSecret: + """Abstraction layer above direct Juju access with caching. + + The data structure is precisely re-using/simulating Juju Secrets behavior, while + also making sure not to fetch a secret multiple times within the same event scope. + """ + + def __init__(self, charm: CharmBase, label: str, secret_uri: Optional[str] = None): + self._secret_meta = None + self._secret_content = {} + self._secret_uri = secret_uri + self.label = label + self.charm = charm + + def add_secret(self, content: Dict[str, str], scope: Scopes) -> Secret: + """Create a new secret.""" + if self._secret_uri: + raise SecretAlreadyExistsError( + "Secret is already defined with uri %s", self._secret_uri + ) + + if scope == APP_SCOPE: + secret = self.charm.app.add_secret(content, label=self.label) + else: + secret = self.charm.unit.add_secret(content, label=self.label) + self._secret_uri = secret.id + self._secret_meta = secret + return self._secret_meta + + @property + def meta(self) -> Optional[Secret]: + """Getting cached secret meta-information.""" + if self._secret_meta: + return self._secret_meta + + if not (self._secret_uri or self.label): + return + + try: + self._secret_meta = self.charm.model.get_secret(label=self.label) + except SecretNotFoundError: + if self._secret_uri: + self._secret_meta = self.charm.model.get_secret( + id=self._secret_uri, label=self.label + ) + return self._secret_meta + + def get_content(self) -> Dict[str, str]: + """Getting cached secret content.""" + if not self._secret_content: + if self.meta: + self._secret_content = self.meta.get_content() + return self._secret_content + + def set_content(self, content: Dict[str, str]) -> None: + """Setting cached secret content.""" + if self.meta: + self.meta.set_content(content) + self._secret_content = content + + def get_info(self) -> Optional[SecretInfo]: + """Wrapper function for get the corresponding call on the Secret object if any.""" + if self.meta: + return self.meta.get_info() + + +class SecretCache: + """A data structure storing CachedSecret objects.""" + + def __init__(self, charm): + self.charm = charm + self._secrets: Dict[str, CachedSecret] = {} + + def get(self, label: str, uri: Optional[str] = None) -> Optional[CachedSecret]: + """Getting a secret from Juju Secret store or cache.""" + if not self._secrets.get(label): + secret = CachedSecret(self.charm, label, uri) + + # Checking if the secret exists, otherwise we don't register it in the cache + if secret.meta: + self._secrets[label] = secret + return self._secrets.get(label) + + def add(self, label: str, content: Dict[str, str], scope: Scopes) -> CachedSecret: + """Adding a secret to Juju Secret.""" + if self._secrets.get(label): + raise SecretAlreadyExistsError(f"Secret {label} already exists") + + secret = CachedSecret(self.charm, label) + secret.add_secret(content, scope) + self._secrets[label] = secret + return self._secrets[label] + + +# END: Secret cache diff --git a/tests/integration/secrets-charm/metadata.yaml b/tests/integration/secrets-charm/metadata.yaml new file mode 100644 index 00000000..bfff5595 --- /dev/null +++ b/tests/integration/secrets-charm/metadata.yaml @@ -0,0 +1,8 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +name: secrets-test +description: | + Data platform libs charm used in integration tests for secrets. +summary: | + Data platform libs database meant to be used + only for testing of the libs in this repository. diff --git a/tests/integration/secrets-charm/requirements.txt b/tests/integration/secrets-charm/requirements.txt new file mode 100644 index 00000000..7f118ed1 --- /dev/null +++ b/tests/integration/secrets-charm/requirements.txt @@ -0,0 +1 @@ + ops >= 2.0.0 diff --git a/tests/integration/secrets-charm/src/charm.py b/tests/integration/secrets-charm/src/charm.py new file mode 100755 index 00000000..be6d20d3 --- /dev/null +++ b/tests/integration/secrets-charm/src/charm.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Application charm that connects to database charms. + +This charm is meant to be used only for testing +of the libraries in this repository. +""" + +import logging + +from ops.charm import ActionEvent, CharmBase +from ops.main import main +from ops.model import ActiveStatus + +from charms.data_platform_libs.v0.data_secrets import SecretCache, generate_secret_label + +logger = logging.getLogger(__name__) + + +class SecretsCharm(CharmBase): + """Application charm that connects to database charms.""" + + def __init__(self, *args): + super().__init__(*args) + self.framework.observe(self.on.start, self._on_start) + + self.framework.observe(self.on.add_secret_action, self._on_add_secret_action) + self.framework.observe(self.on.set_secret_action, self._on_set_secret_action) + self.framework.observe(self.on.get_secret_action, self._on_get_secret_action) + self.secret_cache = SecretCache(self) + + def _on_start(self, _) -> None: + """Only sets an Active status.""" + self.unit.status = ActiveStatus() + + def _on_add_secret_action(self, event: ActionEvent): + label = event.params.get("label") + if not label: + label = generate_secret_label() + content = event.params.get("content") + scope = event.params.get("scope") + self.secret_cache.add(label, content, scope) + + def _on_set_secret_action(self, event: ActionEvent): + label = event.params.get("label") + content = event.params.get("content") + secret = self.secret_cache.get(label) + secret.set_content(content) + + def _on_get_secret_action(self, event: ActionEvent): + """Return the secrets stored in juju secrets backend.""" + label = event.params.get("label") + event.set_results({label: self.secret_cache.get(label).get_content()}) + + +if __name__ == "__main__": + main(SecretsCharm) diff --git a/tests/integration/test_secrets.py b/tests/integration/test_secrets.py new file mode 100644 index 00000000..1b9ce6e0 --- /dev/null +++ b/tests/integration/test_secrets.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +import logging + +import pytest +from pytest_operator.plugin import OpsTest + +from .helpers import get_leader_id, get_non_leader_id + +logger = logging.getLogger(__name__) + +APP_NAME = "secrets-test" + + +@pytest.mark.abort_on_fail +async def test_deploy_charms(ops_test: OpsTest, secrets_charm): + """Deploy both charm to use in the tests.""" + await ops_test.model.deploy( + secrets_charm, application_name=APP_NAME, num_units=2, series="jammy" + ) + await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", wait_for_exact_units=2) + + +@pytest.mark.abort_on_fail +async def test_add_get_secret_app(ops_test: OpsTest): + """Test basic functionality of getting/setting cached secrets.""" + leader_id = await get_leader_id(ops_test, APP_NAME) + leader_name = f"{APP_NAME}/{leader_id}" + action = await ops_test.model.units.get(leader_name).run_action( + "add-secret", + **{ + "label": "grandmas-apple-pie", + "content": {"secret-ingredient": "cinnamon"}, + "scope": "app", + }, + ) + await action.wait() + assert action.results["return-code"] == 0 + + action = await ops_test.model.units.get(leader_name).run_action( + "get-secret", **{"label": "grandmas-apple-pie"} + ) + await action.wait() + assert action.results.get("grandmas-apple-pie") == {"secret-ingredient": "cinnamon"} + + +@pytest.mark.abort_on_fail +@pytest.mark.log_errors_allowed( + 'cannot apply changes: creating secrets: secret with label "grandmas-apple-pie" already exists' +) +async def test_secret_label_unique(ops_test: OpsTest): + """Test basic functionality of getting/setting cached secrets.""" + leader_id = await get_leader_id(ops_test, APP_NAME) + leader_name = f"{APP_NAME}/{leader_id}" + action = await ops_test.model.units.get(leader_name).run_action( + "add-secret", + **{ + "label": "grandmas-apple-pie", + "content": {"secret-ingredient": "cinnamon and grated walnut"}, + "scope": "unit", + }, + ) + await action.wait() + + +@pytest.mark.abort_on_fail +async def test_add_get_secret_unit(ops_test: OpsTest): + unit_id = await get_non_leader_id(ops_test, APP_NAME) + unit_name = f"{APP_NAME}/{unit_id}" + action = await ops_test.model.units.get(unit_name).run_action( + "add-secret", + **{ + "label": "grandmas-cranberry-pie", + "content": {"secret-ingredient": "cranberry jam added!"}, + "scope": "unit", + }, + ) + await action.wait() + + action = await ops_test.model.units.get(unit_name).run_action( + "get-secret", **{"label": "grandmas-cranberry-pie"} + ) + await action.wait() + + assert action.results.get("grandmas-cranberry-pie") == { + "secret-ingredient": "cranberry jam added!" + } + + +@pytest.mark.abort_on_fail +async def test_set_secret(ops_test: OpsTest): + """Test basic functionality of getting/setting cached secrets.""" + leader_id = await get_leader_id(ops_test, APP_NAME) + leader_name = f"{APP_NAME}/{leader_id}" + action = await ops_test.model.units.get(leader_name).run_action( + "add-secret", + **{ + "label": "auntie-susans-muffins", + "content": {"secret-ingredient": "nutella"}, + "scope": "app", + }, + ) + await action.wait() + + action = await ops_test.model.units.get(leader_name).run_action( + "set-secret", + **{ + "label": "auntie-susans-muffins", + "content": { + "secret-ingredient": "nutella and chocolate chips", + "baking-time": "25mins sharp", + }, + "scope": "app", + }, + ) + await action.wait() + + action = await ops_test.model.units.get(leader_name).run_action( + "get-secret", **{"label": "auntie-susans-muffins"} + ) + await action.wait() + assert action.results.get("auntie-susans-muffins") == { + "secret-ingredient": "nutella and chocolate chips", + "baking-time": "25mins sharp", + } diff --git a/tests/unit/test_data_secrets.py b/tests/unit/test_data_secrets.py new file mode 100644 index 00000000..2596bb14 --- /dev/null +++ b/tests/unit/test_data_secrets.py @@ -0,0 +1,75 @@ +import pytest +from ops.charm import CharmBase +from ops.testing import Harness + +from charms.data_platform_libs.v0.data_secrets import ( + CachedSecret, + SecretCache, + generate_secret_label, +) + + +class TestCharm(CharmBase): + """Mock database charm to use in units tests.""" + + def __init__(self, *args): + super().__init__(*args) + + +@pytest.fixture +def harness() -> Harness: + harness = Harness(TestCharm) + harness.set_leader(True) + harness.begin_with_initial_hooks() + return harness + + +@pytest.mark.usefixtures("only_with_juju_secrets") +@pytest.mark.parametrize("scope", [("app"), ("unit")]) +def test_cached_secret_works(scope, harness): + """Testing basic functionalities of the CachedSecret class.""" + secret = CachedSecret(harness.charm, "my-label") + secret.add_secret(content={"rumour": "Community Movie on the way"}, scope=scope) + real_secret = harness.charm.model.get_secret(label="my-label") + + assert real_secret.get_content() == secret.get_content() + assert secret.meta.label == real_secret.label + assert secret.get_info().__dict__ == real_secret.get_info().__dict__ + + +@pytest.mark.usefixtures("only_with_juju_secrets") +def test_cached_secret_is_cached(harness, mocker): + """Testing if no more calls to Juju than planned.""" + secret = CachedSecret(harness.charm, "mylabel") + secret.add_secret(content={"rumour": "Community Movie on the way"}, scope="app") + patched_get = mocker.patch("ops.model.Model.get_secret") + patched_get_content = mocker.patch("ops.Secret.get_content") + + secret2 = CachedSecret(harness.charm, "mylabel") + secret2.get_info() + secret2.get_content() + secret2.set_content({"teaser": "Arcane Season II."}) + assert patched_get.called_once() + assert patched_get_content.called_once() + + +@pytest.mark.usefixtures("only_with_juju_secrets") +def test_secret_cache(harness): + """Testing the SecretCache class.""" + cache = SecretCache(harness.charm) + + cache.add("label1", {"rumour": "Community Movie on the way"}, scope="app") + cache.add("label2", {"teaser1": "Arcane Season II.", "teaser2": "Dune II."}, scope="app") + + assert cache.get("label1").get_content() == {"rumour": "Community Movie on the way"} + assert cache.get("label2").get_content() == { + "teaser1": "Arcane Season II.", + "teaser2": "Dune II.", + } + + +@pytest.mark.usefixtures("only_with_juju_secrets") +def test_generate_secret_label(harness): + """Testing generate_secret_label().""" + assert generate_secret_label(harness.charm, "app") == "test-charm.app" + assert generate_secret_label(harness.charm, "unit") == "test-charm.unit" diff --git a/tox.ini b/tox.ini index 7ae19473..9dd92937 100644 --- a/tox.ini +++ b/tox.ini @@ -133,3 +133,14 @@ deps = -r {tox_root}/requirements.txt commands = pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_opensearch_charm.py + +[testenv:integration-secrets] +description = Run secrets integration tests +deps = + pytest + juju{env:LIBJUJU_VERSION_SPECIFIER:==2.9.42.4} + pytest-operator + pytest-mock + -r {tox_root}/requirements.txt +commands = + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_secrets.py