[DPE-3029] New common library -- secrets usage as shared across charms (

#117)

* New common library -- secrets usage as shared across charms

* Unittests

* Integration tests

* Adding secrets tests to pipeline

.github/workflows/ci.yaml

@@ -76,6 +76,11 @@ jobs:
               {juju-snap-channel: "3.1/stable",
                 juju-bootstrap-option: "3.1.6",
                 libjuju-version: "3.2.2"}}
+          - {tox-environments: integration-secrets,
+              juju-version:
+              {juju-snap-channel: "3.1/stable",
+                juju-bootstrap-option: "3.1.6",
+                libjuju-version: "3.2.2"}}
     name: ${{ matrix.tox-environments }} Juju ${{ matrix.juju-version.juju-snap-channel}} -- libjuju ${{ matrix.juju-version.libjuju-version }}
     needs:
       - lint

lib/charms/data_platform_libs/v0/data_secrets.py

@@ -0,0 +1,143 @@
+"""Secrets related helper classes/functions."""
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from typing import Dict, Literal, Optional
+
+from ops import Secret, SecretInfo
+from ops.charm import CharmBase
+from ops.model import SecretNotFoundError
+
+# The unique Charmhub library identifier, never change it
+LIBID = "d77fb3d01aba41ed88e837d0beab6be5"
+
+# Increment this major API version when introducing breaking changes
+LIBAPI = 0
+
+# Increment this PATCH version before using `charmcraft publish-lib` or reset
+# to 0 if you are raising the major API version
+LIBPATCH = 1
+
+
+APP_SCOPE = "app"
+UNIT_SCOPE = "unit"
+Scopes = Literal["app", "unit"]
+
+
+class DataSecretsError(Exception):
+    """A secret that we want to create already exists."""
+
+
+class SecretAlreadyExistsError(DataSecretsError):
+    """A secret that we want to create already exists."""
+
+
+def generate_secret_label(charm: CharmBase, scope: Scopes) -> str:
+    """Generate unique group_mappings for secrets within a relation context.
+
+    Defined as a standalone function, as the choice on secret labels definition belongs to the
+    Application Logic. To be kept separate from classes below, which are simply to provide a
+    (smart) abstraction layer above Juju Secrets.
+    """
+    members = [charm.app.name, scope]
+    return f"{'.'.join(members)}"
+
+
+# Secret cache
+
+
+class CachedSecret:
+    """Abstraction layer above direct Juju access with caching.
+
+    The data structure is precisely re-using/simulating Juju Secrets behavior, while
+    also making sure not to fetch a secret multiple times within the same event scope.
+    """
+
+    def __init__(self, charm: CharmBase, label: str, secret_uri: Optional[str] = None):
+        self._secret_meta = None
+        self._secret_content = {}
+        self._secret_uri = secret_uri
+        self.label = label
+        self.charm = charm
+
+    def add_secret(self, content: Dict[str, str], scope: Scopes) -> Secret:
+        """Create a new secret."""
+        if self._secret_uri:
+            raise SecretAlreadyExistsError(
+                "Secret is already defined with uri %s", self._secret_uri
+            )
+
+        if scope == APP_SCOPE:
+            secret = self.charm.app.add_secret(content, label=self.label)
+        else:
+            secret = self.charm.unit.add_secret(content, label=self.label)
+        self._secret_uri = secret.id
+        self._secret_meta = secret
+        return self._secret_meta
+
+    @property
+    def meta(self) -> Optional[Secret]:
+        """Getting cached secret meta-information."""
+        if self._secret_meta:
+            return self._secret_meta
+
+        if not (self._secret_uri or self.label):
+            return
+
+        try:
+            self._secret_meta = self.charm.model.get_secret(label=self.label)
+        except SecretNotFoundError:
+            if self._secret_uri:
+                self._secret_meta = self.charm.model.get_secret(
+                    id=self._secret_uri, label=self.label
+                )
+        return self._secret_meta
+
+    def get_content(self) -> Dict[str, str]:
+        """Getting cached secret content."""
+        if not self._secret_content:
+            if self.meta:
+                self._secret_content = self.meta.get_content()
+        return self._secret_content
+
+    def set_content(self, content: Dict[str, str]) -> None:
+        """Setting cached secret content."""
+        if self.meta:
+            self.meta.set_content(content)
+            self._secret_content = content
+
+    def get_info(self) -> Optional[SecretInfo]:
+        """Wrapper function for get the corresponding call on the Secret object if any."""
+        if self.meta:
+            return self.meta.get_info()
+
+
+class SecretCache:
+    """A data structure storing CachedSecret objects."""
+
+    def __init__(self, charm):
+        self.charm = charm
+        self._secrets: Dict[str, CachedSecret] = {}
+
+    def get(self, label: str, uri: Optional[str] = None) -> Optional[CachedSecret]:
+        """Getting a secret from Juju Secret store or cache."""
+        if not self._secrets.get(label):
+            secret = CachedSecret(self.charm, label, uri)
+
+            # Checking if the secret exists, otherwise we don't register it in the cache
+            if secret.meta:
+                self._secrets[label] = secret
+        return self._secrets.get(label)
+
+    def add(self, label: str, content: Dict[str, str], scope: Scopes) -> CachedSecret:
+        """Adding a secret to Juju Secret."""
+        if self._secrets.get(label):
+            raise SecretAlreadyExistsError(f"Secret {label} already exists")
+
+        secret = CachedSecret(self.charm, label)
+        secret.add_secret(content, scope)
+        self._secrets[label] = secret
+        return self._secrets[label]
+
+
+# END: Secret cache

tests/integration/conftest.py

@@ -54,6 +54,14 @@ def copy_s3_library_into_charm(ops_test: OpsTest):
     shutil.copyfile(library_path, install_path_requirer)
 
 
+@pytest.fixture(scope="module", autouse=True)
+def copy_data_secrets_library_into_charm(ops_test: OpsTest):
+    """Copy the data_interfaces library to the different charm folder."""
+    library_path = "lib/charms/data_platform_libs/v0/data_secrets.py"
+    install_path = "tests/integration/secrets-charm/" + library_path
+    shutil.copyfile(library_path, install_path)
+
+
 @pytest.fixture(scope="module")
 async def application_charm(ops_test: OpsTest):
     """Build the application charm."""
@@ -107,6 +115,14 @@ async def opensearch_charm(ops_test: OpsTest):
     return charm
 
 
+@pytest.fixture(scope="module")
+async def secrets_charm(ops_test: OpsTest):
+    """Build the secrets charm."""
+    charm_path = "tests/integration/secrets-charm"
+    charm = await ops_test.build_charm(charm_path)
+    return charm
+
+
 @pytest.fixture(autouse=True)
 async def without_errors(ops_test: OpsTest, request):
     """This fixture is to list all those errors that mustn't occur during execution."""

tests/integration/secrets-charm/actions.yaml

@@ -0,0 +1,26 @@
+add-secret:
+  description: Add a new secret, with label and content specified
+  label:
+    type: str
+    description: Unique part of the identifier of the secret 
+  content:                     
+    type: dict                 
+    description: Content to be added as secret
+  scope:
+    type: str
+    description: Scope of the secret
+
+set-secret:
+  description: Set the content of the secret known by the label specified
+  content:                     
+    type: dict                 
+    description: Content to be set on the secret
+  label:
+    type: str
+    description: Identifier of the secret
+
+get-secret:
+  description: Retrieve a secret by label from the Juju Secret Store
+  label:
+    type: str
+    description: Identifier of the identifier of the secret 

tests/integration/secrets-charm/charmcraft.yaml

@@ -0,0 +1,11 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+type: charm
+bases:
+  - build-on:
+      - name: "ubuntu"
+        channel: "22.04"
+    run-on:
+      - name: "ubuntu"
+        channel: "22.04"

tests/integration/secrets-charm/lib/charms/data_platform_libs/v0/data_secrets.py

@@ -0,0 +1,143 @@
+"""Secrets related helper classes/functions."""
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from typing import Dict, Literal, Optional
+
+from ops import Secret, SecretInfo
+from ops.charm import CharmBase
+from ops.model import SecretNotFoundError
+
+# The unique Charmhub library identifier, never change it
+LIBID = "d77fb3d01aba41ed88e837d0beab6be5"
+
+# Increment this major API version when introducing breaking changes
+LIBAPI = 0
+
+# Increment this PATCH version before using `charmcraft publish-lib` or reset
+# to 0 if you are raising the major API version
+LIBPATCH = 1
+
+
+APP_SCOPE = "app"
+UNIT_SCOPE = "unit"
+Scopes = Literal["app", "unit"]
+
+
+class DataSecretsError(Exception):
+    """A secret that we want to create already exists."""
+
+
+class SecretAlreadyExistsError(DataSecretsError):
+    """A secret that we want to create already exists."""
+
+
+def generate_secret_label(charm: CharmBase, scope: Scopes) -> str:
+    """Generate unique group_mappings for secrets within a relation context.
+
+    Defined as a standalone function, as the choice on secret labels definition belongs to the
+    Application Logic. To be kept separate from classes below, which are simply to provide a
+    (smart) abstraction layer above Juju Secrets.
+    """
+    members = [charm.app.name, scope]
+    return f"{'.'.join(members)}"
+
+
+# Secret cache
+
+
+class CachedSecret:
+    """Abstraction layer above direct Juju access with caching.
+
+    The data structure is precisely re-using/simulating Juju Secrets behavior, while
+    also making sure not to fetch a secret multiple times within the same event scope.
+    """
+
+    def __init__(self, charm: CharmBase, label: str, secret_uri: Optional[str] = None):
+        self._secret_meta = None
+        self._secret_content = {}
+        self._secret_uri = secret_uri
+        self.label = label
+        self.charm = charm
+
+    def add_secret(self, content: Dict[str, str], scope: Scopes) -> Secret:
+        """Create a new secret."""
+        if self._secret_uri:
+            raise SecretAlreadyExistsError(
+                "Secret is already defined with uri %s", self._secret_uri
+            )
+
+        if scope == APP_SCOPE:
+            secret = self.charm.app.add_secret(content, label=self.label)
+        else:
+            secret = self.charm.unit.add_secret(content, label=self.label)
+        self._secret_uri = secret.id
+        self._secret_meta = secret
+        return self._secret_meta
+
+    @property
+    def meta(self) -> Optional[Secret]:
+        """Getting cached secret meta-information."""
+        if self._secret_meta:
+            return self._secret_meta
+
+        if not (self._secret_uri or self.label):
+            return
+
+        try:
+            self._secret_meta = self.charm.model.get_secret(label=self.label)
+        except SecretNotFoundError:
+            if self._secret_uri:
+                self._secret_meta = self.charm.model.get_secret(
+                    id=self._secret_uri, label=self.label
+                )
+        return self._secret_meta
+
+    def get_content(self) -> Dict[str, str]:
+        """Getting cached secret content."""
+        if not self._secret_content:
+            if self.meta:
+                self._secret_content = self.meta.get_content()
+        return self._secret_content
+
+    def set_content(self, content: Dict[str, str]) -> None:
+        """Setting cached secret content."""
+        if self.meta:
+            self.meta.set_content(content)
+            self._secret_content = content
+
+    def get_info(self) -> Optional[SecretInfo]:
+        """Wrapper function for get the corresponding call on the Secret object if any."""
+        if self.meta:
+            return self.meta.get_info()
+
+
+class SecretCache:
+    """A data structure storing CachedSecret objects."""
+
+    def __init__(self, charm):
+        self.charm = charm
+        self._secrets: Dict[str, CachedSecret] = {}
+
+    def get(self, label: str, uri: Optional[str] = None) -> Optional[CachedSecret]:
+        """Getting a secret from Juju Secret store or cache."""
+        if not self._secrets.get(label):
+            secret = CachedSecret(self.charm, label, uri)
+
+            # Checking if the secret exists, otherwise we don't register it in the cache
+            if secret.meta:
+                self._secrets[label] = secret
+        return self._secrets.get(label)
+
+    def add(self, label: str, content: Dict[str, str], scope: Scopes) -> CachedSecret:
+        """Adding a secret to Juju Secret."""
+        if self._secrets.get(label):
+            raise SecretAlreadyExistsError(f"Secret {label} already exists")
+
+        secret = CachedSecret(self.charm, label)
+        secret.add_secret(content, scope)
+        self._secrets[label] = secret
+        return self._secrets[label]
+
+
+# END: Secret cache

tests/integration/secrets-charm/metadata.yaml

@@ -0,0 +1,8 @@
+# Copyright 2022 Canonical Ltd.                                                                                                                                                                                     
+# See LICENSE file for licensing details.
+name: secrets-test 
+description: |
+  Data platform libs charm used in integration tests for secrets.
+summary: |
+  Data platform libs database meant to be used
+  only for testing of the libs in this repository.

tests/integration/secrets-charm/requirements.txt

@@ -0,0 +1 @@
+ ops >= 2.0.0