[DPE-2342] Charm Relation (i.e. 'external') Secrets

tests/conftest.py

@@ -0,0 +1,51 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+import os
+from importlib.metadata import version
+from unittest.mock import PropertyMock
+
+import pytest
+from ops import JujuVersion
+from pytest_mock import MockerFixture
+
+
+@pytest.fixture(autouse=True)
+def juju_has_secrets(mocker: MockerFixture):
+    """This fixture will force the usage of secrets whenever run on Juju 3.x.
+
+    NOTE: This is needed, as normally JujuVersion is set to 0.0.0 in tests
+    (i.e. not the real juju version)
+    """
+    if juju_version := os.environ.get("LIBJUJU_VERSION_SPECIFIER"):
+        juju_version.replace("==", "")
+        juju_version = juju_version[2:].split(".")[0]
+    else:
+        juju_version = version("juju")
+
+    if juju_version < "3":
+        mocker.patch.object(
+            JujuVersion, "has_secrets", new_callable=PropertyMock
+        ).return_value = False
+        return False
+    else:
+        mocker.patch.object(
+            JujuVersion, "has_secrets", new_callable=PropertyMock
+        ).return_value = True
+        return True
+
+
+@pytest.fixture
+def only_with_juju_secrets(juju_has_secrets):
+    """Pretty way to skip Juju 3 tests."""
+    if not juju_has_secrets:
+        pytest.skip("Secrets test only applies on Juju 3.x")
+
+
+@pytest.fixture
+def only_without_juju_secrets(juju_has_secrets):
+    """Pretty way to skip Juju 2-specific tests.
+
+    Typically: to save CI time, when the same check were executed in a Juju 3-specific way already
+    """
+    if juju_has_secrets:
+        pytest.skip("Skipping legacy secrets tests")

tests/integration/helpers.py

@@ -1,12 +1,22 @@
 #!/usr/bin/env python3
 # Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
-from typing import Optional
+import json
+from typing import Dict, Optional
 
 import yaml
+from ops import JujuVersion
 from pytest_operator.plugin import OpsTest
 
 
+async def get_juju_secret(ops_test: OpsTest, secret_uri: str) -> Dict[str, str]:
+    """Retrieve juju secret."""
+    secret_unique_id = secret_uri.split("/")[-1]
+    complete_command = f"show-secret {secret_uri} --reveal --format=json"
+    _, stdout, _ = await ops_test.juju(*complete_command.split())
+    return json.loads(stdout)[secret_unique_id]["content"]["Data"]
+
+
 async def build_connection_string(
     ops_test: OpsTest,
     application_name: str,
@@ -30,15 +40,36 @@ async def build_connection_string(
     """
     # Get the connection data exposed to the application through the relation.
     database = f'{application_name.replace("-", "_")}_{relation_name.replace("-", "_")}'
-    username = await get_application_relation_data(
-        ops_test, application_name, relation_name, "username", relation_id, relation_alias
-    )
-    password = await get_application_relation_data(
-        ops_test, application_name, relation_name, "password", relation_id, relation_alias
-    )
-    endpoints = await get_application_relation_data(
-        ops_test, application_name, relation_name, "endpoints", relation_id, relation_alias
-    )
+
+    if JujuVersion.from_environ().has_secrets:
+        secret_uri = await get_application_relation_data(
+            ops_test, application_name, relation_name, "secret-user", relation_id, relation_alias
+        )
+        secret_data = await get_juju_secret(ops_test, secret_uri)
+        username = secret_data["username"]
+        password = secret_data["password"]
+
+        secret_uri = await get_application_relation_data(
+            ops_test,
+            application_name,
+            relation_name,
+            "secret-endpoints",
+            relation_id,
+            relation_alias,
+        )
+        secret_data = await get_juju_secret(ops_test, secret_uri)
+        endpoints = secret_data["endpoints"]
+
+    else:
+        username = await get_application_relation_data(
+            ops_test, application_name, relation_name, "username", relation_id, relation_alias
+        )
+        password = await get_application_relation_data(
+            ops_test, application_name, relation_name, "password", relation_id, relation_alias
+        )
+        endpoints = await get_application_relation_data(
+            ops_test, application_name, relation_name, "endpoints", relation_id, relation_alias
+        )
     host = endpoints.split(",")[0].split(":")[0]
 
     # Build the complete connection string to connect to the database.

tests/integration/kafka-charm/src/charm.py

@@ -108,7 +108,8 @@ def _on_topic_requested(self, event: TopicRequestedEvent):
     def _on_sync_password(self, event: ActionEvent):
         """Set the password in the data relation databag."""
         logger.info("On sync password")
-        password = event.params["password"]
+
+        password = event.params.get("password")
         self.set_secret("app", "password", password)
         logger.info(f"New password: {password}")
         # set parameters in the secrets
@@ -126,6 +127,7 @@ def _on_sync_username(self, event: ActionEvent):
         """Set the username in the data relation databag."""
         username = event.params["username"]
         self.set_secret("app", "username", username)
+
         # set parameters in the secrets
         # update relation data if the relation is present
         if len(self.kafka_provider.relations) > 0:

tests/integration/test_kafka_charm.py

@@ -7,7 +7,7 @@
 import pytest
 from pytest_operator.plugin import OpsTest
 
-from .helpers import get_application_relation_data
+from .helpers import get_application_relation_data, get_juju_secret
 
 logger = logging.getLogger(__name__)
 
@@ -39,6 +39,7 @@ async def test_deploy_charms(ops_test: OpsTest, application_charm, kafka_charm):
 
 
 @pytest.mark.abort_on_fail
+@pytest.mark.usefixtures("only_without_juju_secrets")
 async def test_kafka_relation_with_charm_libraries(ops_test: OpsTest):
     """Test basic functionality of kafka relation interface."""
     # Relate the charms and wait for them exchanging some connection data.
@@ -77,6 +78,51 @@ async def test_kafka_relation_with_charm_libraries(ops_test: OpsTest):
     assert topic == "test-topic"
 
 
+@pytest.mark.abort_on_fail
+@pytest.mark.usefixtures("only_with_juju_secrets")
+async def test_kafka_relation_with_charm_libraries_secrets(ops_test: OpsTest):
+    """Test basic functionality of kafka relation interface."""
+    # Relate the charms and wait for them exchanging some connection data.
+    await ops_test.model.add_relation(KAFKA_APP_NAME, APPLICATION_APP_NAME)
+    await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active")
+
+    # check unit messagge to check if the topic_created_event is triggered
+    for unit in ops_test.model.applications[APPLICATION_APP_NAME].units:
+        assert unit.workload_status_message == "kafka_topic_created"
+    # check if the topic was granted
+    for unit in ops_test.model.applications[KAFKA_APP_NAME].units:
+        assert "granted" in unit.workload_status_message
+
+    secret_uri = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "secret-user"
+    )
+
+    secret_content = await get_juju_secret(ops_test, secret_uri)
+    username = secret_content["username"]
+    password = secret_content["password"]
+
+    secret_uri = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "secret-endpoints"
+    )
+    secret_content = await get_juju_secret(ops_test, secret_uri)
+    bootstrap_server = secret_content["endpoints"]
+
+    consumer_group_prefix = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "consumer-group-prefix"
+    )
+
+    topic = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "topic"
+    )
+
+    assert username == "admin"
+    assert password == "password"
+    assert bootstrap_server == "host1:port,host2:port"
+    assert consumer_group_prefix == "test-prefix"
+    assert topic == "test-topic"
+
+
+@pytest.mark.usefixtures("only_without_juju_secrets")
 async def test_kafka_bootstrap_server_changed(ops_test: OpsTest):
     """Test that the bootstrap server changed event is correctly triggered."""
     app_unit = ops_test.model.applications[APPLICATION_APP_NAME].units[0]
@@ -115,3 +161,58 @@ async def test_kafka_bootstrap_server_changed(ops_test: OpsTest):
     # check the bootstrap_server_changed event is NOT triggered
     for unit in ops_test.model.applications[APPLICATION_APP_NAME].units:
         assert unit.workload_status_message == ""
+
+
+@pytest.mark.usefixtures("only_with_juju_secrets")
+async def test_kafka_bootstrap_server_changed_secrets(ops_test: OpsTest):
+    """Test that the bootstrap server changed event is correctly triggered."""
+    app_unit = ops_test.model.applications[APPLICATION_APP_NAME].units[0]
+    kafka_unit = ops_test.model.applications[KAFKA_APP_NAME].units[0]
+    # set new bootstrap
+    parameters = {"bootstrap-server": "host1:port,host2:port,host3:port"}
+    action = await kafka_unit.run_action(action_name="sync-bootstrap-server", **parameters)
+    result = await action.wait()
+    await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active")
+    assert result.results["bootstrap-server"] == "host1:port,host2:port,host3:port"
+
+    # check that the new bootstrap-server is set as a secret
+    secret_uri = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "secret-endpoints"
+    )
+    secret_content = await get_juju_secret(ops_test, secret_uri)
+    bootstrap_server = secret_content["endpoints"]
+    assert bootstrap_server == "host1:port,host2:port,host3:port"
+
+    # check that the bootstrap_server_changed event is triggered
+    for unit in ops_test.model.applications[APPLICATION_APP_NAME].units:
+        assert unit.workload_status_message == "kafka_bootstrap_server_changed"
+
+    # reset unit message
+    action = await app_unit.run_action(action_name="reset-unit-status")
+    result = await action.wait()
+    await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active")
+
+    # check if the message is empty
+    for unit in ops_test.model.applications[APPLICATION_APP_NAME].units:
+        assert unit.workload_status_message == ""
+
+    # configure the same bootstrap-server
+    action = await kafka_unit.run_action(action_name="sync-bootstrap-server", **parameters)
+    result = await action.wait()
+    await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active")
+    assert result.results["bootstrap-server"] == "host1:port,host2:port,host3:port"
+    bootstrap_server = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "endpoints"
+    )
+
+    # check that the new bootstrap-server secret
+    secret_uri = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "secret-endpoints"
+    )
+    secret_content = await get_juju_secret(ops_test, secret_uri)
+    bootstrap_server = secret_content["endpoints"]
+    assert bootstrap_server == "host1:port,host2:port,host3:port"
+
+    # check the bootstrap_server_changed event is NOT triggered
+    for unit in ops_test.model.applications[APPLICATION_APP_NAME].units:
+        assert unit.workload_status_message == ""

tests/integration/test_opensearch_charm.py

@@ -7,7 +7,7 @@
 import pytest
 from pytest_operator.plugin import OpsTest
 
-from .helpers import get_application_relation_data
+from .helpers import get_application_relation_data, get_juju_secret
 
 logger = logging.getLogger(__name__)
 
@@ -41,6 +41,7 @@ async def test_deploy_charms(ops_test: OpsTest, application_charm, opensearch_ch
 
 
 @pytest.mark.abort_on_fail
+@pytest.mark.usefixtures("only_without_juju_secrets")
 async def test_opensearch_relation_with_charm_libraries(ops_test: OpsTest):
     """Test basic functionality of opensearch relation interface."""
     # Relate the charms and wait for them exchanging some connection data.
@@ -71,3 +72,42 @@ async def test_opensearch_relation_with_charm_libraries(ops_test: OpsTest):
     assert password == "password"
     assert endpoints == "host1:port,host2:port"
     assert index == "test-index"
+
+
+@pytest.mark.abort_on_fail
+@pytest.mark.usefixtures("only_with_juju_secrets")
+async def test_opensearch_relation_with_charm_libraries_secrets(ops_test: OpsTest):
+    """Test basic functionality of opensearch relation interface."""
+    # Relate the charms and wait for them exchanging some connection data.
+    await ops_test.model.add_relation(OPENSEARCH_APP_NAME, APPLICATION_APP_NAME)
+    await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active")
+
+    # check unit messagge to check if the index_created_event is triggered
+    for unit in ops_test.model.applications[APPLICATION_APP_NAME].units:
+        assert unit.workload_status_message == "opensearch_index_created"
+    # check if index access is granted
+    for unit in ops_test.model.applications[OPENSEARCH_APP_NAME].units:
+        assert "granted" in unit.workload_status_message
+
+    secret_uri = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "secret-user"
+    )
+
+    secret_content = await get_juju_secret(ops_test, secret_uri)
+    username = secret_content["username"]
+    password = secret_content["password"]
+
+    secret_uri = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "secret-endpoints"
+    )
+    secret_content = await get_juju_secret(ops_test, secret_uri)
+    endpoints = secret_content["endpoints"]
+
+    index = await get_application_relation_data(
+        ops_test, APPLICATION_APP_NAME, RELATION_NAME, "index"
+    )
+
+    assert username == "admin"
+    assert password == "password"
+    assert endpoints == "host1:port,host2:port"
+    assert index == "test-index"