Integration tests split

tests/integration/charms/application-charm/src/charm.py

@@ -11,26 +11,29 @@
 import logging
 from importlib import import_module
 
-from ops import JujuVersion, EventBase
+from ops import EventBase, JujuVersion
 from ops.charm import ActionEvent, CharmBase
 from ops.main import main
 from ops.model import ActiveStatus
 
+# The correct library module is supposed to be put in place by the
+# copy_data_interfaces_library_into_charm auto-applied fixture
+# using the LIB_VERSION env variable
 try:
-    df_module = import_module("charms.data_platform_libs.v0.data_interfaces")
+    data_interfaces_module = import_module("charms.data_platform_libs.v0.data_interfaces")
 except ImportError:
-    df_module = import_module("charms.data_platform_libs.v1.data_interfaces")
+    data_interfaces_module = import_module("charms.data_platform_libs.v1.data_interfaces")
 
 
-BootstrapServerChangedEvent = df_module.BootstrapServerChangedEvent
-DatabaseCreatedEvent = df_module.DatabaseCreatedEvent
-DatabaseEndpointsChangedEvent = df_module.DatabaseEndpointsChangedEvent
-DatabaseRequires = df_module.DatabaseRequires
-DataRequires = df_module.DataRequires
-IndexCreatedEvent = df_module.IndexCreatedEvent
-KafkaRequires = df_module.KafkaRequires
-OpenSearchRequires = df_module.OpenSearchRequires
-TopicCreatedEvent = df_module.TopicCreatedEvent
+BootstrapServerChangedEvent = data_interfaces_module.BootstrapServerChangedEvent
+DatabaseCreatedEvent = data_interfaces_module.DatabaseCreatedEvent
+DatabaseEndpointsChangedEvent = data_interfaces_module.DatabaseEndpointsChangedEvent
+DatabaseRequires = data_interfaces_module.DatabaseRequires
+DataRequires = data_interfaces_module.DataRequires
+IndexCreatedEvent = data_interfaces_module.IndexCreatedEvent
+KafkaRequires = data_interfaces_module.KafkaRequires
+OpenSearchRequires = data_interfaces_module.OpenSearchRequires
+TopicCreatedEvent = data_interfaces_module.TopicCreatedEvent
 
 logger = logging.getLogger(__name__)
 
@@ -199,9 +202,7 @@ def _on_cluster_database_created(self, event: DatabaseCreatedEvent) -> None:
         """Event triggered when a database was created for this application."""
         # Retrieve the credentials using the charm library.
         username, password = self._get_username_password(self.database_clusters, event)
-        logger.info(
-            f"cluster {event.relation.app.name} credentials: {username} {password}"
-        )
+        logger.info(f"cluster {event.relation.app.name} credentials: {username} {password}")
         self.unit.status = ActiveStatus(
             f"received database credentials for cluster {event.relation.app.name}"
         )
@@ -253,9 +254,7 @@ def _on_get_plugin_status(self, event: ActionEvent) -> None:
     def _on_kafka_bootstrap_server_changed(self, event: BootstrapServerChangedEvent):
         """Event triggered when a bootstrap server was changed for this application."""
         bootstrap_server = self._get_bootstrap_server(self.kafka, event)
-        logger.info(
-            f"On kafka boostrap-server changed: bootstrap-server: {bootstrap_server}"
-        )
+        logger.info(f"On kafka boostrap-server changed: bootstrap-server: {bootstrap_server}")
         self.unit.status = ActiveStatus("kafka_bootstrap_server_changed")
 
     def _on_kafka_topic_created(self, _: TopicCreatedEvent):

tests/integration/charms/database-charm/src/charm.py

@@ -11,23 +11,26 @@
 import logging
 import secrets
 import string
+from importlib import import_module
 from random import randrange
 from time import sleep
-from importlib import import_module
 
 import psycopg2
 from ops.charm import CharmBase, WorkloadEvent
 from ops.framework import StoredState
 from ops.main import main
 from ops.model import ActiveStatus, MaintenanceStatus
 
+# The correct library module is supposed to be put in place by the
+# copy_data_interfaces_library_into_charm auto-applied fixture
+# using the LIB_VERSION env variable
 try:
-    df_module = import_module("charms.data_platform_libs.v0.data_interfaces")
+    data_interfaces_module = import_module("charms.data_platform_libs.v0.data_interfaces")
 except ImportError:
-    df_module = import_module("charms.data_platform_libs.v1.data_interfaces")
+    data_interfaces_module = import_module("charms.data_platform_libs.v1.data_interfaces")
 
-DatabaseProvides = df_module.DatabaseProvides
-DatabaseRequestedEvent = df_module.DatabaseRequestedEvent
+DatabaseProvides = data_interfaces_module.DatabaseProvides
+DatabaseRequestedEvent = data_interfaces_module.DatabaseRequestedEvent
 
 logger = logging.getLogger(__name__)
 

tests/integration/charms/kafka-charm/src/charm.py

@@ -16,13 +16,16 @@
 from ops.main import main
 from ops.model import ActiveStatus, MaintenanceStatus
 
+# The correct library module is supposed to be put in place by the
+# copy_data_interfaces_library_into_charm auto-applied fixture
+# using the LIB_VERSION env variable
 try:
-    df_module = import_module("charms.data_platform_libs.v0.data_interfaces")
+    data_interfaces_module = import_module("charms.data_platform_libs.v0.data_interfaces")
 except ImportError:
-    df_module = import_module("charms.data_platform_libs.v1.data_interfaces")
+    data_interfaces_module = import_module("charms.data_platform_libs.v1.data_interfaces")
 
-KafkaProvides = df_module.KafkaProvides
-TopicRequestedEvent = df_module.TopicRequestedEvent
+KafkaProvides = data_interfaces_module.KafkaProvides
+TopicRequestedEvent = data_interfaces_module.TopicRequestedEvent
 
 logger = logging.getLogger(__name__)
 

tests/integration/charms/opensearch-charm/src/charm.py

@@ -15,13 +15,16 @@
 from ops.main import main
 from ops.model import ActiveStatus, MaintenanceStatus
 
+# The correct library module is supposed to be put in place by the
+# copy_data_interfaces_library_into_charm auto-applied fixture
+# using the LIB_VERSION env variable
 try:
-    df_module = import_module("charms.data_platform_libs.v0.data_interfaces")
+    data_interfaces_module = import_module("charms.data_platform_libs.v0.data_interfaces")
 except ImportError:
-    df_module = import_module("charms.data_platform_libs.v1.data_interfaces")
+    data_interfaces_module = import_module("charms.data_platform_libs.v1.data_interfaces")
 
-IndexRequestedEvent = df_module.IndexRequestedEvent
-OpenSearchProvides = df_module.OpenSearchProvides
+IndexRequestedEvent = data_interfaces_module.IndexRequestedEvent
+OpenSearchProvides = data_interfaces_module.OpenSearchProvides
 
 logger = logging.getLogger(__name__)
 

tests/integration/conftest.py

@@ -10,8 +10,18 @@
 import pytest
 from pytest_operator.plugin import OpsTest
 
+LIB_VERSION = os.environ.get("LIB_VERSION")
 
-LIB_VERSION = os.environ.get('LIB_VERSION')
+CHARM_DB_PATH = "tests/integration/charms/database-charm/"
+CHARM_APP_PATH = "tests/integration/charms/application-charm/"
+CHARM_KAFKA_PATH = "tests/integration/charms/kafka-charm/"
+CHARM_OPENSEARCH_PATH = "tests/integration/charms/opensearch-charm/"
+
+CHARM_S3_PROVIDES_PATH = "tests/integration/charms/s3-charm/"
+CHARM_S3_REQUIRES_PATH = "tests/integration/charms/application-s3-charm/"
+
+CHARMS_PATH = [CHARM_DB_PATH, CHARM_APP_PATH, CHARM_KAFKA_PATH, CHARM_OPENSEARCH_PATH]
+CHARMS_S3_PATH = [CHARM_S3_PROVIDES_PATH, CHARM_S3_REQUIRES_PATH]
 
 
 @pytest.fixture(scope="module")
@@ -33,72 +43,81 @@ async def build_charm(charm_path, bases_index: int = None) -> Path:
 
 
 @pytest.fixture(scope="module", autouse=True)
-def copy_data_interfaces_library_into_charm(ops_test: OpsTest):
-    """Copy the data_interfaces library to the different charm folder."""
-    library_path = f"lib/charms/data_platform_libs/{LIB_VERSION}/data_interfaces.py"
-    install_path = [
-        "tests/integration/charms/database-charm/",
-        "tests/integration/charms/kafka-charm/",
-        "tests/integration/charms/application-charm/",
-        "tests/integration/charms/opensearch-charm/",
-    ]
-    for path in install_path:
-        install_file = path + library_path
-        shutil.copyfile(library_path, install_file)
+def cleanup_charms_libs(ops_test: OpsTest):
+    """Cleaning up charms libs.
+
+    IMPORTANT to make sure we're running the correct major library version, in a multi-versions space.
+    """
+    for path in CHARMS_PATH + CHARMS_S3_PATH:
+        install_file = path + "lib"
+        shutil.rmtree(install_file, ignore_errors=True)
 
     yield
 
-    for path in install_path:
+    for path in CHARMS_PATH + CHARMS_S3_PATH:
+        install_file = path + "lib"
+        shutil.rmtree(install_file, ignore_errors=True)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def copy_data_interfaces_library_into_charm(ops_test: OpsTest, cleanup_charms_libs):
+    """Copy the data_interfaces library to the different charm folder, clean up after."""
+    library_path = f"lib/charms/data_platform_libs/{LIB_VERSION}/data_interfaces.py"
+
+    for path in CHARMS_PATH:
         install_file = path + library_path
-        shutil.rmtree(install_file)
+        os.makedirs(os.path.dirname(install_file), exist_ok=True)
+        shutil.copyfile(library_path, install_file)
 
 
 @pytest.fixture(scope="module", autouse=True)
-def copy_s3_library_into_charm(ops_test: OpsTest):
+def copy_s3_library_into_charm(ops_test: OpsTest, cleanup_charms_libs):
     """Copy the s3 library to the applications charm folder."""
     library_path = "lib/charms/data_platform_libs/v0/s3.py"
-    install_path_provider = "tests/integration/charms/s3-charm/" + library_path
-    install_path_requirer = "tests/integration/charms/application-s3-charm/" + library_path
-    shutil.copyfile(library_path, install_path_provider)
-    shutil.copyfile(library_path, install_path_requirer)
+    install_path_provider = CHARM_S3_PROVIDES_PATH + library_path
+    install_path_requirer = CHARM_S3_REQUIRES_PATH + library_path
+
+    for dst_lib_path in [install_path_requirer, install_path_provider]:
+        os.makedirs(os.path.dirname(dst_lib_path), exist_ok=True)
+        shutil.copyfile(library_path, dst_lib_path)
 
 
 @pytest.fixture(scope="module")
 async def application_charm(ops_test: OpsTest):
     """Build the application charm."""
-    charm_path = "tests/integration/charms/application-charm"
+    charm_path = CHARM_APP_PATH
     charm = await ops_test.build_charm(charm_path)
     return charm
 
 
 @pytest.fixture(scope="module")
 async def database_charm(ops_test: OpsTest):
     """Build the database charm."""
-    charm_path = "tests/integration/charms/database-charm"
+    charm_path = CHARM_DB_PATH
     charm = await ops_test.build_charm(charm_path)
     return charm
 
 
 @pytest.fixture(scope="module")
 async def application_s3_charm(ops_test: OpsTest):
     """Build the application-s3 charm."""
-    charm_path = "tests/integration/charms/application-s3-charm"
+    charm_path = CHARM_S3_REQUIRES_PATH
     charm = await ops_test.build_charm(charm_path)
     return charm
 
 
 @pytest.fixture(scope="module")
 async def s3_charm(ops_test: OpsTest):
     """Build the S3 charm."""
-    charm_path = "tests/integration/charms/s3-charm"
+    charm_path = CHARM_S3_PROVIDES_PATH
     charm = await ops_test.build_charm(charm_path)
     return charm
 
 
 @pytest.fixture(scope="module")
 async def kafka_charm(ops_test: OpsTest):
     """Build the Kafka charm."""
-    charm_path = "tests/integration/charms/kafka-charm"
+    charm_path = CHARM_KAFKA_PATH
     charm = await ops_test.build_charm(charm_path)
     return charm
 
@@ -111,6 +130,6 @@ async def opensearch_charm(ops_test: OpsTest):
     all these relations. This might be easily achieved by merging this repo with the
     data-integrator charm repo.
     """
-    charm_path = "tests/integration/charms/opensearch-charm"
+    charm_path = CHARM_OPENSEARCH_PATH
     charm = await ops_test.build_charm(charm_path)
     return charm

tests/integration/helpers.py

@@ -41,10 +41,14 @@ async def build_connection_string(
     # Get the connection data exposed to the application through the relation.
     database = f'{application_name.replace("-", "_")}_{relation_name.replace("-", "_")}'
 
+    secret_uri = None
     if JujuVersion.from_environ().has_secrets:
         secret_uri = await get_application_relation_data(
             ops_test, application_name, relation_name, "secret-user", relation_id, relation_alias
         )
+
+    # NOTE: the fact that we're on Juju3 doesn't mean we're using the version with secrets
+    if secret_uri:
         secret_data = await get_juju_secret(ops_test, secret_uri)
         username = secret_data["username"]
         password = secret_data["password"]