test: add additional integration checks

repository.py

@@ -419,6 +419,7 @@ def fmt_cli(
 ):
     """Apply correct formatting to code."""
     files = get_source_dirs(slurm_charms)
+    files.append(str(ROOT_DIR / "tests"))
     logging.info(f"Running black for directories {files}")
     subprocess.run(["black", "--config", "pyproject.toml"] + files, cwd=ROOT_DIR, check=True)
 
@@ -430,6 +431,7 @@ def lint_cli(
 ):
     """Check code against coding style standards."""
     files = get_source_dirs(slurm_charms)
+    files.append(str(ROOT_DIR / "tests"))
     logging.info("Target directories: {files}")
     if fix:
         logging.info("Trying to automatically fix the lint errors.")

tests/integration/conftest.py

@@ -44,6 +44,7 @@ def charm_base(request) -> str:
     """Get slurmctld charm base to use."""
     return request.config.option.charm_base
 
+
 @pytest.fixture(scope="module")
 async def slurmctld_charm(request, ops_test: OpsTest) -> Union[str, Path]:
     """Pack slurmctld charm to use for integration tests.

tests/integration/test_charm.py

@@ -30,6 +30,7 @@
 SLURMRESTD = "slurmrestd"
 DATABASE = "mysql"
 ROUTER = "mysql-router"
+SLURM_APPS = [SLURMCTLD, SLURMD, SLURMDBD, SLURMRESTD]
 
 
 @pytest.mark.abort_on_fail
@@ -44,9 +45,7 @@ async def test_build_and_deploy_against_edge(
     slurmrestd_charm,
 ) -> None:
     """Test that the slurmctld charm can stabilize against slurmd, slurmdbd, slurmrestd, and MySQL."""
-    logger.info(
-        f"Deploying {SLURMCTLD} against {SLURMD}, {SLURMDBD}, {SLURMRESTD}, and {DATABASE}"
-    )
+    logger.info(f"Deploying {', '.join(SLURM_APPS)}, and {DATABASE}")
     # Pack charms and download NHC resource for the slurmd operator.
     slurmctld, slurmd, slurmdbd, slurmrestd = await asyncio.gather(
         slurmctld_charm, slurmd_charm, slurmdbd_charm, slurmrestd_charm
@@ -103,11 +102,9 @@ async def test_build_and_deploy_against_edge(
     await ops_test.model.integrate(f"{SLURMDBD}:database", f"{SLURMDBD}-{ROUTER}:database")
     # Reduce the update status frequency to accelerate the triggering of deferred events.
     async with ops_test.fast_forward():
-        await ops_test.model.wait_for_idle(apps=[SLURMCTLD, SLURMD, SLURMDBD, SLURMRESTD], status="active", timeout=1000)
-        assert ops_test.model.applications["slurmctld"].units[0].workload_status == "active"
-        assert ops_test.model.applications["slurmd"].units[0].workload_status == "active"
-        assert ops_test.model.applications["slurmdbd"].units[0].workload_status == "active"
-        assert ops_test.model.applications["slurmrestd"].units[0].workload_status == "active"
+        await ops_test.model.wait_for_idle(apps=SLURM_APPS, status="active", timeout=1000)
+        for app in SLURM_APPS:
+            assert ops_test.model.applications[app].units[0].workload_status == "active"
 
 
 @pytest.mark.abort_on_fail
@@ -117,12 +114,13 @@ async def test_build_and_deploy_against_edge(
     stop=tenacity.stop_after_attempt(3),
     reraise=True,
 )
-async def test_slurmctld_is_active(ops_test: OpsTest) -> None:
-    """Test that slurmctld is active inside Juju unit."""
-    logger.info("Checking that slurmctld is active inside Juju unit")
-    slurmctld_unit = ops_test.model.applications["slurmctld"].units[0]
-    res = (await slurmctld_unit.ssh("systemctl is-active slurmctld")).strip("\n")
-    assert res == "active"
+async def test_munge_is_active(ops_test: OpsTest) -> None:
+    """Test that munge is active inside all the SLURM units."""
+    for app in SLURM_APPS:
+        logger.info(f"Checking that munge is active inside {app}.")
+        unit = ops_test.model.applications[app].units[0]
+        res = (await unit.ssh("systemctl is-active munge")).strip("\n")
+        assert res == "active"
 
 
 @pytest.mark.abort_on_fail
@@ -132,24 +130,40 @@ async def test_slurmctld_is_active(ops_test: OpsTest) -> None:
     stop=tenacity.stop_after_attempt(3),
     reraise=True,
 )
+async def test_services_are_active(ops_test: OpsTest) -> None:
+    """Test that the SLURM services are active inside the SLURM units."""
+    for app in SLURM_APPS:
+        logger.info(f"Checking that the {app} service is active inside the {app} unit.")
+        unit = ops_test.model.applications[app].units[0]
+        res = (await unit.ssh(f"systemctl is-active {app}")).strip("\n")
+        assert res == "active"
+
+
+@pytest.mark.abort_on_fail
+@pytest.mark.order(4)
+@tenacity.retry(
+    wait=tenacity.wait.wait_exponential(multiplier=2, min=1, max=30),
+    stop=tenacity.stop_after_attempt(3),
+    reraise=True,
+)
 async def test_slurmctld_port_listen(ops_test: OpsTest) -> None:
     """Test that slurmctld is listening on port 6817."""
     logger.info("Checking that slurmctld is listening on port 6817")
-    slurmctld_unit = ops_test.model.applications["slurmctld"].units[0]
+    slurmctld_unit = ops_test.model.applications[SLURMCTLD].units[0]
     res = await slurmctld_unit.ssh("sudo lsof -t -n -iTCP:6817 -sTCP:LISTEN")
     assert res != ""
 
 
 @pytest.mark.abort_on_fail
-@pytest.mark.order(4)
+@pytest.mark.order(5)
 @tenacity.retry(
     wait=tenacity.wait.wait_exponential(multiplier=2, min=1, max=30),
     stop=tenacity.stop_after_attempt(3),
     reraise=True,
 )
-async def test_munge_is_active(ops_test: OpsTest) -> None:
-    """Test that munge is active inside Juju unit."""
-    logger.info("Checking that munge is active inside Juju unit")
-    slurmctld_unit = ops_test.model.applications["slurmctld"].units[0]
-    res = (await slurmctld_unit.ssh("systemctl is-active munge")).strip("\n")
-    assert res == "active"
+async def test_slurmdbd_port_listen(ops_test: OpsTest) -> None:
+    """Test that slurmdbd is listening on port 6819."""
+    logger.info("Checking that slurmdbd is listening on port 6819")
+    slurmdbd_unit = ops_test.model.applications[SLURMDBD].units[0]
+    res = await slurmdbd_unit.ssh("sudo lsof -t -n -iTCP:6819 -sTCP:LISTEN")
+    assert res != ""