test: test both `quay.io/centos-bootc/{centos,fedora}-bootc:{eln,stre…

…am9}`

Now that `bib` supports both fedora and centos bootc we need to start
testing both as part of the integration suite.

plans/all.fmf

@@ -20,7 +20,10 @@ prepare:
     - qemu-kvm
 execute:
   how: tmt
-  script: pytest -s -vv --force-aws-upload
+  script: |
+    pip install --user -r test/requirements.txt
+    pytest -s -vv --force-aws-upload
+  duration: 2h
 finish:
   how: shell
   script: df -h

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
 # do not use /tmp by default as it may be on a tempfs and our tests can
 # generate 10G images (that full of holes so not really 10G but still)
-addopts = --basetemp=/var/tmp
+addopts = --basetemp=/var/tmp/bib-tests

test/containerbuild.py

@@ -17,12 +17,3 @@ def build_container_fixture():
         "-t", container_tag,
     ])
     return container_tag
-
-
-def container_to_build_ref():
-    # TODO: make this another indirect fixture input, e.g. by making
-    # making "image_type" an "image" tuple (type, container_ref_to_test)
-    return os.getenv(
-        "BIB_TEST_BOOTC_CONTAINER_TAG",
-        "quay.io/centos-bootc/fedora-bootc:eln",
-    )

test/test_build.py

@@ -12,7 +12,8 @@
 
 # local test utils
 import testutil
-from containerbuild import build_container_fixture, container_to_build_ref  # noqa: F401
+from containerbuild import build_container_fixture  # noqa: F401
+from testcases import gen_testcases
 from vm import AWS, QEMU
 
 if not testutil.has_executable("podman"):
@@ -27,12 +28,6 @@
     pytest.skip("need x86_64-v3 capable CPU", allow_module_level=True)
 
 
-# image types to test
-DIRECT_BOOT_IMAGE_TYPES = ["qcow2", "ami", "raw"]
-INSTALLER_IMAGE_TYPES = ["iso"]
-SUPPORTED_IMAGE_TYPES = DIRECT_BOOT_IMAGE_TYPES + INSTALLER_IMAGE_TYPES
-
-
 class ImageBuildResult(NamedTuple):
     img_type: str
     img_path: str
@@ -49,7 +44,7 @@ def image_type_fixture(tmpdir_factory, build_container, request, force_aws_uploa
     ImageBuildResult with the resulting image path and user/password
     """
     # image_type is passed via special pytest parameter fixture
-    image_type = request.param
+    container_ref, image_type = request.param.split(",")
 
     username = "test"
     password = "password"
@@ -64,7 +59,7 @@ def image_type_fixture(tmpdir_factory, build_container, request, force_aws_uploa
         "raw": pathlib.Path(output_path) / "image/disk.raw",
         "iso": pathlib.Path(output_path) / "bootiso/install.iso",
     }
-    assert len(artifact) == len(SUPPORTED_IMAGE_TYPES), \
+    assert len(artifact) == len(set(t.split(",")[1] for t in gen_testcases("all"))), \
         "please keep artifact mapping and supported images in sync"
     generated_img = artifact[image_type]
 
@@ -121,7 +116,7 @@ def image_type_fixture(tmpdir_factory, build_container, request, force_aws_uploa
             "-v", "/store",  # share the cache between builds
             *creds_args,
             build_container,
-            container_to_build_ref(),
+            container_ref,
             "--config", "/output/config.json",
             "--type", image_type,
             *upload_args,
@@ -146,14 +141,14 @@ def test_container_builds(build_container):
     assert build_container in output
 
 
-@pytest.mark.parametrize("image_type", SUPPORTED_IMAGE_TYPES, indirect=["image_type"])
+@pytest.mark.parametrize("image_type", gen_testcases("direct-boot"), indirect=["image_type"])
 def test_image_is_generated(image_type):
     assert image_type.img_path.exists(), "output file missing, dir "\
         f"content: {os.listdir(os.fspath(image_type.img_path))}"
 
 
 @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now")
-@pytest.mark.parametrize("image_type", DIRECT_BOOT_IMAGE_TYPES, indirect=["image_type"])
+@pytest.mark.parametrize("image_type", gen_testcases("direct-boot"), indirect=["image_type"])
 def test_image_boots(image_type):
     with QEMU(image_type.img_path) as test_vm:
         exit_status, _ = test_vm.run("true", user=image_type.username, password=image_type.password)
@@ -163,7 +158,7 @@ def test_image_boots(image_type):
         assert "hello" in output
 
 
-@pytest.mark.parametrize("image_type", ["ami"], indirect=["image_type"])
+@pytest.mark.parametrize("image_type", gen_testcases("ami-boot"), indirect=["image_type"])
 def test_ami_boots_in_aws(image_type, force_aws_upload):
     if not testutil.write_aws_creds("/dev/null"):  # we don't care about the file, just the variables being there
         if force_aws_upload:
@@ -213,7 +208,7 @@ def has_selinux():
 
 
 @pytest.mark.skipif(not has_selinux(), reason="selinux not enabled")
-@pytest.mark.parametrize("image_type", SUPPORTED_IMAGE_TYPES, indirect=["image_type"])
+@pytest.mark.parametrize("image_type", gen_testcases("direct-boot"), indirect=["image_type"])
 def test_image_build_without_se_linux_denials(image_type):
     # the journal always contains logs from the image building
     assert image_type.journal_output != ""
@@ -222,7 +217,7 @@ def test_image_build_without_se_linux_denials(image_type):
 
 
 @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now")
-@pytest.mark.parametrize("image_type", INSTALLER_IMAGE_TYPES, indirect=["image_type"])
+@pytest.mark.parametrize("image_type", gen_testcases("iso"), indirect=["image_type"])
 def test_iso_installs(image_type):
     installer_iso_path = image_type.img_path
     test_disk_path = installer_iso_path.with_name("test-disk.img")

test/test_manifest.py

@@ -9,13 +9,17 @@
 if not testutil.has_executable("podman"):
     pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True)
 
-from containerbuild import build_container_fixture, container_to_build_ref  # noqa: F401
+from containerbuild import build_container_fixture  # noqa: F401
+from testcases import gen_testcases
 
 
-def test_manifest_smoke(build_container):
+@pytest.mark.parametrize("image_type", gen_testcases("manifest"))
+def test_manifest_smoke(build_container, image_type):
+    container_ref = image_type.split(",")[0]
+
     output = subprocess.check_output([
         "podman", "run", "--rm",
-        f'--entrypoint=["/usr/bin/bootc-image-builder", "manifest", "{container_to_build_ref()}"]',
+        f'--entrypoint=["/usr/bin/bootc-image-builder", "manifest", "{container_ref}"]',
         build_container,
     ])
     manifest = json.loads(output)

test/testcases.py

@@ -0,0 +1,53 @@
+import os
+
+
+def gen_testcases(what):
+    # supported images that can be directly booted
+    DIRECT_BOOT_IMAGE_TYPES = ("qcow2", "ami", "raw")
+    # supported images that require an install
+    INSTALLER_IMAGE_TYPES = ("iso",)
+
+    # bootc containers that are tested by default
+    CONTAINERS_TO_TEST = {
+        "fedora": "quay.io/centos-bootc/fedora-bootc:eln",
+        "centos": "quay.io/centos-bootc/centos-bootc:stream9",
+    }
+    # allow commandline override, this is used when testing
+    # custom images
+    if os.getenv("BIB_TEST_BOOTC_CONTAINER_TAG"):
+        # TODO: make this more elegant
+        CONTAINERS_TO_TEST = {
+            "centos": os.getenv("BIB_TEST_BOOTC_CONTAINER_TAG"),
+            "fedora": [],
+        }
+
+    if what == "manifest":
+        return CONTAINERS_TO_TEST.values()
+    elif what == "ami-boot":
+        return [cnt + ",ami" for cnt in CONTAINERS_TO_TEST.values()]
+    elif what == "iso":
+        test_cases = []
+        # only fedora right now, centos iso installer is broken right now:
+        # https://github.com/osbuild/bootc-image-builder/issues/157
+        cnt = CONTAINERS_TO_TEST["fedora"]
+        for img_type in INSTALLER_IMAGE_TYPES:
+            test_cases.append(f"{cnt},{img_type}")
+        return test_cases
+    elif what == "direct-boot":
+        # skip some raw/ami tests (they are identical right now) to
+        # avoid overlong test runs but revisit this later and maybe just
+        # do more in parallel?
+        test_cases = [
+            CONTAINERS_TO_TEST["centos"] + "," + DIRECT_BOOT_IMAGE_TYPES[0],
+            CONTAINERS_TO_TEST["fedora"] + "," + DIRECT_BOOT_IMAGE_TYPES[1],
+            CONTAINERS_TO_TEST["centos"] + "," + DIRECT_BOOT_IMAGE_TYPES[2],
+            CONTAINERS_TO_TEST["fedora"] + "," + DIRECT_BOOT_IMAGE_TYPES[0],
+        ]
+        return test_cases
+    elif what == "all":
+        test_cases = []
+        for cnt in CONTAINERS_TO_TEST.values():
+            for img_type in DIRECT_BOOT_IMAGE_TYPES + INSTALLER_IMAGE_TYPES:
+                test_cases.append(f"{cnt},{img_type}")
+        return test_cases
+    raise ValueError(f"unknown test-case type {what}")