From 391db6aea125b64c996ffceddc8be7c031758b52 Mon Sep 17 00:00:00 2001
From: Michael Vogt <michael.vogt@gmail.com>
Date: Fri, 5 Jan 2024 10:58:03 +0100
Subject: [PATCH] github,tmt: run the bib integration tests

This commit runs parametrized `bootc-image-builder` integration
tests as part of the testing. The parametrization requires [0].
The test matrix right now is to generated a "qcow2" and "ami" image
and for both:
1. test_image_is_generated(): just checks that the image can be build
2. test_image_boots(): the image boots in qemu and can be accessed
   via ssh [1]
3. test_image_build_without_se_linux_denials(): ensure that the
   image is build without selinux denials

We can run arbitrary post-boot tests in step (2) if we want, at
this point we have a ssh connection to the booted image so we
can do any post-boot checks we consider important (code for this
needs to be written though but should be easy).

Caveats:
- bootc-image-builder cannot build quay.io/centos-bootc/centos-bootc:stream9
  right now [2]. This is being worked on and we could start with
  testing the quay.io/centos-bootc/fedora-bootc:eln images until bib
  supports centos.

Some ideas for further improvements:
- tests could be extracted from bootc-image-builder into a shared
  bootc-testing repo
- post-boot tests
- upgrade tests, e.g. by building an image from the current "stable"
  bootc first and then upgrading to the bootc container that is
  tested

I hope this is helpful!

[0] https://github.com/osbuild/bootc-image-builder/pull/87
[1] https://github.com/osbuild/bootc-image-builder/blob/main/test/test_smoke.py#L128
[2] https://github.com/osbuild/bootc-image-builder/issues/20
---
 plans/bib-tests.fmf | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 plans/bib-tests.fmf

diff --git a/plans/bib-tests.fmf b/plans/bib-tests.fmf
new file mode 100644
index 00000000..eafa4c08
--- /dev/null
+++ b/plans/bib-tests.fmf
@@ -0,0 +1,34 @@
+summary: Run all tests inside a VM environment
+provision:
+  how: virtual
+  image: fedora:39
+  hardware:
+    virtualization:
+      is-supported: true
+prepare:
+  how: install
+  package:
+    - jq
+    - podman
+    - pytest
+    - python3-flake8
+    - python3-paramiko
+    - qemu-kvm
+execute:
+  how: tmt
+  script: |
+    # XXX: move tests out of main bootc-image-builder repo into a shared one?
+    git clone https://github.com/osbuild/bootc-image-builder
+    cd bootc-image-builder
+
+    echo "$SNAPSHOT" | base64 -d
+    # tmt injects ${SNAPSHOT} as a string
+    IMAGES=$(echo '${SNAPSHOT}' | base64 -d | jq -r '.components[].containerImage')
+
+    for IMAGE in $IMAGES; do
+        echo "Running integration tests for image: $IMAGE"
+        # use "stable" bootc image builder to generate the images
+        BIB_TEST_BUILD_CONTAINER_TAG=quay.io/centos-bootc/bootc-image-builder:latest \
+        BIB_TEST_BOOTC_CONTAINER_TAG="$IMAGE" \
+      pytest -s -vv
+    done