From 927cc3a84875d07d51fa963271c6c1d1c023610c Mon Sep 17 00:00:00 2001
From: Michael Vogt <michael.vogt@gmail.com>
Date: Fri, 5 Jan 2024 10:58:03 +0100
Subject: [PATCH] github,tmt: run the bib integration tests

This commit runs parameterized `bootc-image-builder` integration
tests as part of the testing. The parameterization requires [0].
The test matrix right now is to generated a "qcow2" and "ami" image
and for both:
1. test_image_is_generated(): just checks that the image can be build
2. test_image_boots(): the image boots in qemu and can be accessed
   via ssh [1]
3. test_image_build_without_se_linux_denials(): ensure that the
   image is build without selinux denials

Only test (3) strictly requires testing farm because it can only run
inside a host that runs with a RHEL/fedora kernel and selinux enabled
and it is not relevant for your testing (only required for
bootc-image-builder). This is why this commit also adds a normal
GH action - they support building the images and booting them
with qemu so for this project testing farm is slightly less
important.

We can run arbitrary post-boot tests in step (2) if we want, at
this point we have a ssh connection to the booted image so we
can do any post-boot checks we consider important (code for this
needs to be written though but shoudl be easy).

Caveats:
- bootc-image-builder cannot build quay.io/centos-bootc/centos-bootc:stream9
  right now [2]. This is being worked on and we could start with
  testing the quay.io/centos-bootc/fedora-bootc:eln images until bib
  supports centos.
- `BIB_TEST_BOOTC_CONTAINER_TAG` is not set correctly right now, it
  needs to be parameterized with a reference to the bootc container that
  just got build and now needs testing.

Some ideas for further improvements:
- tests could be extracted from bootc-image-builder into a shared
  bootc-testing repo
- post-boot tests
- upgrade tests, e.g. by building an image from the current "stable"
  bootc first and then upgrading to the bootc container that is
  tested

I hope this is helpful!

[0] https://github.com/osbuild/bootc-image-builder/pull/87
[1] https://github.com/osbuild/bootc-image-builder/blob/main/test/test_smoke.py#L128
[2] https://github.com/osbuild/bootc-image-builder/issues/20
---
 .github/workflows/tests.yml | 37 +++++++++++++++++++++++++++++++++++++
 plans/bib-tests.fmf         | 27 +++++++++++++++++++++++++++
 plans/example.fmf           |  7 -------
 3 files changed, 64 insertions(+), 7 deletions(-)
 create mode 100644 .github/workflows/tests.yml
 create mode 100644 plans/bib-tests.fmf
 delete mode 100644 plans/example.fmf

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 00000000..c0e21a64
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,37 @@
+---
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - main
+  # for merge queue
+  merge_group:
+
+jobs:
+  integration:
+    name: "Integration"
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup up python
+      uses: actions/setup-python@v5
+    - name: Install test dependencies
+      run: |
+        sudo apt install -y podman python3-pytest python3-paramiko flake8 qemu-system-x86
+    - name: Run tests
+      run: |
+        # XXX: move tests out of main bootc-image-builder repo into a shared one?
+        git clone https://github.com/osbuild/bootc-image-builder
+        cd bootc-image-builder
+        BIB_TEST_BUILD_CONTAINER_TAG=quay.io/centos-bootc/bootc-image-builder:latest \
+        # TODO: replace with the freshly generated bootc image that needs to
+        # get tested
+        BIB_TEST_BOOTC_CONTAINER_TAG=quay.io/centos-bootc/fedora-bootc:eln \
+          # podman needs (parts of) the environment but will break when
+          # XDG_RUNTIME_DIR is set.
+          # TODO: figure out what exactly podman needs
+          sudo -E XDG_RUNTIME_DIR= pytest-3 -s -vv
diff --git a/plans/bib-tests.fmf b/plans/bib-tests.fmf
new file mode 100644
index 00000000..71e660d4
--- /dev/null
+++ b/plans/bib-tests.fmf
@@ -0,0 +1,27 @@
+summary: Run all tests inside a VM environment
+provision:
+  how: virtual
+  image: fedora:39
+  hardware:
+    virtualization:
+      is-supported: true
+prepare:
+  how: install
+  package:
+    - podman
+    - pytest
+    - python3-flake8
+    - python3-paramiko
+    - qemu-kvm
+execute:
+  how: tmt
+  script: |
+    # XXX: move tests out of main bootc-image-builder repo into a shared one?
+    git clone https://github.com/osbuild/bootc-image-builder
+    cd bootc-image-builder
+    # testing farm runs as root already
+    BIB_TEST_BUILD_CONTAINER_TAG=quay.io/centos-bootc/bootc-image-builder:latest \
+    # TODO: replace with the freshly generated bootc image that needs to
+    # get tested
+    BIB_TEST_BOOTC_CONTAINER_TAG=quay.io/centos-bootc/fedora-bootc:eln \
+      pytest -s -vv
diff --git a/plans/example.fmf b/plans/example.fmf
deleted file mode 100644
index cc29ed31..00000000
--- a/plans/example.fmf
+++ /dev/null
@@ -1,7 +0,0 @@
-summary: Example test
-execute:
-    how: tmt
-    script: |
-        set +x
-        dnf install -y tmt
-        tmt --help