Boot test AMIs in AWS

.github/workflows/tests.yml

@@ -70,7 +70,7 @@ jobs:
     - name: Install test dependencies
       run: |
         sudo apt update
-        sudo apt install -y podman python3-pytest python3-paramiko flake8 qemu-system-x86
+        sudo apt install -y podman python3-pytest python3-paramiko python3-boto3 flake8 qemu-system-x86
     - name: Run tests
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

plans/all.fmf

@@ -10,9 +10,10 @@ prepare:
   package:
     - podman
     - pytest
+    - python3-boto3
     - python3-flake8
     - python3-paramiko
     - qemu-kvm
 execute:
   how: tmt
-  script: pytest -s -vv
+  script: pytest -s -vv --force-aws-upload

test/conftest.py

@@ -0,0 +1,12 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption("--force-aws-upload", action="store_true", default=False,
+                     help=("Force AWS upload when building AMI, failing if credentials are not set. "
+                           "If not set, the upload will be performed only when credentials are available."))
+
+
+@pytest.fixture(name="force_aws_upload", scope="session")
+def force_aws_upload_fixture(request):
+    return request.config.getoption("--force-aws-upload")

test/requirements.txt

@@ -1,3 +1,4 @@
 pytest==7.4.3
 flake8==6.1.0
 paramiko==2.12.0
+boto3==1.33.13

test/test_smoke.py

@@ -4,13 +4,15 @@
 import platform
 import re
 import subprocess
+import tempfile
+import uuid
 from typing import NamedTuple
 
 import pytest
 
 # local test utils
 import testutil
-from vm import VM
+from vm import AWS, QEMU
 
 if not testutil.has_executable("podman"):
     pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True)
@@ -44,14 +46,16 @@ def build_container_fixture():
 
 
 class ImageBuildResult(NamedTuple):
+    img_type: str
     img_path: str
     username: str
     password: str
     journal_output: str
+    metadata: dict = {}
 
 
 @pytest.fixture(name="image_type", scope="session")
-def image_type_fixture(tmpdir_factory, build_container, request):
+def image_type_fixture(tmpdir_factory, build_container, request, force_aws_upload):
     """
     Build an image inside the passed build_container and return an
     ImageBuildResult with the resulting image path and user/password
@@ -84,7 +88,7 @@ def image_type_fixture(tmpdir_factory, build_container, request):
     # if the fixture already ran and generated an image, use that
     if generated_img.exists():
         journal_output = journal_log_path.read_text(encoding="utf8")
-        return ImageBuildResult(generated_img, username, password, journal_output)
+        return ImageBuildResult(image_type, generated_img, username, password, journal_output)
 
     # no image yet, build it
     CFG = {
@@ -105,22 +109,52 @@ def image_type_fixture(tmpdir_factory, build_container, request):
     config_json_path.write_text(json.dumps(CFG), encoding="utf-8")
 
     cursor = testutil.journal_cursor()
-    # run container to deploy an image into output/qcow2/disk.qcow2
-    subprocess.check_call([
-        "podman", "run", "--rm",
-        "--privileged",
-        "--security-opt", "label=type:unconfined_t",
-        "-v", f"{output_path}:/output",
-        "-v", "/store",  # share the cache between builds
-        build_container,
-        container_to_build_ref,
-        "--config", "/output/config.json",
-        "--type", image_type,
-    ])
+
+    upload_args = []
+    creds_args = []
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        if image_type == "ami":
+            creds_file = pathlib.Path(tempdir) / "aws.creds"
+            if testutil.write_aws_creds(creds_file):
+                creds_args = ["-v", f"{creds_file}:/root/.aws/credentials:ro",
+                              "--env", "AWS_PROFILE=default"]
+
+                upload_args = [
+                    f"--aws-ami-name=bootc-image-builder-test-{str(uuid.uuid4())}",
+                    f"--aws-region={testutil.AWS_REGION}",
+                    "--aws-bucket=bootc-image-builder-ci",
+                ]
+            elif force_aws_upload:
+                # upload forced but credentials aren't set
+                raise RuntimeError("AWS credentials not available (upload forced)")
+
+        # run container to deploy an image into a bootable disk and upload to a cloud service if applicable
+        subprocess.check_call([
+            "podman", "run", "--rm",
+            "--privileged",
+            "--security-opt", "label=type:unconfined_t",
+            "-v", f"{output_path}:/output",
+            "-v", "/store",  # share the cache between builds
+            *creds_args,
+            build_container,
+            container_to_build_ref,
+            "--config", "/output/config.json",
+            "--type", image_type,
+            *upload_args,
+        ])
     journal_output = testutil.journal_after_cursor(cursor)
+    metadata = {}
+    if image_type == "ami" and upload_args:
+        metadata["ami_id"] = parse_ami_id_from_log(journal_output)
+
+        def del_ami():
+            testutil.deregister_ami(metadata["ami_id"])
+        request.addfinalizer(del_ami)
+
     journal_log_path.write_text(journal_output, encoding="utf8")
 
-    return ImageBuildResult(generated_img, username, password, journal_output)
+    return ImageBuildResult(image_type, generated_img, username, password, journal_output, metadata)
 
 
 def test_container_builds(build_container):
@@ -138,7 +172,23 @@ def test_image_is_generated(image_type):
 @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now")
 @pytest.mark.parametrize("image_type", SUPPORTED_IMAGE_TYPES, indirect=["image_type"])
 def test_image_boots(image_type):
-    with VM(image_type.img_path) as test_vm:
+    with QEMU(image_type.img_path) as test_vm:
+        exit_status, _ = test_vm.run("true", user=image_type.username, password=image_type.password)
+        assert exit_status == 0
+        exit_status, output = test_vm.run("echo hello", user=image_type.username, password=image_type.password)
+        assert exit_status == 0
+        assert "hello" in output
+
+
+@pytest.mark.parametrize("image_type", ["ami"], indirect=["image_type"])
+def test_ami_boots_in_aws(image_type, force_aws_upload):
+    if not testutil.write_aws_creds("/dev/null"):  # we don't care about the file, just the variables being there
+        if force_aws_upload:
+            # upload forced but credentials aren't set
+            raise RuntimeError("AWS credentials not available")
+        pytest.skip("AWS credentials not available (upload not forced)")
+
+    with AWS(image_type.metadata["ami_id"]) as test_vm:
         exit_status, _ = test_vm.run("true", user=image_type.username, password=image_type.password)
         assert exit_status == 0
         exit_status, output = test_vm.run("echo hello", user=image_type.username, password=image_type.password)
@@ -151,6 +201,13 @@ def log_has_osbuild_selinux_denials(log):
     return re.search(OSBUID_SELINUX_DENIALS_RE, log)
 
 
+def parse_ami_id_from_log(log_output):
+    ami_id_re = re.compile(r"AMI registered: (?P<ami_id>ami-[a-z0-9]+)\n")
+    ami_ids = ami_id_re.findall(log_output)
+    assert len(ami_ids) > 0
+    return ami_ids[0]
+
+
 def test_osbuild_selinux_denials_re_works():
     fake_log = (
         'Dec 05 07:19:39 other log msg\n'

test/testutil.py

@@ -1,11 +1,16 @@
 import os
 import pathlib
 import platform
-import socket
 import shutil
+import socket
 import subprocess
 import time
 
+import boto3
+from botocore.exceptions import ClientError
+
+AWS_REGION = "us-east-1"
+
 
 def run_journalctl(*args):
     pre = []
@@ -37,12 +42,12 @@ def get_free_port() -> int:
         return s.getsockname()[1]
 
 
-def wait_ssh_ready(port, sleep, max_wait_sec):
+def wait_ssh_ready(address, port, sleep, max_wait_sec):
     for i in range(int(max_wait_sec / sleep)):
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
             s.settimeout(sleep)
             try:
-                s.connect(("localhost", port))
+                s.connect((address, port))
                 data = s.recv(256)
                 if b"OpenSSH" in data:
                     return
@@ -75,3 +80,31 @@ def can_start_rootful_containers():
             return res.stdout.strip() == "true"
         case unknown:
             raise ValueError(f"unknown platform {unknown}")
+
+
+def write_aws_creds(path):
+    key_id = os.environ.get("AWS_ACCESS_KEY_ID")
+    secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
+    if not key_id or not secret_key:
+        return False
+
+    with open(path, mode="w", encoding="utf-8") as creds_file:
+        creds_file.write("[default]\n")
+        creds_file.write(f"aws_access_key_id = {key_id}\n")
+        creds_file.write(f"aws_secret_access_key = {secret_key}\n")
+
+    return True
+
+
+def deregister_ami(ami_id):
+    ec2 = boto3.resource("ec2", region_name=AWS_REGION)
+    try:
+        print(f"Deregistering image {ami_id}")
+        ami = ec2.Image(ami_id)
+        ami.deregister()
+        print("Image deregistered")
+    except ClientError as err:
+        err_code = err.response["Error"]["Code"]
+        err_msg = err.response["Error"]["Message"]
+        print(f"Couldn't deregister image {ami_id}.")
+        print(f"Error {err_code}: {err_msg}")

test/testutil_test.py

@@ -5,7 +5,7 @@
 
 import pytest
 
-from testutil import has_executable, get_free_port, wait_ssh_ready
+from testutil import get_free_port, has_executable, wait_ssh_ready
 
 
 def test_get_free_port():
@@ -21,7 +21,7 @@ def free_port_fixture():
 @patch("time.sleep")
 def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep, free_port):
     with pytest.raises(ConnectionRefusedError):
-        wait_ssh_ready(free_port, sleep=0.1, max_wait_sec=0.35)
+        wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.35)
     assert mocked_sleep.call_args_list == [call(0.1), call(0.1), call(0.1)]
 
 
@@ -45,7 +45,7 @@ def test_wait_ssh_ready_sleeps_wrong_reply(free_port, tmp_path):
         # now connect
         with patch("time.sleep") as mocked_sleep:
             with pytest.raises(ConnectionRefusedError):
-                wait_ssh_ready(free_port, sleep=0.1, max_wait_sec=0.55)
+                wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.55)
             assert mocked_sleep.call_args_list == [
                 call(0.1), call(0.1), call(0.1), call(0.1), call(0.1)]
 
@@ -56,4 +56,4 @@ def test_wait_ssh_ready_integration(free_port, tmp_path):
     with contextlib.ExitStack() as cm:
         p = subprocess.Popen(f"echo OpenSSH | nc -l -p {free_port}", shell=True)
         cm.callback(p.kill)
-        wait_ssh_ready(free_port, sleep=0.1, max_wait_sec=10)
+        wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=10)