diff --git a/conftest.py b/conftest.py index 738b6842..1be3669f 100644 --- a/conftest.py +++ b/conftest.py @@ -169,6 +169,7 @@ def setup_host(hostname_or_ip, *, config=None): vif = host_vm.vifs()[0] mac_address = vif.param_get('MAC') logging.info("Nested host has MAC %s", mac_address) + pxe.arp_clear_for(mac_address) host_vm.start() wait_for(host_vm.is_running, "Wait for nested host VM running") diff --git a/data.py-dist b/data.py-dist index 89be8bb2..60bce6cb 100644 --- a/data.py-dist +++ b/data.py-dist @@ -43,6 +43,7 @@ HOSTS_IP_CONFIG = { 'HOSTS': { # 'DEFAULT': '192.16.0.1', # 'host2': '192.16.0.2', +# 'host3': '192.16.0.3', }, # 'NETMASK': '255.255.0.0', # 'GATEWAY': '192.16.0.254', diff --git a/jobs.py b/jobs.py index aa5e24f1..ac9eeb49 100755 --- a/jobs.py +++ b/jobs.py @@ -2,6 +2,7 @@ import argparse import json +import logging import subprocess import sys from lib.commands import ssh @@ -487,6 +488,8 @@ # running quicktest on zfsvol generates dangling TAP devices that are hard to # cleanup. Bug needs to be fixed before enabling quicktest on zfsvol. "tests/storage/zfsvol/test_zfsvol_sr.py::TestZfsvolVm::test_quicktest", + # not meant to be run from jobs.py (yet) + "tests/install/test_pool.py", ] # Returns the vm filename or None if a host_version is passed and matches the one specified @@ -710,6 +713,7 @@ def action_run(args): sys.exit(1) def main(): + logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.DEBUG) parser = argparse.ArgumentParser(description="Manage test jobs") parser.add_argument("-v", "--host-version", help="host version to match VM filters.") subparsers = parser.add_subparsers(dest="action", metavar="action") diff --git a/lib/commands.py b/lib/commands.py index 40f8634e..fd756ed1 100644 --- a/lib/commands.py +++ b/lib/commands.py @@ -62,14 +62,14 @@ def _ellide_log_lines(log): def _ssh(hostname_or_ip, cmd, check, simple_output, suppress_fingerprint_warnings, background, target_os, decode, options): opts = list(options) - opts.append('-o "BatchMode yes"') + opts.append('-o BatchMode=yes') if suppress_fingerprint_warnings: # Suppress warnings and questions related to host key fingerprints # because on a test network IPs get reused, VMs are reinstalled, etc. # Based on https://unix.stackexchange.com/a/365976/257493 - opts.append('-o "StrictHostKeyChecking no"') - opts.append('-o "LogLevel ERROR"') - opts.append('-o "UserKnownHostsFile /dev/null"') + opts.extend(['-o StrictHostKeyChecking=no', + '-o LogLevel=ERROR', + '-o UserKnownHostsFile=/dev/null']) command = " ".join(cmd) if background and target_os != "windows": @@ -216,7 +216,7 @@ def local_cmd(cmd, check=True, decode=True): errorcode_msg = "" if res.returncode == 0 else " - Got error code: %s" % res.returncode command = " ".join(cmd) - logging.debug(f"[local] {command}{errorcode_msg}{_ellide_log_lines(output_for_logs)}") + logging.debug(f"[local] {command}{errorcode_msg}, output: {_ellide_log_lines(output_for_logs)}") if res.returncode and check: raise LocalCommandFailed(res.returncode, output_for_logs, command) diff --git a/lib/host.py b/lib/host.py index 1cd76667..d7569962 100644 --- a/lib/host.py +++ b/lib/host.py @@ -434,6 +434,17 @@ def yum_restore_saved_state(self): self.saved_packages_list = None self.saved_rollback_id = None + def shutdown(self, verify=False): + logging.info("Shutdown host %s" % self) + try: + self.ssh(['shutdown']) + except commands.SSHCommandFailed as e: + # ssh connection may get killed by the shutdown and terminate with an error code + if "closed by remote host" not in e.stdout: + raise + if verify: + wait_for_not(self.is_enabled, "Wait for host down") + def reboot(self, verify=False): logging.info("Reboot host %s" % self) try: diff --git a/lib/installer.py b/lib/installer.py index 253849e2..6186520e 100644 --- a/lib/installer.py +++ b/lib/installer.py @@ -1,8 +1,10 @@ import logging +import os import time import xml.etree.ElementTree as ET -from lib.commands import ssh, SSHCommandFailed +from lib import pxe +from lib.commands import local_cmd, scp, ssh, SSHCommandFailed from lib.common import wait_for # FIXME should only be used for <7.0 @@ -143,6 +145,56 @@ def monitor_upgrade(*, ip): ).returncode == 1, "Wait for installer to terminate") +# FIXME essentially duplicates vm_booted_with_installer fixture +def perform_upgrade(*, iso, host_vm, host): + vif = host_vm.vifs()[0] + mac_address = vif.param_get('MAC') + logging.info("Host VM has MAC %s", mac_address) + + try: + remote_iso = host.pool.push_iso(iso) + host_vm.insert_cd(os.path.basename(remote_iso)) + + try: + pxe.arp_clear_for(mac_address) + + host_vm.start() + wait_for(host_vm.is_running, "Wait for host VM running") + + # catch host-vm IP address + wait_for(lambda: pxe.arp_addresses_for(mac_address), + "Wait for DHCP server to see Host VM in ARP tables", + timeout_secs=10 * 60) + ips = pxe.arp_addresses_for(mac_address) + logging.info("Host VM has IPs %s", ips) + assert len(ips) == 1 + host_vm.ip = ips[0] + + # host may not be up if ARP cache was filled + wait_for(lambda: local_cmd(["ping", "-c1", host_vm.ip], check=False), + "Wait for host up", timeout_secs=10 * 60, retry_delay_secs=10) + wait_for(lambda: local_cmd(["nc", "-zw5", host_vm.ip, "22"], check=False), + "Wait for ssh up on host", timeout_secs=10 * 60, retry_delay_secs=5) + + yield host_vm + + logging.info("Shutting down Host VM") + poweroff(host_vm.ip) + wait_for(host_vm.is_halted, "Wait for host VM halted") + + except Exception as e: + logging.critical("caught exception %s", e) + host_vm.shutdown(force=True) + raise + except KeyboardInterrupt: + logging.warning("keyboard interrupt") + host_vm.shutdown(force=True) + raise + + host_vm.eject_cd() + finally: + host.pool.remove_iso(remote_iso) + def monitor_restore(*, ip): # wait for "yum install" phase to start wait_for(lambda: ssh(ip, ["grep", diff --git a/lib/pxe.py b/lib/pxe.py index eb004a66..73cf2934 100644 --- a/lib/pxe.py +++ b/lib/pxe.py @@ -45,3 +45,10 @@ def arp_addresses_for(mac_address): ) candidate_ips = output.splitlines() return candidate_ips + +def arp_clear_for(mac_address): + for stray_ip in arp_addresses_for(mac_address): + output = ssh( + PXE_CONFIG_SERVER, + ['arp', '-d', stray_ip] + ) diff --git a/scripts/gen-indirect-installs.sh b/scripts/gen-indirect-installs.sh new file mode 100755 index 00000000..4aaf9e7c --- /dev/null +++ b/scripts/gen-indirect-installs.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +# creation of reference installations to be used for basis of upgrade tests + +# * needs at least --hosts=$NEST +# * ch821 and xs8 needs ISO in the cache + +. $(dirname $0)/lib.bash + +TESTCLASS="tests/install/test.py::TestNested" +for conf in uefi-75+821.1-iso-ext; do + IFS=- read fw versions media sr < <(echo "$conf") + IFS=+ read origversion stepversion < <(echo "$versions") + TESTS=( + $TESTCLASS::test_install[$fw-$origversion-$media-$sr] + $TESTCLASS::test_tune_firstboot[None-$fw-$origversion-host1-$media-$sr] + $TESTCLASS::test_boot_inst[$fw-$origversion-host1-$media-$sr] + + $TESTCLASS::test_upgrade[$fw-$origversion-$stepversion-host1-$media-$sr] + $TESTCLASS::test_boot_upg[$fw-$origversion-$stepversion-host1-$media-$sr] + + #$TESTCLASS::test_upgrade[$fw-$origversion-$stepversion-83nightly-host1-$media-$sr] + ) + run_pytest "$conf" \ + --log-file=test-genref-$conf.log \ + --reruns=5 --only-rerun=TimeoutError \ + "$@" \ + "${TESTS[@]}" +done + +report_failures diff --git a/scripts/lib.bash b/scripts/lib.bash index 4ae2d5c9..312ff16e 100644 --- a/scripts/lib.bash +++ b/scripts/lib.bash @@ -46,6 +46,8 @@ run_pytest() { # reference configurations, to be upgraded to nightly +# FIXME this is also where we take the list to create all ref +# installs, 75/76 should be separated REFVERSIONS=( 83rc1 83b2 83b1 821.1 81 80 diff --git a/tests/install/conftest.py b/tests/install/conftest.py index 162f57ca..a5235a80 100644 --- a/tests/install/conftest.py +++ b/tests/install/conftest.py @@ -116,7 +116,22 @@ def installer_iso(request): @pytest.fixture(scope='function') def install_disk(request): firmware = request.getfixturevalue("firmware") - yield {"uefi": "nvme0n1", "bios": "sda"}[firmware] + if firmware.startswith("uefi"): + yield "nvme0n1" + elif firmware.startswith("bios"): + yield "sda" + else: + assert False, f"unknown firmware {firmware!r}" + +@pytest.fixture(scope='function') +def answerfile_maybe_tweak_parttable(request, answerfile): + firmware = request.getfixturevalue("firmware") + if firmware.endswith("+dell"): + answerfile.top_append(dict(TAG="script", stage="installation-start", + type="url", CONTENTS="file:///root/preinstall-utilitypart.sh")) + if firmware.endswith("+mbr"): + answerfile.top_append(dict(TAG="script", stage="installation-start", + type="url", CONTENTS="file:///root/preinstall-mbrparttable.sh")) # Remasters the ISO sepecified by `installer_iso` mark, with: # - network and ssh support activated, and .ssh/authorized_key so tests can @@ -131,7 +146,7 @@ def install_disk(request): # in contexts where the same IP is reused by successively different MACs # (when cloning VMs from cache) @pytest.fixture(scope='function') -def remastered_iso(installer_iso, answerfile): +def remastered_iso(installer_iso, answerfile, install_disk): iso_file = installer_iso['iso'] unsigned = installer_iso['unsigned'] @@ -230,6 +245,30 @@ def remastered_iso(installer_iso, answerfile): chmod +x "$INSTALLIMG/etc/init.d/S12test-pingpxe" fi +cat > "$INSTALLIMG/root/preinstall-utilitypart.sh" <<'EOF' +#!/bin/sh +set -ex + +# Dell utility partition +sgdisk --zap-all /dev/{install_disk} +sfdisk /dev/{install_disk} << 'EOP' +unit: sectors +p1 : start= 2048, size= 32768, Id=de +EOP +EOF + +cat > "$INSTALLIMG/root/preinstall-mbrparttable.sh" <<'EOF' +#!/bin/sh +set -ex + +# Dell utility partition +sgdisk --zap-all /dev/{install_disk} +sfdisk /dev/{install_disk} << 'EOP' +unit: sectors +p1 : start= 2048, size= 32768, Id=83 +EOP +EOF + cat > "$INSTALLIMG/root/postinstall.sh" <<'EOF' #!/bin/sh set -ex @@ -299,6 +338,8 @@ def vm_booted_with_installer(host, remastered_iso, create_vms): host_vm.insert_cd(os.path.basename(remote_iso)) try: + pxe.arp_clear_for(mac_address) + host_vm.start() wait_for(host_vm.is_running, "Wait for host VM running") diff --git a/tests/install/test-sequences/uefi-82-nosr+pool+rpu83nightly.lst b/tests/install/test-sequences/uefi-82-nosr+pool+rpu83nightly.lst new file mode 100644 index 00000000..af37c15e --- /dev/null +++ b/tests/install/test-sequences/uefi-82-nosr+pool+rpu83nightly.lst @@ -0,0 +1 @@ +tests/install/test_pool.py::test_pool_rpu[uefi-821.1-83nightly] diff --git a/tests/install/test-sequences/uefi-82-nosr+pool.lst b/tests/install/test-sequences/uefi-82-nosr+pool.lst new file mode 100644 index 00000000..cfe2d5ff --- /dev/null +++ b/tests/install/test-sequences/uefi-82-nosr+pool.lst @@ -0,0 +1,2 @@ +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-821.1-host2-iso-nosr] +tests/install/test.py::TestNested::test_boot_inst[uefi-821.1-host2-iso-nosr] diff --git a/tests/install/test-sequences/uefi-82-nosr.lst b/tests/install/test-sequences/uefi-82-nosr.lst new file mode 100644 index 00000000..3f6caffe --- /dev/null +++ b/tests/install/test-sequences/uefi-82-nosr.lst @@ -0,0 +1,3 @@ +tests/install/test.py::TestNested::test_install[uefi-821.1-iso-nosr] +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-821.1-host1-iso-nosr] +tests/install/test.py::TestNested::test_boot_inst[uefi-821.1-host1-iso-nosr] diff --git a/tests/install/test.py b/tests/install/test.py index a13de31c..cfd4f46f 100644 --- a/tests/install/test.py +++ b/tests/install/test.py @@ -32,7 +32,7 @@ class TestNested: "xs8", "ch821.1", "xs70", "xs65", )) - @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.parametrize("firmware", ("uefi", "bios", "bios+dell", "bios+mbr")) @pytest.mark.vm_definitions( lambda firmware: dict( name="vm1", @@ -52,6 +52,8 @@ class TestNested: dict(param_name="platform", key="device-model", value="qemu-upstream-uefi"), ), "bios": (), + "bios+dell": (), + "bios+mbr": (), }[firmware], vdis=[dict(name="vm1 system disk", size="100GiB", device="xvda", userdevice="0")], cd_vbd=dict(device="xvdd", userdevice="3"), @@ -73,7 +75,8 @@ class TestNested: param_mapping={"install_disk": "install_disk", "local_sr": "local_sr", "source_type": "source_type", "version": "iso_version", }) - def test_install(self, vm_booted_with_installer, install_disk, + def test_install(self, answerfile_maybe_tweak_parttable, + vm_booted_with_installer, install_disk, firmware, iso_version, source_type, local_sr): host_vm = vm_booted_with_installer installer.monitor_install(ip=host_vm.ip) @@ -101,7 +104,7 @@ def helper_vm_with_plugged_disk(running_vm, create_vms): @pytest.mark.usefixtures("xcpng_chained") @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm")) @pytest.mark.parametrize("source_type", ("iso", "net")) - @pytest.mark.parametrize("machine", ("host1", "host2")) + @pytest.mark.parametrize("machine", ("host1", "host2", "host3")) @pytest.mark.parametrize("version", ( "83nightly", "83rcnet", "83rc1", "83b2", "83b1", @@ -111,7 +114,7 @@ def helper_vm_with_plugged_disk(running_vm, create_vms): "ch821.1", "xs8", "xs70", "xs65", )) - @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.parametrize("firmware", ("uefi", "bios", "bios+dell", "bios+mbr")) @pytest.mark.continuation_of( lambda version, firmware, local_sr, source_type: [dict( vm="vm1", @@ -123,7 +126,9 @@ def test_tune_firstboot(self, create_vms, helper_vm_with_plugged_disk, firmware, version, machine, local_sr, source_type): helper_vm = helper_vm_with_plugged_disk - helper_vm.ssh(["mount /dev/xvdb1 /mnt"]) + main_part = "/dev/xvdb2" if firmware.endswith("+dell") else "/dev/xvdb1" + + helper_vm.ssh(["mount", main_part, "/mnt"]) try: # hostname logging.info("Setting hostname to %r", machine) @@ -144,7 +149,7 @@ def test_tune_firstboot(self, create_vms, helper_vm_with_plugged_disk, '/mnt/etc/xensource-inventory']) helper_vm.ssh(["grep UUID /mnt/etc/xensource-inventory"]) finally: - helper_vm.ssh(["umount /dev/xvdb1"]) + helper_vm.ssh(["umount", main_part]) def _test_firstboot(self, create_vms, mode, *, machine='DEFAULT'): host_vm = create_vms[0] @@ -294,7 +299,7 @@ def _test_firstboot(self, create_vms, mode, *, machine='DEFAULT'): @pytest.mark.usefixtures("xcpng_chained") @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm")) @pytest.mark.parametrize("source_type", ("iso", "net")) - @pytest.mark.parametrize("machine", ("host1", "host2")) + @pytest.mark.parametrize("machine", ("host1", "host2", "host3")) @pytest.mark.parametrize("version", ( "83nightly", "83rcnet", "83rc1", "83b2", "83b1", @@ -304,7 +309,7 @@ def _test_firstboot(self, create_vms, mode, *, machine='DEFAULT'): "ch821.1", "xs8", "xs70", "xs65", )) - @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.parametrize("firmware", ("uefi", "bios", "bios+dell", "bios+mbr")) @pytest.mark.continuation_of( lambda version, firmware, machine, local_sr, source_type: [ dict(vm="vm1", @@ -334,7 +339,7 @@ def test_boot_inst(self, create_vms, "821.1-821.1", "75-821.1", )) - @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.parametrize("firmware", ("uefi", "bios", "bios+dell", "bios+mbr")) @pytest.mark.continuation_of( lambda mode, firmware, machine, source_type, local_sr: [dict( vm="vm1", @@ -362,7 +367,7 @@ def test_boot_upg(self, create_vms, "83rcnet-83rcnet", "83rcnet-83rcnet-83rcnet", # FIXME "821.1-821.1-821.1", )) - @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.parametrize("firmware", ("uefi", "bios", "bios+dell")) @pytest.mark.continuation_of( lambda mode, firmware, source_type, local_sr: [dict( vm="vm1", @@ -392,7 +397,7 @@ def test_boot_rst(self, create_vms, ("821.1", "821.1"), ("75", "821.1"), ]) - @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.parametrize("firmware", ("uefi", "bios", "bios+dell", "bios+mbr")) @pytest.mark.continuation_of( lambda firmware, params, machine, source_type, local_sr: [dict( vm="vm1", @@ -429,7 +434,7 @@ def test_upgrade(self, vm_booted_with_installer, install_disk, ("83rcnet-83rcnet", "83rcnet"), # FIXME ("821.1-821.1", "821.1"), ]) - @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.parametrize("firmware", ("uefi", "bios", "bios+dell")) @pytest.mark.continuation_of( lambda firmware, params, local_sr, source_type: [dict( vm="vm1", diff --git a/tests/install/test_pool.py b/tests/install/test_pool.py new file mode 100644 index 00000000..8dd9f404 --- /dev/null +++ b/tests/install/test_pool.py @@ -0,0 +1,154 @@ +import logging +import os +import pytest + +from lib import commands, installer, pxe +from lib.common import wait_for, vm_image +from lib.installer import AnswerFile +from lib.pool import Pool + +from data import HOSTS_IP_CONFIG, NFS_DEVICE_CONFIG + +MAINTESTS = "tests/install/test.py::TestNested" + +# FIXME without --ignore-unknown-dependency, SKIPPED +# "because it depends on tests/install/test.py::TestNested::test_firstboot_install[uefi-821.1-host1-iso-nosr]" +@pytest.mark.usefixtures("xcpng_chained") +@pytest.mark.parametrize(("orig_version", "iso_version"), [ + ("821.1", "83nightly"), +]) +@pytest.mark.parametrize("firmware", ("uefi", "bios")) +@pytest.mark.continuation_of( + lambda params, firmware: [ + dict(vm="vm1", + image_test=f"{MAINTESTS}::test_firstboot_install[{firmware}-{params}-host1-iso-nosr]", + scope="session"), + dict(vm="vm2", + image_vm="vm1", + image_test=f"{MAINTESTS}::test_firstboot_install[{firmware}-{params}-host2-iso-nosr]", + scope="session"), + ], + param_mapping={"params": "orig_version", "firmware": "firmware"}) +@pytest.mark.answerfile( + lambda firmware: AnswerFile("UPGRADE").top_append( + {"TAG": "source", "type": "local"}, + {"TAG": "existing-installation", "CONTENTS": {"uefi": "nvme0n1", "bios": "sda"}[firmware]}, + ), + param_mapping={"firmware": "firmware"}) +def test_pool_rpu(host, remastered_iso, create_vms, + firmware, orig_version, iso_version): + (master_vm, slave_vm) = create_vms + master_mac = master_vm.vifs()[0].param_get('MAC') + logging.info("Master VM has MAC %s", master_mac) + slave_mac = slave_vm.vifs()[0].param_get('MAC') + logging.info("Slave VM has MAC %s", slave_mac) + + master_vm.start() + slave_vm.start() + wait_for(master_vm.is_running, "Wait for master VM running") + wait_for(slave_vm.is_running, "Wait for slave VM running") + + master_vm.ip = HOSTS_IP_CONFIG['HOSTS']['DEFAULT'] + logging.info("Expecting master VM to have IP %s", master_vm.ip) + + slave_vm.ip = HOSTS_IP_CONFIG['HOSTS']['host2'] + logging.info("Expecting slave VM to have IP %s", slave_vm.ip) + + wait_for(lambda: not os.system(f"nc -zw5 {master_vm.ip} 22"), + "Wait for ssh up on Master VM", retry_delay_secs=5) + wait_for(lambda: not os.system(f"nc -zw5 {slave_vm.ip} 22"), + "Wait for ssh up on Slave VM", retry_delay_secs=5) + + pool = Pool(master_vm.ip) + + # create pool with shared SR + + slave = Pool(slave_vm.ip).master + slave.join_pool(pool) + + sr = pool.master.sr_create("nfs", "NFS Shared SR", NFS_DEVICE_CONFIG, + shared=True, verify=True) + + # create and start VMs + vms = ( + pool.master.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid), + pool.master.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid), + ) + + for vm in vms: + vm.start() + + wait_for(lambda: all(vm.is_running() for vm in vms), "Wait for VMs running") + wait_for(lambda: all(vm.try_get_and_store_ip() for vm in vms), + "Wait for VM IPs", timeout_secs=5 * 60) + wait_for(lambda: all(vm.is_management_agent_up() for vm in vms), + "Wait for management agents up") + + logging.info("VMs dispatched as %s", [vm.get_residence_host().uuid for vm in vms]) + + # do RPU + + # evacuate master + vms_to_migrate = [vm for vm in vms if vm.get_residence_host().uuid == pool.master.uuid] + logging.info("Expecting migration of %s", ([vm.uuid for vm in vms_to_migrate],)) + pool.master.xe("host-evacuate", {"host": pool.master.uuid}) + wait_for(lambda: all(vm.get_residence_host().uuid != pool.master.uuid for vm in vms_to_migrate), + "Wait for VM migration") + + # upgrade master + pool.master.shutdown() + wait_for(lambda: master_vm.is_halted(), "Wait for Master VM to be halted", timeout_secs=5 * 60) + installer.perform_upgrade(iso=remastered_iso, host_vm=master_vm, host=host) + pxe.arp_clear_for(master_mac) + master_vm.start() + wait_for(master_vm.is_running, "Wait for Master VM running") + + wait_for(lambda: pxe.arp_addresses_for(master_mac), + "Wait for DHCP server to see Master VM in ARP tables", + timeout_secs=10 * 60) + ips = pxe.arp_addresses_for(master_mac) + logging.info("Master VM has IPs %s", ips) + assert len(ips) == 1 + master_vm.ip = ips[0] + + wait_for(lambda: not os.system(f"nc -zw5 {master_vm.ip} 22"), + "Wait for ssh back up on Master VM", retry_delay_secs=5) + wait_for(pool.master.is_enabled, "Wait for XAPI to be ready", timeout_secs=30 * 60) + + # evacuate slave + vms_to_migrate = vms + logging.info("Expecting migration of %s", ([vm.uuid for vm in vms_to_migrate],)) + pool.master.xe("host-evacuate", {"host": slave.uuid}) + wait_for(lambda: all(vm.get_residence_host().uuid != slave.uuid for vm in vms), + "Wait for VM migration") + + # upgrade slave + slave.shutdown() + wait_for(lambda: slave_vm.is_halted(), "Wait for Slave VM to be halted", timeout_secs=5 * 60) + installer.perform_upgrade(iso=remastered_iso, host_vm=slave_vm, host=host) + pxe.arp_clear_for(slave_mac) + slave_vm.start() + wait_for(slave_vm.is_running, "Wait for Slave VM running") + + wait_for(lambda: pxe.arp_addresses_for(slave_mac), + "Wait for DHCP server to see Slave VM in ARP tables", + timeout_secs=10 * 60) + ips = pxe.arp_addresses_for(slave_mac) + logging.info("Slave VM has IPs %s", ips) + assert len(ips) == 1 + slave_vm.ip = ips[0] + + wait_for(lambda: not os.system(f"nc -zw5 {slave_vm.ip} 22"), + "Wait for ssh back up on Slave VM", retry_delay_secs=5) + wait_for(slave.is_enabled, "Wait for XAPI to be ready", timeout_secs=30 * 60) + + logging.info("Migrating a VM back to slave") + vms[1].migrate(slave) + + # cleanup + + slave.shutdown() + pool.master.shutdown() + wait_for(lambda: slave_vm.is_halted(), "Wait for Slave VM to be halted", timeout_secs=5 * 60) + wait_for(lambda: master_vm.is_halted(), "Wait for Master VM to be halted", timeout_secs=5 * 60) + # FIXME destroy shared SR contents