From c511c463c0157e8411943b2d616869de8e542f02 Mon Sep 17 00:00:00 2001 From: Samuel Verschelde Date: Tue, 9 Oct 2018 10:52:11 +0200 Subject: [PATCH 01/38] Import XCP-ng GPG key to yum and rpm post-install Signed-off-by: Samuel Verschelde Orig-commit: 9f0f094409a7f0c19a686cad3ed16f055cbbf155 Orig-commit: eb796dc6a82a0defdcbba1a0833a3356060c5396 --- backend.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/backend.py b/backend.py index 1788db74..555daf1f 100644 --- a/backend.py +++ b/backend.py @@ -170,6 +170,7 @@ def getRepoSequence(ans, repos): def getFinalisationSequence(ans): seq = [ + Task(importYumAndRpmGpgKeys, A(ans, 'mounts'), []), Task(writeResolvConf, A(ans, 'mounts', 'manual-hostname', 'manual-nameservers'), []), Task(writeMachineID, A(ans, 'mounts'), []), Task(writeKeyboardConfiguration, A(ans, 'mounts', 'keymap'), []), @@ -1642,6 +1643,33 @@ def touchSshAuthorizedKeys(mounts): fh = open("%s/root/.ssh/authorized_keys" % mounts['root'], 'a') fh.close() +def importYumAndRpmGpgKeys(mounts): + # Python script that uses yum functions to import the GPG key for our repositories + import_yum_keys = """#!/bin/env python +from __future__ import print_function +from yum import YumBase + +def retTrue(*args, **kwargs): + return True + +base = YumBase() +for repo in base.repos.repos.itervalues(): + if repo.id.startswith('xcp-ng'): + print("*** Importing GPG key for repository %s - %s" % (repo.id, repo.name)) + base.getKeyForRepo(repo, callback=retTrue) +""" + internal_tmp_filepath = '/tmp/import_yum_keys.py' + external_tmp_filepath = mounts['root'] + internal_tmp_filepath + with open(external_tmp_filepath, 'w') as f: + f.write(import_yum_keys) + # bind mount /dev, necessary for NSS initialization without which RPM won't work + util.bindMount('/dev', "%s/dev" % mounts['root']) + try: + util.runCmd2(['chroot', mounts['root'], 'python', internal_tmp_filepath]) + util.runCmd2(['chroot', mounts['root'], 'rpm', '--import', '/etc/pki/rpm-gpg/RPM-GPG-KEY-xcpng']) + finally: + util.umount("%s/dev" % mounts['root']) + os.unlink(external_tmp_filepath) ################################################################################ # OTHER HELPERS From 4418eb6ce83156145891e5736b579371f5806975 Mon Sep 17 00:00:00 2001 From: Samuel Verschelde Date: Wed, 27 Nov 2019 01:59:05 +0100 Subject: [PATCH 02/38] Include mmcblk-devices in disk selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Originally-by: Jöran Malek Signed-off-by: Samuel Verschelde Orig-commit: fc1b4cf120c68cc03ba7a82251a51ba6d9b7533d --- disktools.py | 2 +- diskutil.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/disktools.py b/disktools.py index 749b5922..af7ac40f 100644 --- a/disktools.py +++ b/disktools.py @@ -490,7 +490,7 @@ def diskDevice(partitionDevice): def determineMidfix(device): DISK_PREFIX = '/dev/' - P_STYLE_DISKS = [ 'cciss', 'ida', 'rd', 'sg', 'i2o', 'amiraid', 'iseries', 'emd', 'carmel', 'mapper/', 'nvme', 'md' ] + P_STYLE_DISKS = [ 'cciss', 'ida', 'rd', 'sg', 'i2o', 'amiraid', 'iseries', 'emd', 'carmel', 'mapper/', 'nvme', 'md', 'mmcblk' ] PART_STYLE_DISKS = [ 'disk/by-id' ] for key in P_STYLE_DISKS: diff --git a/diskutil.py b/diskutil.py index 91c5395a..937a8f08 100644 --- a/diskutil.py +++ b/diskutil.py @@ -113,6 +113,9 @@ def mpath_disable(): for major in range(48, 56): disk_nodes += [ (major, x * 8) for x in range(32) ] +# /dev/mmcblk: mmcblk has major 179, each device usually (per kernel) has 7 minors +disk_nodes += [ (179, x * 8) for x in range(32) ] + def getDiskList(): # read the partition tables: parts = open("/proc/partitions") From 5c56d43f64f0310f2d9db5da937e386e604cc836 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 18 Oct 2022 13:01:02 +0200 Subject: [PATCH 03/38] Add support for globally disabling repo_gpgcheck While developing and/or installing from an already-verified ISO, yum's repo_gpgcheck just gets in the way. This provides a general kill-switch for checking repomd.xml.asc, through commandline or answerfile. Signed-off-by: Yann Dirson --- answerfile.py | 1 + backend.py | 4 ++++ doc/answerfile.txt | 15 +++++++++++++++ doc/parameters.txt | 5 +++++ install.py | 3 +++ repository.py | 8 ++++++-- 6 files changed, 34 insertions(+), 2 deletions(-) diff --git a/answerfile.py b/answerfile.py index 490b49ec..62b0615c 100644 --- a/answerfile.py +++ b/answerfile.py @@ -92,6 +92,7 @@ def processAnswerfile(self): else: raise AnswerfileException("Unknown mode, %s" % install_type) + results['repo-gpgcheck'] = getBoolAttribute(self.top_node, ['repo-gpgcheck'], default=True) results.update(self.parseCommon()) elif self.operation == 'restore': results = self.parseRestore() diff --git a/backend.py b/backend.py index 1788db74..0132885e 100644 --- a/backend.py +++ b/backend.py @@ -395,6 +395,10 @@ def add_repos(main_repositories, update_repositories, repos): for i in answers_pristine['sources']: repos = repository.repositoriesFromDefinition(i['media'], i['address']) add_repos(main_repositories, update_repositories, repos) + repo_gpgcheck = answers.get('repo-gpgcheck', True) + for repo in repos: + if repo in main_repositories: + repo.setRepoGpgCheck(repo_gpgcheck) # A single source coming from an interactive install if 'source-media' in answers_pristine and 'source-address' in answers_pristine: diff --git a/doc/answerfile.txt b/doc/answerfile.txt index 1e1d41d4..8309ef71 100644 --- a/doc/answerfile.txt +++ b/doc/answerfile.txt @@ -34,6 +34,21 @@ Restore: ... + +Common Attributes +----------------- + + repo-gpgcheck="false" + + Disable check of repodata signature (`repo_gpgcheck=0` in + `yum.conf`), for all yum repositories that are not Supplemental + Packs (none of which are checked). Don't use this for a network + install of a production server, and make sure to verify the + authenticity of your install media through other means. + + Validity: any operation. + + Elements common to all answerfiles, both 'installation' and 'restore' --------------------------------------------------------------------- diff --git a/doc/parameters.txt b/doc/parameters.txt index c0f78819..5a2294ca 100644 --- a/doc/parameters.txt +++ b/doc/parameters.txt @@ -220,3 +220,8 @@ Installer --cc-preparations Prepare configuration for common criteria security. + + + --no-repo-gpgcheck + + Disable check of repodata signature, for all yum repositories. diff --git a/install.py b/install.py index 825b569e..e090ea48 100755 --- a/install.py +++ b/install.py @@ -128,6 +128,9 @@ def go(ui, args, answerfile_address, answerfile_script): elif opt == "--cc-preparations": constants.CC_PREPARATIONS = True results['network-backend'] = constants.NETWORK_BACKEND_BRIDGE + elif opt == "--no-repo-gpgcheck": + results['repo-gpgcheck'] = False + logger.log("Yum gpg check of repository disabled on command-line") if boot_console and not serial_console: serial_console = boot_console diff --git a/repository.py b/repository.py index b0fb9716..10e8e5a7 100644 --- a/repository.py +++ b/repository.py @@ -242,6 +242,7 @@ def __init__(self, accessor): super(MainYumRepository, self).__init__(accessor) self._identifier = MAIN_REPOSITORY_NAME self.keyfiles = [] + self._repo_gpg_check = True def get_name_version(config_parser, section, name_key, vesion_key): name, version = None, None @@ -313,9 +314,9 @@ def _repo_config(self): outfh.write(infh.read()) return """ gpgcheck=1 -repo_gpgcheck=1 +repo_gpgcheck=%s gpgkey=file://%s -""" % (key_path) +""" % (int(self._repo_gpg_check), key_path) finally: if infh: infh.close() @@ -351,6 +352,9 @@ def getBranding(self, mounts, branding): branding['product-build'] = self._build_number return branding + def setRepoGpgCheck(self, value): + logger.log("%s: setRepoGpgCheck(%s)" % (self, value)) + self._repo_gpg_check = value class UpdateYumRepository(YumRepositoryWithInfo): """Represents a Yum repository containing packages and associated meta data for an update.""" From 8cc2019c5f5f51ff7f43dbc33e3ac3ee1c97e032 Mon Sep 17 00:00:00 2001 From: Samuel Verschelde Date: Fri, 6 Dec 2019 12:52:22 +0100 Subject: [PATCH 04/38] Use xcp-ng-deps instead of groups.xml Signed-off-by: Samuel Verschelde Orig-commit: 2731116cba1ef80a2628c041002f82258e1be5cb --- repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository.py b/repository.py index b0fb9716..d1725a8e 100644 --- a/repository.py +++ b/repository.py @@ -236,7 +236,7 @@ class MainYumRepository(YumRepositoryWithInfo): """Represents a Yum repository containing the main XenServer installation.""" INFO_FILENAME = ".treeinfo" - _targets = ['@xenserver_base', '@xenserver_dom0'] + _targets = ['xcp-ng-deps'] def __init__(self, accessor): super(MainYumRepository, self).__init__(accessor) From afdbe029b9ea253972c79e8090f58576fb5fb6c9 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 18 Oct 2022 15:58:23 +0200 Subject: [PATCH 05/38] Add support for globally disabling gpgcheck Similar to no-repo-gpgcheck but for RPM sigs. Signed-off-by: Yann Dirson --- answerfile.py | 1 + backend.py | 2 ++ doc/answerfile.txt | 8 ++++++++ doc/parameters.txt | 5 +++++ install.py | 3 +++ repository.py | 9 +++++++-- 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/answerfile.py b/answerfile.py index 62b0615c..370b928e 100644 --- a/answerfile.py +++ b/answerfile.py @@ -93,6 +93,7 @@ def processAnswerfile(self): raise AnswerfileException("Unknown mode, %s" % install_type) results['repo-gpgcheck'] = getBoolAttribute(self.top_node, ['repo-gpgcheck'], default=True) + results['gpgcheck'] = getBoolAttribute(self.top_node, ['gpgcheck'], default=True) results.update(self.parseCommon()) elif self.operation == 'restore': results = self.parseRestore() diff --git a/backend.py b/backend.py index 0132885e..4d886ccb 100644 --- a/backend.py +++ b/backend.py @@ -396,9 +396,11 @@ def add_repos(main_repositories, update_repositories, repos): repos = repository.repositoriesFromDefinition(i['media'], i['address']) add_repos(main_repositories, update_repositories, repos) repo_gpgcheck = answers.get('repo-gpgcheck', True) + gpgcheck = answers.get('gpgcheck', True) for repo in repos: if repo in main_repositories: repo.setRepoGpgCheck(repo_gpgcheck) + repo.setGpgCheck(gpgcheck) # A single source coming from an interactive install if 'source-media' in answers_pristine and 'source-address' in answers_pristine: diff --git a/doc/answerfile.txt b/doc/answerfile.txt index 8309ef71..23a7a8d1 100644 --- a/doc/answerfile.txt +++ b/doc/answerfile.txt @@ -48,6 +48,14 @@ Common Attributes Validity: any operation. + gpgcheck="false" + + Disable check of rpm signature (`gpgcheck=0` in `yum.conf`), for + all yum repositories that are not Supplemental Packs (none of + which are checked). Don't use this for a production server. + + Validity: any operation. + Elements common to all answerfiles, both 'installation' and 'restore' --------------------------------------------------------------------- diff --git a/doc/parameters.txt b/doc/parameters.txt index 5a2294ca..3f5f3ed3 100644 --- a/doc/parameters.txt +++ b/doc/parameters.txt @@ -225,3 +225,8 @@ Installer --no-repo-gpgcheck Disable check of repodata signature, for all yum repositories. + + + --no-gpgcheck + + Disable check of rpm signature, for all yum repositories. diff --git a/install.py b/install.py index e090ea48..5afb9c8f 100755 --- a/install.py +++ b/install.py @@ -131,6 +131,9 @@ def go(ui, args, answerfile_address, answerfile_script): elif opt == "--no-repo-gpgcheck": results['repo-gpgcheck'] = False logger.log("Yum gpg check of repository disabled on command-line") + elif opt == "--no-gpgcheck": + results['gpgcheck'] = False + logger.log("Yum gpg check of RPMs disabled on command-line") if boot_console and not serial_console: serial_console = boot_console diff --git a/repository.py b/repository.py index 10e8e5a7..13a20818 100644 --- a/repository.py +++ b/repository.py @@ -243,6 +243,7 @@ def __init__(self, accessor): self._identifier = MAIN_REPOSITORY_NAME self.keyfiles = [] self._repo_gpg_check = True + self._gpg_check = True def get_name_version(config_parser, section, name_key, vesion_key): name, version = None, None @@ -313,10 +314,10 @@ def _repo_config(self): outfh = open(key_path, "w") outfh.write(infh.read()) return """ -gpgcheck=1 +gpgcheck=%s repo_gpgcheck=%s gpgkey=file://%s -""" % (int(self._repo_gpg_check), key_path) +""" % (int(self._gpg_check), int(self._repo_gpg_check), key_path) finally: if infh: infh.close() @@ -356,6 +357,10 @@ def setRepoGpgCheck(self, value): logger.log("%s: setRepoGpgCheck(%s)" % (self, value)) self._repo_gpg_check = value + def setGpgCheck(self, value): + logger.log("%s: setGpgCheck(%s)" % (self, value)) + self._gpg_check = value + class UpdateYumRepository(YumRepositoryWithInfo): """Represents a Yum repository containing packages and associated meta data for an update.""" From 26f61b7983b22f549bf9217a81e09c69cec50c68 Mon Sep 17 00:00:00 2001 From: Samuel Verschelde Date: Wed, 5 Sep 2018 11:03:17 +0200 Subject: [PATCH 06/38] Add software RAID support. Originally-by: Nicolas Raynaud Also adds support for it in the answerfile. the general form is : sda sdb avoid question and display waiting message when running mdadm Signed-off-by: Samuel Verschelde When restoring from backup on a software RAID install, install grub on raid members instead of disk (which is a mdadm device) Signed-off-by: BenjiReis Orig-commit: 6e168b43e0fd31cf3de9f578def1e10ba72ced35 Orig-commit: 7557a0077a65d9353a56d7a21d054ceadc29b796 Orig-commit: ee404b10b9809279600e0be121f88a675c0e2ef4 Orig-commit: e6497d1c602691104a3d750692ffceedb25c1221 Orig-commit: d33fe10fb1c36ee0326e48dbb8059a5fe42fa80b --- answerfile.py | 11 ++++++ backend.py | 7 +++- diskutil.py | 14 ++++++++ doc/answerfile.txt | 10 ++++++ restore.py | 6 +++- tui/installer/screens.py | 73 ++++++++++++++++++++++++++++++++++++---- 6 files changed, 112 insertions(+), 9 deletions(-) diff --git a/answerfile.py b/answerfile.py index 490b49ec..8da3291d 100644 --- a/answerfile.py +++ b/answerfile.py @@ -133,6 +133,7 @@ def parseFreshInstall(self): results['preserve-settings'] = False results['backup-existing-installation'] = False + results.update(self.parseRaid()) results.update(self.parseDisks()) results.update(self.parseInterface()) results.update(self.parseRootPassword()) @@ -293,6 +294,16 @@ def parseDriverSource(self): results['extra-repos'].append((rtype, address)) return results + def parseRaid(self): + results = {} + for raid_node in getElementsByTagName(self.top_node, ['raid']): + disk_device = normalize_disk(getStrAttribute(raid_node, ['device'], mandatory=True)) + disks = [normalize_disk(getText(node)) for node in getElementsByTagName(raid_node, ['disk'])] + if 'raid' not in results: + results['raid'] = {} + results['raid'][disk_device] = disks + return results + def parseDisks(self): results = {} diff --git a/backend.py b/backend.py index 1788db74..6b981039 100644 --- a/backend.py +++ b/backend.py @@ -103,6 +103,7 @@ def getPrepSequence(ans, interactive): Task(util.getUUID, As(ans), ['installation-uuid']), Task(util.getUUID, As(ans), ['control-domain-uuid']), Task(util.randomLabelStr, As(ans), ['disk-label-suffix']), + Task(diskutil.create_raid, A(ans, 'raid'), []), Task(inspectTargetDisk, A(ans, 'primary-disk', 'installation-to-overwrite', 'preserve-first-partition','sr-on-primary'), ['target-boot-mode', 'boot-partnum', 'primary-partnum', 'backup-partnum', 'logs-partnum', 'swap-partnum', 'storage-partnum']), ] @@ -1119,7 +1120,11 @@ def installBootLoader(mounts, disk, boot_partnum, primary_partnum, target_boot_m setEfiBootEntry(mounts, disk, boot_partnum, install_type, branding) else: if location == constants.BOOT_LOCATION_MBR: - installGrub2(mounts, disk, False) + if diskutil.is_raid(disk): + for member in diskutil.getDeviceSlaves(disk): + installGrub2(mounts, member, False) + else: + installGrub2(mounts, disk, False) else: installGrub2(mounts, root_partition, True) diff --git a/diskutil.py b/diskutil.py index 91c5395a..60c85dba 100644 --- a/diskutil.py +++ b/diskutil.py @@ -153,6 +153,20 @@ def getDiskList(): return disks +def create_raid(configuration): + if configuration: + for raid_device, members in configuration.viewitems(): + # allows for idempotence + if not os.path.exists(raid_device): + for dev in members: + util.runCmd2(['mdadm', '--zero-superblock', '--force', dev]) + # let it fail without catching + cmd = ['mdadm', '--create', raid_device, '--run', '--metadata=1.0', '--level=mirror', + '--raid-devices=%s' % (len(members))] + members + rc, out, err = util.runCmd2(cmd, with_stdout=True, with_stderr=True) + if rc != 0: + raise Exception('Error running: %s\n%s\n\n%s' % (' '.join(cmd), out, err)) + def getPartitionList(): disks = getDiskList() rv = [] diff --git a/doc/answerfile.txt b/doc/answerfile.txt index 1e1d41d4..8182efec 100644 --- a/doc/answerfile.txt +++ b/doc/answerfile.txt @@ -156,6 +156,16 @@ Format of 'source' and 'driver-source' (Re)Install Elements -------------------- + + dev1 + dev2 + ? + + Specifies the target disks and md device for creating a + software RAID 1 array. The md device can then be used in + below. (new in xcp-ng 7.5.0-2 and 7.6) + + dev Specifies the target disk for installation. diff --git a/restore.py b/restore.py index 3f4248da..01637f69 100644 --- a/restore.py +++ b/restore.py @@ -135,7 +135,11 @@ def restoreFromBackup(backup, progress=lambda x: ()): backend.setEfiBootEntry(mounts, disk, boot_partnum, constants.INSTALL_TYPE_RESTORE, branding) else: if location == constants.BOOT_LOCATION_MBR: - backend.installGrub2(mounts, disk, False) + if diskutil.is_raid(disk): + for member in diskutil.getDeviceSlaves(disk): + backend.installGrub2(mounts, member, False) + else: + backend.installGrub2(mounts, disk, False) else: backend.installGrub2(mounts, restore_partition, True) else: diff --git a/tui/installer/screens.py b/tui/installer/screens.py index 74bb8cfb..6fd92ec0 100644 --- a/tui/installer/screens.py +++ b/tui/installer/screens.py @@ -504,6 +504,56 @@ def setup_runtime_networking(answers): # Get the answers from the user return tui.network.requireNetworking(answers, defaults) +def raid_array_ui(answers): + disk_entries = sorted_disk_list() + raid_disks = [de for de in disk_entries if diskutil.is_raid(de)] + raid_slaves = [slave for master in raid_disks for slave in diskutil.getDeviceSlaves(master)] + entries = [] + for de in disk_entries: + if de not in raid_slaves and de not in raid_disks: + vendor, model, size = diskutil.getExtendedDiskInfo(de) + string_entry = "%s - %s [%s %s]" % ( + diskutil.getHumanDiskName(de), diskutil.getHumanDiskSize(size), vendor, model) + entries.append((string_entry, de)) + if len(entries) < 2: + return SKIP_SCREEN + text = TextboxReflowed(54, "Do you want to group disks in a software RAID 1 array? \n\n" + + "The array will be created immediately and erase all the target disks.") + buttons = ButtonBar(tui.screen, [('Create', 'create'), ('Back', 'back')]) + scroll, _ = snackutil.scrollHeight(3, len(entries)) + cbt = CheckboxTree(3, scroll) + for (c_text, c_item) in entries: + cbt.append(c_text, c_item, False) + gf = GridFormHelp(tui.screen, 'RAID Array', 'guestdisk:info', 1, 4) + gf.add(text, 0, 0, padding=(0, 0, 0, 1)) + gf.add(cbt, 0, 1, padding=(0, 0, 0, 1)) + gf.add(buttons, 0, 3, growx=1) + gf.addHotKey('F5') + + tui.update_help_line([None, " disk info"]) + loop = True + while loop: + rc = gf.run() + if rc == 'F5': + disk_more_info(cbt.getCurrent()) + else: + loop = False + tui.screen.popWindow() + tui.screen.popHelpLine() + + button = buttons.buttonPressed(rc) + if button == 'create': + selected = cbt.getSelection() + txt = 'The content of the disks %s will be deleted when you activate "Ok"' % (str(selected)) + title = 'RAID array creation' + confirmation = snackutil.ButtonChoiceWindowEx(tui.screen, title, txt, ('Ok', 'Cancel'), 40, default=1) + if confirmation == 'ok': + answers['raid'] = {'/dev/md127': selected} + tui.progress.showMessageDialog("Please wait", "Creating raid array...") + diskutil.create_raid(answers['raid']) + tui.progress.clearModelessDialog() + return REPEAT_STEP + def disk_more_info(context): if not context: return True @@ -532,14 +582,18 @@ def disk_more_info(context): return True def sorted_disk_list(): - return sorted(diskutil.getQualifiedDiskList(), - lambda x, y: len(x) == len(y) and cmp(x,y) or (len(x)-len(y))) + return sorted(set(diskutil.getQualifiedDiskList()), + lambda x, y: len(x) == len(y) and cmp(x, y) or (len(x) - len(y))) + +def filter_out_raid_member(diskEntries): + raid_disks = [de for de in diskEntries if diskutil.is_raid(de)] + raid_slaves = set(member for master in raid_disks for member in diskutil.getDeviceSlaves(master)) + return [e for e in diskEntries if e not in raid_slaves] # select drive to use as the Dom0 disk: def select_primary_disk(answers): button = None - diskEntries = sorted_disk_list() - + diskEntries = filter_out_raid_member(sorted_disk_list()) entries = [] target_is_sr = {} min_primary_disk_size = constants.min_primary_disk_size @@ -588,7 +642,7 @@ def select_primary_disk(answers): You may need to change your system settings to boot from this disk.""" % (MY_PRODUCT_BRAND), entries, - ['Ok', 'Back'], 55, scroll, height, default, help='pridisk:info', + ['Ok', 'Software RAID', 'Back'], 55, scroll, height, default, help='pridisk:info', hotkeys={'F5': disk_more_info}) tui.screen.popHelpLine() @@ -615,7 +669,12 @@ def select_primary_disk(answers): else: answers["preserve-first-partition"] = 'false' - if button is None: return SKIP_SCREEN + # XCP-ng: we replaced `SKIP_SCREEN` by `RIGHT_FORWARDS` for RAID support to avoid a loop after raid creation + if button is None: return RIGHT_FORWARDS + + # XCP-ng + if button == 'software raid': + return raid_array_ui(answers) return RIGHT_FORWARDS @@ -640,7 +699,7 @@ def check_sr_space(answers): return EXIT def select_guest_disks(answers): - diskEntries = sorted_disk_list() + diskEntries = filter_out_raid_member(sorted_disk_list()) # CA-38329: filter out device mapper nodes (except primary disk) as these won't exist # at XenServer boot and therefore cannot be added as physical volumes to Local SR. From a2ac6909457c9c8592d30c1664701d6cf1417a15 Mon Sep 17 00:00:00 2001 From: Samuel Verschelde Date: Thu, 27 Feb 2020 06:40:22 +0000 Subject: [PATCH 07/38] kernel-alt support - show kernel-alt warning Originally-by: Rushikesh Jadhav - Update warning message when installing with kernel-alt - cleaner reboot action on that screen: return EXIT - fix detection of the kernel-alt boot parameter of the installer - install kernel-alt when booting on install-alt - create initrd and update grub configuration as post-install task Signed-off-by: Samuel Verschelde Orig-commit: 243ce81f0ff4ad17c99c1ca6af5f3326af1c94db Orig-commit: d9a8a3eb609b6c0493fb3b6db03fc91bc5321f45 --- backend.py | 27 ++++++++++++++++++++++++++- install.py | 4 ++++ repository.py | 8 +++++--- tui/__init__.py | 3 ++- tui/installer/__init__.py | 1 + tui/installer/screens.py | 22 ++++++++++++++++++++++ 6 files changed, 60 insertions(+), 5 deletions(-) diff --git a/backend.py b/backend.py index 555daf1f..fbf1ab5d 100644 --- a/backend.py +++ b/backend.py @@ -160,7 +160,7 @@ def getMainRepoSequence(ans, repos): def getRepoSequence(ans, repos): seq = [] for repo in repos: - seq.append(Task(repo.installPackages, A(ans, 'mounts'), [], + seq.append(Task(repo.installPackages, A(ans, 'mounts', 'kernel-alt'), [], progress_scale=100, pass_progress_callback=True, progress_text="Installing %s..." % repo.name())) @@ -192,6 +192,7 @@ def getFinalisationSequence(ans): 'boot-partnum', 'primary-partnum', 'target-boot-mode', 'branding', 'disk-label-suffix', 'bootloader-location', 'write-boot-entry', 'install-type', 'serial-console', 'boot-serial', 'host-config', 'fcoe-interfaces'), []), + Task(postInstallAltKernel, A(ans, 'mounts', 'kernel-alt'), []), Task(touchSshAuthorizedKeys, A(ans, 'mounts'), []), Task(setRootPassword, A(ans, 'mounts', 'root-password'), [], args_sensitive=True), Task(setTimeZone, A(ans, 'mounts', 'timezone'), []), @@ -1671,6 +1672,30 @@ def retTrue(*args, **kwargs): util.umount("%s/dev" % mounts['root']) os.unlink(external_tmp_filepath) +def postInstallAltKernel(mounts, kernel_alt): + """ Install our alternate kernel. Must be called after the bootloader installation. """ + if not kernel_alt: + logger.log('kernel-alt not installed') + return + + util.bindMount("/proc", "%s/proc" % mounts['root']) + util.bindMount("/sys", "%s/sys" % mounts['root']) + util.bindMount("/dev", "%s/dev" % mounts['root']) + + try: + rc, out = util.runCmd2(['chroot', mounts['root'], 'rpm', '-q', 'kernel-alt', '--qf', '%{version}'], + with_stdout=True) + version = out + # Generate the initrd as it was disabled during initial installation + util.runCmd2(['chroot', mounts['root'], 'dracut', '-f', '/boot/initrd-%s.img' % version, version]) + + # Update grub + util.runCmd2(['chroot', mounts['root'], 'python', '/usr/lib/python2.7/site-packages/xcp/updategrub.py', 'add', 'kernel-alt', version]) + finally: + util.umount("%s/dev" % mounts['root']) + util.umount("%s/sys" % mounts['root']) + util.umount("%s/proc" % mounts['root']) + ################################################################################ # OTHER HELPERS diff --git a/install.py b/install.py index 825b569e..659382fe 100755 --- a/install.py +++ b/install.py @@ -128,6 +128,10 @@ def go(ui, args, answerfile_address, answerfile_script): elif opt == "--cc-preparations": constants.CC_PREPARATIONS = True results['network-backend'] = constants.NETWORK_BACKEND_BRIDGE + # XCP-ng addition: alternate kernel + elif opt == "--kernel-alt": + results['kernel-alt'] = True + logger.log("Using alternate kernel.") if boot_console and not serial_console: serial_console = boot_console diff --git a/repository.py b/repository.py index d1725a8e..43186119 100644 --- a/repository.py +++ b/repository.py @@ -181,7 +181,7 @@ def record_install(self, answers, installed_repos): installed_repos[str(self)] = self return installed_repos - def _installPackages(self, progress_callback, mounts): + def _installPackages(self, progress_callback, mounts, kernel_alt): assert self._targets is not None url = self._accessor.url() logger.log("URL: " + str(url)) @@ -203,13 +203,15 @@ def _installPackages(self, progress_callback, mounts): yum_conf.write(repo_config) self.disableInitrdCreation(mounts['root']) + if kernel_alt: + self._targets.append('kernel-alt') installFromYum(self._targets, mounts, progress_callback, self._cachedir) self.enableInitrdCreation() - def installPackages(self, progress_callback, mounts): + def installPackages(self, progress_callback, mounts, kernel_alt=False): self._accessor.start() try: - self._installPackages(progress_callback, mounts) + self._installPackages(progress_callback, mounts, kernel_alt) finally: self._accessor.finish() diff --git a/tui/__init__.py b/tui/__init__.py index 84975578..b0d709bd 100644 --- a/tui/__init__.py +++ b/tui/__init__.py @@ -8,6 +8,7 @@ import constants import sys from xcp import logger +import platform screen = None help_pad = [33, 17, 16] @@ -28,7 +29,7 @@ def global_help(screen, context): def init_ui(): global screen screen = SnackScreen() - screen.drawRootText(0, 0, "Welcome to %s - Version %s" % (PRODUCT_BRAND or PLATFORM_NAME, PRODUCT_VERSION_TEXT)) + screen.drawRootText(0, 0, "Welcome to %s - Version %s (Kernel %s)" % (PRODUCT_BRAND or PLATFORM_NAME, PRODUCT_VERSION_TEXT, platform.release())) if PRODUCT_BRAND: if len(COPYRIGHT_YEARS) > 0: screen.drawRootText(0, 1, "Copyright (c) %s %s" % (COPYRIGHT_YEARS, COMPANY_NAME_LEGAL)) diff --git a/tui/installer/__init__.py b/tui/installer/__init__.py index 7a8f588f..c05a5cc0 100644 --- a/tui/installer/__init__.py +++ b/tui/installer/__init__.py @@ -119,6 +119,7 @@ def out_of_order_pool_upgrade_fn(answers): results['preserve-settings'] = False seq = [ + Step(uis.kernel_warning), Step(uis.welcome_screen), Step(uis.eula_screen), Step(uis.hardware_warnings, diff --git a/tui/installer/screens.py b/tui/installer/screens.py index 74bb8cfb..79c7bff3 100644 --- a/tui/installer/screens.py +++ b/tui/installer/screens.py @@ -38,6 +38,28 @@ def selectDefault(key, entries): return text, k return None +# kernel-alt warning +def kernel_warning(answers): + if answers.get("kernel-alt"): + button = snackutil.ButtonChoiceWindowEx( + tui.screen, + "Alternate kernel", + """WARNING: you chose to install our alternative kernel (kernel-alt). + +It is based on our main kernel + upstream kernel.org patches, so it should be stable by construction. However it receives less testing than the main kernel. + +A boot menu entry for kernel-alt will be added, but we will still boot the main kernel by default. + +If kernel-alt works BETTER than the main kernel for you, TELL US so that we may fix the main kernel! +""", + ['Ok', 'Reboot'], width=60) + + if button == 'ok' or button is None: + return True + else: + return EXIT + return True + # welcome screen: def welcome_screen(answers): driver_answers = {'driver-repos': []} From 58e96b09f2b0a6688c5d61c6a349cbc7a5b97201 Mon Sep 17 00:00:00 2001 From: Samuel Verschelde Date: Wed, 5 Sep 2018 11:07:09 +0200 Subject: [PATCH 08/38] Add a netinstall boot option The presence of this parameter changes the installer behavior, preventing local source installation and pre-populating the xcp-ng install url. Originally-by: Nicolas Raynaud Use the `--netinstall` parameter that is automatically generated at preinit stage and translate it into `answers['netinstall'] = True`. Signed-off-by: Samuel Verschelde Orig-commit: 36395c3c68e0d1afdd94ed8ec57d152640dde8b4 Orig-commit: 098a2f404c2d69bdd805d175943c993b88c2b4de Orig-commit: 0b298cd20722ca0bae36ae7079df2b94491a25db --- backend.py | 11 +++++++++++ install.py | 4 ++++ tui/repo.py | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/backend.py b/backend.py index 1788db74..d4bb9417 100644 --- a/backend.py +++ b/backend.py @@ -32,6 +32,7 @@ import version from version import * from constants import * +from diskutil import getRemovableDeviceList MY_PRODUCT_BRAND = PRODUCT_BRAND or PLATFORM_NAME @@ -418,6 +419,16 @@ def add_repos(main_repositories, update_repositories, repos): if r.accessor().canEject(): r.accessor().eject() + # XCP-ng: so, very unfortunately we don't remember with precision why this was added and + # no commit message or comment can help us here. + # It may be related to the fact that the "all_repositories" above doesn't contain + # the installation CD-ROM or USB stick in the case of a netinstall. + # Question: why it is needed at all since there's no repository on the netinstall + # installation media? + if answers.get('netinstall'): + for device in getRemovableDeviceList(): + util.runCmd2(['eject', device]) + if interactive: # Add supp packs in a loop while True: diff --git a/install.py b/install.py index 825b569e..70c65d63 100755 --- a/install.py +++ b/install.py @@ -128,6 +128,10 @@ def go(ui, args, answerfile_address, answerfile_script): elif opt == "--cc-preparations": constants.CC_PREPARATIONS = True results['network-backend'] = constants.NETWORK_BACKEND_BRIDGE + # XCP-ng: netinstall + elif opt == "--netinstall": + results['netinstall'] = True + logger.log("This is a netinstall.") if boot_console and not serial_console: serial_console = boot_console diff --git a/tui/repo.py b/tui/repo.py index a46eda2e..af885cb8 100644 --- a/tui/repo.py +++ b/tui/repo.py @@ -89,6 +89,9 @@ def select_repo_source(answers, title, text, require_base_repo=True): entries = [ ENTRY_LOCAL ] default = ENTRY_LOCAL + if answers.get('netinstall'): + entries = [] + default = ENTRY_URL if len(answers['network-hardware'].keys()) > 0: entries += [ ENTRY_URL, ENTRY_NFS ] @@ -136,6 +139,8 @@ def get_url_location(answers, require_base_repo): user_field.set(answers['source-address'].getUsername()) if answers['source-address'].getPassword() is not None: passwd_field.set(answers['source-address'].getPassword()) + else: + url_field.set('http://mirrors.xcp-ng.org/netinstall/8.2.1') done = False while not done: From 22c5b0b119c1835ed1a253b0a8f2f96c551357ff Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 24 Oct 2022 17:42:21 +0200 Subject: [PATCH 09/38] answerfile: support *gpgcheck override for each Global *gpgcheck flag can change the default from True to False, and these new flags allow to override this default on a per-source basis. Signed-off-by: Yann Dirson --- answerfile.py | 16 +++++++++++++++- backend.py | 6 ++++-- doc/answerfile.txt | 9 +++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/answerfile.py b/answerfile.py index 370b928e..742bf701 100644 --- a/answerfile.py +++ b/answerfile.py @@ -268,7 +268,21 @@ def parseSource(self): if rtype == 'url': address = util.URL(address) - results['sources'].append({'media': rtype, 'address': address}) + # workaround getBoolAttribute() not allowing "None" as + # default, by using a getStrAttribute() call first to + # handle the default situation where the attribute is not + # specified + repo_gpgcheck = (None if getStrAttribute(i, ['repo-gpgcheck'], default=None) is None + else getBoolAttribute(i, ['repo-gpgcheck'])) + gpgcheck = (None if getStrAttribute(i, ['gpgcheck'], default=None) is None + else getBoolAttribute(i, ['gpgcheck'])) + + results['sources'].append({ + 'media': rtype, 'address': address, + 'repo_gpgcheck': repo_gpgcheck, + 'gpgcheck': gpgcheck, + }) + logger.log("parsed source %s" % results['sources'][-1]) return results diff --git a/backend.py b/backend.py index 4d886ccb..d5a2e39a 100644 --- a/backend.py +++ b/backend.py @@ -395,8 +395,10 @@ def add_repos(main_repositories, update_repositories, repos): for i in answers_pristine['sources']: repos = repository.repositoriesFromDefinition(i['media'], i['address']) add_repos(main_repositories, update_repositories, repos) - repo_gpgcheck = answers.get('repo-gpgcheck', True) - gpgcheck = answers.get('gpgcheck', True) + repo_gpgcheck = (answers.get('repo-gpgcheck', True) if i['repo_gpgcheck'] is None + else i['repo_gpgcheck']) + gpgcheck = (answers.get('gpgcheck', True) if i['gpgcheck'] is None + else i['gpgcheck']) for repo in repos: if repo in main_repositories: repo.setRepoGpgCheck(repo_gpgcheck) diff --git a/doc/answerfile.txt b/doc/answerfile.txt index 23a7a8d1..3c86e48e 100644 --- a/doc/answerfile.txt +++ b/doc/answerfile.txt @@ -123,6 +123,15 @@ Elements for 'installation' modes The location of the installation repository or a Supplemental Pack. There may be multiple 'source' elements. + Optional attributes for only: + + repo-gpgcheck=bool + gpgcheck=bool + + Override the global yum gpgcheck setting, respectively for + repodata and RPMs, for this source only. Only applies to + repositories that are not Supplemental Packs (none of which + are checked). grub2|extlinux[D]|grub[D]? From fb73e7166c1d7a0ea04e13d857287dcbc15be7eb Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 8 Dec 2022 15:09:48 +0100 Subject: [PATCH 10/38] RAID support: zap partition tables of disks before building a RAID Failing to do so can leave the kernel's idea of the partitions in weird states. There are likely bugs underlying those weird state, we just don't want to be impacted by them at this point :) Signed-off-by: Yann Dirson --- diskutil.py | 1 + 1 file changed, 1 insertion(+) diff --git a/diskutil.py b/diskutil.py index 60c85dba..e9bd7311 100644 --- a/diskutil.py +++ b/diskutil.py @@ -159,6 +159,7 @@ def create_raid(configuration): # allows for idempotence if not os.path.exists(raid_device): for dev in members: + util.runCmd2(['sgdisk', '--zap-all', dev]) util.runCmd2(['mdadm', '--zero-superblock', '--force', dev]) # let it fail without catching cmd = ['mdadm', '--create', raid_device, '--run', '--metadata=1.0', '--level=mirror', From 77b88425451fc8c105e2c334aaa1e1a9b7579aaa Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 25 Oct 2022 14:32:29 +0200 Subject: [PATCH 11/38] Remove mention of Technical Support Representative in error messages Originally-by: Samuel Verschelde Signed-off-by: Yann Dirson Orig-commit: b20e3033ecc1072f01f1b4cf619f3b2b952e592f --- constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/constants.py b/constants.py index 7b414802..dd0534e2 100644 --- a/constants.py +++ b/constants.py @@ -58,9 +58,9 @@ def error_string(error, logname, with_hd): ) = range(3) ERROR_STRINGS = { - ERROR_STRING_UNKNOWN_ERROR_WITH_HD: "An unrecoverable error has occurred. The details of the error can be found in the log file, which has been written to /tmp/%s (and /root/%s on your hard disk if possible).\n\nPlease refer to your user guide or contact a Technical Support Representative for more details.", - ERROR_STRING_UNKNOWN_ERROR_WITHOUT_HD: "An unrecoverable error has occurred. The details of the error can be found in the log file, which has been written to /tmp/%s.\n\nPlease refer to your user guide or contact a Technical Support Representative for more details.", - ERROR_STRING_KNOWN_ERROR: "An unrecoverable error has occurred. The error was:\n\n%s\n\nPlease refer to your user guide, or contact a Technical Support Representative, for further details." + ERROR_STRING_UNKNOWN_ERROR_WITH_HD: "An unrecoverable error has occurred. The details of the error can be found in the log file, which has been written to /tmp/%s (and /root/%s on your hard disk if possible).", + ERROR_STRING_UNKNOWN_ERROR_WITHOUT_HD: "An unrecoverable error has occurred. The details of the error can be found in the log file, which has been written to /tmp/%s.", + ERROR_STRING_KNOWN_ERROR: "An unrecoverable error has occurred. The error was:\n\n%s" } if error == "": From 5b2c44c0fc5cf7765bcf5423e9f4001b598c06aa Mon Sep 17 00:00:00 2001 From: Samuel Verschelde Date: Thu, 17 Nov 2022 15:35:18 +0100 Subject: [PATCH 12/38] Update URL for netinstall in 8.3 Signed-off-by: Samuel Verschelde --- tui/repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tui/repo.py b/tui/repo.py index af885cb8..2a23d987 100644 --- a/tui/repo.py +++ b/tui/repo.py @@ -140,7 +140,7 @@ def get_url_location(answers, require_base_repo): if answers['source-address'].getPassword() is not None: passwd_field.set(answers['source-address'].getPassword()) else: - url_field.set('http://mirrors.xcp-ng.org/netinstall/8.2.1') + url_field.set('http://mirrors.xcp-ng.org/netinstall/8.3') done = False while not done: From 9c07411cfe8a18bec328f156ce71dc484dc91570 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 14 Nov 2022 11:15:43 +0100 Subject: [PATCH 13/38] Fix interactive install to honor no-*gpgcheck from commandline When originally implementing the per-source gpgcheck flags in answerfile[1], the full code was moved to an anwerfile-only location, breaking the original interactive-install implementation (and then following code review[2] the working code for interactive install disappeared further from the patch series). This moves the Repository flag-setting to `add_repos()` common code, while leaving the flag computation to the caller, since only the answerfile case has to do any non-trivial logic. - [1] https://github.com/xcp-ng/host-installer/commit/06b700789f2c22d1c78285f9583937511e4d3c07 - [2] https://github.com/xcp-ng/host-installer/pull/2#discussion_r1012080594 Signed-off-by: Yann Dirson --- backend.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/backend.py b/backend.py index d5a2e39a..6ffe79bb 100644 --- a/backend.py +++ b/backend.py @@ -373,7 +373,7 @@ def handleRepos(repos, ans): main_repositories = [] update_repositories = [] - def add_repos(main_repositories, update_repositories, repos): + def add_repos(main_repositories, update_repositories, repos, repo_gpgcheck, gpgcheck): """Add repositories to the appropriate list, ensuring no duplicates, that the main repository is at the beginning, and that the order of the rest is maintained.""" @@ -390,28 +390,28 @@ def add_repos(main_repositories, update_repositories, repos): else: repo_list.append(repo) + if repo_list is main_repositories: # i.e., if repo is a "main repository" + repo.setRepoGpgCheck(repo_gpgcheck) + repo.setGpgCheck(gpgcheck) + + default_repo_gpgcheck = answers.get('repo-gpgcheck', True) + default_gpgcheck = answers.get('gpgcheck', True) # A list of sources coming from the answerfile if 'sources' in answers_pristine: for i in answers_pristine['sources']: repos = repository.repositoriesFromDefinition(i['media'], i['address']) - add_repos(main_repositories, update_repositories, repos) - repo_gpgcheck = (answers.get('repo-gpgcheck', True) if i['repo_gpgcheck'] is None - else i['repo_gpgcheck']) - gpgcheck = (answers.get('gpgcheck', True) if i['gpgcheck'] is None - else i['gpgcheck']) - for repo in repos: - if repo in main_repositories: - repo.setRepoGpgCheck(repo_gpgcheck) - repo.setGpgCheck(gpgcheck) + repo_gpgcheck = default_repo_gpgcheck if i['repo_gpgcheck'] is None else i['repo_gpgcheck'] + gpgcheck = default_gpgcheck if i['gpgcheck'] is None else i['gpgcheck'] + add_repos(main_repositories, update_repositories, repos, repo_gpgcheck, gpgcheck) # A single source coming from an interactive install if 'source-media' in answers_pristine and 'source-address' in answers_pristine: repos = repository.repositoriesFromDefinition(answers_pristine['source-media'], answers_pristine['source-address']) - add_repos(main_repositories, update_repositories, repos) + add_repos(main_repositories, update_repositories, repos, default_repo_gpgcheck, default_gpgcheck) for media, address in answers_pristine['extra-repos']: repos = repository.repositoriesFromDefinition(media, address) - add_repos(main_repositories, update_repositories, repos) + add_repos(main_repositories, update_repositories, repos, default_repo_gpgcheck, default_gpgcheck) if not main_repositories or main_repositories[0].identifier() != MAIN_REPOSITORY_NAME: raise RuntimeError("No main repository found") From 4406689f91e0a7cc4ff66ff7976c68b1fc875d95 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 17 May 2023 15:17:58 +0200 Subject: [PATCH 14/38] RAID: only propose creation with at least 2 disks and no RAID Signed-off-by: Yann Dirson --- tui/installer/screens.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tui/installer/screens.py b/tui/installer/screens.py index 6fd92ec0..a2f9f0f4 100644 --- a/tui/installer/screens.py +++ b/tui/installer/screens.py @@ -634,6 +634,15 @@ def select_primary_disk(answers): tui.update_help_line([None, " more info"]) + if len(entries) < 2: + logger.log("not enough disks, not proposing RAID creation") + propose_raid = False + elif any(disk.startswith("/dev/md") for label, disk in entries): + logger.log("existing md found, not proposing RAID creation") + propose_raid = False + else: + propose_raid = True + scroll, height = snackutil.scrollHeight(4, len(entries)) (button, entry) = snackutil.ListboxChoiceWindowEx( tui.screen, @@ -642,7 +651,8 @@ def select_primary_disk(answers): You may need to change your system settings to boot from this disk.""" % (MY_PRODUCT_BRAND), entries, - ['Ok', 'Software RAID', 'Back'], 55, scroll, height, default, help='pridisk:info', + ['Ok', 'Software RAID', 'Back'] if propose_raid else ['Ok', 'Back'], + 55, scroll, height, default, help='pridisk:info', hotkeys={'F5': disk_more_info}) tui.screen.popHelpLine() From a42c2f2a7451eec7e1beb5b5fed5825e30a0d94a Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Tue, 30 May 2023 10:06:07 +0200 Subject: [PATCH 15/38] Write `manual_nameservers` and `domain` for IPv6 too Both fields are used for the 2 IP families and were writen in an IPv4 specific block. Signed-off-by: BenjiReis --- backend.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend.py b/backend.py index 1788db74..49c0eb37 100644 --- a/backend.py +++ b/backend.py @@ -1503,15 +1503,15 @@ def configureNetworking(mounts, admin_iface, admin_bridge, admin_config, hn_conf print >>mc, "NETMASK='%s'" % admin_config.netmask if admin_config.gateway: print >>mc, "GATEWAY='%s'" % admin_config.gateway - if manual_nameservers: - print >>mc, "DNS='%s'" % (','.join(nameservers),) - if domain: - print >>mc, "DOMAIN='%s'" % domain print >>mc, "MODEV6='%s'" % netinterface.NetInterface.getModeStr(admin_config.modev6) if admin_config.modev6 == netinterface.NetInterface.Static: print >>mc, "IPv6='%s'" % admin_config.ipv6addr if admin_config.ipv6_gateway: print >>mc, "IPv6_GATEWAY='%s'" % admin_config.ipv6_gateway + if manual_nameservers: + print >>mc, "DNS='%s'" % (','.join(nameservers),) + if domain: + print >>mc, "DOMAIN='%s'" % domain if admin_config.vlan: print >>mc, "VLAN='%d'" % admin_config.vlan mc.close() From a9c8105a090043049d04681ebe2d72826f51aa4f Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Thu, 1 Jun 2023 13:51:13 +0200 Subject: [PATCH 16/38] NetInterface inherits from Object Signed-off-by: BenjiReis --- netinterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netinterface.py b/netinterface.py index 0cdcad47..af3578b8 100644 --- a/netinterface.py +++ b/netinterface.py @@ -16,7 +16,7 @@ def getTextOrNone(nodelist): rc = rc + node.data return rc == "" and None or rc.strip().encode() -class NetInterface: +class NetInterface(object): """ Represents the configuration of a network interface. """ Static = 1 From ca7dfab34a646279b4e67e531fe1e46baa2460cb Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Tue, 30 May 2023 10:36:16 +0200 Subject: [PATCH 17/38] Add `NetInterfaceV6` to init an IPv6 interface Inherits from `NetInterface` to mutualize the code Signed-off-by: BenjiReis --- netinterface.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/netinterface.py b/netinterface.py index af3578b8..49c0299c 100644 --- a/netinterface.py +++ b/netinterface.py @@ -348,3 +348,22 @@ def loadFromNetDb(jdata, hwaddr): nic.addIPv6(modev6, ipv6addr, gatewayv6) return nic + +class NetInterfaceV6(NetInterface): + def __init__(self, mode, hwaddr, ipaddr=None, netmask=None, gateway=None, dns=None, domain=None, vlan=None): + super(NetInterfaceV6, self).__init__(None, hwaddr, None, None, None, None, None, vlan) + + ipv6addr = None + if mode == self.Static: + assert ipaddr + assert netmask + + ipv6addr = ipaddr + "/" + netmask + if dns == '': + dns = None + elif isinstance(dns, str): + dns = [ dns ] + self.dns = dns + self.domain = domain + + self.addIPv6(mode, ipv6addr=ipv6addr, ipv6gw=gateway) From 611cb247a4ae429b66f7c9096f126990cc25528a Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Tue, 30 May 2023 10:40:33 +0200 Subject: [PATCH 18/38] isStatic helpers by IP family Signed-off-by: BenjiReis --- netinterface.py | 10 +++++++--- netutil.py | 2 +- tui/installer/screens.py | 6 +++--- tui/network.py | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/netinterface.py b/netinterface.py index 49c0299c..168387ff 100644 --- a/netinterface.py +++ b/netinterface.py @@ -122,10 +122,14 @@ def valid(self): return False return self.mode or self.modev6 - def isStatic(self): - """ Returns true if a static interface configuration is represented. """ + def isStatic4(self): + """ Returns true if an IPv4 static interface configuration is represented. """ return self.mode == self.Static + def isStatic6(self): + """ Returns true if an IPv6 static interface configuration is represented. """ + return self.modev6 == self.Static + def isVlan(self): return self.vlan is not None @@ -190,7 +194,7 @@ def writeRHStyleInterface(self, iface): def waitUntilUp(self, iface): - if not self.isStatic(): + if not self.isStatic4(): return True if not self.gateway: return True diff --git a/netutil.py b/netutil.py index 898fe0ea..c975b0d0 100644 --- a/netutil.py +++ b/netutil.py @@ -92,7 +92,7 @@ def writeResolverFile(configuration, filename): for iface in configuration: settings = configuration[iface] - if settings.isStatic() and settings.dns: + if settings.isStatic4() and settings.dns: if settings.dns: for server in settings.dns: outfile.write("nameserver %s\n" % server) diff --git a/tui/installer/screens.py b/tui/installer/screens.py index 74bb8cfb..fef4c4b4 100644 --- a/tui/installer/screens.py +++ b/tui/installer/screens.py @@ -805,7 +805,7 @@ def ns_callback((enabled, )): for entry in [ns1_entry, ns2_entry, ns3_entry]: entry.setFlags(FLAG_DISABLED, enabled) - hide_rb = answers['net-admin-configuration'].isStatic() + hide_rb = answers['net-admin-configuration'].isStatic4() # HOSTNAME: hn_title = Textbox(len("Hostname Configuration"), 1, "Hostname Configuration") @@ -935,7 +935,7 @@ def nsvalue(answers, id): answers['manual-nameservers'][1].append(ns2_entry.value()) if ns3_entry.value() != '': answers['manual-nameservers'][1].append(ns3_entry.value()) - if 'net-admin-configuration' in answers and answers['net-admin-configuration'].isStatic(): + if 'net-admin-configuration' in answers and answers['net-admin-configuration'].isStatic4(): answers['net-admin-configuration'].dns = answers['manual-nameservers'][1] else: answers['manual-nameservers'] = (False, None) @@ -1036,7 +1036,7 @@ def dhcp_change(): for x in [ ntp1_field, ntp2_field, ntp3_field ]: x.setFlags(FLAG_DISABLED, not dhcp_cb.value()) - hide_cb = answers['net-admin-configuration'].isStatic() + hide_cb = answers['net-admin-configuration'].isStatic4() gf = GridFormHelp(tui.screen, 'NTP Configuration', 'ntpconf', 1, 4) text = TextboxReflowed(60, "Please specify details of the NTP servers you wish to use (e.g. pool.ntp.org)?") diff --git a/tui/network.py b/tui/network.py index faf87dc9..0ab6751b 100644 --- a/tui/network.py +++ b/tui/network.py @@ -34,7 +34,7 @@ def dhcp_change(): dns_field = Entry(16) vlan_field = Entry(16) - if defaults and defaults.isStatic(): + if defaults and defaults.isStatic4(): # static configuration defined previously dhcp_rb = SingleRadioButton("Automatic configuration (DHCP)", None, 0) dhcp_rb.setCallback(dhcp_change, ()) From f13f185a47248f54d06f6481d1359646df027174 Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Tue, 30 May 2023 10:46:12 +0200 Subject: [PATCH 19/38] Use `socket.inet_pton` to validate IP Add family specific validator as well Signed-off-by: BenjiReis --- netutil.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/netutil.py b/netutil.py index c975b0d0..efa9bfe7 100644 --- a/netutil.py +++ b/netutil.py @@ -4,12 +4,12 @@ import diskutil import util import re +import socket import subprocess import time import errno from xcp import logger from xcp.net.biosdevname import all_devices_all_names -from socket import inet_ntoa from struct import pack class NIC: @@ -225,16 +225,21 @@ def valid_vlan(vlan): return False return True -def valid_ip_addr(addr): - if not re.match('^\d+\.\d+\.\d+\.\d+$', addr): - return False - els = addr.split('.') - if len(els) != 4: +def valid_ip_address_family(addr, family): + try: + socket.inet_pton(family, addr) + return True + except socket.error: return False - for el in els: - if int(el) > 255: - return False - return True + +def valid_ipv4_addr(addr): + return valid_ip_address_family(addr, socket.AF_INET) + +def valid_ipv6_addr(addr): + return valid_ip_address_family(addr, socket.AF_INET6) + +def valid_ip_addr(addr): + return valid_ipv4_addr(addr) or valid_ipv6_addr(addr) def network(ipaddr, netmask): ip = map(int,ipaddr.split('.',3)) @@ -246,7 +251,7 @@ def prefix2netmask(mask): bits = 0 for i in xrange(32-mask, 32): bits |= (1 << i) - return inet_ntoa(pack('>I', bits)) + return socket.inet_ntoa(pack('>I', bits)) class NetDevices: def __init__(self): From 430ea794cb7cfe695b720ff3ebafa9fbd29351a5 Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Tue, 30 May 2023 10:48:11 +0200 Subject: [PATCH 20/38] Take IPv6 addresses into account to determine an interface is up Signed-off-by: BenjiReis --- netutil.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netutil.py b/netutil.py index efa9bfe7..00af6861 100644 --- a/netutil.py +++ b/netutil.py @@ -137,7 +137,11 @@ def interfaceUp(interface): if rc != 0: return False inets = filter(lambda x: x.startswith(" inet "), out.split("\n")) - return len(inets) == 1 + if len(inets) == 1: + return True + + inet6s = filter(lambda x: x.startswith(" inet6 "), out.split("\n")) + return len(inet6s) > 1 # Not just the fe80:: address # work out if a link is up: def linkUp(interface): From 1c6377231675476739f049989b5b3f919d25d7d5 Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Tue, 30 May 2023 10:56:47 +0200 Subject: [PATCH 21/38] Enable IPv6 in the host when it is configured Signed-off-by: BenjiReis --- backend.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend.py b/backend.py index 49c0eb37..dfada781 100644 --- a/backend.py +++ b/backend.py @@ -1553,12 +1553,18 @@ def configureNetworking(mounts, admin_iface, admin_bridge, admin_config, hn_conf # now we need to write /etc/sysconfig/network nfd = open("%s/etc/sysconfig/network" % mounts["root"], "w") nfd.write("NETWORKING=yes\n") - if admin_config.modev6: + ipv6 = admin_config.modev6 is not None + if ipv6: nfd.write("NETWORKING_IPV6=yes\n") util.runCmd2(['chroot', mounts['root'], 'systemctl', 'enable', 'ip6tables']) else: nfd.write("NETWORKING_IPV6=no\n") netutil.disable_ipv6_module(mounts["root"]) + + with open("%s/etc/sysctl.d/91-net-ipv6.conf" % mounts["root"], "w") as ipv6_conf: + for i in ['all', 'default']: + ipv6_conf.write('net.ipv6.conf.%s.disable_ipv6=%d\n' % (i, int(not ipv6))) + nfd.write("IPV6_AUTOCONF=no\n") nfd.write('NTPSERVERARGS="iburst prefer"\n') nfd.close() From 24268a26fac8f0acd81dc1a2f5bb01a57da28101 Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Tue, 30 May 2023 11:16:57 +0200 Subject: [PATCH 22/38] Write IPv6 conf files for the installer Signed-off-by: BenjiReis --- netinterface.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/netinterface.py b/netinterface.py index 168387ff..729221d9 100644 --- a/netinterface.py +++ b/netinterface.py @@ -168,8 +168,7 @@ def writeRHStyleInterface(self, iface): """ Write a RedHat-style configuration entry for this interface to file object f using interface name iface. """ - assert self.modev6 is None - assert self.mode + assert self.modev6 or self.mode iface_vlan = self.getInterfaceName(iface) f = open('/etc/sysconfig/network-scripts/ifcfg-%s' % iface_vlan, 'w') @@ -178,7 +177,7 @@ def writeRHStyleInterface(self, iface): if self.mode == self.DHCP: f.write("BOOTPROTO=dhcp\n") f.write("PERSISTENT_DHCLIENT=1\n") - else: + elif self.mode == self.Static: # CA-11825: broadcast needs to be determined for non-standard networks bcast = self.getBroadcast() f.write("BOOTPROTO=none\n") @@ -188,6 +187,27 @@ def writeRHStyleInterface(self, iface): f.write("NETMASK=%s\n" % self.netmask) if self.gateway: f.write("GATEWAY=%s\n" % self.gateway) + + if self.modev6: + with open('/etc/sysconfig/network', 'w') as net_conf: + net_conf.write("NETWORKING_IPV6=yes\n") + f.write("IPV6INIT=yes\n") + f.write("IPV6_DEFROUTE=yes\n") + f.write("IPV6_DEFAULTDEV=%s\n" % iface_vlan) + + if self.modev6 == self.DHCP: + f.write("DHCPV6C=yes\n") + f.write("PERSISTENT_DHCLIENT_IPV6=yes\n") + f.write("IPV6_FORCE_ACCEPT_RA=yes\n") + f.write("IPV6_AUTOCONF=no\n") + elif self.modev6 == self.Static: + f.write("IPV6ADDR=%s\n" % self.ipv6addr) + if self.ipv6_gateway: + f.write("IPV6_DEFAULTGW=%s\n" % (self.ipv6_gateway)) + f.write("IPV6_AUTOCONF=no\n") + elif self.modev6 == self.Autoconf: + f.write("IPV6_AUTOCONF=yes\n") + if self.vlan: f.write("VLAN=yes\n") f.close() From 0fa71e235837483c4a232e3233e460bb1e6a7e9c Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Tue, 30 May 2023 11:38:10 +0200 Subject: [PATCH 23/38] Configure IPv6 in TUI Add a screen to choose to configure IPv4, IPv6 or both Display the interface screen for IPv4 and/or IPv6 Signed-off-by: BenjiReis --- tui/network.py | 353 +++++++++++++++++++++++++++++++------------------ 1 file changed, 227 insertions(+), 126 deletions(-) diff --git a/tui/network.py b/tui/network.py index 0ab6751b..126974ee 100644 --- a/tui/network.py +++ b/tui/network.py @@ -9,128 +9,215 @@ from netinterface import * import version import os +import time +import socket from snack import * def get_iface_configuration(nic, txt=None, defaults=None, include_dns=False): - - def use_vlan_cb_change(): - vlan_field.setFlags(FLAG_DISABLED, vlan_cb.value()) - - def dhcp_change(): - for x in [ ip_field, gateway_field, subnet_field, dns_field ]: - x.setFlags(FLAG_DISABLED, not dhcp_rb.selected()) - - gf = GridFormHelp(tui.screen, 'Networking', 'ifconfig', 1, 8) - if txt is None: - txt = "Configuration for %s (%s)" % (nic.name, nic.hwaddr) - text = TextboxReflowed(45, txt) - b = [("Ok", "ok"), ("Back", "back")] - buttons = ButtonBar(tui.screen, b) - - ip_field = Entry(16) - subnet_field = Entry(16) - gateway_field = Entry(16) - dns_field = Entry(16) - vlan_field = Entry(16) - - if defaults and defaults.isStatic4(): - # static configuration defined previously - dhcp_rb = SingleRadioButton("Automatic configuration (DHCP)", None, 0) - dhcp_rb.setCallback(dhcp_change, ()) - static_rb = SingleRadioButton("Static configuration:", dhcp_rb, 1) - static_rb.setCallback(dhcp_change, ()) - if defaults.ipaddr: - ip_field.set(defaults.ipaddr) - if defaults.netmask: - subnet_field.set(defaults.netmask) - if defaults.gateway: - gateway_field.set(defaults.gateway) - if defaults.dns: - dns_field.set(defaults.dns[0]) - else: - dhcp_rb = SingleRadioButton("Automatic configuration (DHCP)", None, 1) + def choose_primary_address_type(nic): + gf = GridFormHelp(tui.screen, 'Networking', 'Address type', 1, 8) + txt = "Choose an address type for %s (%s)" % (nic.name, nic.hwaddr) + text = TextboxReflowed(45, txt) + + b = [("Ok", "ok"), ("Back", "back")] + buttons = ButtonBar(tui.screen, b) + + # IPv4 by default + ipv4_rb = SingleRadioButton("IPv4", None, 1) + ipv6_rb = SingleRadioButton("IPv6", ipv4_rb, 0) + dual_rb = SingleRadioButton("Dual stack (IPv4 primary)", ipv6_rb, 0) + + gf.add(text, 0, 0, padding=(0, 0, 0, 1)) + gf.add(ipv4_rb, 0, 2, anchorLeft=True) + gf.add(ipv6_rb, 0, 3, anchorLeft=True) + gf.add(dual_rb, 0, 4, anchorLeft=True) + gf.add(buttons, 0, 5, growx=1) + + loop = True + direction = LEFT_BACKWARDS + address_type = None + while loop: + result = gf.run() + if buttons.buttonPressed(result) == 'back': + loop = False + elif buttons.buttonPressed(result) == 'ok': + value = None + if ipv4_rb.selected(): + value = "ipv4" + elif ipv6_rb.selected(): + value = "ipv6" + elif dual_rb.selected(): + value = "dual" + loop = False + direction = RIGHT_FORWARDS + address_type = value + + tui.screen.popWindow() + return direction, address_type + + def get_ip_configuration(nic, txt, defaults, include_dns, iface_class): + def use_vlan_cb_change(): + vlan_field.setFlags(FLAG_DISABLED, vlan_cb.value()) + + def dhcp_change(): + for x in [ ip_field, gateway_field, subnet_field, dns_field ]: + x.setFlags(FLAG_DISABLED, static_rb.selected()) + + ipv6 = iface_class == NetInterfaceV6 + + gf = GridFormHelp(tui.screen, 'Networking', 'ifconfig', 1, 10) + if txt is None: + txt = "Configuration for %s (%s)" % (nic.name, nic.hwaddr) + text = TextboxReflowed(45, txt) + b = [("Ok", "ok"), ("Back", "back")] + buttons = ButtonBar(tui.screen, b) + + #TODO? Change size for IPv6? If so which size? + ip_field = Entry(16) + subnet_field = Entry(16) + gateway_field = Entry(16) + dns_field = Entry(16) + vlan_field = Entry(16) + + static = bool(defaults and (defaults.modev6 if ipv6 else defaults.mode) == NetInterface.Static) + dhcp_rb = SingleRadioButton("Automatic configuration (DHCP)", None, not static) dhcp_rb.setCallback(dhcp_change, ()) - static_rb = SingleRadioButton("Static configuration:", dhcp_rb, 0) + static_rb = SingleRadioButton("Static configuration:", dhcp_rb, static) static_rb.setCallback(dhcp_change, ()) - ip_field.setFlags(FLAG_DISABLED, False) - subnet_field.setFlags(FLAG_DISABLED, False) - gateway_field.setFlags(FLAG_DISABLED, False) - dns_field.setFlags(FLAG_DISABLED, False) - - vlan_cb = Checkbox("Use VLAN:", defaults.isVlan() if defaults else False) - vlan_cb.setCallback(use_vlan_cb_change, ()) - if defaults and defaults.isVlan(): - vlan_field.set(str(defaults.vlan)) - else: - vlan_field.setFlags(FLAG_DISABLED, False) - - ip_text = Textbox(15, 1, "IP Address:") - subnet_text = Textbox(15, 1, "Subnet mask:") - gateway_text = Textbox(15, 1, "Gateway:") - dns_text = Textbox(15, 1, "Nameserver:") - vlan_text = Textbox(15, 1, "VLAN (1-4094):") - - entry_grid = Grid(2, include_dns and 4 or 3) - entry_grid.setField(ip_text, 0, 0) - entry_grid.setField(ip_field, 1, 0) - entry_grid.setField(subnet_text, 0, 1) - entry_grid.setField(subnet_field, 1, 1) - entry_grid.setField(gateway_text, 0, 2) - entry_grid.setField(gateway_field, 1, 2) - if include_dns: - entry_grid.setField(dns_text, 0, 3) - entry_grid.setField(dns_field, 1, 3) - - vlan_grid = Grid(2, 1) - vlan_grid.setField(vlan_text, 0, 0) - vlan_grid.setField(vlan_field, 1, 0) - - gf.add(text, 0, 0, padding=(0, 0, 0, 1)) - gf.add(dhcp_rb, 0, 2, anchorLeft=True) - gf.add(static_rb, 0, 3, anchorLeft=True) - gf.add(entry_grid, 0, 4, padding=(0, 0, 0, 1)) - gf.add(vlan_cb, 0, 5, anchorLeft=True) - gf.add(vlan_grid, 0, 6, padding=(0, 0, 0, 1)) - gf.add(buttons, 0, 7, growx=1) - - loop = True - while loop: - result = gf.run() - - if buttons.buttonPressed(result) in ['ok', None]: - # validate input - msg = '' - if static_rb.selected(): - if not netutil.valid_ip_addr(ip_field.value()): - msg = 'IP Address' - elif not netutil.valid_ip_addr(subnet_field.value()): - msg = 'Subnet mask' - elif gateway_field.value() != '' and not netutil.valid_ip_addr(gateway_field.value()): - msg = 'Gateway' - elif dns_field.value() != '' and not netutil.valid_ip_addr(dns_field.value()): - msg = 'Nameserver' - if vlan_cb.selected(): - if not netutil.valid_vlan(vlan_field.value()): - msg = 'VLAN' - if msg != '': - tui.progress.OKDialog("Networking", "Invalid %s, please check the field and try again." % msg) + if ipv6: + autoconf_rb = SingleRadioButton("Automatic configuration (Autoconf)", static_rb, 0) + autoconf_rb.setCallback(dhcp_change, ()) + dhcp_change() + + if defaults: + if ipv6: + if defaults.ipv6addr: + ip6addr, netmask = defaults.ipv6addr.split("/") + ip_field.set(ip6addr) + subnet_field.set(netmask) + if defaults.ipv6_gateway: + gateway_field.set(defaults.ipv6_gateway) + else: + if defaults.ipaddr: + ip_field.set(defaults.ipaddr) + if defaults.netmask: + subnet_field.set(defaults.netmask) + if defaults.gateway: + gateway_field.set(defaults.gateway) + + if defaults.dns: + dns_field.set(defaults.dns[0]) + + vlan_cb = Checkbox("Use VLAN:", defaults.isVlan() if defaults else False) + vlan_cb.setCallback(use_vlan_cb_change, ()) + if defaults and defaults.isVlan(): + vlan_field.set(str(defaults.vlan)) + else: + vlan_field.setFlags(FLAG_DISABLED, False) + + ip_msg = "IPv6 Address" if ipv6 else "IP Address" + mask_msg = "CIDR (4-128)" if ipv6 else "Subnet mask" + ip_text = Textbox(15, 1, "%s:" % ip_msg) + subnet_text = Textbox(15, 1, "%s:" % mask_msg) + gateway_text = Textbox(15, 1, "Gateway:") + dns_text = Textbox(15, 1, "Nameserver:") + vlan_text = Textbox(15, 1, "VLAN (1-4094):") + + entry_grid = Grid(2, include_dns and 4 or 3) + entry_grid.setField(ip_text, 0, 0) + entry_grid.setField(ip_field, 1, 0) + entry_grid.setField(subnet_text, 0, 1) + entry_grid.setField(subnet_field, 1, 1) + entry_grid.setField(gateway_text, 0, 2) + entry_grid.setField(gateway_field, 1, 2) + if include_dns: + entry_grid.setField(dns_text, 0, 3) + entry_grid.setField(dns_field, 1, 3) + + vlan_grid = Grid(2, 1) + vlan_grid.setField(vlan_text, 0, 0) + vlan_grid.setField(vlan_field, 1, 0) + + gf.add(text, 0, 0, padding=(0, 0, 0, 1)) + gf.add(dhcp_rb, 0, 2, anchorLeft=True) + gf.add(static_rb, 0, 3, anchorLeft=True) + gf.add(entry_grid, 0, 4, padding=(0, 0, 0, 1)) + if ipv6: + gf.add(autoconf_rb, 0, 5, anchorLeft=True) + # One more line for IPv6 autoconf + gf.add(vlan_cb, 0, 5 + ipv6, anchorLeft=True) + gf.add(vlan_grid, 0, 6 + ipv6, padding=(0, 0, 0, 1)) + gf.add(buttons, 0, 7 + ipv6, growx=1) + + loop = True + ip_family = socket.AF_INET6 if ipv6 else socket.AF_INET + while loop: + result = gf.run() + + if buttons.buttonPressed(result) in ['ok', None]: + # validate input + msg = '' + if static_rb.selected(): + invalid_subnet = int(subnet_field.value()) > 128 or int(subnet_field.value()) < 4 if ipv6 else not netutil.valid_ipv4_addr(subnet_field.value()) + if not netutil.valid_ip_address_family(ip_field.value(), ip_family): + msg = ip_msg + elif invalid_subnet: + msg = mask_msg + elif gateway_field.value() != '' and not netutil.valid_ip_address_family(gateway_field.value(), ip_family): + msg = 'Gateway' + elif dns_field.value() != '' and not netutil.valid_ip_address_family(dns_field.value(), ip_family): + msg = 'Nameserver' + if vlan_cb.selected(): + if not netutil.valid_vlan(vlan_field.value()): + msg = 'VLAN' + if msg != '': + tui.progress.OKDialog("Networking", "Invalid %s, please check the field and try again." % msg) + else: + loop = False else: loop = False + + tui.screen.popWindow() + + if buttons.buttonPressed(result) == 'back': return LEFT_BACKWARDS, None + + vlan_value = int(vlan_field.value()) if vlan_cb.selected() else None + if dhcp_rb.selected(): + answers = iface_class(NetInterface.DHCP, nic.hwaddr, vlan=vlan_value) + elif ipv6 and autoconf_rb.selected(): + answers = iface_class(NetInterface.Autoconf, nic.hwaddr, vlan=vlan_value) else: - loop = False + answers = iface_class(NetInterface.Static, nic.hwaddr, ip_field.value(), + subnet_field.value(), gateway_field.value(), + dns_field.value(), vlan=vlan_value) - tui.screen.popWindow() + return RIGHT_FORWARDS, answers - if buttons.buttonPressed(result) == 'back': return LEFT_BACKWARDS, None + direction, address_type = choose_primary_address_type(nic) + if direction == LEFT_BACKWARDS: + return LEFT_BACKWARDS, None + + answers = None + if address_type in ["ipv4", "dual"]: + direction, answers = get_ip_configuration(nic, txt, defaults, include_dns, NetInterface) + if direction == LEFT_BACKWARDS: + return LEFT_BACKWARDS, None + + if address_type in ["ipv6", "dual"]: + direction, answers_ipv6 = get_ip_configuration(nic, txt, defaults, include_dns, NetInterfaceV6) + if direction == LEFT_BACKWARDS: + return LEFT_BACKWARDS, None + + if answers == None: + answers = answers_ipv6 + else: + answers.modev6 = answers_ipv6.modev6 + answers.ipv6addr = answers_ipv6.ipv6addr + answers.ipv6_gateway = answers_ipv6.ipv6_gateway + if answers_ipv6.dns != None: + answers.dns = answers_ipv6.dns if answers.dns == None else answers.dns + answers_ipv6.dns - vlan_value = int(vlan_field.value()) if vlan_cb.selected() else None - if bool(dhcp_rb.selected()): - answers = NetInterface(NetInterface.DHCP, nic.hwaddr, vlan=vlan_value) - else: - answers = NetInterface(NetInterface.Static, nic.hwaddr, ip_field.value(), - subnet_field.value(), gateway_field.value(), - dns_field.value(), vlan=vlan_value) return RIGHT_FORWARDS, answers def select_netif(text, conf, offer_existing=False, default=None): @@ -286,23 +373,37 @@ def specify_configuration(answers, txt, defaults): ifaceName = conf_dict['config'].getInterfaceName(conf_dict['interface']) netutil.ifdown(ifaceName) - # check that we have *some* network: - if netutil.ifup(ifaceName) != 0 or not netutil.interfaceUp(ifaceName): + def display_error(): tui.progress.clearModelessDialog() tui.progress.OKDialog("Networking", "The network still does not appear to be active. Please check your settings, and try again.") - direction = REPEAT_STEP - else: - if answers and type(answers) == dict: - # write out results - answers[interface_key] = conf_dict['interface'] - answers[config_key] = conf_dict['config'] - # update cache of manual configurations - manual_config = {} - all_dhcp = False - if 'runtime-iface-configuration' in answers: - manual_config = answers['runtime-iface-configuration'][1] - manual_config[conf_dict['interface']] = conf_dict['config'] - answers['runtime-iface-configuration'] = (all_dhcp, manual_config) - tui.progress.clearModelessDialog() + return REPEAT_STEP + + if netutil.ifup(ifaceName) != 0: + return display_error() + + # For Autoconf wait a bit for network setup + try_nb = 20 if conf_dict['config'].modev6 == NetInterface.Autoconf else 0 + while True: + if try_nb == 0 or netutil.interfaceUp(ifaceName): + break + try_nb -= 1 + time.sleep(0.1) + + # check that we have *some* network: + if not netutil.interfaceUp(ifaceName): + return display_error() + + if answers and type(answers) == dict: + # write out results + answers[interface_key] = conf_dict['interface'] + answers[config_key] = conf_dict['config'] + # update cache of manual configurations + manual_config = {} + all_dhcp = False + if 'runtime-iface-configuration' in answers: + manual_config = answers['runtime-iface-configuration'][1] + manual_config[conf_dict['interface']] = conf_dict['config'] + answers['runtime-iface-configuration'] = (all_dhcp, manual_config) + tui.progress.clearModelessDialog() return direction From 9e9c300c6228bc6587549353ffdce2ab0ba4ab04 Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Thu, 1 Jun 2023 14:30:18 +0200 Subject: [PATCH 24/38] `waitUntilUp` also wait for IPv6 if statically configured Need to add `ndisc6` to the install.img Signed-off-by: BenjiReis --- netinterface.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/netinterface.py b/netinterface.py index 729221d9..dd11fc37 100644 --- a/netinterface.py +++ b/netinterface.py @@ -214,14 +214,18 @@ def writeRHStyleInterface(self, iface): def waitUntilUp(self, iface): - if not self.isStatic4(): - return True - if not self.gateway: - return True - - rc = util.runCmd2(['/usr/sbin/arping', '-f', '-w', '120', '-I', - self.getInterfaceName(iface), self.gateway]) - return rc == 0 + iface_name = self.getInterfaceName(iface) + if self.isStatic4() and self.gateway and util.runCmd2( + ['/usr/sbin/arping', '-f', '-w', '120', '-I', iface_name, self.gateway] + ): + return False + + if self.isStatic6() and self.ipv6_gateway and util.runCmd2( + ['/usr/sbin/ndisc6', '-1', '-w', '120', self.ipv6_gateway, iface_name] + ): + return False + + return True @staticmethod def getModeStr(mode): From 806cfe06336d5bdc6a857af39519534c30ed5c3d Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Thu, 1 Jun 2023 15:17:12 +0200 Subject: [PATCH 25/38] Hide dynamic buttons in tui when in static mode Add `isDynamic` helper to `NetInterface` Signed-off-by: BenjiReis --- netinterface.py | 4 ++++ netutil.py | 2 +- tui/installer/screens.py | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/netinterface.py b/netinterface.py index dd11fc37..50aaef4e 100644 --- a/netinterface.py +++ b/netinterface.py @@ -130,6 +130,10 @@ def isStatic6(self): """ Returns true if an IPv6 static interface configuration is represented. """ return self.modev6 == self.Static + def isDynamic(self): + """ Returns true if a dynamic interface configuration is represented. """ + return self.mode == self.DHCP or self.modev6 == self.DHCP or self.modev6 == self.Autoconf + def isVlan(self): return self.vlan is not None diff --git a/netutil.py b/netutil.py index 00af6861..1719e336 100644 --- a/netutil.py +++ b/netutil.py @@ -92,7 +92,7 @@ def writeResolverFile(configuration, filename): for iface in configuration: settings = configuration[iface] - if settings.isStatic4() and settings.dns: + if (not settings.isDynamic()) and settings.dns: if settings.dns: for server in settings.dns: outfile.write("nameserver %s\n" % server) diff --git a/tui/installer/screens.py b/tui/installer/screens.py index fef4c4b4..fdc4e224 100644 --- a/tui/installer/screens.py +++ b/tui/installer/screens.py @@ -805,7 +805,7 @@ def ns_callback((enabled, )): for entry in [ns1_entry, ns2_entry, ns3_entry]: entry.setFlags(FLAG_DISABLED, enabled) - hide_rb = answers['net-admin-configuration'].isStatic4() + hide_rb = not answers['net-admin-configuration'].isDynamic() # HOSTNAME: hn_title = Textbox(len("Hostname Configuration"), 1, "Hostname Configuration") @@ -935,7 +935,7 @@ def nsvalue(answers, id): answers['manual-nameservers'][1].append(ns2_entry.value()) if ns3_entry.value() != '': answers['manual-nameservers'][1].append(ns3_entry.value()) - if 'net-admin-configuration' in answers and answers['net-admin-configuration'].isStatic4(): + if 'net-admin-configuration' in answers and not answers['net-admin-configuration'].isDynamic(): answers['net-admin-configuration'].dns = answers['manual-nameservers'][1] else: answers['manual-nameservers'] = (False, None) @@ -1036,7 +1036,7 @@ def dhcp_change(): for x in [ ntp1_field, ntp2_field, ntp3_field ]: x.setFlags(FLAG_DISABLED, not dhcp_cb.value()) - hide_cb = answers['net-admin-configuration'].isStatic4() + hide_cb = not answers['net-admin-configuration'].isDynamic() gf = GridFormHelp(tui.screen, 'NTP Configuration', 'ntpconf', 1, 4) text = TextboxReflowed(60, "Please specify details of the NTP servers you wish to use (e.g. pool.ntp.org)?") From a469a6baed467c4540d311a8311daad08c65f9c9 Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Wed, 7 Jun 2023 11:08:38 +0200 Subject: [PATCH 26/38] Keep IPv6 enablement/disablement upon upgrades Add `etc/sysctl.d/91-net-ipv6.conf` in the restore list Signed-off-by: BenjiReis --- upgrade.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/upgrade.py b/upgrade.py index ec8d9104..8a4b0851 100644 --- a/upgrade.py +++ b/upgrade.py @@ -445,6 +445,9 @@ def buildRestoreList(self): self.restore_list += ['var/lib/xcp/verify_certificates'] + # Keep IPv6 enablement/disablement upon upgrades + self.restore_list += ['etc/sysctl.d/91-net-ipv6.conf'] + completeUpgradeArgs = ['mounts', 'installation-to-overwrite', 'primary-disk', 'backup-partnum', 'logs-partnum', 'net-admin-interface', 'net-admin-bridge', 'net-admin-configuration'] def completeUpgrade(self, mounts, prev_install, target_disk, backup_partnum, logs_partnum, admin_iface, admin_bridge, admin_config): From 7cc2d256eec24319b8247fa205ddff6fb528390c Mon Sep 17 00:00:00 2001 From: Samuel Verschelde Date: Wed, 5 Sep 2018 10:59:45 +0200 Subject: [PATCH 27/38] Change checkbox text for the EXT vs LVM option Signed-off-by: Samuel Verschelde Orig-commit: b4ebf8174694799b19eaa5a946ba7f00827eeef5 Orig-commit: 03d3a84ce27976a1d9d6ea9e42a1e7492967f3e6 Orig-commit: d3857b8f23e7e02800e5a531a01b184482023479 Signed-off-by: Yann Dirson --- tui/installer/screens.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tui/installer/screens.py b/tui/installer/screens.py index 74bb8cfb..fe00825a 100644 --- a/tui/installer/screens.py +++ b/tui/installer/screens.py @@ -675,16 +675,19 @@ def select_guest_disks(answers): cbt = CheckboxTree(3, scroll) for (c_text, c_item) in entries: cbt.append(c_text, c_item, c_item in currently_selected) - txt = "Enable thin provisioning" - if len(BRAND_VDI) > 0: - txt += " (Optimized storage for %s)" % BRAND_VDI + txt = "Use EXT instead of LVM for local storage repository" tb = Checkbox(txt, srtype == constants.SR_TYPE_EXT and 1 or 0) - gf = GridFormHelp(tui.screen, 'Virtual Machine Storage', 'guestdisk:info', 1, 4) + explanations = Textbox(54, 2, + "LVM: block based. Thick provisioning.\n" + "EXT: file based. Thin provisioning.") + + gf = GridFormHelp(tui.screen, 'Virtual Machine Storage', 'guestdisk:info', 1, 5) gf.add(text, 0, 0, padding=(0, 0, 0, 1)) gf.add(cbt, 0, 1, padding=(0, 0, 0, 1)) - gf.add(tb, 0, 2, padding=(0, 0, 0, 1)) - gf.add(buttons, 0, 3, growx=1) + gf.add(tb, 0, 2, padding=(0, 0, 0, 0)) + gf.add(explanations, 0, 3, padding=(0, 0, 0, 1)) + gf.add(buttons, 0, 4, growx=1) gf.addHotKey('F5') tui.update_help_line([None, " more info"]) From 37ab2d158812b7626a3cb02d009aab1b537780a6 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 23 Dec 2022 11:13:16 +0100 Subject: [PATCH 28/38] Local SR: use a RadioBar for LVM/EXT choice Signed-off-by: Yann Dirson --- tui/installer/screens.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tui/installer/screens.py b/tui/installer/screens.py index fe00825a..25801e40 100644 --- a/tui/installer/screens.py +++ b/tui/installer/screens.py @@ -675,18 +675,17 @@ def select_guest_disks(answers): cbt = CheckboxTree(3, scroll) for (c_text, c_item) in entries: cbt.append(c_text, c_item, c_item in currently_selected) - txt = "Use EXT instead of LVM for local storage repository" - tb = Checkbox(txt, srtype == constants.SR_TYPE_EXT and 1 or 0) - - explanations = Textbox(54, 2, - "LVM: block based. Thick provisioning.\n" - "EXT: file based. Thin provisioning.") + rb_title = Textbox(15, 1, "Storage type") + rb = RadioBar(tui.screen, (("LVM: block based. Thick provisioning.", + constants.SR_TYPE_LVM, srtype == constants.SR_TYPE_LVM), + ("EXT: file based. Thin provisioning.", + constants.SR_TYPE_EXT, srtype == constants.SR_TYPE_EXT))) gf = GridFormHelp(tui.screen, 'Virtual Machine Storage', 'guestdisk:info', 1, 5) gf.add(text, 0, 0, padding=(0, 0, 0, 1)) gf.add(cbt, 0, 1, padding=(0, 0, 0, 1)) - gf.add(tb, 0, 2, padding=(0, 0, 0, 0)) - gf.add(explanations, 0, 3, padding=(0, 0, 0, 1)) + gf.add(rb_title, 0, 2, padding=(0, 0, 0, 1)) + gf.add(rb, 0, 3, padding=(0, 0, 0, 1)) gf.add(buttons, 0, 4, growx=1) gf.addHotKey('F5') @@ -707,7 +706,7 @@ def select_guest_disks(answers): if button == 'back': return LEFT_BACKWARDS answers['guest-disks'] = cbt.getSelection() - answers['sr-type'] = tb.selected() and constants.SR_TYPE_EXT or constants.SR_TYPE_LVM + answers['sr-type'] = rb.getSelection() answers['sr-on-primary'] = answers['primary-disk'] in answers['guest-disks'] # if the user select no disks for guest storage, check this is what From 3e17337f5bc4675fb8508fa13f47bc8da538efe7 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 16 May 2023 18:03:02 +0200 Subject: [PATCH 29/38] Local SR: make EXT the default and first option instead of LVM Signed-off-by: Yann Dirson --- tui/installer/screens.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tui/installer/screens.py b/tui/installer/screens.py index 25801e40..a4845b84 100644 --- a/tui/installer/screens.py +++ b/tui/installer/screens.py @@ -658,7 +658,7 @@ def select_guest_disks(answers): currently_selected = answers['guest-disks'] else: currently_selected = answers['primary-disk'] - srtype = constants.SR_TYPE_LVM + srtype = constants.SR_TYPE_EXT if 'sr-type' in answers: srtype = answers['sr-type'] @@ -676,10 +676,11 @@ def select_guest_disks(answers): for (c_text, c_item) in entries: cbt.append(c_text, c_item, c_item in currently_selected) rb_title = Textbox(15, 1, "Storage type") - rb = RadioBar(tui.screen, (("LVM: block based. Thick provisioning.", + rb = RadioBar(tui.screen, (("EXT: file based. Thin provisioning.", + constants.SR_TYPE_EXT, srtype == constants.SR_TYPE_EXT), + ("LVM: block based. Thick provisioning.", constants.SR_TYPE_LVM, srtype == constants.SR_TYPE_LVM), - ("EXT: file based. Thin provisioning.", - constants.SR_TYPE_EXT, srtype == constants.SR_TYPE_EXT))) + )) gf = GridFormHelp(tui.screen, 'Virtual Machine Storage', 'guestdisk:info', 1, 5) gf.add(text, 0, 0, padding=(0, 0, 0, 1)) From 0141a6e22952c82ea138943ac1ac7dcf4a6b367c Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 4 Oct 2023 12:34:42 +0200 Subject: [PATCH 30/38] getDiskDeviceSize: don't silently return None (part of #66) Signed-off-by: Yann Dirson --- diskutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/diskutil.py b/diskutil.py index 588f4379..101ccb02 100644 --- a/diskutil.py +++ b/diskutil.py @@ -280,6 +280,9 @@ def getDiskDeviceSize(dev): return int(__readOneLineFile__("/sys/block/%s/device/block/size" % dev)) elif os.path.exists("/sys/block/%s/size" % dev): return int(__readOneLineFile__("/sys/block/%s/size" % dev)) + else: + raise Exception("%s not found as %s or %s" % (dev, "/sys/block/%s/device/block/size", + "/sys/block/%s/size")) def getDiskSerialNumber(dev): # For Multipath nodes return info about 1st slave From 391e2cd7bdcdb578335d442c0ff656ff389c311d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 5 Oct 2023 12:15:43 +0200 Subject: [PATCH 31/38] installFromYum: give more detailed error messages on gpg errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers: 1. repo_gpgcheck: a. wrong system clock putting gpg key creation in the future, causing a yum crash (nothing special happens if the date of the signature is in the future ¯\_(ツ)_/¯) b. other yum crashes due to uncaught gpg exceptions (if any) c. lack of repomd signature (while repo_gpgcheck is in force) d. signature done by other key than the one in ISO ("repomd.xml signature could not be verified" ¯\_(ツ)_/¯) 2. gpgcheck: a. RPM signed with unknown key b. unsigned RPM referenced by unsigned repomd (no-repo-gpgcheck) c. RPM re-signed with unknown key, unsigned repomd (no-repo-gpgcheck) d. RPM overwritten with another RPM signed with known key (diagnosed through hash but, same diag as 2.c) e. delsigned/resigned/etc RPM, unchanged repomd (same diag as 2.c/d) Does not cover notably: - unsigned RPM referenced by (re)signed repomd In some cases Yum does not give an error, but dies because of an uncaught exception, which makes this check quite brittle, but in the worst case if messages change, we still fallback to the original "Error installing packages" message. Signed-off-by: Yann Dirson --- repository.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/repository.py b/repository.py index 98194052..ae5450ce 100644 --- a/repository.py +++ b/repository.py @@ -821,12 +821,59 @@ def installFromYum(targets, mounts, progress_callback, cachedir): rv = p.wait() stderr.seek(0) stderr = stderr.read() + gpg_uncaught_error = 0 + gpg_error_pubring_import = 0 + gpg_error_not_signed = 0 + gpg_error_bad_repo_sig = 0 + gpg_error_rpm_missing_key = None + gpg_error_rpm_not_signed = None + gpg_error_rpm_not_found = None if stderr: logger.log("YUM stderr: %s" % stderr.strip()) + if stderr.find(' in import_key_to_pubring') >= 0: + gpg_error_pubring_import = 1 + # add any other instance of uncaught GpgmeError before this like + elif stderr.find('gpgme.GpgmeError: ') >= 0: + gpg_uncaught_error = 1 + + elif re.search("Couldn't open file [^ ]*/repodata/repomd.xml.asc", stderr): + # would otherwise be mistaken for "pubring import" !? + gpg_error_not_signed = 1 + elif stderr.find('repomd.xml signature could not be verified') >= 0: + gpg_error_bad_repo_sig = 1 + + else: + match = re.search("Public key for ([^ ]*.rpm) is not installed", stderr) + if match: + gpg_error_rpm_missing_key = match.group(1) + match = re.search("Package ([^ ]*.rpm) is not signed", stderr) + if match: + gpg_error_rpm_not_signed = match.group(1) + match = re.search(r" ([^ ]*): \[Errno [0-9]*\] No more mirrors to try", stderr) + if match: + gpg_error_rpm_not_found = match.group(1) + if rv: logger.log("Yum exited with %d" % rv) - raise ErrorInstallingPackage("Error installing packages") + if gpg_error_pubring_import: + errmsg = "Signature key import failed" + elif gpg_uncaught_error: + errmsg = "Cryptography-related yum crash" + elif gpg_error_not_signed: + errmsg = "No signature on repository metadata" + elif gpg_error_bad_repo_sig: + errmsg = "Repository signature verification failure" + elif gpg_error_rpm_missing_key: + errmsg = "Missing key for %s" % (gpg_error_rpm_missing_key,) + elif gpg_error_rpm_not_signed: + errmsg = "Package not signed: %s" % (gpg_error_rpm_not_signed,) + elif gpg_error_rpm_not_found: + # rpm not found or corrupted/re-signed/etc + errmsg = "Cannot find valid rpm for %s" % (gpg_error_rpm_not_found,) + else: + errmsg = "Error installing packages" + raise ErrorInstallingPackage(errmsg) shutil.rmtree(os.path.join(mounts['root'], cachedir)) From 1781d8817f7fef0cf3d23f70501f3e10662005df Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 4 Oct 2023 15:41:56 +0200 Subject: [PATCH 32/38] findXenSourceBackups: log any exception caught - it is bad practice to "catch any" - not logging anything just hides information, which can be especially important here as the reason for a try/catch is not obvious (exceptions thrown by XenServerBackup.__init__?) Signed-off-by: Yann Dirson --- product.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/product.py b/product.py index 5f42f38c..61eb0c99 100644 --- a/product.py +++ b/product.py @@ -540,7 +540,8 @@ def findXenSourceBackups(): if backup.version >= XENSERVER_MIN_VERSION and \ backup.version <= THIS_PLATFORM_VERSION: backups.append(backup) - except: + except Exception as ex: + logger.log("findXenSourceBackups caught exception for partition %s: %s" % (p, ex)) pass if b: b.unmount() From c3f126eaa6515be3b5c4c5cba619c1b354e48113 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 4 Oct 2023 14:48:31 +0200 Subject: [PATCH 33/38] findXenSourceBackups: refactor and log reason for ignoring Previous code structure was that of 2 nested if's, with nominal code path being in the positive branch of each if. Adding another condition will bring a need for logging the reason to ignore a backup, but doing that by converting inner `if` to an `elif` chain (arguably the simplest modification) would bring the nominal code path to switch to the negative/last branch of the inner `elif` chain. At the same time, the outer `if` would not seem special enough to deserve it special place (and the cyclomatic complexity). This commit leverages the existing `try:except:` block to switch to an "error out the loop" pattern, where the nominal code path is just linear. Using `StopIteration` may feel like an abuse, but: - it is the only standard `Exception` that is not a `StandardError` or a `Warning`, and defining a new one could seem overkill - the closest alternative would be a `while True` loop and breaking out in both exceptional code paths and at the end of nominal path, where the `break` statements would "stop the iteration" as well Signed-off-by: Yann Dirson --- product.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/product.py b/product.py index 61eb0c99..0a919365 100644 --- a/product.py +++ b/product.py @@ -534,12 +534,25 @@ def findXenSourceBackups(): b = None try: b = util.TempMount(p, 'backup-', ['ro'], 'ext3') - if os.path.exists(os.path.join(b.mount_point, '.xen-backup-partition')): - backup = XenServerBackup(p, b.mount_point) - logger.log("Found a backup: %s" % (repr(backup),)) - if backup.version >= XENSERVER_MIN_VERSION and \ - backup.version <= THIS_PLATFORM_VERSION: - backups.append(backup) + if not os.path.exists(os.path.join(b.mount_point, '.xen-backup-partition')): + raise StopIteration() + + backup = XenServerBackup(p, b.mount_point) + logger.log("Found a backup: %s" % (repr(backup),)) + + if backup.version < XENSERVER_MIN_VERSION: + logger.log("findXenSourceBackups: ignoring, platform too old: %s < %s" % + (backup.version, XENSERVER_MIN_VERSION)) + raise StopIteration() + if backup.version > THIS_PLATFORM_VERSION: + logger.log("findXenSourceBackups: ignoring later platform: %s > %s" % + (backup.version, THIS_PLATFORM_VERSION)) + raise StopIteration() + + backups.append(backup) + + except StopIteration: + pass except Exception as ex: logger.log("findXenSourceBackups caught exception for partition %s: %s" % (p, ex)) pass From 7a993355d6d9f043eba4dc1c6454c48d32027495 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 4 Oct 2023 14:25:35 +0200 Subject: [PATCH 34/38] XenServerBackup: ignore backups with inaccessible primary disk (#66) Signed-off-by: Yann Dirson --- product.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/product.py b/product.py index 0a919365..27a7eb1a 100644 --- a/product.py +++ b/product.py @@ -548,6 +548,10 @@ def findXenSourceBackups(): logger.log("findXenSourceBackups: ignoring later platform: %s > %s" % (backup.version, THIS_PLATFORM_VERSION)) raise StopIteration() + if not os.path.exists(backup.root_disk): + logger.error("findXenSourceBackups: PRIMARY_DISK=%r does not exist" % + (backup.root_disk,)) + raise StopIteration() backups.append(backup) From 08059cfcbb31735db65b64ef53eee49b7d7b70f6 Mon Sep 17 00:00:00 2001 From: BenjiReis Date: Mon, 17 Jul 2023 10:55:40 +0200 Subject: [PATCH 35/38] Keep user multipath configuration upon upgrade All files matchin `custom.*\.conf` will be kept upon upgrade. See: https://github.com/xapi-project/sm/pull/600 Signed-off-by: BenjiReis --- upgrade.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/upgrade.py b/upgrade.py index 0672b577..9bf45b77 100644 --- a/upgrade.py +++ b/upgrade.py @@ -448,6 +448,9 @@ def buildRestoreList(self): # NRPE service config self.restore_list += ['etc/nagios/nrpe.cfg', {'dir': 'etc/nrpe.d'}] + # Keep user multipath configuration + self.restore_list += [{'dir': 'etc/multipath/conf.d', 're': r'custom.*\.conf'}] + completeUpgradeArgs = ['mounts', 'installation-to-overwrite', 'primary-disk', 'backup-partnum', 'logs-partnum', 'net-admin-interface', 'net-admin-bridge', 'net-admin-configuration'] def completeUpgrade(self, mounts, prev_install, target_disk, backup_partnum, logs_partnum, admin_iface, admin_bridge, admin_config): From 4f7da4283d545464e5ea6b305f0cad84545999ca Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 19 Oct 2023 15:18:16 +0200 Subject: [PATCH 36/38] answerfile: change installation's sr-type default to "ext" This complements the default choice previously changed in the TUI. Signed-off-by: Yann Dirson --- answerfile.py | 2 +- doc/answerfile.txt | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/answerfile.py b/answerfile.py index 490b49ec..9c9c61c1 100644 --- a/answerfile.py +++ b/answerfile.py @@ -324,7 +324,7 @@ def parseDisks(self): results['sr-type'] = getMapAttribute(self.top_node, ['sr-type', 'srtype'], [('lvm', SR_TYPE_LVM), - ('ext', SR_TYPE_EXT)], default='lvm') + ('ext', SR_TYPE_EXT)], default='ext') return results def parseFCoEInterface(self): diff --git a/doc/answerfile.txt b/doc/answerfile.txt index 1e1d41d4..cd01dbeb 100644 --- a/doc/answerfile.txt +++ b/doc/answerfile.txt @@ -285,7 +285,11 @@ Format of 'source' and 'driver-source' Local SR type. - Default: lvm + Default: ext + + Note that while it reflects the default choice in the text UI, + this differs from XenServer and from XCP-ng releases lower than + 8.3, where the default was "lvm". Upgrade Elements From 0e4b38d320f7b74038649aa904c8291c9076c2fe Mon Sep 17 00:00:00 2001 From: Alex Brett Date: Tue, 24 Oct 2023 12:40:38 +0000 Subject: [PATCH 37/38] XSI-1498: Log any exceptions parsing existing bootloader config Previously any exceptions which occurred were silently suppressed, instead log such occurrences to enable easier debugging from the installer log. Signed-off-by: Alex Brett --- product.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/product.py b/product.py index 5f42f38c..5514c529 100644 --- a/product.py +++ b/product.py @@ -393,8 +393,9 @@ def fetchIfaceInfoFromNetworkdbAsDict(bridge, iface=None): pciback = next((x for x in kernel_args if x.startswith('xen-pciback.hide=')), None) if pciback: results['host-config']['xen-pciback.hide'] = pciback - except: - pass + except Exception as e: + logger.log('Exception whilst parsing existing bootloader config:') + logger.logException(e) self.unmount_boot() return results From 09f0562e3456bc7065908c38c9716612731f22bd Mon Sep 17 00:00:00 2001 From: Alex Brett Date: Thu, 23 Nov 2023 09:55:16 +0000 Subject: [PATCH 38/38] CA-385350 Fix handling of NTP configuration over upgrade Previously we were not setting `ntp-config-method` when parsing the NTP configuration from an old version, resulting in any NTP servers that were read being ignored. Set the appropriate `ntp-config-method`, and also filter out any 'default' NTP servers from the `ntp-servers` list so we only load customer specified servers. Note this is done using a fixed list of default domains, as this has changed in the past and may change in future, so cannot be read from the current release config. Signed-off-by: Alex Brett --- constants.py | 3 +++ product.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/constants.py b/constants.py index 525f3616..449beb1f 100644 --- a/constants.py +++ b/constants.py @@ -154,6 +154,9 @@ def error_string(error, logname, with_hd): HYPERVISOR_CAPS_FILE = "/sys/hypervisor/properties/capabilities" SAFE_2_UPGRADE = "var/preserve/safe2upgrade" +# NTP server domains to treat as 'default' servers +DEFAULT_NTP_DOMAINS = [".centos.pool.ntp.org", ".xenserver.pool.ntp.org"] + # timer to exit installer after fatal error AUTO_EXIT_TIMER = 10 * 1000 diff --git a/product.py b/product.py index 5514c529..c07e12ee 100644 --- a/product.py +++ b/product.py @@ -19,6 +19,7 @@ from xcp import logger import xml.dom.minidom import simplejson as json +import glob class SettingsNotAvailable(Exception): pass @@ -168,8 +169,21 @@ def _readSettings(self): ntps = [] for line in lines: if line.startswith("server "): - ntps.append(line[7:].split()[0].strip()) + s = line[7:].split()[0].strip() + if any(s.endswith(d) for d in constants.DEFAULT_NTP_DOMAINS): + continue + ntps.append(s) results['ntp-servers'] = ntps + # ntp-config-method should be set as follows: + # 'dhcp' if dhcp was in use, regardless of server configuration + # 'manual' if we had existing NTP servers defined (other than default servers) + # 'default' if no NTP servers are defined + if self._check_dhcp_ntp_status(): + results['ntp-config-method'] = 'dhcp' + elif ntps: + results['ntp-config-method'] = 'manual' + else: + results['ntp-config-method'] = 'default' # keyboard: keyboard_dict = {} @@ -216,9 +230,6 @@ def _readSettings(self): raise SettingsNotAvailable("no root password found") results['root-password'] = ('pwdhash', root_pwd) - # don't care about this too much. - results['ntp-config-method'] = 'default' - # read network configuration. We only care to find out what the # management interface is, and what its configuration was. # The dev -> MAC mapping for other devices will be preserved in the @@ -400,6 +411,19 @@ def fetchIfaceInfoFromNetworkdbAsDict(bridge, iface=None): return results + def _check_dhcp_ntp_status(self): + """Validate if DHCP was in use and had provided any NTP servers""" + if os.path.exists(self.join_state_path('etc/dhcp/dhclient.d/chrony.sh')) and \ + not (os.stat(self.join_state_path('etc/dhcp/dhclient.d/chrony.sh')).st_mode & stat.S_IXUSR): + # chrony.sh not executable indicates not using DHCP for NTP + return False + + for f in glob.glob(self.join_state_path('var/lib/dhclient/chrony.servers.*')): + if os.path.getsize(f) > 0: + return True + + return False + def mount_boot(self, ro=True): opts = None if ro: