From cb4d20d9486e24f1bc25a8837d5c73ee1213708a Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Fri, 24 Nov 2023 20:47:51 +0800 Subject: [PATCH 01/13] improve KubeVirtNodeDriver.create_node & bugs fix --- libcloud/compute/drivers/kubevirt.py | 826 +++++++++++++++++++++------ 1 file changed, 642 insertions(+), 184 deletions(-) diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index dd475a62a9..85ec611345 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -18,9 +18,13 @@ """ kubevirt driver with support for nodes (vms) """ +import copy import json import time +import uuid import hashlib +import warnings +from typing import Union, Optional from datetime import datetime from libcloud.common.types import LibcloudError @@ -31,6 +35,8 @@ NodeDriver, NodeLocation, StorageVolume, + NodeAuthSSHKey, + NodeAuthPassword, ) from libcloud.compute.types import Provider, NodeState from libcloud.common.kubernetes import ( @@ -39,10 +45,29 @@ KubernetesBasicAuthConnection, ) -__all__ = ["KubeVirtNodeDriver"] +__all__ = [ + "KubeVirtNodeDriver", + "DISK_TYPES", + "KubeVirtNodeSize", + "KubeVirtNodeImage", +] + ROOT_URL = "/api/v1/" KUBEVIRT_URL = "/apis/kubevirt.io/v1alpha3/" +# all valid disk types supported by kubevirt +DISK_TYPES = { + "containerDisk", + "ephemeral", + "configMap", + "dataVolume", + "cloudInitNoCloud", + "persistentVolumeClaim", + "emptyDisk", + "cloudInitConfigDrive", + "hostDisk", +} + class KubeVirtNodeDriver(KubernetesDriverMixin, NodeDriver): type = Provider.KUBEVIRT @@ -179,25 +204,147 @@ def destroy_node(self, node): except Exception: raise + def _create_node_with_template(self, name: str, template: dict, namespace="default"): + """ + Creating a VM defined by the template. + + :param name: A name to give the VM. The VM will be identified by this + name, and it must be the same as the name in the + ``template["metadata"]["name"]``. + Atm, it cannot be changed after it is set. + :type name: ``str`` + + :param template: A dictionary of kubernetes object that defines the + KubeVirt VM. + See also: + - https://kubernetes.io/docs/concepts/overview/working-with-objects/ + - https://kubevirt.io/api-reference/ + :type template: ``dict`` with keys: + -apiVersion: ``str`` + -kind: ``str`` + -metadata: ``dict`` + -spec: ``dict`` + - domain: ``dict`` + - volumes: ``list`` + - ... + + + :param namespace: The namespace where the VM will live. + (default is 'default') + :return: + """ + # k8s object checks + if template.get("apiVersion", "") != "kubevirt.io/v1alpha3": + raise ValueError("The template must have an apiVersion: kubevirt.io/v1alpha3") + if template.get("kind", "") != "VirtualMachine": + raise ValueError("The template must contain kind: VirtualMachine") + if name != template.get("metadata", {}).get("name"): + raise ValueError("The name of the VM must be the same as the name in the template. " + "(name={}, template.metadata.name={})".format( + name, template.get("metadata", {}).get("name"))) + if template.get("spec", {}).get("running", False): + warnings.warn( + "The VM will be created in a stopped state, and then started. " + "Ignoring the `spec.running: True` in the template." + ) + # assert "spec" in template and "running" in template["spec"] + template["spec"]["running"] = False + + vm = template + + method = "POST" + data = json.dumps(vm) + req = KUBEVIRT_URL + "namespaces/" + namespace + "/virtualmachines/" + try: + self.connection.request(req, method=method, data=data) + + except Exception: + raise + # check if new node is present + nodes = self.list_nodes() + for node in nodes: + if node.name == name: + self.start_node(node) + return node + + @staticmethod + def _base_vm_template(name=None): # type: (Optional[str]) -> dict + """ + A skeleton VM template to be used for creating VMs. + + :param name: A name to give the VM. The VM will be identified by this + name. If not provided, a random uuid4 will be used. + The generated name can be gotten from the returned dict + with the key ``["metadata"]["name"]``. + :type name: ``str`` + + :return: dict: A skeleton VM template. + """ + if not name: + name = uuid.uuid4() + return { + "apiVersion": "kubevirt.io/v1alpha3", + "kind": "VirtualMachine", + "metadata": {"labels": {"kubevirt.io/vm": name}, "name": name}, + "spec": { + "running": False, + "template": { + "metadata": {"labels": {"kubevirt.io/vm": name}}, + "spec": { + "domain": { + "devices": { + "disks": [], + "interfaces": [], + "networkInterfaceMultiqueue": False, + }, + "machine": {"type": ""}, + "resources": {"requests": {}, "limits": {}}, + }, + "networks": [], + "terminationGracePeriodSeconds": 0, + "volumes": [], + }, + }, + }, + } + # only has container disk support atm with no persistency def create_node( self, - name, - image, - location=None, - ex_memory=128, - ex_cpu=1, - ex_disks=None, - ex_network=None, - ex_termination_grace_period=0, - ports=None, - ): + name, # type: str + size, # type: Optional[NodeSize] + image, # type: Optional[Union[NodeImage, str]] + location=None, # type: Optional[NodeLocation] + auth=None, # type: Optional[Union[NodeAuthSSHKey, NodeAuthPassword]] + ex_disks=None, # type: Optional[list] + ex_network=None, # type: Optional[dict] + ex_termination_grace_period=0, # type: Optional[int] + ex_ports=None, # type: Optional[dict] + ex_template=None, # type: Optional[dict] + ): # type: (...) -> Node """ Creating a VM with a containerDisk. + :param name: A name to give the VM. The VM will be identified by this name and atm it cannot be changed after it is set. :type name: ``str`` + :param size: The size of the VM in terms of CPU and memory. + :type size: ``NodeSize``: + + >>> size = NodeSize( + >>> # id='small', # str: Size ID + >>> # name='small', # str: Size name. Not used atm + >>> ram=2048, # int: Amount of memory (in MB) + >>> # disk=20, # int: Amount of disk storage (in GB). Not used atm + >>> # bandwidth=0, # int: Amount of bandwidth. Not used atm + >>> # price=0, # float: Price (in US dollars) of running this node for an hour. Not used atm + >>> # driver=KubeVirtNodeDriver # NodeDriver: Driver this size belongs to + >>> extra={ + >>> 'cpu': 1, # int: Number of CPUs provided by this size. + >>> }, + >>> ) + :param image: Either a libcloud NodeImage or a string. In both cases it must point to a Docker image with an embedded disk. @@ -206,179 +353,253 @@ def create_node( https://hub.docker.com/u/URI. For more info visit: https://kubevirt.io/user-guide/docs/latest/creating-virtual-machines/disks-and-volumes.html#containerdisk - :type image: `str` + :type image: ``str`` or ``NodeImage`` + + >>> image = NodeImage( + >>> name='kubevirt/cirros-registry-disk-demo', # str: Image URI + >>> ... # Other attributes are ignored + >>> ) :param location: The namespace where the VM will live. (default is 'default') - :type location: ``str`` + :type location: `NodeLocation`` in which the name is the namespace - :param ex_memory: The RAM in MB to be allocated to the VM - :type ex_memory: ``int`` + >>> location = NodeLocation( + >>> name='default', # str: namespace + >>> ... # Other attributes are ignored + >>> ) - :param ex_cpu: The amount of cpu to be allocated in miliCPUs - ie: 400 will mean 0.4 of a core, 1000 will mean 1 core - and 3000 will mean 3 cores. - :type ex_cpu: ``int`` + :param auth: authentication to a node. + :type auth: ``NodeAuthSSHKey`` or ``NodeAuthPassword``: + - NodeAuthSSHKey(pubkey='...') + - NodeAuthPassword(password='...') :param ex_disks: A list containing disk dictionaries. - Each dictionaries should have the - following optional keys: - -bus: can be "virtio", "sata", or "scsi" - -device: can be "lun" or "disk" + Each dictionary should have the + following optional keys: + + - ``"bus"``: can be "virtio", "sata", or "scsi" + - ``"device"``: can be "lun" or "disk" + The following are required keys: - -disk_type: atm only "persistentVolumeClaim" - is supported - -name: The name of the disk configuration - -claim_name: the name of the - Persistent Volume Claim - - If you wish a new Persistent Volume Claim can be - created by providing the following: - required: - -size: the desired size (implied in GB) - -storage_class_name: the name of the storage class to # NOQA - be used for the creation of the - Persistent Volume Claim. - Make sure it allows for - dymamic provisioning. - optional: - -access_mode: default is ReadWriteOnce - -volume_mode: default is `Filesystem`, - it can also be `Block` - - :type ex_disks: `list` of `dict`. For each `dict` the types - for its keys are: - -bus: `str` - -device: `str` - -disk_type: `str` - -name: `str` - -claim_name: `str` - (for creating a claim:) - -size: `int` - -storage_class_name: `str` - -volume_mode: `str` - -access_mode: `str` - - :param ex_network: Only the pod type is supported, and in the - configuration masquerade or bridge are the + + - ``"disk_type"`` + One of the supported ``DISK_TYPES``. + Atm only "persistentVolumeClaim" and + "containerDisk" are promised to work. + Other types may work but are not tested. + - ``"name"``: + The name of the disk configuration + - ``"voulme_spec"``: + the dictionary that defines the volume + of the disk. + Will be translated to KubeVirt API object: + + volumes: + - name: + : + + + Content is depending on the disk_type: + + For "containerDisk" the dictionary + should have a key "image" with the + value being the image URI. + + For "persistentVolumeClaim" the + dictionary should have a key + "claimName" with the value being + the name of the claim. + If you wish, a new Persistent Volume Claim can be + created by providing the following: + + - required: + - size: the desired size (implied in GB) + - storage_class_name: the name of the storage class to # NOQA + be used for the creation of the + Persistent Volume Claim. + Make sure it allows for + dymamic provisioning. + - optional: + - access_mode: default is ReadWriteOnce + - volume_mode: default is `Filesystem`, + it can also be `Block` + + For other disk types, it will be translated to + a KubeVirt volume object as is: + + {disk_type: , "name": disk_name} + + Please refer to the KubeVirt API documentation + for volume specifications: + + https://kubevirt.io/user-guide/virtual_machines/disks_and_volumes + + :type ex_disks: `list` of `dict`. + For each `dict` the types for its keys are: + - bus: `str` + - device: `str` + - disk_type: `str` + - name: `str` + - voulme_spec: `dict` + + :param ex_network: Only the `pod` type is supported, and in the + configuration `masquerade` or `bridge` are the accepted values. - The parameter must be a tuple or list with - (network_type, interface, name) - :type ex_network: `iterable` (tuple or list) [network_type, interface, name] - network_type: `str` | only "pod" is accepted atm - interface: `str` | "masquerade" or "bridge" - name: `str` - - :param ports: A dictionary with keys: 'ports_tcp' and 'ports_udp' + The parameter must be a dict: + + { + "network_type": "pod", + "interface": "masquerade | bridge", + "name": "network_name" + } + + :type ex_network: `dict` with keys: + + - network_type: `str` | only "pod" is accepted atm + - interface: `str` | "masquerade" or "bridge" + - name: `str` + + :param ex_termination_grace_period: The grace period in seconds before + the VM is forcefully terminated. + (default is 0) + :type ex_termination_grace_period: `int` + + :param ex_ports: A dictionary with keys: 'ports_tcp' and 'ports_udp' 'ports_tcp' value is a list of ints that indicate the ports to be exposed with TCP protocol, and 'ports_udp' is a list of ints that indicate the ports to be exposed with UDP protocol. - :type ports: `dict` with keys + :type ex_ports: `dict` with keys 'ports_tcp`: `list` of `int` 'ports_udp`: `list` of `int` + + :param ex_template: A dictionary of kubernetes object that defines the + KubeVirt VM. This is for advanced vm specifications + that are not covered by the other parameters. + If provided, it will override all other parameters + except for `name` and `location`. + See also: + - https://kubernetes.io/docs/concepts/overview/working-with-objects/ + - https://kubevirt.io/api-reference/ + :type ex_template: ``dict`` with keys: + -apiVersion: ``str`` + -kind: ``str`` + -metadata: ``dict`` + -spec: ``dict`` + - domain: ``dict`` + - volumes: ``list`` + - ... """ - # all valid disk types for which support will be added in the future - DISK_TYPES = { - "containerDisk", - "ephemeral", - "configMap", - "dataVolume", - "cloudInitNoCloud", - "persistentVolumeClaim", - "emptyDisk", - "cloudInitConfigDrive", - "hostDisk", - } - if location is not None: + # location -> namespace + if isinstance(location, NodeLocation): + if location not in self.list_locations(): + raise ValueError("The location must be one of the available namespaces") namespace = location.name else: namespace = "default" + # ex_template or _base_vm_template -> vm + + # ex_template exists, use it to create the vm, ignore other parameters + if ex_template is not None: + assert isinstance(ex_template, dict), "ex_template must be a dictionary" + + other_params = { + "size": size, + "image": image, + "auth": auth, + "ex_disks": ex_disks, + "ex_network": ex_network, + "ex_termination_grace_period": ex_termination_grace_period, + "ex_ports": ex_ports, + } + ignored_non_none_param_keys = list(filter( + lambda x: other_params[x] is not None, + other_params)) + if ignored_non_none_param_keys: + warnings.warn( + "ex_template is provided, ignoring the following non-None " + "parameters: {}" + .format(ignored_non_none_param_keys) + ) + + vm = copy.deepcopy(ex_template) + + if vm.get("metadata") is None: + vm["metadata"] = {} + + if vm["metadata"].get("name") is None: + vm["metadata"]["name"] = name + elif vm["metadata"]["name"] != name: + warnings.warn( + "The name in the ex_template ({}) will be ignored. " + "The name provided in the arguments ({}) will be used." + .format(vm["metadata"]["name"], name) + ) + vm["metadata"]["name"] = name + + return self._create_node_with_template(name=name, + template=vm, + namespace=namespace) + # else (ex_template is None): create a vm with other parameters + # vm template to be populated - vm = { - "apiVersion": "kubevirt.io/v1alpha3", - "kind": "VirtualMachine", - "metadata": {"labels": {"kubevirt.io/vm": name}, "name": name}, - "spec": { - "running": False, - "template": { - "metadata": {"labels": {"kubevirt.io/vm": name}}, - "spec": { - "domain": { - "devices": { - "disks": [], - "interfaces": [], - "networkInterfaceMultiqueue": False, - }, - "machine": {"type": ""}, - "resources": {"requests": {}, "limits": {}}, - }, - "networks": [], - "terminationGracePeriodSeconds": ex_termination_grace_period, # NOQA - "volumes": [], - }, - }, - }, - } - memory = str(ex_memory) + "Mi" - vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] = memory - vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["memory"] = memory - if ex_cpu < 10: - cpu = int(ex_cpu) - vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] = cpu - vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"] = cpu + vm = self._base_vm_template(name=name) + + # size -> cpu and memory limits + + if size is not None: + assert isinstance(size, NodeSize), "size must be a NodeSize" + ex_cpu = size.extra["cpu"] + ex_memory = size.ram else: - cpu = str(ex_cpu) + "m" + ex_cpu = None + ex_memory = None + + if ex_memory is not None: + assert isinstance(ex_memory, int), "ex_memory must be an int in MiB" + memory = str(ex_memory) + "Mi" + vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] = memory + vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["memory"] = memory + + if ex_cpu is not None: + if isinstance(ex_cpu, str) and ex_cpu.endswith("m"): + cpu = ex_cpu + else: + try: + cpu = float(ex_cpu) + except ValueError: + raise ValueError("ex_cpu must be a number or a string ending with 'm'") vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] = cpu vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"] = cpu - i = 0 - for disk in ex_disks: + + # TODO: debug: no boot disk??? + + # ex_disks -> disks and volumes + ex_disks = ex_disks or [] + for i, disk in enumerate(ex_disks): disk_type = disk.get("disk_type") bus = disk.get("bus", "virtio") disk_name = disk.get("name", "disk{}".format(i)) - i += 1 device = disk.get("device", "disk") + if disk_type not in DISK_TYPES: raise ValueError("The possible values for this " "parameter are: ", DISK_TYPES) + # depending on disk_type, in the future, # when more will be supported, # additional elif should be added if disk_type == "containerDisk": try: - image = disk["image"] + image = disk["volume_spec"]["image"] except KeyError: raise KeyError("A container disk needs a " "containerized image") volumes_dict = {"containerDisk": {"image": image}, "name": disk_name} - - if disk_type == "persistentVolumeClaim": - if "claim_name" in disk: - claimName = disk["claim_name"] - if claimName not in self.ex_list_persistent_volume_claims(namespace=namespace): - if "size" not in disk or "storage_class_name" not in disk: - msg = ( - "disk['size'] and " - "disk['storage_class_name'] " - "are both required to create " - "a new claim." - ) - raise KeyError(msg) - size = disk["size"] - storage_class = disk["storage_class_name"] - volume_mode = disk.get("volume_mode", "Filesystem") - access_mode = disk.get("access_mode", "ReadWriteOnce") - self.create_volume( - size=size, - name=claimName, - location=location, - ex_storage_class_name=storage_class, - ex_volume_mode=volume_mode, - ex_access_mode=access_mode, - ) - - else: + elif disk_type == "persistentVolumeClaim": + if "claim_name" not in disk: msg = ( "You must provide either a claim_name of an " "existing claim or if you want one to be " @@ -391,63 +612,151 @@ def create_node( ) raise KeyError(msg) + claim_name = disk["claim_name"] + + if claim_name not in self.ex_list_persistent_volume_claims(namespace=namespace): + if "size" not in disk or "storage_class_name" not in disk: + msg = ( + "disk['size'] and " + "disk['storage_class_name'] " + "are both required to create " + "a new claim." + ) + raise KeyError(msg) + size = disk["size"] + storage_class = disk["storage_class_name"] + volume_mode = disk.get("volume_mode", "Filesystem") + access_mode = disk.get("access_mode", "ReadWriteOnce") + self.create_volume( + size=size, + name=claim_name, + location=location, + ex_storage_class_name=storage_class, + ex_volume_mode=volume_mode, + ex_access_mode=access_mode, + ) + volumes_dict = { - "persistentVolumeClaim": {"claimName": claimName}, + "persistentVolumeClaim": {"claimName": claim_name}, "name": disk_name, } + else: + warnings.warn( + "The disk type {} is not tested. Use at your own risk.".format(disk_type) + ) + volumes_dict = {disk_type: disk["volume_spec"], "name": disk_name} + disk_dict = {device: {"bus": bus}, "name": disk_name} vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) vm["spec"]["template"]["spec"]["volumes"].append(volumes_dict) + # end of for disk in ex_disks + # image -> containerDisk # adding image in a container Disk if isinstance(image, NodeImage): image = image.name - volumes_dict = {"containerDisk": {"image": image}, "name": "boot-disk"} - disk_dict = {"disk": {"bus": "virtio"}, "name": "boot-disk"} + boot_disk_name = "boot-disk-" + str(uuid.uuid4()) + volumes_dict = {"containerDisk": {"image": image}, "name": boot_disk_name} + disk_dict = {"disk": {"bus": "virtio"}, "name": boot_disk_name} vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) vm["spec"]["template"]["spec"]["volumes"].append(volumes_dict) - # network - if ex_network: - interface = ex_network[1] - network_name = ex_network[2] - network_type = ex_network[0] + # auth -> cloud-init + if auth is not None: + # auth requires cloud-init, + # and only one cloud-init volume is supported by kubevirt. + # So if both auth and cloud-init are provided, raise an error. + + for volume in vm["spec"]["template"]["spec"]["volumes"]: + if "cloudInitNoCloud" in volume or "cloudInitConfigDrive" in volume: + raise ValueError( + "Setting auth and cloudInit at the same time is not supported." + "Use deploy_node() instead." + ) + + # cloud-init volume + cloud_init_volume = "auth-cloudinit-" + str(uuid.uuid4()) + disk_dict = {"disk": {"bus": "virtio"}, "name": cloud_init_volume} + volume_dict = { + "name": cloud_init_volume, + "cloudInitNoCloud": { + "userData": "" + }, + } + + # auth + # cloud_init_config reference: https://kubevirt.io/user-guide/virtual_machines/startup_scripts/#injecting-ssh-keys-with-cloud-inits-cloud-config + if isinstance(auth, NodeAuthSSHKey): + public_key = auth.pubkey + cloud_init_config = ("""#cloud-config\n""" + """ssh_authorized_keys:\n""" + """ - {}\n""").format(public_key) + elif isinstance(auth, NodeAuthPassword): + password = auth.password + cloud_init_config = ("""#cloud-config\n""" + """password: {}\n""" + """chpasswd: {{ expire: False }}\n""" + """ssh_pwauth: True\n""").format(password) + else: + raise ValueError("auth must be NodeAuthSSHKey or NodeAuthPassword") + + volume_dict["cloudInitNoCloud"]["userData"] = cloud_init_config + + # add volume + vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) + vm["spec"]["template"]["spec"]["volumes"].append(volume_dict) + + # now, all disks and volumes stuff are done + + # ex_network -> network and interface + + if ex_network is not None: + try: + interface = ex_network["interface"] + network_name = ex_network["name"] + network_type = ex_network["network_type"] + except KeyError: + msg = ( + "ex_network: You must provide a dictionary with keys: " + "'interface', 'name', 'network_type'." + ) + raise KeyError(msg) # add a default network else: interface = "masquerade" network_name = "netw1" network_type = "pod" + network_dict = {network_type: {}, "name": network_name} interface_dict = {interface: {}, "name": network_name} - ports = ports or {} - if ports.get("ports_tcp"): + + # ex_ports -> network.ports + ex_ports = ex_ports or {} + if ex_ports.get("ports_tcp"): ports_to_expose = [] - for port in ports["ports_tcp"]: + for port in ex_ports["ports_tcp"]: ports_to_expose.append({"port": port, "protocol": "TCP"}) interface_dict[interface]["ports"] = ports_to_expose - if ports.get("ports_udp"): + if ex_ports.get("ports_udp"): ports_to_expose = interface_dict[interface].get("ports", []) - for port in ports.get("ports_udp"): + for port in ex_ports.get("ports_udp"): ports_to_expose.append({"port": port, "protocol": "UDP"}) interface_dict[interface]["ports"] = ports_to_expose + vm["spec"]["template"]["spec"]["networks"].append(network_dict) vm["spec"]["template"]["spec"]["domain"]["devices"]["interfaces"].append(interface_dict) - method = "POST" - data = json.dumps(vm) - req = KUBEVIRT_URL + "namespaces/" + namespace + "/virtualmachines/" - try: - self.connection.request(req, method=method, data=data) + # terminationGracePeriodSeconds + if ex_termination_grace_period is not None: + assert isinstance(ex_termination_grace_period, int), ( + "ex_termination_grace_period must be an int" + ) + vm["spec"]["template"]["spec"]["terminationGracePeriodSeconds"] = ( + ex_termination_grace_period + ) - except Exception: - raise - # check if new node is present - nodes = self.list_nodes() - for node in nodes: - if node.name == name: - self.start_node(node) - return node + return self._create_node_with_template(name=name, template=vm, namespace=namespace) def list_images(self, location=None): """ @@ -923,13 +1232,25 @@ def _ex_connection_class_kwargs(self): kwargs["cert_file"] = self.cert_file return kwargs - def _to_node(self, vm, is_stopped=False): - """ """ + def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node + """ + converts a vm from the kubevirt API to a libcloud node + + :param vm: kubevirt vm object + :type vm: dict + + :param is_stopped: if the vm is stopped, it will not have a pod + :type is_stopped: bool + + :return: a libcloud node + :rtype: Node + """ ID = vm["metadata"]["uid"] name = vm["metadata"]["name"] driver = self.connection.driver extra = {"namespace": vm["metadata"]["namespace"]} extra["pvcs"] = [] + memory = 0 if "limits" in vm["spec"]["template"]["spec"]["domain"]["resources"]: if "memory" in vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]: @@ -939,17 +1260,8 @@ def _to_node(self, vm, is_stopped=False): "memory", None ): memory = vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] - if not isinstance(memory, int): - if "M" in memory or "Mi" in memory: - memory = memory.rstrip("M") - memory = memory.rstrip("Mi") - memory = int(memory) - elif "G" in memory: - memory = memory.rstrip("G") - memory = int(memory) // 1000 - elif "Gi" in memory: - memory = memory.rstrip("Gi") - memory = int(memory) // 1024 + memory = _memory_in_MB(memory) + cpu = 1 if vm["spec"]["template"]["spec"]["domain"]["resources"].get("limits", None): if vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"].get("cpu", None): @@ -962,7 +1274,8 @@ def _to_node(self, vm, is_stopped=False): cpu = vm["spec"]["template"]["spec"]["domain"]["cpu"].get("cores", 1) if not isinstance(cpu, int): cpu = int(cpu.rstrip("m")) - extra_size = {"cpus": cpu} + + extra_size = {"cpu": cpu} size_name = "{} vCPUs, {}MB Ram".format(str(cpu), str(memory)) size_id = hashlib.md5(size_name.encode("utf-8")).hexdigest() size = NodeSize( @@ -975,8 +1288,10 @@ def _to_node(self, vm, is_stopped=False): driver=driver, extra=extra_size, ) + extra["memory"] = memory extra["cpu"] = cpu + image_name = "undefined" for volume in vm["spec"]["template"]["spec"]["volumes"]: for k, v in volume.items(): @@ -988,6 +1303,7 @@ def _to_node(self, vm, is_stopped=False): for volume in vm["spec"]["template"]["spec"]["volumes"]: if "persistentVolumeClaim" in volume: extra["pvcs"].append(volume["persistentVolumeClaim"]["claimName"]) + port_forwards = [] services = self.ex_list_services(namespace=extra["namespace"], node_name=name) for service in services: @@ -1009,6 +1325,7 @@ def _to_node(self, vm, is_stopped=False): } ) extra["port_forwards"] = port_forwards + if is_stopped: state = NodeState.STOPPED public_ips = None @@ -1231,3 +1548,144 @@ def ex_delete_service(self, namespace, service_name): except Exception: raise return result.status in VALID_RESPONSE_CODES + + +def _deep_merge_dict(source: dict, destination: dict) -> dict: + """ + Deep merge two dictionaries: source into destination. + For conflicts, prefer source's non-zero values over destination's. + (By non-zero, we mean that bool(value) is True.) + + Extended from https://stackoverflow.com/a/20666342, added zero value handling. + + >>> a = {"domain": {"devices": 0}, "volumes": [1, 2, 3], "network": {}} + >>> b = {"domain": {"machine": "non-exist-in-a", "devices": 1024}, "volumes": [4, 5, 6]} + >>> _deep_merge_dict(a, b) + {'domain': {'machine': 'non-exist-in-a', 'devices': 1024}, 'volumes': [1, 2, 3], 'network': {}} + + In the above example: + + - network: exists in source (a) but not in destination (b): add source (a)'s + - volumes: exists in both, both are non-zero: prefer source (a)'s + - devices: exists in both: source (a) is zero, destination (b) is non-zero: keep destination (b)'s + - machine: exists in destination (b) but not in source (a): reserve destination (b)'s + + :param source: RO: A dict to be merged into another. + Do not use circular dict (e.g. d = {}; d['d'] = d) as source, + otherwise a RecursionError will be raised. + :param destination: RW: A dict to be merged into. (the value will be modified). + + :return: dict: Updated destination. + """ + for key, value in source.items(): + if isinstance(value, dict): # recurse for dicts + node = destination.setdefault(key, {}) # get node or create one + _deep_merge_dict(value, node) + elif key not in destination: # not existing in destination: add it + destination[key] = value + elif value: # existing: update if source's value is non-zero + destination[key] = value + + return destination + + +def _memory_in_MB(memory): # type: (Union[str, int]) -> int + """ + parse k8s memory resource units to MiB or MB (depending on input) + + Note: + + - 1 MiB = 1024 KiB = 1024 * 1024 B = 1048576 B + - 1 MB = 1000 KB = 1000 * 1000 B = 1000000 B + + Reference: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory + + :param memory: Limits and requests for memory are measured in bytes. + You can express memory as a plain integer or as a fixed-point integer using one of these suffixes: + E, P, T, G, M, K, Ei, Pi, Ti, Gi, Mi, Ki + For example, the following represent roughly the same value: + 128974848, 129e6, 129M, 123Mi + :type memory: ``str`` or ``int`` + + :return: memory in MiB (if input is `int` bytes or `str` with suffix `?i`) + or in MB (if input is `str` with suffix `?`) + :rtype: ``int`` + """ + + try: + mem_bytes = int(memory) + return mem_bytes // 1024 // 1024 + except ValueError: + pass + + if not isinstance(memory, str): + raise ValueError("memory must be int or str") + + if memory.endswith("Ei"): + return int(memory.rstrip("Ei")) * 1024 * 1024 * 1024 * 1024 + elif memory.endswith("Pi"): + return int(memory.rstrip("Pi")) * 1024 * 1024 * 1024 + elif memory.endswith("Ti"): + return int(memory.rstrip("Ti")) * 1024 * 1024 + elif memory.endswith("Gi"): + return int(memory.rstrip("Gi")) * 1024 + elif memory.endswith("Mi"): + return int(memory.rstrip("Mi")) + elif memory.endswith("Ki"): + return int(memory.rstrip("Ki")) // 1024 + elif memory.endswith("E"): + return int(memory.rstrip("E")) * 1000 * 1000 * 1000 * 1000 + elif memory.endswith("P"): + return int(memory.rstrip("P")) * 1000 * 1000 * 1000 + elif memory.endswith("T"): + return int(memory.rstrip("T")) * 1000 * 1000 + elif memory.endswith("G"): + return int(memory.rstrip("G")) * 1000 + elif memory.endswith("M"): + return int(memory.rstrip("M")) + elif memory.endswith("K"): + return int(memory.rstrip("K")) // 1000 + else: + raise ValueError("memory unit not supported {}".format(memory)) + + +def KubeVirtNodeSize(cpu, ram): # type: (int, int) -> NodeSize + """ + Create a NodeSize object for KubeVirt driver. + + :param cpu: number of virtual CPUs + :type cpu: ``int`` + + :param ram: amount of RAM in MiB + :type ram: ``int`` + + :return: a NodeSize object + :rtype: :class:`NodeSize` + """ + extra = {"cpu": cpu} + name = "{} vCPUs, {}MB Ram".format(str(cpu), str(ram)) + size_id = hashlib.md5(name.encode("utf-8")).hexdigest() + return NodeSize( + id=size_id, + name=name, + ram=ram, + disk=0, + bandwidth=0, + price=0, + driver=KubeVirtNodeDriver, + extra=extra, + ) + + +def KubeVirtNodeImage(name): # type: (str) -> NodeImage + """ + Create a NodeImage object for KubeVirt driver. + + :param name: image source + :type name: ``str`` + + :return: a NodeImage object + :rtype: :class:`NodeImage` + """ + image_id = hashlib.md5(name.encode("utf-8")).hexdigest() + return NodeImage(id=image_id, name=name, driver=KubeVirtNodeDriver) From 1fa681f44a391c69177692caa83b2cb944fc0233 Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Fri, 24 Nov 2023 21:07:47 +0800 Subject: [PATCH 02/13] fix(KubeVirtNodeDriver): persistentVolumeClaim: disk -> disk['volume_spec'] --- libcloud/compute/drivers/kubevirt.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index 85ec611345..2c7d48aba6 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -599,7 +599,11 @@ def create_node( volumes_dict = {"containerDisk": {"image": image}, "name": disk_name} elif disk_type == "persistentVolumeClaim": - if "claim_name" not in disk: + if "volume_spec" not in disk: + raise KeyError( + "You must provide a volume_spec dictionary" + ) + if "claim_name" not in disk["volume_spec"]: msg = ( "You must provide either a claim_name of an " "existing claim or if you want one to be " @@ -612,21 +616,21 @@ def create_node( ) raise KeyError(msg) - claim_name = disk["claim_name"] + claim_name = disk["volume_spec"]["claim_name"] if claim_name not in self.ex_list_persistent_volume_claims(namespace=namespace): - if "size" not in disk or "storage_class_name" not in disk: + if "size" not in disk["volume_spec"] or "storage_class_name" not in disk["volume_spec"]: msg = ( - "disk['size'] and " - "disk['storage_class_name'] " + "disk['volume_spec']['size'] and " + "disk['volume_spec']['storage_class_name'] " "are both required to create " "a new claim." ) raise KeyError(msg) - size = disk["size"] - storage_class = disk["storage_class_name"] - volume_mode = disk.get("volume_mode", "Filesystem") - access_mode = disk.get("access_mode", "ReadWriteOnce") + size = disk["volume_spec"]["size"] + storage_class = disk["volume_spec"]["storage_class_name"] + volume_mode = disk["volume_spec"].get("volume_mode", "Filesystem") + access_mode = disk["volume_spec"].get("access_mode", "ReadWriteOnce") self.create_volume( size=size, name=claim_name, @@ -644,7 +648,7 @@ def create_node( warnings.warn( "The disk type {} is not tested. Use at your own risk.".format(disk_type) ) - volumes_dict = {disk_type: disk["volume_spec"], "name": disk_name} + volumes_dict = {disk_type: disk.get("volume_spec", {}), "name": disk_name} disk_dict = {device: {"bus": bus}, "name": disk_name} vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) From 13a16a68467a59fee6acad6bc299c5a4bd01eea7 Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Tue, 2 Jan 2024 11:09:21 +0800 Subject: [PATCH 03/13] fix(KubeVirtNodeDriver): put containerDisk (boot disk) as the first disk No more hangs when using ex_disk. The boot disk should be the first one in disks (and volumes) list (/dev/vda), otherwise the vm will not boot. --- libcloud/compute/drivers/kubevirt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index 2c7d48aba6..80f13df132 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -663,8 +663,9 @@ def create_node( boot_disk_name = "boot-disk-" + str(uuid.uuid4()) volumes_dict = {"containerDisk": {"image": image}, "name": boot_disk_name} disk_dict = {"disk": {"bus": "virtio"}, "name": boot_disk_name} - vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) - vm["spec"]["template"]["spec"]["volumes"].append(volumes_dict) + # boot disk should be the first one, otherwise it will not boot + vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].insert(0, disk_dict) + vm["spec"]["template"]["spec"]["volumes"].insert(0, volumes_dict) # auth -> cloud-init if auth is not None: From 51312a39dd8ce8f4fefb4645a18f6340c5a98dfb Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Tue, 2 Jan 2024 15:58:58 +0800 Subject: [PATCH 04/13] docs(compute.drivers.kubevirt): improve docstrs --- libcloud/compute/drivers/kubevirt.py | 556 +++++++++++++++++---------- 1 file changed, 348 insertions(+), 208 deletions(-) diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index 80f13df132..69639ddf30 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -74,6 +74,7 @@ class KubeVirtNodeDriver(KubernetesDriverMixin, NodeDriver): name = "kubevirt" website = "https://www.kubevirt.io" connectionCls = KubernetesBasicAuthConnection + features = {"create_node": ["ssh_key", "password"]} NODE_STATE_MAP = { "pending": NodeState.PENDING, @@ -112,7 +113,14 @@ def list_nodes(self, location=None): return vms def get_node(self, id=None, name=None): - "get a vm by name or id" + """get a vm by name or id. + + :param id: id of the vm + :type id: ``str`` + + :param name: name of the vm + :type name: ``str`` + """ if not id and not name: raise ValueError("This method needs id or name to be specified") nodes = self.list_nodes() @@ -127,6 +135,14 @@ def get_node(self, id=None, name=None): raise ValueError("Node does not exist") def start_node(self, node): + """Starting a VM. + + :param node: The node to be started. + :type node: :class:`Node` + + :return: True if the start was successful, False otherwise. + :rtype: ``bool`` + """ # make sure it is stopped if node.state is NodeState.RUNNING: return True @@ -146,6 +162,14 @@ def start_node(self, node): raise def stop_node(self, node): + """Stopping a VM. + + :param node: The node to be stopped. + :type node: :class:`Node` + + :return: True if the stop was successful, False otherwise. + :rtype: ``bool`` + """ # check if running if node.state is NodeState.STOPPED: return True @@ -167,6 +191,12 @@ def stop_node(self, node): def reboot_node(self, node): """ Rebooting a node. + + :param node: The node to be rebooted. + :type node: :class:`Node` + + :return: True if the reboot was successful, False otherwise. + :rtype: ``bool`` """ namespace = node.extra["namespace"] name = node.name @@ -184,7 +214,13 @@ def reboot_node(self, node): def destroy_node(self, node): """ - Terminating a VMI and deleting the VM resource backing it + Terminating a VMI and deleting the VM resource backing it. + + :param node: The node to be destroyed. + :type node: :class:`Node` + + :return: True if the destruction was successful, False otherwise. + :rtype: ``bool`` """ namespace = node.extra["namespace"] name = node.name @@ -208,26 +244,33 @@ def _create_node_with_template(self, name: str, template: dict, namespace="defau """ Creating a VM defined by the template. + The template must be a dictionary of kubernetes object that defines the + KubeVirt VM. Following are the keys: + + - ``apiVersion``: ``str`` + - ``kind``: ``str`` + - ``metadata``: ``dict`` + - ``spec``: ``dict`` + - ``domain``: ``dict`` + - ``volumes``: ``list`` + - ... + - ... + + See also: + + - https://kubernetes.io/docs/concepts/overview/working-with-objects/ + - https://kubevirt.io/api-reference/ + :param name: A name to give the VM. The VM will be identified by this name, and it must be the same as the name in the ``template["metadata"]["name"]``. Atm, it cannot be changed after it is set. :type name: ``str`` - :param template: A dictionary of kubernetes object that defines the - KubeVirt VM. - See also: - - https://kubernetes.io/docs/concepts/overview/working-with-objects/ - - https://kubevirt.io/api-reference/ + :param template: A dictionary of kubernetes object that defines the VM. :type template: ``dict`` with keys: - -apiVersion: ``str`` - -kind: ``str`` - -metadata: ``dict`` - -spec: ``dict`` - - domain: ``dict`` - - volumes: ``list`` - - ... - + ``apiVersion: str``, ``kind: str``, ``metadata: dict``, + ``spec: dict`` etc. :param namespace: The namespace where the VM will live. (default is 'default') @@ -239,9 +282,12 @@ def _create_node_with_template(self, name: str, template: dict, namespace="defau if template.get("kind", "") != "VirtualMachine": raise ValueError("The template must contain kind: VirtualMachine") if name != template.get("metadata", {}).get("name"): - raise ValueError("The name of the VM must be the same as the name in the template. " - "(name={}, template.metadata.name={})".format( - name, template.get("metadata", {}).get("name"))) + raise ValueError( + "The name of the VM must be the same as the name in the template. " + "(name={}, template.metadata.name={})".format( + name, template.get("metadata", {}).get("name") + ) + ) if template.get("spec", {}).get("running", False): warnings.warn( "The VM will be created in a stopped state, and then started. " @@ -325,171 +371,259 @@ def create_node( """ Creating a VM with a containerDisk. + @inherits: :class:`NodeDriver.create_node` + + The ``size`` parameter should be a ``NodeSize`` object that defines + the CPU and memory limits of the VM. + The ``NodeSize`` object must have the following attributes: + + - ram: int (in MiB) + - extra["cpu"]: int (in cores) + + An example NodeSize for a VM with 1 CPU and 2GiB of memory + (attributes except for ``ram`` and ``extra["cpu"]`` are not supported atm, + but they are required by the ``NodeSize`` object):: + + >>> size = NodeSize( + >>> id='small', # str: Size ID + >>> name='small', # str: Size name. Not used atm + >>> ram=2048, # int: Amount of memory (in MB) + >>> disk=20, # int: Amount of disk storage (in GB). Not used atm + >>> bandwidth=0, # int: Amount of bandwidth. Not used atm + >>> price=0, # float: Price (in US dollars) of running this node for an hour. Not used atm + >>> driver=KubeVirtNodeDriver # NodeDriver: Driver this size belongs to + >>> extra={ + >>> 'cpu': 1, # int: Number of CPUs provided by this size. + >>> }, + >>> ) + + The ``KubeVirtNodeSize`` wrapper can be used to create the ``NodeSize`` + object more easily:: + + >>> size = KubeVirtNodeSize( + >>> cpu=1, # int: Number of CPUs provided by this size. + >>> ram=2048, # int: Amount of memory (in MB) + >>> ) + + The ``image`` parameter can be either a ``NodeImage`` object + with a ``name`` attribute that points to a containerDisk image, + or a string representing the image URI. + + In both cases, it must point to a Docker image with an embedded disk. + May be a URI like `kubevirt/cirros-registry-disk-demo`, + kubevirt will automatically pull it from https://hub.docker.com/u/URI. + + For more info visit: https://kubevirt.io/user-guide/docs/latest/creating-virtual-machines/disks-and-volumes.html#containerdisk + + An example ``NodeImage``:: + + >>> image = NodeImage( + >>> id="something_unique", + >>> name="quay.io/containerdisks/ubuntu:22.04" + >>> driver=KubeVirtNodeDriver, + >>> ) + + The ``KubeVirtNodeImage`` wrapper can be used to create the ``NodeImage`` + object more easily:: + + >>> image = KubeVirtNodeImage( + >>> name="quay.io/containerdisks/ubuntu:22.04" + >>> ) + + The ``location`` parameter is a NodeLocation object + with the name being the kubernetes namespace where the VM will live. + If not provided, the VM will be created in the default namespace. + It can be created as the following:: + + >>> location = NodeLocation( + >>> name='default', # str: namespace + >>> ... # Other attributes are ignored + >>> ) + + The ``auth`` parameter is the authentication method for the VM, + either a ``NodeAuthSSHKey`` or a ``NodeAuthPassword``: + + - ``NodeAuthSSHKey(pubkey='pubkey data here')`` + - ``NodeAuthPassword(password='mysecretpassword')`` + + If the ``auth`` parameter is provided, the VM will be created with + a cloud-init volume that will inject the provided authentication + into the VM. + For details, see: + https://kubevirt.io/user-guide/virtual_machines/startup_scripts/#injecting-ssh-keys-with-cloud-inits-cloud-config + + The ``ex_disks`` parameter is a list of disk dictionaries that define + the disks of the VM. + Each dictionary should have specific keys for disk configuration. + + The following are optional keys: + + - ``"bus"``: can be "virtio", "sata", or "scsi" + - ``"device"``: can be "lun" or "disk" + + The following are required keys: + + - ``"disk_type"``: + One of the supported ``DISK_TYPES``. + Atm only "persistentVolumeClaim" and + "containerDisk" are promised to work. + Other types may work but are not tested. + + - ``"name"``: + The name of the disk configuration + + - ``"voulme_spec"``: + the dictionary that defines the volume of the disk. + The content is depending on the disk_type: + + For `containerDisk` the dictionary should have a key `image` with + the value being the image URI. + + For `persistentVolumeClaim` the dictionary should have a key + `claimName` with the value being the name of the claim. + If you wish, a new Persistent Volume Claim can be + created by providing the following: + + - required: + - size: the desired size (implied in GB) + - storage_class_name: the name of the storage class to # NOQA + be used for the creation of the + Persistent Volume Claim. + Make sure it allows for + dymamic provisioning. + - optional: + - access_mode: default is ReadWriteOnce + - volume_mode: default is `Filesystem`, it can also be `Block` + + For other disk types, it will be translated to + a KubeVirt volume object as is:: + + {disk_type: , "name": disk_name} + + This dict will be translated to KubeVirt API object:: + + volumes: + - name: + : + + + Please refer to the KubeVirt API documentation + for volume specifications: + + https://kubevirt.io/user-guide/virtual_machines/disks_and_volumes + + An example ``ex_disks`` list with a hostDisk that will create a new + img file in the host machine:: + + >>> ex_disks = [ + >>> { + >>> "bus": "virtio", + >>> "device": "disk", + >>> "disk_type": "hostDisk", + >>> "name": "disk114514", + >>> "volume_spec": { + >>> "capacity": "10Gi", + >>> "path": "/tmp/kubevirt/data/disk114514.img", + >>> "type": "DiskOrCreate", + >>> }, + >>> }, + >>> ] + + The ``ex_network`` parameter is a dictionary that defines the network + of the VM. + Only the ``pod`` type is supported, and in the configuration + ``masquerade`` or ``bridge`` are the accepted values. + The parameter must be a dict:: + + { + "network_type": "pod", + "interface": "masquerade | bridge", + "name": "network_name" + } + + For advanced configurations not covered by the other parameters, + the ``ex_template`` parameter can be used to provide a dictionary of + kubernetes object that defines the KubeVirt VM. + Notice, If provided, ``ex_template`` will override all other parameters + except for ``name`` and ``location``. + + The ``ex_template`` parameter is a dictionary of kubernetes object: + + - ``apiVersion``: ``str`` + - ``kind``: ``str`` + - ``metadata``: ``dict`` + - ``spec``: ``dict`` + - ``domain``: ``dict`` + - ``volumes``: ``list`` + - ... + + See also: + + - https://kubernetes.io/docs/concepts/overview/working-with-objects/ + - https://kubevirt.io/api-reference/ + :param name: A name to give the VM. The VM will be identified by this name and atm it cannot be changed after it is set. :type name: ``str`` :param size: The size of the VM in terms of CPU and memory. - :type size: ``NodeSize``: - - >>> size = NodeSize( - >>> # id='small', # str: Size ID - >>> # name='small', # str: Size name. Not used atm - >>> ram=2048, # int: Amount of memory (in MB) - >>> # disk=20, # int: Amount of disk storage (in GB). Not used atm - >>> # bandwidth=0, # int: Amount of bandwidth. Not used atm - >>> # price=0, # float: Price (in US dollars) of running this node for an hour. Not used atm - >>> # driver=KubeVirtNodeDriver # NodeDriver: Driver this size belongs to - >>> extra={ - >>> 'cpu': 1, # int: Number of CPUs provided by this size. - >>> }, - >>> ) + A NodeSize object with the attributes + ``ram`` (int in MiB) and ``extra["cpu"]`` (int in cores). + :type size: ``NodeSize`` with :param image: Either a libcloud NodeImage or a string. - In both cases it must point to a Docker image with an - embedded disk. - May be a URI like `kubevirt/cirros-registry-disk-demo`, - kubevirt will automatically pull it from - https://hub.docker.com/u/URI. - For more info visit: - https://kubevirt.io/user-guide/docs/latest/creating-virtual-machines/disks-and-volumes.html#containerdisk + In both cases, it must a URL to the containerDisk image. :type image: ``str`` or ``NodeImage`` - >>> image = NodeImage( - >>> name='kubevirt/cirros-registry-disk-demo', # str: Image URI - >>> ... # Other attributes are ignored - >>> ) - :param location: The namespace where the VM will live. - (default is 'default') + (default is ``'default'``) :type location: `NodeLocation`` in which the name is the namespace - >>> location = NodeLocation( - >>> name='default', # str: namespace - >>> ... # Other attributes are ignored - >>> ) - :param auth: authentication to a node. - :type auth: ``NodeAuthSSHKey`` or ``NodeAuthPassword``: - - NodeAuthSSHKey(pubkey='...') - - NodeAuthPassword(password='...') + :type auth: ``NodeAuthSSHKey`` or ``NodeAuthPassword``. :param ex_disks: A list containing disk dictionaries. - Each dictionary should have the - following optional keys: - - - ``"bus"``: can be "virtio", "sata", or "scsi" - - ``"device"``: can be "lun" or "disk" - - The following are required keys: - - - ``"disk_type"`` - One of the supported ``DISK_TYPES``. - Atm only "persistentVolumeClaim" and - "containerDisk" are promised to work. - Other types may work but are not tested. - - ``"name"``: - The name of the disk configuration - - ``"voulme_spec"``: - the dictionary that defines the volume - of the disk. - Will be translated to KubeVirt API object: - - volumes: - - name: - : - - - Content is depending on the disk_type: - - For "containerDisk" the dictionary - should have a key "image" with the - value being the image URI. - - For "persistentVolumeClaim" the - dictionary should have a key - "claimName" with the value being - the name of the claim. - If you wish, a new Persistent Volume Claim can be - created by providing the following: - - - required: - - size: the desired size (implied in GB) - - storage_class_name: the name of the storage class to # NOQA - be used for the creation of the - Persistent Volume Claim. - Make sure it allows for - dymamic provisioning. - - optional: - - access_mode: default is ReadWriteOnce - - volume_mode: default is `Filesystem`, - it can also be `Block` - - For other disk types, it will be translated to - a KubeVirt volume object as is: - - {disk_type: , "name": disk_name} - - Please refer to the KubeVirt API documentation - for volume specifications: - - https://kubevirt.io/user-guide/virtual_machines/disks_and_volumes - - :type ex_disks: `list` of `dict`. - For each `dict` the types for its keys are: - - bus: `str` - - device: `str` - - disk_type: `str` - - name: `str` - - voulme_spec: `dict` - - :param ex_network: Only the `pod` type is supported, and in the - configuration `masquerade` or `bridge` are the - accepted values. - The parameter must be a dict: - - { - "network_type": "pod", - "interface": "masquerade | bridge", - "name": "network_name" - } - - :type ex_network: `dict` with keys: - - - network_type: `str` | only "pod" is accepted atm - - interface: `str` | "masquerade" or "bridge" - - name: `str` + Each dictionary should have the following keys: + ``bus``: can be ``"virtio"``, ``"sata"``, or ``"scsi"``; + ``device``: can be ``"lun"`` or ``"disk"``; + ``disk_type``: One of the supported DISK_TYPES; + ``name``: The name of the disk configuration; + ``voulme_spec``: the dictionary that defines the volume of the disk. + + :type ex_disks: ``list`` of ``dict`` with keys: + ``bus: str``, ``device: str``, ``disk_type: str``, ``name: str`` + and ``voulme_spec: dict`` + + :param ex_network: a dictionary that defines the network of the VM. + The following keys are required: + ``network_type``: ``"pod"``; + ``interface``: ``"masquerade"`` or ``"bridge"``; + ``name``: ``"network_name"``. + :type ex_network: ``dict`` with keys: + ``network_type: str``, ``interface: str`` and ``name: str`` :param ex_termination_grace_period: The grace period in seconds before the VM is forcefully terminated. (default is 0) :type ex_termination_grace_period: `int` - :param ex_ports: A dictionary with keys: 'ports_tcp' and 'ports_udp' - 'ports_tcp' value is a list of ints that indicate + :param ex_ports: A dictionary with keys: ``"ports_tcp"`` and ``"ports_udp"`` + ``"ports_tcp"`` value is a list of ints that indicate the ports to be exposed with TCP protocol, - and 'ports_udp' is a list of ints that indicate + and ``"ports_udp"`` is a list of ints that indicate the ports to be exposed with UDP protocol. :type ex_ports: `dict` with keys - 'ports_tcp`: `list` of `int` - 'ports_udp`: `list` of `int` + ``ports_tcp``: ``list`` of ``int``; + ``ports_udp``: ``list`` of ``int``. :param ex_template: A dictionary of kubernetes object that defines the KubeVirt VM. This is for advanced vm specifications that are not covered by the other parameters. If provided, it will override all other parameters except for `name` and `location`. - See also: - - https://kubernetes.io/docs/concepts/overview/working-with-objects/ - - https://kubevirt.io/api-reference/ :type ex_template: ``dict`` with keys: - -apiVersion: ``str`` - -kind: ``str`` - -metadata: ``dict`` - -spec: ``dict`` - - domain: ``dict`` - - volumes: ``list`` - - ... + ``apiVersion: str``, ``kind: str``, ``metadata: dict`` + and ``spec: dict`` """ # location -> namespace @@ -515,14 +649,13 @@ def create_node( "ex_termination_grace_period": ex_termination_grace_period, "ex_ports": ex_ports, } - ignored_non_none_param_keys = list(filter( - lambda x: other_params[x] is not None, - other_params)) + ignored_non_none_param_keys = list( + filter(lambda x: other_params[x] is not None, other_params) + ) if ignored_non_none_param_keys: warnings.warn( "ex_template is provided, ignoring the following non-None " - "parameters: {}" - .format(ignored_non_none_param_keys) + "parameters: {}".format(ignored_non_none_param_keys) ) vm = copy.deepcopy(ex_template) @@ -535,14 +668,13 @@ def create_node( elif vm["metadata"]["name"] != name: warnings.warn( "The name in the ex_template ({}) will be ignored. " - "The name provided in the arguments ({}) will be used." - .format(vm["metadata"]["name"], name) + "The name provided in the arguments ({}) will be used.".format( + vm["metadata"]["name"], name + ) ) vm["metadata"]["name"] = name - return self._create_node_with_template(name=name, - template=vm, - namespace=namespace) + return self._create_node_with_template(name=name, template=vm, namespace=namespace) # else (ex_template is None): create a vm with other parameters # vm template to be populated @@ -600,9 +732,7 @@ def create_node( volumes_dict = {"containerDisk": {"image": image}, "name": disk_name} elif disk_type == "persistentVolumeClaim": if "volume_spec" not in disk: - raise KeyError( - "You must provide a volume_spec dictionary" - ) + raise KeyError("You must provide a volume_spec dictionary") if "claim_name" not in disk["volume_spec"]: msg = ( "You must provide either a claim_name of an " @@ -619,7 +749,10 @@ def create_node( claim_name = disk["volume_spec"]["claim_name"] if claim_name not in self.ex_list_persistent_volume_claims(namespace=namespace): - if "size" not in disk["volume_spec"] or "storage_class_name" not in disk["volume_spec"]: + if ( + "size" not in disk["volume_spec"] + or "storage_class_name" not in disk["volume_spec"] + ): msg = ( "disk['volume_spec']['size'] and " "disk['volume_spec']['storage_class_name'] " @@ -685,24 +818,24 @@ def create_node( disk_dict = {"disk": {"bus": "virtio"}, "name": cloud_init_volume} volume_dict = { "name": cloud_init_volume, - "cloudInitNoCloud": { - "userData": "" - }, + "cloudInitNoCloud": {"userData": ""}, } # auth # cloud_init_config reference: https://kubevirt.io/user-guide/virtual_machines/startup_scripts/#injecting-ssh-keys-with-cloud-inits-cloud-config if isinstance(auth, NodeAuthSSHKey): public_key = auth.pubkey - cloud_init_config = ("""#cloud-config\n""" - """ssh_authorized_keys:\n""" - """ - {}\n""").format(public_key) + cloud_init_config = ( + """#cloud-config\n""" """ssh_authorized_keys:\n""" """ - {}\n""" + ).format(public_key) elif isinstance(auth, NodeAuthPassword): password = auth.password - cloud_init_config = ("""#cloud-config\n""" - """password: {}\n""" - """chpasswd: {{ expire: False }}\n""" - """ssh_pwauth: True\n""").format(password) + cloud_init_config = ( + """#cloud-config\n""" + """password: {}\n""" + """chpasswd: {{ expire: False }}\n""" + """ssh_pwauth: True\n""" + ).format(password) else: raise ValueError("auth must be NodeAuthSSHKey or NodeAuthPassword") @@ -754,12 +887,12 @@ def create_node( # terminationGracePeriodSeconds if ex_termination_grace_period is not None: - assert isinstance(ex_termination_grace_period, int), ( - "ex_termination_grace_period must be an int" - ) - vm["spec"]["template"]["spec"]["terminationGracePeriodSeconds"] = ( - ex_termination_grace_period - ) + assert isinstance( + ex_termination_grace_period, int + ), "ex_termination_grace_period must be an int" + vm["spec"]["template"]["spec"][ + "terminationGracePeriodSeconds" + ] = ex_termination_grace_period return self._create_node_with_template(name=name, template=vm, namespace=namespace) @@ -1248,7 +1381,7 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node :type is_stopped: bool :return: a libcloud node - :rtype: Node + :rtype: :class:`Node` """ ID = vm["metadata"]["uid"] name = vm["metadata"]["name"] @@ -1402,7 +1535,7 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node def ex_list_services(self, namespace="default", node_name=None, service_name=None): """ If node_name is given then the services returned will be those that - concern the node + concern the node. """ params = None if service_name is not None: @@ -1433,41 +1566,41 @@ def ex_create_service( to delete existing exposed ports and expose just the ones in the port variable. - param node: the libcloud node for which the ports will be exposed - type node: libcloud `Node` class + :param node: the libcloud node for which the ports will be exposed + :type node: libcloud `Node` class - param ports: a list of dictionaries with keys --> values: - 'port' --> port to be exposed on the service + :param ports: a list of dictionaries with keys --> values: + 'port' --> port to be exposed on the service; 'target_port' --> port on the pod/node, optional if empty then it gets the same - value as 'port' value - 'protocol' ---> either 'UDP' or 'TCP', defaults to TCP - 'name' --> A name for the service + value as 'port' value; + 'protocol' ---> either 'UDP' or 'TCP', defaults to TCP; + 'name' --> A name for the service; If ports is an empty `list` and a service exists of this type then the service will be deleted. - type ports: `list` of `dict` where each `dict` has keys --> values: - 'port' --> `int` - 'target_port' --> `int` - 'protocol' --> `str` - 'name' --> `str` + :type ports: `list` of `dict` where each `dict` has keys --> values: + 'port' --> `int`; + 'target_port' --> `int`; + 'protocol' --> `str`; + 'name' --> `str`; - param service_type: Valid types are ClusterIP, NodePort, LoadBalancer - type service_type: `str` + :param service_type: Valid types are ClusterIP, NodePort, LoadBalancer + :type service_type: `str` - param cluster_ip: This can be set with an IP string value if you want + :param cluster_ip: This can be set with an IP string value if you want manually set the service's internal IP. If the value is not correct the method will fail, this value can't be updated. - type cluster_ip: `str` + :type cluster_ip: `str` - param override_existing_ports: Set to True if you want to delete the + :param override_existing_ports: Set to True if you want to delete the existing ports exposed by the service and keep just the ones declared in the present ports argument. By default it is false and if the service already exists the ports will be added to the existing ones. - type override_existing_ports: `boolean` + :type override_existing_ports: `boolean` """ # check if service exists first namespace = node.extra.get("namespace", "default") @@ -1563,10 +1696,12 @@ def _deep_merge_dict(source: dict, destination: dict) -> dict: Extended from https://stackoverflow.com/a/20666342, added zero value handling. - >>> a = {"domain": {"devices": 0}, "volumes": [1, 2, 3], "network": {}} - >>> b = {"domain": {"machine": "non-exist-in-a", "devices": 1024}, "volumes": [4, 5, 6]} - >>> _deep_merge_dict(a, b) - {'domain': {'machine': 'non-exist-in-a', 'devices': 1024}, 'volumes': [1, 2, 3], 'network': {}} + Example:: + + >>> a = {"domain": {"devices": 0}, "volumes": [1, 2, 3], "network": {}} + >>> b = {"domain": {"machine": "non-exist-in-a", "devices": 1024}, "volumes": [4, 5, 6]} + >>> _deep_merge_dict(a, b) + {'domain': {'machine': 'non-exist-in-a', 'devices': 1024}, 'volumes': [1, 2, 3], 'network': {}} In the above example: @@ -1658,13 +1793,15 @@ def KubeVirtNodeSize(cpu, ram): # type: (int, int) -> NodeSize """ Create a NodeSize object for KubeVirt driver. + This function is just a shorthand for ``NodeSize(ram=ram, extra={"cpu": cpu})``. + :param cpu: number of virtual CPUs :type cpu: ``int`` :param ram: amount of RAM in MiB :type ram: ``int`` - :return: a NodeSize object + :return: a NodeSize object with ram and extra.cpu set :rtype: :class:`NodeSize` """ extra = {"cpu": cpu} @@ -1686,10 +1823,13 @@ def KubeVirtNodeImage(name): # type: (str) -> NodeImage """ Create a NodeImage object for KubeVirt driver. + This function is just a shorthand for ``NodeImage(name=name)``. + :param name: image source :type name: ``str`` - :return: a NodeImage object + :return: a NodeImage object with the name set to the source to a + containerDisk image (e.g. ``"quay.io/containerdisks/ubuntu:22.04"``) :rtype: :class:`NodeImage` """ image_id = hashlib.md5(name.encode("utf-8")).hexdigest() From e0d832bb2cad846fdae32bd8bbb24e7e49d93f9d Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Tue, 2 Jan 2024 16:34:39 +0800 Subject: [PATCH 05/13] test(KubeVirtNodeDriver): add test_create_node --- libcloud/compute/drivers/kubevirt.py | 20 +- .../compute/fixtures/kubevirt/create_vm.json | 8 +- .../get_default_vms_after_create_vm.json | 182 ++++++++++++++++++ .../compute/fixtures/kubevirt/get_pvcs.json | 150 +++++++++++++++ libcloud/test/compute/test_kubevirt.py | 58 +++++- 5 files changed, 404 insertions(+), 14 deletions(-) create mode 100644 libcloud/test/compute/fixtures/kubevirt/get_default_vms_after_create_vm.json create mode 100644 libcloud/test/compute/fixtures/kubevirt/get_pvcs.json diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index 69639ddf30..475aee253f 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -85,7 +85,12 @@ class KubeVirtNodeDriver(KubernetesDriverMixin, NodeDriver): def list_nodes(self, location=None): namespaces = [] if location is not None: - namespaces.append(location.name) + if isinstance(location, NodeLocation): + namespaces.append(location.name) + elif isinstance(location, str): + namespaces.append(location) + else: + raise ValueError("location must be a NodeLocation or a string") else: for ns in self.list_locations(): namespaces.append(ns.name) @@ -303,16 +308,21 @@ def _create_node_with_template(self, name: str, template: dict, namespace="defau req = KUBEVIRT_URL + "namespaces/" + namespace + "/virtualmachines/" try: self.connection.request(req, method=method, data=data) - except Exception: raise + # check if new node is present - nodes = self.list_nodes() + nodes = self.list_nodes(location=namespace) for node in nodes: if node.name == name: self.start_node(node) return node + raise ValueError( + "The node was not found after creation, " + "create_node may have failed. Please check the kubernetes logs." + ) + @staticmethod def _base_vm_template(name=None): # type: (Optional[str]) -> dict """ @@ -480,7 +490,7 @@ def create_node( the value being the image URI. For `persistentVolumeClaim` the dictionary should have a key - `claimName` with the value being the name of the claim. + `claim_name` with the value being the name of the claim. If you wish, a new Persistent Volume Claim can be created by providing the following: @@ -707,8 +717,6 @@ def create_node( vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] = cpu vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"] = cpu - # TODO: debug: no boot disk??? - # ex_disks -> disks and volumes ex_disks = ex_disks or [] for i, disk in enumerate(ex_disks): diff --git a/libcloud/test/compute/fixtures/kubevirt/create_vm.json b/libcloud/test/compute/fixtures/kubevirt/create_vm.json index 0da20020dd..e86eedd7ba 100644 --- a/libcloud/test/compute/fixtures/kubevirt/create_vm.json +++ b/libcloud/test/compute/fixtures/kubevirt/create_vm.json @@ -4,19 +4,19 @@ "namespace": "default", "resourceVersion": "911058", "generation": 1, - "name": "libcloud-created", + "name": "testcreatenode", "labels": { - "kubevirt.io/vm": "libcloud-created" + "kubevirt.io/vm": "testcreatenode" }, "uid": "e553010d-e904-436f-a92a-396be0f8bd4c", - "selfLink": "/apis/kubevirt.io/v1alpha3/namespaces/default/virtualmachines/libcloud-created" + "selfLink": "/apis/kubevirt.io/v1alpha3/namespaces/default/virtualmachines/testcreatenode" }, "spec": { "running": false, "template": { "metadata": { "labels": { - "kubevirt.io/vm": "libcloud-created" + "kubevirt.io/vm": "testcreatenode" } }, "spec": { diff --git a/libcloud/test/compute/fixtures/kubevirt/get_default_vms_after_create_vm.json b/libcloud/test/compute/fixtures/kubevirt/get_default_vms_after_create_vm.json new file mode 100644 index 0000000000..80a12f07f6 --- /dev/null +++ b/libcloud/test/compute/fixtures/kubevirt/get_default_vms_after_create_vm.json @@ -0,0 +1,182 @@ +{ + "items": [ + { + "spec": { + "template": { + "spec": { + "domain": { + "resources": { + "requests": { + "memory": "64M" + } + }, + "devices": { + "disks": [ + { + "name": "containerdisk", + "disk": { + "bus": "virtio" + } + }, + { + "name": "cloudinitdisk", + "disk": { + "bus": "virtio" + } + } + ], + "interfaces": [ + { + "name": "default", + "bridge": {} + } + ] + }, + "machine": { + "type": "" + } + }, + "networks": [ + { + "name": "default", + "pod": {} + } + ], + "volumes": [ + { + "name": "containerdisk", + "containerDisk": { + "image": "kubevirt/cirros-registry-disk-demo" + } + }, + { + "cloudInitNoCloud": { + "userDataBase64": "SGkuXG4=" + }, + "name": "cloudinitdisk" + } + ] + }, + "metadata": { + "creationTimestamp": null, + "labels": { + "kubevirt.io/domain": "testvm", + "kubevirt.io/size": "small" + } + } + }, + "running": true + }, + "apiVersion": "kubevirt.io/v1alpha3", + "metadata": { + "annotations": { + "kubevirt.io/latest-observed-api-version": "v1alpha3", + "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"kubevirt.io/v1alpha3\",\"kind\":\"VirtualMachine\",\"metadata\":{\"annotations\":{},\"name\":\"testvm\",\"namespace\":\"default\"},\"spec\":{\"running\":false,\"template\":{\"metadata\":{\"labels\":{\"kubevirt.io/domain\":\"testvm\",\"kubevirt.io/size\":\"small\"}},\"spec\":{\"domain\":{\"devices\":{\"disks\":[{\"disk\":{\"bus\":\"virtio\"},\"name\":\"containerdisk\"},{\"disk\":{\"bus\":\"virtio\"},\"name\":\"cloudinitdisk\"}],\"interfaces\":[{\"bridge\":{},\"name\":\"default\"}]},\"resources\":{\"requests\":{\"memory\":\"64M\"}}},\"networks\":[{\"name\":\"default\",\"pod\":{}}],\"volumes\":[{\"containerDisk\":{\"image\":\"kubevirt/cirros-registry-disk-demo\"},\"name\":\"containerdisk\"},{\"cloudInitNoCloud\":{\"userDataBase64\":\"SGkuXG4=\"},\"name\":\"cloudinitdisk\"}]}}}}\n", + "kubevirt.io/storage-observed-api-version": "v1alpha3" + }, + "creationTimestamp": "2019-12-02T15:35:14Z", + "generation": 39, + "namespace": "default", + "name": "testvm", + "selfLink": "/apis/kubevirt.io/v1alpha3/namespaces/default/virtualmachines/testvm", + "resourceVersion": "284863", + "uid": "74fd7665-fbd6-4565-977c-96bd21fb785a" + }, + "kind": "VirtualMachine", + "status": { + "ready": true, + "created": true + } + }, + { + "metadata": { + "creationTimestamp": "2019-12-23T13:33:14Z", + "namespace": "default", + "resourceVersion": "911058", + "generation": 1, + "name": "testcreatenode", + "labels": { + "kubevirt.io/vm": "testcreatenode" + }, + "uid": "e553010d-e904-436f-a92a-396be0f8bd4c", + "selfLink": "/apis/kubevirt.io/v1alpha3/namespaces/default/virtualmachines/testcreatenode" + }, + "spec": { + "running": false, + "template": { + "metadata": { + "labels": { + "kubevirt.io/vm": "testcreatenode" + } + }, + "spec": { + "volumes": [ + { + "persistentVolumeClaim": { + "claimName": "mypvc2" + }, + "name": "anpvc" + }, + { + "name": "boot-disk", + "containerDisk": { + "image": "kubevirt/cirros-registry-disk-demo" + } + } + ], + "networks": [ + { + "name": "netw1", + "pod": {} + } + ], + "terminationGracePeriodSeconds": 0, + "domain": { + "resources": { + "requests": { + "memory": "128M" + } + }, + "cpu": {}, + "devices": { + "interfaces": [ + { + "name": "netw1", + "masquerade": {} + } + ], + "disks": [ + { + "name": "anpvc", + "disk": { + "bus": "virtio" + } + }, + { + "name": "boot-disk", + "disk": { + "bus": "virtio" + } + } + ], + "networkInterfaceMultiqueue": false + }, + "machine": { + "type": "" + } + } + } + } + }, + "kind": "VirtualMachine", + "apiVersion": "kubevirt.io/v1alpha3" + } + ], + "apiVersion": "kubevirt.io/v1alpha3", + "metadata": { + "continue": "", + "selfLink": "/apis/kubevirt.io/v1alpha3/namespaces/default/virtualmachines", + "resourceVersion": "285618" + }, + "kind": "VirtualMachineList" +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/kubevirt/get_pvcs.json b/libcloud/test/compute/fixtures/kubevirt/get_pvcs.json new file mode 100644 index 0000000000..05af01fd8a --- /dev/null +++ b/libcloud/test/compute/fixtures/kubevirt/get_pvcs.json @@ -0,0 +1,150 @@ +{ + "kind": "PersistentVolumeClaimList", + "apiVersion": "v1", + "metadata": { + "selfLink": "/api/v1/namespaces/default/persistentvolumeclaims", + "resourceVersion": "330771690" + }, + "items": [ + { + "metadata": { + "name": "mypvc2", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/persistentvolumeclaims/mypvc2", + "uid": "cd6f5a57-02c8-4012-8461-960ce49f95dd", + "resourceVersion": "324384796", + "creationTimestamp": "2023-12-19T07:17:53Z", + "labels": { + "alerts.k8s.io/KubePersistentVolumeFillingUp": "disabled", + "app": "containerized-data-importer", + "app.kubernetes.io/component": "storage", + "app.kubernetes.io/managed-by": "cdi-controller" + }, + "annotations": { + "cdi.kubevirt.io/storage.condition.bound": "false", + "cdi.kubevirt.io/storage.condition.bound.message": "Claim Pending", + "cdi.kubevirt.io/storage.condition.bound.reason": "Claim Pending", + "cdi.kubevirt.io/storage.contentType": "", + "cdi.kubevirt.io/storage.import.endpoint": "docker://quay.io/containerdisks/ubuntu:22.04", + "cdi.kubevirt.io/storage.import.importPodName": "importer-mypvc2", + "cdi.kubevirt.io/storage.import.requiresScratch": "false", + "cdi.kubevirt.io/storage.import.source": "registry", + "cdi.kubevirt.io/storage.pod.phase": "Pending", + "cdi.kubevirt.io/storage.pod.restarts": "0", + "cdi.kubevirt.io/storage.preallocation.requested": "false", + "volume.beta.kubernetes.io/storage-provisioner": "example.com/external-nfs" + }, + "ownerReferences": [ + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "name": "mypvc2", + "uid": "66470738-e865-4f18-95b1-86c40ff6a3d2", + "controller": true, + "blockOwnerDeletion": true + } + ], + "finalizers": [ + "kubernetes.io/pvc-protection" + ], + "managedFields": [ + { + "manager": "cdi-controller", + "operation": "Update", + "apiVersion": "v1", + "time": "2023-12-19T07:17:53Z", + "fieldsType": "FieldsV1", + "fieldsV1": {"f:metadata":{"f:annotations":{".":{},"f:cdi.kubevirt.io/storage.condition.bound":{},"f:cdi.kubevirt.io/storage.condition.bound.message":{},"f:cdi.kubevirt.io/storage.condition.bound.reason":{},"f:cdi.kubevirt.io/storage.contentType":{},"f:cdi.kubevirt.io/storage.import.endpoint":{},"f:cdi.kubevirt.io/storage.import.importPodName":{},"f:cdi.kubevirt.io/storage.import.requiresScratch":{},"f:cdi.kubevirt.io/storage.import.source":{},"f:cdi.kubevirt.io/storage.pod.phase":{},"f:cdi.kubevirt.io/storage.pod.restarts":{},"f:cdi.kubevirt.io/storage.preallocation.requested":{}},"f:labels":{".":{},"f:alerts.k8s.io/KubePersistentVolumeFillingUp":{},"f:app":{},"f:app.kubernetes.io/component":{},"f:app.kubernetes.io/managed-by":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"66170738-e845-4f58-99b1-88c40ff6a3d2\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:spec":{"f:accessModes":{},"f:resources":{"f:requests":{".":{},"f:storage":{}}},"f:storageClassName":{},"f:volumeMode":{}},"f:status":{"f:phase":{}}} + }, + { + "manager": "kube-controller-manager", + "operation": "Update", + "apiVersion": "v1", + "time": "2023-12-19T07:18:03Z", + "fieldsType": "FieldsV1", + "fieldsV1": {"f:metadata":{"f:annotations":{"f:volume.beta.kubernetes.io/storage-provisioner":{}}}} + } + ] + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "20Gi" + } + }, + "storageClassName": "example-nfs", + "volumeMode": "Filesystem" + }, + "status": { + "phase": "Pending" + } + }, + { + "metadata": { + "name": "mypvc2-scratch", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/persistentvolumeclaims/mypvc2-scratch", + "uid": "cf849c48-a91e-4c34-9639-0591824437dc", + "resourceVersion": "324384797", + "creationTimestamp": "2023-12-19T07:17:53Z", + "labels": { + "app": "containerized-data-importer", + "app.kubernetes.io/component": "storage", + "app.kubernetes.io/managed-by": "cdi-controller" + }, + "annotations": { + "volume.beta.kubernetes.io/storage-provisioner": "example.com/external-nfs" + }, + "ownerReferences": [ + { + "apiVersion": "v1", + "kind": "Pod", + "name": "importer-mypvc2", + "uid": "572c58a2-857d-4dc8-b2f6-0eff76a5a18a", + "controller": true, + "blockOwnerDeletion": true + } + ], + "finalizers": [ + "kubernetes.io/pvc-protection" + ], + "managedFields": [ + { + "manager": "cdi-controller", + "operation": "Update", + "apiVersion": "v1", + "time": "2023-12-19T07:17:53Z", + "fieldsType": "FieldsV1", + "fieldsV1": {"f:metadata":{"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/component":{},"f:app.kubernetes.io/managed-by":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"570c58a2-858d-4dd8-b0f6-0eff76a5a38a\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:spec":{"f:accessModes":{},"f:resources":{"f:requests":{".":{},"f:storage":{}}},"f:storageClassName":{},"f:volumeMode":{}},"f:status":{"f:phase":{}}} + }, + { + "manager": "kube-controller-manager", + "operation": "Update", + "apiVersion": "v1", + "time": "2023-12-19T07:17:53Z", + "fieldsType": "FieldsV1", + "fieldsV1": {"f:metadata":{"f:annotations":{".":{},"f:volume.beta.kubernetes.io/storage-provisioner":{}}}} + } + ] + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "20Gi" + } + }, + "storageClassName": "example-nfs", + "volumeMode": "Filesystem" + }, + "status": { + "phase": "Pending" + } + } + ] +} \ No newline at end of file diff --git a/libcloud/test/compute/test_kubevirt.py b/libcloud/test/compute/test_kubevirt.py index 0d89fac59c..bd110fd8d7 100644 --- a/libcloud/test/compute/test_kubevirt.py +++ b/libcloud/test/compute/test_kubevirt.py @@ -19,7 +19,7 @@ from libcloud.utils.py3 import httplib from libcloud.compute.types import NodeState from libcloud.test.file_fixtures import ComputeFileFixtures -from libcloud.compute.drivers.kubevirt import KubeVirtNodeDriver +from libcloud.compute.drivers.kubevirt import KubeVirtNodeDriver, KubeVirtNodeImage, KubeVirtNodeSize from libcloud.test.common.test_kubernetes import KubernetesAuthTestCaseMixin @@ -81,10 +81,38 @@ def test_reboot_node(self): self.assertTrue(resp) + def test_create_node(self): + node = self.driver.create_node( + name="testcreatenode", + size=KubeVirtNodeSize(cpu=1, ram=128), + image=KubeVirtNodeImage("kubevirt/cirros-registry-disk-demo"), + ex_disks=[ + { + "name": "anpvc", + "bus": "virtio", + "device": "disk", + "disk_type": "persistentVolumeClaim", + "volume_spec": { + "claim_name": "mypvc2" + }, + }, + ], + ex_network={ + "name": "netw1", + "network_type": "pod", + "interface": "masquerade", + }, + ) + self.assertEqual(node.name, "testcreatenode") + self.assertEqual(node.size.extra["cpu"], 1) + self.assertEqual(node.size.ram, 128) + class KubeVirtMockHttp(MockHttp): fixtures = ComputeFileFixtures("kubevirt") + did_create_vm = False + def _api_v1_namespaces(self, method, url, body, headers): if method == "GET": body = self.fixtures.load("_api_v1_namespaces.json") @@ -97,11 +125,15 @@ def _apis_kubevirt_io_v1alpha3_namespaces_default_virtualmachines( self, method, url, body, headers ): if method == "GET": - body = self.fixtures.load("get_default_vms.json") + if self.did_create_vm: + body = self.fixtures.load("get_default_vms_after_create_vm.json") + else: + body = self.fixtures.load("get_default_vms.json") resp = httplib.OK elif method == "POST": body = self.fixtures.load("create_vm.json") resp = httplib.CREATED + self.did_create_vm = True else: AssertionError("Unsupported method") return (resp, body, {}, httplib.responses[httplib.OK]) @@ -144,12 +176,20 @@ def _apis_kubevirt_io_v1alpha3_namespaces_kubevirt_virtualmachines( ): if method == "GET": body = self.fixtures.load("get_kube_public_vms.json") - elif method == "POST": - pass else: AssertionError("Unsupported method") return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _apis_kubevirt_io_v1alpha3_namespaces_default_virtualmachines_testcreatenode( + self, method, url, body, headers + ): + if method == "GET": + body = self.fixtures.load("create_vm.json") + else: + AssertionError("Unsupported method") + + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _apis_kubevirt_io_v1alpha3_namespaces_default_virtualmachines_testvm( self, method, url, body, headers ): @@ -212,6 +252,16 @@ def _api_v1_namespaces_default_services(self, method, url, body, headers): return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _api_v1_namespaces_default_persistentvolumeclaims( + self, method, url, body, headers + ): + if method == "GET": + body = self.fixtures.load("get_pvcs.json") + else: + AssertionError("Unsupported method") + + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + if __name__ == "__main__": sys.exit(unittest.main()) From a92dde7f4edc85ae83f1fcff7cd30b1aa042a681 Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Wed, 3 Jan 2024 13:42:59 +0800 Subject: [PATCH 06/13] apply tox run changes: style & doc --- docs/compute/_supported_methods_main.rst | 2 +- libcloud/test/compute/test_kubevirt.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/compute/_supported_methods_main.rst b/docs/compute/_supported_methods_main.rst index 4ed5f39a49..0fffc48a1b 100644 --- a/docs/compute/_supported_methods_main.rst +++ b/docs/compute/_supported_methods_main.rst @@ -26,7 +26,7 @@ Provider list nodes create node reboot node destroy node s `InternetSolutions`_ yes yes yes yes yes yes yes yes yes `Kamatera`_ yes yes yes yes yes yes yes yes yes `KTUCloud`_ yes yes yes yes no no yes yes yes -`kubevirt`_ yes yes yes yes yes yes yes yes no +`kubevirt`_ yes yes yes yes yes yes yes yes yes `Libvirt`_ yes no yes yes yes yes no no no `Linode`_ yes yes yes yes yes yes yes yes no `Maxihost`_ yes yes yes yes yes yes yes yes no diff --git a/libcloud/test/compute/test_kubevirt.py b/libcloud/test/compute/test_kubevirt.py index bd110fd8d7..f7090ce918 100644 --- a/libcloud/test/compute/test_kubevirt.py +++ b/libcloud/test/compute/test_kubevirt.py @@ -19,7 +19,11 @@ from libcloud.utils.py3 import httplib from libcloud.compute.types import NodeState from libcloud.test.file_fixtures import ComputeFileFixtures -from libcloud.compute.drivers.kubevirt import KubeVirtNodeDriver, KubeVirtNodeImage, KubeVirtNodeSize +from libcloud.compute.drivers.kubevirt import ( + KubeVirtNodeDriver, + KubeVirtNodeImage, + KubeVirtNodeSize, +) from libcloud.test.common.test_kubernetes import KubernetesAuthTestCaseMixin @@ -92,9 +96,7 @@ def test_create_node(self): "bus": "virtio", "device": "disk", "disk_type": "persistentVolumeClaim", - "volume_spec": { - "claim_name": "mypvc2" - }, + "volume_spec": {"claim_name": "mypvc2"}, }, ], ex_network={ @@ -252,9 +254,7 @@ def _api_v1_namespaces_default_services(self, method, url, body, headers): return (httplib.OK, body, {}, httplib.responses[httplib.OK]) - def _api_v1_namespaces_default_persistentvolumeclaims( - self, method, url, body, headers - ): + def _api_v1_namespaces_default_persistentvolumeclaims(self, method, url, body, headers): if method == "GET": body = self.fixtures.load("get_pvcs.json") else: From 3aaa67d00393e4e71c0b1a21e80f76cbb6318819 Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Wed, 3 Jan 2024 14:37:44 +0800 Subject: [PATCH 07/13] support legacy params ex_cpu, ex_memory and the 3-tuple type of ex_network --- libcloud/compute/drivers/kubevirt.py | 41 +++++++++++++++++++++----- libcloud/test/compute/test_kubevirt.py | 4 +-- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index 475aee253f..b59461af80 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -312,6 +312,9 @@ def _create_node_with_template(self, name: str, template: dict, namespace="defau raise # check if new node is present + # But why not just use the resp from the POST request? + # Or self.get_node()? + # I don't think a for loop over list_nodes is necessary. nodes = self.list_nodes(location=namespace) for node in nodes: if node.name == name: @@ -368,10 +371,12 @@ def _base_vm_template(name=None): # type: (Optional[str]) -> dict def create_node( self, name, # type: str - size, # type: Optional[NodeSize] - image, # type: Optional[Union[NodeImage, str]] + size=None, # type: Optional[NodeSize] + image=None, # type: Optional[Union[NodeImage, str]] location=None, # type: Optional[NodeLocation] auth=None, # type: Optional[Union[NodeAuthSSHKey, NodeAuthPassword]] + ex_cpu=None, # type: Optional[Union[int, str]] + ex_memory=None, # type: Optional[Union[int]] ex_disks=None, # type: Optional[list] ex_network=None, # type: Optional[dict] ex_termination_grace_period=0, # type: Optional[int] @@ -415,6 +420,14 @@ def create_node( >>> ram=2048, # int: Amount of memory (in MB) >>> ) + For legacy support, the ``ex_cpu`` and ``ex_memory`` parameters + can be used instead of ``size``: + + - ``ex_cpu``: The number of CPU cores to allocate to the VM. + ex_cpu must be a number (cores) or a string + ending with 'm' (miliCPUs). + - ``ex_memory``: The amount of memory to allocate to the VM in MiB. + The ``image`` parameter can be either a ``NodeImage`` object with a ``name`` attribute that points to a containerDisk image, or a string representing the image URI. @@ -592,6 +605,14 @@ def create_node( :param auth: authentication to a node. :type auth: ``NodeAuthSSHKey`` or ``NodeAuthPassword``. + :param ex_cpu: The number of CPU cores to allocate to the VM. + (Legacy support, consider using ``size`` instead) + :type ex_cpu: ``int`` or ``str`` + + :param ex_memory: The amount of memory to allocate to the VM in MiB. + (Legacy support, consider using ``size`` instead) + :type ex_memory: ``int`` + :param ex_disks: A list containing disk dictionaries. Each dictionary should have the following keys: ``bus``: can be ``"virtio"``, ``"sata"``, or ``"scsi"``; @@ -696,9 +717,6 @@ def create_node( assert isinstance(size, NodeSize), "size must be a NodeSize" ex_cpu = size.extra["cpu"] ex_memory = size.ram - else: - ex_cpu = None - ex_memory = None if ex_memory is not None: assert isinstance(ex_memory, int), "ex_memory must be an int in MiB" @@ -859,9 +877,16 @@ def create_node( if ex_network is not None: try: - interface = ex_network["interface"] - network_name = ex_network["name"] - network_type = ex_network["network_type"] + if isinstance(ex_network, dict): + interface = ex_network["interface"] + network_name = ex_network["name"] + network_type = ex_network["network_type"] + elif isinstance(ex_network, (tuple, list)): # legacy 3-tuple + network_type = ex_network[0] + interface = ex_network[1] + network_name = ex_network[2] + else: + raise KeyError("ex_network must be a dictionary or a tuple/list") except KeyError: msg = ( "ex_network: You must provide a dictionary with keys: " diff --git a/libcloud/test/compute/test_kubevirt.py b/libcloud/test/compute/test_kubevirt.py index f7090ce918..a2df4082cf 100644 --- a/libcloud/test/compute/test_kubevirt.py +++ b/libcloud/test/compute/test_kubevirt.py @@ -20,9 +20,9 @@ from libcloud.compute.types import NodeState from libcloud.test.file_fixtures import ComputeFileFixtures from libcloud.compute.drivers.kubevirt import ( - KubeVirtNodeDriver, - KubeVirtNodeImage, KubeVirtNodeSize, + KubeVirtNodeImage, + KubeVirtNodeDriver, ) from libcloud.test.common.test_kubernetes import KubernetesAuthTestCaseMixin From fa62045d9f01613d7ad02ae5a2456e173909f0c0 Mon Sep 17 00:00:00 2001 From: CDFMLR <45259230+cdfmlr@users.noreply.github.com> Date: Fri, 19 Apr 2024 11:21:18 +0800 Subject: [PATCH 08/13] Documenting breaking changes to KubeVirt driver #1983 --- docs/upgrade_notes.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/upgrade_notes.rst b/docs/upgrade_notes.rst index 5e13616872..217d6e878d 100644 --- a/docs/upgrade_notes.rst +++ b/docs/upgrade_notes.rst @@ -30,6 +30,38 @@ Libcloud 3.9.0 If your code is using those arguments / passing them to the `create_node()` method it needs to be updated and those arguments removed. +* [KubeVirt] Changes to the `create_node()` method: + + - The `ports` argument has been renamed to `ex_ports`. + - The `ex_disks` argument has been redefined to support all volume types. + + The deprecated `ex_disks` format, which only supports `PersistentVolumeClaim`, + is as follows: + + .. sourcecode:: python + + ex_disks=[{"bus": "", "device": "", "disk_type": "", "name": "", "claim_name": "", "size": "", "storage_class_name": "", "volume_mode": "", "access_mode": ""}] + + The new format is: + + .. sourcecode:: python + + ex_disks=[{"bus": "", "device": "", "disk_type": "", "name": "", "volume_spec": {...}}] + + Here, `volume_spec` is the `disk_type` specific settings, which aligns with the + KubeVirt user guide on disks and volumes + (https://kubevirt.io/user-guide/virtual_machines/disks_and_volumes). + + For example, for PVC: + + .. sourcecode:: python + + ex_disks=[{ ..., "volume_spec": {{"claim_name": "", "size": "", "storage_class_name": "", "volume_mode": "", "access_mode": ""} }] + + If your code uses these arguments or passes them to the `create_node()` + method, it will need to be updated accordingly. + + Libcloud 3.8.0 -------------- From f765b4299de9770ac49e9f5343ee28a96d99e26b Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Tue, 18 Jun 2024 12:17:36 +0800 Subject: [PATCH 09/13] test(KubeVirtNodeDriver): covere helpers and unhappy paths --- libcloud/test/compute/test_kubevirt.py | 239 +++++++++++++++++++++++++ 1 file changed, 239 insertions(+) diff --git a/libcloud/test/compute/test_kubevirt.py b/libcloud/test/compute/test_kubevirt.py index a2df4082cf..d8cbee715f 100644 --- a/libcloud/test/compute/test_kubevirt.py +++ b/libcloud/test/compute/test_kubevirt.py @@ -17,12 +17,15 @@ from libcloud.test import MockHttp, unittest from libcloud.utils.py3 import httplib +from libcloud.compute.base import NodeAuthPassword from libcloud.compute.types import NodeState from libcloud.test.file_fixtures import ComputeFileFixtures from libcloud.compute.drivers.kubevirt import ( KubeVirtNodeSize, KubeVirtNodeImage, KubeVirtNodeDriver, + _memory_in_MB, + _deep_merge_dict, ) from libcloud.test.common.test_kubernetes import KubernetesAuthTestCaseMixin @@ -109,6 +112,242 @@ def test_create_node(self): self.assertEqual(node.size.extra["cpu"], 1) self.assertEqual(node.size.ram, 128) + def test_create_node_default_net(self): + node = self.driver.create_node( + name="testcreatenode", + size=KubeVirtNodeSize(cpu=1, ram=128), + image=KubeVirtNodeImage("kubevirt/cirros-registry-disk-demo"), + ex_disks=[ + { + "name": "anpvc", + "bus": "virtio", + "device": "disk", + "disk_type": "persistentVolumeClaim", + "volume_spec": {"claim_name": "mypvc2"}, + }, + ], + ) + self.assertEqual(node.name, "testcreatenode") + self.assertEqual(node.size.extra["cpu"], 1) + self.assertEqual(node.size.ram, 128) + + def test_create_node_legacy_3_tuple_net(self): + node = self.driver.create_node( + name="testcreatenode", + size=KubeVirtNodeSize(cpu=1, ram=128), + image=KubeVirtNodeImage("kubevirt/cirros-registry-disk-demo"), + ex_disks=[ + { + "name": "anpvc", + "bus": "virtio", + "device": "disk", + "disk_type": "persistentVolumeClaim", + "volume_spec": {"claim_name": "mypvc2"}, + }, + ], + ex_network=("pod", "masquerade", "netw1"), + ) + self.assertEqual(node.name, "testcreatenode") + self.assertEqual(node.size.extra["cpu"], 1) + self.assertEqual(node.size.ram, 128) + + def test_create_node_auth_and_cloud_init(self): + try: + self.driver.create_node( + name="testcreatenode", + size=KubeVirtNodeSize(cpu=1, ram=128), + image=KubeVirtNodeImage("kubevirt/cirros-registry-disk-demo"), + auth=NodeAuthPassword("password"), + ex_disks=[ + { + "name": "anpvc", + "bus": "virtio", + "device": "disk", + "disk_type": "persistentVolumeClaim", + "volume_spec": {"claim_name": "mypvc2"}, + }, + { + "name": "cloudinit", + "bus": "virtio", + "device": "cdrom", + "disk_type": "cloudInitConfigDrive", + "volume_spec": { + "cloudInitNoCloud": { + "userData": "echo 'hello world'", + } + }, + }, + ], + ) + except ValueError as e: + self.assertIn("auth and cloudInit at the same time", str(e)) + else: + self.fail("Expected ValueError") + + def test_create_node_bad_pvc(self): + try: + self.driver.create_node( + name="testcreatenode", + size=KubeVirtNodeSize(cpu=1, ram=128), + image=KubeVirtNodeImage("kubevirt/cirros-registry-disk-demo"), + ex_disks=[ + { + "name": "badpvc", + "bus": "virtio", + "device": "disk", + "disk_type": "persistentVolumeClaim", + "volume_spec": { + "claim_name": "notexistnewpvc", + "storage_class_name": "longhorn", + # missing size + }, + }, + ], + ) + except KeyError as e: + self.assertIn("size", str(e)) + self.assertIn("storage_class_name", str(e)) + self.assertIn("both required", str(e)) + else: + self.fail("Expected KeyError") + + def test_create_node_ex_template(self): + # missing the optional metadata & required apiVersion + # metadata will be added by the driver + # and apiVersion missing will raise an error by _create_node_with_template + template = { + "kind": "VirtualMachine", + "spec": { + "running": False, + "template": { + "spec": { + "domain": { + "devices": { + "disks": [], + "interfaces": [], + "networkInterfaceMultiqueue": False, + }, + "machine": {"type": ""}, + "resources": {"requests": {}, "limits": {}}, + }, + "networks": [], + "terminationGracePeriodSeconds": 0, + "volumes": [], + }, + }, + }, + } + try: + self.driver.create_node( + name="testcreatenode", + # size & image should be ignored when ex_template is provided + size=KubeVirtNodeSize(cpu=1, ram=128), + image=KubeVirtNodeImage("kubevirt/cirros-registry-disk-demo"), + ex_template=template, + ) + except ValueError as e: + self.assertEqual(str(e), "The template must have an apiVersion: kubevirt.io/v1alpha3") + else: + self.fail("Expected ValueError") + + def test_memory_in_MB(self): + self.assertEqual(_memory_in_MB("128Mi"), 128) + self.assertEqual(_memory_in_MB("128M"), 128) + + self.assertEqual(_memory_in_MB("134217728"), 128) + self.assertEqual(_memory_in_MB(134217728), 128) + + self.assertEqual(_memory_in_MB("128Gi"), 128 * 1024) + self.assertEqual(_memory_in_MB("128G"), 128 * 1000) + + self.assertEqual(_memory_in_MB("1920Ki"), 1920 // 1024) + self.assertEqual(_memory_in_MB("1920K"), 1920 // 1000) + + self.assertEqual(_memory_in_MB("1Ti"), 1 * 1024 * 1024) + self.assertEqual(_memory_in_MB("1T"), 1 * 1000 * 1000) + + def test_deep_merge_dict(self): + a = {"domain": {"devices": 0}, "volumes": [1, 2, 3], "network": {}} + b = {"domain": {"machine": "non-exist-in-a", "devices": 1024}, "volumes": [4, 5, 6]} + expected_result = { + "domain": {"machine": "non-exist-in-a", "devices": 1024}, + "volumes": [1, 2, 3], + "network": {}, + } + self.assertEqual(_deep_merge_dict(a, b), expected_result) + + a = {"domain": {"devices": 1024}, "volumes": [1, 2, 3], "network": {}} + b = {"domain": {"machine": "non-exist-in-a", "devices": 0}, "volumes": [4, 5, 6]} + expected_result = { + "domain": {"machine": "non-exist-in-a", "devices": 1024}, + "volumes": [1, 2, 3], + "network": {}, + } + self.assertEqual(_deep_merge_dict(a, b), expected_result) + + a = {"domain": {"devices": 0}, "volumes": [1, 2, 3], "network": {}} + b = {"domain": {"machine": "non-exist-in-a", "devices": 0}, "volumes": [4, 5, 6]} + expected_result = { + "domain": {"machine": "non-exist-in-a", "devices": 0}, + "volumes": [1, 2, 3], + "network": {}, + } + self.assertEqual(_deep_merge_dict(a, b), expected_result) + + a = {"domain": {"devices": 1024}, "volumes": [1, 2, 3], "network": {}} + b = {"domain": {"machine": "non-exist-in-a", "devices": 1024}, "volumes": [4, 5, 6]} + expected_result = { + "domain": {"machine": "non-exist-in-a", "devices": 1024}, + "volumes": [1, 2, 3], + "network": {}, + } + self.assertEqual(_deep_merge_dict(a, b), expected_result) + + a = {"domain": {"devices": 1024}, "volumes": [1, 2, 3], "network": {}} + b = {"domain": {"machine": "non-exist-in-a", "devices": 2048}, "volumes": [4, 5, 6]} + expected_result = { + "domain": {"machine": "non-exist-in-a", "devices": 1024}, + "volumes": [1, 2, 3], + "network": {}, + } + self.assertEqual(_deep_merge_dict(a, b), expected_result) + + a = {"domain": {"devices": 1024}, "volumes": [1, 2, 3], "network": {}} + b = { + "domain": {"machine": "non-exist-in-a", "devices": 1024, "foo": "bar"}, + "volumes": [4, 5, 6], + } + expected_result = { + "domain": {"machine": "non-exist-in-a", "devices": 1024, "foo": "bar"}, + "volumes": [1, 2, 3], + "network": {}, + } + self.assertEqual(_deep_merge_dict(a, b), expected_result) + + a = {"domain": {"devices": 1024}, "volumes": [1, 2, 3], "network": {}} + b = { + "domain": {"machine": "non-exist-in-a", "devices": 1024, "foo": ""}, + "volumes": [4, 5, 6], + } + expected_result = { + "domain": {"machine": "non-exist-in-a", "devices": 1024, "foo": ""}, + "volumes": [1, 2, 3], + "network": {}, + } + self.assertEqual(_deep_merge_dict(a, b), expected_result) + + a = {"domain": {"devices": 1024}, "volumes": [1, 2, 3], "network": {}} + b = { + "domain": {"machine": "non-exist-in-a", "devices": 1024, "foo": None}, + "volumes": [4, 5, 6], + } + expected_result = { + "domain": {"machine": "non-exist-in-a", "devices": 1024, "foo": None}, + "volumes": [1, 2, 3], + "network": {}, + } + self.assertEqual(_deep_merge_dict(a, b), expected_result) + class KubeVirtMockHttp(MockHttp): fixtures = ComputeFileFixtures("kubevirt") From 40ec5efd603a75ecb315ea5edd562e4e19baa1cb Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Tue, 18 Jun 2024 16:09:11 +0800 Subject: [PATCH 10/13] refactor(KubeVirtNodeDriver): split _create_node() into multiple smaller methods --- libcloud/compute/drivers/kubevirt.py | 629 ++++++++++++++++----------- 1 file changed, 385 insertions(+), 244 deletions(-) diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index b59461af80..5117e13391 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -367,7 +367,382 @@ def _base_vm_template(name=None): # type: (Optional[str]) -> dict }, } - # only has container disk support atm with no persistency + @staticmethod + def _create_node_vm_from_ex_template( + name, ex_template, other_args + ): # type: (str, dict) -> dict + """ + A part of create_node that deals with the VM template. + Returns the VM template with the name set. + + :param name: A name to give the VM. The VM will be identified by + this name and atm it cannot be changed after it is set. + See also the name parameter in create_node. + :type name: ``str`` + + :param ex_template: A dictionary of kubernetes object that defines the + KubeVirt VM. See also the ex_template parameter in create_node. + :type ex_template: ``dict`` with keys: + ``apiVersion: str``, ``kind: str``, ``metadata: dict`` + and ``spec: dict`` + + :param other_args: Other parameters passed to the create_node method. + This is used to warn the user about ignored parameters. + See also the parameters of create_node. + :type other_args: ``dict`` + + :return: dict: The VM template with the name set. + """ + assert isinstance(ex_template, dict), "ex_template must be a dictionary" + + other_params = { + "size": other_args.get("size"), + "image": other_args.get("image"), + "auth": other_args.get("auth"), + "ex_disks": other_args.get("ex_disks"), + "ex_network": other_args.get("ex_network"), + "ex_termination_grace_period": other_args.get("ex_termination_grace_period"), + "ex_ports": other_args.get("ex_ports"), + } + ignored_non_none_param_keys = list( + filter(lambda x: other_params[x] is not None, other_params) + ) + if ignored_non_none_param_keys: + warnings.warn( + "ex_template is provided, ignoring the following non-None " + "parameters: {}".format(ignored_non_none_param_keys) + ) + + vm = copy.deepcopy(ex_template) + + if vm.get("metadata") is None: + vm["metadata"] = {} + + if vm["metadata"].get("name") is None: + vm["metadata"]["name"] = name + elif vm["metadata"]["name"] != name: + warnings.warn( + "The name in the ex_template ({}) will be ignored. " + "The name provided in the arguments ({}) will be used.".format( + vm["metadata"]["name"], name + ) + ) + vm["metadata"]["name"] = name + + return vm + + @staticmethod + def _create_node_size( + vm, size=None, ex_cpu=None, ex_memory=None + ): # type: (dict, NodeSize, int, int) -> None + """ + A part of create_node that deals with the size of the VM. + It will fill the vm with size information. + + :param size: The size of the VM in terms of CPU and memory. + See also the size parameter in create_node. + :type size: ``NodeSize`` with + + :param ex_cpu: The number of CPU cores to allocate to the VM. + See also the ex_cpu parameter in create_node. + :type ex_cpu: ``int`` or ``str`` + + :param ex_memory: The amount of memory to allocate to the VM in MiB. + See also the ex_memory parameter in create_node. + :type ex_memory: ``int`` + + :return: None + """ + if size is not None: + assert isinstance(size, NodeSize), "size must be a NodeSize" + ex_cpu = size.extra["cpu"] + ex_memory = size.ram + + if ex_memory is not None: + assert isinstance(ex_memory, int), "ex_memory must be an int in MiB" + memory = str(ex_memory) + "Mi" + + vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] = memory + vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["memory"] = memory + + if ex_cpu is not None: + if isinstance(ex_cpu, str) and ex_cpu.endswith("m"): + cpu = ex_cpu + else: + try: + cpu = float(ex_cpu) + except ValueError: + raise ValueError("ex_cpu must be a number or a string ending with 'm'") + + vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] = cpu + vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"] = cpu + + @staticmethod + def _create_node_termination_grace_period( + vm, ex_termination_grace_period + ): # type: (dict, int) -> None + """ + A part of create_node that deals with the termination grace period of the VM. + It will fill the vm with the termination grace period information. + + :param vm: The VM template to be filled. + :type vm: ``dict`` + :param ex_termination_grace_period: The termination grace period of the VM in seconds. + See also the ex_termination_grace_period parameter in create_node. + :type ex_termination_grace_period: ``int`` + :return: None + """ + assert isinstance( + ex_termination_grace_period, int + ), "ex_termination_grace_period must be an int" + + vm["spec"]["template"]["spec"][ + "terminationGracePeriodSeconds" + ] = ex_termination_grace_period + + @staticmethod + def _create_node_network(vm, ex_network, ex_ports): # type: (dict, dict, dict) -> None + """ + A part of create_node that deals with the network of the VM. + It will fill the vm with network information. + + :param vm: The VM template to be filled. + :type vm: ``dict`` + :param ex_network: The network configuration of the VM. + See also the ex_network parameter in create_node. + :type ex_network: ``dict`` + :param ex_ports: The ports to expose in the VM. + See also the ex_ports parameter in create_node. + :type ex_ports: ``dict`` + :return: None + """ + # ex_network -> network and interface + if ex_network is not None: + try: + if isinstance(ex_network, dict): + interface = ex_network["interface"] + network_name = ex_network["name"] + network_type = ex_network["network_type"] + elif isinstance(ex_network, (tuple, list)): # legacy 3-tuple + network_type = ex_network[0] + interface = ex_network[1] + network_name = ex_network[2] + else: + raise KeyError("ex_network must be a dictionary or a tuple/list") + except KeyError: + msg = ( + "ex_network: You must provide a dictionary with keys: " + "'interface', 'name', 'network_type'." + ) + raise KeyError(msg) + # add a default network + else: + interface = "masquerade" + network_name = "netw1" + network_type = "pod" + + network_dict = {network_type: {}, "name": network_name} + interface_dict = {interface: {}, "name": network_name} + + # ex_ports -> network.ports + ex_ports = ex_ports or {} + if ex_ports.get("ports_tcp"): + ports_to_expose = [] + for port in ex_ports["ports_tcp"]: + ports_to_expose.append({"port": port, "protocol": "TCP"}) + interface_dict[interface]["ports"] = ports_to_expose + if ex_ports.get("ports_udp"): + ports_to_expose = interface_dict[interface].get("ports", []) + for port in ex_ports.get("ports_udp"): + ports_to_expose.append({"port": port, "protocol": "UDP"}) + interface_dict[interface]["ports"] = ports_to_expose + + vm["spec"]["template"]["spec"]["networks"].append(network_dict) + vm["spec"]["template"]["spec"]["domain"]["devices"]["interfaces"].append(interface_dict) + + @staticmethod + def _create_node_auth(vm, auth): + """ + A part of create_node that deals with the authentication of the VM. + It will fill the vm with a cloud-init volume that injects the authentication. + + :param vm: The VM template to be filled. + :param auth: The authentication method for the VM. + See also the auth parameter in create_node. + :type auth: ``NodeAuthSSHKey`` or ``NodeAuthPassword`` + :return: None + """ + # auth requires cloud-init, + # and only one cloud-init volume is supported by kubevirt. + # So if both auth and cloud-init are provided, raise an error. + for volume in vm["spec"]["template"]["spec"]["volumes"]: + if "cloudInitNoCloud" in volume or "cloudInitConfigDrive" in volume: + raise ValueError( + "Setting auth and cloudInit at the same time is not supported." + "Use deploy_node() instead." + ) + + # cloud-init volume + cloud_init_volume = "auth-cloudinit-" + str(uuid.uuid4()) + disk_dict = {"disk": {"bus": "virtio"}, "name": cloud_init_volume} + volume_dict = { + "name": cloud_init_volume, + "cloudInitNoCloud": {"userData": ""}, + } + + # cloud_init_config reference: https://kubevirt.io/user-guide/virtual_machines/startup_scripts/#injecting-ssh-keys-with-cloud-inits-cloud-config + if isinstance(auth, NodeAuthSSHKey): + public_key = auth.pubkey + cloud_init_config = ( + """#cloud-config\n""" """ssh_authorized_keys:\n""" """ - {}\n""" + ).format(public_key) + elif isinstance(auth, NodeAuthPassword): + password = auth.password + cloud_init_config = ( + """#cloud-config\n""" + """password: {}\n""" + """chpasswd: {{ expire: False }}\n""" + """ssh_pwauth: True\n""" + ).format(password) + else: + raise ValueError("auth must be NodeAuthSSHKey or NodeAuthPassword") + volume_dict["cloudInitNoCloud"]["userData"] = cloud_init_config + + # add volume + vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) + vm["spec"]["template"]["spec"]["volumes"].append(volume_dict) + + def _create_node_disks(self, vm, ex_disks, image, namespace, location): + """ + A part of create_node that deals with the disks (and volumes) of the VM. + It will fill the vm with disk information. + OS image is added to the disks as well. + + :param vm: The VM template to be filled. + :type vm: ``dict`` + :param ex_disks: The disk configuration of the VM. + See also the ex_disks parameter in create_node. + :type ex_disks: ``list`` + :param image: The image to be used as the OS disk. + See also the image parameter in create_node. + :param namespace: The namespace where the VM will live. + See also the location parameter in create_node. + :type namespace: ``str`` + :param location: The location where the VM will live. + See also the location parameter in create_node. + :type location: ``str`` + + :return: None + """ + # ex_disks -> disks and volumes + + ex_disks = ex_disks or [] + + for i, disk in enumerate(ex_disks): + disk_type = disk.get("disk_type") + bus = disk.get("bus", "virtio") + disk_name = disk.get("name", "disk{}".format(i)) + device = disk.get("device", "disk") + + if disk_type not in DISK_TYPES: + raise ValueError("The possible values for this " "parameter are: ", DISK_TYPES) + + # depending on disk_type, in the future, + # when more will be supported, + # additional elif should be added + if disk_type == "containerDisk": + try: + image = disk["volume_spec"]["image"] + except KeyError: + raise KeyError("A container disk needs a " "containerized image") + + volumes_dict = {"containerDisk": {"image": image}, "name": disk_name} + elif disk_type == "persistentVolumeClaim": + if "volume_spec" not in disk: + raise KeyError("You must provide a volume_spec dictionary") + if "claim_name" not in disk["volume_spec"]: + msg = ( + "You must provide either a claim_name of an " + "existing claim or if you want one to be " + "created you must additionally provide size " + "and the storage_class_name of the " + "cluster, which allows dynamic provisioning, " + "so a Persistent Volume Claim can be created. " + "In the latter case please provide the desired " + "size as well." + ) + raise KeyError(msg) + + claim_name = disk["volume_spec"]["claim_name"] + + if claim_name not in self.ex_list_persistent_volume_claims(namespace=namespace): + if ( + "size" not in disk["volume_spec"] + or "storage_class_name" not in disk["volume_spec"] + ): + msg = ( + "disk['volume_spec']['size'] and " + "disk['volume_spec']['storage_class_name'] " + "are both required to create " + "a new claim." + ) + raise KeyError(msg) + size = disk["volume_spec"]["size"] + storage_class = disk["volume_spec"]["storage_class_name"] + volume_mode = disk["volume_spec"].get("volume_mode", "Filesystem") + access_mode = disk["volume_spec"].get("access_mode", "ReadWriteOnce") + self.create_volume( + size=size, + name=claim_name, + location=location, + ex_storage_class_name=storage_class, + ex_volume_mode=volume_mode, + ex_access_mode=access_mode, + ) + + volumes_dict = { + "persistentVolumeClaim": {"claimName": claim_name}, + "name": disk_name, + } + else: + warnings.warn( + "The disk type {} is not tested. Use at your own risk.".format(disk_type) + ) + volumes_dict = {disk_type: disk.get("volume_spec", {}), "name": disk_name} + + disk_dict = {device: {"bus": bus}, "name": disk_name} + vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) + vm["spec"]["template"]["spec"]["volumes"].append(volumes_dict) + # end of for disk in ex_disks + + # image -> containerDisk -> add as the first disk + self._create_node_image(vm, image) + + @staticmethod + def _create_node_image(vm, image): # type: (dict, NodeImage) -> None + """ + A part of create_node that deals with the image of the VM. + It will fill the vm with the OS image as the first disk. + + :param vm: The VM template to be filled. + :param image: The image to be used as the OS disk. + See also the image parameter in create_node. + :type image: ``NodeImage`` or ``str`` + :return: None + """ + # image -> containerDisk + # adding image in a container Disk + if isinstance(image, NodeImage): + image = image.name + + boot_disk_name = "boot-disk-" + str(uuid.uuid4()) + volumes_dict = {"containerDisk": {"image": image}, "name": boot_disk_name} + disk_dict = {"disk": {"bus": "virtio"}, "name": boot_disk_name} + + # boot disk should be the first one, otherwise it will not boot + vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].insert(0, disk_dict) + vm["spec"]["template"]["spec"]["volumes"].insert(0, volumes_dict) + def create_node( self, name, # type: str @@ -665,267 +1040,33 @@ def create_node( else: namespace = "default" - # ex_template or _base_vm_template -> vm - # ex_template exists, use it to create the vm, ignore other parameters if ex_template is not None: - assert isinstance(ex_template, dict), "ex_template must be a dictionary" - - other_params = { - "size": size, - "image": image, - "auth": auth, - "ex_disks": ex_disks, - "ex_network": ex_network, - "ex_termination_grace_period": ex_termination_grace_period, - "ex_ports": ex_ports, - } - ignored_non_none_param_keys = list( - filter(lambda x: other_params[x] is not None, other_params) + vm = self._create_node_vm_from_ex_template( + name=name, ex_template=ex_template, other_args=locals() ) - if ignored_non_none_param_keys: - warnings.warn( - "ex_template is provided, ignoring the following non-None " - "parameters: {}".format(ignored_non_none_param_keys) - ) - - vm = copy.deepcopy(ex_template) - - if vm.get("metadata") is None: - vm["metadata"] = {} - - if vm["metadata"].get("name") is None: - vm["metadata"]["name"] = name - elif vm["metadata"]["name"] != name: - warnings.warn( - "The name in the ex_template ({}) will be ignored. " - "The name provided in the arguments ({}) will be used.".format( - vm["metadata"]["name"], name - ) - ) - vm["metadata"]["name"] = name - return self._create_node_with_template(name=name, template=vm, namespace=namespace) # else (ex_template is None): create a vm with other parameters - - # vm template to be populated vm = self._base_vm_template(name=name) # size -> cpu and memory limits + self._create_node_size(vm=vm, size=size, ex_cpu=ex_cpu, ex_memory=ex_memory) - if size is not None: - assert isinstance(size, NodeSize), "size must be a NodeSize" - ex_cpu = size.extra["cpu"] - ex_memory = size.ram - - if ex_memory is not None: - assert isinstance(ex_memory, int), "ex_memory must be an int in MiB" - memory = str(ex_memory) + "Mi" - vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] = memory - vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["memory"] = memory - - if ex_cpu is not None: - if isinstance(ex_cpu, str) and ex_cpu.endswith("m"): - cpu = ex_cpu - else: - try: - cpu = float(ex_cpu) - except ValueError: - raise ValueError("ex_cpu must be a number or a string ending with 'm'") - vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] = cpu - vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"] = cpu - - # ex_disks -> disks and volumes - ex_disks = ex_disks or [] - for i, disk in enumerate(ex_disks): - disk_type = disk.get("disk_type") - bus = disk.get("bus", "virtio") - disk_name = disk.get("name", "disk{}".format(i)) - device = disk.get("device", "disk") - - if disk_type not in DISK_TYPES: - raise ValueError("The possible values for this " "parameter are: ", DISK_TYPES) - - # depending on disk_type, in the future, - # when more will be supported, - # additional elif should be added - if disk_type == "containerDisk": - try: - image = disk["volume_spec"]["image"] - except KeyError: - raise KeyError("A container disk needs a " "containerized image") - - volumes_dict = {"containerDisk": {"image": image}, "name": disk_name} - elif disk_type == "persistentVolumeClaim": - if "volume_spec" not in disk: - raise KeyError("You must provide a volume_spec dictionary") - if "claim_name" not in disk["volume_spec"]: - msg = ( - "You must provide either a claim_name of an " - "existing claim or if you want one to be " - "created you must additionally provide size " - "and the storage_class_name of the " - "cluster, which allows dynamic provisioning, " - "so a Persistent Volume Claim can be created. " - "In the latter case please provide the desired " - "size as well." - ) - raise KeyError(msg) - - claim_name = disk["volume_spec"]["claim_name"] - - if claim_name not in self.ex_list_persistent_volume_claims(namespace=namespace): - if ( - "size" not in disk["volume_spec"] - or "storage_class_name" not in disk["volume_spec"] - ): - msg = ( - "disk['volume_spec']['size'] and " - "disk['volume_spec']['storage_class_name'] " - "are both required to create " - "a new claim." - ) - raise KeyError(msg) - size = disk["volume_spec"]["size"] - storage_class = disk["volume_spec"]["storage_class_name"] - volume_mode = disk["volume_spec"].get("volume_mode", "Filesystem") - access_mode = disk["volume_spec"].get("access_mode", "ReadWriteOnce") - self.create_volume( - size=size, - name=claim_name, - location=location, - ex_storage_class_name=storage_class, - ex_volume_mode=volume_mode, - ex_access_mode=access_mode, - ) - - volumes_dict = { - "persistentVolumeClaim": {"claimName": claim_name}, - "name": disk_name, - } - else: - warnings.warn( - "The disk type {} is not tested. Use at your own risk.".format(disk_type) - ) - volumes_dict = {disk_type: disk.get("volume_spec", {}), "name": disk_name} - - disk_dict = {device: {"bus": bus}, "name": disk_name} - vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) - vm["spec"]["template"]["spec"]["volumes"].append(volumes_dict) - # end of for disk in ex_disks - - # image -> containerDisk - # adding image in a container Disk - if isinstance(image, NodeImage): - image = image.name - - boot_disk_name = "boot-disk-" + str(uuid.uuid4()) - volumes_dict = {"containerDisk": {"image": image}, "name": boot_disk_name} - disk_dict = {"disk": {"bus": "virtio"}, "name": boot_disk_name} - # boot disk should be the first one, otherwise it will not boot - vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].insert(0, disk_dict) - vm["spec"]["template"]["spec"]["volumes"].insert(0, volumes_dict) + # ex_disks -> disks and volumes, image -> containerDisk + self._create_node_disks(vm, ex_disks, image, namespace, location) # auth -> cloud-init if auth is not None: - # auth requires cloud-init, - # and only one cloud-init volume is supported by kubevirt. - # So if both auth and cloud-init are provided, raise an error. - - for volume in vm["spec"]["template"]["spec"]["volumes"]: - if "cloudInitNoCloud" in volume or "cloudInitConfigDrive" in volume: - raise ValueError( - "Setting auth and cloudInit at the same time is not supported." - "Use deploy_node() instead." - ) - - # cloud-init volume - cloud_init_volume = "auth-cloudinit-" + str(uuid.uuid4()) - disk_dict = {"disk": {"bus": "virtio"}, "name": cloud_init_volume} - volume_dict = { - "name": cloud_init_volume, - "cloudInitNoCloud": {"userData": ""}, - } - - # auth - # cloud_init_config reference: https://kubevirt.io/user-guide/virtual_machines/startup_scripts/#injecting-ssh-keys-with-cloud-inits-cloud-config - if isinstance(auth, NodeAuthSSHKey): - public_key = auth.pubkey - cloud_init_config = ( - """#cloud-config\n""" """ssh_authorized_keys:\n""" """ - {}\n""" - ).format(public_key) - elif isinstance(auth, NodeAuthPassword): - password = auth.password - cloud_init_config = ( - """#cloud-config\n""" - """password: {}\n""" - """chpasswd: {{ expire: False }}\n""" - """ssh_pwauth: True\n""" - ).format(password) - else: - raise ValueError("auth must be NodeAuthSSHKey or NodeAuthPassword") - - volume_dict["cloudInitNoCloud"]["userData"] = cloud_init_config - - # add volume - vm["spec"]["template"]["spec"]["domain"]["devices"]["disks"].append(disk_dict) - vm["spec"]["template"]["spec"]["volumes"].append(volume_dict) + self._create_node_auth(vm, auth) # now, all disks and volumes stuff are done - # ex_network -> network and interface - - if ex_network is not None: - try: - if isinstance(ex_network, dict): - interface = ex_network["interface"] - network_name = ex_network["name"] - network_type = ex_network["network_type"] - elif isinstance(ex_network, (tuple, list)): # legacy 3-tuple - network_type = ex_network[0] - interface = ex_network[1] - network_name = ex_network[2] - else: - raise KeyError("ex_network must be a dictionary or a tuple/list") - except KeyError: - msg = ( - "ex_network: You must provide a dictionary with keys: " - "'interface', 'name', 'network_type'." - ) - raise KeyError(msg) - # add a default network - else: - interface = "masquerade" - network_name = "netw1" - network_type = "pod" - - network_dict = {network_type: {}, "name": network_name} - interface_dict = {interface: {}, "name": network_name} - - # ex_ports -> network.ports - ex_ports = ex_ports or {} - if ex_ports.get("ports_tcp"): - ports_to_expose = [] - for port in ex_ports["ports_tcp"]: - ports_to_expose.append({"port": port, "protocol": "TCP"}) - interface_dict[interface]["ports"] = ports_to_expose - if ex_ports.get("ports_udp"): - ports_to_expose = interface_dict[interface].get("ports", []) - for port in ex_ports.get("ports_udp"): - ports_to_expose.append({"port": port, "protocol": "UDP"}) - interface_dict[interface]["ports"] = ports_to_expose - - vm["spec"]["template"]["spec"]["networks"].append(network_dict) - vm["spec"]["template"]["spec"]["domain"]["devices"]["interfaces"].append(interface_dict) + # ex_network and ex_ports -> network and interface + self._create_node_network(vm, ex_network, ex_ports) # terminationGracePeriodSeconds if ex_termination_grace_period is not None: - assert isinstance( - ex_termination_grace_period, int - ), "ex_termination_grace_period must be an int" - vm["spec"]["template"]["spec"][ - "terminationGracePeriodSeconds" - ] = ex_termination_grace_period + self._create_node_termination_grace_period(vm, ex_termination_grace_period) return self._create_node_with_template(name=name, template=vm, namespace=namespace) From 4c622eef1236032b3548470f310ae0e9b6ac2d8c Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Tue, 18 Jun 2024 16:45:16 +0800 Subject: [PATCH 11/13] feat(KubeVirtNodeDriver): set cpu/mem requests and limits separately cherry-pick 3a4fe39e8e this feature is required by our internal e2e test. --- libcloud/compute/drivers/kubevirt.py | 74 +++++++++++++++++++++------- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index 5117e13391..bbd4aacd2e 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -453,29 +453,55 @@ def _create_node_size( :return: None """ + # size -> cpu and memory limits / requests + + ex_memory_limit = ex_memory_request = ex_cpu_limit = ex_cpu_request = None + if size is not None: assert isinstance(size, NodeSize), "size must be a NodeSize" - ex_cpu = size.extra["cpu"] - ex_memory = size.ram + ex_cpu_limit = size.extra["cpu"] + ex_memory_limit = size.ram + # optional resc requests: default = limit + ex_cpu_request = size.extra.get("cpu_request", None) or ex_cpu_limit + ex_memory_request = size.extra.get("ram_request", None) or ex_memory_limit - if ex_memory is not None: - assert isinstance(ex_memory, int), "ex_memory must be an int in MiB" - memory = str(ex_memory) + "Mi" + # memory - vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] = memory + if ex_memory is not None: # legacy + ex_memory_limit = ex_memory + ex_memory_request = ex_memory + + def _format_memory(memory_value): # type: (int) -> str + assert isinstance(memory_value, int), "memory must be an int in MiB" + return str(memory_value) + "Mi" + + if ex_memory_limit is not None: + memory = _format_memory(ex_memory_limit) vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["memory"] = memory + if ex_memory_request is not None: + memory = _format_memory(ex_memory_request) + vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] = memory - if ex_cpu is not None: - if isinstance(ex_cpu, str) and ex_cpu.endswith("m"): - cpu = ex_cpu - else: - try: - cpu = float(ex_cpu) - except ValueError: - raise ValueError("ex_cpu must be a number or a string ending with 'm'") + # cpu - vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] = cpu + if ex_cpu is not None: # legacy + ex_cpu_limit = ex_cpu + ex_cpu_request = ex_cpu + + def _format_cpu(cpu_value): # type: (Union[int, str]) -> Union[str, float] + if isinstance(cpu_value, str) and cpu_value.endswith("m"): + return cpu_value + try: + return float(cpu_value) + except ValueError: + raise ValueError("cpu must be a number or a string ending with 'm'") + + if ex_cpu_limit is not None: + cpu = _format_cpu(ex_cpu_limit) vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"] = cpu + if ex_cpu_request is not None: + cpu = _format_cpu(ex_cpu_request) + vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] = cpu @staticmethod def _create_node_termination_grace_period( @@ -1963,22 +1989,34 @@ def _memory_in_MB(memory): # type: (Union[str, int]) -> int raise ValueError("memory unit not supported {}".format(memory)) -def KubeVirtNodeSize(cpu, ram): # type: (int, int) -> NodeSize +def KubeVirtNodeSize( + cpu, ram, cpu_request=None, ram_request=None +): # type: (int, int, Optional[int], Optional[int]) -> NodeSize """ Create a NodeSize object for KubeVirt driver. This function is just a shorthand for ``NodeSize(ram=ram, extra={"cpu": cpu})``. - :param cpu: number of virtual CPUs + :param cpu: number of virtual CPUs (max limit) :type cpu: ``int`` - :param ram: amount of RAM in MiB + :param ram: amount of RAM in MiB (max limit) :type ram: ``int`` + :param cpu_request: number of virtual CPUs (min request) + :type cpu_request: ``int`` + + :param ram_request: amount of RAM in MiB (min request) + :type ram_request: ``int`` + :return: a NodeSize object with ram and extra.cpu set :rtype: :class:`NodeSize` """ extra = {"cpu": cpu} + + extra["cpu_request"] = cpu_request or cpu + extra["ram_request"] = ram_request or ram + name = "{} vCPUs, {}MB Ram".format(str(cpu), str(ram)) size_id = hashlib.md5(name.encode("utf-8")).hexdigest() return NodeSize( From f78d36141939e1e5fe35a94366d4f98143ee5b31 Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Tue, 18 Jun 2024 17:44:35 +0800 Subject: [PATCH 12/13] test(KubeVirtNodeDriver): cpu/mem requests and limits --- libcloud/compute/drivers/kubevirt.py | 23 +- .../fixtures/kubevirt/_api_v1_namespaces.json | 17 + .../fixtures/kubevirt/create_vm_reqlim.json | 218 ++++++++++++ .../get_testreqlim_vms_after_create_vm.json | 317 ++++++++++++++++++ .../kubevirt/get_vm_test_tumbleweed_07.json | 218 ++++++++++++ libcloud/test/compute/test_kubevirt.py | 71 +++- 6 files changed, 859 insertions(+), 5 deletions(-) create mode 100644 libcloud/test/compute/fixtures/kubevirt/create_vm_reqlim.json create mode 100644 libcloud/test/compute/fixtures/kubevirt/get_testreqlim_vms_after_create_vm.json create mode 100644 libcloud/test/compute/fixtures/kubevirt/get_vm_test_tumbleweed_07.json diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index bbd4aacd2e..dda912bca3 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -1060,7 +1060,7 @@ def create_node( # location -> namespace if isinstance(location, NodeLocation): - if location not in self.list_locations(): + if location.name not in map(lambda x: x.name, self.list_locations()): raise ValueError("The location must be one of the available namespaces") namespace = location.name else: @@ -1600,6 +1600,16 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node memory = vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] memory = _memory_in_MB(memory) + memory_req = ( + vm["spec"]["template"]["spec"]["domain"]["resources"] + .get("requests", {}) + .get("memory", None) + ) + if memory_req: + memory_req = _memory_in_MB(memory_req) + else: + memory_req = memory + cpu = 1 if vm["spec"]["template"]["spec"]["domain"]["resources"].get("limits", None): if vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"].get("cpu", None): @@ -1608,12 +1618,21 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node "spec" ]["template"]["spec"]["domain"]["resources"]["requests"].get("cpu", None): cpu = vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] + cpu_req = cpu elif vm["spec"]["template"]["spec"]["domain"].get("cpu", None): cpu = vm["spec"]["template"]["spec"]["domain"]["cpu"].get("cores", 1) if not isinstance(cpu, int): cpu = int(cpu.rstrip("m")) - extra_size = {"cpu": cpu} + cpu_req = ( + vm["spec"]["template"]["spec"]["domain"]["resources"] + .get("requests", {}) + .get("cpu", None) + ) + if cpu_req is None: + cpu_req = cpu + + extra_size = {"cpu": cpu, "cpu_request": cpu_req, "ram": memory, "ram_request": memory_req} size_name = "{} vCPUs, {}MB Ram".format(str(cpu), str(memory)) size_id = hashlib.md5(size_name.encode("utf-8")).hexdigest() size = NodeSize( diff --git a/libcloud/test/compute/fixtures/kubevirt/_api_v1_namespaces.json b/libcloud/test/compute/fixtures/kubevirt/_api_v1_namespaces.json index d15525b55b..3be5559f89 100644 --- a/libcloud/test/compute/fixtures/kubevirt/_api_v1_namespaces.json +++ b/libcloud/test/compute/fixtures/kubevirt/_api_v1_namespaces.json @@ -93,6 +93,23 @@ "status": { "phase": "Active" } + }, + { + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "metadata": { + "name": "testreqlim", + "creationTimestamp": "2019-12-01T15:54:25Z", + "selfLink": "/api/v1/namespaces/default", + "resourceVersion": "146", + "uid": "5341e71d-e8d8-4a1b-a97b-52864eb3dd7d" + }, + "status": { + "phase": "Active" + } } ] } \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/kubevirt/create_vm_reqlim.json b/libcloud/test/compute/fixtures/kubevirt/create_vm_reqlim.json new file mode 100644 index 0000000000..2f23647387 --- /dev/null +++ b/libcloud/test/compute/fixtures/kubevirt/create_vm_reqlim.json @@ -0,0 +1,218 @@ +{ + "apiVersion": "kubevirt.io/v1alpha3", + "kind": "VirtualMachine", + "metadata": { + "annotations": { + "kubevirt.io/latest-observed-api-version": "v1", + "kubevirt.io/storage-observed-api-version": "v1alpha3" + }, + "creationTimestamp": "2024-06-18T08:48:27Z", + "generation": 2, + "labels": { + "kubevirt.io/vm": "vm-test-tumbleweed-07" + }, + "managedFields": [ + { + "apiVersion": "kubevirt.io/v1alpha3", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:labels": { + ".": {}, + "f:kubevirt.io/vm": {} + } + }, + "f:spec": { + ".": {}, + "f:running": {}, + "f:template": { + ".": {}, + "f:metadata": { + ".": {}, + "f:labels": { + ".": {}, + "f:kubevirt.io/vm": {} + } + }, + "f:spec": { + ".": {}, + "f:domain": { + ".": {}, + "f:devices": { + ".": {}, + "f:disks": {}, + "f:interfaces": {}, + "f:networkInterfaceMultiqueue": {} + }, + "f:machine": { + ".": {}, + "f:type": {} + }, + "f:resources": { + ".": {}, + "f:limits": { + ".": {}, + "f:cpu": {}, + "f:memory": {} + }, + "f:requests": { + ".": {}, + "f:cpu": {}, + "f:memory": {} + } + } + }, + "f:networks": {}, + "f:terminationGracePeriodSeconds": {}, + "f:volumes": {} + } + } + } + }, + "manager": "libcloud", + "operation": "Update", + "time": "2024-06-18T08:48:28Z" + }, + { + "apiVersion": "kubevirt.io/v1alpha3", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:kubevirt.io/latest-observed-api-version": {}, + "f:kubevirt.io/storage-observed-api-version": {} + } + }, + "f:status": { + ".": {}, + "f:conditions": {}, + "f:created": {}, + "f:printableStatus": {}, + "f:ready": {}, + "f:volumeSnapshotStatuses": {} + } + }, + "manager": "Go-http-client", + "operation": "Update", + "time": "2024-06-18T08:49:04Z" + } + ], + "name": "vm-test-tumbleweed-07", + "namespace": "testreqlim", + "resourceVersion": "543653381", + "selfLink": "/apis/kubevirt.io/v1/namespaces/testreqlim/virtualmachines/vm-test-tumbleweed-07", + "uid": "2654b6fe-f230-4d18-8d80-c5d540ce360f" + }, + "spec": { + "running": true, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "kubevirt.io/vm": "vm-test-tumbleweed-07" + } + }, + "spec": { + "domain": { + "devices": { + "disks": [ + { + "disk": { + "bus": "virtio" + }, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea" + }, + { + "disk": { + "bus": "virtio" + }, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4" + } + ], + "interfaces": [ + { + "bridge": {}, + "name": "default" + } + ], + "networkInterfaceMultiqueue": false + }, + "machine": { + "type": "q35" + }, + "resources": { + "limits": { + "cpu": "2", + "memory": "4Gi" + }, + "requests": { + "cpu": "1m", + "memory": "1Mi" + } + } + }, + "networks": [ + { + "name": "default", + "pod": {} + } + ], + "terminationGracePeriodSeconds": 0, + "volumes": [ + { + "containerDisk": { + "image": "registry.internal.com/kubevirt-vmidisks/tumbleweed:240531" + }, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea" + }, + { + "cloudInitNoCloud": { + "userData": "#cloud-config\nssh_authorized_keys:\n - ssh-rsa FAKEKEY foo@bar.com\n" + }, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4" + } + ] + } + } + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2024-06-18T08:34:48Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": null, + "message": "cannot migrate VMI which does not use masquerade to connect to the pod network", + "reason": "InterfaceNotLiveMigratable", + "status": "False", + "type": "LiveMigratable" + }, + { + "lastProbeTime": "2024-06-18T08:35:18Z", + "lastTransitionTime": null, + "status": "True", + "type": "AgentConnected" + } + ], + "created": true, + "printableStatus": "Running", + "ready": true, + "volumeSnapshotStatuses": [ + { + "enabled": false, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea", + "reason": "Snapshot is not supported for this volumeSource type [boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea]" + }, + { + "enabled": false, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4", + "reason": "Snapshot is not supported for this volumeSource type [auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4]" + } + ] + } +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/kubevirt/get_testreqlim_vms_after_create_vm.json b/libcloud/test/compute/fixtures/kubevirt/get_testreqlim_vms_after_create_vm.json new file mode 100644 index 0000000000..971583c957 --- /dev/null +++ b/libcloud/test/compute/fixtures/kubevirt/get_testreqlim_vms_after_create_vm.json @@ -0,0 +1,317 @@ +{ + "items": [ + { + "spec": { + "template": { + "spec": { + "domain": { + "resources": { + "requests": { + "memory": "64M" + } + }, + "devices": { + "disks": [ + { + "name": "containerdisk", + "disk": { + "bus": "virtio" + } + }, + { + "name": "cloudinitdisk", + "disk": { + "bus": "virtio" + } + } + ], + "interfaces": [ + { + "name": "default", + "bridge": {} + } + ] + }, + "machine": { + "type": "" + } + }, + "networks": [ + { + "name": "default", + "pod": {} + } + ], + "volumes": [ + { + "name": "containerdisk", + "containerDisk": { + "image": "kubevirt/cirros-registry-disk-demo" + } + }, + { + "cloudInitNoCloud": { + "userDataBase64": "SGkuXG4=" + }, + "name": "cloudinitdisk" + } + ] + }, + "metadata": { + "creationTimestamp": null, + "labels": { + "kubevirt.io/domain": "testvm", + "kubevirt.io/size": "small" + } + } + }, + "running": true + }, + "apiVersion": "kubevirt.io/v1alpha3", + "metadata": { + "annotations": { + "kubevirt.io/latest-observed-api-version": "v1alpha3", + "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"kubevirt.io/v1alpha3\",\"kind\":\"VirtualMachine\",\"metadata\":{\"annotations\":{},\"name\":\"testvm\",\"namespace\":\"default\"},\"spec\":{\"running\":false,\"template\":{\"metadata\":{\"labels\":{\"kubevirt.io/domain\":\"testvm\",\"kubevirt.io/size\":\"small\"}},\"spec\":{\"domain\":{\"devices\":{\"disks\":[{\"disk\":{\"bus\":\"virtio\"},\"name\":\"containerdisk\"},{\"disk\":{\"bus\":\"virtio\"},\"name\":\"cloudinitdisk\"}],\"interfaces\":[{\"bridge\":{},\"name\":\"default\"}]},\"resources\":{\"requests\":{\"memory\":\"64M\"}}},\"networks\":[{\"name\":\"default\",\"pod\":{}}],\"volumes\":[{\"containerDisk\":{\"image\":\"kubevirt/cirros-registry-disk-demo\"},\"name\":\"containerdisk\"},{\"cloudInitNoCloud\":{\"userDataBase64\":\"SGkuXG4=\"},\"name\":\"cloudinitdisk\"}]}}}}\n", + "kubevirt.io/storage-observed-api-version": "v1alpha3" + }, + "creationTimestamp": "2019-12-02T15:35:14Z", + "generation": 39, + "namespace": "default", + "name": "testvm", + "selfLink": "/apis/kubevirt.io/v1alpha3/namespaces/default/virtualmachines/testvm", + "resourceVersion": "284863", + "uid": "74fd7665-fbd6-4565-977c-96bd21fb785a" + }, + "kind": "VirtualMachine", + "status": { + "ready": true, + "created": true + } + }, + { + "apiVersion": "kubevirt.io/v1alpha3", + "kind": "VirtualMachine", + "metadata": { + "annotations": { + "kubevirt.io/latest-observed-api-version": "v1", + "kubevirt.io/storage-observed-api-version": "v1alpha3" + }, + "creationTimestamp": "2024-06-18T08:48:27Z", + "generation": 2, + "labels": { + "kubevirt.io/vm": "vm-test-tumbleweed-07" + }, + "managedFields": [ + { + "apiVersion": "kubevirt.io/v1alpha3", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:labels": { + ".": {}, + "f:kubevirt.io/vm": {} + } + }, + "f:spec": { + ".": {}, + "f:running": {}, + "f:template": { + ".": {}, + "f:metadata": { + ".": {}, + "f:labels": { + ".": {}, + "f:kubevirt.io/vm": {} + } + }, + "f:spec": { + ".": {}, + "f:domain": { + ".": {}, + "f:devices": { + ".": {}, + "f:disks": {}, + "f:interfaces": {}, + "f:networkInterfaceMultiqueue": {} + }, + "f:machine": { + ".": {}, + "f:type": {} + }, + "f:resources": { + ".": {}, + "f:limits": { + ".": {}, + "f:cpu": {}, + "f:memory": {} + }, + "f:requests": { + ".": {}, + "f:cpu": {}, + "f:memory": {} + } + } + }, + "f:networks": {}, + "f:terminationGracePeriodSeconds": {}, + "f:volumes": {} + } + } + } + }, + "manager": "libcloud", + "operation": "Update", + "time": "2024-06-18T08:48:28Z" + }, + { + "apiVersion": "kubevirt.io/v1alpha3", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:kubevirt.io/latest-observed-api-version": {}, + "f:kubevirt.io/storage-observed-api-version": {} + } + }, + "f:status": { + ".": {}, + "f:conditions": {}, + "f:created": {}, + "f:printableStatus": {}, + "f:ready": {}, + "f:volumeSnapshotStatuses": {} + } + }, + "manager": "Go-http-client", + "operation": "Update", + "time": "2024-06-18T08:49:04Z" + } + ], + "name": "vm-test-tumbleweed-07", + "namespace": "testreqlim", + "resourceVersion": "543653381", + "selfLink": "/apis/kubevirt.io/v1/namespaces/testreqlim/virtualmachines/vm-test-tumbleweed-07", + "uid": "2654b6fe-f230-4d18-8d80-c5d540ce360f" + }, + "spec": { + "running": true, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "kubevirt.io/vm": "vm-test-tumbleweed-07" + } + }, + "spec": { + "domain": { + "devices": { + "disks": [ + { + "disk": { + "bus": "virtio" + }, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea" + }, + { + "disk": { + "bus": "virtio" + }, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4" + } + ], + "interfaces": [ + { + "bridge": {}, + "name": "default" + } + ], + "networkInterfaceMultiqueue": false + }, + "machine": { + "type": "q35" + }, + "resources": { + "limits": { + "cpu": "2", + "memory": "4Gi" + }, + "requests": { + "cpu": "1m", + "memory": "1Mi" + } + } + }, + "networks": [ + { + "name": "default", + "pod": {} + } + ], + "terminationGracePeriodSeconds": 0, + "volumes": [ + { + "containerDisk": { + "image": "registry.internal.com/kubevirt-vmidisks/tumbleweed:240531" + }, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea" + }, + { + "cloudInitNoCloud": { + "userData": "#cloud-config\nssh_authorized_keys:\n - ssh-rsa FAKEKEY foo@bar.com\n" + }, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4" + } + ] + } + } + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2024-06-18T08:34:48Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": null, + "message": "cannot migrate VMI which does not use masquerade to connect to the pod network", + "reason": "InterfaceNotLiveMigratable", + "status": "False", + "type": "LiveMigratable" + }, + { + "lastProbeTime": "2024-06-18T08:35:18Z", + "lastTransitionTime": null, + "status": "True", + "type": "AgentConnected" + } + ], + "created": true, + "printableStatus": "Running", + "ready": true, + "volumeSnapshotStatuses": [ + { + "enabled": false, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea", + "reason": "Snapshot is not supported for this volumeSource type [boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea]" + }, + { + "enabled": false, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4", + "reason": "Snapshot is not supported for this volumeSource type [auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4]" + } + ] + } + } + ], + "apiVersion": "kubevirt.io/v1alpha3", + "metadata": { + "continue": "", + "selfLink": "/apis/kubevirt.io/v1alpha3/namespaces/default/virtualmachines", + "resourceVersion": "285618" + }, + "kind": "VirtualMachineList" +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/kubevirt/get_vm_test_tumbleweed_07.json b/libcloud/test/compute/fixtures/kubevirt/get_vm_test_tumbleweed_07.json new file mode 100644 index 0000000000..2f23647387 --- /dev/null +++ b/libcloud/test/compute/fixtures/kubevirt/get_vm_test_tumbleweed_07.json @@ -0,0 +1,218 @@ +{ + "apiVersion": "kubevirt.io/v1alpha3", + "kind": "VirtualMachine", + "metadata": { + "annotations": { + "kubevirt.io/latest-observed-api-version": "v1", + "kubevirt.io/storage-observed-api-version": "v1alpha3" + }, + "creationTimestamp": "2024-06-18T08:48:27Z", + "generation": 2, + "labels": { + "kubevirt.io/vm": "vm-test-tumbleweed-07" + }, + "managedFields": [ + { + "apiVersion": "kubevirt.io/v1alpha3", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:labels": { + ".": {}, + "f:kubevirt.io/vm": {} + } + }, + "f:spec": { + ".": {}, + "f:running": {}, + "f:template": { + ".": {}, + "f:metadata": { + ".": {}, + "f:labels": { + ".": {}, + "f:kubevirt.io/vm": {} + } + }, + "f:spec": { + ".": {}, + "f:domain": { + ".": {}, + "f:devices": { + ".": {}, + "f:disks": {}, + "f:interfaces": {}, + "f:networkInterfaceMultiqueue": {} + }, + "f:machine": { + ".": {}, + "f:type": {} + }, + "f:resources": { + ".": {}, + "f:limits": { + ".": {}, + "f:cpu": {}, + "f:memory": {} + }, + "f:requests": { + ".": {}, + "f:cpu": {}, + "f:memory": {} + } + } + }, + "f:networks": {}, + "f:terminationGracePeriodSeconds": {}, + "f:volumes": {} + } + } + } + }, + "manager": "libcloud", + "operation": "Update", + "time": "2024-06-18T08:48:28Z" + }, + { + "apiVersion": "kubevirt.io/v1alpha3", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:kubevirt.io/latest-observed-api-version": {}, + "f:kubevirt.io/storage-observed-api-version": {} + } + }, + "f:status": { + ".": {}, + "f:conditions": {}, + "f:created": {}, + "f:printableStatus": {}, + "f:ready": {}, + "f:volumeSnapshotStatuses": {} + } + }, + "manager": "Go-http-client", + "operation": "Update", + "time": "2024-06-18T08:49:04Z" + } + ], + "name": "vm-test-tumbleweed-07", + "namespace": "testreqlim", + "resourceVersion": "543653381", + "selfLink": "/apis/kubevirt.io/v1/namespaces/testreqlim/virtualmachines/vm-test-tumbleweed-07", + "uid": "2654b6fe-f230-4d18-8d80-c5d540ce360f" + }, + "spec": { + "running": true, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "kubevirt.io/vm": "vm-test-tumbleweed-07" + } + }, + "spec": { + "domain": { + "devices": { + "disks": [ + { + "disk": { + "bus": "virtio" + }, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea" + }, + { + "disk": { + "bus": "virtio" + }, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4" + } + ], + "interfaces": [ + { + "bridge": {}, + "name": "default" + } + ], + "networkInterfaceMultiqueue": false + }, + "machine": { + "type": "q35" + }, + "resources": { + "limits": { + "cpu": "2", + "memory": "4Gi" + }, + "requests": { + "cpu": "1m", + "memory": "1Mi" + } + } + }, + "networks": [ + { + "name": "default", + "pod": {} + } + ], + "terminationGracePeriodSeconds": 0, + "volumes": [ + { + "containerDisk": { + "image": "registry.internal.com/kubevirt-vmidisks/tumbleweed:240531" + }, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea" + }, + { + "cloudInitNoCloud": { + "userData": "#cloud-config\nssh_authorized_keys:\n - ssh-rsa FAKEKEY foo@bar.com\n" + }, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4" + } + ] + } + } + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2024-06-18T08:34:48Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": null, + "message": "cannot migrate VMI which does not use masquerade to connect to the pod network", + "reason": "InterfaceNotLiveMigratable", + "status": "False", + "type": "LiveMigratable" + }, + { + "lastProbeTime": "2024-06-18T08:35:18Z", + "lastTransitionTime": null, + "status": "True", + "type": "AgentConnected" + } + ], + "created": true, + "printableStatus": "Running", + "ready": true, + "volumeSnapshotStatuses": [ + { + "enabled": false, + "name": "boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea", + "reason": "Snapshot is not supported for this volumeSource type [boot-disk-36c21c0b-e7dc-4ec6-b997-ac1f987e7cea]" + }, + { + "enabled": false, + "name": "auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4", + "reason": "Snapshot is not supported for this volumeSource type [auth-cloudinit-9303dc4a-2464-410a-b7ed-deba78cff2e4]" + } + ] + } +} \ No newline at end of file diff --git a/libcloud/test/compute/test_kubevirt.py b/libcloud/test/compute/test_kubevirt.py index d8cbee715f..184e84fa10 100644 --- a/libcloud/test/compute/test_kubevirt.py +++ b/libcloud/test/compute/test_kubevirt.py @@ -17,7 +17,7 @@ from libcloud.test import MockHttp, unittest from libcloud.utils.py3 import httplib -from libcloud.compute.base import NodeAuthPassword +from libcloud.compute.base import NodeLocation, NodeAuthSSHKey, NodeAuthPassword from libcloud.compute.types import NodeState from libcloud.test.file_fixtures import ComputeFileFixtures from libcloud.compute.drivers.kubevirt import ( @@ -42,7 +42,7 @@ def setUp(self): def test_list_locations(self): locations = self.driver.list_locations() - self.assertEqual(len(locations), 5) + self.assertEqual(len(locations), 6) self.assertEqual(locations[0].name, "default") self.assertEqual(locations[1].name, "kube-node-lease") self.assertEqual(locations[2].name, "kube-public") @@ -54,7 +54,7 @@ def test_list_locations(self): self.assertEqual(id4, "e6d3d7e8-0ee5-428b-8e17-5187779e5627") def test_list_nodes(self): - nodes = self.driver.list_nodes() + nodes = self.driver.list_nodes(location="default") id0 = "74fd7665-fbd6-4565-977c-96bd21fb785a" self.assertEqual(len(nodes), 1) @@ -250,6 +250,37 @@ def test_create_node_ex_template(self): else: self.fail("Expected ValueError") + def test_create_node_req_lim(self): + node = self.driver.create_node( + name="vm-test-tumbleweed-07", + size=KubeVirtNodeSize( + cpu=2, + ram=4096, + cpu_request="1m", + ram_request=1, + ), + image=KubeVirtNodeImage( + name="registry.internal.com/kubevirt-vmidisks/tumbleweed:240531" + ), + location=NodeLocation( + id="5341e71d-e8d8-4a1b-a97b-52864eb3dd7d", + name="testreqlim", + country="", + driver=self.driver, + ), + auth=NodeAuthSSHKey("ssh-rsa FAKEKEY foo@bar.com"), + ex_network={ + "network_type": "pod", + "interface": "bridge", + "name": "default", + }, + ) + self.assertEqual(node.name, "vm-test-tumbleweed-07") + self.assertEqual(node.size.extra["cpu"], 2) + self.assertEqual(node.size.extra["cpu_request"], "1m") + self.assertEqual(node.size.ram, 4096) + self.assertEqual(node.size.extra["ram_request"], 1) + def test_memory_in_MB(self): self.assertEqual(_memory_in_MB("128Mi"), 128) self.assertEqual(_memory_in_MB("128M"), 128) @@ -353,6 +384,7 @@ class KubeVirtMockHttp(MockHttp): fixtures = ComputeFileFixtures("kubevirt") did_create_vm = False + did_create_vm_test_size_req_lim = False def _api_v1_namespaces(self, method, url, body, headers): if method == "GET": @@ -501,6 +533,39 @@ def _api_v1_namespaces_default_persistentvolumeclaims(self, method, url, body, h return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _apis_kubevirt_io_v1alpha3_namespaces_testreqlim_virtualmachines( + self, method, url, body, headers + ): + if method == "GET": + if self.did_create_vm_test_size_req_lim: + body = self.fixtures.load("get_testreqlim_vms_after_create_vm.json") + else: + body = self.fixtures.load("get_default_vms.json") + resp = httplib.OK + elif method == "POST": + body = self.fixtures.load("create_vm_reqlim.json") + resp = httplib.CREATED + self.did_create_vm_test_size_req_lim = True + else: + AssertionError("Unsupported method") + return (resp, body, {}, httplib.responses[httplib.OK]) + + def _api_v1_namespaces_testreqlim_services(self, method, url, body, headers): + return self._api_v1_namespaces_default_services(method, url, body, headers) + + def _api_v1_namespaces_testreqlim_pods(self, method, url, body, headers): + return self._api_v1_namespaces_default_pods(method, url, body, headers) + + def _apis_kubevirt_io_v1alpha3_namespaces_testreqlim_virtualmachines_vm_test_tumbleweed_07( + self, method, url, body, headers + ): + if method == "GET": + body = self.fixtures.load("get_vm_test_tumbleweed_07.json") + else: + AssertionError("Unsupported method") + + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + if __name__ == "__main__": sys.exit(unittest.main()) From 7d7c1021970db7ef8d7deba4305822cb33b204ce Mon Sep 17 00:00:00 2001 From: CDFMLR Date: Wed, 19 Jun 2024 10:02:11 +0800 Subject: [PATCH 13/13] fix(KubeVirtNodeDriver): prevent possible YAML injections --- libcloud/compute/drivers/kubevirt.py | 6 ++- libcloud/test/compute/test_kubevirt.py | 70 +++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index dda912bca3..c24ea21187 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -618,12 +618,14 @@ def _create_node_auth(vm, auth): # cloud_init_config reference: https://kubevirt.io/user-guide/virtual_machines/startup_scripts/#injecting-ssh-keys-with-cloud-inits-cloud-config if isinstance(auth, NodeAuthSSHKey): - public_key = auth.pubkey + public_key = auth.pubkey.strip() + public_key = json.dumps(public_key) cloud_init_config = ( """#cloud-config\n""" """ssh_authorized_keys:\n""" """ - {}\n""" ).format(public_key) elif isinstance(auth, NodeAuthPassword): - password = auth.password + password = auth.password.strip() + password = json.dumps(password) cloud_init_config = ( """#cloud-config\n""" """password: {}\n""" diff --git a/libcloud/test/compute/test_kubevirt.py b/libcloud/test/compute/test_kubevirt.py index 184e84fa10..d983cb17d6 100644 --- a/libcloud/test/compute/test_kubevirt.py +++ b/libcloud/test/compute/test_kubevirt.py @@ -12,8 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import sys +import copy +import json from libcloud.test import MockHttp, unittest from libcloud.utils.py3 import httplib @@ -379,6 +380,73 @@ def test_deep_merge_dict(self): } self.assertEqual(_deep_merge_dict(a, b), expected_result) + def test_create_node_auth(self): + mock_vm = { + "spec": {"template": {"spec": {"domain": {"devices": {"disks": []}}, "volumes": []}}} + } + cases = [ + NodeAuthPassword("password"), + NodeAuthSSHKey("ssh-rsa FAKEKEY foo@bar.com"), + NodeAuthPassword("bad\npassword\nwith\nnew\nline"), + NodeAuthPassword("bad\npassword\n\fwith\tnot\b\b\b printable\a\n\rcharacters\b\n"), + NodeAuthPassword("bad\npassword\nwith\n\"double\" and 'single' quotes"), + NodeAuthSSHKey("ssh-rsa bad\nkey\nwith new line injected hacker@yaml.security"), + NodeAuthSSHKey( + "ssh-rsa bad\n\akey\b\b\b\nwith many\a \"injected' chars hacker@yaml.security" + ), + ] + for a in cases: + try: + vm = copy.deepcopy(mock_vm) + self.driver._create_node_auth(vm, a) + user_data = vm["spec"]["template"]["spec"]["volumes"][0]["cloudInitNoCloud"][ + "userData" + ] + self.assertTrue(isinstance(user_data, str)) + # 1. make sure there are no newlines escaped + if isinstance(a, NodeAuthSSHKey): + # >>> public_key = "ssh-rsa FAKEKEY foo@bar.com" + # >>> a = ( + # ... """#cloud-config\n""" """ssh_authorized_keys:\n""" """ - {}\n""" + # ... ).format(public_key) + # >>> len(a.splitlines()) + # 3 + self.assertEqual(len(user_data.splitlines()), 3) + elif isinstance(a, NodeAuthPassword): + # >>> password = "password" + # >>> a = ( + # ... """#cloud-config\n""" + # ... """password: {}\n""" + # ... """chpasswd: {{ expire: False }}\n""" + # ... """ssh_pwauth: True\n""" + # ... ).format(password) + # >>> len(a.splitlines()) + # 4 + self.assertEqual(len(user_data.splitlines()), 4) + # 2. check if the quotes are well-escaped + for line in user_data.splitlines(): + key = "" # public key or password + if line.startswith(" - "): + key = line[4:] + self.assertEqual(key, json.dumps(a.pubkey.strip())) + elif line.startswith("password: "): + key = line[10:] + self.assertEqual(key, json.dumps(a.password.strip())) + else: + continue + self.assertTrue(key.startswith('"')) + self.assertTrue(key.endswith('"')) + # all double quotes inside must be escaped + for i, c in enumerate(key[1:-1]): + if c == '"': + # since enumerate starts from key[1:], + # so c is key[i+1], + # and key[i] is the previous char + self.assertEqual(key[i], "\\") + + except Exception as e: + self.fail(f"Failed to create node auth for {a}: {e}") + class KubeVirtMockHttp(MockHttp): fixtures = ComputeFileFixtures("kubevirt")