From c6c13f05bf8199691a168b668e1cac5a1ac7bc4a Mon Sep 17 00:00:00 2001 From: wukent Date: Sat, 13 Jul 2019 19:35:57 +0000 Subject: [PATCH] openstack-crowbar: Add pci passthough option Add Jenkins parameter 'want_pci_passthrough' to allow running tests to validate pci passthough. To test pci passthrough locally with mkcloud, simply just export 'want_pci_passthrouh=1' and append 'rebootcloud testpcipassthru' to 'plain' step, e.g. your_mkcloud.sh plain rebootcloud testpcipassthru The rebootcloud step will ensure intel_iommu=on kernel param enabled. The testpcipassthru step will prepare the environment and spin up a nested vm (L2) on compute (L1) with a virtio disk passing from compute vm (L1). Noted that this is not passing any device from the host (L0) itself. There will be another PR to construct job definition and other prep works for new ci. 1. add environment prep and verify function in qa_crowbarsetup.sh 2. create legacy ci job template 3. introduce testpcipassthru step in mkcloud --- ...cloud-mkcloud-job-pcipassthru-template.yml | 28 ++ scripts/mkcloud | 12 +- scripts/qa_crowbarsetup.sh | 277 +++++++++++++++++- 3 files changed, 315 insertions(+), 2 deletions(-) create mode 100644 jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml diff --git a/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml new file mode 100644 index 0000000000..5d53f391ea --- /dev/null +++ b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml @@ -0,0 +1,28 @@ +- job-template: + name: 'cloud-mkcloud{version}-job-pcipassthru-{arch}' + node: cloud-trigger + disabled: '{obj:disabled}' + + triggers: + - timed: 'H 20 * * *' + + logrotate: + numToKeep: -1 + daysToKeep: 7 + + builders: + - trigger-builds: + - project: openstack-mkcloud + condition: SUCCESS + block: true + current-parameters: true + predefined-parameters: | + TESTHEAD=1 + cloudsource=develcloud{version} + libvirt_type=kvm + nodenumber=2 + mkcloudtarget=all testpcipassthru + label={label} + custom_settings='export want_pci_passthrough=1' + job_name=cloud-mkcloud{version}-job-pcipassthru-{arch} + diff --git a/scripts/mkcloud b/scripts/mkcloud index f1e009e5a9..cb4e04058a 100755 --- a/scripts/mkcloud +++ b/scripts/mkcloud @@ -1029,6 +1029,14 @@ function rebootcrowbar return $? } +# Need to run rebootcloud step to enable iommu (e.g., plain rebootcloud +# testpcipassthru) +function testpcipassthru +{ + onadmin verify_pci_passthru + return $? +} + function rebootcloud { # reboot compute nodes @@ -1455,6 +1463,8 @@ Optional If set, does not use the --insecure flag in openstack CLI commands. want_monasca_tsdb (Cloud9+ only) Allows setting time-series DB used for Monasca [influxdb|cassandra]. + want_pci_passthrough (default=0) + Deploy node with an extra disk to test pci passthrough. Cloud9+ only. want_ipv6 (Currently in development) Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set the firmware_type to UEFI as it's required to netboot nodes over IPv6. @@ -1619,7 +1629,7 @@ allcmds="$step_aliases _test_setuphost \ lonelynode_nfs_server setupironicnodes\ restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \ cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \ - testpreupgrade testpostupgrade deployexternalceph" + testpreupgrade testpostupgrade deployexternalceph testpcipassthru" wantedcmds=$@ function expand_steps diff --git a/scripts/qa_crowbarsetup.sh b/scripts/qa_crowbarsetup.sh index a8d0bad121..e777f32efd 100644 --- a/scripts/qa_crowbarsetup.sh +++ b/scripts/qa_crowbarsetup.sh @@ -57,10 +57,12 @@ fi # global variables that are set within this script novacontroller= +novacompute_kvm= horizonserver= horizonservice= manila_service_vm_uuid= manila_tenant_vm_ip= +pci_passthru_vm_name="pci-passthru-instance" clusternodesdrbd= clusternodesdata= clusternodesnetwork= @@ -3315,6 +3317,45 @@ function onadmin_proposal done set_dashboard_alias + + # prepare env to test pci_passthrough if condition met + prepare_pci_passthru_env +} + +function prepare_pci_passthru_env +{ + if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then + # Place nova configuration to verify pci_passthru + get_novacontroller + safely oncontroller create_pci_passthru_conf + append_pci_passthru_filter + get_novacompute_kvm + safely oncompute create_pci_passthru_conf + safely oncompute load_vfio_module_onboot + safely oncompute bind_vfio_module_onboot + # TODO: need support for amd param + safely oncompute add_intel_iommu_param + fi +} + +function onadmin_verify_pci_passthru +{ + # spin up vm and pass device from compute host to vm to verify pci + # passthru functionality. + local cirros_image_name=cirros-0.4.0-x86_64-disk.img + local cirros_image_url=$imageserver_url/$arch/openstack/$cirros_image_name + get_novacontroller + safely oncontroller retrieve_image $cirros_image_name $cirros_image_url + safely oncontroller upload_image_to_glance "cirros-0.4.0-x86_64-disk" $cirros_image_name + safely oncontroller setup_pci_passthru_vm "cirros-0.4.0-x86_64-disk" + + # Get instance name (e.g.,instance-00000001) + pci_passthru_instance_name=$(ssh "$novacontroller" "source /root/.openrc; + openstack server show -c OS-EXT-SRV-ATTR:instance_name -f value $pci_passthru_vm_name") + get_novacompute_kvm + safely oncompute unload_pci_device "$pci_passthru_instance_name" + safely oncompute reload_pci_device "$pci_passthru_instance_name" + safely oncompute parse_vm_xml "$pci_passthru_instance_name" } function set_node_alias @@ -3417,6 +3458,15 @@ function get_novacontroller novacontroller=`resolve_element_to_hostname "$element"` } +function get_novacompute_kvm +{ + local element=`crowbar nova proposal show default | \ + rubyjsonparse " + puts j['deployment']['nova']\ + ['elements']['nova-compute-kvm']"` + novacompute_kvm=`resolve_element_to_hostname "$element"` +} + function get_horizon { local element=`crowbar horizon proposal show default | \ @@ -3860,6 +3910,131 @@ function oncontroller_heat_image_setup() fi } +function oncompute_parse_vm_xml +{ + local instance_name=$1 + echo $instance_name + virsh list --all | grep $instance_name + [ $? != 0 ] && complain 53 "failed to find pci-passthru-instance" + + # verify node xml to see if it has pci passthru section which has the + # specific address passing from compute host + virsh dumpxml $instance_name | grep -E "&1 >/dev/null) + if [[ $ret =~ "200 OK" ]]; then + echo $ret + elif [[ $ret =~ "Not Found" ]]; then + complain 73 "$image_name image not found: $ret" + else + complain 74 "failed to retrieve $image_name image: $ret" + fi + fi +} + +function oncontroller_upload_image_to_glance +{ + local image_name=$1 + local image_path=$2 + shift ; shift + local image_params=$@ + + if [ -z "$image_params" ]; then + image_params="--disk-format qcow2 --property hypervisor_type=kvm" + fi + + . .openrc + + # using list subcommand because show requires an ID + if ! openstack image list --format value -c Name | grep -q "$image_name"; then + openstack image create --file $image_path \ + $image_params --container-format bare --public \ + $image_name + fi +} + +function oncontroller_setup_pci_passthru_vm +{ + local image_name=$1 + local sec_group="pci-passthru" + + . .openrc + + # create flavor with pci-passthru property so that the vm using this flavor + # will aquire the pci device passing from the host (compute) + openstack flavor create --id 200 --ram 512 --ephemeral 0 \ + --vcpus 1 --property "pci_passthrough:alias"="a1:1" \ + pci-passthru-flavor + + if iscloudver 9plus && ! openstack security group show $sec_group 2>/dev/null ; then + openstack security group create --description "$sec_group description" $sec_group + openstack security group rule create --protocol icmp $sec_group + openstack security group rule create --protocol tcp --dst-port 22 $sec_group + fi + + fixed_net_id=`neutron net-show fixed -f value -c id` + timeout 10m openstack server create --flavor 200 \ + --image $image_name \ + --security-group $sec_group \ + --nic net-id=$fixed_net_id $pci_passthru_vm_name + + [ $? != 0 ] && complain 43 "nova boot $pci_passthru_vm_name failed" + + # Check status of the vm + wait_for 60 1 "openstack server show -c status -f value $pci_passthru_vm_name | grep '^ACTIVE$'" \ + "pci passthru VM booted and is in ACTIVE state" \ + "echo \"ERROR: pci passthru VM is not in ACTIVE state.\"" + + local floatingip=$(addfloatingip $pci_passthru_vm_name) + [ -z "$floatingip" ] && complain 44 "adding a floating ip to $pci_passthru_vm_name VM failed" +} + +function oncompute_unload_pci_device +{ + local vm_domain=$1 + local pci_nodedev=$(virsh nodedev-list --cap pci|grep 00_1d) + if [ -z "$pci_nodedev" ] ; then + pci_nodedev="pci_0000_00_1d_0" + fi + virt-xml $vm_domain --remove-device --update --hostdev $pci_nodedev + + # avoid nova to revert vm state for the change on pci device + systemctl stop openstack-nova-compute + + # make change effect + virsh destroy $vm_domain + virsh start $vm_domain +} + +function oncompute_reload_pci_device +{ + local vm_domain=$1 + local pci_nodedev=$(virsh nodedev-list --cap pci|grep 00_1d) + if [ -z "$pci_nodedev" ] ; then + pci_nodedev="pci_0000_00_1d_0" + fi + virt-xml $vm_domain --add-device --update --hostdev $pci_nodedev + + systemctl start openstack-nova-compute + # $vm_domain shall be pingable from now on +} + function oncontroller_manila_generic_driver_setup() { if [[ $wantxenpv ]] ; then @@ -4478,6 +4653,107 @@ function oncontroller run_on "$novacontroller" "oncontroller_$func $@" } +function oncompute +{ + local func=$1 ; shift + run_on "$novacompute_kvm" "oncompute_$func $@" +} + +function oncompute_add_intel_iommu_param +{ + cat /etc/default/grub | grep iommu >/dev/null 2>&1 + if [ $? != 0 ]; then + # enabled intel_iommu in kernel param + sed -in "s/\(^GRUB_CMDLINE_LINUX_DEFAULT=.*\)\"$/\1 intel_iommu=on\"/g" /etc/default/grub + update-bootloader + fi +} + +function oncompute_load_vfio_module_onboot +{ + if [ ! -e "/etc/modules-load.d/pci-passthru.conf" ]; then + # vfio-pci module is not loaded by default, however, we need it to bind to + # our candidate device to verify pci passthru + cat > /etc/modules-load.d/pci-passthru.conf < /etc/udev/rules.d/99-pci-passthru.rules < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf </dev/null 2>&1 + if [ $? != 0 ]; then + enabled_filters="${enabled_filters},PciPassthroughFilter" + + # append PciPassthroughFilter to enabled_filters + proposal_set_value nova default \ + "['attributes']['nova']['scheduler']['enabled_filters']" \ + "'$enabled_filters'" + + crowbar nova proposal --file=$pfile edit default ||\ + complain 88 "'crowbar nova proposal --file=$pfile edit default' failed with exit code: $?" + + crowbar_proposal_commit nova + + # Make the changes effective right away + run_on "$novacontroller" 'sed -in "s/\(enabled_filters = .*\)$/\1,PciPassthroughFilter/g"' $nova_conf + run_on "$novacontroller" "systemctl restart openstack-nova-scheduler.service" + fi +} + function install_suse_ca { # trust build key - workaround https://bugzilla.opensuse.org/show_bug.cgi?id=935020 @@ -4832,7 +5108,6 @@ function onadmin_testsetup if iscloudver 7plus; then update_keystone_password fi - wantcephtestsuite=0 if [[ $deployceph ]]; then get_ceph_nodes