diff --git a/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml new file mode 100644 index 0000000000..5d53f391ea --- /dev/null +++ b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml @@ -0,0 +1,28 @@ +- job-template: + name: 'cloud-mkcloud{version}-job-pcipassthru-{arch}' + node: cloud-trigger + disabled: '{obj:disabled}' + + triggers: + - timed: 'H 20 * * *' + + logrotate: + numToKeep: -1 + daysToKeep: 7 + + builders: + - trigger-builds: + - project: openstack-mkcloud + condition: SUCCESS + block: true + current-parameters: true + predefined-parameters: | + TESTHEAD=1 + cloudsource=develcloud{version} + libvirt_type=kvm + nodenumber=2 + mkcloudtarget=all testpcipassthru + label={label} + custom_settings='export want_pci_passthrough=1' + job_name=cloud-mkcloud{version}-job-pcipassthru-{arch} + diff --git a/scripts/mkcloud b/scripts/mkcloud index f1e009e5a9..cb4e04058a 100755 --- a/scripts/mkcloud +++ b/scripts/mkcloud @@ -1029,6 +1029,14 @@ function rebootcrowbar return $? } +# Need to run rebootcloud step to enable iommu (e.g., plain rebootcloud +# testpcipassthru) +function testpcipassthru +{ + onadmin verify_pci_passthru + return $? +} + function rebootcloud { # reboot compute nodes @@ -1455,6 +1463,8 @@ Optional If set, does not use the --insecure flag in openstack CLI commands. want_monasca_tsdb (Cloud9+ only) Allows setting time-series DB used for Monasca [influxdb|cassandra]. + want_pci_passthrough (default=0) + Deploy node with an extra disk to test pci passthrough. Cloud9+ only. want_ipv6 (Currently in development) Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set the firmware_type to UEFI as it's required to netboot nodes over IPv6. @@ -1619,7 +1629,7 @@ allcmds="$step_aliases _test_setuphost \ lonelynode_nfs_server setupironicnodes\ restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \ cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \ - testpreupgrade testpostupgrade deployexternalceph" + testpreupgrade testpostupgrade deployexternalceph testpcipassthru" wantedcmds=$@ function expand_steps diff --git a/scripts/qa_crowbarsetup.sh b/scripts/qa_crowbarsetup.sh index a8d0bad121..e777f32efd 100644 --- a/scripts/qa_crowbarsetup.sh +++ b/scripts/qa_crowbarsetup.sh @@ -57,10 +57,12 @@ fi # global variables that are set within this script novacontroller= +novacompute_kvm= horizonserver= horizonservice= manila_service_vm_uuid= manila_tenant_vm_ip= +pci_passthru_vm_name="pci-passthru-instance" clusternodesdrbd= clusternodesdata= clusternodesnetwork= @@ -3315,6 +3317,45 @@ function onadmin_proposal done set_dashboard_alias + + # prepare env to test pci_passthrough if condition met + prepare_pci_passthru_env +} + +function prepare_pci_passthru_env +{ + if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then + # Place nova configuration to verify pci_passthru + get_novacontroller + safely oncontroller create_pci_passthru_conf + append_pci_passthru_filter + get_novacompute_kvm + safely oncompute create_pci_passthru_conf + safely oncompute load_vfio_module_onboot + safely oncompute bind_vfio_module_onboot + # TODO: need support for amd param + safely oncompute add_intel_iommu_param + fi +} + +function onadmin_verify_pci_passthru +{ + # spin up vm and pass device from compute host to vm to verify pci + # passthru functionality. + local cirros_image_name=cirros-0.4.0-x86_64-disk.img + local cirros_image_url=$imageserver_url/$arch/openstack/$cirros_image_name + get_novacontroller + safely oncontroller retrieve_image $cirros_image_name $cirros_image_url + safely oncontroller upload_image_to_glance "cirros-0.4.0-x86_64-disk" $cirros_image_name + safely oncontroller setup_pci_passthru_vm "cirros-0.4.0-x86_64-disk" + + # Get instance name (e.g.,instance-00000001) + pci_passthru_instance_name=$(ssh "$novacontroller" "source /root/.openrc; + openstack server show -c OS-EXT-SRV-ATTR:instance_name -f value $pci_passthru_vm_name") + get_novacompute_kvm + safely oncompute unload_pci_device "$pci_passthru_instance_name" + safely oncompute reload_pci_device "$pci_passthru_instance_name" + safely oncompute parse_vm_xml "$pci_passthru_instance_name" } function set_node_alias @@ -3417,6 +3458,15 @@ function get_novacontroller novacontroller=`resolve_element_to_hostname "$element"` } +function get_novacompute_kvm +{ + local element=`crowbar nova proposal show default | \ + rubyjsonparse " + puts j['deployment']['nova']\ + ['elements']['nova-compute-kvm']"` + novacompute_kvm=`resolve_element_to_hostname "$element"` +} + function get_horizon { local element=`crowbar horizon proposal show default | \ @@ -3860,6 +3910,131 @@ function oncontroller_heat_image_setup() fi } +function oncompute_parse_vm_xml +{ + local instance_name=$1 + echo $instance_name + virsh list --all | grep $instance_name + [ $? != 0 ] && complain 53 "failed to find pci-passthru-instance" + + # verify node xml to see if it has pci passthru section which has the + # specific address passing from compute host + virsh dumpxml $instance_name | grep -E "&1 >/dev/null) + if [[ $ret =~ "200 OK" ]]; then + echo $ret + elif [[ $ret =~ "Not Found" ]]; then + complain 73 "$image_name image not found: $ret" + else + complain 74 "failed to retrieve $image_name image: $ret" + fi + fi +} + +function oncontroller_upload_image_to_glance +{ + local image_name=$1 + local image_path=$2 + shift ; shift + local image_params=$@ + + if [ -z "$image_params" ]; then + image_params="--disk-format qcow2 --property hypervisor_type=kvm" + fi + + . .openrc + + # using list subcommand because show requires an ID + if ! openstack image list --format value -c Name | grep -q "$image_name"; then + openstack image create --file $image_path \ + $image_params --container-format bare --public \ + $image_name + fi +} + +function oncontroller_setup_pci_passthru_vm +{ + local image_name=$1 + local sec_group="pci-passthru" + + . .openrc + + # create flavor with pci-passthru property so that the vm using this flavor + # will aquire the pci device passing from the host (compute) + openstack flavor create --id 200 --ram 512 --ephemeral 0 \ + --vcpus 1 --property "pci_passthrough:alias"="a1:1" \ + pci-passthru-flavor + + if iscloudver 9plus && ! openstack security group show $sec_group 2>/dev/null ; then + openstack security group create --description "$sec_group description" $sec_group + openstack security group rule create --protocol icmp $sec_group + openstack security group rule create --protocol tcp --dst-port 22 $sec_group + fi + + fixed_net_id=`neutron net-show fixed -f value -c id` + timeout 10m openstack server create --flavor 200 \ + --image $image_name \ + --security-group $sec_group \ + --nic net-id=$fixed_net_id $pci_passthru_vm_name + + [ $? != 0 ] && complain 43 "nova boot $pci_passthru_vm_name failed" + + # Check status of the vm + wait_for 60 1 "openstack server show -c status -f value $pci_passthru_vm_name | grep '^ACTIVE$'" \ + "pci passthru VM booted and is in ACTIVE state" \ + "echo \"ERROR: pci passthru VM is not in ACTIVE state.\"" + + local floatingip=$(addfloatingip $pci_passthru_vm_name) + [ -z "$floatingip" ] && complain 44 "adding a floating ip to $pci_passthru_vm_name VM failed" +} + +function oncompute_unload_pci_device +{ + local vm_domain=$1 + local pci_nodedev=$(virsh nodedev-list --cap pci|grep 00_1d) + if [ -z "$pci_nodedev" ] ; then + pci_nodedev="pci_0000_00_1d_0" + fi + virt-xml $vm_domain --remove-device --update --hostdev $pci_nodedev + + # avoid nova to revert vm state for the change on pci device + systemctl stop openstack-nova-compute + + # make change effect + virsh destroy $vm_domain + virsh start $vm_domain +} + +function oncompute_reload_pci_device +{ + local vm_domain=$1 + local pci_nodedev=$(virsh nodedev-list --cap pci|grep 00_1d) + if [ -z "$pci_nodedev" ] ; then + pci_nodedev="pci_0000_00_1d_0" + fi + virt-xml $vm_domain --add-device --update --hostdev $pci_nodedev + + systemctl start openstack-nova-compute + # $vm_domain shall be pingable from now on +} + function oncontroller_manila_generic_driver_setup() { if [[ $wantxenpv ]] ; then @@ -4478,6 +4653,107 @@ function oncontroller run_on "$novacontroller" "oncontroller_$func $@" } +function oncompute +{ + local func=$1 ; shift + run_on "$novacompute_kvm" "oncompute_$func $@" +} + +function oncompute_add_intel_iommu_param +{ + cat /etc/default/grub | grep iommu >/dev/null 2>&1 + if [ $? != 0 ]; then + # enabled intel_iommu in kernel param + sed -in "s/\(^GRUB_CMDLINE_LINUX_DEFAULT=.*\)\"$/\1 intel_iommu=on\"/g" /etc/default/grub + update-bootloader + fi +} + +function oncompute_load_vfio_module_onboot +{ + if [ ! -e "/etc/modules-load.d/pci-passthru.conf" ]; then + # vfio-pci module is not loaded by default, however, we need it to bind to + # our candidate device to verify pci passthru + cat > /etc/modules-load.d/pci-passthru.conf < /etc/udev/rules.d/99-pci-passthru.rules < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf </dev/null 2>&1 + if [ $? != 0 ]; then + enabled_filters="${enabled_filters},PciPassthroughFilter" + + # append PciPassthroughFilter to enabled_filters + proposal_set_value nova default \ + "['attributes']['nova']['scheduler']['enabled_filters']" \ + "'$enabled_filters'" + + crowbar nova proposal --file=$pfile edit default ||\ + complain 88 "'crowbar nova proposal --file=$pfile edit default' failed with exit code: $?" + + crowbar_proposal_commit nova + + # Make the changes effective right away + run_on "$novacontroller" 'sed -in "s/\(enabled_filters = .*\)$/\1,PciPassthroughFilter/g"' $nova_conf + run_on "$novacontroller" "systemctl restart openstack-nova-scheduler.service" + fi +} + function install_suse_ca { # trust build key - workaround https://bugzilla.opensuse.org/show_bug.cgi?id=935020 @@ -4832,7 +5108,6 @@ function onadmin_testsetup if iscloudver 7plus; then update_keystone_password fi - wantcephtestsuite=0 if [[ $deployceph ]]; then get_ceph_nodes