From cff77af16ed1a8b4d2273f4dba8d8ae98c471755 Mon Sep 17 00:00:00 2001 From: wukent Date: Sat, 13 Jul 2019 19:35:57 +0000 Subject: [PATCH] openstack-crowbar: Add pci passthough option Add Jenkins parameter 'want_pci_passthrough' to allow running tests to validate pci passthough. 1. add environment prep and verify function in qa_crowbarsetup.sh 2. create ci job template --- jenkins/ci.suse.de/openstack-mkcloud.yaml | 5 + ...cloud-mkcloud-job-pcipassthru-template.yml | 29 +++ scripts/mkcloud | 12 +- scripts/qa_crowbarsetup.sh | 205 ++++++++++++++++++ 4 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml diff --git a/jenkins/ci.suse.de/openstack-mkcloud.yaml b/jenkins/ci.suse.de/openstack-mkcloud.yaml index 738ccd7582..c7a1c6c772 100644 --- a/jenkins/ci.suse.de/openstack-mkcloud.yaml +++ b/jenkins/ci.suse.de/openstack-mkcloud.yaml @@ -401,6 +401,11 @@ default: '1' description: set to 0 to not deploy aodh + - string: + name: want_pci_passthrough + default: + description: set to 1 to deploy extra disk to test pci passthrough + ################################################ # misc diff --git a/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml new file mode 100644 index 0000000000..5a0ba0d61a --- /dev/null +++ b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml @@ -0,0 +1,29 @@ +- job-template: + name: 'cloud-mkcloud{version}-job-pcipassthru-{arch}' + node: cloud-trigger + disabled: '{obj:disabled}' + + triggers: + - timed: 'H 20 * * *' + + logrotate: + numToKeep: -1 + daysToKeep: 7 + + builders: + - trigger-builds: + - project: openstack-mkcloud + condition: SUCCESS + block: true + current-parameters: true + predefined-parameters: | + TESTHEAD=1 + cloudsource=develcloud{version} + libvirt_type=kvm + nodenumber=2 + mkcloudtarget=all + want_pci_passthrough=1 + vcpus=4 + label={label} + job_name=cloud-mkcloud{version}-job-pcipassthru-{arch} + diff --git a/scripts/mkcloud b/scripts/mkcloud index 6112481be7..63bd5964c5 100755 --- a/scripts/mkcloud +++ b/scripts/mkcloud @@ -1029,6 +1029,14 @@ function rebootcrowbar return $? } +# Need to run rebootcloud step to enable iommu (e.g., plain rebootcloud +# testpcipassthru) +function testpcipassthru +{ + onadmin verify_pci_passthru + return $? +} + function rebootcloud { # reboot compute nodes @@ -1448,6 +1456,8 @@ Optional If set, does not use the --insecure flag in openstack CLI commands. want_monasca_tsdb (Cloud9+ only) Allows setting time-series DB used for Monasca [influxdb|cassandra]. + want_pci_passthrough (default=0) + Deploy node with an extra disk to test pci passthrough. Cloud9+ only. want_ipv6 (Currently in development) Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set the firmware_type to UEFI as it's required to netboot nodes over IPv6. @@ -1612,7 +1622,7 @@ allcmds="$step_aliases _test_setuphost \ lonelynode_nfs_server setupironicnodes\ restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \ cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \ - testpreupgrade testpostupgrade deployexternalceph" + testpreupgrade testpostupgrade deployexternalceph testpcipassthru" wantedcmds=$@ function expand_steps diff --git a/scripts/qa_crowbarsetup.sh b/scripts/qa_crowbarsetup.sh index 9b4f91b73c..117aaf28a0 100644 --- a/scripts/qa_crowbarsetup.sh +++ b/scripts/qa_crowbarsetup.sh @@ -57,10 +57,12 @@ fi # global variables that are set within this script novacontroller= +novacompute_kvm= horizonserver= horizonservice= manila_service_vm_uuid= manila_tenant_vm_ip= +pci_passthru_vm_name="pci-passthru-instance" clusternodesdrbd= clusternodesdata= clusternodesnetwork= @@ -2662,6 +2664,16 @@ function custom_configuration proposal_set_value nova default "['deployment']['nova']['elements']['nova-compute-${libvirt_type}']" "[]" proposal_set_value nova default "['deployment']['nova']['elements']['nova-compute-kvm']" "[]" fi + if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then + # add PciPassthroughFilter to enabled_filters for scheduler + proposal_set_value nova default "['attributes']['nova']['scheduler']['enabled_filters']" \ +"'RetryFilter,AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter,PciPassthroughFilter'" +# local has_itxt=$(proposal_get_value nova default "['attributes']['nova']['itxt_instance']") +# if [ -z "$has_itxt" ] || [ "$has_itxt" == "none" ] ; then +# proposal_set_value nova default "['attributes']['nova']['scheduler']['enabled_filters']" \ +#"'RetryFilter,AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter,PciPassthroughFilter'" +# fi + fi ;; horizon|nova_dashboard) [[ $want_ldap = 1 ]] && iscloudver 7plus && proposal_set_value $proposal default "['attributes']['$proposal']['multi_domain_support']" "true" @@ -3309,6 +3321,36 @@ function onadmin_proposal done set_dashboard_alias + + prepare_pci_passthru_env +} + +function prepare_pci_passthru_env() +{ + if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then + # Place nova configuration to verify pci_passthru + get_novacontroller + safely oncontroller create_pci_passthru_conf + get_novacompute_kvm + safely oncompute create_pci_passthru_conf + safely oncompute load_vfio_module_onboot + safely oncompute bind_vfio_module_onboot + # TODO: need support for amd param + safely oncompute add_intel_iommu_param + fi +} + +function onadmin_verify_pci_passthru() +{ + # spin up vm and pass device from compute host to vm to verify pci + # passthru functionality. + get_novacontroller + safely oncontroller setup_pci_passthru_vm + # Get instance name (e.g.,instance-00000001) + pci_passthru_instance_name=$(ssh "$novacontroller" "source /root/.openrc; + openstack server show -c OS-EXT-SRV-ATTR:instance_name -f value $pci_passthru_vm_name") + get_novacompute_kvm + safely oncompute parse_vm_xml "$pci_passthru_instance_name" } function set_node_alias @@ -3411,6 +3453,15 @@ function get_novacontroller novacontroller=`resolve_element_to_hostname "$element"` } +function get_novacompute_kvm +{ + local element=`crowbar nova proposal show default | \ + rubyjsonparse " + puts j['deployment']['nova']\ + ['elements']['nova-compute-kvm']"` + novacompute_kvm=`resolve_element_to_hostname "$element"` +} + function get_horizon { local element=`crowbar horizon proposal show default | \ @@ -3854,6 +3905,99 @@ function oncontroller_heat_image_setup() fi } +function oncompute_parse_vm_xml() +{ + local instance_name=$1 + echo $instance_name + virsh list --all | grep $instance_name + [ $? != 0 ] && complain 53 "failed to find pci-passthru-instance" + + # verify node xml to see if it has pci passthru section which has the + # specific address passing from compute host + virsh dumpxml $instance_name | grep -E "&1 >/dev/null) + if [[ $ret =~ "200 OK" ]]; then + echo $ret + elif [[ $ret =~ "Not Found" ]]; then + complain 73 "cirros image not found: $ret" + else + complain 74 "failed to retrieve cirros image: $ret" + fi + fi + + . .openrc + + # using list subcommand because show requires an ID + if ! openstack image list --format value -c Name | grep -q "^cirros-0.4.0-x86_64-disk$"; then + openstack image create --file $cirros_image_name \ + $cirros_image_params --container-format bare --public \ + cirros-0.4.0-x86_64-disk + fi + + # create flavor with pci-passthru property so that the vm using this flavor + # will aquire the pci device passing from the host (compute) + openstack flavor create --id 200 --ram 512 --ephemeral 0 \ + --vcpus 1 --property "pci_passthrough:alias"="a1:1" \ + pci-passthru-flavor + + if iscloudver 9plus && ! openstack security group show $sec_group 2>/dev/null ; then + openstack security group create --description "$sec_group description" $sec_group + openstack security group rule create --protocol icmp $sec_group + openstack security group rule create --protocol tcp --dst-port 22 $sec_group + fi + + fixed_net_id=`neutron net-show fixed -f value -c id` + timeout 10m openstack server create --flavor 200 \ + --image cirros-0.4.0-x86_64-disk \ + --security-group $sec_group \ + --nic net-id=$fixed_net_id $pci_passthru_vm_name + + [ $? != 0 ] && complain 43 "nova boot pci-passthru-instance failed" + + # Check status of the vm + wait_for 60 1 "openstack server show -c status -f value pci-passthru-instance | grep '^ACTIVE$'" \ + "pci passthru VM booted and is in ACTIVE state" \ + "echo \"ERROR: pci passthru VM is not in ACTIVE state.\"" + +# # Create a floating IP for the instance and add it to the VM +# oscclient_ver=`rpm -q --queryformat '%{VERSION}' python-openstackclient` +# if [ ${oscclient_ver:0:1} -ge 3 ]; then +# # >= Newton +# pci_passthru_vm_ip=`openstack floating ip create floating -f value -c floating_ip_address` +# openstack server add floating ip pci-passthru-instance $pci_passthru_vm_ip +# else +# pci_passthru_vm_ip=`openstack ip floating create floating -f value -c ip` +# openstack ip floating add $pci_passthru_vm_ip pci-passthru-instance +# fi +# +# [ $? != 0 ] && complain 44 "adding a floating ip to the pci passthru VM failed" +# +# # check that the service VM is pingable. +# wait_for 300 1 "nc -w 1 -z $pci_passthru_vm_ip 22" \ +# "pci passthru VM booted and ssh port open" \ +# "echo \"ERROR: pci passthru VM not listening on ssh port.\"" +} + function oncontroller_manila_generic_driver_setup() { if [[ $wantxenpv ]] ; then @@ -4472,6 +4616,67 @@ function oncontroller run_on "$novacontroller" "oncontroller_$func $@" } +function oncompute +{ + local func=$1 ; shift + run_on "$novacompute_kvm" "oncompute_$func $@" +} + +function oncompute_add_intel_iommu_param +{ + # enabled intel_iommu in kernel param + sed -in "s/\(^GRUB_CMDLINE_LINUX_DEFAULT=.*\)\"$/\1 intel_iommu=on\"/g" /etc/default/grub + update-bootloader +} + +function oncompute_load_vfio_module_onboot +{ + # vfio-pci module is not loaded by default, however, we need it to bind to + # our candidate device to verify pci passthru + cat > /etc/modules-load.d/pci-passthru.conf < /etc/udev/rules.d/99-pci-passthru.rules < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf <