From 47cfb6cc1529a4f32141121d1c895677f0860366 Mon Sep 17 00:00:00 2001
From: wukent <kwu@suse.com>
Date: Thu, 30 May 2019 19:01:36 +0000
Subject: [PATCH] Add mocked disk device to test pci passthrough

We need to specify q35 as the machine type in order to support IOMMU and
tweaking pcie controllers to place our virtio device properly.
1. Add option for pci passthrough
2. Add script to construct node template
---
 scripts/lib/libvirt/compute-config            |  4 +-
 scripts/lib/libvirt/libvirt_setup.py          | 47 ++++++++++++++++---
 .../lib/libvirt/templates/compute-node.xml    |  3 ++
 .../templates/cpu-intel-pcipassthrough.xml    | 11 +++++
 .../templates/iommu-device-default.xml        |  3 ++
 .../templates/pcie-root-bridge-default.xml    | 14 ++++++
 scripts/lib/mkcloud-driver-libvirt.sh         | 11 +++++
 scripts/mkcloud                               | 12 +++++
 8 files changed, 98 insertions(+), 7 deletions(-)
 create mode 100644 scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml
 create mode 100644 scripts/lib/libvirt/templates/iommu-device-default.xml
 create mode 100644 scripts/lib/libvirt/templates/pcie-root-bridge-default.xml

diff --git a/scripts/lib/libvirt/compute-config b/scripts/lib/libvirt/compute-config
index 422f5ba570..ff9226ccd6 100755
--- a/scripts/lib/libvirt/compute-config
+++ b/scripts/lib/libvirt/compute-config
@@ -47,9 +47,11 @@ def main():
                         help="Local Repository Directory Target")
     parser.add_argument("--ipmi", action='store_true',
                         help="Whether to simulate IPMI BMC devices")
+    parser.add_argument("--pcipassthrough", action='store_true',
+                        help="Whether to simulate PCI passthrough devices")
     args = parser.parse_args()
 
-    print(libvirt_setup.compute_config(args, libvirt_setup.cpuflags()))
+    print(libvirt_setup.compute_config(args, libvirt_setup.cpuflags(args.pcipassthrough)))
 
 
 if __name__ == "__main__":
diff --git a/scripts/lib/libvirt/libvirt_setup.py b/scripts/lib/libvirt/libvirt_setup.py
index 122131cce2..594cf9db25 100644
--- a/scripts/lib/libvirt/libvirt_setup.py
+++ b/scripts/lib/libvirt/libvirt_setup.py
@@ -28,13 +28,13 @@ def remove_files(files):
         os.remove(f)
 
 
-def cpuflags():
+def cpuflags(pcipassthrough=False):
     cpu_template = "cpu-default.xml"
     cpu_info = readfile("/proc/cpuinfo")
     if re.search("^CPU architecture.* 8", cpu_info, re.MULTILINE):
         cpu_template = "cpu-arm64.xml"
     elif re.search("^vendor_id.*GenuineIntel", cpu_info, re.MULTILINE):
-        cpu_template = "cpu-intel.xml"
+        cpu_template = get_intel_cputemplate(pcipassthrough)
     elif re.search("^vendor_id.*AuthenticAMD", cpu_info, re.MULTILINE):
         cpu_template = "cpu-amd.xml"
     elif re.search("^vendor_id.*IBM/S390", cpu_info, re.MULTILINE):
@@ -43,6 +43,13 @@ def cpuflags():
     return readfile(os.path.join(TEMPLATE_DIR, cpu_template))
 
 
+def get_intel_cputemplate(pcipassthrough=False):
+    cpu_template = "cpu-intel.xml"
+    if pcipassthrough:
+        cpu_template = "cpu-intel-pcipassthrough.xml"
+    return cpu_template
+
+
 def hypervisor_has_virtio(libvirt_type):
     return libvirt_type == "kvm"
 
@@ -257,7 +264,7 @@ def compute_config(args, cpu_flags=cpuflags()):
             "bus='{0}' target='0' unit='0'/>"
 
     # override nic model for ironic setups
-    if args.ironicnic >= 0:
+    if args.ironicnic >= 0 and not args.pcipassthrough:
         configopts['nicmodel'] = 'e1000'
 
     controller_raid_volumes = args.controller_raid_volumes
@@ -320,7 +327,32 @@ def compute_config(args, cpu_flags=cpuflags()):
             'target_address': target_address.format('0x1f')},
             configopts))
 
-    if args.ipmi:
+    machine = ""
+    machine = get_default_machine(args.emulator)
+    pciecontrollers = ""
+    iommudevice = ""
+    extravolume = ""
+    if args.pcipassthrough and not args.drbdserial:
+        volume_template = string.Template(
+            readfile(os.path.join(TEMPLATE_DIR, "extra-volume.xml")))
+        extravolume = volume_template.substitute(merge_dicts({
+            'volume_serial': "{0}-node{1}-extra".format(
+                args.cloud,
+                args.nodecounter),
+            'source_dev': "{0}/{1}.node{2}-extra".format(
+                args.vdiskdir,
+                args.cloud,
+                args.nodecounter),
+            'target_dev': targetdevprefix + ''.join(next(alldevices)),
+            'target_address': target_address.format('0x1d')},
+            configopts))
+        iommudevice = readfile(os.path.join(
+            TEMPLATE_DIR, 'iommu-device-default.xml'))
+        machine = "q35"
+        pciecontrollers = readfile(os.path.join(
+            TEMPLATE_DIR, 'pcie-root-bridge-default.xml'))
+
+    if args.ipmi and not args.pcipassthrough:
         values = dict(
           nodecounter=args.nodecounter
         )
@@ -329,7 +361,7 @@ def compute_config(args, cpu_flags=cpuflags()):
     else:
         ipmi_config = ''
 
-    if not hypervisor_has_virtio(libvirt_type):
+    if not hypervisor_has_virtio(libvirt_type) and not args.pcipassthrough:
         target_address = target_address.format('0')
         # map virtio addr to ide:
         raidvolume = raidvolume.replace("bus='0x17'", "bus='1'")
@@ -342,13 +374,16 @@ def compute_config(args, cpu_flags=cpuflags()):
         nodememory=nodememory,
         vcpus=args.vcpus,
         march=get_machine_arch(),
-        machine=get_default_machine(args.emulator),
+        machine=machine,
         osloader=get_os_loader(firmware_type=args.firmwaretype),
         cpuflags=cpu_flags,
         consoletype=get_console_type(),
         raidvolume=raidvolume,
         cephvolume=cephvolume,
         drbdvolume=drbdvolume,
+        pciecontrollers=pciecontrollers,
+        extravolume=extravolume,
+        iommudevice=iommudevice,
         nics=net_interfaces_config(args, configopts["nicmodel"]),
         maindiskaddress=get_maindisk_address(),
         videodevices=get_video_devices(),
diff --git a/scripts/lib/libvirt/templates/compute-node.xml b/scripts/lib/libvirt/templates/compute-node.xml
index 18a07900b2..e141f4aabc 100644
--- a/scripts/lib/libvirt/templates/compute-node.xml
+++ b/scripts/lib/libvirt/templates/compute-node.xml
@@ -25,6 +25,9 @@ $cpuflags
 $raidvolume
 $cephvolume
 $drbdvolume
+$extravolume
+$pciecontrollers
+$iommudevice
 $nics
 $serialdevice
     <console type='pty'>
diff --git a/scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml b/scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml
new file mode 100644
index 0000000000..240c38c5bd
--- /dev/null
+++ b/scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml
@@ -0,0 +1,11 @@
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+    <ioapic driver='qemu'/>
+  </features>
+  <cpu mode='custom' match='exact'>
+    <model fallback='allow'>Haswell-noTSX</model>
+    <feature policy='require' name='vmx'/>
+    <feature policy='disable' name='monitor'/>
+  </cpu>
diff --git a/scripts/lib/libvirt/templates/iommu-device-default.xml b/scripts/lib/libvirt/templates/iommu-device-default.xml
new file mode 100644
index 0000000000..db83c22481
--- /dev/null
+++ b/scripts/lib/libvirt/templates/iommu-device-default.xml
@@ -0,0 +1,3 @@
+    <iommu model='intel'>
+      <driver intremap='on' caching_mode='on'/>
+    </iommu>
diff --git a/scripts/lib/libvirt/templates/pcie-root-bridge-default.xml b/scripts/lib/libvirt/templates/pcie-root-bridge-default.xml
new file mode 100644
index 0000000000..9fc31e8c7f
--- /dev/null
+++ b/scripts/lib/libvirt/templates/pcie-root-bridge-default.xml
@@ -0,0 +1,14 @@
+    <controller type='pci' index='0' model='pcie-root'>
+      <alias name='pcie.0'/>
+    </controller>
+    <controller type='pci' index='1' model='dmi-to-pci-bridge'>
+      <model name='i82801b11-bridge'/>
+      <alias name='pci.1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x1e' function='0x0'/>
+    </controller>
+    <controller type='pci' index='2' model='pci-bridge'>
+      <model name='pci-bridge'/>
+      <target chassisNr='2'/>
+      <alias name='pci.2'/>
+      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
+    </controller>
diff --git a/scripts/lib/mkcloud-driver-libvirt.sh b/scripts/lib/mkcloud-driver-libvirt.sh
index f45ac76788..958a151601 100644
--- a/scripts/lib/mkcloud-driver-libvirt.sh
+++ b/scripts/lib/mkcloud-driver-libvirt.sh
@@ -285,6 +285,17 @@ function libvirt_do_create_cloud_lvm()
         done
     fi
 
+    # create an extra volume to test pci passthrough
+    if [ $extravolumenumber -gt 0 ] ; then
+        for i in $(nodes ids all) ; do
+            for n in $(seq 1 $extravolumenumber) ; do
+                onhost_get_next_pv_device
+                hdd_size=${extravolume_hdd_size}
+                _lvcreate $cloud.node$i-extra $hdd_size $cloudvg $next_pv_device
+            done
+        done
+    fi
+
     # create volumes for drbd
     if [ $drbd_hdd_size != 0 ] ; then
         for i in `seq 1 2`; do
diff --git a/scripts/mkcloud b/scripts/mkcloud
index f509e33757..6112481be7 100755
--- a/scripts/mkcloud
+++ b/scripts/mkcloud
@@ -112,6 +112,12 @@ iscloudver 7plus && : ${controller_node_memory:=12582912}
 : ${controller_ceph_hdd_size:=25}
 : ${lonelynode_hdd_size:=20}
 : ${ironicnode_hdd_size:=20}
+: ${want_pci_passthrough:=0}
+if [[ $want_pci_passthrough = 1 ]]; then
+    # Create extra volume to test pci passthrough
+    : ${extravolumenumber:=1}
+    : ${extravolume_hdd_size:=1}
+fi
 
 if [[ $hacloud = 1 ]] && ( [[ "$want_database_sql_engine" == "postgresql" ]] || iscloudver 6minus ) ; then
     : ${drbd_hdd_size:=15}
@@ -739,6 +745,11 @@ function setupnodes
             localrepos_params="--localreposrc ${localreposdir_src} --localrepotgt ${localreposdir_target}"
         fi
 
+        local pcipassthrough_params=""
+        if [[ $want_pci_passthrough = 1 ]]; then
+            pcipassthrough_params="--pcipassthrough"
+        fi
+
         local ipmi_params=""
         if [[ $want_ipmi = 1 ]] ; then
             ipmi_params="--ipmi"
@@ -757,6 +768,7 @@ function setupnodes
                         $mac_params \
                         $ironic_params \
                         $ipmi_params \
+                        $pcipassthrough_params \
                         --cephvolumenumber $cephvolumenumber \
                         --drbdserial "$drbd_serial"\
                         --computenodememory $compute_node_memory \