Skip to content

Commit

Permalink
vhost-vdpa support
Browse files Browse the repository at this point in the history 
SriovNetworkNodePolicy API: VDPA device type can take now the following values: "virtio" and "vhost"
Vhost/vdpa: Same behaviour of virtio/vdpa apart for the drivers that are loaded in the kernel (vhost-vdpa instead of virtio-vdpa)

Signed-off-by: Leonardo Milleri <[email protected]>
  • Loading branch information
@lmilleri
lmilleri committed Jul 13, 2023
1 parent 37ddcd4 commit 4175d3b
Show file tree
Hide file tree
Showing 8 changed files with 124 additions and 19 deletions.
4 changes: 2 additions & 2 deletions api/v1/sriovnetworknodepolicy_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ type SriovNetworkNodePolicySpec struct {
// +kubebuilder:validation:Enum=legacy;switchdev
// NIC Device Mode. Allowed value "legacy","switchdev".
EswitchMode string `json:"eSwitchMode,omitempty"`
// +kubebuilder:validation:Enum=virtio
// VDPA device type. Allowed value "virtio"
// +kubebuilder:validation:Enum=virtio;vhost
// VDPA device type. Allowed value "virtio", "vhost"
VdpaType string `json:"vdpaType,omitempty"`
// Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false.
ExcludeTopology bool `json:"excludeTopology,omitempty"`
Expand Down
2 changes: 1 addition & 1 deletion bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ contents:
do
extract_min_max_ids
vdpaType=$(jq -c '.vdpaType' -r <<< $group)
if [ $vfid -le $maxId ] && [ $vfid -ge $minId ] && [ $vdpaType == "virtio" ]; then
if [ $vfid -le $maxId ] && [ $vfid -ge $minId ] && [ $vdpaType != "" ]; then
vdpa_cmd="vdpa dev add name vdpa:"${VfPciAddr}" mgmtdev pci/"${VfPciAddr}
eval $vdpa_cmd
fi
Expand Down
3 changes: 2 additions & 1 deletion config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,10 @@ spec:
description: SRIOV Network device plugin endpoint resource name
type: string
vdpaType:
description: VDPA device type. Allowed value "virtio"
description: VDPA device type. Allowed value "virtio", "vhost"
enum:
- virtio
- vhost
type: string
required:
- nicSelector
Expand Down
3 changes: 2 additions & 1 deletion ...yment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,10 @@ spec:
description: SRIOV Network device plugin endpoint resource name
type: string
vdpaType:
description: VDPA device type. Allowed value "virtio"
description: VDPA device type. Allowed value "virtio", "vhost"
enum:
- virtio
- vhost
type: string
required:
- nicSelector
Expand Down
1 change: 1 addition & 0 deletions pkg/consts/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,5 @@ const (
DeviceTypeVfioPci = "vfio-pci"
DeviceTypeNetDevice = "netdevice"
VdpaTypeVirtio = "virtio"
VdpaTypeVhost = "vhost"
)
32 changes: 30 additions & 2 deletions pkg/plugins/generic/generic_plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type GenericPlugin struct {
LoadVirtioVdpaDriver uint
RunningOnHost bool
HostManager host.HostManagerInterface
LoadVhostVdpaDriver uint
}

const scriptsPath = "bindata/scripts/enable-kargs.sh"
Expand All @@ -47,6 +48,7 @@ func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) {
LoadVirtioVdpaDriver: unloaded,
RunningOnHost: runningOnHost,
HostManager: host.NewHostManager(runningOnHost),
LoadVhostVdpaDriver: unloaded,
}, nil
}

Expand All @@ -69,7 +71,7 @@ func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeSt
p.DesireState = new

needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces)
needReboot = needRebootNode(new, &p.LoadVfioDriver, &p.LoadVirtioVdpaDriver)
needReboot = needRebootNode(new, &p.LoadVfioDriver, &p.LoadVirtioVdpaDriver, &p.LoadVhostVdpaDriver)

if needReboot {
needDrain = true
Expand All @@ -96,6 +98,14 @@ func (p *GenericPlugin) Apply() error {
p.LoadVirtioVdpaDriver = loaded
}

if p.LoadVhostVdpaDriver == loading {
if err := p.HostManager.LoadKernelModule("vhost_vdpa"); err != nil {
glog.Errorf("generic-plugin Apply(): fail to load vhost_vdpa kmod: %v", err)
return err
}
p.LoadVhostVdpaDriver = loaded
}

if p.LastState != nil {
glog.Infof("generic-plugin Apply(): lastStat=%v", p.LastState.Spec)
if reflect.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) {
Expand Down Expand Up @@ -151,6 +161,17 @@ func needVirtioVdpaDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool {
return false
}

func needVhostVdpaDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool {
for _, iface := range state.Spec.Interfaces {
for i := range iface.VfGroups {
if iface.VfGroups[i].VdpaType == constants.VdpaTypeVhost {
return true
}
}
}
return false
}

func tryEnableIommuInKernelArgs() (bool, error) {
glog.Info("generic-plugin tryEnableIommuInKernelArgs()")
args := [2]string{"intel_iommu=on", "iommu=pt"}
Expand Down Expand Up @@ -215,7 +236,8 @@ func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.Int
return
}

func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint, loadVirtioVdpaDriver *uint) (needReboot bool) {
func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint, loadVirtioVdpaDriver *uint,
loadVhostVdpaDriver *uint) (needReboot bool) {
needReboot = false
if *loadVfioDriver != loaded {
if needVfioDriver(state) {
Expand All @@ -237,6 +259,12 @@ func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver
}
}

if *loadVhostVdpaDriver != loaded {
if needVhostVdpaDriver(state) {
*loadVhostVdpaDriver = loading
}
}

update, err := utils.WriteSwitchdevConfFile(state)
if err != nil {
glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file")
Expand Down
12 changes: 6 additions & 6 deletions pkg/webhook/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,12 +203,12 @@ func staticValidateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePol
}

// vdpa: deviceType must be set to 'netdevice'
if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && cr.Spec.VdpaType == constants.VdpaTypeVirtio {
return false, fmt.Errorf("'deviceType: %s' conflicts with 'vdpaType: virtio'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType)
if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) {
return false, fmt.Errorf("'deviceType: %s' conflicts with '%s'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType, cr.Spec.VdpaType)
}
// vdpa: device must be configured in switchdev mode
if cr.Spec.VdpaType == constants.VdpaTypeVirtio && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev {
return false, fmt.Errorf("virtio/vdpa requires the device to be configured in switchdev mode")
if (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev {
return false, fmt.Errorf("vdpa requires the device to be configured in switchdev mode")
}
return true, nil
}
Expand Down Expand Up @@ -286,8 +286,8 @@ func validatePolicyForNodeState(policy *sriovnetworkv1.SriovNetworkNodePolicy, s
return fmt.Errorf("numVfs(%d) in CR %s exceed the maximum allowed value(%d)", policy.Spec.NumVfs, policy.GetName(), MlxMaxVFs)
}
// vdpa: only mellanox cards are supported
if policy.Spec.VdpaType == constants.VdpaTypeVirtio && iface.Vendor != MellanoxID {
return fmt.Errorf("vendor(%s) in CR %s not supported for virtio-vdpa", iface.Vendor, policy.GetName())
if (policy.Spec.VdpaType == constants.VdpaTypeVirtio || policy.Spec.VdpaType == constants.VdpaTypeVhost) && iface.Vendor != MellanoxID {
return fmt.Errorf("vendor(%s) in CR %s not supported for vdpa", iface.Vendor, policy.GetName())
}
}
}
Expand Down
86 changes: 80 additions & 6 deletions pkg/webhook/validate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,7 +538,7 @@ func TestStaticValidateSriovNetworkNodePolicyWithConflictIsRdmaAndDeviceType(t *
g.Expect(ok).To(Equal(false))
}

func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVdpaType(t *testing.T) {
func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVirtioVdpaType(t *testing.T) {
policy := &SriovNetworkNodePolicy{
Spec: SriovNetworkNodePolicySpec{
DeviceType: constants.DeviceTypeVfioPci,
Expand All @@ -558,11 +558,35 @@ func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVdpaType(t
}
g := NewGomegaWithT(t)
ok, err := staticValidateSriovNetworkNodePolicy(policy)
g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'vdpaType: virtio'")))
g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'virtio'")))
g.Expect(ok).To(Equal(false))
}

func TestStaticValidateSriovNetworkNodePolicyVdpaMustSpecifySwitchDev(t *testing.T) {
func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVhostVdpaType(t *testing.T) {
policy := &SriovNetworkNodePolicy{
Spec: SriovNetworkNodePolicySpec{
DeviceType: constants.DeviceTypeVfioPci,
NicSelector: SriovNetworkNicSelector{
Vendor: "15b3",
DeviceID: "101d",
},
NodeSelector: map[string]string{
"feature.node.kubernetes.io/network-sriov.capable": "true",
},
NumVfs: 1,
Priority: 99,
ResourceName: "p0",
VdpaType: constants.VdpaTypeVhost,
EswitchMode: "switchdev",
},
}
g := NewGomegaWithT(t)
ok, err := staticValidateSriovNetworkNodePolicy(policy)
g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'vhost'")))
g.Expect(ok).To(Equal(false))
}

func TestStaticValidateSriovNetworkNodePolicyVirtioVdpaMustSpecifySwitchDev(t *testing.T) {
policy := &SriovNetworkNodePolicy{
Spec: SriovNetworkNodePolicySpec{
DeviceType: "netdevice",
Expand All @@ -581,11 +605,34 @@ func TestStaticValidateSriovNetworkNodePolicyVdpaMustSpecifySwitchDev(t *testing
}
g := NewGomegaWithT(t)
ok, err := staticValidateSriovNetworkNodePolicy(policy)
g.Expect(err).To(MatchError(ContainSubstring("virtio/vdpa requires the device to be configured in switchdev mode")))
g.Expect(err).To(MatchError(ContainSubstring("vdpa requires the device to be configured in switchdev mode")))
g.Expect(ok).To(Equal(false))
}

func TestValidatePolicyForNodeStateVdpaWithNotSupportedVendor(t *testing.T) {
func TestStaticValidateSriovNetworkNodePolicyVhostVdpaMustSpecifySwitchDev(t *testing.T) {
policy := &SriovNetworkNodePolicy{
Spec: SriovNetworkNodePolicySpec{
DeviceType: "netdevice",
NicSelector: SriovNetworkNicSelector{
Vendor: "15b3",
DeviceID: "101d",
},
NodeSelector: map[string]string{
"feature.node.kubernetes.io/network-sriov.capable": "true",
},
NumVfs: 1,
Priority: 99,
ResourceName: "p0",
VdpaType: constants.VdpaTypeVhost,
},
}
g := NewGomegaWithT(t)
ok, err := staticValidateSriovNetworkNodePolicy(policy)
g.Expect(err).To(MatchError(ContainSubstring("vdpa requires the device to be configured in switchdev mode")))
g.Expect(ok).To(Equal(false))
}

func TestValidatePolicyForNodeStateVirtioVdpaWithNotSupportedVendor(t *testing.T) {
state := newNodeState()
policy := &SriovNetworkNodePolicy{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -609,7 +656,34 @@ func TestValidatePolicyForNodeStateVdpaWithNotSupportedVendor(t *testing.T) {
}
g := NewGomegaWithT(t)
err := validatePolicyForNodeState(policy, state, NewNode())
g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for virtio-vdpa", state.Status.Interfaces[0].Vendor, policy.Name))))
g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for vdpa", state.Status.Interfaces[0].Vendor, policy.Name))))
}

func TestValidatePolicyForNodeStateVhostVdpaWithNotSupportedVendor(t *testing.T) {
state := newNodeState()
policy := &SriovNetworkNodePolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "p1",
},
Spec: SriovNetworkNodePolicySpec{
DeviceType: "netdevice",
VdpaType: "vhost",
NicSelector: SriovNetworkNicSelector{
PfNames: []string{"ens803f0"},
RootDevices: []string{"0000:86:00.0"},
Vendor: "8086",
},
NodeSelector: map[string]string{
"feature.node.kubernetes.io/network-sriov.capable": "true",
},
NumVfs: 4,
Priority: 99,
ResourceName: "p0",
},
}
g := NewGomegaWithT(t)
err := validatePolicyForNodeState(policy, state, NewNode())
g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for vdpa", state.Status.Interfaces[0].Vendor, policy.Name))))
}

func TestValidatePolicyForNodeStateWithInvalidDevice(t *testing.T) {
Expand Down

0 comments on commit 4175d3b

Please sign in to comment.