Skip to content

Commit

Permalink
Merge pull request #435 from zeeke/exclude-topology
Browse files Browse the repository at this point in the history 
Allow creating policies that ignore NUMA topology
  • Loading branch information
@adrianchiris
adrianchiris authored Jun 1, 2023
2 parents 6b81c4e + f55a17e commit 03cca92
Show file tree
Hide file tree
Showing 11 changed files with 1,259 additions and 143 deletions.
2 changes: 2 additions & 0 deletions api/v1/sriovnetworknodepolicy_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ type SriovNetworkNodePolicySpec struct {
// +kubebuilder:validation:Enum=virtio
// VDPA device type. Allowed value "virtio"
VdpaType string `json:"vdpaType,omitempty"`
// Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false.
ExcludeTopology bool `json:"excludeTopology,omitempty"`
}

type SriovNetworkNicSelector struct {
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ spec:
- legacy
- switchdev
type: string
excludeTopology:
description: Exclude device's NUMA node when advertising this resource
by SRIOV network device plugin. Default to false.
type: boolean
isRdma:
description: RDMA mode. Defaults to false.
type: boolean
Expand Down
270 changes: 151 additions & 119 deletions controllers/sriovnetworknodepolicy_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,135 +588,25 @@ func (r *SriovNetworkNodePolicyReconciler) renderDevicePluginConfigData(ctx cont
continue
}

found, i := resourceNameInList(p.Spec.ResourceName, &rcl)
netDeviceSelectors := dptypes.NetDeviceSelectors{}
if found {
if err := json.Unmarshal(*rcl.ResourceList[i].Selectors, &netDeviceSelectors); err != nil {
return rcl, err
}

if p.Spec.NicSelector.Vendor != "" && !sriovnetworkv1.StringInArray(p.Spec.NicSelector.Vendor, netDeviceSelectors.Vendors) {
netDeviceSelectors.Vendors = append(netDeviceSelectors.Vendors, p.Spec.NicSelector.Vendor)
}
if p.Spec.NicSelector.DeviceID != "" {
var deviceID string
if p.Spec.NumVfs == 0 {
deviceID = p.Spec.NicSelector.DeviceID
} else {
deviceID = sriovnetworkv1.GetVfDeviceID(p.Spec.NicSelector.DeviceID)
}
nodeState := &sriovnetworkv1.SriovNetworkNodeState{}
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: node.Name}, nodeState)
if err != nil {
return rcl, err
}

if !sriovnetworkv1.StringInArray(deviceID, netDeviceSelectors.Devices) && deviceID != "" {
netDeviceSelectors.Devices = append(netDeviceSelectors.Devices, deviceID)
}
}
if len(p.Spec.NicSelector.PfNames) > 0 {
netDeviceSelectors.PfNames = sriovnetworkv1.UniqueAppend(netDeviceSelectors.PfNames, p.Spec.NicSelector.PfNames...)
}
// vfio-pci device link type is not detectable
if p.Spec.DeviceType != constants.DeviceTypeVfioPci {
if p.Spec.LinkType != "" {
linkType := constants.LinkTypeEthernet
if strings.EqualFold(p.Spec.LinkType, constants.LinkTypeIB) {
linkType = constants.LinkTypeInfiniband
}
if !sriovnetworkv1.StringInArray(linkType, netDeviceSelectors.LinkTypes) {
netDeviceSelectors.LinkTypes = sriovnetworkv1.UniqueAppend(netDeviceSelectors.LinkTypes, linkType)
}
}
}
if len(p.Spec.NicSelector.RootDevices) > 0 {
netDeviceSelectors.RootDevices = sriovnetworkv1.UniqueAppend(netDeviceSelectors.RootDevices, p.Spec.NicSelector.RootDevices...)
}
// Removed driver constraint for "netdevice" DeviceType
if p.Spec.DeviceType == constants.DeviceTypeVfioPci {
netDeviceSelectors.Drivers = sriovnetworkv1.UniqueAppend(netDeviceSelectors.Drivers, p.Spec.DeviceType)
}
// Enable the selection of devices using NetFilter
if p.Spec.NicSelector.NetFilter != "" {
nodeState := &sriovnetworkv1.SriovNetworkNodeState{}
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: node.Name}, nodeState)
if err == nil {
// Loop through interfaces status to find a match for NetworkID or NetworkTag
for _, intf := range nodeState.Status.Interfaces {
if sriovnetworkv1.NetFilterMatch(p.Spec.NicSelector.NetFilter, intf.NetFilter) {
// Found a match add the Interfaces PciAddress
netDeviceSelectors.PciAddresses = sriovnetworkv1.UniqueAppend(netDeviceSelectors.PciAddresses, intf.PciAddress)
}
}
}
}
found, i := resourceNameInList(p.Spec.ResourceName, &rcl)

netDeviceSelectorsMarshal, err := json.Marshal(netDeviceSelectors)
if found {
err := updateDevicePluginResource(ctx, &rcl.ResourceList[i], &p, nodeState)
if err != nil {
return rcl, err
}
rawNetDeviceSelectors := json.RawMessage(netDeviceSelectorsMarshal)
rcl.ResourceList[i].Selectors = &rawNetDeviceSelectors
logger.Info("Update resource", "Resource", rcl.ResourceList[i])
} else {
rc := &dptypes.ResourceConfig{
ResourceName: p.Spec.ResourceName,
}
netDeviceSelectors.IsRdma = p.Spec.IsRdma
netDeviceSelectors.NeedVhostNet = p.Spec.NeedVhostNet
netDeviceSelectors.VdpaType = dptypes.VdpaType(p.Spec.VdpaType)

if p.Spec.NicSelector.Vendor != "" {
netDeviceSelectors.Vendors = append(netDeviceSelectors.Vendors, p.Spec.NicSelector.Vendor)
}
if p.Spec.NicSelector.DeviceID != "" {
var deviceID string
if p.Spec.NumVfs == 0 {
deviceID = p.Spec.NicSelector.DeviceID
} else {
deviceID = sriovnetworkv1.GetVfDeviceID(p.Spec.NicSelector.DeviceID)
}

if !sriovnetworkv1.StringInArray(deviceID, netDeviceSelectors.Devices) && deviceID != "" {
netDeviceSelectors.Devices = append(netDeviceSelectors.Devices, deviceID)
}
}
if len(p.Spec.NicSelector.PfNames) > 0 {
netDeviceSelectors.PfNames = append(netDeviceSelectors.PfNames, p.Spec.NicSelector.PfNames...)
}
// vfio-pci device link type is not detectable
if p.Spec.DeviceType != constants.DeviceTypeVfioPci {
if p.Spec.LinkType != "" {
linkType := constants.LinkTypeEthernet
if strings.EqualFold(p.Spec.LinkType, constants.LinkTypeIB) {
linkType = constants.LinkTypeInfiniband
}
netDeviceSelectors.LinkTypes = sriovnetworkv1.UniqueAppend(netDeviceSelectors.LinkTypes, linkType)
}
}
if len(p.Spec.NicSelector.RootDevices) > 0 {
netDeviceSelectors.RootDevices = append(netDeviceSelectors.RootDevices, p.Spec.NicSelector.RootDevices...)
}
// Removed driver constraint for "netdevice" DeviceType
if p.Spec.DeviceType == constants.DeviceTypeVfioPci {
netDeviceSelectors.Drivers = append(netDeviceSelectors.Drivers, p.Spec.DeviceType)
}
// Enable the selection of devices using NetFilter
if p.Spec.NicSelector.NetFilter != "" {
nodeState := &sriovnetworkv1.SriovNetworkNodeState{}
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: node.Name}, nodeState)
if err == nil {
// Loop through interfaces status to find a match for NetworkID or NetworkTag
for _, intf := range nodeState.Status.Interfaces {
if sriovnetworkv1.NetFilterMatch(p.Spec.NicSelector.NetFilter, intf.NetFilter) {
// Found a match add the Interfaces PciAddress
netDeviceSelectors.PciAddresses = sriovnetworkv1.UniqueAppend(netDeviceSelectors.PciAddresses, intf.PciAddress)
}
}
}
}
netDeviceSelectorsMarshal, err := json.Marshal(netDeviceSelectors)
rc, err := createDevicePluginResource(ctx, &p, nodeState)
if err != nil {
return rcl, err
}
rawNetDeviceSelectors := json.RawMessage(netDeviceSelectorsMarshal)
rc.Selectors = &rawNetDeviceSelectors
rcl.ResourceList = append(rcl.ResourceList, *rc)
logger.Info("Add resource", "Resource", *rc, "Resource list", rcl.ResourceList)
}
Expand All @@ -732,3 +622,145 @@ func resourceNameInList(name string, rcl *dptypes.ResourceConfList) (bool, int)
}
return false, 0
}

func createDevicePluginResource(
ctx context.Context,
p *sriovnetworkv1.SriovNetworkNodePolicy,
nodeState *sriovnetworkv1.SriovNetworkNodeState) (*dptypes.ResourceConfig, error) {
netDeviceSelectors := dptypes.NetDeviceSelectors{}

rc := &dptypes.ResourceConfig{
ResourceName: p.Spec.ResourceName,
}
netDeviceSelectors.IsRdma = p.Spec.IsRdma
netDeviceSelectors.NeedVhostNet = p.Spec.NeedVhostNet
netDeviceSelectors.VdpaType = dptypes.VdpaType(p.Spec.VdpaType)

if p.Spec.NicSelector.Vendor != "" {
netDeviceSelectors.Vendors = append(netDeviceSelectors.Vendors, p.Spec.NicSelector.Vendor)
}
if p.Spec.NicSelector.DeviceID != "" {
var deviceID string
if p.Spec.NumVfs == 0 {
deviceID = p.Spec.NicSelector.DeviceID
} else {
deviceID = sriovnetworkv1.GetVfDeviceID(p.Spec.NicSelector.DeviceID)
}

if !sriovnetworkv1.StringInArray(deviceID, netDeviceSelectors.Devices) && deviceID != "" {
netDeviceSelectors.Devices = append(netDeviceSelectors.Devices, deviceID)
}
}
if len(p.Spec.NicSelector.PfNames) > 0 {
netDeviceSelectors.PfNames = append(netDeviceSelectors.PfNames, p.Spec.NicSelector.PfNames...)
}
// vfio-pci device link type is not detectable
if p.Spec.DeviceType != constants.DeviceTypeVfioPci {
if p.Spec.LinkType != "" {
linkType := constants.LinkTypeEthernet
if strings.EqualFold(p.Spec.LinkType, constants.LinkTypeIB) {
linkType = constants.LinkTypeInfiniband
}
netDeviceSelectors.LinkTypes = sriovnetworkv1.UniqueAppend(netDeviceSelectors.LinkTypes, linkType)
}
}
if len(p.Spec.NicSelector.RootDevices) > 0 {
netDeviceSelectors.RootDevices = append(netDeviceSelectors.RootDevices, p.Spec.NicSelector.RootDevices...)
}
// Removed driver constraint for "netdevice" DeviceType
if p.Spec.DeviceType == constants.DeviceTypeVfioPci {
netDeviceSelectors.Drivers = append(netDeviceSelectors.Drivers, p.Spec.DeviceType)
}
// Enable the selection of devices using NetFilter
if p.Spec.NicSelector.NetFilter != "" {
// Loop through interfaces status to find a match for NetworkID or NetworkTag
for _, intf := range nodeState.Status.Interfaces {
if sriovnetworkv1.NetFilterMatch(p.Spec.NicSelector.NetFilter, intf.NetFilter) {
// Found a match add the Interfaces PciAddress
netDeviceSelectors.PciAddresses = sriovnetworkv1.UniqueAppend(netDeviceSelectors.PciAddresses, intf.PciAddress)
}
}
}

netDeviceSelectorsMarshal, err := json.Marshal(netDeviceSelectors)
if err != nil {
return nil, err
}
rawNetDeviceSelectors := json.RawMessage(netDeviceSelectorsMarshal)
rc.Selectors = &rawNetDeviceSelectors

rc.ExcludeTopology = p.Spec.ExcludeTopology

return rc, nil
}

func updateDevicePluginResource(
ctx context.Context,
rc *dptypes.ResourceConfig,
p *sriovnetworkv1.SriovNetworkNodePolicy,
nodeState *sriovnetworkv1.SriovNetworkNodeState) error {
netDeviceSelectors := dptypes.NetDeviceSelectors{}

if err := json.Unmarshal(*rc.Selectors, &netDeviceSelectors); err != nil {
return err
}

if p.Spec.NicSelector.Vendor != "" && !sriovnetworkv1.StringInArray(p.Spec.NicSelector.Vendor, netDeviceSelectors.Vendors) {
netDeviceSelectors.Vendors = append(netDeviceSelectors.Vendors, p.Spec.NicSelector.Vendor)
}
if p.Spec.NicSelector.DeviceID != "" {
var deviceID string
if p.Spec.NumVfs == 0 {
deviceID = p.Spec.NicSelector.DeviceID
} else {
deviceID = sriovnetworkv1.GetVfDeviceID(p.Spec.NicSelector.DeviceID)
}

if !sriovnetworkv1.StringInArray(deviceID, netDeviceSelectors.Devices) && deviceID != "" {
netDeviceSelectors.Devices = append(netDeviceSelectors.Devices, deviceID)
}
}
if len(p.Spec.NicSelector.PfNames) > 0 {
netDeviceSelectors.PfNames = sriovnetworkv1.UniqueAppend(netDeviceSelectors.PfNames, p.Spec.NicSelector.PfNames...)
}
// vfio-pci device link type is not detectable
if p.Spec.DeviceType != constants.DeviceTypeVfioPci {
if p.Spec.LinkType != "" {
linkType := constants.LinkTypeEthernet
if strings.EqualFold(p.Spec.LinkType, constants.LinkTypeIB) {
linkType = constants.LinkTypeInfiniband
}
if !sriovnetworkv1.StringInArray(linkType, netDeviceSelectors.LinkTypes) {
netDeviceSelectors.LinkTypes = sriovnetworkv1.UniqueAppend(netDeviceSelectors.LinkTypes, linkType)
}
}
}
if len(p.Spec.NicSelector.RootDevices) > 0 {
netDeviceSelectors.RootDevices = sriovnetworkv1.UniqueAppend(netDeviceSelectors.RootDevices, p.Spec.NicSelector.RootDevices...)
}
// Removed driver constraint for "netdevice" DeviceType
if p.Spec.DeviceType == constants.DeviceTypeVfioPci {
netDeviceSelectors.Drivers = sriovnetworkv1.UniqueAppend(netDeviceSelectors.Drivers, p.Spec.DeviceType)
}
// Enable the selection of devices using NetFilter
if p.Spec.NicSelector.NetFilter != "" {
// Loop through interfaces status to find a match for NetworkID or NetworkTag
for _, intf := range nodeState.Status.Interfaces {
if sriovnetworkv1.NetFilterMatch(p.Spec.NicSelector.NetFilter, intf.NetFilter) {
// Found a match add the Interfaces PciAddress
netDeviceSelectors.PciAddresses = sriovnetworkv1.UniqueAppend(netDeviceSelectors.PciAddresses, intf.PciAddress)
}
}
}

netDeviceSelectorsMarshal, err := json.Marshal(netDeviceSelectors)
if err != nil {
return err
}
rawNetDeviceSelectors := json.RawMessage(netDeviceSelectorsMarshal)
rc.Selectors = &rawNetDeviceSelectors

rc.ExcludeTopology = p.Spec.ExcludeTopology

return nil
}
Loading

0 comments on commit 03cca92

Please sign in to comment.