From a1703ca13298c086543746e53d5024f521ef752a Mon Sep 17 00:00:00 2001 From: Leonardo Milleri Date: Mon, 22 Aug 2022 14:47:35 +0100 Subject: [PATCH] Integrating govdpa changes from release v0.1.7 Signed-off-by: Leonardo Milleri --- go.mod | 2 + go.sum | 4 +- pkg/netdevice/vdpa.go | 15 +- pkg/utils/vdpa_provider.go | 11 +- .../govdpa/pkg/kvdpa/device.go | 322 ++++++++++++++++++ .../govdpa/pkg/kvdpa/kvdpa.go | 242 ------------- .../govdpa/pkg/kvdpa/mgmtdev.go | 114 +++++++ .../govdpa/pkg/kvdpa/netlink.go | 182 ++++++++++ .../govdpa/pkg/kvdpa/vhost.go | 62 ++++ .../govdpa/pkg/kvdpa/virtio.go | 69 ++++ vendor/modules.txt | 2 +- 11 files changed, 778 insertions(+), 247 deletions(-) create mode 100644 vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/device.go delete mode 100644 vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/kvdpa.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/mgmtdev.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/netlink.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/vhost.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/virtio.go diff --git a/go.mod b/go.mod index 3766a3679..7ae60cae8 100644 --- a/go.mod +++ b/go.mod @@ -80,3 +80,5 @@ replace ( github.com/containernetworking/cni => github.com/containernetworking/cni v0.8.1 github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2 ) + +replace github.com/k8snetworkplumbingwg/govdpa v0.1.3 => github.com/lmilleri/govdpa v0.1.7 diff --git a/go.sum b/go.sum index 6068fff40..1e7c8026f 100644 --- a/go.sum +++ b/go.sum @@ -275,8 +275,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k8snetworkplumbingwg/govdpa v0.1.3 h1:FZRhTMB1e3yWwSEy+l4eS73WioyMaL+vmFZ8JNwn+Uk= -github.com/k8snetworkplumbingwg/govdpa v0.1.3/go.mod h1:Jx2rlMquENdCd8M5Oc51xHCt10bQIXTloDU8F4nS4T4= github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d h1:9QbZltQGRFe7temwcTDjj8rIbow48Gv6mIKOxuks+OI= github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d/go.mod h1:+1DpV8uIwteAhxNO0lgRox8gHkTG6w3OeDfAlg+qqjA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -292,6 +290,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lmilleri/govdpa v0.1.7 h1:O2+6PwdfgIZBqRAS+IDoYIKwZyjU1FTOBM9EQQ1NbZY= +github.com/lmilleri/govdpa v0.1.7/go.mod h1:zC+FdDzIqMN38LAuA8L2oADVZjVX5lnHGHpHMNeQuI4= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= diff --git a/pkg/netdevice/vdpa.go b/pkg/netdevice/vdpa.go index 61be12c18..b6f569b74 100644 --- a/pkg/netdevice/vdpa.go +++ b/pkg/netdevice/vdpa.go @@ -40,7 +40,7 @@ type vdpaDevice struct { // GetType returns the VdpaType associated with the VdpaDevice func (v *vdpaDevice) GetType() types.VdpaType { - currentDriver := v.GetDriver() + currentDriver := v.VdpaDevice.Driver() for vtype, driver := range supportedVdpaTypes { if driver == currentDriver { return vtype @@ -49,6 +49,19 @@ func (v *vdpaDevice) GetType() types.VdpaType { return types.VdpaInvalidType } +func (v *vdpaDevice) GetParent() string { + return v.VdpaDevice.Name() +} + +func (v *vdpaDevice) GetPath() string { + path, err := v.ParentDevicePath() + if err != nil { + glog.Infof("%s - No path for vDPA device found: %s", v.Name(), err) + return "" + } + return path +} + // GetVdpaDevice returns a VdpaDevice from a given VF PCI address func GetVdpaDevice(pciAddr string) types.VdpaDevice { detailVdpaDev, err := utils.GetVdpaProvider().GetVdpaDeviceByPci(pciAddr) diff --git a/pkg/utils/vdpa_provider.go b/pkg/utils/vdpa_provider.go index 49366df46..dc7f61a75 100644 --- a/pkg/utils/vdpa_provider.go +++ b/pkg/utils/vdpa_provider.go @@ -1,6 +1,8 @@ package utils import ( + "fmt" + vdpa "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" ) @@ -25,5 +27,12 @@ func GetVdpaProvider() VdpaProvider { } func (defaultVdpaProvider) GetVdpaDeviceByPci(pciAddr string) (vdpa.VdpaDevice, error) { - return vdpa.GetVdpaDeviceByPci(pciAddr) + vdpaDevices, err := vdpa.GetVdpaDevicesByPciAddress(pciAddr) + if err != nil { + return nil, err + } + if len(vdpaDevices) == 0 { + return nil, fmt.Errorf("no vdpa device associated to pciAddress %v", pciAddr) + } + return vdpaDevices[0], nil } diff --git a/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/device.go b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/device.go new file mode 100644 index 000000000..a9e3f7296 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/device.go @@ -0,0 +1,322 @@ +package kvdpa + +import ( + "errors" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" +) + +// Exported constants +const ( + VhostVdpaDriver = "vhost_vdpa" + VirtioVdpaDriver = "virtio_vdpa" +) + +// Private constants +const ( + vdpaBusDevDir = "/sys/bus/vdpa/devices" + vdpaVhostDevDir = "/dev" + rootDevDir = "/sys/devices" +) + +// VdpaDevice contains information about a Vdpa Device +type VdpaDevice interface { + Driver() string + Name() string + MgmtDev() MgmtDev + VirtioNet() VirtioNet + VhostVdpa() VhostVdpa + ParentDevicePath() (string, error) +} + +// vdpaDev implements VdpaDevice interface +type vdpaDev struct { + name string + driver string + mgmtDev *mgmtDev + virtioNet VirtioNet + vhostVdpa VhostVdpa +} + +// Driver resturns de device's driver name +func (vd *vdpaDev) Driver() string { + return vd.driver +} + +// Driver resturns de device's name +func (vd *vdpaDev) Name() string { + return vd.name +} + +// MgmtDev returns the device's management device +func (vd *vdpaDev) MgmtDev() MgmtDev { + return vd.mgmtDev +} + +// VhostVdpa returns the VhostVdpa device information associated +// or nil if the device is not bound to the vhost_vdpa driver +func (vd *vdpaDev) VhostVdpa() VhostVdpa { + return vd.vhostVdpa +} + +// Virtionet returns the VirtioNet device information associated +// or nil if the device is not bound to the virtio_vdpa driver +func (vd *vdpaDev) VirtioNet() VirtioNet { + return vd.virtioNet +} + +// getBusInfo populates the vdpa bus information +// the vdpa device must have at least the name prepopulated +func (vd *vdpaDev) getBusInfo() error { + driverLink, err := os.Readlink(filepath.Join(vdpaBusDevDir, vd.name, "driver")) + if err != nil { + // No error if driver is not present. The device is simply not bound to any. + return nil + } + + vd.driver = filepath.Base(driverLink) + + switch vd.driver { + case VhostVdpaDriver: + vd.vhostVdpa, err = vd.getVhostVdpaDev() + if err != nil { + return err + } + case VirtioVdpaDriver: + vd.virtioNet, err = vd.getVirtioVdpaDev() + if err != nil { + return err + } + } + + return nil +} + +// parseAttributes populates the vdpa device information from netlink attributes +func (vd *vdpaDev) parseAttributes(attrs []syscall.NetlinkRouteAttr) error { + mgmtDev := &mgmtDev{} + for _, a := range attrs { + switch a.Attr.Type { + case VdpaAttrDevName: + vd.name = string(a.Value[:len(a.Value)-1]) + case VdpaAttrMgmtDevBusName: + mgmtDev.busName = string(a.Value[:len(a.Value)-1]) + case VdpaAttrMgmtDevDevName: + mgmtDev.devName = string(a.Value[:len(a.Value)-1]) + } + } + vd.mgmtDev = mgmtDev + return nil +} + +/* Finds the vhost vdpa device of a vdpa device and returns it's path */ +func (vd *vdpaDev) getVhostVdpaDev() (VhostVdpa, error) { + // vhost vdpa devices live in the vdpa device's path + path := filepath.Join(vdpaBusDevDir, vd.name) + return GetVhostVdpaDevInPath(path) +} + +/* ParentDevice returns the path of the parent device (e.g: PCI) of the device */ +func (vd *vdpaDev) ParentDevicePath() (string, error) { + vdpaDevicePath := filepath.Join(vdpaBusDevDir, vd.name) + + /* For pci devices we have: + /sys/bud/vdpa/devices/vdpaX -> + ../../../devices/pci0000:00/.../0000:05:00:1/vdpaX + + Resolving the symlinks should give us the parent PCI device. + */ + devicePath, err := filepath.EvalSymlinks(vdpaDevicePath) + if err != nil { + return "", err + } + + /* If the parent device is the root device /sys/devices, there is + no parent (e.g: vdpasim). + */ + parent := filepath.Dir(devicePath) + if parent == rootDevDir { + return devicePath, nil + } + + return parent, nil +} + +/* Finds the virtio vdpa device of a vdpa device and returns it's path +Currently, PCI-based devices have the following sysfs structure: +/sys/bus/vdpa/devices/ + vdpa1 -> ../../../devices/pci0000:00/0000:00:03.2/0000:05:00.2/vdpa1 + +In order to find the virtio device we look for virtio* devices inside the parent device: + sys/devices/pci0000:00/0000:00:03.2/0000:05:00.2/virtio{N} + +We also check the virtio device exists in the virtio bus: +/sys/bus/virtio/devices + virtio{N} -> ../../../devices/pci0000:00/0000:00:03.2/0000:05:00.2/virtio{N} +*/ +func (vd *vdpaDev) getVirtioVdpaDev() (VirtioNet, error) { + parentPath, err := vd.ParentDevicePath() + if err != nil { + return nil, err + } + return GetVirtioNetInPath(parentPath) +} + +/*GetVdpaDevice returns the vdpa device information by a vdpa device name */ +func GetVdpaDevice(name string) (VdpaDevice, error) { + nameAttr, err := GetNetlinkOps().NewAttribute(VdpaAttrDevName, name) + if err != nil { + return nil, err + } + + msgs, err := GetNetlinkOps(). + RunVdpaNetlinkCmd(VdpaCmdDevGet, 0, []*nl.RtAttr{nameAttr}) + if err != nil { + return nil, err + } + + vdpaDevs, err := parseDevLinkVdpaDevList(msgs) + if err != nil { + return nil, err + } + return vdpaDevs[0], nil +} + +/*GetVdpaDevicesByMgmtDev returns the VdpaDevice objects whose MgmtDev +has the given bus and device names. +*/ +func GetVdpaDevicesByMgmtDev(busName, devName string) ([]VdpaDevice, error) { + result := []VdpaDevice{} + devices, err := ListVdpaDevices() + if err != nil { + return nil, err + } + for _, device := range devices { + if device.MgmtDev() != nil && + device.MgmtDev().BusName() == busName && + device.MgmtDev().DevName() == devName { + result = append(result, device) + } + } + if len(result) == 0 { + return nil, unix.ENODEV + } + return result, nil +} + +/*GetVdpaDevicesByPciAddress returns the VdpaDevice objects for the given pciAddress */ +func GetVdpaDevicesByPciAddress(pciAddress string) ([]VdpaDevice, error) { + busName, mgmtDeviceName, err := extractBusNameAndMgmtDeviceName(pciAddress) + if err != nil { + return nil, unix.EINVAL + } + + return GetVdpaDevicesByMgmtDev(busName, mgmtDeviceName) +} + +/*ListVdpaDevices returns a list of all available vdpa devices */ +func ListVdpaDevices() ([]VdpaDevice, error) { + msgs, err := GetNetlinkOps().RunVdpaNetlinkCmd(VdpaCmdDevGet, unix.NLM_F_DUMP, nil) + if err != nil { + return nil, err + } + + vdpaDevs, err := parseDevLinkVdpaDevList(msgs) + if err != nil { + return nil, err + } + return vdpaDevs, nil +} + +func extractBusNameAndMgmtDeviceName(fullMgmtDeviceName string) (busName string, mgmtDeviceName string, err error) { + numSlashes := strings.Count(fullMgmtDeviceName, "/") + if numSlashes > 1 { + return "", "", errors.New("expected mgmtDeviceName to be either in the format / or ") + } else if numSlashes == 0 { + return "", fullMgmtDeviceName, nil + } else { + values := strings.Split(fullMgmtDeviceName, "/") + return values[0], values[1], nil + } +} + +/*AddVdpaDevice adds a new vdpa device to the given management device */ +func AddVdpaDevice(mgmtDeviceName string, vdpaDeviceName string) ([][]byte, error) { + if mgmtDeviceName == "" || vdpaDeviceName == "" { + return nil, unix.EINVAL + } + + busName, mgmtDeviceName, err := extractBusNameAndMgmtDeviceName(mgmtDeviceName) + if err != nil { + return nil, unix.EINVAL + } + + var busNameAttr *nl.RtAttr + if busName != "" { + busNameAttr, err = GetNetlinkOps().NewAttribute(VdpaAttrMgmtDevBusName, busName) + if err != nil { + return nil, err + } + } + + mgmtAttr, err := GetNetlinkOps().NewAttribute(VdpaAttrMgmtDevDevName, mgmtDeviceName) + if err != nil { + return nil, err + } + + nameAttr, err := GetNetlinkOps().NewAttribute(VdpaAttrDevName, vdpaDeviceName) + if err != nil { + return nil, err + } + + msgs, err := GetNetlinkOps().RunVdpaNetlinkCmd(VdpaCmdDevNew, unix.NLM_F_ACK|unix.NLM_F_REQUEST, []*nl.RtAttr{busNameAttr, mgmtAttr, nameAttr}) + if err != nil { + return nil, err + } + + return msgs, nil +} + +/*DeleteVdpaDevice deletes a vdpa device */ +func DeleteVdpaDevice(vdpaDeviceName string) ([][]byte, error) { + if vdpaDeviceName == "" { + return nil, unix.EINVAL + } + + nameAttr, err := GetNetlinkOps().NewAttribute(VdpaAttrDevName, vdpaDeviceName) + if err != nil { + return nil, err + } + + msgs, err := GetNetlinkOps().RunVdpaNetlinkCmd(VdpaCmdDevDel, unix.NLM_F_ACK|unix.NLM_F_REQUEST, []*nl.RtAttr{nameAttr}) + if err != nil { + return nil, err + } + + return msgs, nil +} + +func parseDevLinkVdpaDevList(msgs [][]byte) ([]VdpaDevice, error) { + devices := make([]VdpaDevice, 0, len(msgs)) + + for _, m := range msgs { + attrs, err := nl.ParseRouteAttr(m[nl.SizeofGenlmsg:]) + if err != nil { + return nil, err + } + dev := &vdpaDev{} + if err = dev.parseAttributes(attrs); err != nil { + return nil, err + } + if err = dev.getBusInfo(); err != nil { + return nil, err + } + devices = append(devices, dev) + } + return devices, nil +} diff --git a/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/kvdpa.go b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/kvdpa.go deleted file mode 100644 index 24471adc2..000000000 --- a/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/kvdpa.go +++ /dev/null @@ -1,242 +0,0 @@ -package kvdpa - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" -) - -/*Exported constants */ -const ( - VhostVdpaDriver = "vhost_vdpa" - VirtioVdpaDriver = "virtio_vdpa" -) - -/*Private constants */ -const ( - vdpaBusDevDir = "/sys/bus/vdpa/devices" - pciBusDevDir = "/sys/bus/pci/devices" - vdpaVhostDevDir = "/dev" - virtioDevDir = "/sys/bus/virtio/devices" - rootDevDir = "/sys/devices" -) - -/*VdpaDevice contains information about a Vdpa Device*/ -type VdpaDevice interface { - GetDriver() string - GetParent() string - GetPath() string - GetNetDev() string -} - -/*vdpaDevimplements VdpaDevice interface */ -type vdpaDev struct { - name string - driver string - path string // Path of the vhost or virtio device - netdev string // VirtioNet netdev (only for virtio-vdpa devices) -} - -func (vd *vdpaDev) GetDriver() string { - return vd.driver -} - -func (vd *vdpaDev) GetParent() string { - return vd.name -} - -func (vd *vdpaDev) GetPath() string { - return vd.path -} - -func (vd *vdpaDev) GetNetDev() string { - return vd.netdev -} - -/*GetVdpaDeviceList returns a list of all available vdpa devices */ -func GetVdpaDeviceList() ([]VdpaDevice, error) { - vdpaDevList := make([]VdpaDevice, 0) - fd, err := os.Open(vdpaBusDevDir) - if err != nil { - return nil, err - } - defer fd.Close() - - fileInfos, err := fd.Readdir(-1) - if err != nil { - return nil, err - } - var errors []string - for _, file := range fileInfos { - if vdpaDev, err := GetVdpaDeviceByName(file.Name()); err != nil { - errors = append(errors, err.Error()) - } else { - vdpaDevList = append(vdpaDevList, vdpaDev) - } - } - - if len(errors) > 0 { - return vdpaDevList, fmt.Errorf(strings.Join(errors, ";")) - } - return vdpaDevList, nil -} - -/*GetVdpaDeviceByName returns the vdpa device information by a vdpa device name */ -func GetVdpaDeviceByName(name string) (VdpaDevice, error) { - var err error - var path string - var netdev string - - driverLink, err := os.Readlink(filepath.Join(vdpaBusDevDir, name, "driver")) - if err != nil { - return nil, err - } - - driver := filepath.Base(driverLink) - switch driver { - case VhostVdpaDriver: - path, err = getVhostVdpaDev(name) - if err != nil { - return nil, err - } - case VirtioVdpaDriver: - path, err = getVirtioVdpaDev(name) - if err != nil { - return nil, err - } - virtioNetDir := filepath.Join(path, "net") - netDeviceFiles, err := ioutil.ReadDir(virtioNetDir) - if err != nil || len(netDeviceFiles) != 1 { - return nil, fmt.Errorf("failed to get network device name from vdpa device in %v %v", name, err) - } - netdev = strings.TrimSpace(netDeviceFiles[0].Name()) - default: - return nil, fmt.Errorf("Unknown vdpa bus driver %s", driver) - } - - return &vdpaDev{ - name: name, - driver: driver, - path: path, - netdev: netdev, - }, nil -} - -/* Finds the vhost vdpa device of a vdpa device and returns it's path */ -func getVhostVdpaDev(name string) (string, error) { - file := filepath.Join(vdpaBusDevDir, name) - fd, err := os.Open(file) - if err != nil { - return "", err - } - defer fd.Close() - - fileInfos, err := fd.Readdir(-1) - for _, file := range fileInfos { - if strings.Contains(file.Name(), "vhost-vdpa") && - file.IsDir() { - devicePath := filepath.Join(vdpaVhostDevDir, file.Name()) - info, err := os.Stat(devicePath) - if err != nil { - return "", err - } - if info.Mode()&os.ModeDevice == 0 { - return "", fmt.Errorf("vhost device %s is not a valid device", devicePath) - } - return devicePath, nil - } - } - return "", fmt.Errorf("vhost device not found for vdpa device %s", name) -} - -/*GetVdpaDeviceByPci returns the vdpa device information corresponding to a PCI device*/ -/* Based on the following directory hiearchy: -/sys/bus/pci/devices/{PCIDev}/ - /vdpa{N}/ - -/sys/bus/vdpa/devices/vdpa{N} -> ../../../devices/pci.../{PCIDev}/vdpa{N} -*/ -func GetVdpaDeviceByPci(pciAddr string) (VdpaDevice, error) { - path, err := filepath.EvalSymlinks(filepath.Join(pciBusDevDir, pciAddr)) - if err != nil { - return nil, err - } - fd, err := os.Open(path) - if err != nil { - return nil, err - } - defer fd.Close() - - fileInfos, err := fd.Readdir(-1) - for _, file := range fileInfos { - if strings.Contains(file.Name(), "vdpa") { - parent, err := getParentDevice(filepath.Join(vdpaBusDevDir, file.Name())) - if err != nil { - return nil, err - } - if parent != path { - return nil, fmt.Errorf("vdpa device %s parent (%s) does not match containing dir (%s)", - file.Name(), parent, path) - } - return GetVdpaDeviceByName(file.Name()) - } - } - return nil, fmt.Errorf("PCI address %s does not contain a vdpa device", pciAddr) -} - -/* Finds the virtio vdpa device of a vdpa device and returns it's path -Currently, PCI-based devices have the following sysfs structure: -/sys/bus/vdpa/devices/ - vdpa1 -> ../../../devices/pci0000:00/0000:00:03.2/0000:05:00.2/vdpa1 - -In order to find the virtio device we look for virtio* devices inside the parent device: - sys/devices/pci0000:00/0000:00:03.2/0000:05:00.2/virtio{N} - -We also check the virtio device exists in the virtio bus: -/sys/bus/virtio/devices - virtio{N} -> ../../../devices/pci0000:00/0000:00:03.2/0000:05:00.2/virtio{N} -*/ -func getVirtioVdpaDev(name string) (string, error) { - vdpaDevicePath := filepath.Join(vdpaBusDevDir, name) - parentPath, err := getParentDevice(vdpaDevicePath) - if err != nil { - return "", err - } - - fd, err := os.Open(parentPath) - if err != nil { - return "", err - } - defer fd.Close() - - fileInfos, err := fd.Readdir(-1) - for _, file := range fileInfos { - if strings.Contains(file.Name(), "virtio") && - file.IsDir() { - virtioDevPath := filepath.Join(virtioDevDir, file.Name()) - if _, err := os.Stat(virtioDevPath); os.IsNotExist(err) { - return "", fmt.Errorf("virtio device %s does not exist", virtioDevPath) - } - return virtioDevPath, nil - } - } - - return "", fmt.Errorf("virtio device not found for vdpa device %s", name) -} - -/* getParentDevice returns the parent's path of a vdpa device path */ -func getParentDevice(path string) (string, error) { - devicePath, err := filepath.EvalSymlinks(path) - if err != nil { - return "", err - } - - parent := filepath.Dir(devicePath) - // if the "parent" is sys/devices, we have reached the "root" device - if parent == rootDevDir { - return devicePath, nil - } - return parent, nil -} diff --git a/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/mgmtdev.go b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/mgmtdev.go new file mode 100644 index 000000000..81e3b00fe --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/mgmtdev.go @@ -0,0 +1,114 @@ +package kvdpa + +import ( + "strings" + "syscall" + + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" +) + +// MgmtDev represents a Vdpa Management Device +type MgmtDev interface { + BusName() string // Optional + DevName() string // + Name() string // The MgmtDevName is BusName/DevName + // TODO: SupportedClasses() []string +} + +type mgmtDev struct { + busName string + devName string + // TODO: classes []string +} + +// BusName returns the MgmtDev's bus name +func (m *mgmtDev) BusName() string { + return m.busName +} + +// BusName returns the MgmtDev's device name +func (m *mgmtDev) DevName() string { + return m.devName +} + +// BusName returns the MgmtDev's name: [BusName/]DeviceName +func (m *mgmtDev) Name() string { + if m.busName != "" { + return strings.Join([]string{m.busName, m.devName}, "/") + } + return m.devName +} + +// parseAttributes parses the netlink attributes and populates the fields accordingly +func (m *mgmtDev) parseAttributes(attrs []syscall.NetlinkRouteAttr) error { + for _, a := range attrs { + switch a.Attr.Type { + case VdpaAttrMgmtDevBusName: + m.busName = string(a.Value[:len(a.Value)-1]) + case VdpaAttrMgmtDevDevName: + m.devName = string(a.Value[:len(a.Value)-1]) + } + } + return nil +} + +// ListVdpaMgmtDevices returns the list of all available MgmtDevs +func ListVdpaMgmtDevices() ([]MgmtDev, error) { + msgs, err := GetNetlinkOps().RunVdpaNetlinkCmd(VdpaCmdMgmtDevGet, unix.NLM_F_DUMP, nil) + if err != nil { + return nil, err + } + + mgtmDevs, err := parseDevLinkVdpaMgmtDevList(msgs) + if err != nil { + return nil, err + } + return mgtmDevs, nil +} + +// GetVdpaMgmtDevices returns a MgmtDev based on a busName and deviceName +func GetVdpaMgmtDevices(busName, devName string) (MgmtDev, error) { + data := []*nl.RtAttr{} + if busName != "" { + bus, err := GetNetlinkOps().NewAttribute(VdpaAttrMgmtDevBusName, busName) + if err != nil { + return nil, err + } + data = append(data, bus) + } + + dev, err := GetNetlinkOps().NewAttribute(VdpaAttrMgmtDevDevName, devName) + if err != nil { + return nil, err + } + data = append(data, dev) + + msgs, err := GetNetlinkOps().RunVdpaNetlinkCmd(VdpaCmdMgmtDevGet, 0, data) + if err != nil { + return nil, err + } + + mgtmDevs, err := parseDevLinkVdpaMgmtDevList(msgs) + if err != nil { + return nil, err + } + return mgtmDevs[0], nil +} + +func parseDevLinkVdpaMgmtDevList(msgs [][]byte) ([]MgmtDev, error) { + devices := make([]MgmtDev, 0, len(msgs)) + + for _, m := range msgs { + attrs, err := nl.ParseRouteAttr(m[nl.SizeofGenlmsg:]) + if err != nil { + return nil, err + } + dev := &mgmtDev{} + if err = dev.parseAttributes(attrs); err != nil { + return nil, err + } + devices = append(devices, dev) + } + return devices, nil +} diff --git a/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/netlink.go b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/netlink.go new file mode 100644 index 000000000..7035974bd --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/netlink.go @@ -0,0 +1,182 @@ +package kvdpa + +import ( + "fmt" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" +) + +/* Vdpa Netlink Name */ +const ( + VdpaGenlName = "vdpa" +) + +/* VDPA Netlink Commands */ +const ( + VdpaCmdUnspec uint8 = iota + VdpaCmdMgmtDevNew + VdpaCmdMgmtDevGet /* can dump */ + VdpaCmdDevNew + VdpaCmdDevDel + VdpaCmdDevGet /* can dump */ + VdpaCmdDevConfigGet /* can dump */ +) + +/* VDPA Netlink Attributes */ +const ( + VdpaAttrUnspec = iota + + /* bus name (optional) + dev name together make the parent device handle */ + VdpaAttrMgmtDevBusName /* string */ + VdpaAttrMgmtDevDevName /* string */ + VdpaAttrMgmtDevSupportedClasses /* u64 */ + + VdpaAttrDevName /* string */ + VdpaAttrDevID /* u32 */ + VdpaAttrDevVendorID /* u32 */ + VdpaAttrDevMaxVqs /* u32 */ + VdpaAttrDevMaxVqSize /* u16 */ + VdpaAttrDevMinVqSize /* u16 */ + + VdpaAttrDevNetCfgMacAddr /* binary */ + VdpaAttrDevNetStatus /* u8 */ + VdpaAttrDevNetCfgMaxVqp /* u16 */ + VdpaAttrGetNetCfgMTU /* u16 */ + + /* new attributes must be added above here */ + VdpaAttrMax +) + +var ( + commonNetlinkFlags = unix.NLM_F_REQUEST | unix.NLM_F_ACK +) + +// NetlinkOps defines the Netlink Operations +type NetlinkOps interface { + RunVdpaNetlinkCmd(command uint8, flags int, data []*nl.RtAttr) ([][]byte, error) + NewAttribute(attrType int, data interface{}) (*nl.RtAttr, error) +} + +type defaultNetlinkOps struct { +} + +var netlinkOps NetlinkOps = &defaultNetlinkOps{} + +// SetNetlinkOps method would be used by unit tests +func SetNetlinkOps(mockInst NetlinkOps) { + netlinkOps = mockInst +} + +// GetNetlinkOps will be invoked by functions in other packages that would need access to the sriovnet library methods. +func GetNetlinkOps() NetlinkOps { + return netlinkOps +} + +// RunVdpaNerlinkCmd runs a vdpa netlink command and returns the response +func (defaultNetlinkOps) RunVdpaNetlinkCmd(command uint8, flags int, data []*nl.RtAttr) ([][]byte, error) { + f, err := netlink.GenlFamilyGet(VdpaGenlName) + if err != nil { + return nil, err + } + + msg := &nl.Genlmsg{ + Command: command, + Version: nl.GENL_CTRL_VERSION, + } + req := nl.NewNetlinkRequest(int(f.ID), commonNetlinkFlags|flags) + + req.AddData(msg) + for _, d := range data { + req.AddData(d) + } + + msgs, err := req.Execute(unix.NETLINK_GENERIC, 0) + if err != nil { + return nil, err + } + return msgs, nil +} + +// NewAttribute returns a new netlink attribute based on the provided data +func (defaultNetlinkOps) NewAttribute(attrType int, data interface{}) (*nl.RtAttr, error) { + switch attrType { + case VdpaAttrMgmtDevBusName, VdpaAttrMgmtDevDevName, VdpaAttrDevName: + strData, ok := data.(string) + if !ok { + return nil, fmt.Errorf("attribute type %d requires string data", attrType) + } + bytes := make([]byte, len(strData)+1) + copy(bytes, strData) + return nl.NewRtAttr(attrType, bytes), nil + /* TODO + case: + VdpaAttrMgmtDevBusName string + VdpaAttrMgmtDevDevName string + VdpaAttrMgmtDevSupportedClasses u64 + + VdpaAttrDevName string + VdpaAttrDevID u32 + VdpaAttrDevVendorID u32 + VdpaAttrDevMaxVqs u32 + VdpaAttrDevMaxVqSize u16 + VdpaAttrDevMinVqSize u16 + + VdpaAttrDevNetCfgMacAddr binary + VdpaAttrDevNetStatus u8 + VdpaAttrDevNetCfgMaxVqp u16 + VdpaAttrGetNetCfgMTU u16 + */ + default: + return nil, fmt.Errorf("invalid attribute type %d", attrType) + } + +} + +func newMockSingleMessage(command uint8, attrs []*nl.RtAttr) []byte { + b := make([]byte, 0) + dataBytes := make([][]byte, len(attrs)+1) + + msg := &nl.Genlmsg{ + Command: command, + Version: nl.GENL_CTRL_VERSION, + } + dataBytes[0] = msg.Serialize() + + for i, attr := range attrs { + dataBytes[i+1] = attr.Serialize() + } + next := 0 + for _, data := range dataBytes { + for _, dataByte := range data { + b = append(b, dataByte) + next = next + 1 + } + } + return b + /* + nlm := &nl.NetlinkRequest{ + NlMsghdr: unix.NlMsghdr{ + Len: uint32(unix.SizeofNlMsghdr), + Type: 0xa, + Flags: 0, + Seq: 1, + }, + } + for _, a := range attrs { + nlm.AddData(a) + } + return nlm.Serialize() + */ +} + +// Used for unit tests +func newMockNetLinkResponse(command uint8, data [][]*nl.RtAttr) [][]byte { + msgs := make([][]byte, len(data)) + for i, msgData := range data { + msgDataBytes := newMockSingleMessage(command, msgData) + msgs[i] = msgDataBytes + } + return msgs +} diff --git a/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/vhost.go b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/vhost.go new file mode 100644 index 000000000..5067dcf9e --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/vhost.go @@ -0,0 +1,62 @@ +package kvdpa + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// VhostVdpa is the vhost-vdpa device information +type VhostVdpa interface { + Name() string + Path() string +} + +// vhostVdpa implements VhostVdpa interface +type vhostVdpa struct { + name string + path string +} + +// Name returns the vhost device's name +func (v *vhostVdpa) Name() string { + return v.name +} + +// Name returns the vhost device's path +func (v *vhostVdpa) Path() string { + return v.path +} + +// GetVhostVdpaDevInPath returns the VhostVdpa found in the provided parent device's path +func GetVhostVdpaDevInPath(parentPath string) (VhostVdpa, error) { + fd, err := os.Open(parentPath) + if err != nil { + return nil, err + } + defer fd.Close() + + fileInfos, err := fd.Readdir(-1) + if err != nil { + return nil, err + } + for _, file := range fileInfos { + if strings.Contains(file.Name(), "vhost-vdpa") && + file.IsDir() { + devicePath := filepath.Join(vdpaVhostDevDir, file.Name()) + info, err := os.Stat(devicePath) + if err != nil { + return nil, err + } + if info.Mode()&os.ModeDevice == 0 { + return nil, fmt.Errorf("vhost device %s is not a valid device", devicePath) + } + return &vhostVdpa{ + name: file.Name(), + path: devicePath, + }, nil + } + } + return nil, fmt.Errorf("no VhostVdpa device foiund in path %s", parentPath) +} diff --git a/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/virtio.go b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/virtio.go new file mode 100644 index 000000000..3b5a35e7f --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/virtio.go @@ -0,0 +1,69 @@ +package kvdpa + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const ( + virtioDevDir = "/sys/bus/virtio/devices" +) + +// VirtioNet is the virtio-net device information +type VirtioNet interface { + Name() string + NetDev() string +} + +// virtioNet implements VirtioNet interface +type virtioNet struct { + name string + netDev string +} + +// Name returns the virtio device's name (as appears in the virtio bus) +func (v *virtioNet) Name() string { + return v.name +} + +// NetDev returns the virtio-net netdev name +func (v *virtioNet) NetDev() string { + return v.netDev +} + +// GetVirtioNetInPath returns the VirtioNet found in the provided parent device's path +func GetVirtioNetInPath(parentPath string) (VirtioNet, error) { + fd, err := os.Open(parentPath) + if err != nil { + return nil, err + } + defer fd.Close() + + fileInfos, err := fd.Readdir(-1) + if err != nil { + return nil, err + } + for _, file := range fileInfos { + if strings.Contains(file.Name(), "virtio") && + file.IsDir() { + virtioDevPath := filepath.Join(virtioDevDir, file.Name()) + if _, err := os.Stat(virtioDevPath); os.IsNotExist(err) { + return nil, fmt.Errorf("virtio device %s does not exist", virtioDevPath) + } + var netdev string + // Read the "net" directory in the virtio device path + netDeviceFiles, err := ioutil.ReadDir(filepath.Join(virtioDevPath, "net")) + if err == nil && len(netDeviceFiles) == 1 { + netdev = strings.TrimSpace(netDeviceFiles[0].Name()) + } + return &virtioNet{ + name: file.Name(), + netDev: netdev, + }, nil + } + } + return nil, fmt.Errorf("no VirtioNet device found in path %s", parentPath) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6d07261ab..9533f61cc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -94,7 +94,7 @@ github.com/josharian/intern # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/k8snetworkplumbingwg/govdpa v0.1.3 +# github.com/k8snetworkplumbingwg/govdpa v0.1.3 => github.com/lmilleri/govdpa v0.1.7 ## explicit; go 1.14 github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa # github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d