From 2886ca62cb7ec3efb3cb3e6ad42055c84e478820 Mon Sep 17 00:00:00 2001 From: amaslennikov Date: Tue, 26 Mar 2024 10:49:23 +0300 Subject: [PATCH] Create instance of IB interface in the host object Signed-off-by: amaslennikov --- pkg/helper/host.go | 6 ++++- pkg/helper/mock/mock_helper.go | 14 ++++++++++ pkg/host/internal/sriov/sriov.go | 37 +++++++++++++++------------ pkg/host/internal/sriov/sriov_test.go | 22 +++++----------- pkg/host/manager.go | 14 +++++++--- pkg/host/mock/mock_host.go | 14 ++++++++++ pkg/platforms/platforms.go | 8 +++++- pkg/plugins/k8s/k8s_plugin_test.go | 2 +- 8 files changed, 78 insertions(+), 39 deletions(-) diff --git a/pkg/helper/host.go b/pkg/helper/host.go index 5829d161a..2e93a30d3 100644 --- a/pkg/helper/host.go +++ b/pkg/helper/host.go @@ -35,7 +35,11 @@ func NewHostHelpers(utilsHelper utils.CmdInterface, func NewDefaultHostHelpers() (HostHelpersInterface, error) { utilsHelper := utils.New() mlxHelper := mlx.New(utilsHelper) - hostManager := host.NewHostManager(utilsHelper) + hostManager, err := host.NewHostManager(utilsHelper) + if err != nil { + log.Log.Error(err, "failed to create host manager") + return nil, err + } storeManager, err := store.NewManager() if err != nil { log.Log.Error(err, "failed to create store manager") diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go index a45e6e9e4..23b32a1d4 100644 --- a/pkg/helper/mock/mock_helper.go +++ b/pkg/helper/mock/mock_helper.go @@ -194,6 +194,20 @@ func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovInterfaces(storeManag return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, skipVFConfiguration) } +// ConfigureVfGUID mocks base method. +func (m *MockHostHelpersInterface) ConfigureVfGUID(vfAddr, pfAddr string, vfID int, pfLink netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigureVfGUID", vfAddr, pfAddr, vfID, pfLink) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigureVfGUID indicates an expected call of ConfigureVfGUID. +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureVfGUID", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigureVfGUID), vfAddr, pfAddr, vfID, pfLink) +} + // CreateVDPADevice mocks base method. func (m *MockHostHelpersInterface) CreateVDPADevice(pciAddr, vdpaType string) error { m.ctrl.T.Helper() diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index 0d7b4c262..64f1fa537 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -33,16 +33,16 @@ type interfaceToConfigure struct { } type sriov struct { - utilsHelper utils.CmdInterface - kernelHelper types.KernelInterface - networkHelper types.NetworkInterface - udevHelper types.UdevInterface - vdpaHelper types.VdpaInterface + utilsHelper utils.CmdInterface + kernelHelper types.KernelInterface + networkHelper types.NetworkInterface + udevHelper types.UdevInterface + vdpaHelper types.VdpaInterface infinibandHelper types.InfinibandInterface - netlinkLib netlinkPkg.NetlinkLib - dputilsLib dputilsPkg.DPUtilsLib - sriovnetLib sriovnetPkg.SriovnetLib - ghwLib ghwPkg.GHWLib + netlinkLib netlinkPkg.NetlinkLib + dputilsLib dputilsPkg.DPUtilsLib + sriovnetLib sriovnetPkg.SriovnetLib + ghwLib ghwPkg.GHWLib } func New(utilsHelper utils.CmdInterface, @@ -56,15 +56,15 @@ func New(utilsHelper utils.CmdInterface, sriovnetLib sriovnetPkg.SriovnetLib, ghwLib ghwPkg.GHWLib) types.SriovInterface { return &sriov{utilsHelper: utilsHelper, - kernelHelper: kernelHelper, - networkHelper: networkHelper, - udevHelper: udevHelper, - vdpaHelper: vdpaHelper, + kernelHelper: kernelHelper, + networkHelper: networkHelper, + udevHelper: udevHelper, + vdpaHelper: vdpaHelper, infinibandHelper: infinibandHelper, - netlinkLib: netlinkLib, - dputilsLib: dputilsLib, - sriovnetLib: sriovnetLib, - ghwLib: ghwLib, + netlinkLib: netlinkLib, + dputilsLib: dputilsLib, + sriovnetLib: sriovnetLib, + ghwLib: ghwLib, } } @@ -476,6 +476,9 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error { if err := s.infinibandHelper.ConfigureVfGUID(addr, iface.PciAddress, vfID, pfLink); err != nil { return err } + if err := s.kernelHelper.Unbind(iface.PciAddress); err != nil { + return err + } } else { vfLink, err := s.VFIsReady(addr) if err != nil { diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index 6d2e43a71..f89ecd3c5 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -35,8 +35,7 @@ var _ = Describe("SRIOV", func() { ghwLibMock *ghwMockPkg.MockGHWLib hostMock *hostMockPkg.MockHostManagerInterface storeManagerMode *hostStoreMockPkg.MockManagerInterface - - testCtrl *gomock.Controller + testCtrl *gomock.Controller testError = fmt.Errorf("test") ) @@ -50,7 +49,7 @@ var _ = Describe("SRIOV", func() { hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl) - s = New(nil, hostMock, hostMock, hostMock, hostMock, nil, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock) + s = New(nil, hostMock, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock) }) AfterEach(func() { @@ -227,7 +226,6 @@ var _ = Describe("SRIOV", func() { pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"}) netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).Times(0) dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") @@ -296,16 +294,14 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) - dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) - hostMock.EXPECT().Unbind("0000:d8:00.2").Return(nil) + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(1) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(true, "test").Times(2) hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.2", true).Return(nil) hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) hostMock.EXPECT().SetNetdevMTU("0000:d8:00.2", 2000).Return(nil) - vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) - netlinkLibMock.EXPECT().LinkSetVfNodeGUID(vf0LinkMock, 0, gomock.Any()).Return(nil) - netlinkLibMock.EXPECT().LinkSetVfPortGUID(vf0LinkMock, 0, gomock.Any()).Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() + hostMock.EXPECT().ConfigureVfGUID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + + hostMock.EXPECT().Unbind(gomock.Any()).Return(nil).Times(1) storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) @@ -372,7 +368,6 @@ var _ = Describe("SRIOV", func() { hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil) hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa") hostMock.EXPECT().LoadUdevRules().Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) @@ -400,7 +395,6 @@ var _ = Describe("SRIOV", func() { It("externally managed - wrong VF count", func() { dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() Expect(s.ConfigSriovInterfaces(storeManagerMode, []sriovnetworkv1.Interface{{ @@ -426,7 +420,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().GetNetdevMTU("0000:d8:00.0") Expect(s.ConfigSriovInterfaces(storeManagerMode, @@ -463,7 +456,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) @@ -482,7 +474,6 @@ var _ = Describe("SRIOV", func() { helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "0") }) It("reset device - skip external", func() { - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{ Name: "enp216s0f0np0", PciAddress: "0000:d8:00.0", @@ -511,7 +502,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) diff --git a/pkg/host/manager.go b/pkg/host/manager.go index de7f6e3b6..b70f6b387 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -1,6 +1,7 @@ package host import ( + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/infiniband" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool" @@ -26,6 +27,7 @@ type HostManagerInterface interface { types.UdevInterface types.SriovInterface types.VdpaInterface + types.InfinibandInterface } type hostManager struct { @@ -36,9 +38,10 @@ type hostManager struct { types.UdevInterface types.SriovInterface types.VdpaInterface + types.InfinibandInterface } -func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { +func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, error) { dpUtils := dputils.New() netlinkLib := netlink.New() ethtoolLib := ethtool.New() @@ -49,7 +52,11 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { sv := service.New(utilsInterface) u := udev.New(utilsInterface) v := vdpa.New(k, netlinkLib) - sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils, sriovnetLib, ghwLib) + ib, err := infiniband.New(netlinkLib, k, n) + if err != nil { + return nil, err + } + sr := sriov.New(utilsInterface, k, n, u, v, ib, netlinkLib, dpUtils, sriovnetLib, ghwLib) return &hostManager{ utilsInterface, @@ -59,5 +66,6 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { u, sr, v, - } + ib, + }, nil } diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go index d787a4640..72155f557 100644 --- a/pkg/host/mock/mock_host.go +++ b/pkg/host/mock/mock_host.go @@ -164,6 +164,20 @@ func (mr *MockHostManagerInterfaceMockRecorder) ConfigSriovInterfaces(storeManag return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, skipVFConfiguration) } +// ConfigureVfGUID mocks base method. +func (m *MockHostManagerInterface) ConfigureVfGUID(vfAddr, pfAddr string, vfID int, pfLink netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigureVfGUID", vfAddr, pfAddr, vfID, pfLink) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigureVfGUID indicates an expected call of ConfigureVfGUID. +func (mr *MockHostManagerInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureVfGUID", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigureVfGUID), vfAddr, pfAddr, vfID, pfLink) +} + // CreateVDPADevice mocks base method. func (m *MockHostManagerInterface) CreateVDPADevice(pciAddr, vdpaType string) error { m.ctrl.T.Helper() diff --git a/pkg/platforms/platforms.go b/pkg/platforms/platforms.go index 19c74cef3..3320dbaac 100644 --- a/pkg/platforms/platforms.go +++ b/pkg/platforms/platforms.go @@ -1,6 +1,8 @@ package platforms import ( + "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openstack" @@ -24,7 +26,11 @@ func NewDefaultPlatformHelper() (Interface, error) { return nil, err } utilsHelper := utils.New() - hostManager := host.NewHostManager(utilsHelper) + hostManager, err := host.NewHostManager(utilsHelper) + if err != nil { + log.Log.Error(err, "failed to create host manager") + return nil, err + } openstackContext := openstack.New(hostManager) return &platformHelper{ diff --git a/pkg/plugins/k8s/k8s_plugin_test.go b/pkg/plugins/k8s/k8s_plugin_test.go index 28a2e85c1..b4344b95f 100644 --- a/pkg/plugins/k8s/k8s_plugin_test.go +++ b/pkg/plugins/k8s/k8s_plugin_test.go @@ -79,7 +79,7 @@ var _ = Describe("K8s plugin", func() { testCtrl = gomock.NewController(GinkgoT()) hostHelper = mock_helper.NewMockHostHelpersInterface(testCtrl) - realHostMgr := host.NewHostManager(hostHelper) + realHostMgr, _ := host.NewHostManager(hostHelper) // proxy some functions to real host manager to simplify testing and to additionally validate manifests for _, f := range []string{