diff --git a/internal/controllers/nmc_reconciler.go b/internal/controllers/nmc_reconciler.go index c20ec20ed..106314dbd 100644 --- a/internal/controllers/nmc_reconciler.go +++ b/internal/controllers/nmc_reconciler.go @@ -55,6 +55,9 @@ const ( volumeNameConfig = "config" workerContainerName = "worker" globalPullSecretName = "global-pull-secret" + initContainerName = "image-extractor" + sharedFilesDir = "/tmp" + volNameTmp = "tmp" ) //+kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=nodemodulesconfigs,verbs=get;list;watch @@ -834,7 +837,7 @@ func (p *podManagerImpl) ListWorkerPodsOnNode(ctx context.Context, nodeName stri } func (p *podManagerImpl) LoaderPodTemplate(ctx context.Context, nmc client.Object, nms *kmmv1beta1.NodeModuleSpec) (*v1.Pod, error) { - pod, err := p.baseWorkerPod(ctx, nmc.GetName(), &nms.ModuleItem, nmc) + pod, err := p.baseWorkerPod(ctx, nmc, &nms.ModuleItem, &nms.Config) if err != nil { return nil, fmt.Errorf("could not create the base Pod: %v", err) } @@ -856,6 +859,13 @@ func (p *podManagerImpl) LoaderPodTemplate(ctx context.Context, nmc client.Objec } args = append(args, "--"+worker.FlagFirmwarePath, *firmwareHostPath) + + firmwarePathContainerImg := filepath.Join(nms.Config.Modprobe.FirmwarePath, "*") + firmwarePathWorkerImg := filepath.Join(sharedFilesDir, nms.Config.Modprobe.FirmwarePath) + if err = addCopyCommand(pod, firmwarePathContainerImg, firmwarePathWorkerImg); err != nil { + return nil, fmt.Errorf("could not add the copy command to the init container: %v", err) + } + if err = setFirmwareVolume(pod, firmwareHostPath); err != nil { return nil, fmt.Errorf("could not map host volume needed for firmware loading: %v", err) } @@ -881,7 +891,7 @@ func (p *podManagerImpl) LoaderPodTemplate(ctx context.Context, nmc client.Objec } func (p *podManagerImpl) UnloaderPodTemplate(ctx context.Context, nmc client.Object, nms *kmmv1beta1.NodeModuleStatus) (*v1.Pod, error) { - pod, err := p.baseWorkerPod(ctx, nmc.GetName(), &nms.ModuleItem, nmc) + pod, err := p.baseWorkerPod(ctx, nmc, &nms.ModuleItem, &nms.Config) if err != nil { return nil, fmt.Errorf("could not create the base Pod: %v", err) } @@ -908,6 +918,13 @@ func (p *podManagerImpl) UnloaderPodTemplate(ctx context.Context, nmc client.Obj return nil, fmt.Errorf("firmwareHostPath was not set while the Module requires firmware unloading") } args = append(args, "--"+worker.FlagFirmwarePath, *firmwareHostPath) + + firmwarePathContainerImg := filepath.Join(nms.Config.Modprobe.FirmwarePath, "*") + firmwarePathWorkerImg := filepath.Join(sharedFilesDir, nms.Config.Modprobe.FirmwarePath) + if err = addCopyCommand(pod, firmwarePathContainerImg, firmwarePathWorkerImg); err != nil { + return nil, fmt.Errorf("could not add the copy command to the init container: %v", err) + } + if err = setFirmwareVolume(pod, firmwareHostPath); err != nil { return nil, fmt.Errorf("could not map host volume needed for firmware unloading: %v", err) } @@ -933,12 +950,26 @@ var ( } ) -func (p *podManagerImpl) baseWorkerPod( - ctx context.Context, - nodeName string, - item *kmmv1beta1.ModuleItem, - owner client.Object, -) (*v1.Pod, error) { +func addCopyCommand(pod *v1.Pod, src, dst string) error { + + container, _ := podcmd.FindContainerByName(pod, initContainerName) + if container == nil { + return errors.New("could not find the init container") + } + + const template = ` +mkdir -p %s; +cp -R %s %s; +` + copyCommand := fmt.Sprintf(template, dst, src, dst) + container.Args[0] = strings.Join([]string{container.Args[0], copyCommand}, "") + + return nil +} + +func (p *podManagerImpl) baseWorkerPod(ctx context.Context, nmc client.Object, item *kmmv1beta1.ModuleItem, + moduleConfig *kmmv1beta1.ModuleConfig) (*v1.Pod, error) { + const ( trustedCAVolumeName = "trusted-ca" volNameEtcContainers = "etc-containers" @@ -1048,6 +1079,12 @@ func (p *podManagerImpl) baseWorkerPod( }, }, }, + { + Name: volNameTmp, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, } volumeMounts := []v1.VolumeMount{ @@ -1082,8 +1119,14 @@ func (p *podManagerImpl) baseWorkerPod( ReadOnly: true, MountPath: filepath.Join(worker.PullSecretsDir, "_global", v1.DockerConfigJsonKey), }, + { + Name: volNameTmp, + MountPath: sharedFilesDir, + ReadOnly: true, + }, } + nodeName := nmc.GetName() pod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: item.Namespace, @@ -1096,6 +1139,24 @@ func (p *podManagerImpl) baseWorkerPod( }, }, Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: initContainerName, + Image: moduleConfig.ContainerImage, + Command: []string{"/bin/sh", "-c"}, + Args: []string{""}, + VolumeMounts: []v1.VolumeMount{ + { + Name: volNameTmp, + MountPath: sharedFilesDir, + }, + }, + Resources: v1.ResourceRequirements{ + Requests: requests, + Limits: limits, + }, + }, + }, Containers: []v1.Container{ { Name: workerContainerName, @@ -1114,10 +1175,16 @@ func (p *podManagerImpl) baseWorkerPod( }, } - if err = ctrl.SetControllerReference(owner, &pod, p.scheme); err != nil { + if err = ctrl.SetControllerReference(nmc, &pod, p.scheme); err != nil { return nil, fmt.Errorf("could not set the owner as controller: %v", err) } + kmodsPathContainerImg := filepath.Join(moduleConfig.Modprobe.DirName, "lib", "modules", moduleConfig.KernelVersion) + kmodsPathWorkerImg := filepath.Join(sharedFilesDir, moduleConfig.Modprobe.DirName, "lib", "modules") + if err = addCopyCommand(&pod, kmodsPathContainerImg, kmodsPathWorkerImg); err != nil { + return nil, fmt.Errorf("could not add the copy command to the init container: %v", err) + } + controllerutil.AddFinalizer(&pod, nodeModulesConfigFinalizer) return &pod, nil @@ -1227,6 +1294,7 @@ func setFirmwareVolume(pod *v1.Pod, firmwareHostPath *string) error { pod.Spec.Volumes = append(pod.Spec.Volumes, firmwareVolume) container.VolumeMounts = append(container.VolumeMounts, firmwareVolumeMount) + return nil } diff --git a/internal/controllers/nmc_reconciler_test.go b/internal/controllers/nmc_reconciler_test.go index 971d2905a..dbd555e49 100644 --- a/internal/controllers/nmc_reconciler_test.go +++ b/internal/controllers/nmc_reconciler_test.go @@ -259,7 +259,7 @@ var _ = Describe("NodeModulesConfigReconciler_Reconcile", func() { }) var moduleConfig = kmmv1beta1.ModuleConfig{ - KernelVersion: "kernel version", + KernelVersion: "kernel-version", ContainerImage: "container image", InsecurePull: true, InTreeModulesToRemove: []string{"intree1", "intree2"}, @@ -1791,7 +1791,7 @@ var _ = Describe("podManagerImpl_CreateUnloaderPod", func() { It("should work as expected", func() { - expected := getBaseWorkerPod("unload", nmc, ptr.To("some-path"), true, false) + expected := getBaseWorkerPod("unload", nmc, ptr.To("/var/lib/firmware"), true, false) container, _ := podcmd.FindContainerByName(expected, "worker") Expect(container).NotTo(BeNil()) @@ -1817,7 +1817,8 @@ var _ = Describe("podManagerImpl_CreateUnloaderPod", func() { ) workerCfg := *workerCfg - workerCfg.FirmwareHostPath = ptr.To("some-path") + workerCfg.FirmwareHostPath = ptr.To("/var/lib/firmware") + pm := newPodManager(client, workerImage, scheme, caHelper, &workerCfg) pm.(*podManagerImpl).psh = psh @@ -1946,7 +1947,7 @@ inTreeModulesToRemove: - intree1 - intree2 insecurePull: true -kernelVersion: kernel version +kernelVersion: kernel-version modprobe: dirName: /dir firmwarePath: /firmware-path @@ -1961,11 +1962,22 @@ modprobe: ` modulesOrderValue := `softdep a pre: b softdep b pre: c +` + + var initContainerArg = ` +mkdir -p /tmp/dir/lib/modules; +cp -R /dir/lib/modules/kernel-version /tmp/dir/lib/modules; +` + + const initContainerArgFirmwareAddition = ` +mkdir -p /tmp/firmware-path; +cp -R /firmware-path/* /tmp/firmware-path; ` args := []string{"kmod", subcommand, "/etc/kmm-worker/config.yaml"} if withFirmware { args = append(args, "--firmware-path", *firmwareHostPath) + initContainerArg = strings.Join([]string{initContainerArg, initContainerArgFirmwareAddition}, "") } else { configAnnotationValue = strings.ReplaceAll(configAnnotationValue, "firmwarePath: /firmware-path\n ", "") } @@ -1986,6 +1998,24 @@ softdep b pre: c }, }, Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "image-extractor", + Image: "container image", + Command: []string{"/bin/sh", "-c"}, + Args: []string{initContainerArg}, + Resources: v1.ResourceRequirements{ + Limits: limits, + Requests: requests, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: volNameTmp, + MountPath: sharedFilesDir, + }, + }, + }, + }, Containers: []v1.Container{ { Name: "worker", @@ -2027,6 +2057,11 @@ softdep b pre: c ReadOnly: true, MountPath: filepath.Join(worker.PullSecretsDir, "_global", v1.DockerConfigJsonKey), }, + { + Name: volNameTmp, + MountPath: sharedFilesDir, + ReadOnly: true, + }, { Name: volNameModulesOrder, ReadOnly: true, @@ -2121,6 +2156,12 @@ softdep b pre: c }, }, }, + { + Name: volNameTmp, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, { Name: volNameModulesOrder, VolumeSource: v1.VolumeSource{ diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 1ce679510..61d499e20 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -35,13 +35,9 @@ func NewWorker(im ImageMounter, mr ModprobeRunner, logger logr.Logger) Worker { } } -func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, firmwareMountPath string) error { - imageName := cfg.ContainerImage +const sharedFilesDir = "/tmp" - fsDir, err := w.im.MountImage(ctx, imageName, cfg) - if err != nil { - return fmt.Errorf("failed to mount image %s: %v", imageName, err) - } +func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, firmwareMountPath string) error { inTreeModulesToRemove := cfg.InTreeModulesToRemove // [TODO] - remove handling cfg.InTreeModuleToRemove once we cease to support it @@ -53,14 +49,14 @@ func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, fir w.logger.Info("Unloading in-tree modules", "names", inTreeModulesToRemove) runArgs := append([]string{"-rv"}, inTreeModulesToRemove...) - if err = w.mr.Run(ctx, runArgs...); err != nil { + if err := w.mr.Run(ctx, runArgs...); err != nil { return fmt.Errorf("could not remove in-tree modules %s: %v", strings.Join(inTreeModulesToRemove, ""), err) } } // prepare firmware if cfg.Modprobe.FirmwarePath != "" { - imageFirmwarePath := filepath.Join(fsDir, cfg.Modprobe.FirmwarePath) + imageFirmwarePath := filepath.Join(sharedFilesDir, cfg.Modprobe.FirmwarePath) w.logger.Info("preparing firmware for loading", "image directory", imageFirmwarePath, "host mount directory", firmwareMountPath) options := cp.Options{ OnError: func(src, dest string, err error) error { @@ -70,7 +66,7 @@ func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, fir return nil }, } - if err = cp.Copy(imageFirmwarePath, firmwareMountPath, options); err != nil { + if err := cp.Copy(imageFirmwarePath, firmwareMountPath, options); err != nil { return fmt.Errorf("failed to copy firmware from path %s to path %s: %v", imageFirmwarePath, firmwareMountPath, err) } } @@ -82,7 +78,7 @@ func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, fir if cfg.Modprobe.RawArgs != nil { args = cfg.Modprobe.RawArgs.Load } else { - args = []string{"-vd", filepath.Join(fsDir, cfg.Modprobe.DirName)} + args = []string{"-vd", filepath.Join(sharedFilesDir, cfg.Modprobe.DirName)} if cfg.Modprobe.Args != nil { args = append(args, cfg.Modprobe.Args.Load...) @@ -120,12 +116,6 @@ func (w *worker) SetFirmwareClassPath(value string) error { } func (w *worker) UnloadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, firmwareMountPath string) error { - imageName := cfg.ContainerImage - - fsDir, err := w.im.MountImage(ctx, imageName, cfg) - if err != nil { - return fmt.Errorf("failed to mount image %s: %v", imageName, err) - } moduleName := cfg.Modprobe.ModuleName @@ -134,7 +124,7 @@ func (w *worker) UnloadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, f if cfg.Modprobe.RawArgs != nil { args = cfg.Modprobe.RawArgs.Unload } else { - args = []string{"-rvd", filepath.Join(fsDir, cfg.Modprobe.DirName)} + args = []string{"-rvd", filepath.Join(sharedFilesDir, cfg.Modprobe.DirName)} if cfg.Modprobe.Args != nil { args = append(args, cfg.Modprobe.Args.Unload...) @@ -145,14 +135,14 @@ func (w *worker) UnloadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, f w.logger.Info("Unloading module", "name", moduleName) - if err = w.mr.Run(ctx, args...); err != nil { + if err := w.mr.Run(ctx, args...); err != nil { return fmt.Errorf("could not unload module %s: %v", moduleName, err) } //remove firmware files only (no directories) if cfg.Modprobe.FirmwarePath != "" { - imageFirmwarePath := filepath.Join(fsDir, cfg.Modprobe.FirmwarePath) - err = filepath.Walk(imageFirmwarePath, func(path string, info os.FileInfo, err error) error { + imageFirmwarePath := filepath.Join(sharedFilesDir, cfg.Modprobe.FirmwarePath) + err := filepath.Walk(imageFirmwarePath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } diff --git a/internal/worker/worker_test.go b/internal/worker/worker_test.go index 5d70ba48b..1488b21bc 100644 --- a/internal/worker/worker_test.go +++ b/internal/worker/worker_test.go @@ -49,22 +49,6 @@ var _ = Describe("worker_LoadKmod", func() { moduleName = "test" ) - It("should return an error if the image could not be pulled", func() { - cfg := v1beta1.ModuleConfig{ - ContainerImage: imageName, - } - im. - EXPECT(). - MountImage(ctx, imageName, &cfg). - Return("", errors.New("random error")) - - Expect( - w.LoadKmod(ctx, &cfg, ""), - ).To( - HaveOccurred(), - ) - }) - It("should return an error if modprobe failed", func() { cfg := v1beta1.ModuleConfig{ ContainerImage: imageName, @@ -74,10 +58,7 @@ var _ = Describe("worker_LoadKmod", func() { }, } - gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg), - mr.EXPECT().Run(ctx, "-vd", dirName, moduleName).Return(errors.New("random error")), - ) + mr.EXPECT().Run(ctx, "-vd", filepath.Join(sharedFilesDir, dirName), moduleName).Return(errors.New("random error")) Expect( w.LoadKmod(ctx, &cfg, ""), @@ -99,9 +80,8 @@ var _ = Describe("worker_LoadKmod", func() { } gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg), mr.EXPECT().Run(ctx, "-rv", "intree1", "intree2"), - mr.EXPECT().Run(ctx, "-vd", dirName, moduleName), + mr.EXPECT().Run(ctx, "-vd", filepath.Join(sharedFilesDir, dirName), moduleName), ) Expect( @@ -123,9 +103,8 @@ var _ = Describe("worker_LoadKmod", func() { } gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg), mr.EXPECT().Run(ctx, "-rv", "intreeToRemove"), - mr.EXPECT().Run(ctx, "-vd", dirName, moduleName), + mr.EXPECT().Run(ctx, "-vd", filepath.Join(sharedFilesDir, dirName), moduleName), ) Expect( @@ -145,17 +124,14 @@ var _ = Describe("worker_LoadKmod", func() { }, } - err := os.MkdirAll(imageDir+"/"+"firmwareDir/binDir", 0750) + err := os.MkdirAll(filepath.Join(sharedFilesDir, "firmwareDir", "binDir"), 0750) Expect(err).Should(BeNil()) - err = os.WriteFile(imageDir+"/"+"firmwareDir/firwmwareFile1", []byte("some data 1"), 0660) + err = os.WriteFile(filepath.Join(sharedFilesDir, "firmwareDir", "firwmwareFile1"), []byte("some data 1"), 0660) Expect(err).Should(BeNil()) - err = os.WriteFile(imageDir+"/"+"firmwareDir/binDir/firwmwareFile2", []byte("some data 2"), 0660) + err = os.WriteFile(filepath.Join(sharedFilesDir, "firmwareDir", "binDir", "firwmwareFile2"), []byte("some data 2"), 0660) Expect(err).Should(BeNil()) - gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg).Return(imageDir, nil), - mr.EXPECT().Run(ctx, "-vd", imageDir+dirName, moduleName), - ) + mr.EXPECT().Run(ctx, "-vd", filepath.Join(sharedFilesDir, dirName), moduleName) Expect( w.LoadKmod(ctx, &cfg, hostDir), @@ -180,10 +156,7 @@ var _ = Describe("worker_LoadKmod", func() { }, } - gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg), - mr.EXPECT().Run(ctx, ToInterfaceSlice(rawArgs)...), - ) + mr.EXPECT().Run(ctx, ToInterfaceSlice(rawArgs)...) Expect( w.LoadKmod(ctx, &cfg, ""), @@ -203,12 +176,7 @@ var _ = Describe("worker_LoadKmod", func() { }, } - gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg), - mr. - EXPECT(). - Run(ctx, "-vd", dirName, "a", "b", "c", moduleName, "key0=value0", "key1=value1"), - ) + mr.EXPECT().Run(ctx, "-vd", filepath.Join(sharedFilesDir, dirName), "a", "b", "c", moduleName, "key0=value0", "key1=value1") Expect( w.LoadKmod(ctx, &cfg, ""), @@ -301,23 +269,6 @@ var _ = Describe("worker_UnloadKmod", func() { moduleName = "test" ) - It("should return an error if the image could not be pulled", func() { - cfg := v1beta1.ModuleConfig{ - ContainerImage: imageName, - } - - im. - EXPECT(). - MountImage(ctx, imageName, &cfg). - Return("", errors.New("random error")) - - Expect( - w.UnloadKmod(ctx, &cfg, ""), - ).To( - HaveOccurred(), - ) - }) - It("should return an error if modprobe failed", func() { cfg := v1beta1.ModuleConfig{ ContainerImage: imageName, @@ -327,10 +278,7 @@ var _ = Describe("worker_UnloadKmod", func() { }, } - gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg), - mr.EXPECT().Run(ctx, "-rvd", dirName, moduleName).Return(errors.New("random error")), - ) + mr.EXPECT().Run(ctx, "-rvd", filepath.Join(sharedFilesDir, dirName), moduleName).Return(errors.New("random error")) Expect( w.UnloadKmod(ctx, &cfg, ""), @@ -349,10 +297,7 @@ var _ = Describe("worker_UnloadKmod", func() { }, } - gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg), - mr.EXPECT().Run(ctx, ToInterfaceSlice(rawArgs)...), - ) + mr.EXPECT().Run(ctx, ToInterfaceSlice(rawArgs)...) Expect( w.UnloadKmod(ctx, &cfg, ""), @@ -371,12 +316,7 @@ var _ = Describe("worker_UnloadKmod", func() { }, } - gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg), - mr. - EXPECT(). - Run(ctx, "-rvd", dirName, "a", "b", "c", moduleName), - ) + mr.EXPECT().Run(ctx, "-rvd", filepath.Join(sharedFilesDir, dirName), "a", "b", "c", moduleName) Expect( w.UnloadKmod(ctx, &cfg, ""), @@ -411,12 +351,7 @@ var _ = Describe("worker_UnloadKmod", func() { err = os.WriteFile(hostDir+"/binDir/firmwareFile2", []byte("some data 2"), 0660) Expect(err).Should(BeNil()) - gomock.InOrder( - im.EXPECT().MountImage(ctx, imageName, &cfg).Return(imageDir, nil), - mr. - EXPECT(). - Run(ctx, "-rvd", imageDir+dirName, moduleName), - ) + mr.EXPECT().Run(ctx, "-rvd", filepath.Join(sharedFilesDir, dirName), moduleName) Expect( w.UnloadKmod(ctx, &cfg, hostDir),