Skip to content

Commit

Permalink
Adding an init container to the worker pod. (#1194)
Browse files Browse the repository at this point in the history 
This init container will be pulled by the container-runtime of the
cluster and will copy all `.ko` and firmware files to a shared volume
accessible by the worker container to load.

Signed-off-by: Yoni Bettan <[email protected]>
  • Loading branch information
@ybettan
ybettan authored Sep 3, 2024
1 parent 573da4d commit a15af73
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 111 deletions.
86 changes: 77 additions & 9 deletions internal/controllers/nmc_reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ const (
volumeNameConfig = "config"
workerContainerName = "worker"
globalPullSecretName = "global-pull-secret"
initContainerName = "image-extractor"
sharedFilesDir = "/tmp"
volNameTmp = "tmp"
)

//+kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=nodemodulesconfigs,verbs=get;list;watch
Expand Down Expand Up @@ -834,7 +837,7 @@ func (p *podManagerImpl) ListWorkerPodsOnNode(ctx context.Context, nodeName stri
}

func (p *podManagerImpl) LoaderPodTemplate(ctx context.Context, nmc client.Object, nms *kmmv1beta1.NodeModuleSpec) (*v1.Pod, error) {
pod, err := p.baseWorkerPod(ctx, nmc.GetName(), &nms.ModuleItem, nmc)
pod, err := p.baseWorkerPod(ctx, nmc, &nms.ModuleItem, &nms.Config)
if err != nil {
return nil, fmt.Errorf("could not create the base Pod: %v", err)
}
Expand All @@ -856,6 +859,13 @@ func (p *podManagerImpl) LoaderPodTemplate(ctx context.Context, nmc client.Objec
}

args = append(args, "--"+worker.FlagFirmwarePath, *firmwareHostPath)

firmwarePathContainerImg := filepath.Join(nms.Config.Modprobe.FirmwarePath, "*")
firmwarePathWorkerImg := filepath.Join(sharedFilesDir, nms.Config.Modprobe.FirmwarePath)
if err = addCopyCommand(pod, firmwarePathContainerImg, firmwarePathWorkerImg); err != nil {
return nil, fmt.Errorf("could not add the copy command to the init container: %v", err)
}

if err = setFirmwareVolume(pod, firmwareHostPath); err != nil {
return nil, fmt.Errorf("could not map host volume needed for firmware loading: %v", err)
}
Expand All @@ -881,7 +891,7 @@ func (p *podManagerImpl) LoaderPodTemplate(ctx context.Context, nmc client.Objec
}

func (p *podManagerImpl) UnloaderPodTemplate(ctx context.Context, nmc client.Object, nms *kmmv1beta1.NodeModuleStatus) (*v1.Pod, error) {
pod, err := p.baseWorkerPod(ctx, nmc.GetName(), &nms.ModuleItem, nmc)
pod, err := p.baseWorkerPod(ctx, nmc, &nms.ModuleItem, &nms.Config)
if err != nil {
return nil, fmt.Errorf("could not create the base Pod: %v", err)
}
Expand All @@ -908,6 +918,13 @@ func (p *podManagerImpl) UnloaderPodTemplate(ctx context.Context, nmc client.Obj
return nil, fmt.Errorf("firmwareHostPath was not set while the Module requires firmware unloading")
}
args = append(args, "--"+worker.FlagFirmwarePath, *firmwareHostPath)

firmwarePathContainerImg := filepath.Join(nms.Config.Modprobe.FirmwarePath, "*")
firmwarePathWorkerImg := filepath.Join(sharedFilesDir, nms.Config.Modprobe.FirmwarePath)
if err = addCopyCommand(pod, firmwarePathContainerImg, firmwarePathWorkerImg); err != nil {
return nil, fmt.Errorf("could not add the copy command to the init container: %v", err)
}

if err = setFirmwareVolume(pod, firmwareHostPath); err != nil {
return nil, fmt.Errorf("could not map host volume needed for firmware unloading: %v", err)
}
Expand All @@ -933,12 +950,26 @@ var (
}
)

func (p *podManagerImpl) baseWorkerPod(
ctx context.Context,
nodeName string,
item *kmmv1beta1.ModuleItem,
owner client.Object,
) (*v1.Pod, error) {
func addCopyCommand(pod *v1.Pod, src, dst string) error {

container, _ := podcmd.FindContainerByName(pod, initContainerName)
if container == nil {
return errors.New("could not find the init container")
}

const template = `
mkdir -p %s;
cp -R %s %s;
`
copyCommand := fmt.Sprintf(template, dst, src, dst)
container.Args[0] = strings.Join([]string{container.Args[0], copyCommand}, "")

return nil
}

func (p *podManagerImpl) baseWorkerPod(ctx context.Context, nmc client.Object, item *kmmv1beta1.ModuleItem,
moduleConfig *kmmv1beta1.ModuleConfig) (*v1.Pod, error) {

const (
trustedCAVolumeName = "trusted-ca"
volNameEtcContainers = "etc-containers"
Expand Down Expand Up @@ -1048,6 +1079,12 @@ func (p *podManagerImpl) baseWorkerPod(
},
},
},
{
Name: volNameTmp,
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
}

volumeMounts := []v1.VolumeMount{
Expand Down Expand Up @@ -1082,8 +1119,14 @@ func (p *podManagerImpl) baseWorkerPod(
ReadOnly: true,
MountPath: filepath.Join(worker.PullSecretsDir, "_global", v1.DockerConfigJsonKey),
},
{
Name: volNameTmp,
MountPath: sharedFilesDir,
ReadOnly: true,
},
}

nodeName := nmc.GetName()
pod := v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: item.Namespace,
Expand All @@ -1096,6 +1139,24 @@ func (p *podManagerImpl) baseWorkerPod(
},
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
{
Name: initContainerName,
Image: moduleConfig.ContainerImage,
Command: []string{"/bin/sh", "-c"},
Args: []string{""},
VolumeMounts: []v1.VolumeMount{
{
Name: volNameTmp,
MountPath: sharedFilesDir,
},
},
Resources: v1.ResourceRequirements{
Requests: requests,
Limits: limits,
},
},
},
Containers: []v1.Container{
{
Name: workerContainerName,
Expand All @@ -1114,10 +1175,16 @@ func (p *podManagerImpl) baseWorkerPod(
},
}

if err = ctrl.SetControllerReference(owner, &pod, p.scheme); err != nil {
if err = ctrl.SetControllerReference(nmc, &pod, p.scheme); err != nil {
return nil, fmt.Errorf("could not set the owner as controller: %v", err)
}

kmodsPathContainerImg := filepath.Join(moduleConfig.Modprobe.DirName, "lib", "modules", moduleConfig.KernelVersion)
kmodsPathWorkerImg := filepath.Join(sharedFilesDir, moduleConfig.Modprobe.DirName, "lib", "modules")
if err = addCopyCommand(&pod, kmodsPathContainerImg, kmodsPathWorkerImg); err != nil {
return nil, fmt.Errorf("could not add the copy command to the init container: %v", err)
}

controllerutil.AddFinalizer(&pod, nodeModulesConfigFinalizer)

return &pod, nil
Expand Down Expand Up @@ -1227,6 +1294,7 @@ func setFirmwareVolume(pod *v1.Pod, firmwareHostPath *string) error {

pod.Spec.Volumes = append(pod.Spec.Volumes, firmwareVolume)
container.VolumeMounts = append(container.VolumeMounts, firmwareVolumeMount)

return nil
}

Expand Down
49 changes: 45 additions & 4 deletions internal/controllers/nmc_reconciler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ var _ = Describe("NodeModulesConfigReconciler_Reconcile", func() {
})

var moduleConfig = kmmv1beta1.ModuleConfig{
KernelVersion: "kernel version",
KernelVersion: "kernel-version",
ContainerImage: "container image",
InsecurePull: true,
InTreeModulesToRemove: []string{"intree1", "intree2"},
Expand Down Expand Up @@ -1791,7 +1791,7 @@ var _ = Describe("podManagerImpl_CreateUnloaderPod", func() {

It("should work as expected", func() {

expected := getBaseWorkerPod("unload", nmc, ptr.To("some-path"), true, false)
expected := getBaseWorkerPod("unload", nmc, ptr.To("/var/lib/firmware"), true, false)

container, _ := podcmd.FindContainerByName(expected, "worker")
Expect(container).NotTo(BeNil())
Expand All @@ -1817,7 +1817,8 @@ var _ = Describe("podManagerImpl_CreateUnloaderPod", func() {
)

workerCfg := *workerCfg
workerCfg.FirmwareHostPath = ptr.To("some-path")
workerCfg.FirmwareHostPath = ptr.To("/var/lib/firmware")

pm := newPodManager(client, workerImage, scheme, caHelper, &workerCfg)
pm.(*podManagerImpl).psh = psh

Expand Down Expand Up @@ -1946,7 +1947,7 @@ inTreeModulesToRemove:
- intree1
- intree2
insecurePull: true
kernelVersion: kernel version
kernelVersion: kernel-version
modprobe:
dirName: /dir
firmwarePath: /firmware-path
Expand All @@ -1961,11 +1962,22 @@ modprobe:
`
modulesOrderValue := `softdep a pre: b
softdep b pre: c
`

var initContainerArg = `
mkdir -p /tmp/dir/lib/modules;
cp -R /dir/lib/modules/kernel-version /tmp/dir/lib/modules;
`

const initContainerArgFirmwareAddition = `
mkdir -p /tmp/firmware-path;
cp -R /firmware-path/* /tmp/firmware-path;
`

args := []string{"kmod", subcommand, "/etc/kmm-worker/config.yaml"}
if withFirmware {
args = append(args, "--firmware-path", *firmwareHostPath)
initContainerArg = strings.Join([]string{initContainerArg, initContainerArgFirmwareAddition}, "")
} else {
configAnnotationValue = strings.ReplaceAll(configAnnotationValue, "firmwarePath: /firmware-path\n ", "")
}
Expand All @@ -1986,6 +1998,24 @@ softdep b pre: c
},
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
{
Name: "image-extractor",
Image: "container image",
Command: []string{"/bin/sh", "-c"},
Args: []string{initContainerArg},
Resources: v1.ResourceRequirements{
Limits: limits,
Requests: requests,
},
VolumeMounts: []v1.VolumeMount{
{
Name: volNameTmp,
MountPath: sharedFilesDir,
},
},
},
},
Containers: []v1.Container{
{
Name: "worker",
Expand Down Expand Up @@ -2027,6 +2057,11 @@ softdep b pre: c
ReadOnly: true,
MountPath: filepath.Join(worker.PullSecretsDir, "_global", v1.DockerConfigJsonKey),
},
{
Name: volNameTmp,
MountPath: sharedFilesDir,
ReadOnly: true,
},
{
Name: volNameModulesOrder,
ReadOnly: true,
Expand Down Expand Up @@ -2121,6 +2156,12 @@ softdep b pre: c
},
},
},
{
Name: volNameTmp,
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: volNameModulesOrder,
VolumeSource: v1.VolumeSource{
Expand Down
30 changes: 10 additions & 20 deletions internal/worker/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,9 @@ func NewWorker(im ImageMounter, mr ModprobeRunner, logger logr.Logger) Worker {
}
}

func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, firmwareMountPath string) error {
imageName := cfg.ContainerImage
const sharedFilesDir = "/tmp"

fsDir, err := w.im.MountImage(ctx, imageName, cfg)
if err != nil {
return fmt.Errorf("failed to mount image %s: %v", imageName, err)
}
func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, firmwareMountPath string) error {

inTreeModulesToRemove := cfg.InTreeModulesToRemove
// [TODO] - remove handling cfg.InTreeModuleToRemove once we cease to support it
Expand All @@ -53,14 +49,14 @@ func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, fir
w.logger.Info("Unloading in-tree modules", "names", inTreeModulesToRemove)

runArgs := append([]string{"-rv"}, inTreeModulesToRemove...)
if err = w.mr.Run(ctx, runArgs...); err != nil {
if err := w.mr.Run(ctx, runArgs...); err != nil {
return fmt.Errorf("could not remove in-tree modules %s: %v", strings.Join(inTreeModulesToRemove, ""), err)
}
}

// prepare firmware
if cfg.Modprobe.FirmwarePath != "" {
imageFirmwarePath := filepath.Join(fsDir, cfg.Modprobe.FirmwarePath)
imageFirmwarePath := filepath.Join(sharedFilesDir, cfg.Modprobe.FirmwarePath)
w.logger.Info("preparing firmware for loading", "image directory", imageFirmwarePath, "host mount directory", firmwareMountPath)
options := cp.Options{
OnError: func(src, dest string, err error) error {
Expand All @@ -70,7 +66,7 @@ func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, fir
return nil
},
}
if err = cp.Copy(imageFirmwarePath, firmwareMountPath, options); err != nil {
if err := cp.Copy(imageFirmwarePath, firmwareMountPath, options); err != nil {
return fmt.Errorf("failed to copy firmware from path %s to path %s: %v", imageFirmwarePath, firmwareMountPath, err)
}
}
Expand All @@ -82,7 +78,7 @@ func (w *worker) LoadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, fir
if cfg.Modprobe.RawArgs != nil {
args = cfg.Modprobe.RawArgs.Load
} else {
args = []string{"-vd", filepath.Join(fsDir, cfg.Modprobe.DirName)}
args = []string{"-vd", filepath.Join(sharedFilesDir, cfg.Modprobe.DirName)}

if cfg.Modprobe.Args != nil {
args = append(args, cfg.Modprobe.Args.Load...)
Expand Down Expand Up @@ -120,12 +116,6 @@ func (w *worker) SetFirmwareClassPath(value string) error {
}

func (w *worker) UnloadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, firmwareMountPath string) error {
imageName := cfg.ContainerImage

fsDir, err := w.im.MountImage(ctx, imageName, cfg)
if err != nil {
return fmt.Errorf("failed to mount image %s: %v", imageName, err)
}

moduleName := cfg.Modprobe.ModuleName

Expand All @@ -134,7 +124,7 @@ func (w *worker) UnloadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, f
if cfg.Modprobe.RawArgs != nil {
args = cfg.Modprobe.RawArgs.Unload
} else {
args = []string{"-rvd", filepath.Join(fsDir, cfg.Modprobe.DirName)}
args = []string{"-rvd", filepath.Join(sharedFilesDir, cfg.Modprobe.DirName)}

if cfg.Modprobe.Args != nil {
args = append(args, cfg.Modprobe.Args.Unload...)
Expand All @@ -145,14 +135,14 @@ func (w *worker) UnloadKmod(ctx context.Context, cfg *kmmv1beta1.ModuleConfig, f

w.logger.Info("Unloading module", "name", moduleName)

if err = w.mr.Run(ctx, args...); err != nil {
if err := w.mr.Run(ctx, args...); err != nil {
return fmt.Errorf("could not unload module %s: %v", moduleName, err)
}

//remove firmware files only (no directories)
if cfg.Modprobe.FirmwarePath != "" {
imageFirmwarePath := filepath.Join(fsDir, cfg.Modprobe.FirmwarePath)
err = filepath.Walk(imageFirmwarePath, func(path string, info os.FileInfo, err error) error {
imageFirmwarePath := filepath.Join(sharedFilesDir, cfg.Modprobe.FirmwarePath)
err := filepath.Walk(imageFirmwarePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit a15af73

Please sign in to comment.