From f1813b1ef37b26122715fe741e3d81072e9fa108 Mon Sep 17 00:00:00 2001 From: wondywang Date: Tue, 8 Aug 2023 08:29:42 +0800 Subject: [PATCH] add featuregate KubeApiAccessSupport --- .../mutatorplugin/podkubeapiaccessmutator.go | 269 ++++++++++++++++++ .../podkubeapiaccessmutator_test.go | 204 +++++++++++++ .../pod/mutatorplugin/podrootcacertmutator.go | 257 +---------------- .../pkg/syncer/util/featuregate/gate.go | 5 + 4 files changed, 484 insertions(+), 251 deletions(-) create mode 100644 virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podkubeapiaccessmutator.go create mode 100644 virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podkubeapiaccessmutator_test.go diff --git a/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podkubeapiaccessmutator.go b/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podkubeapiaccessmutator.go new file mode 100644 index 00000000..811ea3d7 --- /dev/null +++ b/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podkubeapiaccessmutator.go @@ -0,0 +1,269 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutatorplugin + +import ( + "context" + "fmt" + "math/rand" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/storage/names" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/constants" + "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/conversion" + "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/util/featuregate" + uplugin "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/util/plugin" +) + +const ( + // DefaultServiceAccountName is the name of the default service account to set on pods which do not specify a service account + DefaultServiceAccountName = "default" // #nosec G101 + + // ServiceAccountVolumeName is the prefix name that will be added to volumes that mount ServiceAccount secrets + ServiceAccountVolumeName = "kube-api-access" // #nosec G101 + + // DefaultAPITokenMountPath is the path that ServiceAccountToken secrets are automounted to. + // The token file would then be accessible at /var/run/secrets/kubernetes.io/serviceaccount + DefaultAPITokenMountPath = "/var/run/secrets/kubernetes.io/serviceaccount" // #nosec G101 + + defaultAttemptTimes = 10 +) + +func init() { + MutatorRegister.Register(&uplugin.Registration{ + ID: "00_PodKubeAPIAccessMutator", + InitFn: func(ctx *uplugin.InitContext) (interface{}, error) { + return NewPodKubeAPIAccessMutatorPlugin(ctx) + }, + }) +} + +type PodKubeAPIAccessMutatorPlugin struct { + client kubernetes.Interface + generateName func(string) string +} + +func NewPodKubeAPIAccessMutatorPlugin(ctx *uplugin.InitContext) (*PodKubeAPIAccessMutatorPlugin, error) { + plugin := &PodKubeAPIAccessMutatorPlugin{ + client: ctx.Client, + generateName: names.SimpleNameGenerator.GenerateName, + } + return plugin, nil +} + +func (pl *PodKubeAPIAccessMutatorPlugin) Mutator() conversion.PodMutator { + return func(p *conversion.PodMutateCtx) error { + if !featuregate.DefaultFeatureGate.Enabled(featuregate.KubeAPIAccessSupport) { + return nil + } + + // Don't modify the spec of mirror pods. + // That makes the kubelet very angry and confused, and it immediately deletes the pod (because the spec doesn't match) + // That said, don't allow mirror pods to reference ServiceAccounts or SecretVolumeSources either + if _, isMirrorPod := p.PPod.Annotations[corev1.MirrorPodAnnotationKey]; isMirrorPod { + return nil + } + + // Set the default service account if needed + if len(p.PPod.Spec.ServiceAccountName) == 0 { + p.PPod.Spec.ServiceAccountName = DefaultServiceAccountName + } + + targetNamespace := conversion.ToSuperClusterNamespace(p.ClusterName, p.PPod.Namespace) + serviceAccount, err := pl.getServiceAccount(targetNamespace, p.PPod.Spec.ServiceAccountName) + if err != nil { + return fmt.Errorf("error looking up serviceAccount %s/%s: %v", targetNamespace, p.PPod.Spec.ServiceAccountName, err) + } + + secret, err := pl.getSecret(p.ClusterName, targetNamespace, serviceAccount) + if err != nil || secret == nil { + klog.Errorf("not found serviceAccount %s/%s token secret, err: %v", targetNamespace, p.PPod.Spec.ServiceAccountName, err) + return fmt.Errorf("error looking up secret of serviceAccount %s/%s: %v", targetNamespace, p.PPod.Spec.ServiceAccountName, err) + } + + if shouldAutomount(serviceAccount, p.PPod) { + pl.mountServiceAccountToken(secret, p.PPod) + } + return nil + } +} + +// getServiceAccount returns the ServiceAccount for the given namespace and name if it exists +func (pl *PodKubeAPIAccessMutatorPlugin) getServiceAccount(namespace string, name string) (*corev1.ServiceAccount, error) { + serviceAccount, err := pl.client.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil { + return serviceAccount, nil + } + if !apierrors.IsNotFound(err) { + return nil, err + } + + // Could not find in cache, attempt to look up directly + numAttempts := 1 + if name == DefaultServiceAccountName { + // If this is the default serviceaccount, attempt more times, since it should be auto-created by the controller + numAttempts = defaultAttemptTimes + } + retryInterval := time.Duration(rand.Int63n(100)+int64(100)) * time.Millisecond // #nosec G404 + for i := 0; i < numAttempts; i++ { + serviceAccount, err := pl.client.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil { + return serviceAccount, nil + } + if !apierrors.IsNotFound(err) { + return nil, err + } + time.Sleep(retryInterval) + } + + return nil, apierrors.NewNotFound(corev1.Resource("serviceaccount"), name) +} + +func (pl *PodKubeAPIAccessMutatorPlugin) getSecret(cluster, namespace string, sa *corev1.ServiceAccount) (*corev1.Secret, error) { + secrets, err := pl.client.CoreV1().Secrets(namespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("error listing secret from super control plane informer cache: %v", err) + return nil, err + } + for _, secret := range secrets.Items { + if secret.Type != corev1.SecretTypeOpaque { + continue + } + annotations := secret.GetAnnotations() + if annotations[constants.LabelCluster] != cluster || annotations[corev1.ServiceAccountNameKey] != sa.Name { + continue + } + return &secret, nil + } + return nil, nil +} + +func (pl *PodKubeAPIAccessMutatorPlugin) mountServiceAccountToken(secret *corev1.Secret, pod *corev1.Pod) { + // Determine a volume name for the ServiceAccountTokenSecret in case we need it + tokenVolumeName := pl.generateName(ServiceAccountVolumeName + "-") + klog.V(4).Infof("generate a new VolumeMount.name: %s", tokenVolumeName) + + // Create the prototypical VolumeMount + tokenVolumeMount := corev1.VolumeMount{ + Name: tokenVolumeName, + ReadOnly: true, + MountPath: DefaultAPITokenMountPath, + } + + // Find the volume and volume name for the ServiceAccountTokenSecret if it already exists + serviceAccountVolumeExist := false + allVolumeNames := sets.NewString() + for i, volume := range pod.Spec.Volumes { + allVolumeNames.Insert(volume.Name) + if strings.HasPrefix(volume.Name, ServiceAccountVolumeName+"-") { + for _, source := range volume.Projected.Sources { + if source.ServiceAccountToken != nil { + klog.V(4).Infof("pod: %s/%s volume: %s mount service account token, mutate it!", pod.Namespace, pod.Name, volume.Name) + pod.Spec.Volumes[i] = corev1.Volume{ + Name: tokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: TokenVolumeSource(secret), + }, + } + pl.MutateAutoKubeAPIAccessVolumeMounts(volume.Name, tokenVolumeName, pod) + serviceAccountVolumeExist = true + break + } + } + break + } + } + + // Ensure every container mounts the APISecret volume + for i, container := range pod.Spec.InitContainers { + existingContainerMount := false + for _, volumeMount := range container.VolumeMounts { + // Existing mounts at the default mount path prevent mounting of the API token + if volumeMount.MountPath == DefaultAPITokenMountPath { + existingContainerMount = true + break + } + } + if serviceAccountVolumeExist && !existingContainerMount { + pod.Spec.InitContainers[i].VolumeMounts = append(pod.Spec.InitContainers[i].VolumeMounts, tokenVolumeMount) + } + } + for i, container := range pod.Spec.Containers { + existingContainerMount := false + for _, volumeMount := range container.VolumeMounts { + // Existing mounts at the default mount path prevent mounting of the API token + if volumeMount.MountPath == DefaultAPITokenMountPath { + existingContainerMount = true + break + } + } + if serviceAccountVolumeExist && !existingContainerMount { + pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, tokenVolumeMount) + } + } +} + +func (pl *PodKubeAPIAccessMutatorPlugin) MutateAutoKubeAPIAccessVolumeMounts(old, new string, pod *corev1.Pod) { + for i, container := range pod.Spec.InitContainers { + for j := 0; j < len(container.VolumeMounts); j++ { + if container.VolumeMounts[j].Name == old { + klog.V(4).Infof("mutate initContainer %s volumeMount.name from %s to %s", container.Name, old, new) + pod.Spec.InitContainers[i].VolumeMounts[j].Name = new + continue + } + } + } + + for i, container := range pod.Spec.Containers { + for j := 0; j < len(container.VolumeMounts); j++ { + if container.VolumeMounts[j].Name == old { + klog.V(4).Infof("mutate containers %s volumeMount.name from %s to %s", container.Name, old, new) + pod.Spec.Containers[i].VolumeMounts[j].Name = new + continue + } + } + } +} + +func shouldAutomount(sa *corev1.ServiceAccount, pod *corev1.Pod) bool { + // Fixme: Now we will always return true. + // Perhaps in the future, some distinction between pods needs to be made here. + return true +} + +// TokenVolumeSource returns the projected volume source for service account token. +func TokenVolumeSource(secret *corev1.Secret) *corev1.ProjectedVolumeSource { + return &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret.Name, + }, + }, + }, + }, + } +} diff --git a/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podkubeapiaccessmutator_test.go b/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podkubeapiaccessmutator_test.go new file mode 100644 index 00000000..94a05a07 --- /dev/null +++ b/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podkubeapiaccessmutator_test.go @@ -0,0 +1,204 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutatorplugin + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + + "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/constants" + "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/conversion" + "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/util/featuregate" + util "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/util/test" + uplugin "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/util/plugin" +) + +func tenantKubeAPIAccessPod(name, namespace, uid string) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID(uid), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "default", + InitContainers: []corev1.Container{ + { + Image: "busybox", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "kube-api-access-l945t", + MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + ReadOnly: true, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Image: "busybox", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "kube-api-access-l945t", + MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + ReadOnly: true, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "kube-api-access-l945t", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: "token", + }, + ConfigMap: &corev1.ConfigMapProjection{ + Items: []corev1.KeyToPath{ + { + Key: "ca.crt", + Path: "ca.crt", + }, + }, + LocalObjectReference: corev1.LocalObjectReference{Name: constants.RootCACertConfigMapName}, + }, + DownwardAPI: &corev1.DownwardAPIProjection{ + Items: []corev1.DownwardAPIVolumeFile{ + { + Path: "namespace", + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + return p +} + +func TestPodKubeAPIAccessMutatorPlugin_Mutator(t *testing.T) { + defer util.SetFeatureGateDuringTest(t, featuregate.DefaultFeatureGate, featuregate.KubeAPIAccessSupport, true)() + + tests := []struct { + name string + pPod *corev1.Pod + existingObjectInSuper []runtime.Object + }{ + { + "Test RootCACert Mutator", + tenantKubeAPIAccessPod("test", "default", "123-456-789"), + []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-token-pkm85", + Namespace: "cluster1-default", + Annotations: map[string]string{ + constants.LabelCluster: "cluster1", + corev1.ServiceAccountNameKey: "default", + corev1.ServiceAccountUIDKey: "7374a172-c35d-45b1-9c8e-bf5c5b614937", + }, + }, + Type: corev1.SecretTypeOpaque, + }, + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "cluster1-default", + UID: "7374a172-c35d-45b1-9c8e-bf5c5b614937", + }, + Secrets: []corev1.ObjectReference{ + { + Name: "default-token-pkm85", + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.existingObjectInSuper...) + informer := informers.NewSharedInformerFactory(client, 0) + ctx := &uplugin.InitContext{ + Client: client, + Informer: informer, + } + pl, err := NewPodKubeAPIAccessMutatorPlugin(ctx) + if err != nil { + t.Errorf("mutator failed processing the pod") + } + mutator := pl.Mutator() + + if err := mutator(&conversion.PodMutateCtx{ClusterName: "cluster1", PPod: tt.pPod}); err != nil { + t.Errorf("mutator failed processing the pod, %v", err) + } + + kubeAPIAccessVolume := "" + for _, volume := range tt.pPod.Spec.Volumes { + if strings.HasPrefix(volume.Name, ServiceAccountVolumeName) && volume.Projected != nil { + kubeAPIAccessVolume = volume.Name + for _, source := range volume.Projected.Sources { + if source.Secret == nil { + t.Errorf("tt.pPod.Spec.Volumes[*].Name = %v, want to mount secret, but nil", volume.Name) + } + } + } + } + for c := range tt.pPod.Spec.Containers { + found := false + for e := range tt.pPod.Spec.Containers[c].VolumeMounts { + if tt.pPod.Spec.Containers[c].VolumeMounts[e].Name == kubeAPIAccessVolume { + found = true + } + } + if !found { + t.Errorf("tt.pPod.Spec.Containers[c].VolumeMounts[e] want to mount %s, but not found", kubeAPIAccessVolume) + } + } + + for c := range tt.pPod.Spec.InitContainers { + found := false + for e := range tt.pPod.Spec.InitContainers[c].VolumeMounts { + if tt.pPod.Spec.InitContainers[c].VolumeMounts[e].Name == kubeAPIAccessVolume { + found = true + } + } + if !found { + t.Errorf("tt.pPod.Spec.InitContainers[c].VolumeMounts[e] want to mount %s, but not found", kubeAPIAccessVolume) + } + } + }) + } +} diff --git a/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podrootcacertmutator.go b/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podrootcacertmutator.go index dffab6ec..f80a3c4c 100644 --- a/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podrootcacertmutator.go +++ b/virtualcluster/pkg/syncer/resources/pod/mutatorplugin/podrootcacertmutator.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,72 +17,23 @@ limitations under the License. package mutatorplugin import ( - "context" - "fmt" - "k8s.io/client-go/kubernetes" - "math/rand" - "strings" - "time" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/sets" - listersv1 "k8s.io/client-go/listers/core/v1" - "k8s.io/klog/v2" - "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/constants" "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/conversion" "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/util/featuregate" uplugin "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/util/plugin" ) -const ( - // DefaultServiceAccountName is the name of the default service account to set on pods which do not specify a service account - DefaultServiceAccountName = "default" - - // EnforceMountableSecretsAnnotation is a default annotation that indicates that a service account should enforce mountable secrets. - // The value must be true to have this annotation take effect - EnforceMountableSecretsAnnotation = "kubernetes.io/enforce-mountable-secrets" - - // ServiceAccountVolumeName is the prefix name that will be added to volumes that mount ServiceAccount secrets - ServiceAccountVolumeName = "kube-api-access" - - // DefaultAPITokenMountPath is the path that ServiceAccountToken secrets are automounted to. - // The token file would then be accessible at /var/run/secrets/kubernetes.io/serviceaccount - DefaultAPITokenMountPath = "/var/run/secrets/kubernetes.io/serviceaccount" - - defaultAttemptTimes = 10 -) - func init() { MutatorRegister.Register(&uplugin.Registration{ ID: "00_PodRootCACertMutator", InitFn: func(ctx *uplugin.InitContext) (interface{}, error) { - return NewMutatorPlugin(ctx) + return &PodRootCACertMutatorPlugin{}, nil }, }) } -type PodRootCACertMutatorPlugin struct { - client kubernetes.Interface - secretLister listersv1.SecretLister - serviceAccountLister listersv1.ServiceAccountLister - - generateName func(string) string -} - -func NewMutatorPlugin(ctx *uplugin.InitContext) (*PodRootCACertMutatorPlugin, error) { - secretInformer := ctx.Informer.Core().V1().Secrets() - serviceAccountInformer := ctx.Informer.Core().V1().ServiceAccounts() - plugin := &PodRootCACertMutatorPlugin{ - client: ctx.Client, - secretLister: secretInformer.Lister(), - serviceAccountLister: serviceAccountInformer.Lister(), - } - return plugin, nil -} +type PodRootCACertMutatorPlugin struct{} // Mutator will automatically reassign configmap references for configmaps named // kube-root-ca.crt in the pod spec, these places are @@ -95,188 +46,13 @@ func (pl *PodRootCACertMutatorPlugin) Mutator() conversion.PodMutator { return nil } - // Don't modify the spec of mirror pods. - // That makes the kubelet very angry and confused, and it immediately deletes the pod (because the spec doesn't match) - // That said, don't allow mirror pods to reference ServiceAccounts or SecretVolumeSources either - if _, isMirrorPod := p.PPod.Annotations[corev1.MirrorPodAnnotationKey]; isMirrorPod { - return nil - } - - // Set the default service account if needed - if len(p.PPod.Spec.ServiceAccountName) == 0 { - p.PPod.Spec.ServiceAccountName = DefaultServiceAccountName - } - - serviceAccount, err := pl.getServiceAccount(p.PPod.Namespace, p.PPod.Spec.ServiceAccountName) - if err != nil { - return fmt.Errorf("error looking up serviceAccount %s/%s: %v", p.PPod.Namespace, p.PPod.Spec.ServiceAccountName, err) - } - - secret, err := pl.getSecret(p.ClusterName, p.PPod.Namespace, p.PPod.Spec.ServiceAccountName) - if err != nil || secret == nil { - klog.Errorf("not found serviceAccount %s token secret, err: %v", p.PPod.Spec.ServiceAccountName, err) - return fmt.Errorf("error looking up secret of serviceAccount %s/%s: %v", p.PPod.Namespace, p.PPod.Spec.ServiceAccountName, err) - } - klog.V(6).Infof("mutate pod: %s/%s, found service account: %s, secret: %s", p.PPod.Namespace, p.PPod.Name, serviceAccount.Name, secret.Name) - - if shouldAutomount(serviceAccount, p.PPod) { - pl.mountServiceAccountToken(secret, p.PPod) - pl.mountKubeRootCAConfigMap(p.PPod) - } + p.PPod.Spec.Containers = pl.containersMutator(p.PPod.Spec.Containers) + p.PPod.Spec.InitContainers = pl.containersMutator(p.PPod.Spec.InitContainers) + p.PPod.Spec.Volumes = pl.volumesMutator(p.PPod.Spec.Volumes) return nil } } -// getServiceAccount returns the ServiceAccount for the given namespace and name if it exists -func (p *PodRootCACertMutatorPlugin) getServiceAccount(namespace string, name string) (*corev1.ServiceAccount, error) { - serviceAccount, err := p.serviceAccountLister.ServiceAccounts(namespace).Get(name) - if err == nil { - return serviceAccount, nil - } - if !errors.IsNotFound(err) { - return nil, err - } - - // Could not find in cache, attempt to look up directly - numAttempts := 1 - if name == DefaultServiceAccountName { - // If this is the default serviceaccount, attempt more times, since it should be auto-created by the controller - numAttempts = defaultAttemptTimes - } - retryInterval := time.Duration(rand.Int63n(100)+int64(100)) * time.Millisecond - for i := 0; i < numAttempts; i++ { - if i != 0 { - time.Sleep(retryInterval) - } - serviceAccount, err := p.client.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err == nil { - return serviceAccount, nil - } - if !errors.IsNotFound(err) { - return nil, err - } - } - - return nil, errors.NewNotFound(corev1.Resource("serviceaccount"), name) -} - -func (p *PodRootCACertMutatorPlugin) getSecret(cluster, namespace, serviceAccount string) (*corev1.Secret, error) { - secrets, err := p.secretLister.Secrets(namespace).List(labels.Everything()) - if err != nil { - klog.Errorf("error listing secret from super control plane informer cache: %v", err) - return nil, err - } - for _, secret := range secrets { - if secret.Type != corev1.SecretTypeOpaque { - continue - } - annotations := secret.GetAnnotations() - if annotations[constants.LabelCluster] != cluster || annotations[corev1.ServiceAccountNameKey] != serviceAccount { - continue - } - return secret, nil - } - return nil, nil -} - -func (p *PodRootCACertMutatorPlugin) mountServiceAccountToken(secret *corev1.Secret, pod *corev1.Pod) { - // Determine a volume name for the ServiceAccountTokenSecret in case we need it - tokenVolumeName := p.generateName(ServiceAccountVolumeName + "-") - klog.V(4).Infof("generate a new VolumeMount.name: %s", tokenVolumeName) - - // Create the prototypical VolumeMount - tokenVolumeMount := corev1.VolumeMount{ - Name: tokenVolumeName, - ReadOnly: true, - MountPath: DefaultAPITokenMountPath, - } - - // Find the volume and volume name for the ServiceAccountTokenSecret if it already exists - serviceAccountVolumeExist := false - allVolumeNames := sets.NewString() - for i, volume := range pod.Spec.Volumes { - allVolumeNames.Insert(volume.Name) - if strings.HasPrefix(volume.Name, ServiceAccountVolumeName+"-") { - for _, source := range volume.Projected.Sources { - if source.ServiceAccountToken != nil { - klog.V(4).Infof("pod: %s/%s volume: %s mount service account token, mutate it!", pod.Namespace, pod.Name, volume.Name) - pod.Spec.Volumes[i] = corev1.Volume{ - Name: tokenVolumeName, - VolumeSource: corev1.VolumeSource{ - Projected: TokenVolumeSource(secret), - }, - } - p.MutateAutoKubeApiAccessVolumeMounts(volume.Name, tokenVolumeName, pod) - serviceAccountVolumeExist = true - break - } - } - break - } - } - - // Ensure every container mounts the APISecret volume - for i, container := range pod.Spec.InitContainers { - existingContainerMount := false - for _, volumeMount := range container.VolumeMounts { - // Existing mounts at the default mount path prevent mounting of the API token - if volumeMount.MountPath == DefaultAPITokenMountPath { - existingContainerMount = true - break - } - } - if serviceAccountVolumeExist && !existingContainerMount { - pod.Spec.InitContainers[i].VolumeMounts = append(pod.Spec.InitContainers[i].VolumeMounts, tokenVolumeMount) - } - } - for i, container := range pod.Spec.Containers { - existingContainerMount := false - for _, volumeMount := range container.VolumeMounts { - // Existing mounts at the default mount path prevent mounting of the API token - if volumeMount.MountPath == DefaultAPITokenMountPath { - existingContainerMount = true - break - } - } - if serviceAccountVolumeExist && !existingContainerMount { - pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, tokenVolumeMount) - } - } -} - -// Mutator will automatically reassign configmap references for configmaps named -// kube-root-ca.crt in the pod spec, these places are -// * volumes -// * env -// * envFrom -func (p *PodRootCACertMutatorPlugin) mountKubeRootCAConfigMap(pod *corev1.Pod) { - pod.Spec.Containers = p.containersMutator(pod.Spec.Containers) - pod.Spec.InitContainers = p.containersMutator(pod.Spec.InitContainers) - pod.Spec.Volumes = p.volumesMutator(pod.Spec.Volumes) -} - -func (p *PodRootCACertMutatorPlugin) MutateAutoKubeApiAccessVolumeMounts(old, new string, pod *corev1.Pod) { - for i, container := range pod.Spec.InitContainers { - for j := 0; j < len(container.VolumeMounts); j++ { - if container.VolumeMounts[j].Name == old { - klog.V(6).Infof("mutate initContainer %s volumeMount.name from %s to %s", container.Name, old, new) - pod.Spec.InitContainers[i].VolumeMounts[j].Name = new - continue - } - } - } - - for i, container := range pod.Spec.Containers { - for j := 0; j < len(container.VolumeMounts); j++ { - if container.VolumeMounts[j].Name == old { - klog.V(6).Infof("mutate containers %s volumeMount.name from %s to %s", container.Name, old, new) - pod.Spec.Containers[i].VolumeMounts[j].Name = new - continue - } - } - } -} - func (pl *PodRootCACertMutatorPlugin) containersMutator(containers []corev1.Container) []corev1.Container { for i, container := range containers { for j, env := range container.Env { @@ -311,24 +87,3 @@ func (pl *PodRootCACertMutatorPlugin) volumesMutator(volumes []corev1.Volume) [] } return volumes } - -func shouldAutomount(sa *corev1.ServiceAccount, pod *corev1.Pod) bool { - // Fixme: Now we will always return true. - // Perhaps in the future, some distinction between pods needs to be made here. - return true -} - -// TokenVolumeSource returns the projected volume source for service account token. -func TokenVolumeSource(secret *corev1.Secret) *corev1.ProjectedVolumeSource { - return &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{ - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret.Name, - }, - }, - }, - }, - } -} diff --git a/virtualcluster/pkg/syncer/util/featuregate/gate.go b/virtualcluster/pkg/syncer/util/featuregate/gate.go index 7b0a514a..68e778fa 100644 --- a/virtualcluster/pkg/syncer/util/featuregate/gate.go +++ b/virtualcluster/pkg/syncer/util/featuregate/gate.go @@ -88,6 +88,10 @@ const ( // add clusterIP of pService to vService's externalIPs. // So that vService can be resolved by using the k8s_external plugin in coredns. VServiceExternalIP = "VServiceExternalIP" + + // KubeAPIAccessSupport is an experimental feature that allows clusters +1.21 to support + // kube-api-access volume mount + KubeAPIAccessSupport = "KubeAPIAccessSupport" ) var defaultFeatures = FeatureList{ @@ -103,6 +107,7 @@ var defaultFeatures = FeatureList{ DisableCRDPreserveUnknownFields: {Default: false}, RootCACertConfigMapSupport: {Default: false}, VServiceExternalIP: {Default: false}, + KubeAPIAccessSupport: {Default: false}, } type Feature string