From 91e7def0260480364991eafc4b1d01ea71b6c47a Mon Sep 17 00:00:00 2001 From: Stephen Kitt Date: Mon, 6 Dec 2021 16:53:24 +0100 Subject: [PATCH] Mount the Broker secret when available ... and pass its name to the pods which need it. Fixes: #1685 Signed-off-by: Stephen Kitt --- .../servicediscovery_controller.go | 20 ++++++++++ controllers/submariner/gateway_resources.go | 38 ++++++++++++++----- .../submariner/submariner_controller_test.go | 1 + go.mod | 2 +- go.sum | 4 +- internal/restconfig/restconfig.go | 23 ++++++----- 6 files changed, 65 insertions(+), 23 deletions(-) diff --git a/controllers/servicediscovery/servicediscovery_controller.go b/controllers/servicediscovery/servicediscovery_controller.go index 182fd1af2f..743093cbda 100644 --- a/controllers/servicediscovery/servicediscovery_controller.go +++ b/controllers/servicediscovery/servicediscovery_controller.go @@ -211,6 +211,23 @@ func newLighthouseAgent(cr *submarinerv1alpha1.ServiceDiscovery) *appsv1.Deploym terminationGracePeriodSeconds := int64(0) + volumeMounts := []corev1.VolumeMount{} + volumes := []corev1.Volume{} + + if cr.Spec.BrokerK8sSecret != "" { + // We've got a secret, mount it where the syncer expects it + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "brokersecret", + MountPath: broker.SecretPath(cr.Spec.BrokerK8sSecret), + ReadOnly: true, + }) + + volumes = append(volumes, corev1.Volume{ + Name: "brokersecret", + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: cr.Spec.BrokerK8sSecret}}, + }) + } + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: cr.Namespace, @@ -243,12 +260,15 @@ func newLighthouseAgent(cr *submarinerv1alpha1.ServiceDiscovery) *appsv1.Deploym {Name: broker.EnvironmentVariable("RemoteNamespace"), Value: cr.Spec.BrokerK8sRemoteNamespace}, {Name: broker.EnvironmentVariable("CA"), Value: cr.Spec.BrokerK8sCA}, {Name: broker.EnvironmentVariable("Insecure"), Value: strconv.FormatBool(cr.Spec.BrokerK8sInsecure)}, + {Name: broker.EnvironmentVariable("Secret"), Value: cr.Spec.BrokerK8sSecret}, }, + VolumeMounts: volumeMounts, }, }, ServiceAccountName: "submariner-lighthouse-agent", TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + Volumes: volumes, }, }, }, diff --git a/controllers/submariner/gateway_resources.go b/controllers/submariner/gateway_resources.go index fe3aed3239..47d209dbaa 100644 --- a/controllers/submariner/gateway_resources.go +++ b/controllers/submariner/gateway_resources.go @@ -108,6 +108,31 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner) corev1.PodTemplateSpec { nattPort, _ := strconv.ParseInt(submarinerv1.DefaultNATTDiscoveryPort, 10, 32) + volumeMounts := []corev1.VolumeMount{ + {Name: "ipsecd", MountPath: "/etc/ipsec.d", ReadOnly: false}, + {Name: "ipsecnss", MountPath: "/var/lib/ipsec/nss", ReadOnly: false}, + {Name: "libmodules", MountPath: "/lib/modules", ReadOnly: true}, + } + volumes := []corev1.Volume{ + {Name: "ipsecd", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "ipsecnss", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "libmodules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + } + + if cr.Spec.BrokerK8sSecret != "" { + // We've got a secret, mount it where the syncer expects it + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "brokersecret", + MountPath: broker.SecretPath(cr.Spec.BrokerK8sSecret), + ReadOnly: true, + }) + + volumes = append(volumes, corev1.Volume{ + Name: "brokersecret", + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: cr.Spec.BrokerK8sSecret}}, + }) + } + podTemplate := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -170,6 +195,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner) corev1.PodTemplateSpec { {Name: broker.EnvironmentVariable("RemoteNamespace"), Value: cr.Spec.BrokerK8sRemoteNamespace}, {Name: broker.EnvironmentVariable("CA"), Value: cr.Spec.BrokerK8sCA}, {Name: broker.EnvironmentVariable("Insecure"), Value: strconv.FormatBool(cr.Spec.BrokerK8sInsecure)}, + {Name: broker.EnvironmentVariable("Secret"), Value: cr.Spec.BrokerK8sSecret}, {Name: "CE_IPSEC_PSK", Value: cr.Spec.CeIPSecPSK}, {Name: "CE_IPSEC_DEBUG", Value: strconv.FormatBool(cr.Spec.CeIPSecDebug)}, {Name: "SUBMARINER_HEALTHCHECKENABLED", Value: strconv.FormatBool(healthCheckEnabled)}, @@ -186,11 +212,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner) corev1.PodTemplateSpec { }, }}, }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "ipsecd", MountPath: "/etc/ipsec.d", ReadOnly: false}, - {Name: "ipsecnss", MountPath: "/var/lib/ipsec/nss", ReadOnly: false}, - {Name: "libmodules", MountPath: "/lib/modules", ReadOnly: true}, - }, + VolumeMounts: volumeMounts, }, }, // TODO: Use SA submariner-gateway or submariner? @@ -201,11 +223,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner) corev1.PodTemplateSpec { DNSPolicy: corev1.DNSClusterFirst, // The gateway engine must be able to run on any flagged node, regardless of existing taints Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}}, - Volumes: []corev1.Volume{ - {Name: "ipsecd", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - {Name: "ipsecnss", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - {Name: "libmodules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - }, + Volumes: volumes, }, } if cr.Spec.CeIPSecIKEPort != 0 { diff --git a/controllers/submariner/submariner_controller_test.go b/controllers/submariner/submariner_controller_test.go index 072f09e935..e0160cdddd 100644 --- a/controllers/submariner/submariner_controller_test.go +++ b/controllers/submariner/submariner_controller_test.go @@ -407,6 +407,7 @@ func verifyGatewayDaemonSet(ctx context.Context, submariner *submariner_v1.Subma Expect(envMap).To(HaveKeyWithValue(broker.EnvironmentVariable("ApiServerToken"), submariner.Spec.BrokerK8sApiServerToken)) Expect(envMap).To(HaveKeyWithValue(broker.EnvironmentVariable("CA"), submariner.Spec.BrokerK8sCA)) Expect(envMap).To(HaveKeyWithValue(broker.EnvironmentVariable("Insecure"), strconv.FormatBool(submariner.Spec.BrokerK8sInsecure))) + Expect(envMap).To(HaveKeyWithValue(broker.EnvironmentVariable("Secret"), submariner.Spec.BrokerK8sSecret)) Expect(envMap).To(HaveKeyWithValue("SUBMARINER_BROKER", submariner.Spec.Broker)) Expect(envMap).To(HaveKeyWithValue("SUBMARINER_NATENABLED", strconv.FormatBool(submariner.Spec.NatEnabled))) Expect(envMap).To(HaveKeyWithValue("SUBMARINER_CLUSTERID", submariner.Spec.ClusterID)) diff --git a/go.mod b/go.mod index 721e71b539..8cb8d0a5ff 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/prometheus/client_golang v1.11.0 github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/cobra v1.2.1 - github.com/submariner-io/admiral v0.12.0-m1.0.20211209141450-f39009d93c9d + github.com/submariner-io/admiral v0.12.0-m1.0.20211216212848-d630c38c2fd2 github.com/submariner-io/cloud-prepare v0.12.0-m1 github.com/submariner-io/lighthouse v0.12.0-m1 github.com/submariner-io/shipyard v0.12.0-m1 diff --git a/go.sum b/go.sum index 1e6eced5c5..1e1dbfb4e4 100644 --- a/go.sum +++ b/go.sum @@ -1335,8 +1335,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/submariner-io/admiral v0.12.0-m1/go.mod h1:vjW5SFdXOQIVo6wvvLVh8uipruHnopP1gYInOhxy9gk= -github.com/submariner-io/admiral v0.12.0-m1.0.20211209141450-f39009d93c9d h1:Y6odrLRFwvSWv4u9yOEmX5+9x68wdIVsqhp0P6hBq/8= -github.com/submariner-io/admiral v0.12.0-m1.0.20211209141450-f39009d93c9d/go.mod h1:vjW5SFdXOQIVo6wvvLVh8uipruHnopP1gYInOhxy9gk= +github.com/submariner-io/admiral v0.12.0-m1.0.20211216212848-d630c38c2fd2 h1:S7YyoOIIkBoQFImVyWMhyAVJTdaX318O/uMJboV1410= +github.com/submariner-io/admiral v0.12.0-m1.0.20211216212848-d630c38c2fd2/go.mod h1:vjW5SFdXOQIVo6wvvLVh8uipruHnopP1gYInOhxy9gk= github.com/submariner-io/cloud-prepare v0.12.0-m1 h1:u034PljM3NQTb4p4nf5/yPbWdcLoaA3fW8DvbDJ9XtY= github.com/submariner-io/cloud-prepare v0.12.0-m1/go.mod h1:bMLl0JUT94idqHj9MKZATtEfETDkV4lt5pE3VRfb0H0= github.com/submariner-io/lighthouse v0.12.0-m1 h1:EdCZtoiEfXSTIUMDAQTaTCmYohZaGF7ZFDOJ7qF58oI= diff --git a/internal/restconfig/restconfig.go b/internal/restconfig/restconfig.go index c5d6c1f1b4..4c23858580 100644 --- a/internal/restconfig/restconfig.go +++ b/internal/restconfig/restconfig.go @@ -151,33 +151,36 @@ func (rcp *Producer) ForClusters() ([]RestConfig, error) { } func ForBroker(submariner *v1alpha1.Submariner, serviceDisc *v1alpha1.ServiceDiscovery) (*rest.Config, string, error) { + var restConfig *rest.Config + var namespace string + var err error + + // This is used in subctl; the broker secret isn't available mounted, so we use the old strings for now if submariner != nil { // Try to authorize against the submariner Cluster resource as we know the CRD should exist and the credentials // should allow read access. - restConfig, _, err := resource.GetAuthorizedRestConfig(submariner.Spec.BrokerK8sApiServer, submariner.Spec.BrokerK8sApiServerToken, + restConfig, _, err = resource.GetAuthorizedRestConfigFromData(submariner.Spec.BrokerK8sApiServer, + submariner.Spec.BrokerK8sApiServerToken, submariner.Spec.BrokerK8sCA, &rest.TLSClientConfig{}, schema.GroupVersionResource{ Group: subv1.SchemeGroupVersion.Group, Version: subv1.SchemeGroupVersion.Version, Resource: "clusters", }, submariner.Spec.BrokerK8sRemoteNamespace) - - return restConfig, submariner.Spec.BrokerK8sRemoteNamespace, errors.Wrap(err, "error getting auth rest config") - } - - if serviceDisc != nil { + namespace = submariner.Spec.BrokerK8sRemoteNamespace + } else if serviceDisc != nil { // Try to authorize against the ServiceImport resource as we know the CRD should exist and the credentials // should allow read access. - restConfig, _, err := resource.GetAuthorizedRestConfig(serviceDisc.Spec.BrokerK8sApiServer, serviceDisc.Spec.BrokerK8sApiServerToken, + restConfig, _, err = resource.GetAuthorizedRestConfigFromData(serviceDisc.Spec.BrokerK8sApiServer, + serviceDisc.Spec.BrokerK8sApiServerToken, serviceDisc.Spec.BrokerK8sCA, &rest.TLSClientConfig{}, schema.GroupVersionResource{ Group: "multicluster.x-k8s.io", Version: "v1alpha1", Resource: "serviceimports", }, serviceDisc.Spec.BrokerK8sRemoteNamespace) - - return restConfig, serviceDisc.Spec.BrokerK8sRemoteNamespace, errors.Wrap(err, "error getting auth rest config") + namespace = serviceDisc.Spec.BrokerK8sRemoteNamespace } - return nil, "", nil + return restConfig, namespace, errors.Wrap(err, "error getting auth rest config") } func clientConfigAndClusterName(rules *clientcmd.ClientConfigLoadingRules, overrides *clientcmd.ConfigOverrides) (RestConfig, error) {