Mount the Broker secret when available

... and pass its name to the pods which need it.

Fixes: #1685
Signed-off-by: Stephen Kitt <[email protected]>

controllers/servicediscovery/servicediscovery_controller.go

@@ -211,6 +211,23 @@ func newLighthouseAgent(cr *submarinerv1alpha1.ServiceDiscovery) *appsv1.Deploym
 
 	terminationGracePeriodSeconds := int64(0)
 
+	volumeMounts := []corev1.VolumeMount{}
+	volumes := []corev1.Volume{}
+
+	if cr.Spec.BrokerK8sSecret != "" {
+		// We've got a secret, mount it where the syncer expects it
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "brokersecret",
+			MountPath: broker.SecretPath(cr.Spec.BrokerK8sSecret),
+			ReadOnly:  true,
+		})
+
+		volumes = append(volumes, corev1.Volume{
+			Name:         "brokersecret",
+			VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: cr.Spec.BrokerK8sSecret}},
+		})
+	}
+
 	return &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: cr.Namespace,
@@ -243,12 +260,15 @@ func newLighthouseAgent(cr *submarinerv1alpha1.ServiceDiscovery) *appsv1.Deploym
 								{Name: broker.EnvironmentVariable("RemoteNamespace"), Value: cr.Spec.BrokerK8sRemoteNamespace},
 								{Name: broker.EnvironmentVariable("CA"), Value: cr.Spec.BrokerK8sCA},
 								{Name: broker.EnvironmentVariable("Insecure"), Value: strconv.FormatBool(cr.Spec.BrokerK8sInsecure)},
+								{Name: broker.EnvironmentVariable("Secret"), Value: cr.Spec.BrokerK8sSecret},
 							},
+							VolumeMounts: volumeMounts,
 						},
 					},
 
 					ServiceAccountName:            "submariner-lighthouse-agent",
 					TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,
+					Volumes:                       volumes,
 				},
 			},
 		},

controllers/submariner/gateway_resources.go

@@ -108,6 +108,31 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner) corev1.PodTemplateSpec {
 
 	nattPort, _ := strconv.ParseInt(submarinerv1.DefaultNATTDiscoveryPort, 10, 32)
 
+	volumeMounts := []corev1.VolumeMount{
+		{Name: "ipsecd", MountPath: "/etc/ipsec.d", ReadOnly: false},
+		{Name: "ipsecnss", MountPath: "/var/lib/ipsec/nss", ReadOnly: false},
+		{Name: "libmodules", MountPath: "/lib/modules", ReadOnly: true},
+	}
+	volumes := []corev1.Volume{
+		{Name: "ipsecd", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
+		{Name: "ipsecnss", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
+		{Name: "libmodules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}},
+	}
+
+	if cr.Spec.BrokerK8sSecret != "" {
+		// We've got a secret, mount it where the syncer expects it
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "brokersecret",
+			MountPath: broker.SecretPath(cr.Spec.BrokerK8sSecret),
+			ReadOnly:  true,
+		})
+
+		volumes = append(volumes, corev1.Volume{
+			Name:         "brokersecret",
+			VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: cr.Spec.BrokerK8sSecret}},
+		})
+	}
+
 	podTemplate := corev1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: labels,
@@ -170,6 +195,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner) corev1.PodTemplateSpec {
 						{Name: broker.EnvironmentVariable("RemoteNamespace"), Value: cr.Spec.BrokerK8sRemoteNamespace},
 						{Name: broker.EnvironmentVariable("CA"), Value: cr.Spec.BrokerK8sCA},
 						{Name: broker.EnvironmentVariable("Insecure"), Value: strconv.FormatBool(cr.Spec.BrokerK8sInsecure)},
+						{Name: broker.EnvironmentVariable("Secret"), Value: cr.Spec.BrokerK8sSecret},
 						{Name: "CE_IPSEC_PSK", Value: cr.Spec.CeIPSecPSK},
 						{Name: "CE_IPSEC_DEBUG", Value: strconv.FormatBool(cr.Spec.CeIPSecDebug)},
 						{Name: "SUBMARINER_HEALTHCHECKENABLED", Value: strconv.FormatBool(healthCheckEnabled)},
@@ -186,11 +212,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner) corev1.PodTemplateSpec {
 							},
 						}},
 					},
-					VolumeMounts: []corev1.VolumeMount{
-						{Name: "ipsecd", MountPath: "/etc/ipsec.d", ReadOnly: false},
-						{Name: "ipsecnss", MountPath: "/var/lib/ipsec/nss", ReadOnly: false},
-						{Name: "libmodules", MountPath: "/lib/modules", ReadOnly: true},
-					},
+					VolumeMounts: volumeMounts,
 				},
 			},
 			// TODO: Use SA submariner-gateway or submariner?
@@ -201,11 +223,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner) corev1.PodTemplateSpec {
 			DNSPolicy:                     corev1.DNSClusterFirst,
 			// The gateway engine must be able to run on any flagged node, regardless of existing taints
 			Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}},
-			Volumes: []corev1.Volume{
-				{Name: "ipsecd", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
-				{Name: "ipsecnss", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
-				{Name: "libmodules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}},
-			},
+			Volumes:     volumes,
 		},
 	}
 	if cr.Spec.CeIPSecIKEPort != 0 {

controllers/submariner/submariner_controller_test.go

@@ -407,6 +407,7 @@ func verifyGatewayDaemonSet(ctx context.Context, submariner *submariner_v1.Subma
 	Expect(envMap).To(HaveKeyWithValue(broker.EnvironmentVariable("ApiServerToken"), submariner.Spec.BrokerK8sApiServerToken))
 	Expect(envMap).To(HaveKeyWithValue(broker.EnvironmentVariable("CA"), submariner.Spec.BrokerK8sCA))
 	Expect(envMap).To(HaveKeyWithValue(broker.EnvironmentVariable("Insecure"), strconv.FormatBool(submariner.Spec.BrokerK8sInsecure)))
+	Expect(envMap).To(HaveKeyWithValue(broker.EnvironmentVariable("Secret"), submariner.Spec.BrokerK8sSecret))
 	Expect(envMap).To(HaveKeyWithValue("SUBMARINER_BROKER", submariner.Spec.Broker))
 	Expect(envMap).To(HaveKeyWithValue("SUBMARINER_NATENABLED", strconv.FormatBool(submariner.Spec.NatEnabled)))
 	Expect(envMap).To(HaveKeyWithValue("SUBMARINER_CLUSTERID", submariner.Spec.ClusterID))

go.mod

@@ -77,3 +77,5 @@ replace (
 	k8s.io/cloud-provider => k8s.io/cloud-provider v0.19.10
 	sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.7.0
 )
+
+replace github.com/submariner-io/admiral => github.com/skitt/admiral v0.0.0-20211215160040-faaee27f1510

go.sum

@@ -1278,6 +1278,8 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/skitt/admiral v0.0.0-20211215160040-faaee27f1510 h1:EBPO5jCBJD/30STwToO+1AX6ikf0jYIDcvJ8CCchEG8=
+github.com/skitt/admiral v0.0.0-20211215160040-faaee27f1510/go.mod h1:vjW5SFdXOQIVo6wvvLVh8uipruHnopP1gYInOhxy9gk=
 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
@@ -1334,9 +1336,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/submariner-io/admiral v0.12.0-m1/go.mod h1:vjW5SFdXOQIVo6wvvLVh8uipruHnopP1gYInOhxy9gk=
-github.com/submariner-io/admiral v0.12.0-m1.0.20211209141450-f39009d93c9d h1:Y6odrLRFwvSWv4u9yOEmX5+9x68wdIVsqhp0P6hBq/8=
-github.com/submariner-io/admiral v0.12.0-m1.0.20211209141450-f39009d93c9d/go.mod h1:vjW5SFdXOQIVo6wvvLVh8uipruHnopP1gYInOhxy9gk=
 github.com/submariner-io/cloud-prepare v0.12.0-m1 h1:u034PljM3NQTb4p4nf5/yPbWdcLoaA3fW8DvbDJ9XtY=
 github.com/submariner-io/cloud-prepare v0.12.0-m1/go.mod h1:bMLl0JUT94idqHj9MKZATtEfETDkV4lt5pE3VRfb0H0=
 github.com/submariner-io/lighthouse v0.12.0-m1 h1:EdCZtoiEfXSTIUMDAQTaTCmYohZaGF7ZFDOJ7qF58oI=

internal/restconfig/restconfig.go

@@ -151,33 +151,36 @@ func (rcp *Producer) ForClusters() ([]RestConfig, error) {
 }
 
 func ForBroker(submariner *v1alpha1.Submariner, serviceDisc *v1alpha1.ServiceDiscovery) (*rest.Config, string, error) {
+	var restConfig *rest.Config
+	var namespace string
+	var err error
+
+	// This is used in subctl; the broker secret isn't available mounted, so we use the old strings for now
 	if submariner != nil {
 		// Try to authorize against the submariner Cluster resource as we know the CRD should exist and the credentials
 		// should allow read access.
-		restConfig, _, err := resource.GetAuthorizedRestConfig(submariner.Spec.BrokerK8sApiServer, submariner.Spec.BrokerK8sApiServerToken,
+		restConfig, _, err = resource.GetAuthorizedRestConfigFromData(submariner.Spec.BrokerK8sApiServer,
+			submariner.Spec.BrokerK8sApiServerToken,
 			submariner.Spec.BrokerK8sCA, &rest.TLSClientConfig{}, schema.GroupVersionResource{
 				Group:    subv1.SchemeGroupVersion.Group,
 				Version:  subv1.SchemeGroupVersion.Version,
 				Resource: "clusters",
 			}, submariner.Spec.BrokerK8sRemoteNamespace)
-
-		return restConfig, submariner.Spec.BrokerK8sRemoteNamespace, errors.Wrap(err, "error getting auth rest config")
-	}
-
-	if serviceDisc != nil {
+		namespace = submariner.Spec.BrokerK8sRemoteNamespace
+	} else if serviceDisc != nil {
 		// Try to authorize against the ServiceImport resource as we know the CRD should exist and the credentials
 		// should allow read access.
-		restConfig, _, err := resource.GetAuthorizedRestConfig(serviceDisc.Spec.BrokerK8sApiServer, serviceDisc.Spec.BrokerK8sApiServerToken,
+		restConfig, _, err = resource.GetAuthorizedRestConfigFromData(serviceDisc.Spec.BrokerK8sApiServer,
+			serviceDisc.Spec.BrokerK8sApiServerToken,
 			serviceDisc.Spec.BrokerK8sCA, &rest.TLSClientConfig{}, schema.GroupVersionResource{
 				Group:    "multicluster.x-k8s.io",
 				Version:  "v1alpha1",
 				Resource: "serviceimports",
 			}, serviceDisc.Spec.BrokerK8sRemoteNamespace)
-
-		return restConfig, serviceDisc.Spec.BrokerK8sRemoteNamespace, errors.Wrap(err, "error getting auth rest config")
+		namespace = serviceDisc.Spec.BrokerK8sRemoteNamespace
 	}
 
-	return nil, "", nil
+	return restConfig, namespace, errors.Wrap(err, "error getting auth rest config")
 }
 
 func clientConfigAndClusterName(rules *clientcmd.ClientConfigLoadingRules, overrides *clientcmd.ConfigOverrides) (RestConfig, error) {