OVNKubernetes CNI IC deployment support

Signed-off-by: Aswin Suryanarayanan <[email protected]>

config/rbac/submariner-operator/cluster_role.yaml

@@ -34,6 +34,7 @@ rules:
       - pods
       - services
       - nodes
+      - endpoints
     verbs:
       - get
       - list

controllers/submariner/cleanup.go

@@ -58,7 +58,7 @@ func (r *Reconciler) runComponentCleanup(ctx context.Context, instance *operator
 		},
 		{
 			Resource:          newDaemonSet(names.RouteAgentComponent, instance.Namespace),
-			UninstallResource: newRouteAgentDaemonSet(instance, names.AppendUninstall(names.RouteAgentComponent)),
+			UninstallResource: newRouteAgentDaemonSet(instance, clusterNetwork, names.AppendUninstall(names.RouteAgentComponent)),
 		},
 		{
 			Resource:          newDaemonSet(names.GlobalnetComponent, instance.Namespace),

controllers/submariner/route_agent_resources.go

@@ -20,6 +20,7 @@ package submariner
 
 import (
 	"context"
+	"github.com/submariner-io/submariner-operator/pkg/discovery/network"
 	"strconv"
 
 	"github.com/go-logr/logr"
@@ -35,21 +36,21 @@ import (
 )
 
 //nolint:wrapcheck // No need to wrap errors here.
-func (r *Reconciler) reconcileRouteagentDaemonSet(ctx context.Context, instance *v1alpha1.Submariner, reqLogger logr.Logger,
-) (*appsv1.DaemonSet, error) {
-	return apply.DaemonSet(ctx, instance, newRouteAgentDaemonSet(instance, names.RouteAgentComponent), reqLogger, r.config.ScopedClient,
+func (r *Reconciler) reconcileRouteagentDaemonSet(ctx context.Context, instance *v1alpha1.Submariner,
+	clusterNetwork *network.ClusterNetwork, reqLogger logr.Logger) (*appsv1.DaemonSet, error) {
+	return apply.DaemonSet(ctx, instance, newRouteAgentDaemonSet(instance, clusterNetwork, names.RouteAgentComponent), reqLogger, r.config.ScopedClient,
 		r.config.Scheme)
 }
 
-func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.DaemonSet {
+func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, clusterNetwork *network.ClusterNetwork, name string) *appsv1.DaemonSet {
 	labels := map[string]string{
 		"app":       name,
 		"component": "routeagent",
 	}
 
 	maxUnavailable := intstr.FromString("100%")
 
-	return &appsv1.DaemonSet{
+	ds := &appsv1.DaemonSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: cr.Namespace,
 			Name:      name,
@@ -128,4 +129,22 @@ func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.Daemon
 			},
 		},
 	}
+
+	if clusterNetwork.PluginSettings != nil {
+		if ovndb, ok := clusterNetwork.PluginSettings[network.OvnNBDB]; ok {
+			ds.Spec.Template.Spec.Containers[0].Env = append(
+				ds.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
+					Name: network.OvnNBDB, Value: ovndb,
+				})
+		}
+
+		if ovnsb, ok := clusterNetwork.PluginSettings[network.OvnSBDB]; ok {
+			ds.Spec.Template.Spec.Containers[0].Env = append(
+				ds.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
+					Name: network.OvnSBDB, Value: ovnsb,
+				})
+		}
+	}
+
+	return ds
 }

controllers/submariner/submariner_controller.go

@@ -172,7 +172,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
 		}
 	}
 
-	routeagentDaemonSet, err := r.reconcileRouteagentDaemonSet(ctx, instance, reqLogger)
+	routeagentDaemonSet, err := r.reconcileRouteagentDaemonSet(ctx, instance, clusterNetwork, reqLogger)
 	if err != nil {
 		return reconcile.Result{}, err
 	}

pkg/discovery/network/ovnkubernetes.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/submariner-io/submariner/pkg/cni"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -40,39 +41,113 @@ const (
 func discoverOvnKubernetesNetwork(ctx context.Context, client controllerClient.Client) (*ClusterNetwork, error) {
 	ovnDBPod, err := FindPod(ctx, client, "name=ovnkube-db")
 
-	if err != nil || ovnDBPod == nil {
-		return nil, err
+	var clusterNetwork *ClusterNetwork
+
+	if err == nil && ovnDBPod != nil {
+		clusterNetwork, err = discoverOvnDBClusterNetwork(ctx, client, ovnDBPod)
+	} else {
+		clusterNetwork, err = discoverOvnNodeClusterNetwork(ctx, client)
 	}
 
-	err = client.Get(ctx, types.NamespacedName{Namespace: ovnDBPod.Namespace, Name: ovnKubeService}, &corev1.Service{})
 	if err != nil {
-		return nil, fmt.Errorf("error finding %q service in %q namespace", ovnKubeService, ovnDBPod.Namespace)
+		return nil, err
 	}
 
-	dbConnectionProtocol := "tcp"
+	clusterNetwork.NetworkPlugin = cni.OVNKubernetes
 
-	for i := range ovnDBPod.Spec.Containers {
-		for _, envVar := range ovnDBPod.Spec.Containers[i].Env {
-			if envVar.Name == "OVN_SSL_ENABLE" {
-				if !strings.EqualFold(envVar.Value, "NO") {
-					dbConnectionProtocol = "ssl"
-				}
-			}
-		}
+	return clusterNetwork, nil
+}
+
+func discoverOvnDBClusterNetwork(ctx context.Context, client controllerClient.Client, ovnDBPod *corev1.Pod) (*ClusterNetwork, error) {
+	err := client.Get(ctx, types.NamespacedName{Namespace: ovnDBPod.Namespace, Name: ovnKubeService}, &corev1.Service{})
+	if err != nil {
+		return nil, fmt.Errorf("error finding %q service in %q namespace", ovnKubeService, ovnDBPod.Namespace)
 	}
 
+	dbConnectionProtocol := findProtocol(ovnDBPod)
+
 	clusterNetwork := &ClusterNetwork{
-		NetworkPlugin: cni.OVNKubernetes,
 		PluginSettings: map[string]string{
 			OvnNBDB: fmt.Sprintf("%s:%s.%s:%d", dbConnectionProtocol, ovnKubeService, ovnDBPod.Namespace, OvnNBDBDefaultPort),
 			OvnSBDB: fmt.Sprintf("%s:%s.%s:%d", dbConnectionProtocol, ovnKubeService, ovnDBPod.Namespace, OvnSBDBDefaultPort),
 		},
 	}
 
-	// If the cluster/service CIDRs weren't found we leave it to the generic functions to figure out later
+	return clusterNetwork, nil
+}
+
+func discoverOvnNodeClusterNetwork(ctx context.Context, client controllerClient.Client) (*ClusterNetwork, error) {
+	// In OVN IC deployments, the ovn DB will be a part of ovnkube-node
+	ovnPod, err := FindPod(ctx, client, "name=ovnkube-node")
+	if err != nil || ovnPod == nil {
+		return nil, err
+	}
+
+	endpointList, err := FindEndpoint(ctx, client, ovnPod.Namespace)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error retrieving the endpoints from namespace %q", ovnPod.Namespace)
+	}
+
+	var clusterNetwork *ClusterNetwork
+
+	if endpointList == nil || len(endpointList.Items) == 0 {
+		clusterNetwork, err = createLocalClusterNetwork(), nil
+	} else {
+		clusterNetwork, err = createClusterNetworkWithEndpoints(endpointList.Items), nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	err = updateClusterNetworkFromConfigMap(ctx, client, ovnPod.Namespace, clusterNetwork)
+	if err != nil {
+		return nil, err
+	}
+
+	return clusterNetwork, nil
+}
+
+func createLocalClusterNetwork() *ClusterNetwork {
+	return &ClusterNetwork{
+		PluginSettings: map[string]string{
+			OvnNBDB: "local",
+			OvnSBDB: "local",
+		},
+	}
+}
+
+func createClusterNetworkWithEndpoints(endPoints []corev1.Endpoints) *ClusterNetwork {
+	pluginSettings := map[string]string{}
+	var OvnNBDBIPs, OVNSBDBIps string
+	for index := 0; index < len(endPoints); index++ {
+		for _, subset := range endPoints[index].Subsets {
+			for _, port := range subset.Ports {
+				if strings.Contains(port.Name, "north") {
+					OvnNBDBIPs = OvnNBDBIPs + fmt.Sprintf("%s:%s:%s:%s:%d,",
+						"IC:", endPoints[index].Name, port.Protocol, subset.Addresses[0].IP, OvnNBDBDefaultPort)
+				} else if strings.Contains(port.Name, "south") {
+					OVNSBDBIps = OVNSBDBIps + fmt.Sprintf("%s:%s:%s:%s:%d,",
+						"IC:", endPoints[index].Name, port.Protocol, subset.Addresses[0].IP, OvnSBDBDefaultPort)
+				}
+			}
+		}
+	}
+
+	pluginSettings[OvnNBDB] = OvnNBDBIPs
+	pluginSettings[OvnSBDB] = OVNSBDBIps
+
+	return &ClusterNetwork{
+		PluginSettings: pluginSettings,
+	}
+}
+
+func updateClusterNetworkFromConfigMap(ctx context.Context, client controllerClient.Client, ovnPodNamespace string,
+	clusterNetwork *ClusterNetwork,
+) error {
 	ovnConfig := &corev1.ConfigMap{}
 
-	err = client.Get(ctx, types.NamespacedName{Namespace: ovnDBPod.Namespace, Name: "ovn-config"}, ovnConfig)
+	err := client.Get(ctx, types.NamespacedName{Namespace: ovnPodNamespace, Name: "ovn-config"}, ovnConfig)
 	if err == nil {
 		if netCidr, ok := ovnConfig.Data["net_cidr"]; ok {
 			clusterNetwork.PodCIDRs = []string{netCidr}
@@ -83,5 +158,32 @@ func discoverOvnKubernetesNetwork(ctx context.Context, client controllerClient.C
 		}
 	}
 
-	return clusterNetwork, nil
+	return errors.Wrap(err, "error getting the pod and service CIDRS")
+}
+
+func FindEndpoint(ctx context.Context, client controllerClient.Client, endpointNameSpace string) (*corev1.EndpointsList, error) {
+	endpointsList := &corev1.EndpointsList{}
+	listOptions := &controllerClient.ListOptions{
+		Namespace: endpointNameSpace,
+	}
+
+	err := client.List(ctx, endpointsList, listOptions)
+
+	return endpointsList, errors.WithMessagef(err, "error listing endpoints in namespace %q", endpointNameSpace)
+}
+
+func findProtocol(pod *corev1.Pod) string {
+	dbConnectionProtocol := "tcp"
+
+	for i := range pod.Spec.Containers {
+		for _, envVar := range pod.Spec.Containers[i].Env {
+			if envVar.Name == "OVN_SSL_ENABLE" {
+				if !strings.EqualFold(envVar.Value, "NO") {
+					dbConnectionProtocol = "ssl"
+				}
+			}
+		}
+	}
+
+	return dbConnectionProtocol
 }

pkg/embeddedyamls/yamls.go

@@ -2493,6 +2493,7 @@ rules:
       - pods
       - services
       - nodes
+      - endpoints
     verbs:
       - get
       - list