From 376ac21a48c18e2d53ac1d546e2ef4426a4489b0 Mon Sep 17 00:00:00 2001 From: Aswin Suryanarayanan Date: Wed, 12 Jul 2023 17:48:54 -0400 Subject: [PATCH] OVNKubernetes CNI IC deployment support Signed-off-by: Aswin Suryanarayanan --- controllers/submariner/cleanup.go | 2 +- .../submariner/route_agent_resources.go | 34 ++++- .../submariner/submariner_controller.go | 2 +- pkg/discovery/network/ovnkubernetes.go | 137 +++++++++++++++--- 4 files changed, 151 insertions(+), 24 deletions(-) diff --git a/controllers/submariner/cleanup.go b/controllers/submariner/cleanup.go index f5bd7c2939..bf409307c6 100644 --- a/controllers/submariner/cleanup.go +++ b/controllers/submariner/cleanup.go @@ -58,7 +58,7 @@ func (r *Reconciler) runComponentCleanup(ctx context.Context, instance *operator }, { Resource: newDaemonSet(names.RouteAgentComponent, instance.Namespace), - UninstallResource: newRouteAgentDaemonSet(instance, names.AppendUninstall(names.RouteAgentComponent)), + UninstallResource: newRouteAgentDaemonSet(instance, clusterNetwork, names.AppendUninstall(names.RouteAgentComponent)), }, { Resource: newDaemonSet(names.GlobalnetComponent, instance.Namespace), diff --git a/controllers/submariner/route_agent_resources.go b/controllers/submariner/route_agent_resources.go index 4d7954b576..2bffa45907 100644 --- a/controllers/submariner/route_agent_resources.go +++ b/controllers/submariner/route_agent_resources.go @@ -25,6 +25,7 @@ import ( "github.com/go-logr/logr" "github.com/submariner-io/submariner-operator/api/v1alpha1" "github.com/submariner-io/submariner-operator/controllers/apply" + "github.com/submariner-io/submariner-operator/pkg/discovery/network" "github.com/submariner-io/submariner-operator/pkg/images" "github.com/submariner-io/submariner-operator/pkg/names" appsv1 "k8s.io/api/apps/v1" @@ -35,13 +36,14 @@ import ( ) //nolint:wrapcheck // No need to wrap errors here. -func (r *Reconciler) reconcileRouteagentDaemonSet(ctx context.Context, instance *v1alpha1.Submariner, reqLogger logr.Logger, +func (r *Reconciler) reconcileRouteagentDaemonSet(ctx context.Context, instance *v1alpha1.Submariner, + clusterNetwork *network.ClusterNetwork, reqLogger logr.Logger, ) (*appsv1.DaemonSet, error) { - return apply.DaemonSet(ctx, instance, newRouteAgentDaemonSet(instance, names.RouteAgentComponent), reqLogger, r.config.ScopedClient, - r.config.Scheme) + return apply.DaemonSet(ctx, instance, newRouteAgentDaemonSet(instance, clusterNetwork, names.RouteAgentComponent), + reqLogger, r.config.ScopedClient, r.config.Scheme) } -func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.DaemonSet { +func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, clusterNetwork *network.ClusterNetwork, name string) *appsv1.DaemonSet { labels := map[string]string{ "app": name, "component": "routeagent", @@ -49,7 +51,7 @@ func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.Daemon maxUnavailable := intstr.FromString("100%") - return &appsv1.DaemonSet{ + ds := &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: cr.Namespace, Name: name, @@ -83,6 +85,9 @@ func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.Daemon {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ Path: "/sys", }}}, + {Name: "host-var-run-openvswitch-nbdb-sock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/run/openvswitch/ovnnb_db.sock", + }}}, }, Containers: []corev1.Container{ { @@ -102,6 +107,7 @@ func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.Daemon {Name: "host-sys", MountPath: "/sys", ReadOnly: true}, {Name: "host-run-xtables-lock", MountPath: "/run/xtables.lock"}, {Name: "host-run-openvswitch-db-sock", MountPath: "/run/openvswitch/db.sock"}, + {Name: "host-var-run-openvswitch-nbdb-sock", MountPath: "/var/run/openvswitch/ovnnb_db.sock"}, }, Env: []corev1.EnvVar{ {Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.Namespace}, @@ -128,4 +134,22 @@ func newRouteAgentDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.Daemon }, }, } + + if clusterNetwork.PluginSettings != nil { + if ovndb, ok := clusterNetwork.PluginSettings[network.OvnNBDB]; ok { + ds.Spec.Template.Spec.Containers[0].Env = append( + ds.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{ + Name: network.OvnNBDB, Value: ovndb, + }) + } + + if ovnsb, ok := clusterNetwork.PluginSettings[network.OvnSBDB]; ok { + ds.Spec.Template.Spec.Containers[0].Env = append( + ds.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{ + Name: network.OvnSBDB, Value: ovnsb, + }) + } + } + + return ds } diff --git a/controllers/submariner/submariner_controller.go b/controllers/submariner/submariner_controller.go index b379b03673..81ff80ff4f 100644 --- a/controllers/submariner/submariner_controller.go +++ b/controllers/submariner/submariner_controller.go @@ -172,7 +172,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( } } - routeagentDaemonSet, err := r.reconcileRouteagentDaemonSet(ctx, instance, reqLogger) + routeagentDaemonSet, err := r.reconcileRouteagentDaemonSet(ctx, instance, clusterNetwork, reqLogger) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/discovery/network/ovnkubernetes.go b/pkg/discovery/network/ovnkubernetes.go index e36077b623..7273a7dcf1 100644 --- a/pkg/discovery/network/ovnkubernetes.go +++ b/pkg/discovery/network/ovnkubernetes.go @@ -23,6 +23,7 @@ import ( "fmt" "strings" + "github.com/pkg/errors" "github.com/submariner-io/submariner/pkg/cni" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -39,40 +40,117 @@ const ( func discoverOvnKubernetesNetwork(ctx context.Context, client controllerClient.Client) (*ClusterNetwork, error) { ovnDBPod, err := FindPod(ctx, client, "name=ovnkube-db") + if err != nil { + return nil, err + } - if err != nil || ovnDBPod == nil { + var clusterNetwork *ClusterNetwork + + if ovnDBPod != nil { + clusterNetwork, err = discoverOvnDBClusterNetwork(ctx, client, ovnDBPod) + } else { + clusterNetwork, err = discoverOvnNodeClusterNetwork(ctx, client) + } + + if err != nil || clusterNetwork == nil { return nil, err } - err = client.Get(ctx, types.NamespacedName{Namespace: ovnDBPod.Namespace, Name: ovnKubeService}, &corev1.Service{}) + clusterNetwork.NetworkPlugin = cni.OVNKubernetes + + return clusterNetwork, nil +} + +func discoverOvnDBClusterNetwork(ctx context.Context, client controllerClient.Client, ovnDBPod *corev1.Pod) (*ClusterNetwork, error) { + err := client.Get(ctx, types.NamespacedName{Namespace: ovnDBPod.Namespace, Name: ovnKubeService}, &corev1.Service{}) if err != nil { return nil, fmt.Errorf("error finding %q service in %q namespace", ovnKubeService, ovnDBPod.Namespace) } - dbConnectionProtocol := "tcp" - - for i := range ovnDBPod.Spec.Containers { - for _, envVar := range ovnDBPod.Spec.Containers[i].Env { - if envVar.Name == "OVN_SSL_ENABLE" { - if !strings.EqualFold(envVar.Value, "NO") { - dbConnectionProtocol = "ssl" - } - } - } - } + dbConnectionProtocol := findProtocol(ovnDBPod) clusterNetwork := &ClusterNetwork{ - NetworkPlugin: cni.OVNKubernetes, PluginSettings: map[string]string{ OvnNBDB: fmt.Sprintf("%s:%s.%s:%d", dbConnectionProtocol, ovnKubeService, ovnDBPod.Namespace, OvnNBDBDefaultPort), OvnSBDB: fmt.Sprintf("%s:%s.%s:%d", dbConnectionProtocol, ovnKubeService, ovnDBPod.Namespace, OvnSBDBDefaultPort), }, } - // If the cluster/service CIDRs weren't found we leave it to the generic functions to figure out later + updateClusterNetworkFromConfigMap(ctx, client, ovnDBPod.Namespace, clusterNetwork) + + return clusterNetwork, nil +} + +func discoverOvnNodeClusterNetwork(ctx context.Context, client controllerClient.Client) (*ClusterNetwork, error) { + // In OVN IC deployments, the ovn DB will be a part of ovnkube-node + ovnPod, err := FindPod(ctx, client, "name=ovnkube-node") + if err != nil || ovnPod == nil { + return nil, err + } + + endpointList, err := FindEndpoint(ctx, client, ovnPod.Namespace) + if err != nil { + return nil, errors.Wrapf(err, "Error retrieving the endpoints from namespace %q", ovnPod.Namespace) + } + + var clusterNetwork *ClusterNetwork + + if endpointList == nil || len(endpointList.Items) == 0 { + clusterNetwork, err = createLocalClusterNetwork(), nil + } else { + clusterNetwork, err = createClusterNetworkWithEndpoints(endpointList.Items), nil + } + + if err != nil { + return nil, err + } + + updateClusterNetworkFromConfigMap(ctx, client, ovnPod.Namespace, clusterNetwork) + + return clusterNetwork, nil +} + +func createLocalClusterNetwork() *ClusterNetwork { + return &ClusterNetwork{ + PluginSettings: map[string]string{ + OvnNBDB: "local", + OvnSBDB: "local", + }, + } +} + +func createClusterNetworkWithEndpoints(endPoints []corev1.Endpoints) *ClusterNetwork { + pluginSettings := map[string]string{} + var OvnNBDBIPs, OVNSBDBIps string + + for index := 0; index < len(endPoints); index++ { + for _, subset := range endPoints[index].Subsets { + for _, port := range subset.Ports { + if strings.Contains(port.Name, "north") { + OvnNBDBIPs += fmt.Sprintf("%s:%s:%s:%s:%d,", + "IC:", endPoints[index].Name, port.Protocol, subset.Addresses[0].IP, OvnNBDBDefaultPort) + } else if strings.Contains(port.Name, "south") { + OVNSBDBIps += fmt.Sprintf("%s:%s:%s:%s:%d,", + "IC:", endPoints[index].Name, port.Protocol, subset.Addresses[0].IP, OvnSBDBDefaultPort) + } + } + } + } + + pluginSettings[OvnNBDB] = OvnNBDBIPs + pluginSettings[OvnSBDB] = OVNSBDBIps + + return &ClusterNetwork{ + PluginSettings: pluginSettings, + } +} + +func updateClusterNetworkFromConfigMap(ctx context.Context, client controllerClient.Client, ovnPodNamespace string, + clusterNetwork *ClusterNetwork, +) { ovnConfig := &corev1.ConfigMap{} - err = client.Get(ctx, types.NamespacedName{Namespace: ovnDBPod.Namespace, Name: "ovn-config"}, ovnConfig) + err := client.Get(ctx, types.NamespacedName{Namespace: ovnPodNamespace, Name: "ovn-config"}, ovnConfig) if err == nil { if netCidr, ok := ovnConfig.Data["net_cidr"]; ok { clusterNetwork.PodCIDRs = []string{netCidr} @@ -82,6 +160,31 @@ func discoverOvnKubernetesNetwork(ctx context.Context, client controllerClient.C clusterNetwork.ServiceCIDRs = []string{svcCidr} } } +} - return clusterNetwork, nil +func FindEndpoint(ctx context.Context, client controllerClient.Client, endpointNameSpace string) (*corev1.EndpointsList, error) { + endpointsList := &corev1.EndpointsList{} + listOptions := &controllerClient.ListOptions{ + Namespace: endpointNameSpace, + } + + err := client.List(ctx, endpointsList, listOptions) + + return endpointsList, errors.WithMessagef(err, "error listing endpoints in namespace %q", endpointNameSpace) +} + +func findProtocol(pod *corev1.Pod) string { + dbConnectionProtocol := "tcp" + + for i := range pod.Spec.Containers { + for _, envVar := range pod.Spec.Containers[i].Env { + if envVar.Name == "OVN_SSL_ENABLE" { + if !strings.EqualFold(envVar.Value, "NO") { + dbConnectionProtocol = "ssl" + } + } + } + } + + return dbConnectionProtocol }