[WIP] A prototype implementation to improve startup performance

controllers/ingress/group_controller.go

@@ -2,7 +2,10 @@ package ingress
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
@@ -168,12 +171,24 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
 		return nil, nil, err
 	}
 	r.logger.Info("successfully built model", "model", stackJSON)
+	currentCheckpoint := r.computeReconcileCheckpoint(ctx, stackJSON)
+	originalCheckpoint := r.getReconcileCheckpoint(ctx, ingGroup)
+	if currentCheckpoint == originalCheckpoint {
+		r.logger.Info("ingress hasn't changed, skip deploying model")
+		// TODO: this whole bit code needs some refactors,
+		// e.g. when skip deploy model we should log a "Successfully reconciled" event.
+		return nil, nil, nil
+	}
 
 	if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
 		r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
 		return nil, nil, err
 	}
 	r.logger.Info("successfully deployed model", "ingressGroup", ingGroup.ID)
+	if err := r.saveReconcileCheckpoint(ctx, ingGroup, currentCheckpoint); err != nil {
+		return nil, nil, err
+	}
+
 	r.secretsManager.MonitorSecrets(ingGroup.ID.String(), secrets)
 	var inactiveResources []types.NamespacedName
 	inactiveResources = append(inactiveResources, k8s.ToSliceOfNamespacedNames(ingGroup.InactiveMembers)...)
@@ -218,6 +233,42 @@ func (r *groupReconciler) updateIngressStatus(ctx context.Context, lbDNS string,
 	return nil
 }
 
+// computeReconcileCheckpoint computes a checkpoint based on generated stack json.
+func (r *groupReconciler) computeReconcileCheckpoint(_ context.Context, stackJSON string) string {
+	checkpointHash := sha256.New()
+	_, _ = checkpointHash.Write([]byte(stackJSON))
+	return base64.RawURLEncoding.EncodeToString(checkpointHash.Sum(nil))
+}
+
+// getReconcileCheckpoint retrieves the last known reconciled checkpoint for the ingress group.
+func (r *groupReconciler) getReconcileCheckpoint(_ context.Context, ingGroup ingress.Group) string {
+	checkpoints := sets.NewString()
+	for _, member := range ingGroup.Members {
+		if ingCheckpoint, ok := member.Ing.Annotations[annotations.AnnotationCheckPoint]; ok {
+			checkpoints.Insert(ingCheckpoint)
+		}
+	}
+	if len(checkpoints) == 1 {
+		checkpoint, _ := checkpoints.PopAny()
+		return checkpoint
+	}
+	return ""
+}
+
+// saveReconcileCheckpoint persists reconcile checkpoint into ingressGroup as annotation
+func (r *groupReconciler) saveReconcileCheckpoint(ctx context.Context, ingGroup ingress.Group, checkpoint string) error {
+	for _, member := range ingGroup.Members {
+		if ingCheckpoint := member.Ing.Annotations[annotations.AnnotationCheckPoint]; ingCheckpoint != checkpoint {
+			ingNew := member.Ing.DeepCopy()
+			ingNew.Annotations[annotations.AnnotationCheckPoint] = checkpoint
+			if err := r.k8sClient.Patch(ctx, ingNew, client.MergeFrom(member.Ing)); err != nil {
+				return errors.Wrapf(err, "failed to update reconcile checkpoint: %v", k8s.NamespacedName(ingNew))
+			}
+		}
+	}
+	return nil
+}
+
 func (r *groupReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, clientSet *kubernetes.Clientset) error {
 	c, err := controller.New(controllerName, mgr, controller.Options{
 		MaxConcurrentReconciles: r.maxConcurrentReconciles,

controllers/service/service_controller.go

@@ -2,8 +2,9 @@ package service
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
-
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -97,29 +98,42 @@ func (r *serviceReconciler) reconcile(ctx context.Context, req ctrl.Request) err
 	if err := r.k8sClient.Get(ctx, req.NamespacedName, svc); err != nil {
 		return client.IgnoreNotFound(err)
 	}
-	stack, lb, backendSGRequired, err := r.buildModel(ctx, svc)
+	stack, lb, backendSGRequired, stackJSON, err := r.buildModel(ctx, svc)
 	if err != nil {
 		return err
 	}
 	if lb == nil {
 		return r.cleanupLoadBalancerResources(ctx, svc, stack)
 	}
-	return r.reconcileLoadBalancerResources(ctx, svc, stack, lb, backendSGRequired)
+
+	currentCheckpoint := r.computeReconcileCheckpoint(ctx, stackJSON)
+	originalCheckpoint := r.getReconcileCheckpoint(ctx, svc)
+	if currentCheckpoint == originalCheckpoint {
+		r.logger.Info("service hasn't changed, skip deploying model")
+		// TODO: this whole bit code in this file needs some refactors,
+		// e.g. stackJson shouldn't be returned from buildModel
+		return nil
+	}
+
+	if err := r.reconcileLoadBalancerResources(ctx, svc, stack, lb, backendSGRequired); err != nil {
+		return err
+	}
+	return r.saveReconcileCheckpoint(ctx, svc, currentCheckpoint)
 }
 
-func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, bool, error) {
+func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, bool, string, error) {
 	stack, lb, backendSGRequired, err := r.modelBuilder.Build(ctx, svc)
 	if err != nil {
 		r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
-		return nil, nil, false, err
+		return nil, nil, false, "", err
 	}
 	stackJSON, err := r.stackMarshaller.Marshal(stack)
 	if err != nil {
 		r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
-		return nil, nil, false, err
+		return nil, nil, false, "", err
 	}
 	r.logger.Info("successfully built model", "model", stackJSON)
-	return stack, lb, backendSGRequired, nil
+	return stack, lb, backendSGRequired, stackJSON, nil
 }
 
 func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error {
@@ -208,6 +222,28 @@ func (r *serviceReconciler) cleanupServiceStatus(ctx context.Context, svc *corev
 	return nil
 }
 
+func (r *serviceReconciler) computeReconcileCheckpoint(_ context.Context, stackJSON string) string {
+	checkpointHash := sha256.New()
+	_, _ = checkpointHash.Write([]byte(stackJSON))
+	return base64.RawURLEncoding.EncodeToString(checkpointHash.Sum(nil))
+}
+
+func (r *serviceReconciler) getReconcileCheckpoint(_ context.Context, svc *corev1.Service) string {
+	if svcCheckpoint, ok := svc.Annotations[annotations.AnnotationCheckPoint]; ok {
+		return svcCheckpoint
+	}
+	return ""
+}
+
+func (r *serviceReconciler) saveReconcileCheckpoint(ctx context.Context, svc *corev1.Service, checkpoint string) error {
+	svcNew := svc.DeepCopy()
+	svcNew.Annotations[annotations.AnnotationCheckPoint] = checkpoint
+	if err := r.k8sClient.Patch(ctx, svcNew, client.MergeFrom(svc)); err != nil {
+		return errors.Wrapf(err, "failed to update reconcile checkpoint: %v", k8s.NamespacedName(svcNew))
+	}
+	return nil
+}
+
 func (r *serviceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	svcEventHandler := eventhandlers.NewEnqueueRequestForServiceEvent(r.eventRecorder,
 		r.serviceUtils, r.logger.WithName("eventHandlers").WithName("service"))

pkg/annotations/constants.go

@@ -1,9 +1,16 @@
 package annotations
 
 const (
-	// IngressClass
-	IngressClass = "kubernetes.io/ingress.class"
+	// Controller-Managed annotations
+
+	// AnnotationCheckPoint is the annotation used to store a checkpoint for resources.
+	// It contains an opaque value that represents the last known reconciled state.
+	AnnotationCheckPoint = "elbv2.k8s.aws/checkpoint"
 
+	// User-Defined annotations
+
+	// IngressClass
+	IngressClass            = "kubernetes.io/ingress.class"
 	AnnotationPrefixIngress = "alb.ingress.kubernetes.io"
 	// Ingress annotation suffixes
 	IngressSuffixLoadBalancerName             = "load-balancer-name"