Add label selectors to informer's cache (#361)

* Informers caches with label selector

* wip

* wip

* docs 2

* wip

* wip

* Apply suggestions from code review

Co-authored-by: Iwona Langer <[email protected]>

* Apply suggestions from code review

Co-authored-by: Iwona Langer <[email protected]>

* Apply suggestions from code review

Co-authored-by: Iwona Langer <[email protected]>

---------

Co-authored-by: Iwona Langer <[email protected]>

Makefile

@@ -148,7 +148,7 @@ module-image: docker-build docker-push ## Build the Module Image and push it to
 .PHONY: module-build
 module-build: kyma kustomize ## Build the Module and push it to a registry defined in MODULE_REGISTRY
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KYMA) alpha create module --channel=${MODULE_CHANNEL} --name kyma.project.io/module/$(MODULE_NAME) --version $(MODULE_VERSION) --target="remote" --path . $(MODULE_CREATION_FLAGS)
+	$(KYMA) alpha create module --channel=${MODULE_CHANNEL} --name kyma.project.io/module/$(MODULE_NAME) --version $(MODULE_VERSION) --path . $(MODULE_CREATION_FLAGS)
 
 .PHONY: module-template-push
 module-template-push: ## Pushes the ModuleTemplate referencing the Image on MODULE_REGISTRY

controllers/cache.go

@@ -0,0 +1,30 @@
+package controllers
+
+import (
+	"fmt"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func CacheCreator(conf *rest.Config, opts cache.Options) (cache.Cache, error) {
+	labelSelector, err := labels.Parse("app.kubernetes.io/managed-by in (btp-manager,kcp-kyma-environment-broker)")
+	if err != nil {
+		panic(fmt.Sprintf("unable to parse label selector: %s", err))
+	}
+	objSelector := cache.ObjectSelector{
+		Label: labelSelector,
+	}
+
+	opts.SelectorsByObject = map[client.Object]cache.ObjectSelector{
+		&corev1.Secret{}:    objSelector,
+		&corev1.ConfigMap{}: objSelector,
+		&admissionregistrationv1.ValidatingWebhookConfiguration{}: objSelector,
+		&admissionregistrationv1.MutatingWebhookConfiguration{}:   objSelector,
+	}
+
+	return cache.New(conf, opts)
+}

controllers/suite_test.go

@@ -154,6 +154,7 @@ var _ = SynchronizedBeforeSuite(func() {
 		Scheme:                 scheme.Scheme,
 		MetricsBindAddress:     "0",
 		HealthProbeBindAddress: "0",
+		NewCache:               CacheCreator,
 	})
 	Expect(err).ToNot(HaveOccurred())
 

controllers/testdata/test-secret.yaml

@@ -4,6 +4,8 @@ type: Opaque
 metadata:
   name: sap-btp-manager
   namespace: kyma-system
+  labels:
+    app.kubernetes.io/managed-by: kcp-kyma-environment-broker
 data:
   clientid: dGVzdF9jbGllbnRpZA==
   clientsecret: dGVzdF9jbGllbnRzZWNyZXQ=

controllers/utils_test.go

@@ -398,6 +398,7 @@ func initConfig(data map[string]string) *corev1.ConfigMap {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      ConfigName,
 			Namespace: ChartNamespace,
+			Labels:    map[string]string{managedByLabelKey: operatorName},
 		},
 		Data: data,
 	}

docs/contributor/01-20-configuration.md

@@ -62,6 +62,8 @@ kind: ConfigMap
 metadata:
   name: sap-btp-manager
   namespace: kyma-system
+  labels:
+    app.kubernetes.io/managed-by: btp-manager
 data:
   ChartPath: ./module-chart/chart
   ChartNamespace: kyma-system

docs/contributor/02-10-operations.md

@@ -34,6 +34,8 @@ apiVersion: operator.kyma-project.io/v1alpha1
 kind: BtpOperator
 metadata:
   name: btpoperator
+  labels:
+    app.kubernetes.io/managed-by: btp-manager
 EOF
 ```
 
@@ -44,7 +46,7 @@ reconciliation proceeds.
 Otherwise, it is given the `Error` state with the condition reason `OlderCRExists` and message containing details
 about the CR responsible for reconciling the operand.
 
-Next, the reconciler looks for a `sap-btp-manager` Secret in the `kyma-system` Namespace. This Secret contains Service
+Next, the reconciler looks for a `sap-btp-manager` Secret in the `kyma-system` Namespace with the label `app.kubernetes.io/managed-by: kcp-kyma-environment-broker`. This Secret contains the Service
 Manager credentials for SAP BTP Service Operator and should be delivered to the cluster by KEB. If the Secret is
 missing, an error is thrown, the reconciler sets `Warning` state (with the condition reason `MissingSecret`) in the CR and stops the reconciliation until the Secret
 is created. When the Secret is present in the cluster, the reconciler verifies whether it contains required data. The

docs/contributor/07-10-informer-cache.md

@@ -0,0 +1,3 @@
+# Informer's cache
+
+The controller manager uses informers with a cache. All observed resources (btpoperator, Secret, ConfigMap etc.) are stored in the cache. Because of the out of memory risk, the cache is configured with a label selector `app.kubernetes.io/managed-by in (btp-manager,kcp-kyma-environment-broker)`. See [cache.go](/controllers/cache.go).

examples/btp-manager-secret.yaml

@@ -4,6 +4,8 @@ type: Opaque
 metadata:
   name: sap-btp-manager
   namespace: kyma-system
+  labels:
+    app.kubernetes.io/managed-by: kcp-kyma-environment-broker
 data:
   clientid: dGVzdF9jbGllbnRpZA==
   clientsecret: dGVzdF9jbGllbnRzZWNyZXQ=

examples/btp-operator-configmap.yaml

@@ -3,6 +3,8 @@ kind: ConfigMap
 metadata:
   name: sap-btp-manager
   namespace: kyma-system
+  labels:
+    app.kubernetes.io/managed-by: kcp-kyma-environment-broker
 data:
   ChartPath: ./module-chart/chart
   ChartNamespace: kyma-system

main.go

@@ -18,10 +18,8 @@ package main
 
 import (
 	"flag"
-	"os"
-
 	btpmanagermetrics "github.com/kyma-project/btp-manager/internal/metrics"
-
+	"os"
 	//test
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -86,22 +84,12 @@ func main() {
 	restCfg := ctrl.GetConfigOrDie()
 	mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
 		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
-		Port:                   9443,
-		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "ec023d38.kyma-project.io",
-		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
-		// when the Manager ends. This requires the binary to immediately end when the
-		// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
-		// speeds up voluntary leader transitions as the new leader don't have to wait
-		// LeaseDuration time first.
-		//
-		// In the default scaffold provided, the program ends immediately after
-		// the manager stops, so would be fine to enable this option. However,
-		// if you are doing or is intended to do any operation such as perform cleanups
-		// after the manager stops then its usage might be unsafe.
-		// LeaderElectionReleaseOnCancel: true,
+		MetricsBindAddress:     metricsAddr,
+		HealthProbeBindAddress: probeAddr,
+		Port:                   9443,
+		NewCache:               controllers.CacheCreator,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

scripts/testing/create-secrets-configmaps.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Stress testing in regard to memory consumption - could cause OOM (but should not).
+# Creates btp-operator and numerous Service Instances and Service Bindings in the current context.
+#
+# The script has the following arguments:
+#     - number of Config Maps and Secrets (the provided number is multiplied by 10)
+# Example (creates 1000 CMs and Secrets):
+#     ./stress-mem.sh 100
+
+N=${1-100}
+YAML_DIR=./scripts/testing/yaml
+
+echo -e "\n---Creating ${N} config maps and secrets"
+
+for ((i=1; i <= N ; i++))
+  do
+      CM_NAME=cm-$i
+      SECRET_NAME=secret-$i
+
+      export CM_NAME
+      export SECRET_NAME
+
+      envsubst <${YAML_DIR}/stress-cm.yaml | kubectl apply -f - >/dev/null
+      envsubst <${YAML_DIR}/stress-secret.yaml | kubectl apply -f - >/dev/null
+done

scripts/testing/yaml/e2e-test-configmap.yaml

@@ -3,5 +3,7 @@ kind: ConfigMap
 metadata:
   name: sap-btp-manager
   namespace: kyma-system
+  labels:
+    app.kubernetes.io/managed-by: btp-manager
 data:
   HardDeleteTimeout: "10s"

scripts/testing/yaml/e2e-test-secret.yaml

@@ -4,6 +4,8 @@ type: Opaque
 metadata:
   name: sap-btp-manager
   namespace: kyma-system
+  labels:
+    app.kubernetes.io/managed-by: kcp-kyma-environment-broker
 data:
   clientid: ${SM_CLIENT_ID}
   clientsecret: ${SM_CLIENT_SECRET}

scripts/testing/yaml/stress-cm.yaml

@@ -0,0 +1,79 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-001
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-002
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-003
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-004
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-005
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-006
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-007
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-008
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-009
+  namespace: kyma-system
+data:
+  testData: "abcd"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CM_NAME}-000
+  namespace: kyma-system
+data:
+  testData: "abcd"

scripts/testing/yaml/stress-secret.yaml

@@ -0,0 +1,89 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-001
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-002
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-003
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-004
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-005
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-006
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-007
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-008
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-009
+  namespace: kyma-system
+data:
+  testData: test
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: ${SECRET_NAME}-000
+  namespace: kyma-system
+data:
+  testData: test