diff --git a/Makefile b/Makefile index d43eb5bfc..867d2349b 100644 --- a/Makefile +++ b/Makefile @@ -148,7 +148,7 @@ module-image: docker-build docker-push ## Build the Module Image and push it to .PHONY: module-build module-build: kyma kustomize ## Build the Module and push it to a registry defined in MODULE_REGISTRY cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KYMA) alpha create module --channel=${MODULE_CHANNEL} --name kyma.project.io/module/$(MODULE_NAME) --version $(MODULE_VERSION) --target="remote" --path . $(MODULE_CREATION_FLAGS) + $(KYMA) alpha create module --channel=${MODULE_CHANNEL} --name kyma.project.io/module/$(MODULE_NAME) --version $(MODULE_VERSION) --path . $(MODULE_CREATION_FLAGS) .PHONY: module-template-push module-template-push: ## Pushes the ModuleTemplate referencing the Image on MODULE_REGISTRY diff --git a/controllers/cache.go b/controllers/cache.go new file mode 100644 index 000000000..8c6e00873 --- /dev/null +++ b/controllers/cache.go @@ -0,0 +1,30 @@ +package controllers + +import ( + "fmt" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func CacheCreator(conf *rest.Config, opts cache.Options) (cache.Cache, error) { + labelSelector, err := labels.Parse("app.kubernetes.io/managed-by in (btp-manager,kcp-kyma-environment-broker)") + if err != nil { + panic(fmt.Sprintf("unable to parse label selector: %s", err)) + } + objSelector := cache.ObjectSelector{ + Label: labelSelector, + } + + opts.SelectorsByObject = map[client.Object]cache.ObjectSelector{ + &corev1.Secret{}: objSelector, + &corev1.ConfigMap{}: objSelector, + &admissionregistrationv1.ValidatingWebhookConfiguration{}: objSelector, + &admissionregistrationv1.MutatingWebhookConfiguration{}: objSelector, + } + + return cache.New(conf, opts) +} diff --git a/controllers/suite_test.go b/controllers/suite_test.go index dbf3c3f6b..a8f028158 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -154,6 +154,7 @@ var _ = SynchronizedBeforeSuite(func() { Scheme: scheme.Scheme, MetricsBindAddress: "0", HealthProbeBindAddress: "0", + NewCache: CacheCreator, }) Expect(err).ToNot(HaveOccurred()) diff --git a/controllers/testdata/test-secret.yaml b/controllers/testdata/test-secret.yaml index efe57dbfc..43a30b501 100644 --- a/controllers/testdata/test-secret.yaml +++ b/controllers/testdata/test-secret.yaml @@ -4,6 +4,8 @@ type: Opaque metadata: name: sap-btp-manager namespace: kyma-system + labels: + app.kubernetes.io/managed-by: kcp-kyma-environment-broker data: clientid: dGVzdF9jbGllbnRpZA== clientsecret: dGVzdF9jbGllbnRzZWNyZXQ= diff --git a/controllers/utils_test.go b/controllers/utils_test.go index 8aa9eaed4..f3de1402a 100644 --- a/controllers/utils_test.go +++ b/controllers/utils_test.go @@ -398,6 +398,7 @@ func initConfig(data map[string]string) *corev1.ConfigMap { ObjectMeta: metav1.ObjectMeta{ Name: ConfigName, Namespace: ChartNamespace, + Labels: map[string]string{managedByLabelKey: operatorName}, }, Data: data, } diff --git a/docs/contributor/01-20-configuration.md b/docs/contributor/01-20-configuration.md index ab6746326..d348c1891 100644 --- a/docs/contributor/01-20-configuration.md +++ b/docs/contributor/01-20-configuration.md @@ -62,6 +62,8 @@ kind: ConfigMap metadata: name: sap-btp-manager namespace: kyma-system + labels: + app.kubernetes.io/managed-by: btp-manager data: ChartPath: ./module-chart/chart ChartNamespace: kyma-system diff --git a/docs/contributor/02-10-operations.md b/docs/contributor/02-10-operations.md index a1801e8d9..65e157443 100644 --- a/docs/contributor/02-10-operations.md +++ b/docs/contributor/02-10-operations.md @@ -34,6 +34,8 @@ apiVersion: operator.kyma-project.io/v1alpha1 kind: BtpOperator metadata: name: btpoperator + labels: + app.kubernetes.io/managed-by: btp-manager EOF ``` @@ -44,7 +46,7 @@ reconciliation proceeds. Otherwise, it is given the `Error` state with the condition reason `OlderCRExists` and message containing details about the CR responsible for reconciling the operand. -Next, the reconciler looks for a `sap-btp-manager` Secret in the `kyma-system` Namespace. This Secret contains Service +Next, the reconciler looks for a `sap-btp-manager` Secret in the `kyma-system` Namespace with the label `app.kubernetes.io/managed-by: kcp-kyma-environment-broker`. This Secret contains the Service Manager credentials for SAP BTP Service Operator and should be delivered to the cluster by KEB. If the Secret is missing, an error is thrown, the reconciler sets `Warning` state (with the condition reason `MissingSecret`) in the CR and stops the reconciliation until the Secret is created. When the Secret is present in the cluster, the reconciler verifies whether it contains required data. The diff --git a/docs/contributor/07-10-informer-cache.md b/docs/contributor/07-10-informer-cache.md new file mode 100644 index 000000000..5961bad9c --- /dev/null +++ b/docs/contributor/07-10-informer-cache.md @@ -0,0 +1,3 @@ +# Informer's cache + +The controller manager uses informers with a cache. All observed resources (btpoperator, Secret, ConfigMap etc.) are stored in the cache. Because of the out of memory risk, the cache is configured with a label selector `app.kubernetes.io/managed-by in (btp-manager,kcp-kyma-environment-broker)`. See [cache.go](/controllers/cache.go). \ No newline at end of file diff --git a/examples/btp-manager-secret.yaml b/examples/btp-manager-secret.yaml index efe57dbfc..43a30b501 100644 --- a/examples/btp-manager-secret.yaml +++ b/examples/btp-manager-secret.yaml @@ -4,6 +4,8 @@ type: Opaque metadata: name: sap-btp-manager namespace: kyma-system + labels: + app.kubernetes.io/managed-by: kcp-kyma-environment-broker data: clientid: dGVzdF9jbGllbnRpZA== clientsecret: dGVzdF9jbGllbnRzZWNyZXQ= diff --git a/examples/btp-operator-configmap.yaml b/examples/btp-operator-configmap.yaml index 0787c940a..3d119cd76 100644 --- a/examples/btp-operator-configmap.yaml +++ b/examples/btp-operator-configmap.yaml @@ -3,6 +3,8 @@ kind: ConfigMap metadata: name: sap-btp-manager namespace: kyma-system + labels: + app.kubernetes.io/managed-by: kcp-kyma-environment-broker data: ChartPath: ./module-chart/chart ChartNamespace: kyma-system diff --git a/main.go b/main.go index 606269f07..da2048f4a 100644 --- a/main.go +++ b/main.go @@ -18,10 +18,8 @@ package main import ( "flag" - "os" - btpmanagermetrics "github.com/kyma-project/btp-manager/internal/metrics" - + "os" //test // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -86,22 +84,12 @@ func main() { restCfg := ctrl.GetConfigOrDie() mgr, err := ctrl.NewManager(restCfg, ctrl.Options{ Scheme: scheme, - MetricsBindAddress: metricsAddr, - Port: 9443, - HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "ec023d38.kyma-project.io", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, + MetricsBindAddress: metricsAddr, + HealthProbeBindAddress: probeAddr, + Port: 9443, + NewCache: controllers.CacheCreator, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/scripts/testing/create-secrets-configmaps.sh b/scripts/testing/create-secrets-configmaps.sh new file mode 100755 index 000000000..19e850462 --- /dev/null +++ b/scripts/testing/create-secrets-configmaps.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Stress testing in regard to memory consumption - could cause OOM (but should not). +# Creates btp-operator and numerous Service Instances and Service Bindings in the current context. +# +# The script has the following arguments: +# - number of Config Maps and Secrets (the provided number is multiplied by 10) +# Example (creates 1000 CMs and Secrets): +# ./stress-mem.sh 100 + +N=${1-100} +YAML_DIR=./scripts/testing/yaml + +echo -e "\n---Creating ${N} config maps and secrets" + +for ((i=1; i <= N ; i++)) + do + CM_NAME=cm-$i + SECRET_NAME=secret-$i + + export CM_NAME + export SECRET_NAME + + envsubst <${YAML_DIR}/stress-cm.yaml | kubectl apply -f - >/dev/null + envsubst <${YAML_DIR}/stress-secret.yaml | kubectl apply -f - >/dev/null +done \ No newline at end of file diff --git a/scripts/testing/yaml/e2e-test-configmap.yaml b/scripts/testing/yaml/e2e-test-configmap.yaml index 99df3892f..f3e986a55 100644 --- a/scripts/testing/yaml/e2e-test-configmap.yaml +++ b/scripts/testing/yaml/e2e-test-configmap.yaml @@ -3,5 +3,7 @@ kind: ConfigMap metadata: name: sap-btp-manager namespace: kyma-system + labels: + app.kubernetes.io/managed-by: btp-manager data: HardDeleteTimeout: "10s" diff --git a/scripts/testing/yaml/e2e-test-secret.yaml b/scripts/testing/yaml/e2e-test-secret.yaml index a5b91a939..effc3a913 100644 --- a/scripts/testing/yaml/e2e-test-secret.yaml +++ b/scripts/testing/yaml/e2e-test-secret.yaml @@ -4,6 +4,8 @@ type: Opaque metadata: name: sap-btp-manager namespace: kyma-system + labels: + app.kubernetes.io/managed-by: kcp-kyma-environment-broker data: clientid: ${SM_CLIENT_ID} clientsecret: ${SM_CLIENT_SECRET} diff --git a/scripts/testing/yaml/stress-cm.yaml b/scripts/testing/yaml/stress-cm.yaml new file mode 100644 index 000000000..9c8406bab --- /dev/null +++ b/scripts/testing/yaml/stress-cm.yaml @@ -0,0 +1,79 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-001 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-002 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-003 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-004 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-005 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-006 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-007 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-008 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-009 + namespace: kyma-system +data: + testData: "abcd" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CM_NAME}-000 + namespace: kyma-system +data: + testData: "abcd" \ No newline at end of file diff --git a/scripts/testing/yaml/stress-secret.yaml b/scripts/testing/yaml/stress-secret.yaml new file mode 100644 index 000000000..a6a64263f --- /dev/null +++ b/scripts/testing/yaml/stress-secret.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-001 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-002 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-003 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-004 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-005 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-006 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-007 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-008 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-009 + namespace: kyma-system +data: + testData: test +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: ${SECRET_NAME}-000 + namespace: kyma-system +data: + testData: test