feat: add autoscaler admission webhook

charts/nebula-operator/templates/_helpers.tpl

@@ -97,6 +97,20 @@ Admission webhook name of the chart.
 {{ include "nebula-operator.name" . }}-webhook
 {{- end }}
 
+{{/*
+Controller Manager Admission webhook name.
+*/}}
+{{- define "controller-manager-admission-webhook.name" -}}
+controller-manager-{{ include "nebula-operator.name" . }}-webhook
+{{- end }}
+
+{{/*
+Autoscaler Admission webhook name.
+*/}}
+{{- define "autoscaler-admission-webhook.name" -}}
+autoscaler-{{ include "nebula-operator.name" . }}-webhook
+{{- end }}
+
 {{/*
 Admission webhook selector labels
 */}}

charts/nebula-operator/templates/admission-webhook-certificate.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) .Values.admissionWebhook.useCertManager }}
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
@@ -9,8 +9,10 @@ metadata:
   {{- include "admission-webhook.labels" . | nindent 4 }}
 spec:
   dnsNames:
-    - {{ template "admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
-    - {{ template "admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
+    - {{ template "controller-manager-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
+    - {{ template "controller-manager-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
+    - {{ template "autoscaler-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
+    - {{ template "autoscaler-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
   issuerRef:
     kind: Issuer
     name: {{ template "admission-webhook.name" . }}-issuer

charts/nebula-operator/templates/admission-webhook-registration.yaml

@@ -1,20 +1,23 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
+  {{- if .Values.admissionWebhook.useCertManager }}
   annotations:
     cert-manager.io/inject-ca-from: {{ template "nebula-operator.namespace" . }}/{{ template "admission-webhook.name" . }}-cert
+  {{- end }}
   name: {{ template "admission-webhook.name" . }}-validating
   labels:
   {{- include "admission-webhook.labels" . | nindent 4 }}
 webhooks:
+  {{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
   - name: nebulaclustervalidating.nebula-graph.io
     admissionReviewVersions:
       - v1
     clientConfig:
       service:
-        name: {{ template "admission-webhook.name" . }}-service
+        name: {{ template "controller-manager-admission-webhook.name" . }}-service
         namespace: {{ template "nebula-operator.namespace" . }}
         path: /validate-nebulacluster
     failurePolicy: Fail
@@ -32,5 +35,31 @@ webhooks:
         scope: "*"
     sideEffects: None
     timeoutSeconds: 3
+  {{- end }}
+
+  {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+  - name: nebulaautoscalingvalidating.nebula-graph.io
+    admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: {{ template "autoscaler-admission-webhook.name" . }}-service
+        namespace: {{ template "nebula-operator.namespace" . }}
+        path: /validate-nebulaautoscaler
+    failurePolicy: Fail
+    rules:
+      - apiGroups:
+          - autoscaling.nebula-graph.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - nebulaautoscalers
+        scope: "*"
+    sideEffects: None
+    timeoutSeconds: 3
+  {{- end }}
 {{- end }}
 

charts/nebula-operator/templates/autoscaler-admission-webhook-service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "autoscaler-admission-webhook.name" . }}-service
+  namespace: {{ template "nebula-operator.namespace" . }}
+  labels:
+  {{- include "admission-webhook.labels" . | nindent 4 }}
+spec:
+  ports:
+    - port: 443
+      targetPort: 9448
+  selector:
+  {{- include "admission-webhook.matchLabels" . | nindent 4 }}
+{{- end }}
+

charts/nebula-operator/templates/admission-webhook-service.yaml → charts/nebula-operator/templates/controller-manager-admission-webhook-service.yaml

@@ -1,9 +1,9 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "admission-webhook.name" . }}-service
+  name: {{ template "controller-manager-admission-webhook.name" . }}-service
   namespace: {{ template "nebula-operator.namespace" . }}
   labels:
   {{- include "admission-webhook.labels" . | nindent 4 }}

charts/nebula-operator/templates/controller-manager-deployment.yaml

@@ -1,4 +1,17 @@
 {{- if .Values.controllerManager.create }}
+{{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "admission-webhook.name" . }}-secret
+  namespace: {{ template "nebula-operator.namespace" . }}
+type: kubernetes.io/tls
+data:
+  tls.crt: ""
+  tls.key: ""
+  ca.crt: ""
+{{- end }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -43,8 +56,15 @@ spec:
             - --concurrent-nebulabackup-syncs={{ .Values.concurrentNebulaBackupSyncs }}
             - --leader-elect
             - --leader-elect-resource-namespace={{ template "nebula-operator.namespace" . }}
-            - --enable-admission-webhook={{ .Values.admissionWebhook.create }}
-            - --webhook-secure-port={{ .Values.admissionWebhook.webhookBindPort }}
+            - --enable-admission-webhook={{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
+            - --webhook-secure-port={{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.webhookBindPort }}
+            - --webhook-namespace={{ template "nebula-operator.namespace" . }}
+            - --webhook-server-name={{ template "admission-webhook.name" . }}-validating
+            - --webhook-names={{ template "controller-manager-admission-webhook.name" . }}-service,{{ template "autoscaler-admission-webhook.name" . }}-service
+            - --certificate-validity={{ .Values.admissionWebhook.certValidity }}
+            - --secret-namespace={{ template "nebula-operator.namespace" . }}
+            - --secret-name={{ template "admission-webhook.name" . }}-secret
+            - --kube-domain={{ default "cluster.local" .Values.kubernetesClusterDomain }}
             - --enable-kruise-scheme={{ .Values.enableKruiseScheme }}
             - --v={{ .Values.controllerManager.verbosity }}
           {{- if or .Values.kubernetesClusterDomain .Values.controllerManager.env }}
@@ -55,9 +75,9 @@ spec:
           {{- end }}
           {{- if .Values.controllerManager.env }}{{ toYaml .Values.controllerManager.env | nindent 12 }}{{- end }}
           {{- end }}
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
           ports:
-            - containerPort: {{ .Values.admissionWebhook.webhookBindPort | default 9443 }}
+            - containerPort: {{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.webhookBindPort | default 9443 }}
               name: webhook-server
               protocol: TCP
           {{- end }}
@@ -77,9 +97,9 @@ spec:
             periodSeconds: 10
           securityContext:
             allowPrivilegeEscalation: false
-          {{- if or .Values.controllerManager.extraVolumeMounts .Values.admissionWebhook.create }}
+          {{- if or .Values.controllerManager.extraVolumeMounts .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
           volumeMounts:
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
             - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: cert
               readOnly: true
@@ -100,6 +120,15 @@ spec:
             - --leader-elect
             - --leader-elect-resource-namespace={{ template "nebula-operator.namespace" . }}
             - --v={{ .Values.controllerManager.verbosity }}
+            - --enable-admission-webhook={{ .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+            - --webhook-secure-port={{ .Values.admissionWebhook.autoscalerAdmissionWebhook.webhookBindPort }}
+            - --webhook-namespace={{ template "nebula-operator.namespace" . }}
+            - --webhook-server-name={{ template "admission-webhook.name" . }}-validating
+            - --webhook-names={{ template "controller-manager-admission-webhook.name" . }}-service,{{ template "autoscaler-admission-webhook.name" . }}-service
+            - --certificate-validity={{ .Values.admissionWebhook.certValidity }}
+            - --secret-namespace={{ template "nebula-operator.namespace" . }}
+            - --secret-name={{ template "admission-webhook.name" . }}-secret
+            - --kube-domain={{ default "cluster.local" .Values.kubernetesClusterDomain }}
           {{- if or .Values.kubernetesClusterDomain .Values.controllerManager.env }}
           env:
           {{- if .Values.kubernetesClusterDomain }}
@@ -108,6 +137,12 @@ spec:
           {{- end }}
           {{- if .Values.controllerManager.env }}{{ toYaml .Values.controllerManager.env | nindent 12 }}{{- end }}
           {{- end }}
+          {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+          ports:
+            - containerPort: {{ .Values.admissionWebhook.autoscalerAdmissionWebhook.webhookBindPort | default 9448 }}
+              name: webhook-server
+              protocol: TCP
+          {{- end }}
           resources:
           {{- toYaml .Values.controllerManager.resources | nindent 12 }}
           livenessProbe:
@@ -124,6 +159,12 @@ spec:
             periodSeconds: 10
           securityContext:
             allowPrivilegeEscalation: false
+          {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+          volumeMounts:
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
+              name: cert
+              readOnly: true
+          {{- end }}
     {{- with .Values.controllerManager.sidecarContainers }}
       {{- range $name, $spec :=  $.Values.controllerManager.sidecarContainers }}
         - name: {{ $name }}
@@ -153,9 +194,9 @@ spec:
       tolerations:
       {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if or .Values.controllerManager.extraVolumes .Values.admissionWebhook.create }}
+      {{- if or .Values.controllerManager.extraVolumes (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) }}
       volumes:
-      {{- if .Values.admissionWebhook.create }}
+      {{- if or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create}}
       - name: cert
         secret:
           defaultMode: 420

charts/nebula-operator/templates/controller-manager-rbac.yaml

@@ -37,6 +37,19 @@ rules:
     verbs:
       - create
       - patch
+  {{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+  {{- end}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -387,6 +400,18 @@ rules:
     verbs:
       - get
       - list
+  {{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - validatingwebhookconfigurations
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+  {{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

charts/nebula-operator/values.yaml

@@ -64,9 +64,17 @@ controllerManager:
 #    runAsNonRoot: true
 
 admissionWebhook:
-  create: false
-  # The TCP port the Webhook server binds to. (default 9443)
-  webhookBindPort: 9443
+  contollerManagerAdmissionWebhook:
+    create: false
+    # The TCP port the Webhook server binds to. (default 9443)
+    webhookBindPort: 9443
+  autoscalerAdmissionWebhook:
+    create: true
+    # The TCP port the Webhook server binds to. (default 9448)
+    webhookBindPort: 9448
+  useCertManager: false
+  # Number of days. Only needed if useCertManager is false. (default 1)
+  certValidity: 1
 
 scheduler:
   create: true

cmd/autoscaler/app/autoscaler.go

@@ -19,6 +19,7 @@ package app
 import (
 	"context"
 	"flag"
+	"net/http"
 
 	"github.com/spf13/cobra"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -28,14 +29,18 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	"github.com/vesoft-inc/nebula-operator/apis/autoscaling/scheme"
 	"github.com/vesoft-inc/nebula-operator/apis/autoscaling/v1alpha1"
 	"github.com/vesoft-inc/nebula-operator/cmd/autoscaler/app/options"
+	certrot "github.com/vesoft-inc/nebula-operator/pkg/cert-rotation"
 	"github.com/vesoft-inc/nebula-operator/pkg/controller/autoscaler"
 	klogflag "github.com/vesoft-inc/nebula-operator/pkg/flag/klog"
 	profileflag "github.com/vesoft-inc/nebula-operator/pkg/flag/profile"
 	"github.com/vesoft-inc/nebula-operator/pkg/version"
+	nawebhook "github.com/vesoft-inc/nebula-operator/pkg/webhook/autoscaler"
 )
 
 // NewAutoscalerCommand creates a *cobra.Command object with default parameters
@@ -106,6 +111,17 @@ func Run(ctx context.Context, opts *options.Options) error {
 		},
 	}
 
+	if opts.EnableAdmissionWebhook {
+		ctrlOptions.WebhookServer = webhook.NewServer(webhook.Options{
+			Host:          opts.WebhookOpts.BindAddress,
+			Port:          opts.WebhookOpts.SecurePort,
+			CertDir:       opts.WebhookOpts.CertDir,
+			CertName:      opts.WebhookOpts.CertName,
+			KeyName:       opts.WebhookOpts.KeyName,
+			TLSMinVersion: opts.WebhookOpts.TLSMinVersion,
+		})
+	}
+
 	mgr, err := ctrlruntime.NewManager(cfg, ctrlOptions)
 	if err != nil {
 		klog.Errorf("Failed to build nebula-autoscaler: %v", err)
@@ -126,6 +142,30 @@ func Run(ctx context.Context, opts *options.Options) error {
 		return err
 	}
 
+	if opts.EnableAdmissionWebhook {
+		decoder := admission.NewDecoder(mgr.GetScheme())
+		klog.Info("Registering webhooks to nebula-auto-scaler")
+		hookServer := mgr.GetWebhookServer()
+		hookServer.Register("/validate-nebulaautoscaler",
+			&webhook.Admission{Handler: &nawebhook.ValidatingAdmission{Decoder: decoder}})
+		hookServer.WebhookMux().Handle("/readyz/", http.StripPrefix("/readyz/", &healthz.Handler{}))
+
+		// Start certificate rotation
+		certGenerator := certrot.CertGenerator{
+			LeaderElection:    opts.LeaderElection,
+			WebhookNames:      opts.WebhookOpts.WebhookNames,
+			WebhookServerName: opts.WebhookOpts.WebhookServerName,
+			WebhookNamespace:  opts.WebhookOpts.WebhookNamespace,
+			CertDir:           opts.WebhookOpts.CertDir,
+			CertValidity:      opts.WebhookOpts.CertValidity,
+			SecretName:        opts.WebhookOpts.SecretName,
+			SecretNamespace:   opts.WebhookOpts.SecretNamespace,
+			KubernetesDomain:  opts.WebhookOpts.KubernetesDomain,
+		}
+
+		certGenerator.Run(ctx)
+	}
+
 	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
 		klog.Errorf("failed to add health check endpoint: %v", err)
 		return err