fed: setup the br-federation-manager (#4996)

Makefile

@@ -36,8 +36,9 @@ default: build
 docker-push: docker
 	docker push "${DOCKER_REPO}/tidb-operator:${IMAGE_TAG}"
 	docker push "${DOCKER_REPO}/tidb-backup-manager:${IMAGE_TAG}"
+	docker push "${DOCKER_REPO}/br-federation-manager:${IMAGE_TAG}"
 
-docker: operator-docker backup-docker
+docker: operator-docker backup-docker br-federation-docker
 
 ifeq ($(NO_BUILD),y)
 operator-docker:
@@ -51,7 +52,7 @@ else
 	docker build --tag "${DOCKER_REPO}/tidb-operator:${IMAGE_TAG}" --build-arg=TARGETARCH=$(GOARCH) images/tidb-operator
 endif
 
-build: controller-manager scheduler discovery admission-webhook backup-manager
+build: controller-manager scheduler discovery admission-webhook backup-manager br-federation-manager
 
 controller-manager:
 ifeq ($(E2E),y)
@@ -88,6 +89,13 @@ else
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o images/tidb-backup-manager/bin/$(GOARCH)/tidb-backup-manager cmd/backup-manager/main.go
 endif
 
+br-federation-manager:
+ifeq ($(E2E),y)
+	$(GO_TEST) -ldflags '$(LDFLAGS)' -c -o images/br-federation-manager/bin/br-federation-manager ./cmd/br-federation-manager
+else
+	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o images/br-federation-manager/bin/$(GOARCH)/br-federation-manager ./cmd/br-federation-manager
+endif
+
 ifeq ($(NO_BUILD),y)
 backup-docker:
 	@echo "NO_BUILD=y, skip build for $@"
@@ -100,6 +108,18 @@ else
 	docker build --tag "${DOCKER_REPO}/tidb-backup-manager:${IMAGE_TAG}" --build-arg=TARGETARCH=$(GOARCH) images/tidb-backup-manager
 endif
 
+ifeq ($(NO_BUILD),y)
+br-federation-docker:
+	@echo "NO_BUILD=y, skip build for $@"
+else
+br-federation-docker: br-federation-manager
+endif
+ifeq ($(E2E),y)
+	docker build --tag "${DOCKER_REPO}/br-federation-manager:${IMAGE_TAG}" -f images/br-federation-manager/Dockerfile.e2e images/br-federation-manager
+else
+	docker build --tag "${DOCKER_REPO}/br-federation-manager:${IMAGE_TAG}" --build-arg=TARGETARCH=$(GOARCH) images/br-federation-manager
+endif
+
 e2e-docker-push: e2e-docker
 	docker push "${DOCKER_REPO}/tidb-operator-e2e:${IMAGE_TAG}"
 

charts/br-federation/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

charts/br-federation/Chart.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+description: br-federation Helm chart for Kubernetes
+name: br-federation
+version: v1-canary
+appVersion: v1-canary
+home: https://github.com/pingcap/tidb-operator
+sources:
+  - https://github.com/pingcap/tidb-operator
+keywords:
+  - operator
+  - newsql
+  - htap
+  - database
+  - mysql
+  - raft
+maintainers:
+- name: csuzhangxc
+  email: [email protected]
+- name: WangLe1321
+  email: [email protected]

charts/br-federation/templates/NOTES.txt

@@ -0,0 +1,3 @@
+Make sure br-federation components are running:
+
+    kubectl get pods --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }}

charts/br-federation/templates/_helpers.tpl

@@ -0,0 +1,24 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "chart.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "br-federation.fullname" -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "helm-toolkit.utils.template" -}}
+{{- $name := index . 0 -}}
+{{- $context := index . 1 -}}
+{{- $last := base $context.Template.Name }}
+{{- $wtf := $context.Template.Name | replace $last $name -}}
+{{ include $wtf $context }}
+{{- end }}

charts/br-federation/templates/controller-manager-deployment.yaml

@@ -0,0 +1,125 @@
+{{- if (hasKey .Values.brFederationManager "create" | ternary .Values.brFederationManager.create true) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  {{- if eq .Values.appendReleaseSuffix true}}
+  name: br-federation-manager-{{.Release.Name }}
+  {{- else }}
+  name: br-federation-manager
+  {{- end }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: br-federation-manager
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  replicas: {{ .Values.brFederationManager.replicas }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "chart.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: br-federation-manager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "chart.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: br-federation-manager
+{{- if .Values.brFederationManager.podAnnotations }}
+      annotations:
+{{ toYaml .Values.brFederationManager.podAnnotations | indent 8 }}
+{{ end }}
+    spec:
+    {{- if .Values.brFederationManager.serviceAccount }}
+      {{- if eq .Values.appendReleaseSuffix true}}
+      serviceAccount: {{ .Values.brFederationManager.serviceAccount }}-{{ .Release.Name }}
+      {{- else }}
+      serviceAccount: {{ .Values.brFederationManager.serviceAccount }}
+      {{- end }}
+    {{- end }}
+    {{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+  {{ toYaml .Values.imagePullSecrets | indent 6 }}
+    {{- end }}
+      containers:
+      - name: br-federation-manager
+        image: {{ .Values.image }}
+        imagePullPolicy: {{ .Values.imagePullPolicy | default "IfNotPresent" }}
+        {{- if .Values.brFederationManager.resources }}
+        resources:
+{{ toYaml .Values.brFederationManager.resources | indent 12 }}
+        {{- end }}
+        livenessProbe:
+          tcpSocket:
+            port: 6060
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          failureThreshold: 10
+        command:
+          - /usr/local/bin/br-federation-manager
+          - -v={{ .Values.brFederationManager.logLevel }}
+          {{- if .Values.brFederationManager.workers }}
+          - -workers={{ .Values.brFederationManager.workers | default 5 }}
+          {{- end }}
+         {{- if .Values.brFederationManager.leaderLeaseDuration }}
+          - -leader-lease-duration={{ .Values.brFederationManager.leaderLeaseDuration }}
+         {{- end }}
+         {{- if .Values.brFederationManager.leaderRenewDeadline }}
+          - -leader-renew-deadline={{ .Values.brFederationManager.leaderRenewDeadline }}
+         {{- end }}
+         {{- if .Values.brFederationManager.leaderRetryPeriod }}
+          - -leader-retry-period={{ .Values.brFederationManager.leaderRetryPeriod }}
+         {{- end }}
+         {{- if .Values.brFederationManager.kubeClientQPS }}
+          - -kube-client-qps={{ .Values.brFederationManager.kubeClientQPS }}
+         {{- end }}
+         {{- if .Values.brFederationManager.kubeClientBurst }}
+          - -kube-client-burst={{ .Values.brFederationManager.kubeClientBurst }}
+         {{- end }}
+        env:
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: TZ
+            value: {{ .Values.timezone | default "UTC" }}
+          {{- if eq .Values.appendReleaseSuffix true}}
+          - name: HELM_RELEASE
+            value: {{ .Release.Name }}
+          {{- end }}
+          {{- with .Values.brFederationManager.env }}
+{{ toYaml . | indent 10 }}
+          {{- end }}
+        volumeMounts:
+        - name: federation-kubeconfig
+          mountPath: /etc/br-federation/federation-kubeconfig
+          readOnly: true
+      volumes:
+      - name: federation-kubeconfig
+        secret:
+          secretName: {{ .Values.brFederationManager.federationKubeconfigSecret }}
+      {{- with .Values.brFederationManager.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+
+      {{- with .Values.brFederationManager.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+
+      {{- with .Values.brFederationManager.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+
+      {{- if .Values.brFederationManager.priorityClassName }}
+      priorityClassName: {{ .Values.brFederationManager.priorityClassName }}
+      {{- end }}
+    {{- with .Values.brFederationManager.securityContext }}
+      securityContext:
+{{ toYaml . | indent 8 }}
+    {{- end}}
+{{- end }}

charts/br-federation/templates/controller-manager-rbac.yaml

@@ -0,0 +1,58 @@
+{{- if and .Values.rbac.create (hasKey .Values.brFederationManager "create" | ternary .Values.brFederationManager.create true) }}
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  {{- if eq .Values.appendReleaseSuffix true}}
+  name: {{ .Values.brFederationManager.serviceAccount }}-{{ .Release.Name }}
+  {{- else }}
+  name: {{ .Values.brFederationManager.serviceAccount }}
+  {{- end }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: br-federation-manager
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Release.Name }}:br-federation-manager
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: br-federation-manager
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+rules:
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["create", "get", "list", "watch", "update", "delete"]
+- apiGroups: ["federation.pingcap.com"]
+  resources: ["*"]
+  verbs: ["*"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Release.Name }}:br-federation-manager
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: br-federation-manager
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+subjects:
+- kind: ServiceAccount
+  {{- if eq .Values.appendReleaseSuffix true}}
+  name: {{ .Values.brFederationManager.serviceAccount }}-{{ .Release.Name }}
+  {{- else }}
+  name: {{ .Values.brFederationManager.serviceAccount }}
+  {{- end }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ .Release.Name }}:br-federation-manager
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/br-federation/values.yaml

@@ -0,0 +1,77 @@
+# Default values for br-federation
+
+rbac:
+  create: true
+
+# timezone is the default system timzone
+timezone: UTC
+
+image: pingcap/br-federation-manager:v1.5.0-beta.1
+imagePullPolicy: IfNotPresent
+# imagePullSecrets: []
+
+appendReleaseSuffix: false
+
+brFederationManager:
+  create: true
+  # With rbac.create=false, the user is responsible for creating this account
+  # With rbac.create=true, this service account will be created
+  # Also see rbac.create
+  serviceAccount: br-federation-manager
+
+  # Secret name of the kubeconfig for the federation Kubernetes clusters
+  # The data item key is the cluster name, and the data item value is the base64 encoded kubeconfig
+  federationKubeconfigSecret: br-federation-kubeconfig
+
+  logLevel: 2
+  replicas: 1
+  resources:
+    requests:
+      cpu: 80m
+      memory: 50Mi
+  # REF: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+  # priorityClassName: system-cluster-critical
+
+
+  # REF: https://pkg.go.dev/k8s.io/client-go/tools/leaderelection#LeaderElectionConfig
+  ## leaderLeaseDuration is the duration that non-leader candidates will wait to force acquire leadership
+  # leaderLeaseDuration: 15s
+  ## leaderRenewDeadline is the duration that the acting master will retry refreshing leadership before giving up
+  # leaderRenewDeadline: 10s
+  ## leaderRetryPeriod is the duration the LeaderElector clients should wait between tries of actions
+  # leaderRetryPeriod: 2s
+
+  ## number of workers that are allowed to sync concurrently. default 5
+  # workers: 5
+
+  ## affinity defines pod scheduling rules,affinity default settings is empty.
+  ## please read the affinity document before set your scheduling rule:
+  ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+  affinity: {}
+  ## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels
+  ## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  nodeSelector: {}
+  ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints.
+  ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration
+  tolerations: []
+  ## Env define environments for the controller manager.
+  ## NOTE that the following env names is reserved: 
+  ##  - NAMESPACE
+  ##  - TZ
+  ##  - HELM_RELEASE
+  env: []
+  # - name: AWS_REGION
+  #   value: us-west-2
+  # SecurityContext is security config of this component, it will set template.spec.securityContext
+  # Refer to https://kubernetes.io/docs/tasks/configure-pod-container/security-context
+  securityContext: {}
+  # runAsUser: 1000
+  # runAsGroup: 2000
+  # fsGroup: 2000
+  # PodAnnotations will set template.metadata.annotations
+  # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+  podAnnotations: {}
+  ## KubeClientQPS indicates the maximum QPS to the kubenetes API server from client.
+  # kubeClientQPS: 5
+  ## Maximum burst for throttle.
+  # kubeClientBurst: 10