Merge branch 'master' into feat/backup-incremental-size

pingcap · Jul 28, 2023 · 3ab633d · 3ab633d
2 parents 4d7d04a + 87b2874
commit 3ab633d
Show file tree

Hide file tree

Showing 126 changed files with 1,908 additions and 102,772 deletions.
diff --git a/.github/workflows/chaos.yaml b/.github/workflows/chaos.yaml
@@ -53,8 +53,8 @@ jobs:
     - name: Run chaos mesh
       uses: chaos-mesh/chaos-mesh-actions@master
       env:
-        CFG_BASE64: YXBpVmVyc2lvbjogY2hhb3MtbWVzaC5vcmcvdjFhbHBoYTEKa2luZDogUG9kQ2hhb3MKbWV0YWRhdGE6CiAgbmFtZTogcG9kLWZhaWx1cmUKICBuYW1lc3BhY2U6IHBpbmdjYXAKc3BlYzoKICBhY3Rpb246IHBvZC1mYWlsdXJlIAogIG1vZGU6IG9uZSAKICBkdXJhdGlvbjogIjYwcyIKICBzZWxlY3RvcjoKICAgIGxhYmVsU2VsZWN0b3JzOgogICAgICAiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjogImNvbnRyb2xsZXItbWFuYWdlciIKICBzY2hlZHVsZXI6CiAgICBjcm9uOiAiQGV2ZXJ5IDVtIgo=
-        CHAOS_MESH_VERSION: v1.0.0
+        CFG_BASE64: YXBpVmVyc2lvbjogY2hhb3MtbWVzaC5vcmcvdjFhbHBoYTEKa2luZDogU2NoZWR1bGUKbWV0YWRhdGE6CiAgbmFtZTogcG9kLWZhaWx1cmUKICBuYW1lc3BhY2U6IHBpbmdjYXAKc3BlYzoKICBzY2hlZHVsZTogJ0BldmVyeSA1bScKICB0eXBlOiBQb2RDaGFvcwogIGhpc3RvcnlMaW1pdDogNQogIGNvbmN1cnJlbmN5UG9saWN5OiBGb3JiaWQKICBwb2RDaGFvczoKICAgIGFjdGlvbjogcG9kLWZhaWx1cmUKICAgIG1vZGU6IG9uZQogICAgZHVyYXRpb246IDYwcwogICAgc2VsZWN0b3I6CiAgICAgIGxhYmVsU2VsZWN0b3JzOgogICAgICAgIGFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudDogY29udHJvbGxlci1tYW5hZ2VyCg==
+        CHAOS_MESH_VERSION: v2.1.0
     - name: Run e2e
       run: |
         SKIP_UP=y ./hack/e2e.sh -- --ginkgo.focus='Restarter'

diff --git a/charts/br-federation/templates/controller-manager-deployment.yaml b/charts/br-federation/templates/controller-manager-deployment.yaml
@@ -78,6 +78,9 @@ spec:
          {{- if .Values.brFederationManager.kubeClientBurst }}
           - -kube-client-burst={{ .Values.brFederationManager.kubeClientBurst }}
          {{- end }}
+         {{- if .Values.brFederationManager.federationKubeconfigSecretKey }}
+          - -federation-kubeconfig-path=/etc/br-federation/federation-kubeconfig/{{ .Values.brFederationManager.federationKubeconfigSecretKey }}
+         {{- end }}
         env:
           - name: NAMESPACE
             valueFrom:

diff --git a/charts/br-federation/values.yaml b/charts/br-federation/values.yaml
@@ -20,8 +20,11 @@ brFederationManager:
   serviceAccount: br-federation-manager
 
   # Secret name of the kubeconfig for the federation Kubernetes clusters
-  # The data item key is the cluster name, and the data item value is the base64 encoded kubeconfig
   federationKubeconfigSecret: br-federation-kubeconfig
+  # which data item is the kubeconfig file, and the data item value is the base64 encoded kubeconfig
+  # if you have multiple kubernetes clusters, you should merge them in one kubeconfig
+  # we use the context name in the kubeconfig as the k8sClusterName in volume backup/restore CR
+  federationKubeconfigSecretKey: kubeconfig
 
   logLevel: 2
   replicas: 1

diff --git a/charts/tidb-operator/templates/admission/admission-webhook-registration.yaml b/charts/tidb-operator/templates/admission/admission-webhook-registration.yaml
@@ -41,7 +41,7 @@ webhooks:
     objectSelector:
       matchLabels:
         "app.kubernetes.io/managed-by": "tidb-operator"
-    admissionReviewVersions: ["v1beta1"]
+    admissionReviewVersions: ["v1"]
     failurePolicy: {{ .Values.admissionWebhook.failurePolicy.validation | default "Fail" }}
     sideEffects: None
     clientConfig:
@@ -79,7 +79,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 webhooks:
   - name: validating.admission.tidb.pingcap.com
-    admissionReviewVersions: ["v1beta1"]
+    admissionReviewVersions: ["v1"]
     failurePolicy: {{ .Values.admissionWebhook.failurePolicy.validation | default "Fail" }}
     sideEffects: None
     clientConfig:
@@ -113,7 +113,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 webhooks:
   - name: defaulting.admission.tidb.pingcap.com
-    admissionReviewVersions: ["v1beta1"]
+    admissionReviewVersions: ["v1"]
     failurePolicy: {{ .Values.admissionWebhook.failurePolicy.mutation | default "Fail" }}
     sideEffects: None
     clientConfig:

diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml
@@ -189,8 +189,7 @@ scheduler:
 # AdvancedStatefulSet controller.
 # Note that AdvancedStatefulSet CRD must be installed manually via the following
 # command:
-#   kubectl apply -f manifests/advanced-statefulset-crd.v1beta1.yaml # k8s version < 1.16.0
-#   kubectl apply -f manifests/advanced-statefulset-crd.v1.yaml      # k8s version >= 1.16.0
+#   kubectl apply -f manifests/advanced-statefulset-crd.v1.yaml
 advancedStatefulset:
   create: false
   image: pingcap/advanced-statefulset:v0.4.0

diff --git a/ci/pull_e2e_kind.groovy b/ci/pull_e2e_kind.groovy
@@ -150,8 +150,8 @@ e2ePodResources = [
         storage: "250Gi"
     ],
     limits: [
-        cpu: "16",
-        memory: "32Gi",
+        cpu: "24",
+        memory: "48Gi",
         storage: "250Gi"
     ],
 ]

diff --git a/cmd/backup-manager/app/clean/clean.go b/cmd/backup-manager/app/clean/clean.go
@@ -84,11 +84,12 @@ func (bo *Options) deleteSnapshotsAndBackupMeta(ctx context.Context, backup *v1a
 	}()
 
 	contents, err := os.ReadFile(metaFile)
+
 	if errors.Is(err, os.ErrNotExist) {
-		klog.Warningf("read metadata file %s failed, err: %s, a mannual check or delete aciton require.", metaFile, err)
-		return nil
+		klog.Errorf("read metadata file %s failed, err: %s, a manual check or delete action required.", metaFile, err)
+		return err
 	} else if err != nil { // will retry it
-		klog.Warningf("read metadata file %s failed, err: %s", metaFile, err)
+		klog.Errorf("read metadata file %s failed, err: %s", metaFile, err)
 		return err
 	}
 

diff --git a/cmd/br-federation-manager/main.go b/cmd/br-federation-manager/main.go
@@ -20,7 +20,6 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"reflect"
 	"syscall"
 
@@ -219,18 +218,16 @@ func main() {
 }
 
 func initFederationKubeClients(cliCfg *controller.BrFedCLIConfig) (map[string]fedversioned.Interface, error) {
-	files, err := os.ReadDir(cliCfg.FederationKubeConfigPath)
+	kubeConfig, err := clientcmd.LoadFromFile(cliCfg.FederationKubeConfigPath)
 	if err != nil {
 		return nil, err
 	}
 
 	clients := make(map[string]fedversioned.Interface)
-	for _, f := range files {
-		if f.IsDir() || f.Name() == "..data" {
-			continue
-		}
-
-		cfg, err := clientcmd.BuildConfigFromFlags("", filepath.Join(cliCfg.FederationKubeConfigPath, f.Name()))
+	for contextName := range kubeConfig.Contexts {
+		cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+			&clientcmd.ClientConfigLoadingRules{ExplicitPath: cliCfg.FederationKubeConfigPath},
+			&clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig()
 		if err != nil {
 			return nil, err // return error if any kube client init failed
 		}
@@ -243,7 +240,7 @@ func initFederationKubeClients(cliCfg *controller.BrFedCLIConfig) (map[string]fe
 		if err != nil {
 			return nil, err
 		}
-		clients[f.Name()] = cli
+		clients[contextName] = cli
 	}
 
 	return clients, nil

diff --git a/cmd/to-crdgen/main.go b/cmd/to-crdgen/main.go
diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md
@@ -5642,7 +5642,7 @@ string
 <td>
 <code>AdditionalPrinterColums</code></br>
 <em>
-[]k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1.CustomResourceColumnDefinition
+[]k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1.CustomResourceColumnDefinition
 </em>
 </td>
 <td>

diff --git a/examples/advanced/tidb-monitor.yaml b/examples/advanced/tidb-monitor.yaml
@@ -274,7 +274,7 @@ spec:
     ## ImagePullPolicy of grafana.
     # imagePullPolicy: IfNotPresent
     ## Grafana version.
-    version: v7.5.11
+    version: 7.5.11
     ## Describes the compute resource requirements and limits of grafana.
     ## Ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
     # requests:

diff --git a/examples/heterogeneous-tls/README.md b/examples/heterogeneous-tls/README.md
@@ -5,9 +5,6 @@ The following steps will create a basic TLS TiDB cluster , then we can create a
 
 ## Install Cert Manager 
 Reference [document](https://cert-manager.io/docs/installation/).
-```bash
-kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.7.1/cert-manager.yaml
-```
 
 ## Install TLS certificate
 The following commands is assumed to be executed in this directory.

diff --git a/examples/monitor-multiple-cluster-tls/cert-manager/ns1/cluster-client-secret.yaml b/examples/monitor-multiple-cluster-tls/cert-manager/ns1/cluster-client-secret.yaml
@@ -7,8 +7,9 @@ spec:
   secretName: ns1-cluster-client-secret
   duration: 8760h # 365d
   renewBefore: 360h # 15d
-  organization:
-    - PingCAP
+  subject:
+    organizations:
+      - PingCAP
   commonName: "TiDB"
   usages:
     - client auth

diff --git a/examples/monitor-multiple-cluster-tls/cert-manager/ns1/pd-cluster-secret.yaml b/examples/monitor-multiple-cluster-tls/cert-manager/ns1/pd-cluster-secret.yaml
@@ -7,8 +7,9 @@ spec:
   secretName: ns1-pd-cluster-secret
   duration: 8760h # 365d
   renewBefore: 360h # 15d
-  organization:
-    - PingCAP
+  subject:
+    organizations:
+      - PingCAP
   commonName: "TiDB"
   usages:
     - server auth

diff --git a/examples/monitor-multiple-cluster-tls/cert-manager/ns1/tidb-client-cert.yaml b/examples/monitor-multiple-cluster-tls/cert-manager/ns1/tidb-client-cert.yaml
@@ -7,8 +7,9 @@ spec:
   secretName: ns1-tidb-client-secret
   duration: 8760h # 365d
   renewBefore: 360h # 15d
-  organization:
-    - PingCAP
+  subject:
+    organizations:
+      - PingCAP
   commonName: "TiDB Client"
   usages:
     - client auth

diff --git a/examples/monitor-multiple-cluster-tls/cert-manager/ns1/tidb-cluster-secret.yaml b/examples/monitor-multiple-cluster-tls/cert-manager/ns1/tidb-cluster-secret.yaml
@@ -7,8 +7,9 @@ spec:
   secretName: ns1-tidb-cluster-secret
   duration: 8760h # 365d
   renewBefore: 360h # 15d
-  organization:
-    - PingCAP
+  subject:
+    organizations:
+      - PingCAP
   commonName: "TiDB"
   usages:
     - server auth

diff --git a/examples/monitor-multiple-cluster-tls/cert-manager/ns1/tidb-server-secret.yaml b/examples/monitor-multiple-cluster-tls/cert-manager/ns1/tidb-server-secret.yaml
@@ -7,8 +7,9 @@ spec:
   secretName: ns1-tidb-server-secret
   duration: 8760h # 365d
   renewBefore: 360h # 15d
-  organization:
-    - PingCAP
+  subject:
+    organizations:
+      - PingCAP
   commonName: "TiDB Server"
   usages:
     - server auth

diff --git a/examples/monitor-multiple-cluster-tls/cert-manager/ns1/tikv-cluster-secret.yaml b/examples/monitor-multiple-cluster-tls/cert-manager/ns1/tikv-cluster-secret.yaml
@@ -7,8 +7,9 @@ spec:
   secretName: ns1-tikv-cluster-secret
   duration: 8760h # 365d
   renewBefore: 360h # 15d
-  organization:
-    - PingCAP
+  subject:
+    organizations:
+      - PingCAP
   commonName: "TiDB"
   usages:
     - server auth

diff --git a/examples/selfsigned-tls/selfsigned-ca.yaml b/examples/selfsigned-tls/selfsigned-ca.yaml
@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: selfsigned-ca-cert

diff --git a/examples/selfsigned-tls/selfsigned-cert-issuer.yaml b/examples/selfsigned-tls/selfsigned-cert-issuer.yaml
@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: selfsigned-cert-issuer

diff --git a/examples/selfsigned-tls/selfsigned-issuer.yaml b/examples/selfsigned-tls/selfsigned-issuer.yaml
@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: selfsigned-issuer

diff --git a/examples/selfsigned-tls/tidb-client-cert.yaml b/examples/selfsigned-tls/tidb-client-cert.yaml
@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: tidb-client-cert
@@ -7,8 +7,8 @@ spec:
   subject:
     organizationalUnits:
       - "TiDB Operator"
-  organization:
-    - "PingCAP"
+    organizations:
+      - PingCAP
   duration: "8760h" # 364 days
   # If you want verify server cert Common Name (e.g. --ssl-verify-server-cert
   # flag in MySQL CLI), you must configure the HostName you used to connect the

diff --git a/examples/selfsigned-tls/tidb-server-cert.yaml b/examples/selfsigned-tls/tidb-server-cert.yaml
@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: tidb-server-cert
@@ -7,8 +7,8 @@ spec:
   subject:
     organizationalUnits:
     - "TiDB Operator"
-  organization:
-  - "PingCAP"
+    organizations:
+      - PingCAP
   duration: "8760h" # 364 days
   # If you want verify server cert Common Name (e.g. --ssl-verify-server-cert
   # flag in MySQL CLI), you must configure the HostName you used to connect the