Add support to customize spare replicas during VolumeReplace (#5666)

Co-authored-by: Xuecheng Zhang <[email protected]>
pingcap · Jun 28, 2024 · 32166b6 · 32166b6
1 parent dfd5ac2
commit 32166b6
Show file tree

Hide file tree

Showing 9 changed files with 97 additions and 3 deletions.
diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md
@@ -12331,6 +12331,21 @@ string
 <p>Mode is the mode of PD cluster</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>spareVolReplaceReplicas</code></br>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The default number of spare replicas to scale up when using VolumeReplace feature.
+In multi-az deployments with topology spread constraints you may need to set this to number of zones to avoid
+zone skew after volume replace (total replicas always whole multiples of zones).
+Optional: Defaults to 1</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="pdstatus">PDStatus</h3>
@@ -22653,6 +22668,21 @@ ScalePolicy
 <p>ScalePolicy is the scale configuration for TiKV</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>spareVolReplaceReplicas</code></br>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The default number of spare replicas to scale up when using VolumeReplace feature.
+In multi-az deployments with topology spread constraints you may need to set this to number of zones to avoid
+zone skew after volume replace (total replicas always whole multiples of zones).
+Optional: Defaults to 1</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="tikvstatus">TiKVStatus</h3>

diff --git a/manifests/crd.yaml b/manifests/crd.yaml
@@ -24779,6 +24779,10 @@ spec:
                     type: object
                   serviceAccount:
                     type: string
+                  spareVolReplaceReplicas:
+                    format: int32
+                    minimum: 0
+                    type: integer
                   startTimeout:
                     default: 30
                     type: integer
@@ -41162,6 +41166,10 @@ spec:
                     type: boolean
                   serviceAccount:
                     type: string
+                  spareVolReplaceReplicas:
+                    format: int32
+                    minimum: 0
+                    type: integer
                   statefulSetUpdateStrategy:
                     type: string
                   storageClassName:

diff --git a/manifests/crd/v1/pingcap.com_tidbclusters.yaml b/manifests/crd/v1/pingcap.com_tidbclusters.yaml
@@ -5725,6 +5725,10 @@ spec:
                     type: object
                   serviceAccount:
                     type: string
+                  spareVolReplaceReplicas:
+                    format: int32
+                    minimum: 0
+                    type: integer
                   startTimeout:
                     default: 30
                     type: integer
@@ -22108,6 +22112,10 @@ spec:
                     type: boolean
                   serviceAccount:
                     type: string
+                  spareVolReplaceReplicas:
+                    format: int32
+                    minimum: 0
+                    type: integer
                   statefulSetUpdateStrategy:
                     type: string
                   storageClassName:

diff --git a/pkg/apis/pingcap/v1alpha1/defaulting/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/defaulting/tidbcluster.go
@@ -115,6 +115,9 @@ func setTikvSpecDefault(tc *v1alpha1.TidbCluster) {
 	if tc.Spec.TiKV.MaxFailoverCount == nil {
 		tc.Spec.TiKV.MaxFailoverCount = pointer.Int32Ptr(3)
 	}
+	if tc.Spec.TiKV.SpareVolReplaceReplicas == nil {
+		tc.Spec.TiKV.SpareVolReplaceReplicas = pointer.Int32Ptr(1)
+	}
 }
 
 func setPdSpecDefault(tc *v1alpha1.TidbCluster) {
@@ -126,6 +129,9 @@ func setPdSpecDefault(tc *v1alpha1.TidbCluster) {
 	if tc.Spec.PD.MaxFailoverCount == nil {
 		tc.Spec.PD.MaxFailoverCount = pointer.Int32Ptr(3)
 	}
+	if tc.Spec.PD.SpareVolReplaceReplicas == nil {
+		tc.Spec.PD.SpareVolReplaceReplicas = pointer.Int32Ptr(1)
+	}
 }
 
 func setPDMSSpecDefault(tc *v1alpha1.TidbCluster) {

diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go
diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/tidbcluster.go
@@ -631,7 +631,7 @@ func (tc *TidbCluster) PDStsDesiredReplicas() int32 {
 	}
 	var spareReplaceReplicas int32 = 0
 	if tc.Status.PD.VolReplaceInProgress {
-		spareReplaceReplicas = 1
+		spareReplaceReplicas = *tc.Spec.PD.SpareVolReplaceReplicas
 	}
 	return tc.Spec.PD.Replicas + tc.GetPDDeletedFailureReplicas() + spareReplaceReplicas
 }
@@ -709,7 +709,7 @@ func (tc *TidbCluster) TiKVStsDesiredReplicas() int32 {
 	}
 	var spareReplaceReplicas int32 = 0
 	if tc.Status.TiKV.VolReplaceInProgress {
-		spareReplaceReplicas = 1
+		spareReplaceReplicas = *tc.Spec.TiKV.SpareVolReplaceReplicas
 	}
 	return tc.Spec.TiKV.Replicas + int32(len(tc.Status.TiKV.FailureStores)) + spareReplaceReplicas
 }

diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go
@@ -576,6 +576,14 @@ type PDSpec struct {
 	// +optional
 	// +kubebuilder:validation:Enum:="";"ms"
 	Mode string `json:"mode,omitempty"`
+
+	// The default number of spare replicas to scale up when using VolumeReplace feature.
+	// In multi-az deployments with topology spread constraints you may need to set this to number of zones to avoid
+	// zone skew after volume replace (total replicas always whole multiples of zones).
+	// Optional: Defaults to 1
+	// +kubebuilder:validation:Minimum=0
+	// +optional
+	SpareVolReplaceReplicas *int32 `json:"spareVolReplaceReplicas,omitempty"`
 }
 
 // +k8s:openapi-gen=true
@@ -758,6 +766,14 @@ type TiKVSpec struct {
 	// ScalePolicy is the scale configuration for TiKV
 	// +optional
 	ScalePolicy ScalePolicy `json:"scalePolicy,omitempty"`
+
+	// The default number of spare replicas to scale up when using VolumeReplace feature.
+	// In multi-az deployments with topology spread constraints you may need to set this to number of zones to avoid
+	// zone skew after volume replace (total replicas always whole multiples of zones).
+	// Optional: Defaults to 1
+	// +kubebuilder:validation:Minimum=0
+	// +optional
+	SpareVolReplaceReplicas *int32 `json:"spareVolReplaceReplicas,omitempty"`
 }
 
 // TiFlashSpec contains details of TiFlash members

diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/manager/member/tikv_member_manager_test.go b/pkg/manager/member/tikv_member_manager_test.go
@@ -413,6 +413,7 @@ func TestTiKVMemberManagerSyncUpdate(t *testing.T) {
 			modify: func(tc *v1alpha1.TidbCluster) {
 				// Random test change to affect pod spec template.
 				tc.Spec.TiKV.ServiceAccount = "test_new_account"
+				tc.Spec.TiKV.SpareVolReplaceReplicas = pointer.Int32(1)
 				tc.Status.TiKV.VolReplaceInProgress = true
 			},
 			pdStores:                     &pdapi.StoresInfo{Count: 0, Stores: []*pdapi.StoreInfo{}},
@@ -2149,7 +2150,8 @@ func TestGetNewTiKVSetForTidbCluster(t *testing.T) {
 				},
 				Spec: v1alpha1.TidbClusterSpec{
 					TiKV: &v1alpha1.TiKVSpec{
-						Replicas: 3,
+						Replicas:                3,
+						SpareVolReplaceReplicas: pointer.Int32(1),
 					},
 					PD:   &v1alpha1.PDSpec{},
 					TiDB: &v1alpha1.TiDBSpec{},