From c0ad8d66809e8dc530859902e779c1e85d2fee97 Mon Sep 17 00:00:00 2001
From: Xuecheng Zhang <csuzhangxc@gmail.com>
Date: Fri, 30 Jun 2023 09:00:23 +0800
Subject: [PATCH] opt: do not evict leader when changing PVC size (#5101)

---
 pkg/manager/volumes/pvc_modifier.go | 55 ++++++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 8 deletions(-)

diff --git a/pkg/manager/volumes/pvc_modifier.go b/pkg/manager/volumes/pvc_modifier.go
index 68f57a8a05..f674462038 100644
--- a/pkg/manager/volumes/pvc_modifier.go
+++ b/pkg/manager/volumes/pvc_modifier.go
@@ -284,15 +284,21 @@ func (p *pvcModifier) tryToModifyPVC(ctx *componentVolumeContext) error {
 			// try to evict leader if need to modify
 			isEvicted := isLeaderEvictedOrTimeout(ctx.tc, pod)
 			if !isEvicted {
-				if ensureTiKVLeaderEvictionCondition(ctx.tc, metav1.ConditionTrue) {
-					// return to sync tc
-					return fmt.Errorf("try to evict leader for tidbcluster %s/%s", ctx.tc.Namespace, ctx.tc.Name)
+				// do not evict leader when resizing PVC (increasing size)
+				// as if the storage size is not enough, the leader eviction will be blocked (never finished)
+				if !skipEvictLeaderForSizeModify(actual) {
+					if ensureTiKVLeaderEvictionCondition(ctx.tc, metav1.ConditionTrue) {
+						// return to sync tc
+						return fmt.Errorf("try to evict leader for tidbcluster %s/%s", ctx.tc.Namespace, ctx.tc.Name)
+					}
+					if err := p.evictLeader(ctx.tc, pod); err != nil {
+						return err
+					}
+
+					return fmt.Errorf("wait for leader eviction of %s/%s completed", pod.Namespace, pod.Name)
+				} else {
+					klog.Infof("skip evicting leader for %s/%s as the storage size is changing", pod.Namespace, pod.Name)
 				}
-				if err := p.evictLeader(ctx.tc, pod); err != nil {
-					return err
-				}
-
-				return fmt.Errorf("wait for leader eviction of %s/%s completed", pod.Namespace, pod.Name)
 			}
 		}
 
@@ -315,6 +321,39 @@ func (p *pvcModifier) tryToModifyPVC(ctx *componentVolumeContext) error {
 	return nil
 }
 
+// skip evict leader if the storage size should be modified or is in modifying phase
+func skipEvictLeaderForSizeModify(actual []ActualVolume) bool {
+	for _, vol := range actual {
+		if vol.PVC == nil || vol.Desired == nil {
+			continue
+		}
+
+		annoStatusSize, ok := vol.PVC.Annotations[annoKeyPVCStatusStorageSize]
+		if ok {
+			// modified by the PVC Modifier before (with status size annotation)
+			if annoStatusSize == vol.Desired.Size.String() {
+				continue // already up to date, no need to modify size
+			}
+			return true // need to modify size
+		}
+
+		// not modified by the PVC modifier before (without status size annotation)
+		quantity := vol.GetStorageSize()
+		statusSize := quantity.String()
+		if statusSize == vol.Desired.Size.String() {
+			// special case: skip evict leader (again) as the PVC is in modfiying phase (and the status size annotation is not set yet)
+			if vol.Phase == VolumePhaseModifying {
+				return true
+			}
+			// already modified, in fact, the status size annotation should already be set
+			continue
+		}
+		// need to modify size
+		return true
+	}
+	return false
+}
+
 func ensureTiKVLeaderEvictionCondition(tc *v1alpha1.TidbCluster, status metav1.ConditionStatus) bool {
 	if meta.IsStatusConditionPresentAndEqual(tc.Status.TiKV.Conditions, v1alpha1.ConditionTypeLeaderEvicting, status) {
 		return false