try to Graceful Drain TiCDC for improper constraint image tag (#5173)

pingcap · Jul 17, 2023 · 4dd9e40 · 4dd9e40
1 parent df734da
commit 4dd9e40
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 16 deletions.
diff --git a/pkg/controller/ticdc_control.go b/pkg/controller/ticdc_control.go
@@ -115,8 +115,9 @@ func (c *defaultTiCDCControl) DrainCapture(tc *v1alpha1.TidbCluster, ordinal int
 		// Let caller retry drain capture.
 		return 0, true, nil
 	}
-	if len(captures) == 1 {
+	if len(captures) == 1 || len(captures) == 0 {
 		// No way to drain a single node TiCDC cluster, ignore.
+		// Can't get capture info, ignore.
 		return 0, false, nil
 	}
 
@@ -186,8 +187,9 @@ func (c *defaultTiCDCControl) ResignOwner(tc *v1alpha1.TidbCluster, ordinal int3
 		// Let caller retry resign owner.
 		return false, nil
 	}
-	if len(captures) == 1 {
+	if len(captures) == 1 || len(captures) == 0 {
 		// No way to resign owner in a single node TiCDC cluster, ignore.
+		// Can't get capture info, ignore.
 		return true, nil
 	}
 
@@ -237,6 +239,10 @@ func (c *defaultTiCDCControl) IsHealthy(tc *v1alpha1.TidbCluster, ordinal int32)
 		// Let caller retry.
 		return false, nil
 	}
+	if len(captures) == 0 {
+		// No way to get capture info, ignore.
+		return true, nil
+	}
 
 	_, owner := getOrdinalAndOwnerCaptureInfo(tc, ordinal, captures)
 	if owner == nil {

diff --git a/pkg/manager/member/ticdc_scaler.go b/pkg/manager/member/ticdc_scaler.go
@@ -235,15 +235,17 @@ func isTiCDCPodSupportGracefulUpgrade(
 	currentVersion := tc.TiCDCVersion()
 	ge, err := cmpver.Compare(podVersion, cmpver.GreaterOrEqual, currentVersion)
 	if err != nil {
-		klog.Errorf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
+		klog.Warningf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, still try to graceful shutdown",
 			action, podVersion, currentVersion, err)
-		return false, nil
+		// if parse version failed, we still try to graftfully shutdown TiCDC,
+		// as we already checked `http.StatusNotFound` in `DrainCapture`, `ResignOwner` and `IsHealthy`
+		return true, nil
 	}
 	le, err := cmpver.Compare(podVersion, cmpver.LessOrEqual, currentVersion)
 	if err != nil {
-		klog.Errorf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
+		klog.Warningf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, still try to graceful shutdown",
 			action, podVersion, currentVersion, err)
-		return false, nil
+		return true, nil
 	}
 	// Reload TiCDC if the current version matches pod version.
 	if ge && le {
@@ -252,7 +254,7 @@ func isTiCDCPodSupportGracefulUpgrade(
 	// We are performing cross version upgrade.
 	lessThan63, err := cmpver.Compare(podVersion, cmpver.Less, ticdcCrossUpgradeVersion)
 	if err != nil {
-		klog.Errorf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
+		klog.Warningf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
 			action, podVersion, ticdcCrossUpgradeVersion, err)
 		return false, nil
 	}
@@ -271,7 +273,7 @@ func isTiCDCPodSupportGracefulUpgrade(
 	podVersionPlus2 := podVer.IncMajor().IncMajor()
 	withInTwoMajorVersion, err := cmpver.Compare(currentVersion, cmpver.Less, podVersionPlus2.String())
 	if err != nil {
-		klog.Errorf("ticdc.%s: fail to compare version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
+		klog.Warningf("ticdc.%s: fail to compare version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
 			action, currentVersion, ticdcCrossUpgradeVersion, err)
 		return false, nil
 	}

diff --git a/pkg/manager/member/ticdc_scaler_test.go b/pkg/manager/member/ticdc_scaler_test.go
@@ -708,7 +708,7 @@ func TestTiCDCIsSupportGracefulUpgrade(t *testing.T) {
 			expectedErr: false,
 		},
 		{
-			caseName: "v6.3.0 -> latest, skip graceful upgrade",
+			caseName: "v6.3.0 -> latest, still try to graceful upgrade",
 			cdcCtl: &controller.FakeTiCDCControl{
 				GetStatusFn: func(tc *v1alpha1.TidbCluster, ordinal int32) (*controller.CaptureStatus, error) {
 					return &controller.CaptureStatus{
@@ -720,7 +720,7 @@ func TestTiCDCIsSupportGracefulUpgrade(t *testing.T) {
 				v := "latest"
 				tc.Spec.TiCDC.Version = &v
 			},
-			expectedOk:  false,
+			expectedOk:  true,
 			expectedErr: false,
 		},
 		{
@@ -734,19 +734,19 @@ func TestTiCDCIsSupportGracefulUpgrade(t *testing.T) {
 			expectedErr: true,
 		},
 		{
-			caseName: "malformed pod version, skip graceful upgrade",
+			caseName: "malformed pod version, still try to graceful upgrade",
 			cdcCtl: &controller.FakeTiCDCControl{
 				GetStatusFn: func(tc *v1alpha1.TidbCluster, ordinal int32) (*controller.CaptureStatus, error) {
 					return &controller.CaptureStatus{
 						Version: "malformed",
 					}, nil
 				},
 			},
-			expectedOk:  false,
+			expectedOk:  true,
 			expectedErr: false,
 		},
 		{
-			caseName: "malformed tc version, skip graceful upgrade",
+			caseName: "malformed tc version, still try to graceful upgrade",
 			cdcCtl: &controller.FakeTiCDCControl{
 				GetStatusFn: func(tc *v1alpha1.TidbCluster, ordinal int32) (*controller.CaptureStatus, error) {
 					return &controller.CaptureStatus{
@@ -757,11 +757,11 @@ func TestTiCDCIsSupportGracefulUpgrade(t *testing.T) {
 			changeTc: func(tc *v1alpha1.TidbCluster) {
 				tc.Spec.Version = "malformed"
 			},
-			expectedOk:  false,
+			expectedOk:  true,
 			expectedErr: false,
 		},
 		{
-			caseName: "malformed ticdc spec version, skip graceful upgrade",
+			caseName: "malformed ticdc spec version, still try to graceful upgrade",
 			cdcCtl: &controller.FakeTiCDCControl{
 				GetStatusFn: func(tc *v1alpha1.TidbCluster, ordinal int32) (*controller.CaptureStatus, error) {
 					return &controller.CaptureStatus{
@@ -773,7 +773,7 @@ func TestTiCDCIsSupportGracefulUpgrade(t *testing.T) {
 				v := "malformed"
 				tc.Spec.TiCDC.Version = &v
 			},
-			expectedOk:  false,
+			expectedOk:  true,
 			expectedErr: false,
 		},
 		{