change hot upgrade type

pkg/control/sidecarcontrol/sidecarset_control.go

@@ -21,14 +21,21 @@ import (
 
 	"github.com/openkruise/kruise/apis/apps/pub"
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
+	"github.com/openkruise/kruise/pkg/features"
 	"github.com/openkruise/kruise/pkg/util"
+	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 )
 
+const (
+	ContainerNotReady = "NotReady"
+	ContainerReady    = "Ready"
+)
+
 type commonControl struct {
 	*appsv1alpha1.SidecarSet
 }
@@ -136,6 +143,25 @@ func (c *commonControl) UpdatePodAnnotationsInUpgrade(changedContainers []string
 	sidecarUpdateStates[sidecarSet.Name] = inPlaceUpdateState
 	by, _ := json.Marshal(sidecarUpdateStates)
 	pod.Annotations[SidecarsetInplaceUpdateStateKey] = string(by)
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.SidecarsetHotupgradeIgnoreMainContainerReadyStatus) {
+		return
+	}
+	mainContainers := c.GetPodMainContainerStatuses(pod)
+	mainContainerStatuses := make(map[string]string)
+	for _, mainContainer := range mainContainers {
+		if mainContainer.Name == "" {
+			continue
+		}
+		if mainContainer.Ready {
+			mainContainerStatuses[mainContainer.Name] = ContainerReady
+		} else {
+			mainContainerStatuses[mainContainer.Name] = ContainerNotReady
+		}
+	}
+	by, _ = json.Marshal(mainContainerStatuses)
+	pod.Annotations[SidecarSetBeforeHotUpgradeStateKey] = string(by)
+	return
 }
 
 // only check sidecar container is consistent
@@ -264,3 +290,15 @@ func IsSidecarContainerUpdateCompleted(pod *v1.Pod, sidecarSets, containers sets
 
 	return true
 }
+
+func (c *commonControl) GetPodMainContainerStatuses(pod *v1.Pod) []v1.ContainerStatus {
+	sidecarset := c.GetSidecarset()
+	sidecarContainers := GetSidecarContainersInPod(sidecarset)
+	mainContainerStatuses := make([]v1.ContainerStatus, 0)
+	for _, containerStatus := range pod.Status.ContainerStatuses {
+		if !sidecarContainers.Has(containerStatus.Name) {
+			mainContainerStatuses = append(mainContainerStatuses, containerStatus)
+		}
+	}
+	return mainContainerStatuses
+}

pkg/control/sidecarcontrol/util.go

@@ -64,6 +64,8 @@ const (
 
 	// SidecarSetUpgradable is a pod condition to indicate whether the pod's sidecarset is upgradable
 	SidecarSetUpgradable corev1.PodConditionType = "SidecarSetUpgradable"
+
+	SidecarSetBeforeHotUpgradeStateKey = "kruise.io/sidecarset-before-hotupgrade-main-container-state"
 )
 
 var (

pkg/controller/sidecarset/sidecarset_hotupgrade.go

@@ -18,11 +18,14 @@ package sidecarset
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
 	"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
+	"github.com/openkruise/kruise/pkg/features"
 	"github.com/openkruise/kruise/pkg/util"
+	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -31,6 +34,8 @@ import (
 	"k8s.io/klog/v2"
 )
 
+type SidecarsetMainContainerStatus map[string]string
+
 func (p *Processor) flipHotUpgradingContainers(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) error {
 	for _, pod := range pods {
 		if err := p.flipPodSidecarContainer(control, pod); err != nil {
@@ -105,27 +110,63 @@ func isSidecarSetHasHotUpgradeContainer(sidecarSet *appsv1alpha1.SidecarSet) boo
 	return false
 }
 
+func getPreviousMainContainerStatus(pod *corev1.Pod) map[string]bool {
+	result := make(map[string]bool)
+	v, ok := pod.Annotations[sidecarcontrol.SidecarSetBeforeHotUpgradeStateKey]
+	if !ok {
+		return nil
+	}
+	tmp := make(SidecarsetMainContainerStatus)
+	err := json.Unmarshal([]byte(v), &tmp)
+	if err != nil {
+		klog.Errorf("error unmarshaling pod(%s.%s)'s previous main container status, return nil", pod.Namespace, pod.Name)
+		return nil
+	}
+	for k, v := range tmp {
+		if v == sidecarcontrol.ContainerReady {
+			result[k] = true
+		} else {
+			result[k] = false
+		}
+	}
+	return result
+}
+
 func isHotUpgradingReady(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) bool {
 	if util.IsRunningAndReady(pod) {
 		return true
 	}
 
+	IgnoreMainContainerReadyStatusEnabled := utilfeature.DefaultFeatureGate.Enabled(features.SidecarsetHotupgradeIgnoreMainContainerReadyStatus)
+
+	sidecarContainers := sets.NewString()
 	emptyContainers := sets.NewString()
 	for _, sidecarContainer := range sidecarSet.Spec.Containers {
 		if sidecarcontrol.IsHotUpgradeContainer(&sidecarContainer) {
-			_, emptyContainer := sidecarcontrol.GetPodHotUpgradeContainers(sidecarContainer.Name, pod)
+			sidecarContainer, emptyContainer := sidecarcontrol.GetPodHotUpgradeContainers(sidecarContainer.Name, pod)
+			sidecarContainers.Insert(emptyContainer)
+			sidecarContainers.Insert(sidecarContainer)
 			emptyContainers.Insert(emptyContainer)
 		}
 	}
 
+	previousStatuses := getPreviousMainContainerStatus(pod)
+
 	for _, containerStatus := range pod.Status.ContainerStatuses {
 		// ignore empty sidecar container status
 		if emptyContainers.Has(containerStatus.Name) {
 			continue
 		}
-		// if container is not ready, then return false
+
+		// if container is not ready, when IgnoreMainContainerReadyStatusEnabled is on, then need to know if the container is main container
 		if !containerStatus.Ready {
-			return false
+			if IgnoreMainContainerReadyStatusEnabled && !sidecarContainers.Has(containerStatus.Name) {
+				if previousStatuses[containerStatus.Name] && !containerStatus.Ready {
+					return false
+				}
+			} else {
+				return false
+			}
 		}
 	}
 	// all containers with exception of empty sidecar containers are ready, then return true

pkg/controller/sidecarset/sidecarset_hotupgrade_test.go

@@ -22,6 +22,8 @@ import (
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
 	"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
+	"github.com/openkruise/kruise/pkg/features"
+	"github.com/openkruise/kruise/pkg/util/feature"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -294,6 +296,153 @@ func testUpdateHotUpgradeSidecar(t *testing.T, hotUpgradeEmptyImage string, side
 	}
 }
 
+func TestUpdateHotUpgradeSidecarWithMainContainerNotReady(t *testing.T) {
+	sidecarSetInput := sidecarSetHotUpgrade.DeepCopy()
+	handlers := map[string]HandlePod{
+		"test-sidecar-2 container is upgrading": func(pods []*corev1.Pod) {
+			pods[0].Status.ContainerStatuses[2].Image = "test-image:v2"
+			pods[0].Status.ContainerStatuses[2].ImageID = testImageV2ImageID
+		},
+		"test-sidecar-2 container upgrade complete, and reset test-sidecar-1 empty image": func(pods []*corev1.Pod) {
+			pods[0].Status.ContainerStatuses[1].Image = hotUpgradeEmptyImage
+			pods[0].Status.ContainerStatuses[1].ImageID = hotUpgradeEmptyImageID
+		},
+	}
+	testUpdateHotUpgradeSidecarWithMainContainerNotReady(t, hotUpgradeEmptyImage, sidecarSetInput, handlers)
+}
+
+func testUpdateHotUpgradeSidecarWithMainContainerNotReady(t *testing.T, hotUpgradeEmptyImage string, sidecarSetInput *appsv1alpha1.SidecarSet, handlers map[string]HandlePod) {
+	podInput := podHotUpgrade.DeepCopy()
+	podInput.Name = "juruo-test"
+	podInput.Status.Phase = corev1.PodPending
+	podInput.Status.Conditions[0].Status = corev1.ConditionFalse
+	podInput.Status.ContainerStatuses[0].Ready = false
+	feature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%v=%v", features.SidecarsetHotupgradeIgnoreMainContainerReadyStatus, "true"))
+
+	cases := []struct {
+		name          string
+		getPods       func() []*corev1.Pod
+		getSidecarset func() *appsv1alpha1.SidecarSet
+		// container.name -> infos []string
+		expectedInfo map[string][]string
+		// MatchedPods, UpdatedPods, ReadyPods, AvailablePods, UnavailablePods
+		expectedStatus []int32
+	}{
+		{
+			name: "sidecarset hot update test-sidecar container test-image:v2",
+			getPods: func() []*corev1.Pod {
+				pods := []*corev1.Pod{
+					podInput.DeepCopy(),
+				}
+				return pods
+			},
+			getSidecarset: func() *appsv1alpha1.SidecarSet {
+				return sidecarSetInput.DeepCopy()
+			},
+			expectedInfo: map[string][]string{
+				"test-sidecar-1": {"test-image:v1"},
+				"test-sidecar-2": {"test-image:v2"},
+			},
+			expectedStatus: []int32{1, 0, 0, 0},
+		},
+		{
+			name: "test-sidecar-2 container is upgrading",
+			getPods: func() []*corev1.Pod {
+				pods := []*corev1.Pod{
+					podInput.DeepCopy(),
+				}
+				return pods
+			},
+			getSidecarset: func() *appsv1alpha1.SidecarSet {
+				return sidecarSetInput.DeepCopy()
+			},
+			expectedInfo: map[string][]string{
+				"test-sidecar-1": {"test-image:v1"},
+				"test-sidecar-2": {"test-image:v2"},
+			},
+			expectedStatus: []int32{1, 1, 0, 0},
+		},
+		{
+			name: "test-sidecar-2 container upgrade complete, and reset test-sidecar-1 empty image",
+			getPods: func() []*corev1.Pod {
+				return []*corev1.Pod{podInput.DeepCopy()}
+			},
+			getSidecarset: func() *appsv1alpha1.SidecarSet {
+				return sidecarSetInput.DeepCopy()
+			},
+			expectedInfo: map[string][]string{
+				"test-sidecar-1": {hotUpgradeEmptyImage},
+				"test-sidecar-2": {"test-image:v2"},
+			},
+			expectedStatus: []int32{1, 1, 0, 0},
+		},
+		{
+			name: "sidecarset hot update test-sidecar container test-image:v2 complete",
+			getPods: func() []*corev1.Pod {
+				return []*corev1.Pod{podInput.DeepCopy()}
+			},
+			getSidecarset: func() *appsv1alpha1.SidecarSet {
+				return sidecarSetInput.DeepCopy()
+			},
+			expectedInfo: map[string][]string{
+				"test-sidecar-1": {hotUpgradeEmptyImage},
+				"test-sidecar-2": {"test-image:v2"},
+			},
+			expectedStatus: []int32{1, 1, 0, 0},
+		},
+	}
+	for _, cs := range cases {
+		t.Run(cs.name, func(t *testing.T) {
+			pod := cs.getPods()[0]
+			sidecarset := cs.getSidecarset()
+			fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarset, pod).Build()
+			processor := NewSidecarSetProcessor(fakeClient, record.NewFakeRecorder(10))
+			_, err := processor.UpdateSidecarSet(sidecarset)
+			if err != nil {
+				t.Errorf("processor update sidecarset failed: %s", err.Error())
+			}
+			podOutput, err := getLatestPod(fakeClient, pod)
+			if err != nil {
+				t.Errorf("get latest pod(%s) failed: %s", pod.Name, err.Error())
+			}
+			podInput = podOutput.DeepCopy()
+			for cName, infos := range cs.expectedInfo {
+				sidecarContainer := getPodContainerByName(cName, podOutput)
+				if infos[0] != sidecarContainer.Image {
+					t.Fatalf("expect pod(%s) container(%s) image(%s), but get image(%s)", pod.Name, sidecarContainer.Name, infos[0], sidecarContainer.Image)
+				}
+			}
+
+			sidecarsetOutput, err := getLatestSidecarSet(fakeClient, sidecarset)
+			if err != nil {
+				t.Errorf("get latest sidecarset(%s) failed: %s", sidecarset.Name, err.Error())
+			}
+			sidecarSetInput = sidecarsetOutput.DeepCopy()
+			for k, v := range cs.expectedStatus {
+				var actualValue int32
+				switch k {
+				case 0:
+					actualValue = sidecarsetOutput.Status.MatchedPods
+				case 1:
+					actualValue = sidecarsetOutput.Status.UpdatedPods
+				case 2:
+					actualValue = sidecarsetOutput.Status.ReadyPods
+				case 3:
+					actualValue = sidecarsetOutput.Status.UpdatedReadyPods
+				}
+
+				if v != actualValue {
+					t.Fatalf("except sidecarset status(%d:%d), but get value(%d)", k, v, actualValue)
+				}
+			}
+			//handle potInput
+			if handle, ok := handlers[cs.name]; ok {
+				handle([]*corev1.Pod{podInput})
+			}
+		})
+	}
+}
+
 func getPodContainerByName(cName string, pod *corev1.Pod) *corev1.Container {
 	for _, container := range pod.Spec.Containers {
 		if cName == container.Name {

pkg/features/kruise_features.go

@@ -76,6 +76,9 @@ const (
 	// If TemplateNoDefaults is false, webhook should inject default fields only when the template changed.
 	TemplateNoDefaults featuregate.Feature = "TemplateNoDefaults"
 
+	// Controls whether hotupgrading process could be done when sidecarset pod are not ready at start.
+	SidecarsetHotupgradeIgnoreMainContainerReadyStatus featuregate.Feature = "SidecarsetHotupgradeIgnoreMainContainerReadyStatus"
+
 	// InPlaceUpdateEnvFromMetadata enables Kruise to in-place update a container in Pod
 	// when its env from labels/annotations changed and pod is in-place updating.
 	InPlaceUpdateEnvFromMetadata featuregate.Feature = "InPlaceUpdateEnvFromMetadata"
@@ -112,23 +115,24 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
 	KruiseDaemon:      {Default: true, PreRelease: featuregate.Beta},
 	DaemonWatchingPod: {Default: true, PreRelease: featuregate.Beta},
 
-	CloneSetShortHash:                         {Default: false, PreRelease: featuregate.Alpha},
-	KruisePodReadinessGate:                    {Default: false, PreRelease: featuregate.Alpha},
-	PreDownloadImageForInPlaceUpdate:          {Default: false, PreRelease: featuregate.Alpha},
-	CloneSetPartitionRollback:                 {Default: false, PreRelease: featuregate.Alpha},
-	ResourcesDeletionProtection:               {Default: true, PreRelease: featuregate.Alpha},
-	WorkloadSpread:                            {Default: true, PreRelease: featuregate.Alpha},
-	PodUnavailableBudgetDeleteGate:            {Default: true, PreRelease: featuregate.Alpha},
-	PodUnavailableBudgetUpdateGate:            {Default: false, PreRelease: featuregate.Alpha},
-	TemplateNoDefaults:                        {Default: false, PreRelease: featuregate.Alpha},
-	InPlaceUpdateEnvFromMetadata:              {Default: true, PreRelease: featuregate.Alpha},
-	StatefulSetAutoDeletePVC:                  {Default: true, PreRelease: featuregate.Alpha},
-	SidecarSetPatchPodMetadataDefaultsAllowed: {Default: false, PreRelease: featuregate.Alpha},
-	SidecarTerminator:                         {Default: false, PreRelease: featuregate.Alpha},
-	PodProbeMarkerGate:                        {Default: true, PreRelease: featuregate.Alpha},
-	PreDownloadImageForDaemonSetUpdate:        {Default: false, PreRelease: featuregate.Alpha},
-	CloneSetEventHandlerOptimization:          {Default: false, PreRelease: featuregate.Alpha},
-	PreparingUpdateAsUpdate:                   {Default: false, PreRelease: featuregate.Alpha},
+	CloneSetShortHash:                                  {Default: false, PreRelease: featuregate.Alpha},
+	KruisePodReadinessGate:                             {Default: false, PreRelease: featuregate.Alpha},
+	PreDownloadImageForInPlaceUpdate:                   {Default: false, PreRelease: featuregate.Alpha},
+	CloneSetPartitionRollback:                          {Default: false, PreRelease: featuregate.Alpha},
+	ResourcesDeletionProtection:                        {Default: true, PreRelease: featuregate.Alpha},
+	WorkloadSpread:                                     {Default: true, PreRelease: featuregate.Alpha},
+	PodUnavailableBudgetDeleteGate:                     {Default: true, PreRelease: featuregate.Alpha},
+	PodUnavailableBudgetUpdateGate:                     {Default: false, PreRelease: featuregate.Alpha},
+	TemplateNoDefaults:                                 {Default: false, PreRelease: featuregate.Alpha},
+	SidecarsetHotupgradeIgnoreMainContainerReadyStatus: {Default: false, PreRelease: featuregate.Alpha},
+	InPlaceUpdateEnvFromMetadata:                       {Default: true, PreRelease: featuregate.Alpha},
+	StatefulSetAutoDeletePVC:                           {Default: true, PreRelease: featuregate.Alpha},
+	SidecarSetPatchPodMetadataDefaultsAllowed:          {Default: false, PreRelease: featuregate.Alpha},
+	SidecarTerminator:                                  {Default: false, PreRelease: featuregate.Alpha},
+	PodProbeMarkerGate:                                 {Default: true, PreRelease: featuregate.Alpha},
+	PreDownloadImageForDaemonSetUpdate:                 {Default: false, PreRelease: featuregate.Alpha},
+	CloneSetEventHandlerOptimization:                   {Default: false, PreRelease: featuregate.Alpha},
+	PreparingUpdateAsUpdate:                            {Default: false, PreRelease: featuregate.Alpha},
 }
 
 func init() {