From 99e10a51a7df1ab5618924740fafcf08d52d468d Mon Sep 17 00:00:00 2001 From: yunbo Date: Thu, 1 Aug 2024 13:54:13 +0800 Subject: [PATCH] add restriction for traffic configuration of partition-style step Signed-off-by: yunbo --- .../rollout_create_update_handler.go | 91 +- .../rollout_create_update_handler_test.go | 253 ++ .../validating/validate_v1alphal_rollout.go | 52 +- test/e2e/rollout_test.go | 2841 +-------------- test/e2e/rollout_v1beta1_test.go | 3095 +++++++++++++++-- .../rollout/rollout_canary_base.yaml | 2 + .../rollout_v1beta1_partition_base.yaml | 12 +- 7 files changed, 3099 insertions(+), 3247 deletions(-) diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler.go b/pkg/webhook/rollout/validating/rollout_create_update_handler.go index 08bfacf2..fa7b378b 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler.go @@ -18,6 +18,7 @@ package validating import ( "context" + "flag" "fmt" "net/http" "reflect" @@ -47,7 +48,29 @@ type RolloutCreateUpdateHandler struct { Decoder *admission.Decoder } -var _ admission.Handler = &RolloutCreateUpdateHandler{} +var ( + _ admission.Handler = &RolloutCreateUpdateHandler{} + // PartitionReplicasLimitWithTraffic represents the maximum percentage of replicas + // allowed for a step of partition-style release, if traffic/matches specified. + // If a step is configured with a number of replicas exceeding this percentage, the traffic strategy for that step + // must not be specified. If this rule is violated, the Rollout webhook will block the creation or modification of the Rollout. + // The default limit is set to 50%. + // Here is why we set this limit for partition style release: + // In rollback and continuous scenarios, usually we expect the Rollout to route all traffic to the stable version first. + // However, if the stable version's pods are relatively few (less than 1-PartitionReplicasLimitWithTraffic), this might overload the stable version's pods. + // Therefore, even if users were allowed to configure traffic when the new version's replicas exceed this threshold, + // during rollback, traffic will still be evenly distributed across all backend pods if the stable version's pods are relatively few. + PartitionReplicasLimitWithTraffic = 50 +) + +func init() { + flag.IntVar(&PartitionReplicasLimitWithTraffic, "partition-percent-limit", 50, "represents the maximum percentage of replicas allowed for a step of partition-style release, if traffic/matches specified.") +} + +// record upper level information about the rollout +type validateContext struct { + style string +} // Handle handles admission requests. func (h *RolloutCreateUpdateHandler) Handle(ctx context.Context, req admission.Request) admission.Response { @@ -155,7 +178,7 @@ func (h *RolloutCreateUpdateHandler) validateRolloutUpdate(oldObj, newObj *appsv } func (h *RolloutCreateUpdateHandler) validateRollout(rollout *appsv1beta1.Rollout) field.ErrorList { - errList := validateRolloutSpec(rollout, field.NewPath("Spec")) + errList := validateRolloutSpec(GetContextFromv1beta1Rollout(rollout), rollout, field.NewPath("Spec")) errList = append(errList, h.validateRolloutConflict(rollout, field.NewPath("Conflict Checker"))...) return errList } @@ -178,9 +201,9 @@ func (h *RolloutCreateUpdateHandler) validateRolloutConflict(rollout *appsv1beta return nil } -func validateRolloutSpec(rollout *appsv1beta1.Rollout, fldPath *field.Path) field.ErrorList { +func validateRolloutSpec(c *validateContext, rollout *appsv1beta1.Rollout, fldPath *field.Path) field.ErrorList { errList := validateRolloutSpecObjectRef(&rollout.Spec.WorkloadRef, fldPath.Child("ObjectRef")) - errList = append(errList, validateRolloutSpecStrategy(&rollout.Spec.Strategy, fldPath.Child("Strategy"))...) + errList = append(errList, validateRolloutSpecStrategy(c, &rollout.Spec.Strategy, fldPath.Child("Strategy"))...) return errList } @@ -196,7 +219,7 @@ func validateRolloutSpecObjectRef(workloadRef *appsv1beta1.ObjectRef, fldPath *f return nil } -func validateRolloutSpecStrategy(strategy *appsv1beta1.RolloutStrategy, fldPath *field.Path) field.ErrorList { +func validateRolloutSpecStrategy(c *validateContext, strategy *appsv1beta1.RolloutStrategy, fldPath *field.Path) field.ErrorList { if strategy.Canary == nil && strategy.BlueGreen == nil { return field.ErrorList{field.Invalid(fldPath, nil, "Canary and BlueGreen cannot both be empty")} } @@ -204,25 +227,13 @@ func validateRolloutSpecStrategy(strategy *appsv1beta1.RolloutStrategy, fldPath return field.ErrorList{field.Invalid(fldPath, nil, "Canary and BlueGreen cannot both be set")} } if strategy.BlueGreen != nil { - return validateRolloutSpecBlueGreenStrategy(strategy.BlueGreen, fldPath.Child("BlueGreen")) + return validateRolloutSpecBlueGreenStrategy(c, strategy.BlueGreen, fldPath.Child("BlueGreen")) } - return validateRolloutSpecCanaryStrategy(strategy.Canary, fldPath.Child("Canary")) + return validateRolloutSpecCanaryStrategy(c, strategy.Canary, fldPath.Child("Canary")) } -type TrafficRule string - -const ( - TrafficRuleCanary TrafficRule = "Canary" - TrafficRuleBlueGreen TrafficRule = "BlueGreen" - NoTraffic TrafficRule = "NoTraffic" -) - -func validateRolloutSpecCanaryStrategy(canary *appsv1beta1.CanaryStrategy, fldPath *field.Path) field.ErrorList { - trafficRule := NoTraffic - if len(canary.TrafficRoutings) > 0 { - trafficRule = TrafficRuleCanary - } - errList := validateRolloutSpecCanarySteps(canary.Steps, fldPath.Child("Steps"), trafficRule) +func validateRolloutSpecCanaryStrategy(c *validateContext, canary *appsv1beta1.CanaryStrategy, fldPath *field.Path) field.ErrorList { + errList := validateRolloutSpecCanarySteps(c, canary.Steps, fldPath.Child("Steps")) if len(canary.TrafficRoutings) > 1 { errList = append(errList, field.Invalid(fldPath, canary.TrafficRoutings, "Rollout currently only support single TrafficRouting.")) } @@ -232,12 +243,8 @@ func validateRolloutSpecCanaryStrategy(canary *appsv1beta1.CanaryStrategy, fldPa return errList } -func validateRolloutSpecBlueGreenStrategy(blueGreen *appsv1beta1.BlueGreenStrategy, fldPath *field.Path) field.ErrorList { - trafficRule := NoTraffic - if len(blueGreen.TrafficRoutings) > 0 { - trafficRule = TrafficRuleBlueGreen - } - errList := validateRolloutSpecCanarySteps(blueGreen.Steps, fldPath.Child("Steps"), trafficRule) +func validateRolloutSpecBlueGreenStrategy(c *validateContext, blueGreen *appsv1beta1.BlueGreenStrategy, fldPath *field.Path) field.ErrorList { + errList := validateRolloutSpecCanarySteps(c, blueGreen.Steps, fldPath.Child("Steps")) if len(blueGreen.TrafficRoutings) > 1 { errList = append(errList, field.Invalid(fldPath, blueGreen.TrafficRoutings, "Rollout currently only support single TrafficRouting.")) } @@ -275,7 +282,7 @@ func validateRolloutSpecCanaryTraffic(traffic appsv1beta1.TrafficRoutingRef, fld return errList } -func validateRolloutSpecCanarySteps(steps []appsv1beta1.CanaryStep, fldPath *field.Path, trafficRule TrafficRule) field.ErrorList { +func validateRolloutSpecCanarySteps(c *validateContext, steps []appsv1beta1.CanaryStep, fldPath *field.Path) field.ErrorList { stepCount := len(steps) if stepCount == 0 { return field.ErrorList{field.Invalid(fldPath, steps, "The number of Canary.Steps cannot be empty")} @@ -293,13 +300,24 @@ func validateRolloutSpecCanarySteps(steps []appsv1beta1.CanaryStep, fldPath *fie return field.ErrorList{field.Invalid(fldPath.Index(i).Child("Replicas"), s.Replicas, `replicas must be positive number, or a percentage with "0%" < canaryReplicas <= "100%"`)} } - if trafficRule == NoTraffic || s.Traffic == nil { + // no traffic strategy is configured for current step + if s.Traffic == nil && len(s.Matches) == 0 { + continue + } + // replicas is percentage + if c.style == string(appsv1beta1.PartitionRollingStyle) && IsPercentageCanaryReplicasType(s.Replicas) { + currCanaryReplicas, _ := intstr.GetScaledValueFromIntOrPercent(s.Replicas, 100, true) + if currCanaryReplicas > PartitionReplicasLimitWithTraffic { + return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `For partition style rollout: step[x].replicas must not greater than partition-percent-limit if traffic specified`)} + } + } + if s.Traffic == nil { continue } is := intstr.FromString(*s.Traffic) weight, err := intstr.GetScaledValueFromIntOrPercent(&is, 100, true) - switch trafficRule { - case TrafficRuleBlueGreen: + switch c.style { + case string(appsv1beta1.BlueGreenRollingStyle): // traffic "0%" is allowed in blueGreen strategy if err != nil || weight < 0 || weight > 100 { return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `traffic must be percentage with "0%" <= traffic <= "100%" in blueGreen strategy`)} @@ -355,3 +373,14 @@ func (h *RolloutCreateUpdateHandler) InjectDecoder(d *admission.Decoder) error { h.Decoder = d return nil } + +func GetContextFromv1beta1Rollout(rollout *appsv1beta1.Rollout) *validateContext { + if rollout.Spec.Strategy.Canary == nil && rollout.Spec.Strategy.BlueGreen == nil { + return nil + } + style := rollout.Spec.Strategy.GetRollingStyle() + if appsv1beta1.IsRealPartition(rollout) { + style = appsv1beta1.PartitionRollingStyle + } + return &validateContext{style: string(style)} +} diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go index 27ea4531..12e6f9f4 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" rolloutapi "github.com/openkruise/rollouts/api" + appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" appsv1beta1 "github.com/openkruise/rollouts/api/v1beta1" apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -102,6 +103,63 @@ var ( }, }, } + rolloutV1alpha1 = appsv1alpha1.Rollout{ + TypeMeta: metav1.TypeMeta{ + APIVersion: appsv1alpha1.SchemeGroupVersion.String(), + Kind: "Rollout", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rollout-demo", + Namespace: "namespace-unit-test", + Annotations: map[string]string{}, + }, + Spec: appsv1alpha1.RolloutSpec{ + ObjectRef: appsv1alpha1.ObjectRef{ + WorkloadRef: &appsv1alpha1.WorkloadRef{ + APIVersion: apps.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: "deployment-demo", + }, + }, + Strategy: appsv1alpha1.RolloutStrategy{ + Canary: &appsv1alpha1.CanaryStrategy{ + Steps: []appsv1alpha1.CanaryStep{ + { + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(10), + }, + Pause: appsv1alpha1.RolloutPause{}, + }, + { + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(30), + }, + Pause: appsv1alpha1.RolloutPause{Duration: utilpointer.Int32(1 * 24 * 60 * 60)}, + }, + { + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, + }, + }, + TrafficRoutings: []appsv1alpha1.TrafficRoutingRef{ + { + Service: "service-demo", + Ingress: &appsv1alpha1.IngressTrafficRouting{ + ClassType: "nginx", + Name: "ingress-nginx-demo", + }, + }, + }, + }, + }, + }, + Status: appsv1alpha1.RolloutStatus{ + CanaryStatus: &appsv1alpha1.CanaryStatus{ + CurrentStepState: appsv1alpha1.CanaryStepStateCompleted, + }, + }, + } ) func init() { @@ -262,6 +320,74 @@ func TestRolloutValidateCreate(t *testing.T) { return []client.Object{object} }, }, + { + Name: "test with replicasLimitWithTraffic - 1", + Succeed: true, + GetObject: func() []client.Object { + object := rollout.DeepCopy() + n := len(object.Spec.Strategy.Canary.Steps) + object.Spec.Strategy.Canary.Steps[n-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "30%"} + return []client.Object{object} + }, + }, + { + Name: "test with replicasLimitWithTraffic - 2", + Succeed: false, + GetObject: func() []client.Object { + object := rollout.DeepCopy() + n := len(object.Spec.Strategy.Canary.Steps) + object.Spec.Strategy.Canary.Steps[n-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "31%"} + return []client.Object{object} + }, + }, + { + Name: "test with replicasLimitWithTraffic - 2 - canary style", + Succeed: true, + GetObject: func() []client.Object { + object := rollout.DeepCopy() + object.Spec.Strategy.Canary.EnableExtraWorkloadForCanary = true + n := len(object.Spec.Strategy.Canary.Steps) + object.Spec.Strategy.Canary.Steps[n-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "31%"} + return []client.Object{object} + }, + }, + { + Name: "test with replicasLimitWithTraffic - 2 - cloneset", + Succeed: false, + GetObject: func() []client.Object { + object := rollout.DeepCopy() + object.Spec.WorkloadRef = appsv1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "whatever", + } + n := len(object.Spec.Strategy.Canary.Steps) + object.Spec.Strategy.Canary.Steps[n-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "31%"} + return []client.Object{object} + }, + }, + { + Name: "test with replicasLimitWithTraffic - 3", + Succeed: true, + GetObject: func() []client.Object { + object := rollout.DeepCopy() + PartitionReplicasLimitWithTraffic = 100 + n := len(object.Spec.Strategy.Canary.Steps) + object.Spec.Strategy.Canary.Steps[n-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "100%"} + return []client.Object{object} + }, + }, + { + Name: "test with replicasLimitWithTraffic - 4", + Succeed: false, + GetObject: func() []client.Object { + object := rollout.DeepCopy() + PartitionReplicasLimitWithTraffic = 50 + n := len(object.Spec.Strategy.Canary.Steps) + object.Spec.Strategy.Canary.Steps[n-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "51%"} + return []client.Object{object} + }, + }, //{ // Name: "The last Steps.Traffic is not 100", // Succeed: false, @@ -335,6 +461,133 @@ func TestRolloutValidateCreate(t *testing.T) { errList := handler.validateRollout(objects[0].(*appsv1beta1.Rollout)) t.Log(errList) Expect(len(errList) == 0).Should(Equal(cs.Succeed)) + // restore PartitionReplicasLimitWithTraffic after each case + PartitionReplicasLimitWithTraffic = 30 + }) + } +} + +func TestRolloutV1alpha1ValidateCreate(t *testing.T) { + RegisterFailHandler(Fail) + + cases := []struct { + Name string + Succeed bool + GetObject func() []client.Object + }{ + { + Name: "Canary style", + Succeed: true, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + return []client.Object{obj} + }, + }, + { + Name: "Partition style without replicas - 1", + Succeed: false, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + obj.Annotations[appsv1alpha1.RolloutStyleAnnotation] = string(appsv1alpha1.PartitionRollingStyle) + return []client.Object{obj} + }, + }, + { + Name: "Partition style without replicas - 2", + Succeed: true, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + PartitionReplicasLimitWithTraffic = 100 + obj.Annotations[appsv1alpha1.RolloutStyleAnnotation] = string(appsv1alpha1.PartitionRollingStyle) + return []client.Object{obj} + }, + }, + { + Name: "Partition style without replicas - 3", + Succeed: false, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Weight = utilpointer.Int32(32) + PartitionReplicasLimitWithTraffic = 31 + obj.Annotations[appsv1alpha1.RolloutStyleAnnotation] = string(appsv1alpha1.PartitionRollingStyle) + return []client.Object{obj} + }, + }, + { + Name: "Partition style without replicas- 3 - canary style", + Succeed: true, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Weight = utilpointer.Int32(32) + PartitionReplicasLimitWithTraffic = 31 + return []client.Object{obj} + }, + }, + { + Name: "Partition style without replicas - 3 - cloneset", + Succeed: false, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + obj.Spec.ObjectRef.WorkloadRef = &appsv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "whatever", + } + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Weight = utilpointer.Int32(32) + PartitionReplicasLimitWithTraffic = 31 + obj.Annotations[appsv1alpha1.RolloutStyleAnnotation] = string(appsv1alpha1.PartitionRollingStyle) + return []client.Object{obj} + }, + }, + { + Name: "Partition style with replicas - 1", + Succeed: false, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Weight = utilpointer.Int32(50) + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "32%"} + PartitionReplicasLimitWithTraffic = 31 + obj.Annotations[appsv1alpha1.RolloutStyleAnnotation] = string(appsv1alpha1.PartitionRollingStyle) + return []client.Object{obj} + }, + }, + { + Name: "Partition style with replicas - 2", + Succeed: true, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Weight = utilpointer.Int32(50) + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "31%"} + PartitionReplicasLimitWithTraffic = 31 + obj.Annotations[appsv1alpha1.RolloutStyleAnnotation] = string(appsv1alpha1.PartitionRollingStyle) + return []client.Object{obj} + }, + }, + { + Name: "Partition style with replicas - 3", + Succeed: true, + GetObject: func() []client.Object { + obj := rolloutV1alpha1.DeepCopy() + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Weight = nil + obj.Spec.Strategy.Canary.Steps[len(obj.Spec.Strategy.Canary.Steps)-1].Replicas = &intstr.IntOrString{Type: intstr.String, StrVal: "50%"} + obj.Annotations[appsv1alpha1.RolloutStyleAnnotation] = string(appsv1alpha1.PartitionRollingStyle) + return []client.Object{obj} + }, + }, + } + + for _, cs := range cases { + t.Run(cs.Name, func(t *testing.T) { + objects := cs.GetObject() + cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + handler := RolloutCreateUpdateHandler{ + Client: cli, + } + errList := handler.validateV1alpha1Rollout(objects[0].(*appsv1alpha1.Rollout)) + t.Log(errList) + Expect(len(errList) == 0).Should(Equal(cs.Succeed)) + // restore PartitionReplicasLimitWithTraffic after each case + PartitionReplicasLimitWithTraffic = 30 }) } } diff --git a/pkg/webhook/rollout/validating/validate_v1alphal_rollout.go b/pkg/webhook/rollout/validating/validate_v1alphal_rollout.go index 19c230a7..e9c2d3e6 100644 --- a/pkg/webhook/rollout/validating/validate_v1alphal_rollout.go +++ b/pkg/webhook/rollout/validating/validate_v1alphal_rollout.go @@ -25,6 +25,7 @@ import ( appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/util" utilclient "github.com/openkruise/rollouts/pkg/util/client" + apps "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/validation/field" @@ -70,7 +71,7 @@ func (h *RolloutCreateUpdateHandler) validateV1alpha1RolloutUpdate(oldObj, newOb } func (h *RolloutCreateUpdateHandler) validateV1alpha1Rollout(rollout *appsv1alpha1.Rollout) field.ErrorList { - errList := validateV1alpha1RolloutSpec(rollout, field.NewPath("Spec")) + errList := validateV1alpha1RolloutSpec(GetContextFromv1alpha1Rollout(rollout), rollout, field.NewPath("Spec")) errList = append(errList, h.validateV1alpha1RolloutConflict(rollout, field.NewPath("Conflict Checker"))...) return errList } @@ -93,10 +94,10 @@ func (h *RolloutCreateUpdateHandler) validateV1alpha1RolloutConflict(rollout *ap return nil } -func validateV1alpha1RolloutSpec(rollout *appsv1alpha1.Rollout, fldPath *field.Path) field.ErrorList { +func validateV1alpha1RolloutSpec(c *validateContext, rollout *appsv1alpha1.Rollout, fldPath *field.Path) field.ErrorList { errList := validateV1alpha1RolloutSpecObjectRef(&rollout.Spec.ObjectRef, fldPath.Child("ObjectRef")) errList = append(errList, validateV1alpha1RolloutRollingStyle(rollout, field.NewPath("RollingStyle"))...) - errList = append(errList, validateV1alpha1RolloutSpecStrategy(&rollout.Spec.Strategy, fldPath.Child("Strategy"))...) + errList = append(errList, validateV1alpha1RolloutSpecStrategy(c, &rollout.Spec.Strategy, fldPath.Child("Strategy"))...) return errList } @@ -122,16 +123,16 @@ func validateV1alpha1RolloutSpecObjectRef(objectRef *appsv1alpha1.ObjectRef, fld return nil } -func validateV1alpha1RolloutSpecStrategy(strategy *appsv1alpha1.RolloutStrategy, fldPath *field.Path) field.ErrorList { - return validateV1alpha1RolloutSpecCanaryStrategy(strategy.Canary, fldPath.Child("Canary")) +func validateV1alpha1RolloutSpecStrategy(c *validateContext, strategy *appsv1alpha1.RolloutStrategy, fldPath *field.Path) field.ErrorList { + return validateV1alpha1RolloutSpecCanaryStrategy(c, strategy.Canary, fldPath.Child("Canary")) } -func validateV1alpha1RolloutSpecCanaryStrategy(canary *appsv1alpha1.CanaryStrategy, fldPath *field.Path) field.ErrorList { +func validateV1alpha1RolloutSpecCanaryStrategy(c *validateContext, canary *appsv1alpha1.CanaryStrategy, fldPath *field.Path) field.ErrorList { if canary == nil { return field.ErrorList{field.Invalid(fldPath, nil, "Canary cannot be empty")} } - errList := validateV1alpha1RolloutSpecCanarySteps(canary.Steps, fldPath.Child("Steps"), len(canary.TrafficRoutings) > 0) + errList := validateV1alpha1RolloutSpecCanarySteps(c, canary.Steps, fldPath.Child("Steps"), len(canary.TrafficRoutings) > 0) if len(canary.TrafficRoutings) > 1 { errList = append(errList, field.Invalid(fldPath, canary.TrafficRoutings, "Rollout currently only support single TrafficRouting.")) } @@ -169,7 +170,7 @@ func validateV1alpha1RolloutSpecCanaryTraffic(traffic appsv1alpha1.TrafficRoutin return errList } -func validateV1alpha1RolloutSpecCanarySteps(steps []appsv1alpha1.CanaryStep, fldPath *field.Path, isTraffic bool) field.ErrorList { +func validateV1alpha1RolloutSpecCanarySteps(c *validateContext, steps []appsv1alpha1.CanaryStep, fldPath *field.Path, isTraffic bool) field.ErrorList { stepCount := len(steps) if stepCount == 0 { return field.ErrorList{field.Invalid(fldPath, steps, "The number of Canary.Steps cannot be empty")} @@ -188,6 +189,25 @@ func validateV1alpha1RolloutSpecCanarySteps(steps []appsv1alpha1.CanaryStep, fld return field.ErrorList{field.Invalid(fldPath.Index(i).Child("Replicas"), s.Replicas, `replicas must be positive number, or a percentage with "0%" < canaryReplicas <= "100%"`)} } + // is partiton-style release + // and has traffic strategy for this step + // and replicas is in percentage format and greater than replicasLimitWithTraffic + if c.style == string(appsv1alpha1.PartitionRollingStyle) && + IsPercentageCanaryReplicasType(s.Replicas) && canaryReplicas > PartitionReplicasLimitWithTraffic && + (s.Matches != nil || s.Weight != nil) { + return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, + `For patition style rollout: step[x].replicas must not greater than replicasLimitWithTraffic if traffic or matches specified`)} + } + } else { + // replicas is nil, weight is not nil + if c.style == string(appsv1alpha1.PartitionRollingStyle) && *s.Weight > int32(PartitionReplicasLimitWithTraffic) { + return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, + `For patition style rollout: step[x].weight must not greater than replicasLimitWithTraffic if replicas is not specified`)} + } + if *s.Weight <= 0 || *s.Weight > 100 { + return field.ErrorList{field.Invalid(fldPath.Index(i).Child("Weight"), s.Weight, + `weight must be positive number, and less than or equal to replicasLimitWithTraffic (defaults to 30)`)} + } } } @@ -225,3 +245,19 @@ func IsSameV1alpha1WorkloadRefGVKName(a, b *appsv1alpha1.WorkloadRef) bool { } return reflect.DeepEqual(a, b) } + +func GetContextFromv1alpha1Rollout(rollout *appsv1alpha1.Rollout) *validateContext { + if rollout.Spec.Strategy.Canary == nil { + return nil + } + style := appsv1alpha1.PartitionRollingStyle + switch strings.ToLower(rollout.Annotations[appsv1alpha1.RolloutStyleAnnotation]) { + case "", strings.ToLower(string(appsv1alpha1.CanaryRollingStyle)): + targetRef := rollout.Spec.ObjectRef.WorkloadRef + if targetRef.APIVersion == apps.SchemeGroupVersion.String() && targetRef.Kind == reflect.TypeOf(apps.Deployment{}).Name() { + style = appsv1alpha1.CanaryRollingStyle + } + } + + return &validateContext{style: string(style)} +} diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index cc135caa..45fd3ce3 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -16,6 +16,10 @@ limitations under the License. package e2e +//READ - instead of adding test cases or modifying existing test cases in this file, +// we recommend to create new test cases or rewrite existing test cases in test/e2e/rollout_v1beta1_test.go, +// which use v1beta1 Rollout instead of v1alpha1. + import ( "context" "fmt" @@ -163,42 +167,6 @@ var _ = SIGDescribe("Rollout", func() { return daemon } - UpdateNativeStatefulSet := func(object *apps.StatefulSet) *apps.StatefulSet { - var clone *apps.StatefulSet - Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { - clone = &apps.StatefulSet{} - err := GetObject(object.Name, clone) - if err != nil { - return err - } - clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) - clone.Spec.Template = *object.Spec.Template.DeepCopy() - clone.Labels = mergeMap(clone.Labels, object.Labels) - clone.Annotations = mergeMap(clone.Annotations, object.Annotations) - return k8sClient.Update(context.TODO(), clone) - })).NotTo(HaveOccurred()) - - return clone - } - - UpdateAdvancedStatefulSet := func(object *appsv1beta1.StatefulSet) *appsv1beta1.StatefulSet { - var clone *appsv1beta1.StatefulSet - Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { - clone = &appsv1beta1.StatefulSet{} - err := GetObject(object.Name, clone) - if err != nil { - return err - } - clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) - clone.Spec.Template = *object.Spec.Template.DeepCopy() - clone.Labels = mergeMap(clone.Labels, object.Labels) - clone.Annotations = mergeMap(clone.Annotations, object.Annotations) - return k8sClient.Update(context.TODO(), clone) - })).NotTo(HaveOccurred()) - - return clone - } - UpdateRollout := func(object *v1alpha1.Rollout) *v1alpha1.Rollout { var clone *v1alpha1.Rollout Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -269,24 +237,6 @@ var _ = SIGDescribe("Rollout", func() { }, 5*time.Minute, time.Second).Should(BeTrue()) } - WaitNativeStatefulSetPodsReady := func(statefulset *apps.StatefulSet) { - Eventually(func() bool { - set := &apps.StatefulSet{} - Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) - return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && - *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas - }, 20*time.Minute, 3*time.Second).Should(BeTrue()) - } - - WaitAdvancedStatefulSetPodsReady := func(statefulset *appsv1beta1.StatefulSet) { - Eventually(func() bool { - set := &appsv1beta1.StatefulSet{} - Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) - return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && - *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas - }, 20*time.Minute, 3*time.Second).Should(BeTrue()) - } - WaitDaemonSetAllPodsReady := func(daemonset *appsv1alpha1.DaemonSet) { Eventually(func() bool { daemon := &appsv1alpha1.DaemonSet{} @@ -386,54 +336,6 @@ var _ = SIGDescribe("Rollout", func() { return apps, nil } - CheckPodBatchLabel := func(namespace string, labelSelector *metav1.LabelSelector, rolloutID, batchID string, expected int) { - pods, err := ListPods(namespace, labelSelector) - Expect(err).NotTo(HaveOccurred()) - - count := 0 - for _, pod := range pods { - if pod.Labels[v1alpha1.RolloutIDLabel] == rolloutID && - pod.Labels[v1alpha1.RolloutBatchIDLabel] == batchID { - count++ - } - } - Expect(count).Should(BeNumerically("==", expected)) - } - - ListReplicaSet := func(d *apps.Deployment) []*apps.ReplicaSet { - var rss []*apps.ReplicaSet - rsLister := &apps.ReplicaSetList{} - selectorOpt, _ := metav1.LabelSelectorAsSelector(d.Spec.Selector) - err := k8sClient.List(context.TODO(), rsLister, &client.ListOptions{LabelSelector: selectorOpt, Namespace: d.Namespace}) - Expect(err).NotTo(HaveOccurred()) - for i := range rsLister.Items { - rs := &rsLister.Items[i] - if !rs.DeletionTimestamp.IsZero() { - continue - } - rss = append(rss, rs) - } - return rss - } - - GetStableRSRevision := func(d *apps.Deployment) string { - rss := ListReplicaSet(d) - _, stable := util.FindCanaryAndStableReplicaSet(rss, d) - if stable != nil { - return stable.Labels[apps.DefaultDeploymentUniqueLabelKey] - } - return "" - } - - GetCanaryRSRevision := func(d *apps.Deployment) string { - rss := ListReplicaSet(d) - canary, _ := util.FindCanaryAndStableReplicaSet(rss, d) - if canary != nil { - return canary.Labels[apps.DefaultDeploymentUniqueLabelKey] - } - return "" - } - BeforeEach(func() { namespace = randomNamespaceName("rollout") ns := v1.Namespace{ @@ -2905,2741 +2807,6 @@ var _ = SIGDescribe("Rollout", func() { }) }) - KruiseDescribe("CloneSet canary rollout with Ingress", func() { - It("CloneSet V1->V2: Percentage, 20%,60% Succeeded", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - } - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress := &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - - // check stable, canary service & ingress - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - - // resume rollout - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitCloneSetAllPodsReady(workload) - By("rollout completed, and check") - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version2")) - } - } - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) - - // scale up replicas 5 -> 6 - workload.Spec.Replicas = utilpointer.Int32(6) - UpdateCloneSet(workload) - By("Update cloneSet replicas from(5) -> to(6)") - time.Sleep(time.Second * 2) - - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - - It("V1->V2: Percentage, 20%, and rollback(v1)", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - // check rollout status - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - time.Sleep(time.Second * 20) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 0)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStateUpgrade)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - time.Sleep(time.Second * 15) - - // rollback -> v1 - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) - workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:latest" - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") - time.Sleep(time.Second * 2) - - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitCloneSetAllPodsReady(workload) - By("rollout completed, and check") - // check progressing canceled - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - Expect(string(cond.Status)).Should(Equal("False")) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress := &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) - cService := &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(stableRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(stableRevision)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version1")) - } - } - }) - - It("Cloneset V1->V2: Percentage, 20%,40% and continuous release v3", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - time.Sleep(time.Second * 15) - - // v1 -> v2 -> v3, continuous release - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - By("Update cloneSet env NODE_NAME from(version2) -> to(version3)") - time.Sleep(time.Second * 10) - - // wait step 0 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - //Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) - //Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - canaryRevisionV2 := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevisionV2)) - // canary ingress - cIngress := &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitCloneSetAllPodsReady(workload) - By("rollout completed, and check") - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevisionV2)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevisionV2)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version3")) - } - } - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - - It("V1->V2: disable quickly rollback policy without traffic routing", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - rollout.Annotations = map[string]string{ - v1alpha1.RollbackInBatchAnnotation: "true", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // workload - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // v1 -> v2 -> v1, continuous release - By("Update cloneSet env NODE_NAME from(version2) -> to(version1)") - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 3) - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - - // make sure CloneSet is rolling back in batch - By("Wait step 1 paused") - WaitRolloutCanaryStepPaused(rollout.Name, 1) - By("Wait step 2 paused") - ResumeRolloutCanary(rollout.Name) - By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitCloneSetAllPodsReady(workload) - - By("rollout completed, and check") - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version1")) - } - } - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - - It("CloneSet V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - workload.Spec.UpdateStrategy.Type = appsv1alpha1.InPlaceOnlyCloneSetUpdateStrategyType - workload.Spec.UpdateStrategy.MaxUnavailable = &intstr.IntOrString{ - Type: intstr.Int, - IntVal: 1, - } - workload.Spec.UpdateStrategy.MaxSurge = nil - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - //newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" - UpdateCloneSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitCloneSetAllPodsReady(workload) - By("rollout completed, and check") - - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) - - // scale up replicas 5 -> 6 - workload.Spec.Replicas = utilpointer.Int32(6) - UpdateCloneSet(workload) - By("Update cloneSet replicas from(5) -> to(6)") - time.Sleep(time.Second * 2) - - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - }) - - KruiseDescribe("StatefulSet canary rollout with Ingress", func() { - - KruiseDescribe("Native StatefulSet rollout canary with Ingress", func() { - It("V1->V2: Percentage, 20%,60% Succeeded", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - } - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // headless service - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &apps.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitNativeStatefulSetPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - By("Update statefulset env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateNativeStatefulSet(workload) - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", *workload.Spec.Replicas-1)) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress := &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - - // check stable, canary service & ingress - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", *workload.Spec.Replicas-3)) - - // resume rollout - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitNativeStatefulSetPodsReady(workload) - By("rollout completed, and check") - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version2")) - } - } - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) - - // scale up replicas 5 -> 6 - workload.Spec.Replicas = utilpointer.Int32(6) - UpdateNativeStatefulSet(workload) - By("Update cloneSet replicas from(5) -> to(6)") - time.Sleep(time.Second * 2) - - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - - It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &apps.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitNativeStatefulSetPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateNativeStatefulSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - time.Sleep(time.Second * 15) - - // v1 -> v2 -> v3, continuous release - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateNativeStatefulSet(workload) - By("Update cloneSet env NODE_NAME from(version2) -> to(version3)") - time.Sleep(time.Second * 10) - - // wait step 0 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - canaryRevisionV2 := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(canaryRevisionV2)) - // canary ingress - cIngress := &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitNativeStatefulSetPodsReady(workload) - By("rollout completed, and check") - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevisionV2)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevisionV2)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version3")) - } - } - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - - It("V1->V2: Percentage, 20%, and rollback(v1)", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // service - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &apps.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitNativeStatefulSetPodsReady(workload) - - // check rollout status - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateNativeStatefulSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - time.Sleep(time.Minute * 2) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStateUpgrade)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - time.Sleep(time.Second * 15) - - // rollback -> v1 - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) - workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:latest" - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateNativeStatefulSet(workload) - By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") - time.Sleep(time.Second * 2) - - // StatefulSet will not remove the broken pod with failed image, we should delete it manually - brokenPod := &v1.Pod{} - Expect(GetObject(fmt.Sprintf("%v-%v", workload.Name, *workload.Spec.Replicas-1), brokenPod)).NotTo(HaveOccurred()) - Expect(k8sClient.Delete(context.TODO(), brokenPod)).NotTo(HaveOccurred()) - - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitNativeStatefulSetPodsReady(workload) - By("rollout completed, and check") - // check progressing canceled - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress := &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal("")) - cService := &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", 0)) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(stableRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(stableRevision)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version1")) - } - } - }) - - It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // headless-service - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &apps.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitNativeStatefulSetPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - //newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" - UpdateNativeStatefulSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitNativeStatefulSetPodsReady(workload) - By("rollout completed, and check") - - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) - - // scale up replicas 5 -> 6 - workload.Spec.Replicas = utilpointer.Int32(6) - UpdateNativeStatefulSet(workload) - By("Update cloneSet replicas from(5) -> to(6)") - time.Sleep(time.Second * 2) - - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - }) - - KruiseDescribe("Advanced StatefulSet rollout canary with Ingress", func() { - It("V1->V2: Percentage, 20%,60% Succeeded", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - } - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1beta1", - Kind: "StatefulSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // headless service - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1beta1.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitAdvancedStatefulSetPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - By("Update statefulset env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateAdvancedStatefulSet(workload) - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", *workload.Spec.Replicas-1)) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress := &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - - // check stable, canary service & ingress - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) - // workload - time.Sleep(time.Second * 10) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", *workload.Spec.Replicas-3)) - - // resume rollout - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitAdvancedStatefulSetPodsReady(workload) - By("rollout completed, and check") - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version2")) - } - } - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) - - // scale up replicas 5 -> 6 - workload.Spec.Replicas = utilpointer.Int32(6) - UpdateAdvancedStatefulSet(workload) - By("Update cloneSet replicas from(5) -> to(6)") - time.Sleep(time.Second * 2) - - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - - It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1beta1", - Kind: "StatefulSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1beta1.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitAdvancedStatefulSetPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateAdvancedStatefulSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - time.Sleep(time.Second * 15) - - // v1 -> v2 -> v3, continuous release - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateAdvancedStatefulSet(workload) - By("Update cloneSet env NODE_NAME from(version2) -> to(version3)") - time.Sleep(time.Second * 10) - - // wait step 0 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - canaryRevisionV2 := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(canaryRevisionV2)) - // canary ingress - cIngress := &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitAdvancedStatefulSetPodsReady(workload) - By("rollout completed, and check") - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevisionV2)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevisionV2)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version3")) - } - } - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - - It("V1->V2: Percentage, 20%, and rollback(v1)", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1beta1", - Kind: "StatefulSet", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // service - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1beta1.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitAdvancedStatefulSetPodsReady(workload) - - // check rollout status - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateAdvancedStatefulSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - time.Sleep(time.Minute * 2) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStateUpgrade)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - time.Sleep(time.Second * 15) - - // rollback -> v1 - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) - workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:latest" - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateAdvancedStatefulSet(workload) - By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") - time.Sleep(time.Second * 2) - - // StatefulSet will not remove the broken pod with failed image, we should delete it manually - brokenPod := &v1.Pod{} - Expect(GetObject(fmt.Sprintf("%v-%v", workload.Name, *workload.Spec.Replicas-1), brokenPod)).NotTo(HaveOccurred()) - Expect(k8sClient.Delete(context.TODO(), brokenPod)).NotTo(HaveOccurred()) - - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitAdvancedStatefulSetPodsReady(workload) - By("rollout completed, and check") - // check progressing canceled - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress := &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal("")) - cService := &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", 0)) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(stableRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(stableRevision)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version1")) - } - } - }) - - It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1beta1", - Kind: "StatefulSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // headless-service - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1beta1.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitAdvancedStatefulSetPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.CanaryStatus.StableRevision - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - //newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" - UpdateAdvancedStatefulSet(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) - canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - - // resume rollout - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitAdvancedStatefulSetPodsReady(workload) - By("rollout completed, and check") - - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) - Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) - - // scale up replicas 5 -> 6 - workload.Spec.Replicas = utilpointer.Int32(6) - UpdateAdvancedStatefulSet(workload) - By("Update cloneSet replicas from(5) -> to(6)") - time.Sleep(time.Second * 2) - - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - }) - }) - - KruiseDescribe("Others", func() { - It("Patch batch id to pods: normal case", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1beta1", - Kind: "StatefulSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // headless-service - headlessService := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) - CreateObject(headlessService) - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1beta1.StatefulSet{} - Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) - workload.Labels[v1alpha1.RolloutIDLabel] = "1" - CreateObject(workload) - WaitAdvancedStatefulSetPodsReady(workload) - - By("Update statefulset env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateAdvancedStatefulSet(workload) - - // wait step 1 complete - By("wait step(1) pause") - WaitRolloutCanaryStepPaused(rollout.Name, 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) - - // resume rollout - ResumeRolloutCanary(rollout.Name) - By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitAdvancedStatefulSetPodsReady(workload) - - // check batch id after rollout - By("rollout completed, and check pod batch label") - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "4", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "5", 1) - }) - - It("patch batch id to pods: scaling case", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(40), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - } - CreateObject(rollout) - By("Creating workload and waiting for all pods ready...") - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - workload.Labels[v1alpha1.RolloutIDLabel] = "1" - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - // v1 -> v2, start rollout action - By("Update cloneset env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - time.Sleep(time.Second * 2) - - // wait step 2 complete - By("wait step(2) pause") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) - - // scale up replicas, 5 -> 10 - By("scaling up CloneSet from 5 -> 10") - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - workload.Spec.Replicas = utilpointer.Int32(10) - UpdateCloneSet(workload) - Eventually(func() bool { - object := &v1alpha1.Rollout{} - Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred()) - return object.Status.CanaryStatus.CanaryReadyReplicas == 4 - }, 5*time.Minute, time.Second).Should(BeTrue()) - - // check pod batch label after scale - By("check pod batch label after scale") - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 3) - - // resume rollout canary - By("check rollout canary status success, resume rollout, and wait rollout canary complete") - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitCloneSetAllPodsReady(workload) - - By("rollout completed, and check pod batch label") - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 3) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 2) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "4", 4) - }) - - It("patch batch id to pods: rollback case", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - rollout.Annotations = map[string]string{ - v1alpha1.RollbackInBatchAnnotation: "true", - } - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(40), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(80), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{ - Duration: utilpointer.Int32(0), - }, - }, - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - workload.Labels[v1alpha1.RolloutIDLabel] = "1" - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - // v1 -> v2, start rollout action - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - - By("wait step(1) pause") - WaitRolloutCanaryStepPaused(rollout.Name, 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - - By("wait step(2) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 2) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) - - By("wait step(3) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 3) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 1) - - By("Update cloneSet env NODE_NAME from(version2) -> to(version1)") - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) - workload.Labels[v1alpha1.RolloutIDLabel] = "2" - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - time.Sleep(10 * time.Second) - - // make sure disable quickly rollback policy - By("Wait step (1) paused") - WaitRolloutCanaryStepPaused(rollout.Name, 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) - - By("wait step(2) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 2) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) - - By("wait step(3) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 3) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) - - By("wait step(4) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 4) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) - - By("Wait rollout complete") - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "5", 1) - }) - - It("patch batch id to pods: only rollout-id changes", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - rollout.Annotations = map[string]string{ - v1alpha1.RollbackInBatchAnnotation: "true", - } - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(40), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(80), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{ - Duration: utilpointer.Int32(0), - }, - }, - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - By("Only update rollout id = '1', and start rollout") - workload.Labels[v1alpha1.RolloutIDLabel] = "1" - workload.Annotations[util.InRolloutProgressingAnnotation] = "true" - UpdateCloneSet(workload) - - By("wait step(1) pause") - WaitRolloutCanaryStepPaused(rollout.Name, 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - - By("wait step(2) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 2) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) - - By("wait step(3) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 3) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 1) - - By("Only update rollout id = '2', and check batch label again") - workload.Labels[v1alpha1.RolloutIDLabel] = "2" - UpdateCloneSet(workload) - - By("wait step(3) pause again") - WaitRolloutCanaryStepPaused(rollout.Name, 3) - time.Sleep(30 * time.Second) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) - - By("wait step(4) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 4) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) - - By("Wait rollout complete") - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "5", 1) - }) - - It("patch batch id to pods: only change rollout-id after rolling the first step", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - rollout.Annotations = map[string]string{ - v1alpha1.RollbackInBatchAnnotation: "true", - } - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(40), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(80), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{ - Duration: utilpointer.Int32(0), - }, - }, - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - By("Only update rollout id = '1', and start rollout") - workload.Labels[v1alpha1.RolloutIDLabel] = "1" - workload.Annotations[util.InRolloutProgressingAnnotation] = "true" - UpdateCloneSet(workload) - - By("wait step(1) pause") - WaitRolloutCanaryStepPaused(rollout.Name, 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - - By("Only update rollout id = '2', and check batch label again") - workload.Labels[v1alpha1.RolloutIDLabel] = "2" - UpdateCloneSet(workload) - - By("wait 30s") - time.Sleep(30 * time.Second) - - By("wait step(1) pause") - WaitRolloutCanaryStepPaused(rollout.Name, 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) - - By("wait step(2) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 2) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) - - By("wait step(3) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 3) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) - - By("wait step(4) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 4) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) - - By("Wait rollout complete") - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "5", 1) - }) - }) - - KruiseDescribe("Test", func() { - It("failure threshold", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps.kruise.io/v1alpha1", - Kind: "CloneSet", - Name: "echoserver", - } - rollout.Spec.Strategy.Canary = &v1alpha1.CanaryStrategy{ - FailureThreshold: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, - Steps: []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(10), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(30), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{}, - }, - }, - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - workload := &appsv1alpha1.CloneSet{} - Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - workload.Spec.Replicas = utilpointer.Int32(10) - workload.Spec.UpdateStrategy.MaxUnavailable = &intstr.IntOrString{Type: intstr.String, StrVal: "100%"} - CreateObject(workload) - WaitCloneSetAllPodsReady(workload) - - checkUpdateReadyPods := func(lower, upper int32) bool { - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - return lower <= workload.Status.UpdatedReadyReplicas && workload.Status.UpdatedReadyReplicas <= upper - } - - By("start rollout") - workload.Labels[v1alpha1.RolloutIDLabel] = "1" - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSet(workload) - - By("wait step(1) pause") - WaitRolloutCanaryStepPaused(rollout.Name, 1) - Expect(checkUpdateReadyPods(1, 1)).Should(BeTrue()) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - - By("wait step(2) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 2) - Expect(checkUpdateReadyPods(2, 3)).Should(BeTrue()) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 2) - - By("wait step(3) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 3) - Expect(checkUpdateReadyPods(4, 6)).Should(BeTrue()) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 3) - - By("wait step(4) pause") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 4) - Expect(checkUpdateReadyPods(8, 10)).Should(BeTrue()) - CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "4", 4) - }) - }) - - KruiseDescribe("Advanced Deployment canary rollout with Ingress", func() { - It("advanced deployment rolling with traffic case", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Annotations = map[string]string{ - v1alpha1.RolloutStyleAnnotation: string(v1alpha1.PartitionRollingStyle), - } - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - } - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - // service - service := &v1.Service{} - Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) - CreateObject(service) - // ingress - ingress := &netv1.Ingress{} - Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) - CreateObject(ingress) - // workload - workload := &apps.Deployment{} - Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitDeploymentAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeployment(workload) - By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - stableRevision := GetStableRSRevision(workload) - By(stableRevision) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - strategy := util.GetDeploymentStrategy(workload) - extraStatus := util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(strategy.Paused).Should(BeFalse()) - By("check cloneSet status & paused success") - - // check rollout status - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) - canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress := &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - - // check stable, canary service & ingress - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - strategy = util.GetDeploymentStrategy(workload) - extraStatus = util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) - Expect(strategy.Paused).Should(BeFalse()) - - // resume rollout - ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitDeploymentAllPodsReady(workload) - By("rollout completed, and check") - - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // cloneset - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version2")) - } - } - time.Sleep(time.Second * 3) - - // check progressing succeed - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) - - // scale up replicas 5 -> 6 - workload.Spec.Replicas = utilpointer.Int32(6) - UpdateDeployment(workload) - By("Update cloneSet replicas from(5) -> to(6)") - time.Sleep(time.Second * 2) - - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - }) - - It("advanced deployment continuous rolling case", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Annotations = map[string]string{ - v1alpha1.RolloutStyleAnnotation: string(v1alpha1.PartitionRollingStyle), - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, - }, - } - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - workload := &apps.Deployment{} - Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitDeploymentAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - By("update workload env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeployment(workload) - - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - stableRevision := GetStableRSRevision(workload) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - strategy := util.GetDeploymentStrategy(workload) - extraStatus := util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(strategy.Paused).Should(BeFalse()) - By("check workload status & paused success") - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - strategy = util.GetDeploymentStrategy(workload) - extraStatus = util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) - Expect(strategy.Paused).Should(BeFalse()) - - By("update workload env NODE_NAME from(version2) -> to(version3)") - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeployment(workload) - - WaitRolloutCanaryStepPaused(rollout.Name, 1) - stableRevision = workload.Labels[v1alpha1.DeploymentStableRevisionLabel] - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - strategy = util.GetDeploymentStrategy(workload) - extraStatus = util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(strategy.Paused).Should(BeFalse()) - By("check workload status & paused success") - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - strategy = util.GetDeploymentStrategy(workload) - extraStatus = util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) - Expect(strategy.Paused).Should(BeFalse()) - }) - - It("advanced deployment rollback case", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Annotations = map[string]string{ - v1alpha1.RolloutStyleAnnotation: string(v1alpha1.PartitionRollingStyle), - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, - }, - } - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - workload := &apps.Deployment{} - Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitDeploymentAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - By("update cloneSet env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeployment(workload) - - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - stableRevision := GetStableRSRevision(workload) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - strategy := util.GetDeploymentStrategy(workload) - extraStatus := util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(strategy.Paused).Should(BeFalse()) - By("check workload status & paused success") - - // resume rollout canary - ResumeRolloutCanary(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutCanaryStepPaused(rollout.Name, 2) - - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - strategy = util.GetDeploymentStrategy(workload) - extraStatus = util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) - Expect(strategy.Paused).Should(BeFalse()) - - By("update workload env NODE_NAME from(version2) -> to(version1)") - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeployment(workload) - - WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) - WaitDeploymentAllPodsReady(workload) - }) - - It("advanced deployment delete rollout case", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Annotations = map[string]string{ - v1alpha1.RolloutStyleAnnotation: string(v1alpha1.PartitionRollingStyle), - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, - }, - } - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - workload := &apps.Deployment{} - Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitDeploymentAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - By("update workload env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeployment(workload) - - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - stableRevision := GetStableRSRevision(workload) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - strategy := util.GetDeploymentStrategy(workload) - extraStatus := util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(strategy.Paused).Should(BeFalse()) - By("check workload status & paused success") - - By("delete rollout and check deployment") - k8sClient.Delete(context.TODO(), rollout) - WaitRolloutNotFound(rollout.Name) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) - Expect(workload.Spec.Paused).Should(BeFalse()) - WaitDeploymentAllPodsReady(workload) - }) - - It("advanced deployment scaling case", func() { - By("Creating Rollout...") - rollout := &v1alpha1.Rollout{} - Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Annotations = map[string]string{ - v1alpha1.RolloutStyleAnnotation: string(v1alpha1.PartitionRollingStyle), - } - rollout.Spec.Strategy.Canary.TrafficRoutings = nil - rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(20), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(60), - }, - Pause: v1alpha1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ - Weight: utilpointer.Int32(100), - }, - Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, - }, - } - rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "echoserver", - } - CreateObject(rollout) - - By("Creating workload and waiting for all pods ready...") - workload := &apps.Deployment{} - Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) - CreateObject(workload) - WaitDeploymentAllPodsReady(workload) - - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) - By("check rollout status & paused success") - - // v1 -> v2, start rollout action - By("update workload env NODE_NAME from(version1) -> to(version2)") - newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeployment(workload) - - // wait step 1 complete - WaitRolloutCanaryStepPaused(rollout.Name, 1) - stableRevision := GetStableRSRevision(workload) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) - strategy := util.GetDeploymentStrategy(workload) - extraStatus := util.GetDeploymentExtraStatus(workload) - Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(strategy.Paused).Should(BeFalse()) - By("check workload status & paused success") - - By("scale up workload from 5 to 10, and check") - workload.Spec.Replicas = utilpointer.Int32(10) - UpdateDeployment(workload) - Eventually(func() bool { - object := &v1alpha1.Rollout{} - Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred()) - return object.Status.CanaryStatus.CanaryReadyReplicas == 2 - }, 5*time.Minute, time.Second).Should(BeTrue()) - - By("scale down workload from 10 to 5, and check") - workload.Spec.Replicas = utilpointer.Int32(5) - UpdateDeployment(workload) - Eventually(func() bool { - object := &v1alpha1.Rollout{} - Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred()) - return object.Status.CanaryStatus.CanaryReadyReplicas == 1 - }, 5*time.Minute, time.Second).Should(BeTrue()) - - By("rolling deployment to be completed") - ResumeRolloutCanary(rollout.Name) - WaitRolloutCanaryStepPaused(rollout.Name, 2) - ResumeRolloutCanary(rollout.Name) - WaitDeploymentAllPodsReady(workload) - }) - }) - KruiseDescribe("DaemonSet canary rollout", func() { It("DaemonSet V1->V2: 1,100% Succeeded", func() { By("Creating Rollout...") diff --git a/test/e2e/rollout_v1beta1_test.go b/test/e2e/rollout_v1beta1_test.go index 31b1c1dc..799b62e0 100644 --- a/test/e2e/rollout_v1beta1_test.go +++ b/test/e2e/rollout_v1beta1_test.go @@ -32,6 +32,7 @@ import ( apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -149,41 +150,41 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // return daemon // } - // UpdateNativeStatefulSet := func(object *apps.StatefulSet) *apps.StatefulSet { - // var clone *apps.StatefulSet - // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { - // clone = &apps.StatefulSet{} - // err := GetObject(object.Name, clone) - // if err != nil { - // return err - // } - // clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) - // clone.Spec.Template = *object.Spec.Template.DeepCopy() - // clone.Labels = mergeMap(clone.Labels, object.Labels) - // clone.Annotations = mergeMap(clone.Annotations, object.Annotations) - // return k8sClient.Update(context.TODO(), clone) - // })).NotTo(HaveOccurred()) + UpdateNativeStatefulSet := func(object *apps.StatefulSet) *apps.StatefulSet { + var clone *apps.StatefulSet + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &apps.StatefulSet{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) - // return clone - // } + return clone + } - // UpdateAdvancedStatefulSet := func(object *appsv1beta1.StatefulSet) *appsv1beta1.StatefulSet { - // var clone *appsv1beta1.StatefulSet - // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { - // clone = &appsv1beta1.StatefulSet{} - // err := GetObject(object.Name, clone) - // if err != nil { - // return err - // } - // clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) - // clone.Spec.Template = *object.Spec.Template.DeepCopy() - // clone.Labels = mergeMap(clone.Labels, object.Labels) - // clone.Annotations = mergeMap(clone.Annotations, object.Annotations) - // return k8sClient.Update(context.TODO(), clone) - // })).NotTo(HaveOccurred()) + UpdateAdvancedStatefulSet := func(object *appsv1beta1.StatefulSet) *appsv1beta1.StatefulSet { + var clone *appsv1beta1.StatefulSet + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &appsv1beta1.StatefulSet{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) - // return clone - // } + return clone + } UpdateRollout := func(object *v1beta1.Rollout) *v1beta1.Rollout { var clone *v1beta1.Rollout @@ -263,23 +264,23 @@ var _ = SIGDescribe("Rollout v1beta1", func() { }, 5*time.Minute, time.Second).Should(BeTrue()) } - // WaitNativeStatefulSetPodsReady := func(statefulset *apps.StatefulSet) { - // Eventually(func() bool { - // set := &apps.StatefulSet{} - // Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) - // return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && - // *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas - // }, 20*time.Minute, 3*time.Second).Should(BeTrue()) - // } + WaitNativeStatefulSetPodsReady := func(statefulset *apps.StatefulSet) { + Eventually(func() bool { + set := &apps.StatefulSet{} + Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) + return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && + *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas + }, 20*time.Minute, 3*time.Second).Should(BeTrue()) + } - // WaitAdvancedStatefulSetPodsReady := func(statefulset *appsv1beta1.StatefulSet) { - // Eventually(func() bool { - // set := &appsv1beta1.StatefulSet{} - // Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) - // return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && - // *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas - // }, 20*time.Minute, 3*time.Second).Should(BeTrue()) - // } + WaitAdvancedStatefulSetPodsReady := func(statefulset *appsv1beta1.StatefulSet) { + Eventually(func() bool { + set := &appsv1beta1.StatefulSet{} + Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) + return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && + *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas + }, 20*time.Minute, 3*time.Second).Should(BeTrue()) + } // WaitDaemonSetAllPodsReady := func(daemonset *appsv1alpha1.DaemonSet) { // Eventually(func() bool { @@ -333,20 +334,20 @@ var _ = SIGDescribe("Rollout v1beta1", func() { }, time.Minute, time.Second).Should(BeTrue()) } - // WaitRolloutNotFound := func(name string) { - // Eventually(func() bool { - // clone := &v1beta1.Rollout{} - // err := GetObject(name, clone) - // if err == nil { - // return false - // } else if errors.IsNotFound(err) { - // return true - // } else { - // Expect(err).NotTo(HaveOccurred()) - // return false - // } - // }, 5*time.Minute, time.Second).Should(BeTrue()) - // } + WaitRolloutNotFound := func(name string) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + err := GetObject(name, clone) + if err == nil { + return false + } else if errors.IsNotFound(err) { + return true + } else { + Expect(err).NotTo(HaveOccurred()) + return false + } + }, 5*time.Minute, time.Second).Should(BeTrue()) + } GetCanaryDeployment := func(stable *apps.Deployment) (*apps.Deployment, error) { canaryList := &apps.DeploymentList{} @@ -363,36 +364,36 @@ var _ = SIGDescribe("Rollout v1beta1", func() { return &canaryList.Items[0], nil } - // ListPods := func(namespace string, labelSelector *metav1.LabelSelector) ([]*v1.Pod, error) { - // appList := &v1.PodList{} - // selector, _ := metav1.LabelSelectorAsSelector(labelSelector) - // err := k8sClient.List(context.TODO(), appList, &client.ListOptions{Namespace: namespace, LabelSelector: selector}) - // if err != nil { - // return nil, err - // } - // apps := make([]*v1.Pod, 0) - // for i := range appList.Items { - // pod := &appList.Items[i] - // if pod.DeletionTimestamp.IsZero() { - // apps = append(apps, pod) - // } - // } - // return apps, nil - // } + ListPods := func(namespace string, labelSelector *metav1.LabelSelector) ([]*v1.Pod, error) { + appList := &v1.PodList{} + selector, _ := metav1.LabelSelectorAsSelector(labelSelector) + err := k8sClient.List(context.TODO(), appList, &client.ListOptions{Namespace: namespace, LabelSelector: selector}) + if err != nil { + return nil, err + } + apps := make([]*v1.Pod, 0) + for i := range appList.Items { + pod := &appList.Items[i] + if pod.DeletionTimestamp.IsZero() { + apps = append(apps, pod) + } + } + return apps, nil + } - // CheckPodBatchLabel := func(namespace string, labelSelector *metav1.LabelSelector, rolloutID, batchID string, expected int) { - // pods, err := ListPods(namespace, labelSelector) - // Expect(err).NotTo(HaveOccurred()) + CheckPodBatchLabel := func(namespace string, labelSelector *metav1.LabelSelector, rolloutID, batchID string, expected int) { + pods, err := ListPods(namespace, labelSelector) + Expect(err).NotTo(HaveOccurred()) - // count := 0 - // for _, pod := range pods { - // if pod.Labels[v1beta1.RolloutIDLabel] == rolloutID && - // pod.Labels[v1beta1.RolloutBatchIDLabel] == batchID { - // count++ - // } - // } - // Expect(count).Should(BeNumerically("==", expected)) - // } + count := 0 + for _, pod := range pods { + if pod.Labels[v1beta1.RolloutIDLabel] == rolloutID && + pod.Labels[v1beta1.RolloutBatchIDLabel] == batchID { + count++ + } + } + Expect(count).Should(BeNumerically("==", expected)) + } ListReplicaSet := func(d *apps.Deployment) []*apps.ReplicaSet { var rss []*apps.ReplicaSet @@ -428,6 +429,51 @@ var _ = SIGDescribe("Rollout v1beta1", func() { return "" } + CheckIngressRestored := func(sname string) { + // stable service + service := &v1.Service{} + Expect(GetObject(sname, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(BeEmpty()) + //canary service + cService := &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).To(HaveOccurred()) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).To(HaveOccurred()) + } + type trafficContext struct { + stableRevision string + canaryRevision string + service *v1.Service + selectorKey string + } + CheckIngressConfigured := func(ctx *trafficContext, step *v1beta1.CanaryStep) { + selectorKey := ctx.selectorKey + if selectorKey == "" { + selectorKey = apps.DefaultDeploymentUniqueLabelKey + } + sname := ctx.service.Name + // stable service + service := &v1.Service{} + Expect(GetObject(sname, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[selectorKey]).Should(Equal(ctx.stableRevision)) + //canary service + cService := &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[selectorKey]).Should(Equal(ctx.canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + if step.Traffic != nil { + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*step.Traffic))) + } + if step.Matches != nil { + By("Fatal: Unimplemented to check match for ingress!") + Expect(false).Should(BeTrue()) + } + + } BeforeEach(func() { namespace = randomNamespaceName("rollout") ns := v1.Namespace{ @@ -830,7 +876,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { cIngress := &netv1.Ingress{} Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) // wait step 2 complete @@ -856,24 +901,11 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) - + // if network configuration has restored + CheckIngressRestored(service.Name) // wait step 3 complete By("wait step(3) pause") ResumeRolloutCanary(rollout.Name) @@ -897,23 +929,11 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + // if network configuration has restored + CheckIngressRestored(service.Name) // wait step 4 complete By("wait step(4) pause") @@ -938,23 +958,11 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 4)) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 5)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[3].Traffic))) + // if network configuration has restored + CheckIngressRestored(service.Name) // Jump to step 3 By("Jump to step 3") @@ -980,71 +988,24 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + // if network configuration has restored + CheckIngressRestored(service.Name) // Change traffic of current step, which shouldn't cause jump By("Change traffic of step 3") Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) // update rollout step configuration - rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("21%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, - Pause: v1beta1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("41%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, - Pause: v1beta1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("61%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, - Pause: v1beta1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("81%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, - Pause: v1beta1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, + rollout.Spec.Strategy.Canary.Steps[2] = v1beta1.CanaryStep{ + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("60%"), }, - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("100%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, - Pause: v1beta1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, + // change to format of integer, so that traffic can be set + Replicas: &intstr.IntOrString{Type: intstr.Int, IntVal: 3}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), }, } rollout = UpdateRollout(rollout) @@ -1115,23 +1076,11 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + // if network configuration has restored + CheckIngressRestored(service.Name) // Jump to step 1 By("Jump to step 1") @@ -1183,16 +1132,8 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // rollout Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", -1)) - // check service & ingress & deployment - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // if network configuration has restored + CheckIngressRestored(service.Name) // deployment Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Spec.Paused).Should(BeFalse()) @@ -1305,114 +1246,55 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + // if network configuration has restored + CheckIngressRestored(service.Name) // wait step 3 complete By("wait step(3) pause") ResumeRolloutCanary(rollout.Name) WaitRolloutCanaryStepPaused(rollout.Name, 3) - // check workload status & paused - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) - By("check cloneSet status & paused success") - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) - Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) - Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) - - // Change traffic of current step, which shouldn't cause jump - By("Change traffic of step 3") - Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // update rollout step configuration - rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("21%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, - Pause: v1beta1.RolloutPause{}, - }, - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("41%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, - Pause: v1beta1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("61%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, - Pause: v1beta1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("81%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, - Pause: v1beta1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - { - TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ - Traffic: utilpointer.StringPtr("100%"), - }, - Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, - Pause: v1beta1.RolloutPause{ - Duration: utilpointer.Int32(10), - }, - }, - } - rollout = UpdateRollout(rollout) - By("update rollout configuration, and wait rollout re-run current step(3)") - time.Sleep(time.Second * 3) - WaitRolloutCanaryStepPaused(rollout.Name, 3) - // batch release - batch := &v1beta1.BatchRelease{} - Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // if network configuration has restored + CheckIngressRestored(service.Name) + + // Change traffic of current step, which shouldn't cause jump + By("Change traffic of step 3") + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // update rollout step configuration + rollout.Spec.Strategy.Canary.Steps[2] = v1beta1.CanaryStep{ + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("60%"), + }, + // change to format of integer, so that traffic can be set + Replicas: &intstr.IntOrString{Type: intstr.Int, IntVal: 3}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + } + rollout = UpdateRollout(rollout) + By("update rollout configuration, and wait rollout re-run current step(3)") + time.Sleep(time.Second * 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // batch release + batch := &v1beta1.BatchRelease{} + Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) @@ -1459,23 +1341,11 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 4)) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 5)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[3].Traffic))) + // if network configuration has restored + CheckIngressRestored(service.Name) // Jump to step 3 By("Jump to step 3") @@ -1527,23 +1397,11 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) - canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService = &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress = &netv1.Ingress{} - Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + // check if network configuration has restored + CheckIngressRestored(service.Name) // Jump to step 1 By("Jump to step 1") @@ -1588,17 +1446,9 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // rollout Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", -1)) - // check service & ingress - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress = &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // clonese + // check if network configuration has restored + CheckIngressRestored(service.Name) + // cloneset Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) @@ -1620,15 +1470,2534 @@ var _ = SIGDescribe("Rollout v1beta1", func() { }) }) -}) -func removePercentageSign(input string) string { - if input == "0" { - return "0" - } - if strings.HasSuffix(input, "%") { - return strings.TrimSuffix(input, "%") - } - fmt.Printf("input(%s) has no percentage sign!", input) - return "" + KruiseDescribe("CloneSet canary rollout with Ingress", func() { + It("CloneSet V1->V2: Percentage, 20%,60% Succeeded", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + } + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + CheckIngressConfigured(&trafficContext{ + stableRevision: stableRevision, + canaryRevision: canaryRevision, + service: service, + }, &rollout.Spec.Strategy.Canary.Steps[0]) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + + // check stable, canary service & ingress + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + By("rollout completed, and check") + + // check if network configuration has restored + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) + + // scale up replicas 5 -> 6 + workload.Spec.Replicas = utilpointer.Int32(6) + UpdateCloneSet(workload) + By("Update cloneSet replicas from(5) -> to(6)") + time.Sleep(time.Second * 2) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("V1->V2: Percentage, 20%, and rollback(v1)", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + time.Sleep(time.Second * 20) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1beta1.CanaryStepStateUpgrade)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // rollback -> v1 + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:latest" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") + time.Sleep(time.Second * 2) + + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + By("rollout completed, and check") + // check progressing canceled + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + Expect(string(cond.Status)).Should(Equal("False")) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(stableRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(stableRevision)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version1")) + } + } + }) + + It("Cloneset V1->V2: Percentage, 20%,40% and continuous release v3", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // v1 -> v2 -> v3, continuous release + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version2) -> to(version3)") + time.Sleep(time.Second * 10) + + // wait step 0 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + //Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) + //Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevisionV2 := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + // check stable, canary service & ingress + CheckIngressConfigured(&trafficContext{ + stableRevision: stableRevision, + canaryRevision: canaryRevisionV2, + service: service, + }, &rollout.Spec.Strategy.Canary.Steps[0]) + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + By("rollout completed, and check") + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevisionV2)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevisionV2)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version3")) + } + } + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("V1->V2: disable quickly rollback policy without traffic routing", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + addAnnotation(rollout, v1beta1.RollbackInBatchAnnotation, "true") + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // v1 -> v2 -> v1, continuous release + By("Update cloneSet env NODE_NAME from(version2) -> to(version1)") + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + + // make sure CloneSet is rolling back in batch + By("Wait step 1 paused") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + By("Wait step 2 paused") + ResumeRolloutCanary(rollout.Name) + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + + By("rollout completed, and check") + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version1")) + } + } + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("CloneSet V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + workload.Spec.UpdateStrategy.Type = appsv1alpha1.InPlaceOnlyCloneSetUpdateStrategyType + workload.Spec.UpdateStrategy.MaxUnavailable = &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + } + workload.Spec.UpdateStrategy.MaxSurge = nil + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + //newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + By("rollout completed, and check") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) + + // scale up replicas 5 -> 6 + workload.Spec.Replicas = utilpointer.Int32(6) + UpdateCloneSet(workload) + By("Update cloneSet replicas from(5) -> to(6)") + time.Sleep(time.Second * 2) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + }) + + KruiseDescribe("Advanced Deployment canary rollout with Ingress", func() { + It("advanced deployment rolling with traffic case", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + } + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision := GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + strategy := util.GetDeploymentStrategy(workload) + extraStatus := util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + CheckIngressConfigured(&trafficContext{ + stableRevision: stableRevision, + canaryRevision: canaryRevision, + service: service, + }, &rollout.Spec.Strategy.Canary.Steps[0]) + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + + // check stable, canary service & ingress + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(strategy.Paused).Should(BeFalse()) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitDeploymentAllPodsReady(workload) + By("rollout completed, and check") + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) + + // scale up replicas 5 -> 6 + workload.Spec.Replicas = utilpointer.Int32(6) + UpdateDeployment(workload) + By("Update cloneSet replicas from(5) -> to(6)") + time.Sleep(time.Second * 2) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("advanced deployment continuous rolling case", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{Duration: utilpointer.Int32(0)}, + }, + } + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + By("update workload env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision := GetStableRSRevision(workload) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + strategy := util.GetDeploymentStrategy(workload) + extraStatus := util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(strategy.Paused).Should(BeFalse()) + By("check workload status & paused success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(strategy.Paused).Should(BeFalse()) + + By("update workload env NODE_NAME from(version2) -> to(version3)") + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision = workload.Labels[v1beta1.DeploymentStableRevisionLabel] + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(strategy.Paused).Should(BeFalse()) + By("check workload status & paused success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(strategy.Paused).Should(BeFalse()) + }) + + It("advanced deployment rollback case", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{Duration: utilpointer.Int32(0)}, + }, + } + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + By("update cloneSet env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision := GetStableRSRevision(workload) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + strategy := util.GetDeploymentStrategy(workload) + extraStatus := util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(strategy.Paused).Should(BeFalse()) + By("check workload status & paused success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(strategy.Paused).Should(BeFalse()) + + By("update workload env NODE_NAME from(version2) -> to(version1)") + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitDeploymentAllPodsReady(workload) + }) + + It("advanced deployment delete rollout case", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{Duration: utilpointer.Int32(0)}, + }, + } + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + By("update workload env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision := GetStableRSRevision(workload) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + strategy := util.GetDeploymentStrategy(workload) + extraStatus := util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(strategy.Paused).Should(BeFalse()) + By("check workload status & paused success") + + By("delete rollout and check deployment") + k8sClient.Delete(context.TODO(), rollout) + WaitRolloutNotFound(rollout.Name) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) + Expect(workload.Spec.Paused).Should(BeFalse()) + WaitDeploymentAllPodsReady(workload) + }) + + It("advanced deployment scaling case", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{Duration: utilpointer.Int32(0)}, + }, + } + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + By("update workload env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision := GetStableRSRevision(workload) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + strategy := util.GetDeploymentStrategy(workload) + extraStatus := util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(strategy.Paused).Should(BeFalse()) + By("check workload status & paused success") + + By("scale up workload from 5 to 10, and check") + workload.Spec.Replicas = utilpointer.Int32(10) + UpdateDeployment(workload) + Eventually(func() bool { + object := &v1beta1.Rollout{} + Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred()) + return object.Status.CanaryStatus.CanaryReadyReplicas == 2 + }, 5*time.Minute, time.Second).Should(BeTrue()) + + By("scale down workload from 10 to 5, and check") + workload.Spec.Replicas = utilpointer.Int32(5) + UpdateDeployment(workload) + Eventually(func() bool { + object := &v1beta1.Rollout{} + Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred()) + return object.Status.CanaryStatus.CanaryReadyReplicas == 1 + }, 5*time.Minute, time.Second).Should(BeTrue()) + + By("rolling deployment to be completed") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + ResumeRolloutCanary(rollout.Name) + WaitDeploymentAllPodsReady(workload) + }) + }) + + KruiseDescribe("Others", func() { + It("Patch batch id to pods: normal case", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless-service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1beta1.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) + workload.Labels[v1beta1.RolloutIDLabel] = "1" + CreateObject(workload) + WaitAdvancedStatefulSetPodsReady(workload) + + By("Update statefulset env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateAdvancedStatefulSet(workload) + + // wait step 1 complete + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitAdvancedStatefulSetPodsReady(workload) + + // check batch id after rollout + By("rollout completed, and check pod batch label") + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "4", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "5", 1) + }) + + It("patch batch id to pods: scaling case", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + } + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + workload.Labels[v1beta1.RolloutIDLabel] = "1" + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // v1 -> v2, start rollout action + By("Update cloneset env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + time.Sleep(time.Second * 2) + + // wait step 2 complete + By("wait step(2) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) + + // scale up replicas, 5 -> 10 + By("scaling up CloneSet from 5 -> 10") + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + workload.Spec.Replicas = utilpointer.Int32(10) + UpdateCloneSet(workload) + Eventually(func() bool { + object := &v1beta1.Rollout{} + Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred()) + return object.Status.CanaryStatus.CanaryReadyReplicas == 4 + }, 5*time.Minute, time.Second).Should(BeTrue()) + + // check pod batch label after scale + By("check pod batch label after scale") + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 3) + + // resume rollout canary + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + + By("rollout completed, and check pod batch label") + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 3) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 2) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "4", 4) + }) + + It("patch batch id to pods: rollback case", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + addAnnotation(rollout, v1beta1.RollbackInBatchAnnotation, "true") + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{Duration: utilpointer.Int32(0)}, + }, + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + workload.Labels[v1beta1.RolloutIDLabel] = "1" + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // v1 -> v2, start rollout action + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) + + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 1) + + By("Update cloneSet env NODE_NAME from(version2) -> to(version1)") + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Labels[v1beta1.RolloutIDLabel] = "2" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + time.Sleep(10 * time.Second) + + // make sure disable quickly rollback policy + By("Wait step (1) paused") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) + + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) + + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) + + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) + + By("Wait rollout complete") + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "5", 1) + }) + + It("patch batch id to pods: only rollout-id changes", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + addAnnotation(rollout, v1beta1.RollbackInBatchAnnotation, "true") + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{Duration: utilpointer.Int32(0)}, + }, + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + By("Only update rollout id = '1', and start rollout") + workload.Labels[v1beta1.RolloutIDLabel] = "1" + workload.Annotations[util.InRolloutProgressingAnnotation] = "true" + UpdateCloneSet(workload) + + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 1) + + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 1) + + By("Only update rollout id = '2', and check batch label again") + workload.Labels[v1beta1.RolloutIDLabel] = "2" + UpdateCloneSet(workload) + + By("wait step(3) pause again") + WaitRolloutCanaryStepPaused(rollout.Name, 3) + time.Sleep(30 * time.Second) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) + + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) + + By("Wait rollout complete") + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "5", 1) + }) + + It("patch batch id to pods: only change rollout-id after rolling the first step", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + addAnnotation(rollout, v1beta1.RollbackInBatchAnnotation, "true") + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{Duration: utilpointer.Int32(0)}, + }, + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + By("Only update rollout id = '1', and start rollout") + workload.Labels[v1beta1.RolloutIDLabel] = "1" + workload.Annotations[util.InRolloutProgressingAnnotation] = "true" + UpdateCloneSet(workload) + + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + + By("Only update rollout id = '2', and check batch label again") + workload.Labels[v1beta1.RolloutIDLabel] = "2" + UpdateCloneSet(workload) + + By("wait 30s") + time.Sleep(30 * time.Second) + + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) + + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) + + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) + + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) + + By("Wait rollout complete") + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "4", 1) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "5", 1) + }) + }) + + KruiseDescribe("StatefulSet canary rollout with Ingress", func() { + KruiseDescribe("Native StatefulSet rollout canary with Ingress", func() { + It("V1->V2: Percentage, 20%,60% Succeeded", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + } + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitNativeStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + By("Update statefulset env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateNativeStatefulSet(workload) + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", *workload.Spec.Replicas-1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + CheckIngressConfigured(&trafficContext{ + stableRevision: stableRevision, + canaryRevision: canaryRevision, + service: service, + selectorKey: apps.ControllerRevisionHashLabelKey, + }, &rollout.Spec.Strategy.Canary.Steps[0]) + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + + // check stable, canary service & ingress + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", *workload.Spec.Replicas-3)) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitNativeStatefulSetPodsReady(workload) + By("rollout completed, and check") + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) + + // scale up replicas 5 -> 6 + workload.Spec.Replicas = utilpointer.Int32(6) + UpdateNativeStatefulSet(workload) + By("Update cloneSet replicas from(5) -> to(6)") + time.Sleep(time.Second * 2) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitNativeStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateNativeStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // v1 -> v2 -> v3, continuous release + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateNativeStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version2) -> to(version3)") + time.Sleep(time.Second * 10) + + // wait step 0 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + canaryRevisionV2 := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + // check stable, canary service & ingress + CheckIngressConfigured(&trafficContext{ + stableRevision: stableRevision, + canaryRevision: canaryRevisionV2, + service: service, + selectorKey: apps.ControllerRevisionHashLabelKey, + }, &rollout.Spec.Strategy.Canary.Steps[0]) + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitNativeStatefulSetPodsReady(workload) + By("rollout completed, and check") + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevisionV2)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevisionV2)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version3")) + } + } + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("V1->V2: Percentage, 20%, and rollback(v1)", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitNativeStatefulSetPodsReady(workload) + + // check rollout status + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateNativeStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + time.Sleep(time.Minute * 2) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1beta1.CanaryStepStateUpgrade)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // rollback -> v1 + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:latest" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateNativeStatefulSet(workload) + By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") + time.Sleep(time.Second * 2) + + // StatefulSet will not remove the broken pod with failed image, we should delete it manually + brokenPod := &v1.Pod{} + Expect(GetObject(fmt.Sprintf("%v-%v", workload.Name, *workload.Spec.Replicas-1), brokenPod)).NotTo(HaveOccurred()) + Expect(k8sClient.Delete(context.TODO(), brokenPod)).NotTo(HaveOccurred()) + + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitNativeStatefulSetPodsReady(workload) + By("rollout completed, and check") + // check progressing canceled + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", 0)) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(stableRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(stableRevision)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version1")) + } + } + }) + + It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless-service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitNativeStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + //newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" + UpdateNativeStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitNativeStatefulSetPodsReady(workload) + By("rollout completed, and check") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) + + // scale up replicas 5 -> 6 + workload.Spec.Replicas = utilpointer.Int32(6) + UpdateNativeStatefulSet(workload) + By("Update cloneSet replicas from(5) -> to(6)") + time.Sleep(time.Second * 2) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + }) + + KruiseDescribe("Advanced StatefulSet rollout canary with Ingress", func() { + It("V1->V2: Percentage, 20%,60% Succeeded", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("20%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + } + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1beta1.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitAdvancedStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + By("Update statefulset env NODE_NAME from(version1) -> to(version2)") + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateAdvancedStatefulSet(workload) + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", *workload.Spec.Replicas-1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + CheckIngressConfigured(&trafficContext{ + stableRevision: stableRevision, + canaryRevision: canaryRevision, + service: service, + selectorKey: apps.ControllerRevisionHashLabelKey, + }, &rollout.Spec.Strategy.Canary.Steps[0]) + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutCanaryStepPaused(rollout.Name, 2) + + // check stable, canary service & ingress + CheckIngressRestored(service.Name) + // workload + time.Sleep(time.Second * 10) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", *workload.Spec.Replicas-3)) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitAdvancedStatefulSetPodsReady(workload) + By("rollout completed, and check") + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) + + // scale up replicas 5 -> 6 + workload.Spec.Replicas = utilpointer.Int32(6) + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet replicas from(5) -> to(6)") + time.Sleep(time.Second * 2) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1beta1.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitAdvancedStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // v1 -> v2 -> v3, continuous release + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version2) -> to(version3)") + time.Sleep(time.Second * 10) + + // wait step 0 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + canaryRevisionV2 := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + // check stable, canary service & ingress + CheckIngressConfigured(&trafficContext{ + stableRevision: stableRevision, + canaryRevision: canaryRevisionV2, + service: service, + selectorKey: apps.ControllerRevisionHashLabelKey, + }, &rollout.Spec.Strategy.Canary.Steps[0]) + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitAdvancedStatefulSetPodsReady(workload) + By("rollout completed, and check") + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevisionV2)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevisionV2)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version3")) + } + } + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("V1->V2: Percentage, 20%, and rollback(v1)", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1beta1.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitAdvancedStatefulSetPodsReady(workload) + + // check rollout status + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + time.Sleep(time.Minute * 2) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1beta1.CanaryStepStateUpgrade)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // rollback -> v1 + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:latest" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateAdvancedStatefulSet(workload) + By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") + time.Sleep(time.Second * 2) + + // StatefulSet will not remove the broken pod with failed image, we should delete it manually + brokenPod := &v1.Pod{} + Expect(GetObject(fmt.Sprintf("%v-%v", workload.Name, *workload.Spec.Replicas-1), brokenPod)).NotTo(HaveOccurred()) + Expect(k8sClient.Delete(context.TODO(), brokenPod)).NotTo(HaveOccurred()) + + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitAdvancedStatefulSetPodsReady(workload) + By("rollout completed, and check") + // check progressing canceled + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + + // check service & ingress & deployment + CheckIngressRestored(service.Name) + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", 0)) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(stableRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(stableRevision)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version1")) + } + } + }) + + It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless-service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1beta1.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitAdvancedStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + //newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitAdvancedStatefulSetPodsReady(workload) + By("rollout completed, and check") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + //Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) + + // scale up replicas 5 -> 6 + workload.Spec.Replicas = utilpointer.Int32(6) + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet replicas from(5) -> to(6)") + time.Sleep(time.Second * 2) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + }) + }) + // seems the next never be tested + KruiseDescribe("Test", func() { + It("failure threshold", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary = &v1beta1.CanaryStrategy{ + FailureThreshold: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Steps: []v1beta1.CanaryStep{ + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "10%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "30%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{}, + }, + }, + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + workload.Spec.Replicas = utilpointer.Int32(10) + workload.Spec.UpdateStrategy.MaxUnavailable = &intstr.IntOrString{Type: intstr.String, StrVal: "100%"} + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + checkUpdateReadyPods := func(lower, upper int32) bool { + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + return lower <= workload.Status.UpdatedReadyReplicas && workload.Status.UpdatedReadyReplicas <= upper + } + + By("start rollout") + workload.Labels[v1beta1.RolloutIDLabel] = "1" + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + Expect(checkUpdateReadyPods(1, 1)).Should(BeTrue()) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) + + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + Expect(checkUpdateReadyPods(2, 3)).Should(BeTrue()) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "2", 2) + + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + Expect(checkUpdateReadyPods(4, 6)).Should(BeTrue()) + CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 3) + + // By("wait step(4) pause") + // ResumeRolloutCanary(rollout.Name) + // WaitRolloutCanaryStepPaused(rollout.Name, 4) + // Expect(checkUpdateReadyPods(8, 10)).Should(BeTrue()) + // CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "4", 4) + }) + }) + +}) + +func removePercentageSign(input string) string { + if input == "0" { + return "0" + } + if strings.HasSuffix(input, "%") { + return strings.TrimSuffix(input, "%") + } + fmt.Printf("input(%s) has no percentage sign!", input) + return "" +} + +func addAnnotation(obj metav1.Object, key, value string) { + if obj.GetAnnotations() == nil { + obj.SetAnnotations(make(map[string]string)) + } + obj.GetAnnotations()[key] = value } diff --git a/test/e2e/test_data/rollout/rollout_canary_base.yaml b/test/e2e/test_data/rollout/rollout_canary_base.yaml index fb60df39..ad461ccd 100644 --- a/test/e2e/test_data/rollout/rollout_canary_base.yaml +++ b/test/e2e/test_data/rollout/rollout_canary_base.yaml @@ -1,3 +1,5 @@ +# we recommend that new test cases or modifications to existing test cases should +# use v1beta1 Rollout, eg. use rollout_v1beta1_canary_base.yaml apiVersion: rollouts.kruise.io/v1alpha1 kind: Rollout metadata: diff --git a/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml b/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml index 2b492ea7..ce3f4498 100644 --- a/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml +++ b/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml @@ -14,17 +14,13 @@ spec: - traffic: 20% replicas: 20% pause: {} - - traffic: 40% - replicas: 40% + - replicas: 40% pause: {duration: 10} - - traffic: 60% - replicas: 60% + - replicas: 60% pause: {duration: 10} - - traffic: 80% - replicas: 80% + - replicas: 80% pause: {duration: 10} - - traffic: 100% - replicas: 100% + - replicas: 100% pause: {duration: 0} trafficRoutings: - service: echoserver