From bb518724e410d0c3ac28c6a3653cb9b66890456d Mon Sep 17 00:00:00 2001 From: yunbo Date: Thu, 12 Sep 2024 11:12:10 +0800 Subject: [PATCH] support bluegreen release: webhook update Signed-off-by: yunbo --- api/v1alpha1/conversion.go | 4 +- config/rbac/role.yaml | 20 ++ config/webhook/manifests.yaml | 102 -------- config/webhook/patch_manifests.yaml | 54 ++-- main.go | 1 + .../batchrelease/batchrelease_controller.go | 1 + pkg/webhook/server.go | 1 + pkg/webhook/util/revision/revision.go | 242 ++++++++++++++++++ pkg/webhook/util/writer/fs.go | 2 +- pkg/webhook/workload/mutating/webhooks.go | 17 +- .../mutating/workload_update_handler.go | 71 ++++- .../mutating/workload_update_handler_test.go | 146 ++++++++++- 12 files changed, 512 insertions(+), 149 deletions(-) create mode 100644 pkg/webhook/util/revision/revision.go diff --git a/api/v1alpha1/conversion.go b/api/v1alpha1/conversion.go index 28ad4a9c..442218ff 100644 --- a/api/v1alpha1/conversion.go +++ b/api/v1alpha1/conversion.go @@ -172,7 +172,9 @@ func (dst *Rollout) ConvertFrom(src conversion.Hub) error { srcV1beta1 := src.(*v1beta1.Rollout) dst.ObjectMeta = srcV1beta1.ObjectMeta if !srcV1beta1.Spec.Strategy.IsCanaryStragegy() { - return fmt.Errorf("v1beta1 Rollout with %s strategy cannot be converted to v1alpha1", srcV1beta1.Spec.Strategy.GetRollingStyle()) + // only v1beta1 supports bluegreen strategy + // Don't log the message because it will print too often + return nil } // spec dst.Spec = RolloutSpec{ diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d5ed5866..fba7b6a4 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -44,6 +44,14 @@ rules: - patch - update - watch +- apiGroups: + - apps + resources: + - controllerrevisions + verbs: + - get + - list + - watch - apiGroups: - apps resources: @@ -162,6 +170,18 @@ rules: - get - patch - update +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 64ccd607..845e3f17 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -6,108 +6,6 @@ metadata: creationTimestamp: null name: mutating-webhook-configuration webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-apps-kruise-io-v1alpha1-cloneset - failurePolicy: Fail - name: mcloneset.kb.io - rules: - - apiGroups: - - apps.kruise.io - apiVersions: - - v1alpha1 - operations: - - UPDATE - resources: - - clonesets - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-apps-kruise-io-v1alpha1-daemonset - failurePolicy: Fail - name: mdaemonset.kb.io - rules: - - apiGroups: - - apps.kruise.io - apiVersions: - - v1alpha1 - operations: - - UPDATE - resources: - - daemonsets - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-apps-v1-deployment - failurePolicy: Fail - name: mdeployment.kb.io - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-apps-v1-statefulset - failurePolicy: Fail - name: mstatefulset.kb.io - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-apps-kruise-io-statefulset - failurePolicy: Fail - name: madvancedstatefulset.kb.io - rules: - - apiGroups: - - apps.kruise.io - apiVersions: - - v1alpha1 - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/config/webhook/patch_manifests.yaml b/config/webhook/patch_manifests.yaml index 5022bcab..4ad163cc 100644 --- a/config/webhook/patch_manifests.yaml +++ b/config/webhook/patch_manifests.yaml @@ -4,31 +4,6 @@ metadata: name: mutating-webhook-configuration webhooks: - name: munifiedworload.kb.io - objectSelector: - matchExpressions: - - key: rollouts.kruise.io/workload-type - operator: Exists - - name: mcloneset.kb.io - objectSelector: - matchExpressions: - - key: rollouts.kruise.io/workload-type - operator: Exists - - name: mdaemonset.kb.io - objectSelector: - matchExpressions: - - key: rollouts.kruise.io/workload-type - operator: Exists - - name: mstatefulset.kb.io - objectSelector: - matchExpressions: - - key: rollouts.kruise.io/workload-type - operator: Exists - - name: madvancedstatefulset.kb.io - objectSelector: - matchExpressions: - - key: rollouts.kruise.io/workload-type - operator: Exists - - name: mdeployment.kb.io objectSelector: matchExpressions: - key: control-plane @@ -37,3 +12,32 @@ webhooks: - controller-manager - key: rollouts.kruise.io/workload-type operator: Exists + # - name: mcloneset.kb.io + # objectSelector: + # matchExpressions: + # - key: rollouts.kruise.io/workload-type + # operator: Exists + # - name: mdaemonset.kb.io + # objectSelector: + # matchExpressions: + # - key: rollouts.kruise.io/workload-type + # operator: Exists + # - name: mstatefulset.kb.io + # objectSelector: + # matchExpressions: + # - key: rollouts.kruise.io/workload-type + # operator: Exists + # - name: madvancedstatefulset.kb.io + # objectSelector: + # matchExpressions: + # - key: rollouts.kruise.io/workload-type + # operator: Exists + # - name: mdeployment.kb.io + # objectSelector: + # matchExpressions: + # - key: control-plane + # operator: NotIn + # values: + # - controller-manager + # - key: rollouts.kruise.io/workload-type + # operator: Exists diff --git a/main.go b/main.go index cb5a409d..6be2ec42 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(kruisev1aplphal1.AddToScheme(clientgoscheme.Scheme)) utilruntime.Must(kruisev1aplphal1.AddToScheme(scheme)) utilruntime.Must(kruisev1beta1.AddToScheme(scheme)) utilruntime.Must(rolloutapi.AddToScheme(scheme)) diff --git a/pkg/controller/batchrelease/batchrelease_controller.go b/pkg/controller/batchrelease/batchrelease_controller.go index 2eaf2faa..1cfcdbbb 100644 --- a/pkg/controller/batchrelease/batchrelease_controller.go +++ b/pkg/controller/batchrelease/batchrelease_controller.go @@ -148,6 +148,7 @@ type BatchReleaseReconciler struct { // +kubebuilder:rbac:groups=apps.kruise.io,resources=statefulsets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=apps.kruise.io,resources=daemonsets,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=apps.kruise.io,resources=daemonsets/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete // Reconcile reads that state of the cluster for a Rollout object and makes changes based on the state read // and what is in the Rollout.Spec diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index 3a9c9e11..03484243 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -105,6 +105,7 @@ func SetupWithManager(mgr manager.Manager) error { // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list;watch func initialize(ctx context.Context, cfg *rest.Config) error { c, err := webhookcontroller.New(cfg, HandlerMap) diff --git a/pkg/webhook/util/revision/revision.go b/pkg/webhook/util/revision/revision.go new file mode 100644 index 00000000..d22b8ad3 --- /dev/null +++ b/pkg/webhook/util/revision/revision.go @@ -0,0 +1,242 @@ +package revision + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "reflect" + "unsafe" + + kruiseappsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" + appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + patchCodec = clientgoscheme.Codecs.LegacyCodec(kruiseappsv1alpha1.SchemeGroupVersion) +) + +func IsContinuousRelease(cli client.Client, oldObj, newObj *kruiseappsv1alpha1.CloneSet) bool { + // if it has not in rollout progressing, just return false + if _, ok := oldObj.Annotations[util.InRolloutProgressingAnnotation]; !ok { + klog.InfoS("not in rollout progressing", "cloneSet", newObj.Name) + return false + } + newSepc, err := getPatch(newObj) + if err != nil || len(newSepc) == 0 { + klog.ErrorS(err, "Error getting patch for newObj", "cloneSet", newObj.Name) + return false + } + + oldSepc, err := GetStablePodTemplate(cli, oldObj) + //REVIEW - error handling + if err != nil || len(oldSepc) == 0 { + klog.ErrorS(err, "Error getting patch for oldObj", "cloneSet", oldObj.Name) + return false + } + // if equal, it means rollback + if bytes.Equal(oldSepc, newSepc) { + klog.InfoS("webook detected rollback: the old Spec and new Spec are equal", "oldSpec", oldSepc, "newSpec", newSepc) + return false + } + klog.InfoS("webook detected continuous release: the old Spec and new Spec are not equal", "oldSpec", oldSepc, "newSpec", newSepc) + return true +} +func GetStablePodTemplate(cli client.Client, cs *kruiseappsv1alpha1.CloneSet) ([]byte, error) { + cr, err := GetCurrentRevision(cli, cs) + if err != nil || cr == nil { + klog.ErrorS(err, "Error getting revision", "revision", cr.Name) + return []byte{}, err + } + if len(cr.Data.Raw) == 0 { + err = fmt.Errorf("revision %s Data.Raw is empty", cr.Name) + klog.ErrorS(err, "Error getting revision Data.Raw", "revision", cr.Name) + return []byte{}, err + } + klog.Infof("revision GetStablePodTemplate: %s", cr.Data.Raw) + return cr.Data.Raw, nil +} + +func GetCurrentRevision(cli client.Client, cs *kruiseappsv1alpha1.CloneSet) (*apps.ControllerRevision, error) { + currRevision := cs.Status.CurrentRevision + if currRevision == "" { + return nil, nil + } + selector, err := validatedLabelSelectorAsSelector(cs.Spec.Selector) + if err != nil { + klog.ErrorS(err, "Error converting CloneSet selector", "cloneSet") + return nil, err + } + revisions, err := ListControllerRevisions(cli, cs, selector) + if err != nil { + klog.ErrorS(err, "Error listing revisions", "cloneSet", cs.Name) + return nil, err + } + // find revision by currRevision + for i := range revisions { + if revisions[i].Name == currRevision { + return revisions[i], nil + } + } + klog.ErrorS(err, "Error getting revision", "revision", currRevision) + return nil, nil +} + +func GetStableReplicaSet(cli client.Client, OldObj *apps.Deployment, newObj *apps.Deployment) (*apps.ReplicaSet, error) { + if stableRevision := OldObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel]; len(stableRevision) != 0 { + stableRSLabel := map[string]string{ + apps.DefaultDeploymentUniqueLabelKey: stableRevision, + } + return getReplicaSetByLabel(cli, newObj, &stableRSLabel) + } + return nil, fmt.Errorf("annotation %s is empty", appsv1alpha1.DeploymentStableRevisionLabel) +} + +// get the replicaSet by pod-template-hash label +func getReplicaSetByLabel(cli client.Client, obj *apps.Deployment, rsLable *map[string]string) (*apps.ReplicaSet, error) { + rss := &apps.ReplicaSetList{} + listOpts := []client.ListOption{ + client.InNamespace(obj.Namespace), + client.MatchingLabels(obj.Spec.Selector.MatchLabels), + client.MatchingLabels(*rsLable), + } + + if err := cli.List(context.TODO(), rss, listOpts...); err != nil { + klog.Warningf("get Repliaset by label failed, because"+"%s", err.Error()) + return nil, err + } + + allRSs := rss.Items + // select rs owner by current deployment + ownedRSs := make([]*apps.ReplicaSet, 0) + for i := range allRSs { + rs := &allRSs[i] + if !rs.DeletionTimestamp.IsZero() { + continue + } + + if metav1.IsControlledBy(rs, obj) { + ownedRSs = append(ownedRSs, rs) + } + } + if len(ownedRSs) != 1 { + klog.Warningf("amount of stable replicaset for deployment %s/%s is not 1", obj.Namespace, obj.Name) + return nil, nil + } + return ownedRSs[0], nil +} + +// ******************the following code is copied from open kruise****************** +func ListControllerRevisions(cli client.Client, parent metav1.Object, selector labels.Selector) ([]*apps.ControllerRevision, error) { + // List all revisions in the namespace that match the selector + revisions := apps.ControllerRevisionList{} + err := cli.List(context.TODO(), &revisions, &client.ListOptions{Namespace: parent.GetNamespace(), LabelSelector: selector}) + if err != nil { + return nil, err + } + var owned []*apps.ControllerRevision + for i := range revisions.Items { + ref := metav1.GetControllerOf(&revisions.Items[i]) + if ref == nil || ref.UID == parent.GetUID() { + owned = append(owned, &revisions.Items[i]) + } + } + return owned, err +} + +func validatedLabelSelectorAsSelector(ps *metav1.LabelSelector) (labels.Selector, error) { + if ps == nil { + return labels.Nothing(), nil + } + if len(ps.MatchLabels)+len(ps.MatchExpressions) == 0 { + return labels.Everything(), nil + } + + selector := labels.NewSelector() + for k, v := range ps.MatchLabels { + r, err := newRequirement(k, selection.Equals, []string{v}) + if err != nil { + return nil, err + } + selector = selector.Add(*r) + } + for _, expr := range ps.MatchExpressions { + var op selection.Operator + switch expr.Operator { + case metav1.LabelSelectorOpIn: + op = selection.In + case metav1.LabelSelectorOpNotIn: + op = selection.NotIn + case metav1.LabelSelectorOpExists: + op = selection.Exists + case metav1.LabelSelectorOpDoesNotExist: + op = selection.DoesNotExist + default: + return nil, fmt.Errorf("%q is not a valid pod selector operator", expr.Operator) + } + r, err := newRequirement(expr.Key, op, append([]string(nil), expr.Values...)) + if err != nil { + return nil, err + } + selector = selector.Add(*r) + } + return selector, nil +} + +func newRequirement(key string, op selection.Operator, vals []string) (*labels.Requirement, error) { + sel := &labels.Requirement{} + selVal := reflect.ValueOf(sel) + val := reflect.Indirect(selVal) + + keyField := val.FieldByName("key") + keyFieldPtr := (*string)(unsafe.Pointer(keyField.UnsafeAddr())) + *keyFieldPtr = key + + opField := val.FieldByName("operator") + opFieldPtr := (*selection.Operator)(unsafe.Pointer(opField.UnsafeAddr())) + *opFieldPtr = op + + if len(vals) > 0 { + valuesField := val.FieldByName("strValues") + valuesFieldPtr := (*[]string)(unsafe.Pointer(valuesField.UnsafeAddr())) + *valuesFieldPtr = vals + } + + return sel, nil +} + +// getPatch returns a strategic merge patch that can be applied to restore a CloneSet to a +// previous version. If the returned error is nil the patch is valid. The current state that we save is just the +// PodSpecTemplate. We can modify this later to encompass more state (or less) and remain compatible with previously +// recorded patches. +func getPatch(cs *kruiseappsv1alpha1.CloneSet) ([]byte, error) { + str, err := runtime.Encode(patchCodec, cs) + if err != nil { + return nil, err + } + var raw map[string]interface{} + _ = json.Unmarshal(str, &raw) + objCopy := make(map[string]interface{}) + specCopy := make(map[string]interface{}) + spec := raw["spec"].(map[string]interface{}) + template := spec["template"].(map[string]interface{}) + + SetRevisionTemplate(specCopy, template) + objCopy["spec"] = specCopy + patch, err := json.Marshal(objCopy) + return patch, err +} + +func SetRevisionTemplate(revisionSpec map[string]interface{}, template map[string]interface{}) { + revisionSpec["template"] = template + template["$patch"] = "replace" +} diff --git a/pkg/webhook/util/writer/fs.go b/pkg/webhook/util/writer/fs.go index c003caa7..543e94f3 100644 --- a/pkg/webhook/util/writer/fs.go +++ b/pkg/webhook/util/writer/fs.go @@ -128,7 +128,7 @@ func prepareToWrite(dir string) error { // TODO: figure out if we can reduce the permission. (Now it's 0777) err = os.MkdirAll(dir, 0777) if err != nil { - return fmt.Errorf("can't create dir: %v", dir) + return fmt.Errorf("can't create dir: %v, err: %s", dir, err.Error()) } case err != nil: return err diff --git a/pkg/webhook/workload/mutating/webhooks.go b/pkg/webhook/workload/mutating/webhooks.go index 6ff5d4b2..51079aaf 100644 --- a/pkg/webhook/workload/mutating/webhooks.go +++ b/pkg/webhook/workload/mutating/webhooks.go @@ -20,21 +20,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -// +kubebuilder:webhook:path=/mutate-apps-kruise-io-v1alpha1-cloneset,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.kruise.io,resources=clonesets,verbs=update,versions=v1alpha1,name=mcloneset.kb.io -// +kubebuilder:webhook:path=/mutate-apps-kruise-io-v1alpha1-daemonset,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.kruise.io,resources=daemonsets,verbs=update,versions=v1alpha1,name=mdaemonset.kb.io -// +kubebuilder:webhook:path=/mutate-apps-v1-deployment,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps,resources=deployments,verbs=update,versions=v1,name=mdeployment.kb.io -// +kubebuilder:webhook:path=/mutate-apps-v1-statefulset,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps,resources=statefulsets,verbs=update,versions=v1,name=mstatefulset.kb.io -// +kubebuilder:webhook:path=/mutate-apps-kruise-io-statefulset,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.kruise.io,resources=statefulsets,verbs=create;update,versions=v1alpha1;v1beta1,name=madvancedstatefulset.kb.io // +kubebuilder:webhook:path=/mutate-unified-workload,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=*,resources=*,verbs=create;update,versions=*,name=munifiedworload.kb.io var ( // HandlerMap contains admission webhook handlers HandlerMap = map[string]admission.Handler{ - "mutate-apps-kruise-io-v1alpha1-cloneset": &WorkloadHandler{}, - "mutate-apps-v1-deployment": &WorkloadHandler{}, - "mutate-apps-v1-statefulset": &WorkloadHandler{}, - "mutate-apps-kruise-io-statefulset": &WorkloadHandler{}, - "mutate-unified-workload": &WorkloadHandler{}, - "mutate-apps-kruise-io-v1alpha1-daemonset": &WorkloadHandler{}, + // "mutate-apps-kruise-io-v1alpha1-cloneset": &WorkloadHandler{}, + // "mutate-apps-v1-deployment": &WorkloadHandler{}, + // "mutate-apps-v1-statefulset": &WorkloadHandler{}, + // "mutate-apps-kruise-io-statefulset": &WorkloadHandler{}, + "mutate-unified-workload": &WorkloadHandler{}, + // "mutate-apps-kruise-io-v1alpha1-daemonset": &WorkloadHandler{}, } ) diff --git a/pkg/webhook/workload/mutating/workload_update_handler.go b/pkg/webhook/workload/mutating/workload_update_handler.go index 45ade337..61f37bd9 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler.go +++ b/pkg/webhook/workload/mutating/workload_update_handler.go @@ -19,6 +19,7 @@ package mutating import ( "context" "encoding/json" + "fmt" "math" "net/http" "strings" @@ -30,6 +31,7 @@ import ( utilclient "github.com/openkruise/rollouts/pkg/util/client" util2 "github.com/openkruise/rollouts/pkg/webhook/util" "github.com/openkruise/rollouts/pkg/webhook/util/configuration" + "github.com/openkruise/rollouts/pkg/webhook/util/revision" admissionv1 "k8s.io/api/admission/v1" v1 "k8s.io/api/admissionregistration/v1" apps "k8s.io/api/apps/v1" @@ -260,16 +262,18 @@ func (h *WorkloadHandler) handleStatefulSetLikeWorkload(newObj, oldObj *unstruct } func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (bool, error) { + klog.Info("webhook handles Deployment") // in rollout progressing if newObj.Annotations[util.InRolloutProgressingAnnotation] != "" { + klog.InfoS("webhook handles Deployment, in rollout progressing") modified := false - if !newObj.Spec.Paused { - modified = true - newObj.Spec.Paused = true - } strategy := util.GetDeploymentStrategy(newObj) - switch strings.ToLower(string(strategy.RollingStyle)) { - case strings.ToLower(string(appsv1alpha1.PartitionRollingStyle)): + // partition + if strings.EqualFold(string(strategy.RollingStyle), string(appsv1alpha1.PartitionRollingStyle)) { + if !newObj.Spec.Paused { + modified = true + newObj.Spec.Paused = true + } // Make sure it is always Recreate to disable native controller if newObj.Spec.Strategy.Type == apps.RollingUpdateDeploymentStrategyType { modified = true @@ -287,7 +291,44 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo } appsv1alpha1.SetDefaultDeploymentStrategy(&strategy) setDeploymentStrategyAnnotation(strategy, newObj) - default: + // bluegreenStyle + } else if len(newObj.GetAnnotations()[appsv1beta1.OriginalDeploymentStrategyAnnotation]) > 0 { + if isEffectiveDeploymentRevisionChange(oldObj, newObj) { + modified = true + newObj.Spec.Paused = true + klog.Info("successive release or rollback detected") + if rs, err := revision.GetStableReplicaSet(h.Client, oldObj, newObj); rs != nil && err == nil { + // we don't allow to do continuous release for bluegreen release + if !util.EqualIgnoreHash(&newObj.Spec.Template, &rs.Spec.Template) { + err = fmt.Errorf("successive release is not supported for bluegreen for now, rollback first") + klog.Warningf(err.Error()) + // newObj.Spec.Template = oldObj.Spec.Template + return false, err + } + // for rollback, we set the stable rs minReadySeconds to infinity, + // to set pods of the stable rs unavailable, we can prevent the new version pods + // from being deleted immediately (we want to keep them until traffic is switched to the stable version) + body := fmt.Sprintf(`{"spec":{"minReadySeconds":%v}}`, appsv1beta1.MaxReadySeconds) + if err = h.Client.Patch(context.TODO(), rs, client.RawPatch(types.MergePatchType, []byte(body))); err == nil { + klog.Infof("patch rs %s/%s minReadySeconds to %v success", rs.Namespace, rs.Name, appsv1beta1.MaxReadySeconds) + } else { + klog.Warningf("patch rs %s/%s minReadySeconds to %v failed, err: %v", rs.Namespace, rs.Name, appsv1beta1.MaxReadySeconds, err) + } + } else { + klog.Warningf("Cannot find stable replicaset for deployment %s/%s", newObj.Namespace, newObj.Name) + } + } + // not allow to modify Strategy.Type to Recreate + if newObj.Spec.Strategy.Type != apps.RollingUpdateDeploymentStrategyType { + modified = true + newObj.Spec.Strategy.Type = oldObj.Spec.Strategy.Type + klog.Warningf("Not allow to modify Strategy.Type to Recreate") + } + } else { // default + if !newObj.Spec.Paused { + modified = true + newObj.Spec.Paused = true + } // Do not allow to modify strategy as Recreate during rolling if newObj.Spec.Strategy.Type == apps.RecreateDeploymentStrategyType { modified = true @@ -351,6 +392,7 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo } func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.CloneSet) (bool, error) { + klog.Info("webhook handles CloneSet") // indicate whether the workload can enter the rollout process // when cloneSet don't contain any pods, no need to enter rollout progressing if newObj.Spec.Replicas != nil && *newObj.Spec.Replicas == 0 { @@ -369,6 +411,21 @@ func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.Clon } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } + /* + continuous release (or successive release) is not supported for bluegreen release, especially for cloneset, + here is why: + suppose we are releasing a cloneset, which has pods of both v1 and v2 for now. If we release v3 before + v2 release is done, the cloneset controller might scale down pods without distinguishing between v1 and v2. + This is because our implementation is based on the minReadySeconds, pods of both v1 and v2 are "unavailable" + in the progress of rollout. + Deployment actually has the same problem, however it is possible to bypass this issue for Deployment by setting + minReadySeconds for replicaset separately; unfortunately this workaround seems not work for cloneset + */ + if rollout.Spec.Strategy.IsBlueGreenRelease() && revision.IsContinuousRelease(h.Client, oldObj, newObj) { + err = fmt.Errorf("successive release is not supported for bluegreen for now, rollback first") + return false, fmt.Errorf(err.Error()) + } + // if traffic routing, there must only be one version of Pods if rollout.Spec.Strategy.HasTrafficRoutings() && newObj.Status.Replicas != newObj.Status.UpdatedReplicas { klog.Warningf("Because cloneSet(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name) diff --git a/pkg/webhook/workload/mutating/workload_update_handler_test.go b/pkg/webhook/workload/mutating/workload_update_handler_test.go index 117d32ba..7e70ee04 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler_test.go +++ b/pkg/webhook/workload/mutating/workload_update_handler_test.go @@ -130,6 +130,51 @@ var ( }, } + rsDemoV2 = &apps.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "ReplicaSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver-v2", + Labels: map[string]string{ + "app": "echoserver", + "pod-template-hash": "verision2", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deploymentDemo, schema.GroupVersionKind{ + Group: apps.SchemeGroupVersion.Group, + Version: apps.SchemeGroupVersion.Version, + Kind: "Deployment", + }), + }, + }, + Spec: apps.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "echoserver", + }, + }, + Replicas: pointer.Int32(5), + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "echoserver", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "echoserver", + Image: "echoserver:v2", + }, + }, + }, + }, + }, + } + cloneSetDemo = &kruisev1aplphal.CloneSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps.kruise.io/v1alpha1", @@ -519,6 +564,100 @@ func TestHandlerDeployment(t *testing.T) { return obj }, }, + { + name: "bluegreen: normal release", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + newObj := deploymentDemo.DeepCopy() + newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + obj.Spec.Paused = true + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + return []*apps.ReplicaSet{rs} + }, + getRollout: func() *appsv1beta1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{} + return obj + }, + isError: false, + }, + { + name: "bluegreen: rollback", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + oldObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + oldObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + oldObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel] = "5b494f7bf" + oldObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + newObj := deploymentDemo.DeepCopy() + newObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + newObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v1" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v1" + obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + obj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + obj.Spec.Paused = true + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + rs2 := rsDemoV2.DeepCopy() + return []*apps.ReplicaSet{rs, rs2} + }, + getRollout: func() *appsv1beta1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{} + return obj + }, + isError: false, + }, + { + name: "bluegreen: successive release", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + oldObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + oldObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + oldObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel] = "5b494f7bf" + oldObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + newObj := deploymentDemo.DeepCopy() + newObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + newObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v3" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + obj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + obj.Spec.Paused = true + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + rs2 := rsDemoV2.DeepCopy() + return []*apps.ReplicaSet{rs, rs2} + }, + getRollout: func() *appsv1beta1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{} + return obj + }, + isError: true, + }, } decoder, _ := admission.NewDecoder(scheme) @@ -542,8 +681,11 @@ func TestHandlerDeployment(t *testing.T) { oldObj, newObj := cs.getObjs() _, err := h.handleDeployment(newObj, oldObj) - if cs.isError && err == nil { - t.Fatal("handlerDeployment failed") + if cs.isError { + if err == nil { + t.Fatal("handlerDeployment failed") + } + return //no need to check again } else if !cs.isError && err != nil { t.Fatalf(err.Error()) }