diff --git a/.github/workflows/e2e-advanced-deployment-1.19.yaml b/.github/workflows/e2e-advanced-deployment-1.19.yaml index 65847510..bc5285d3 100644 --- a/.github/workflows/e2e-advanced-deployment-1.19.yaml +++ b/.github/workflows/e2e-advanced-deployment-1.19.yaml @@ -81,13 +81,13 @@ jobs: done set +e PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) - kubectl get node -o yaml - kubectl get all -n kruise-rollout -o yaml set -e if [ "$PODS" -eq "1" ]; then echo "Wait for kruise-rollout ready successfully" else echo "Timeout to wait for kruise-rollout ready" + kubectl get pod -n kruise-rollout --no-headers | awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + kubectl get pod -n kruise-rollout --no-headers | awk '{print $1}' | xargs kubectl logs -n kruise-rollout exit 1 fi - name: Run E2E Tests For Deployment Controller diff --git a/api/v1alpha1/rollout_types.go b/api/v1alpha1/rollout_types.go index 4661058c..3c97f6b3 100644 --- a/api/v1alpha1/rollout_types.go +++ b/api/v1alpha1/rollout_types.go @@ -51,6 +51,12 @@ const ( // Defaults to "canary" to Deployment. // Defaults to "partition" to the others. RolloutStyleAnnotation = "rollouts.kruise.io/rolling-style" + + // TrafficRoutingAnnotation is the TrafficRouting Name, and it is the Rollout's TrafficRouting. + // The Rollout release will trigger the TrafficRouting release. For example: + // A microservice consists of three applications, and the invocation relationship is as follows: a -> b -> c, + // and application(a, b, c)'s gateway is trafficRouting. Any application(a, b or b) release will trigger TrafficRouting release. + TrafficRoutingAnnotation = "rollouts.kruise.io/trafficrouting" ) // RolloutSpec defines the desired state of Rollout @@ -72,19 +78,8 @@ type ObjectRef struct { // WorkloadRef contains enough information to let you identify a workload for Rollout // Batch release of the bypass WorkloadRef *WorkloadRef `json:"workloadRef,omitempty"` - - // revisionRef - // Fully managed batch publishing capability - //RevisionRef *ControllerRevisionRef `json:"revisionRef,omitempty"` } -type ObjectRefType string - -const ( - WorkloadRefType = "workloadRef" - RevisionRefType = "revisionRef" -) - // WorkloadRef holds a references to the Kubernetes object type WorkloadRef struct { // API Version of the referent @@ -102,50 +97,33 @@ type RolloutStrategy struct { Paused bool `json:"paused,omitempty"` // +optional Canary *CanaryStrategy `json:"canary,omitempty"` - // +optional - // BlueGreen *BlueGreenStrategy `json:"blueGreen,omitempty"` } -type RolloutStrategyType string - -const ( - RolloutStrategyCanary RolloutStrategyType = "canary" - RolloutStrategyBlueGreen RolloutStrategyType = "blueGreen" -) - // CanaryStrategy defines parameters for a Replica Based Canary type CanaryStrategy struct { // Steps define the order of phases to execute release in batches(20%, 40%, 60%, 80%, 100%) // +optional Steps []CanaryStep `json:"steps,omitempty"` // TrafficRoutings hosts all the supported service meshes supported to enable more fine-grained traffic routing - // todo current only support one TrafficRouting - TrafficRoutings []*TrafficRouting `json:"trafficRoutings,omitempty"` + // and current only support one TrafficRouting + TrafficRoutings []TrafficRoutingRef `json:"trafficRoutings,omitempty"` // FailureThreshold indicates how many failed pods can be tolerated in all upgraded pods. // Only when FailureThreshold are satisfied, Rollout can enter ready state. // If FailureThreshold is nil, Rollout will use the MaxUnavailable of workload as its // FailureThreshold. // Defaults to nil. FailureThreshold *intstr.IntOrString `json:"failureThreshold,omitempty"` - // MetricsAnalysis *MetricsAnalysisBackground `json:"metricsAnalysis,omitempty"` } // CanaryStep defines a step of a canary workload. type CanaryStep struct { - // Weight indicate how many percentage of traffic the canary pods should receive - // +optional - Weight *int32 `json:"weight,omitempty"` + TrafficRoutingStrategy `json:",inline"` // Replicas is the number of expected canary pods in this batch // it can be an absolute number (ex: 5) or a percentage of total pods. Replicas *intstr.IntOrString `json:"replicas,omitempty"` // Pause defines a pause stage for a rollout, manual or auto // +optional Pause RolloutPause `json:"pause,omitempty"` - // Matches define conditions used for matching the incoming HTTP requests to canary service. - // Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied. - // If Gateway API, current only support one match. - // And cannot support both weight and matches, if both are configured, then matches takes precedence. - Matches []HttpRouteMatch `json:"matches,omitempty"` } type HttpRouteMatch struct { @@ -163,37 +141,6 @@ type RolloutPause struct { Duration *int32 `json:"duration,omitempty"` } -// TrafficRouting hosts all the different configuration for supported service meshes to enable more fine-grained traffic routing -type TrafficRouting struct { - // Service holds the name of a service which selects pods with stable version and don't select any pods with canary version. - Service string `json:"service"` - // Optional duration in seconds the traffic provider(e.g. nginx ingress controller) consumes the service, ingress configuration changes gracefully. - GracePeriodSeconds int32 `json:"gracePeriodSeconds,omitempty"` - // Ingress holds Ingress specific configuration to route traffic, e.g. Nginx, Alb. - Ingress *IngressTrafficRouting `json:"ingress,omitempty"` - // Gateway holds Gateway specific configuration to route traffic - // Gateway configuration only supports >= v0.4.0 (v1alpha2). - Gateway *GatewayTrafficRouting `json:"gateway,omitempty"` -} - -// IngressTrafficRouting configuration for ingress controller to control traffic routing -type IngressTrafficRouting struct { - // ClassType refers to the type of `Ingress`. - // current support nginx, aliyun-alb. default is nginx. - // +optional - ClassType string `json:"classType,omitempty"` - // Name refers to the name of an `Ingress` resource in the same namespace as the `Rollout` - Name string `json:"name"` -} - -// GatewayTrafficRouting configuration for gateway api -type GatewayTrafficRouting struct { - // HTTPRouteName refers to the name of an `HTTPRoute` resource in the same namespace as the `Rollout` - HTTPRouteName *string `json:"httpRouteName,omitempty"` - // TCPRouteName *string `json:"tcpRouteName,omitempty"` - // UDPRouteName *string `json:"udpRouteName,omitempty"` -} - // RolloutStatus defines the observed state of Rollout type RolloutStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster diff --git a/api/v1alpha1/trafficrouting_types.go b/api/v1alpha1/trafficrouting_types.go new file mode 100644 index 00000000..4a9a15c7 --- /dev/null +++ b/api/v1alpha1/trafficrouting_types.go @@ -0,0 +1,153 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + ProgressingRolloutFinalizerPrefix = "progressing.rollouts.kruise.io" +) + +// TrafficRoutingRef hosts all the different configuration for supported service meshes to enable more fine-grained traffic routing +type TrafficRoutingRef struct { + // Service holds the name of a service which selects pods with stable version and don't select any pods with canary version. + Service string `json:"service"` + // Optional duration in seconds the traffic provider(e.g. nginx ingress controller) consumes the service, ingress configuration changes gracefully. + GracePeriodSeconds int32 `json:"gracePeriodSeconds,omitempty"` + // Ingress holds Ingress specific configuration to route traffic, e.g. Nginx, Alb. + Ingress *IngressTrafficRouting `json:"ingress,omitempty"` + // Gateway holds Gateway specific configuration to route traffic + // Gateway configuration only supports >= v0.4.0 (v1alpha2). + Gateway *GatewayTrafficRouting `json:"gateway,omitempty"` +} + +// IngressTrafficRouting configuration for ingress controller to control traffic routing +type IngressTrafficRouting struct { + // ClassType refers to the type of `Ingress`. + // current support nginx, aliyun-alb. default is nginx. + // +optional + ClassType string `json:"classType,omitempty"` + // Name refers to the name of an `Ingress` resource in the same namespace as the `Rollout` + Name string `json:"name"` +} + +// GatewayTrafficRouting configuration for gateway api +type GatewayTrafficRouting struct { + // HTTPRouteName refers to the name of an `HTTPRoute` resource in the same namespace as the `Rollout` + HTTPRouteName *string `json:"httpRouteName,omitempty"` + // TCPRouteName *string `json:"tcpRouteName,omitempty"` + // UDPRouteName *string `json:"udpRouteName,omitempty"` +} + +type TrafficRoutingSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // ObjectRef indicates trafficRouting ref + ObjectRef []TrafficRoutingRef `json:"objectRef"` + // trafficrouting strategy + Strategy TrafficRoutingStrategy `json:"strategy"` +} + +type TrafficRoutingStrategy struct { + // Weight indicate how many percentage of traffic the canary pods should receive + // +optional + Weight *int32 `json:"weight,omitempty"` + // Set overwrites the request with the given header (name, value) + // before the action. + // + // Input: + // GET /foo HTTP/1.1 + // my-header: foo + // + // requestHeaderModifier: + // set: + // - name: "my-header" + // value: "bar" + // + // Output: + // GET /foo HTTP/1.1 + // my-header: bar + // + // +optional + // RequestHeaderModifier *gatewayv1alpha2.HTTPRequestHeaderFilter `json:"requestHeaderModifier,omitempty"` + // Matches define conditions used for matching the incoming HTTP requests to canary service. + // Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied. + // If Gateway API, current only support one match. + // And cannot support both weight and matches, if both are configured, then matches takes precedence. + Matches []HttpRouteMatch `json:"matches,omitempty"` +} + +type TrafficRoutingStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // observedGeneration is the most recent generation observed for this Rollout. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // Phase is the trafficRouting phase. + Phase TrafficRoutingPhase `json:"phase,omitempty"` + // Message provides details on why the rollout is in its current phase + Message string `json:"message,omitempty"` +} + +// TrafficRoutingPhase are a set of phases that this rollout +type TrafficRoutingPhase string + +const ( + // TrafficRoutingPhaseInitial indicates a traffic routing is Initial + TrafficRoutingPhaseInitial TrafficRoutingPhase = "Initial" + // TrafficRoutingPhaseHealthy indicates a traffic routing is healthy. + // This means that Ingress and Service Resources exist. + TrafficRoutingPhaseHealthy TrafficRoutingPhase = "Healthy" + // TrafficRoutingPhaseProgressing indicates a traffic routing is not yet healthy but still making progress towards a healthy state + TrafficRoutingPhaseProgressing TrafficRoutingPhase = "Progressing" + // TrafficRoutingPhaseFinalizing indicates the trafficRouting progress is complete, and is running recycle operations. + TrafficRoutingPhaseFinalizing TrafficRoutingPhase = "Finalizing" + // TrafficRoutingPhaseTerminating indicates a traffic routing is terminated + TrafficRoutingPhaseTerminating TrafficRoutingPhase = "Terminating" +) + +// +genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="STATUS",type="string",JSONPath=".status.phase",description="The TrafficRouting status phase" +// +kubebuilder:printcolumn:name="MESSAGE",type="string",JSONPath=".status.message",description="The TrafficRouting canary status message" +// +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=".metadata.creationTimestamp" + +// TrafficRouting is the Schema for the TrafficRoutings API +type TrafficRouting struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TrafficRoutingSpec `json:"spec,omitempty"` + Status TrafficRoutingStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// TrafficRoutingList contains a list of TrafficRouting +type TrafficRoutingList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TrafficRouting `json:"items"` +} + +func init() { + SchemeBuilder.Register(&TrafficRouting{}, &TrafficRoutingList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index fcf7dc17..a5e87273 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -178,24 +178,13 @@ func (in *CanaryStatus) DeepCopy() *CanaryStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CanaryStep) DeepCopyInto(out *CanaryStep) { *out = *in - if in.Weight != nil { - in, out := &in.Weight, &out.Weight - *out = new(int32) - **out = **in - } + in.TrafficRoutingStrategy.DeepCopyInto(&out.TrafficRoutingStrategy) if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas *out = new(intstr.IntOrString) **out = **in } in.Pause.DeepCopyInto(&out.Pause) - if in.Matches != nil { - in, out := &in.Matches, &out.Matches - *out = make([]HttpRouteMatch, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryStep. @@ -240,13 +229,9 @@ func (in *CanaryStrategy) DeepCopyInto(out *CanaryStrategy) { } if in.TrafficRoutings != nil { in, out := &in.TrafficRoutings, &out.TrafficRoutings - *out = make([]*TrafficRouting, len(*in)) + *out = make([]TrafficRoutingRef, len(*in)) for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = new(TrafficRouting) - (*in).DeepCopyInto(*out) - } + (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.FailureThreshold != nil { @@ -783,16 +768,10 @@ func (in *ServiceInfo) DeepCopy() *ServiceInfo { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TrafficRouting) DeepCopyInto(out *TrafficRouting) { *out = *in - if in.Ingress != nil { - in, out := &in.Ingress, &out.Ingress - *out = new(IngressTrafficRouting) - **out = **in - } - if in.Gateway != nil { - in, out := &in.Gateway, &out.Gateway - *out = new(GatewayTrafficRouting) - (*in).DeepCopyInto(*out) - } + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRouting. @@ -805,6 +784,14 @@ func (in *TrafficRouting) DeepCopy() *TrafficRouting { return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TrafficRouting) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TrafficRoutingInfo) DeepCopyInto(out *TrafficRoutingInfo) { *out = *in @@ -830,6 +817,128 @@ func (in *TrafficRoutingInfo) DeepCopy() *TrafficRoutingInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TrafficRoutingList) DeepCopyInto(out *TrafficRoutingList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TrafficRouting, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingList. +func (in *TrafficRoutingList) DeepCopy() *TrafficRoutingList { + if in == nil { + return nil + } + out := new(TrafficRoutingList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TrafficRoutingList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TrafficRoutingRef) DeepCopyInto(out *TrafficRoutingRef) { + *out = *in + if in.Ingress != nil { + in, out := &in.Ingress, &out.Ingress + *out = new(IngressTrafficRouting) + **out = **in + } + if in.Gateway != nil { + in, out := &in.Gateway, &out.Gateway + *out = new(GatewayTrafficRouting) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingRef. +func (in *TrafficRoutingRef) DeepCopy() *TrafficRoutingRef { + if in == nil { + return nil + } + out := new(TrafficRoutingRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TrafficRoutingSpec) DeepCopyInto(out *TrafficRoutingSpec) { + *out = *in + if in.ObjectRef != nil { + in, out := &in.ObjectRef, &out.ObjectRef + *out = make([]TrafficRoutingRef, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Strategy.DeepCopyInto(&out.Strategy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingSpec. +func (in *TrafficRoutingSpec) DeepCopy() *TrafficRoutingSpec { + if in == nil { + return nil + } + out := new(TrafficRoutingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TrafficRoutingStatus) DeepCopyInto(out *TrafficRoutingStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingStatus. +func (in *TrafficRoutingStatus) DeepCopy() *TrafficRoutingStatus { + if in == nil { + return nil + } + out := new(TrafficRoutingStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TrafficRoutingStrategy) DeepCopyInto(out *TrafficRoutingStrategy) { + *out = *in + if in.Weight != nil { + in, out := &in.Weight, &out.Weight + *out = new(int32) + **out = **in + } + if in.Matches != nil { + in, out := &in.Matches, &out.Matches + *out = make([]HttpRouteMatch, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingStrategy. +func (in *TrafficRoutingStrategy) DeepCopy() *TrafficRoutingStrategy { + if in == nil { + return nil + } + out := new(TrafficRoutingStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadInfo) DeepCopyInto(out *WorkloadInfo) { *out = *in diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index 10850a4e..4d375139 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -111,13 +111,20 @@ spec: description: CanaryStep defines a step of a canary workload. properties: matches: - description: Matches define conditions used for matching - the incoming HTTP requests to canary service. Each - match is independent, i.e. this rule will be matched - if **any** one of the matches is satisfied. If Gateway - API, current only support one match. And cannot support - both weight and matches, if both are configured, then - matches takes precedence. + description: "Set overwrites the request with the given + header (name, value) before the action. \n Input: + \ GET /foo HTTP/1.1 my-header: foo \n requestHeaderModifier: + \ set: - name: \"my-header\" value: \"bar\" + \n Output: GET /foo HTTP/1.1 my-header: bar \n + RequestHeaderModifier *gatewayv1alpha2.HTTPRequestHeaderFilter + `json:\"requestHeaderModifier,omitempty\"` Matches + define conditions used for matching the incoming HTTP + requests to canary service. Each match is independent, + i.e. this rule will be matched if **any** one of the + matches is satisfied. If Gateway API, current only + support one match. And cannot support both weight + and matches, if both are configured, then matches + takes precedence." items: properties: headers: @@ -209,9 +216,9 @@ spec: trafficRoutings: description: TrafficRoutings hosts all the supported service meshes supported to enable more fine-grained traffic routing - todo current only support one TrafficRouting + and current only support one TrafficRouting items: - description: TrafficRouting hosts all the different configuration + description: TrafficRoutingRef hosts all the different configuration for supported service meshes to enable more fine-grained traffic routing properties: diff --git a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml new file mode 100644 index 00000000..f8ef5907 --- /dev/null +++ b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml @@ -0,0 +1,205 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: trafficroutings.rollouts.kruise.io +spec: + group: rollouts.kruise.io + names: + kind: TrafficRouting + listKind: TrafficRoutingList + plural: trafficroutings + singular: trafficrouting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The TrafficRouting status phase + jsonPath: .status.phase + name: STATUS + type: string + - description: The TrafficRouting canary status message + jsonPath: .status.message + name: MESSAGE + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TrafficRouting is the Schema for the TrafficRoutings API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + objectRef: + description: ObjectRef indicates trafficRouting ref + items: + description: TrafficRoutingRef hosts all the different configuration + for supported service meshes to enable more fine-grained traffic + routing + properties: + gateway: + description: Gateway holds Gateway specific configuration to + route traffic Gateway configuration only supports >= v0.4.0 + (v1alpha2). + properties: + httpRouteName: + description: HTTPRouteName refers to the name of an `HTTPRoute` + resource in the same namespace as the `Rollout` + type: string + type: object + gracePeriodSeconds: + description: Optional duration in seconds the traffic provider(e.g. + nginx ingress controller) consumes the service, ingress configuration + changes gracefully. + format: int32 + type: integer + ingress: + description: Ingress holds Ingress specific configuration to + route traffic, e.g. Nginx, Alb. + properties: + classType: + description: ClassType refers to the type of `Ingress`. + current support nginx, aliyun-alb. default is nginx. + type: string + name: + description: Name refers to the name of an `Ingress` resource + in the same namespace as the `Rollout` + type: string + required: + - name + type: object + service: + description: Service holds the name of a service which selects + pods with stable version and don't select any pods with canary + version. + type: string + required: + - service + type: object + type: array + strategy: + description: trafficrouting strategy + properties: + matches: + description: "Set overwrites the request with the given header + (name, value) before the action. \n Input: GET /foo HTTP/1.1 + \ my-header: foo \n requestHeaderModifier: set: - name: + \"my-header\" value: \"bar\" \n Output: GET /foo HTTP/1.1 + \ my-header: bar \n RequestHeaderModifier *gatewayv1alpha2.HTTPRequestHeaderFilter + `json:\"requestHeaderModifier,omitempty\"` Matches define conditions + used for matching the incoming HTTP requests to canary service. + Each match is independent, i.e. this rule will be matched if + **any** one of the matches is satisfied. If Gateway API, current + only support one match. And cannot support both weight and matches, + if both are configured, then matches takes precedence." + items: + properties: + headers: + description: Headers specifies HTTP request header matchers. + Multiple match values are ANDed together, meaning, a request + must match all the specified headers to select the route. + items: + description: HTTPHeaderMatch describes how to select a + HTTP route by matching HTTP request headers. + properties: + name: + description: "Name is the name of the HTTP Header + to be matched. Name matching MUST be case insensitive. + (See https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries specify equivalent header + names, only the first entry with an equivalent name + MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. + Due to the case-insensitivity of header names, \"foo\" + and \"Foo\" are considered equivalent. \n When a + header is repeated in an HTTP request, it is implementation-specific + behavior as to how this is represented. Generally, + proxies should follow the guidance from the RFC: + https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 + regarding processing a repeated header, with special + handling for \"Set-Cookie\"." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: "Type specifies how to match against + the value of the header. \n Support: Core (Exact) + \n Support: Custom (RegularExpression) \n Since + RegularExpression HeaderMatchType has custom conformance, + implementations can support POSIX, PCRE or any other + dialects of regular expressions. Please read the + implementation's documentation to determine the + supported dialect." + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP Header to + be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + type: object + type: array + weight: + description: Weight indicate how many percentage of traffic the + canary pods should receive + format: int32 + type: integer + type: object + required: + - objectRef + - strategy + type: object + status: + properties: + message: + description: Message provides details on why the rollout is in its + current phase + type: string + observedGeneration: + description: observedGeneration is the most recent generation observed + for this Rollout. + format: int64 + type: integer + phase: + description: Phase is the trafficRouting phase. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index c11d35da..205d70d7 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -5,6 +5,7 @@ resources: - bases/rollouts.kruise.io_rollouts.yaml - bases/rollouts.kruise.io_batchreleases.yaml - bases/rollouts.kruise.io_rollouthistories.yaml +- bases/rollouts.kruise.io_trafficroutings.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index bba31e50..50e35f69 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -340,3 +340,28 @@ rules: - get - patch - update +- apiGroups: + - rollouts.kruise.io + resources: + - trafficroutings + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - rollouts.kruise.io + resources: + - trafficroutings/finalizers + verbs: + - update +- apiGroups: + - rollouts.kruise.io + resources: + - trafficroutings/status + verbs: + - get + - patch + - update diff --git a/lua_configuration/trafficrouting_ingress/mse.lua b/lua_configuration/trafficrouting_ingress/mse.lua new file mode 100644 index 00000000..ecaf3a00 --- /dev/null +++ b/lua_configuration/trafficrouting_ingress/mse.lua @@ -0,0 +1,45 @@ +function split(input, delimiter) + local arr = {} + string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end) + return arr +end + +annotations = obj.annotations +annotations["nginx.ingress.kubernetes.io/canary"] = "true" +annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil +annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil +if ( obj.weight ~= "-1" ) +then + annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight +end +if ( obj.requestHeaderModifier ) +then + local str = '' + for _,header in ipairs(obj.requestHeaderModifier.set) do + str = str..string.format("%s %s", header.name, header.value) + end + annotations["mse.ingress.kubernetes.io/request-header-control-update"] = str +end +if ( not obj.matches ) +then + return annotations +end +for _,match in ipairs(obj.matches) do + header = match.headers[1] + if ( header.name == "canary-by-cookie" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name + if ( header.type == "RegularExpression" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value + end + end +end +return annotations diff --git a/main.go b/main.go index 11dd9f4f..46682654 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ import ( "github.com/openkruise/rollouts/pkg/controller/deployment" "github.com/openkruise/rollouts/pkg/controller/rollout" "github.com/openkruise/rollouts/pkg/controller/rollouthistory" + "github.com/openkruise/rollouts/pkg/controller/trafficrouting" utilclient "github.com/openkruise/rollouts/pkg/util/client" utilfeature "github.com/openkruise/rollouts/pkg/util/feature" "github.com/openkruise/rollouts/pkg/webhook" @@ -110,6 +111,15 @@ func main() { os.Exit(1) } + if err = (&trafficrouting.TrafficRoutingReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("trafficrouting-controller"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TrafficRouting") + os.Exit(1) + } + if err = br.Add(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "BatchRelease") os.Exit(1) diff --git a/pkg/controller/batchrelease/control/canarystyle/deployment/canary.go b/pkg/controller/batchrelease/control/canarystyle/deployment/canary.go index d91c16f6..9e3f7884 100644 --- a/pkg/controller/batchrelease/control/canarystyle/deployment/canary.go +++ b/pkg/controller/batchrelease/control/canarystyle/deployment/canary.go @@ -132,6 +132,7 @@ func (r *realCanaryController) create(release *v1alpha1.BatchRelease, template * // spec canary.Spec = *template.Spec.DeepCopy() + // todo, patch canary pod metadata canary.Spec.Replicas = pointer.Int32Ptr(0) canary.Spec.Paused = false @@ -179,6 +180,7 @@ func filterCanaryDeployment(ds []*apps.Deployment, template *corev1.PodTemplateS return ds[0] } for _, d := range ds { + // todo, remove the canary pod metadata if util.EqualIgnoreHash(template, &d.Spec.Template) { return d } diff --git a/pkg/controller/batchrelease/control/canarystyle/deployment/control.go b/pkg/controller/batchrelease/control/canarystyle/deployment/control.go index cf5a968c..04938d0e 100644 --- a/pkg/controller/batchrelease/control/canarystyle/deployment/control.go +++ b/pkg/controller/batchrelease/control/canarystyle/deployment/control.go @@ -73,7 +73,6 @@ func (rc *realController) BuildCanaryController(release *v1alpha1.BatchRelease) if client.IgnoreNotFound(err) != nil { return rc, err } - rc.canaryObject = filterCanaryDeployment(util.FilterActiveDeployment(ds), template) if rc.canaryObject == nil { return rc, control.GenerateNotFoundError(fmt.Sprintf("%v-canary", rc.stableKey), "Deployment") diff --git a/pkg/controller/rollout/rollout_canary.go b/pkg/controller/rollout/rollout_canary.go index 3ab2b960..4ae89517 100644 --- a/pkg/controller/rollout/rollout_canary.go +++ b/pkg/controller/rollout/rollout_canary.go @@ -45,7 +45,7 @@ type canaryReleaseManager struct { recorder record.EventRecorder } -func (m *canaryReleaseManager) runCanary(c *util.RolloutContext) error { +func (m *canaryReleaseManager) runCanary(c *RolloutContext) error { canaryStatus := c.NewStatus.CanaryStatus if br, err := m.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name); err != nil && !errors.IsNotFound(err) { klog.Errorf("rollout(%s/%s) fetch batchRelease failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) @@ -70,7 +70,9 @@ func (m *canaryReleaseManager) runCanary(c *util.RolloutContext) error { // We need to clean up the canary-related resources first and then rollout the rest of the batch. currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] if currentStep.Weight == nil && len(currentStep.Matches) == 0 { - done, err := m.trafficRoutingManager.FinalisingTrafficRouting(c, false) + tr := newTrafficRoutingContext(c) + done, err := m.trafficRoutingManager.FinalisingTrafficRouting(tr, false) + c.NewStatus.CanaryStatus.LastUpdateTime = tr.LastUpdateTime if err != nil { return err } else if !done { @@ -95,7 +97,9 @@ func (m *canaryReleaseManager) runCanary(c *util.RolloutContext) error { case v1alpha1.CanaryStepStateTrafficRouting: klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1alpha1.CanaryStepStateTrafficRouting) - done, err := m.trafficRoutingManager.DoTrafficRouting(c) + tr := newTrafficRoutingContext(c) + done, err := m.trafficRoutingManager.DoTrafficRouting(tr) + c.NewStatus.CanaryStatus.LastUpdateTime = tr.LastUpdateTime if err != nil { return err } else if done { @@ -142,6 +146,7 @@ func (m *canaryReleaseManager) runCanary(c *util.RolloutContext) error { klog.Infof("rollout(%s/%s) canary run all steps, and completed", c.Rollout.Namespace, c.Rollout.Name) canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} canaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + return nil } klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex, v1alpha1.CanaryStepStateReady, canaryStatus.CurrentStepState) @@ -153,7 +158,7 @@ func (m *canaryReleaseManager) runCanary(c *util.RolloutContext) error { return nil } -func (m *canaryReleaseManager) doCanaryUpgrade(c *util.RolloutContext) (bool, error) { +func (m *canaryReleaseManager) doCanaryUpgrade(c *RolloutContext) (bool, error) { // verify whether batchRelease configuration is the latest steps := len(c.Rollout.Spec.Strategy.Canary.Steps) canaryStatus := c.NewStatus.CanaryStatus @@ -184,12 +189,12 @@ func (m *canaryReleaseManager) doCanaryUpgrade(c *util.RolloutContext) (bool, er return true, nil } -func (m *canaryReleaseManager) doCanaryMetricsAnalysis(c *util.RolloutContext) (bool, error) { +func (m *canaryReleaseManager) doCanaryMetricsAnalysis(c *RolloutContext) (bool, error) { // todo return true, nil } -func (m *canaryReleaseManager) doCanaryPaused(c *util.RolloutContext) (bool, error) { +func (m *canaryReleaseManager) doCanaryPaused(c *RolloutContext) (bool, error) { canaryStatus := c.NewStatus.CanaryStatus currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] steps := len(c.Rollout.Spec.Strategy.Canary.Steps) @@ -223,7 +228,7 @@ func (m *canaryReleaseManager) doCanaryPaused(c *util.RolloutContext) (bool, err } // cleanup after rollout is completed or finished -func (m *canaryReleaseManager) doCanaryFinalising(c *util.RolloutContext) (bool, error) { +func (m *canaryReleaseManager) doCanaryFinalising(c *RolloutContext) (bool, error) { // when CanaryStatus is nil, which means canary action hasn't started yet, don't need doing cleanup if c.NewStatus.CanaryStatus == nil { return true, nil @@ -233,8 +238,10 @@ func (m *canaryReleaseManager) doCanaryFinalising(c *util.RolloutContext) (bool, if err != nil { return false, err } + tr := newTrafficRoutingContext(c) // 2. remove stable service the pod revision selector, so stable service will be selector all version pods. - done, err := m.trafficRoutingManager.FinalisingTrafficRouting(c, true) + done, err := m.trafficRoutingManager.FinalisingTrafficRouting(tr, true) + c.NewStatus.CanaryStatus.LastUpdateTime = tr.LastUpdateTime if err != nil || !done { return done, err } @@ -244,7 +251,8 @@ func (m *canaryReleaseManager) doCanaryFinalising(c *util.RolloutContext) (bool, return done, err } // 4. modify network api(ingress or gateway api) configuration, and route 100% traffic to stable pods. - done, err = m.trafficRoutingManager.FinalisingTrafficRouting(c, false) + done, err = m.trafficRoutingManager.FinalisingTrafficRouting(tr, false) + c.NewStatus.CanaryStatus.LastUpdateTime = tr.LastUpdateTime if err != nil || !done { return done, err } @@ -260,7 +268,7 @@ func (m *canaryReleaseManager) doCanaryFinalising(c *util.RolloutContext) (bool, return true, nil } -func (m *canaryReleaseManager) removeRolloutProgressingAnnotation(c *util.RolloutContext) error { +func (m *canaryReleaseManager) removeRolloutProgressingAnnotation(c *RolloutContext) error { if c.Workload == nil { return nil } @@ -270,10 +278,8 @@ func (m *canaryReleaseManager) removeRolloutProgressingAnnotation(c *util.Rollou workloadRef := c.Rollout.Spec.ObjectRef.WorkloadRef workloadGVK := schema.FromAPIVersionAndKind(workloadRef.APIVersion, workloadRef.Kind) obj := util.GetEmptyWorkloadObject(workloadGVK) - if err := m.Get(context.TODO(), types.NamespacedName{Name: c.Workload.Name, Namespace: c.Workload.Namespace}, obj); err != nil { - klog.Errorf("getting updated workload(%s.%s) failed: %s", c.Workload.Namespace, c.Workload.Name, err.Error()) - return err - } + obj.SetNamespace(c.Workload.Namespace) + obj.SetName(c.Workload.Name) body := fmt.Sprintf(`{"metadata":{"annotations":{"%s":null}}}`, util.InRolloutProgressingAnnotation) if err := m.Patch(context.TODO(), obj, client.RawPatch(types.MergePatchType, []byte(body))); err != nil { klog.Errorf("rollout(%s/%s) patch workload(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, c.Workload.Name, err.Error()) @@ -359,6 +365,7 @@ func createBatchRelease(rollout *v1alpha1.Rollout, rolloutID string, batch int32 RolloutID: rolloutID, BatchPartition: utilpointer.Int32Ptr(batch), FailureThreshold: rollout.Spec.Strategy.Canary.FailureThreshold, + // PatchPodTemplateMetadata: rollout.Spec.Strategy.Canary.PatchPodTemplateMetadata, }, }, } @@ -375,7 +382,7 @@ func createBatchRelease(rollout *v1alpha1.Rollout, rolloutID string, batch int32 return br } -func (m *canaryReleaseManager) removeBatchRelease(c *util.RolloutContext) (bool, error) { +func (m *canaryReleaseManager) removeBatchRelease(c *RolloutContext) (bool, error) { batch := &v1alpha1.BatchRelease{} err := m.Get(context.TODO(), client.ObjectKey{Namespace: c.Rollout.Namespace, Name: c.Rollout.Name}, batch) if err != nil && errors.IsNotFound(err) { @@ -399,7 +406,7 @@ func (m *canaryReleaseManager) removeBatchRelease(c *util.RolloutContext) (bool, return false, nil } -func (m *canaryReleaseManager) finalizingBatchRelease(c *util.RolloutContext) (bool, error) { +func (m *canaryReleaseManager) finalizingBatchRelease(c *RolloutContext) (bool, error) { br, err := m.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name) if err != nil { if errors.IsNotFound(err) { diff --git a/pkg/controller/rollout/rollout_canary_test.go b/pkg/controller/rollout/rollout_canary_test.go index 3ad2d658..0bf93690 100644 --- a/pkg/controller/rollout/rollout_canary_test.go +++ b/pkg/controller/rollout/rollout_canary_test.go @@ -233,13 +233,16 @@ func TestRunCanary(t *testing.T) { trafficRoutingManager: r.trafficRoutingManager, recorder: r.Recorder, } - workload, _ := r.finder.GetWorkloadForRef(rollout) - c := &util.RolloutContext{ + workload, err := r.finder.GetWorkloadForRef(rollout) + if err != nil { + t.Fatalf("GetWorkloadForRef failed: %s", err.Error()) + } + c := &RolloutContext{ Rollout: rollout, NewStatus: rollout.Status.DeepCopy(), Workload: workload, } - err := r.canaryManager.runCanary(c) + err = r.canaryManager.runCanary(c) if err != nil { t.Fatalf("reconcileRolloutProgressing failed: %s", err.Error()) } @@ -310,7 +313,7 @@ func TestRunCanaryPaused(t *testing.T) { trafficRoutingManager: r.trafficRoutingManager, recorder: r.Recorder, } - c := &util.RolloutContext{ + c := &RolloutContext{ Rollout: rollout, NewStatus: rollout.Status.DeepCopy(), } diff --git a/pkg/controller/rollout/rollout_controller_test.go b/pkg/controller/rollout/rollout_controller_test.go index 0093781e..8afa8346 100644 --- a/pkg/controller/rollout/rollout_controller_test.go +++ b/pkg/controller/rollout/rollout_controller_test.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" clientgoscheme "k8s.io/client-go/kubernetes/scheme" utilpointer "k8s.io/utils/pointer" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) var ( @@ -57,23 +58,31 @@ var ( Canary: &v1alpha1.CanaryStrategy{ Steps: []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(5), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(5), + }, Replicas: &intstr.IntOrString{IntVal: 1}, }, { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Replicas: &intstr.IntOrString{IntVal: 2}, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, Replicas: &intstr.IntOrString{IntVal: 6}, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Replicas: &intstr.IntOrString{IntVal: 10}, }, }, - TrafficRoutings: []*v1alpha1.TrafficRouting{ + TrafficRoutings: []v1alpha1.TrafficRoutingRef{ { Service: "echoserver", Ingress: &v1alpha1.IngressTrafficRouting{ @@ -308,6 +317,36 @@ var ( `, }, } + + demoTR = &v1alpha1.TrafficRouting{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tr-demo", + Labels: map[string]string{}, + }, + Spec: v1alpha1.TrafficRoutingSpec{ + ObjectRef: []v1alpha1.TrafficRoutingRef{ + { + Service: "echoserver", + Ingress: &v1alpha1.IngressTrafficRouting{ + Name: "echoserver", + }, + }, + }, + Strategy: v1alpha1.TrafficRoutingStrategy{ + Matches: []v1alpha1.HttpRouteMatch{ + // header + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123456", + }, + }, + }, + }, + }, + }, + } ) func init() { diff --git a/pkg/controller/rollout/rollout_progressing.go b/pkg/controller/rollout/rollout_progressing.go index dbf2ec5c..9635d6d0 100644 --- a/pkg/controller/rollout/rollout_progressing.go +++ b/pkg/controller/rollout/rollout_progressing.go @@ -17,20 +17,36 @@ limitations under the License. package rollout import ( + "context" + "fmt" "strconv" "time" "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting" "github.com/openkruise/rollouts/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) var defaultGracePeriodSeconds int32 = 3 +type RolloutContext struct { + Rollout *v1alpha1.Rollout + NewStatus *v1alpha1.RolloutStatus + // related workload + Workload *util.Workload + // reconcile RequeueAfter recheckTime + RecheckTime *time.Time + // wait stable workload pods ready + WaitReady bool +} + // parameter1 retryReconcile, parameter2 error func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutStatus) (*time.Time, error) { cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) @@ -46,7 +62,7 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1alpha1.Rollou klog.Infof("rollout(%s/%s) workload status is inconsistent, then wait a moment", rollout.Namespace, rollout.Name) return nil, nil } - rolloutContext := &util.RolloutContext{Rollout: rollout, NewStatus: newStatus, Workload: workload} + rolloutContext := &RolloutContext{Rollout: rollout, NewStatus: newStatus, Workload: workload} switch cond.Reason { case v1alpha1.ProgressingReasonInitializing: klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) @@ -132,10 +148,10 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1alpha1.Rollou return rolloutContext.RecheckTime, nil } -func (r *RolloutReconciler) doProgressingInitializing(c *util.RolloutContext) (bool, error) { +func (r *RolloutReconciler) doProgressingInitializing(c *RolloutContext) (bool, error) { // Traffic routing if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { - if err := r.trafficRoutingManager.InitializeTrafficRouting(c); err != nil { + if err := r.trafficRoutingManager.InitializeTrafficRouting(newTrafficRoutingContext(c)); err != nil { return false, err } } @@ -149,10 +165,15 @@ func (r *RolloutReconciler) doProgressingInitializing(c *util.RolloutContext) (b klog.Infof("verify rollout(%s/%s) TrafficRouting, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) return false, nil } + + // TrafficRouting indicates the gateway traffic routing, and rollout release will trigger the trafficRouting + if c.Rollout.Annotations[v1alpha1.TrafficRoutingAnnotation] != "" { + return r.handleTrafficRouting(c.Rollout.Namespace, c.Rollout.Name, c.Rollout.Annotations[v1alpha1.TrafficRoutingAnnotation]) + } return true, nil } -func (r *RolloutReconciler) doProgressingInRolling(c *util.RolloutContext) error { +func (r *RolloutReconciler) doProgressingInRolling(c *RolloutContext) error { // Handle the 5 special cases firstly, and we had better keep the order of following cases: switch { @@ -185,7 +206,7 @@ func (r *RolloutReconciler) handleRolloutPaused(rollout *v1alpha1.Rollout, newSt return nil } -func (r *RolloutReconciler) handleContinuousRelease(c *util.RolloutContext) error { +func (r *RolloutReconciler) handleContinuousRelease(c *RolloutContext) error { r.Recorder.Eventf(c.Rollout, corev1.EventTypeNormal, "Progressing", "workload continuous publishing canaryRevision, then restart publishing") klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing", c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.CanaryStatus.CanaryRevision, c.Workload.CanaryRevision) @@ -226,7 +247,7 @@ func (r *RolloutReconciler) handleRollbackInBatches(rollout *v1alpha1.Rollout, w return nil } -func (r *RolloutReconciler) handleRolloutPlanChanged(c *util.RolloutContext) error { +func (r *RolloutReconciler) handleRolloutPlanChanged(c *RolloutContext) error { newStepIndex, err := r.recalculateCanaryStep(c) if err != nil { klog.Errorf("rollout(%s/%s) reCalculate Canary StepIndex failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) @@ -242,13 +263,62 @@ func (r *RolloutReconciler) handleRolloutPlanChanged(c *util.RolloutContext) err return nil } -func (r *RolloutReconciler) handleNormalRolling(c *util.RolloutContext) error { - //check if canary is done +func (r *RolloutReconciler) handleNormalRolling(c *RolloutContext) error { + // check if canary is done if c.NewStatus.CanaryStatus.CurrentStepState == v1alpha1.CanaryStepStateCompleted { klog.Infof("rollout(%s/%s) progressing rolling done", c.Rollout.Namespace, c.Rollout.Name) progressingStateTransition(c.NewStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonFinalising, "Rollout has been completed and some closing work is being done") - } else { // rollout is in rolling - return r.canaryManager.runCanary(c) + return nil + } + return r.canaryManager.runCanary(c) +} + +// name is rollout name, tr is trafficRouting name +func (r *RolloutReconciler) handleTrafficRouting(namespace, name, tr string) (bool, error) { + obj := &v1alpha1.TrafficRouting{} + err := r.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: tr}, obj) + if err != nil { + if errors.IsNotFound(err) { + klog.Warningf("rollout(%s/%s) trafficRouting(%s) Not Found, and wait a moment", namespace, name, tr) + return false, nil + } + return false, err + } + if controllerutil.ContainsFinalizer(obj, util.ProgressingRolloutFinalizer(name)) { + return true, nil + } + if obj.Status.Phase == v1alpha1.TrafficRoutingPhaseFinalizing || obj.Status.Phase == v1alpha1.TrafficRoutingPhaseTerminating { + klog.Infof("rollout(%s/%s) trafficRouting(%s) phase(%s), and wait a moment", namespace, name, tr, obj.Status.Phase) + return false, nil + } + err = util.UpdateFinalizer(r.Client, obj, util.AddFinalizerOpType, util.ProgressingRolloutFinalizer(name)) + if err != nil { + klog.Errorf("rollout(%s/%s) add trafficRouting(%s) finalizer failed: %s", namespace, name, tr, err.Error()) + return false, err + } + klog.Infof("rollout(%s/%s) add trafficRouting(%s) finalizer(%s) success", namespace, name, tr, util.ProgressingRolloutFinalizer(name)) + return false, nil +} + +func (r *RolloutReconciler) finalizeTrafficRouting(namespace, name, tr string) error { + obj := &v1alpha1.TrafficRouting{} + err := r.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: tr}, obj) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + // progressing.rollouts.kruise.io/rollout-b + finalizer := util.ProgressingRolloutFinalizer(name) + if controllerutil.ContainsFinalizer(obj, finalizer) { + err = util.UpdateFinalizer(r.Client, obj, util.RemoveFinalizerOpType, finalizer) + if err != nil { + klog.Errorf("rollout(%s/%s) remove trafficRouting(%s) finalizer failed: %s", namespace, name, tr, err.Error()) + return err + } + klog.Infof("rollout(%s/%s) remove trafficRouting(%s) remove finalizer(%s) success", namespace, name, tr, finalizer) + return nil } return nil } @@ -288,10 +358,12 @@ func isRollingBackInBatches(rollout *v1alpha1.Rollout, workload *util.Workload) // 1. modify network api(ingress or gateway api) configuration, and route 100% traffic to stable pods // 2. remove batchRelease CR. -func (r *RolloutReconciler) doProgressingReset(c *util.RolloutContext) (bool, error) { +func (r *RolloutReconciler) doProgressingReset(c *RolloutContext) (bool, error) { if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { // modify network api(ingress or gateway api) configuration, and route 100% traffic to stable pods - done, err := r.trafficRoutingManager.FinalisingTrafficRouting(c, false) + tr := newTrafficRoutingContext(c) + done, err := r.trafficRoutingManager.FinalisingTrafficRouting(tr, false) + c.NewStatus.CanaryStatus.LastUpdateTime = tr.LastUpdateTime if err != nil || !done { return done, err } @@ -306,7 +378,7 @@ func (r *RolloutReconciler) doProgressingReset(c *util.RolloutContext) (bool, er return true, nil } -func (r *RolloutReconciler) recalculateCanaryStep(c *util.RolloutContext) (int32, error) { +func (r *RolloutReconciler) recalculateCanaryStep(c *RolloutContext) (int32, error) { batch, err := r.canaryManager.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name) if errors.IsNotFound(err) { return 1, nil @@ -332,8 +404,15 @@ func (r *RolloutReconciler) recalculateCanaryStep(c *util.RolloutContext) (int32 return stepIndex, nil } -func (r *RolloutReconciler) doFinalising(c *util.RolloutContext) (bool, error) { +func (r *RolloutReconciler) doFinalising(c *RolloutContext) (bool, error) { klog.Infof("reconcile rollout(%s/%s) doFinalising", c.Rollout.Namespace, c.Rollout.Name) + // TrafficRouting indicates the gateway traffic routing, and rollout finalizer will trigger the trafficRouting finalizer + if c.Rollout.Annotations[v1alpha1.TrafficRoutingAnnotation] != "" { + err := r.finalizeTrafficRouting(c.Rollout.Namespace, c.Rollout.Name, c.Rollout.Annotations[v1alpha1.TrafficRoutingAnnotation]) + if err != nil { + return false, err + } + } done, err := r.canaryManager.doCanaryFinalising(c) if err != nil { klog.Errorf("rollout(%s/%s) Progressing failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) @@ -370,3 +449,22 @@ func setRolloutSucceededCondition(status *v1alpha1.RolloutStatus, condStatus cor } util.SetRolloutCondition(status, *cond) } + +func newTrafficRoutingContext(c *RolloutContext) *trafficrouting.TrafficRoutingContext { + currentStep := c.Rollout.Spec.Strategy.Canary.Steps[c.NewStatus.CanaryStatus.CurrentStepIndex-1] + var revisionLabelKey string + if c.Workload != nil { + revisionLabelKey = c.Workload.RevisionLabelKey + } + return &trafficrouting.TrafficRoutingContext{ + Key: fmt.Sprintf("Rollout(%s/%s)", c.Rollout.Namespace, c.Rollout.Name), + Namespace: c.Rollout.Namespace, + ObjectRef: c.Rollout.Spec.Strategy.Canary.TrafficRoutings, + Strategy: currentStep.TrafficRoutingStrategy, + OwnerRef: *metav1.NewControllerRef(c.Rollout, rolloutControllerKind), + RevisionLabelKey: revisionLabelKey, + StableRevision: c.NewStatus.CanaryStatus.StableRevision, + CanaryRevision: c.NewStatus.CanaryStatus.PodTemplateHash, + LastUpdateTime: c.NewStatus.CanaryStatus.LastUpdateTime, + } +} diff --git a/pkg/controller/rollout/rollout_progressing_test.go b/pkg/controller/rollout/rollout_progressing_test.go index 57e4db73..c1443b0e 100644 --- a/pkg/controller/rollout/rollout_progressing_test.go +++ b/pkg/controller/rollout/rollout_progressing_test.go @@ -40,9 +40,41 @@ func TestReconcileRolloutProgressing(t *testing.T) { name string getObj func() ([]*apps.Deployment, []*apps.ReplicaSet) getNetwork func() ([]*corev1.Service, []*netv1.Ingress) - getRollout func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) + getRollout func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) expectStatus func() *v1alpha1.RolloutStatus + expectTr func() *v1alpha1.TrafficRouting }{ + { + name: "ReconcileRolloutProgressing init trafficRouting", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + rs1 := rsDemo.DeepCopy() + return []*apps.Deployment{dep1}, []*apps.ReplicaSet{rs1} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { + obj := rolloutDemo.DeepCopy() + obj.Annotations[v1alpha1.TrafficRoutingAnnotation] = "tr-demo" + return obj, nil, demoTR.DeepCopy() + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "56855c89f9" + s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + return s + }, + expectTr: func() *v1alpha1.TrafficRouting { + tr := demoTR.DeepCopy() + tr.Finalizers = []string{util.ProgressingRolloutFinalizer(rolloutDemo.Name)} + return tr + }, + }, { name: "ReconcileRolloutProgressing init -> rolling", getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { @@ -53,9 +85,11 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() - return obj, nil + tr := demoTR.DeepCopy() + tr.Finalizers = []string{util.ProgressingRolloutFinalizer(rolloutDemo.Name)} + return obj, nil, tr }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -98,7 +132,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" @@ -109,7 +143,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling util.SetRolloutCondition(&obj.Status, *cond) - return obj, nil + return obj, nil, nil }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -153,7 +187,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" @@ -165,7 +199,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling util.SetRolloutCondition(&obj.Status, *cond) - return obj, nil + return obj, nil, nil }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -201,8 +235,9 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() + obj.Annotations[v1alpha1.TrafficRoutingAnnotation] = "tr-demo" obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" @@ -220,7 +255,9 @@ func TestReconcileRolloutProgressing(t *testing.T) { CanaryReplicas: intstr.FromInt(1), }, } - return obj, br + tr := demoTR.DeepCopy() + tr.Finalizers = []string{util.ProgressingRolloutFinalizer(rolloutDemo.Name)} + return obj, br, tr }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -237,6 +274,10 @@ func TestReconcileRolloutProgressing(t *testing.T) { util.SetRolloutCondition(s, *cond) return s }, + expectTr: func() *v1alpha1.TrafficRouting { + tr := demoTR.DeepCopy() + return tr + }, }, { name: "ReconcileRolloutProgressing finalizing2", @@ -256,7 +297,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" @@ -276,7 +317,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { }, } br.Status.Phase = v1alpha1.RolloutPhaseCompleted - return obj, br + return obj, br, nil }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -312,7 +353,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" @@ -325,7 +366,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { cond.Reason = v1alpha1.ProgressingReasonFinalising cond.Status = corev1.ConditionTrue util.SetRolloutCondition(&obj.Status, *cond) - return obj, nil + return obj, nil, nil }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -348,7 +389,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { }, }, { - name: "ReconcileRolloutProgressing rolling -> rollback", + name: "ReconcileRolloutProgressing rolling -> rollback1", getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { dep1 := deploymentDemo.DeepCopy() dep1.Spec.Template.Spec.Containers[0].Image = "echoserver:v1" @@ -375,7 +416,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" @@ -387,7 +428,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling util.SetRolloutCondition(&obj.Status, *cond) - return obj, nil + return obj, nil, nil }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -405,7 +446,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { }, }, { - name: "ReconcileRolloutProgressing rolling -> rollback", + name: "ReconcileRolloutProgressing rolling -> rollback2", getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { dep1 := deploymentDemo.DeepCopy() dep1.Spec.Template.Spec.Containers[0].Image = "echoserver:v1" @@ -432,7 +473,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" @@ -444,7 +485,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling util.SetRolloutCondition(&obj.Status, *cond) - return obj, nil + return obj, nil, nil }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -489,7 +530,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} }, - getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease, *v1alpha1.TrafficRouting) { obj := rolloutDemo.DeepCopy() obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" @@ -503,7 +544,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling util.SetRolloutCondition(&obj.Status, *cond) - return obj, nil + return obj, nil, nil }, expectStatus: func() *v1alpha1.RolloutStatus { s := rolloutDemo.Status.DeepCopy() @@ -519,7 +560,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { deps, rss := cs.getObj() - rollout, br := cs.getRollout() + rollout, br, tr := cs.getRollout() fc := fake.NewClientBuilder().WithScheme(scheme).WithObjects(rollout, demoConf.DeepCopy()).Build() for _, rs := range rss { _ = fc.Create(context.TODO(), rs) @@ -530,6 +571,9 @@ func TestReconcileRolloutProgressing(t *testing.T) { if br != nil { _ = fc.Create(context.TODO(), br) } + if tr != nil { + _ = fc.Create(context.TODO(), tr) + } ss, in := cs.getNetwork() for _, obj := range ss { _ = fc.Create(context.TODO(), obj) @@ -556,6 +600,17 @@ func TestReconcileRolloutProgressing(t *testing.T) { } _ = r.updateRolloutStatusInternal(rollout, *newStatus) checkRolloutEqual(fc, t, client.ObjectKey{Name: rollout.Name}, cs.expectStatus()) + if cs.expectTr != nil { + expectTr := cs.expectTr() + obj := &v1alpha1.TrafficRouting{} + err = fc.Get(context.TODO(), client.ObjectKey{Name: expectTr.Name}, obj) + if err != nil { + t.Fatalf("get object failed: %s", err.Error()) + } + if !reflect.DeepEqual(obj.Finalizers, expectTr.Finalizers) { + t.Fatalf("expect(%s), but get(%s)", expectTr.Finalizers, obj.Finalizers) + } + } }) } } @@ -604,13 +659,19 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { obj := rolloutDemo.DeepCopy() obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, }, { - Weight: utilpointer.Int32(50), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(50), + }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, }, } return obj @@ -643,13 +704,19 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { obj := rolloutDemo.DeepCopy() obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, }, { - Weight: utilpointer.Int32(40), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, }, } return obj @@ -682,13 +749,19 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { obj := rolloutDemo.DeepCopy() obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(40), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, }, } return obj @@ -721,13 +794,19 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { obj := rolloutDemo.DeepCopy() obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(10), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(10), + }, }, { - Weight: utilpointer.Int32(30), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(30), + }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, }, } return obj @@ -760,14 +839,18 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { obj := rolloutDemo.DeepCopy() obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(2), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(2), + }, Replicas: &intstr.IntOrString{ Type: intstr.String, StrVal: "10%", }, }, { - Weight: utilpointer.Int32(3), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(3), + }, Replicas: &intstr.IntOrString{ Type: intstr.String, StrVal: "10%", @@ -819,7 +902,7 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { if err != nil { t.Fatalf(err.Error()) } - c := &util.RolloutContext{Rollout: rollout, Workload: workload} + c := &RolloutContext{Rollout: rollout, Workload: workload} newStepIndex, err := reconciler.recalculateCanaryStep(c) if err != nil { t.Fatalf(err.Error()) diff --git a/pkg/controller/rollout/rollout_status.go b/pkg/controller/rollout/rollout_status.go index bf62ccf1..f1bd8c10 100644 --- a/pkg/controller/rollout/rollout_status.go +++ b/pkg/controller/rollout/rollout_status.go @@ -66,6 +66,8 @@ func (r *RolloutReconciler) calculateRolloutStatus(rollout *v1alpha1.Rollout) (r klog.Infof("rollout(%s/%s) workload not found, and reset status be Initial", rollout.Namespace, rollout.Name) return false, newStatus, nil } + klog.V(5).Infof("rollout(%s/%s) workload(%s)", rollout.Namespace, rollout.Name, util.DumpJSON(workload)) + // todo, patch workload webhook labels // workload status generation is not equal to workload.generation if !workload.IsStatusConsistent { klog.Infof("rollout(%s/%s) workload status is inconsistent, then wait a moment", rollout.Namespace, rollout.Name) @@ -187,7 +189,7 @@ func (r *RolloutReconciler) reconcileRolloutTerminating(rollout *v1alpha1.Rollou klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) return nil, err } - c := &util.RolloutContext{Rollout: rollout, NewStatus: newStatus, Workload: workload} + c := &RolloutContext{Rollout: rollout, NewStatus: newStatus, Workload: workload} done, err := r.doFinalising(c) if err != nil { return nil, err diff --git a/pkg/controller/rollout/rollout_status_test.go b/pkg/controller/rollout/rollout_status_test.go index 0d4316cd..548087fe 100644 --- a/pkg/controller/rollout/rollout_status_test.go +++ b/pkg/controller/rollout/rollout_status_test.go @@ -63,10 +63,14 @@ func TestCalculateRolloutHash(t *testing.T) { obj := rolloutDemo.DeepCopy() obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(50), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(50), + }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, }, } return obj diff --git a/pkg/controller/rollouthistory/rollouthistory_controller_test.go b/pkg/controller/rollouthistory/rollouthistory_controller_test.go index efaea08f..52a73bd0 100644 --- a/pkg/controller/rollouthistory/rollouthistory_controller_test.go +++ b/pkg/controller/rollouthistory/rollouthistory_controller_test.go @@ -137,23 +137,29 @@ var ( Canary: &rolloutv1alpha1.CanaryStrategy{ Steps: []rolloutv1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(5), + TrafficRoutingStrategy: rolloutv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(5), + }, Pause: rolloutv1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, }, { - Weight: utilpointer.Int32(40), - Pause: rolloutv1alpha1.RolloutPause{}, + TrafficRoutingStrategy: rolloutv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, + Pause: rolloutv1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: rolloutv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: rolloutv1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, }, }, - TrafficRoutings: []*rolloutv1alpha1.TrafficRouting{ + TrafficRoutings: []rolloutv1alpha1.TrafficRoutingRef{ { Service: "service-demo", Ingress: &rolloutv1alpha1.IngressTrafficRouting{ diff --git a/pkg/controller/trafficrouting/trafficrouting_controller.go b/pkg/controller/trafficrouting/trafficrouting_controller.go new file mode 100644 index 00000000..eb62d3c2 --- /dev/null +++ b/pkg/controller/trafficrouting/trafficrouting_controller.go @@ -0,0 +1,225 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trafficrouting + +import ( + "context" + "flag" + "fmt" + "reflect" + "strings" + "time" + + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting" + "github.com/openkruise/rollouts/pkg/util" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var ( + concurrentReconciles = 3 + defaultGracePeriodSeconds int32 = 3 + trControllerKind = v1alpha1.SchemeGroupVersion.WithKind("TrafficRouting") +) + +func init() { + flag.IntVar(&concurrentReconciles, "trafficrouting-workers", 3, "Max concurrent workers for trafficrouting controller.") +} + +// TrafficRoutingReconciler reconciles a TrafficRouting object +type TrafficRoutingReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + trafficRoutingManager *trafficrouting.Manager +} + +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=trafficroutings,verbs=get;list;watch;create;update;patch +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=trafficroutings/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=trafficroutings/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the Rollout object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile +func (r *TrafficRoutingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // Fetch the Rollout instance + tr := &v1alpha1.TrafficRouting{} + err := r.Get(context.TODO(), req.NamespacedName, tr) + if err != nil { + if errors.IsNotFound(err) { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + klog.Infof("Begin to reconcile TrafficRouting %v", util.DumpJSON(tr)) + + // handle finalizer + err = r.handleFinalizer(tr) + if err != nil { + return ctrl.Result{}, err + } + newStatus := tr.Status.DeepCopy() + if newStatus.Phase == "" { + newStatus.Phase = v1alpha1.TrafficRoutingPhaseInitial + } + if !tr.DeletionTimestamp.IsZero() { + newStatus.Phase = v1alpha1.TrafficRoutingPhaseTerminating + } + var done = true + switch newStatus.Phase { + case v1alpha1.TrafficRoutingPhaseInitial: + err = r.trafficRoutingManager.InitializeTrafficRouting(newTrafficRoutingContext(tr)) + if err == nil { + newStatus.Phase = v1alpha1.TrafficRoutingPhaseHealthy + newStatus.Message = "TrafficRouting is Healthy" + } + case v1alpha1.TrafficRoutingPhaseHealthy: + if rolloutProgressingFinalizer(tr).Len() > 0 { + newStatus.Phase = v1alpha1.TrafficRoutingPhaseProgressing + newStatus.Message = "TrafficRouting is Progressing" + } + case v1alpha1.TrafficRoutingPhaseProgressing: + if rolloutProgressingFinalizer(tr).Len() == 0 { + newStatus.Phase = v1alpha1.TrafficRoutingPhaseFinalizing + newStatus.Message = "TrafficRouting is Finalizing" + } else { + done, err = r.trafficRoutingManager.DoTrafficRouting(newTrafficRoutingContext(tr)) + } + case v1alpha1.TrafficRoutingPhaseFinalizing: + done, err = r.trafficRoutingManager.FinalisingTrafficRouting(newTrafficRoutingContext(tr), false) + if done { + newStatus.Phase = v1alpha1.TrafficRoutingPhaseHealthy + newStatus.Message = "TrafficRouting is Healthy" + } + case v1alpha1.TrafficRoutingPhaseTerminating: + done, err = r.trafficRoutingManager.FinalisingTrafficRouting(newTrafficRoutingContext(tr), false) + if done { + // remove trafficRouting finalizer + err = r.handleFinalizer(tr) + } + } + if err != nil { + return ctrl.Result{}, err + } else if !done { + recheckTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) + return ctrl.Result{RequeueAfter: time.Until(recheckTime)}, nil + } + newStatus.ObservedGeneration = tr.Generation + return ctrl.Result{}, r.updateTrafficRoutingStatus(tr, *newStatus) +} + +func (r *TrafficRoutingReconciler) updateTrafficRoutingStatus(tr *v1alpha1.TrafficRouting, newStatus v1alpha1.TrafficRoutingStatus) error { + if reflect.DeepEqual(tr.Status, newStatus) { + return nil + } + var err error + trClone := tr.DeepCopy() + if err = r.Client.Get(context.TODO(), types.NamespacedName{Namespace: tr.Namespace, Name: tr.Name}, trClone); err != nil { + klog.Errorf("error getting updated trafficRouting(%s/%s) from client", tr.Namespace, tr.Name) + return err + } + trClone.Status = newStatus + if err = r.Client.Status().Update(context.TODO(), trClone); err != nil { + klog.Errorf("update trafficRouting(%s/%s) status failed: %s", tr.Namespace, tr.Name, err.Error()) + return err + } + klog.Infof("trafficRouting(%s/%s) status from(%s) -> to(%s) success", tr.Namespace, tr.Name, util.DumpJSON(tr.Status), util.DumpJSON(newStatus)) + return nil +} + +// handle adding and handle finalizer logic, it turns if we should continue to reconcile +func (r *TrafficRoutingReconciler) handleFinalizer(tr *v1alpha1.TrafficRouting) error { + // delete trafficRouting crd, remove finalizer + if !tr.DeletionTimestamp.IsZero() { + err := util.UpdateFinalizer(r.Client, tr, util.RemoveFinalizerOpType, util.TrafficRoutingFinalizer) + if err != nil { + klog.Errorf("remove trafficRouting(%s/%s) finalizer failed: %s", tr.Namespace, tr.Name, err.Error()) + return err + } + klog.Infof("remove trafficRouting(%s/%s) finalizer success", tr.Namespace, tr.Name) + return nil + } + + // create trafficRouting crd, add finalizer + if !controllerutil.ContainsFinalizer(tr, util.TrafficRoutingFinalizer) { + err := util.UpdateFinalizer(r.Client, tr, util.AddFinalizerOpType, util.TrafficRoutingFinalizer) + if err != nil { + klog.Errorf("register trafficRouting(%s/%s) finalizer failed: %s", tr.Namespace, tr.Name, err.Error()) + return err + } + klog.Infof("register trafficRouting(%s/%s) finalizer success", tr.Namespace, tr.Name) + } + return nil +} + +func rolloutProgressingFinalizer(tr *v1alpha1.TrafficRouting) sets.String { + progressing := sets.String{} + for _, s := range tr.GetFinalizers() { + if strings.Contains(s, v1alpha1.ProgressingRolloutFinalizerPrefix) { + progressing.Insert(s) + } + } + return progressing +} + +func (r *TrafficRoutingReconciler) SetupWithManager(mgr ctrl.Manager) error { + // Create a new controller + c, err := controller.New("trafficrouting-controller", mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + return err + } + // Watch for changes to trafficrouting + if err = c.Watch(&source.Kind{Type: &v1alpha1.TrafficRouting{}}, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + r.trafficRoutingManager = trafficrouting.NewTrafficRoutingManager(mgr.GetClient()) + return nil +} + +func newTrafficRoutingContext(tr *v1alpha1.TrafficRouting) *trafficrouting.TrafficRoutingContext { + return &trafficrouting.TrafficRoutingContext{ + Key: fmt.Sprintf("TrafficRouting(%s/%s)", tr.Namespace, tr.Name), + Namespace: tr.Namespace, + ObjectRef: tr.Spec.ObjectRef, + Strategy: tr.Spec.Strategy, + OwnerRef: *metav1.NewControllerRef(tr, trControllerKind), + OnlyTrafficRouting: true, + } +} diff --git a/pkg/controller/trafficrouting/trafficrouting_controller_test.go b/pkg/controller/trafficrouting/trafficrouting_controller_test.go new file mode 100644 index 00000000..6485a3f2 --- /dev/null +++ b/pkg/controller/trafficrouting/trafficrouting_controller_test.go @@ -0,0 +1,427 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trafficrouting + +import ( + "context" + "fmt" + "reflect" + "testing" + + kruisev1aplphal "github.com/openkruise/kruise-api/apps/v1alpha1" + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +var ( + scheme *runtime.Scheme + nginxIngressAnnotationDefaultPrefix = "nginx.ingress.kubernetes.io" + + demoService = corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + }, + Selector: map[string]string{ + "app": "echoserver", + }, + }, + } + + demoIngress = netv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "networking.k8s.io/v1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: netv1.IngressSpec{ + Rules: []netv1.IngressRule{ + { + Host: "echoserver.example.com", + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: "/apis/echo", + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "echoserver", + Port: netv1.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + demoTR = &v1alpha1.TrafficRouting{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rollouts.kruise.io/v1alpha1", + Kind: "TrafficRouting", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tr-demo", + Labels: map[string]string{}, + }, + Spec: v1alpha1.TrafficRoutingSpec{ + ObjectRef: []v1alpha1.TrafficRoutingRef{ + { + Service: "echoserver", + Ingress: &v1alpha1.IngressTrafficRouting{ + Name: "echoserver", + }, + }, + }, + Strategy: v1alpha1.TrafficRoutingStrategy{ + Matches: []v1alpha1.HttpRouteMatch{ + // header + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123456", + }, + }, + }, + }, + }, + }, + } + + demoConf = corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configuration.RolloutConfigurationName, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.nginx", configuration.LuaTrafficRoutingIngressTypePrefix): ` + annotations = obj.annotations + annotations["nginx.ingress.kubernetes.io/canary"] = "true" + annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil + annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil + annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil + annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil + annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil + if ( obj.weight ~= "-1" ) + then + annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight + end + if ( not obj.matches ) + then + return annotations + end + for _,match in ipairs(obj.matches) do + header = match.headers[1] + if ( header.name == "canary-by-cookie" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name + if ( header.type == "RegularExpression" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value + end + end + end + return annotations + `, + }, + } +) + +func init() { + scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = kruisev1aplphal.AddToScheme(scheme) + _ = v1alpha1.AddToScheme(scheme) +} + +func TestTrafficRoutingTest(t *testing.T) { + cases := []struct { + name string + getObj func() ([]*corev1.Service, []*netv1.Ingress) + getTrafficRouting func() *v1alpha1.TrafficRouting + expectObj func() ([]*corev1.Service, []*netv1.Ingress, *v1alpha1.TrafficRouting) + expectDone bool + }{ + { + name: "TrafficRouting reconcile Initial->Healthy", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getTrafficRouting: func() *v1alpha1.TrafficRouting { + return demoTR.DeepCopy() + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress, *v1alpha1.TrafficRouting) { + s1 := demoService.DeepCopy() + tr := demoTR.DeepCopy() + tr.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseHealthy, + } + return []*corev1.Service{s1}, []*netv1.Ingress{demoIngress.DeepCopy()}, tr + }, + expectDone: true, + }, + { + name: "TrafficRouting reconcile Initial->Progressing", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getTrafficRouting: func() *v1alpha1.TrafficRouting { + obj := demoTR.DeepCopy() + obj.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseHealthy, + } + obj.Finalizers = []string{fmt.Sprintf("%s/rollout-test", v1alpha1.ProgressingRolloutFinalizerPrefix), + util.TrafficRoutingFinalizer} + return obj + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress, *v1alpha1.TrafficRouting) { + s1 := demoService.DeepCopy() + tr := demoTR.DeepCopy() + tr.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseProgressing, + } + return []*corev1.Service{s1}, []*netv1.Ingress{demoIngress.DeepCopy()}, tr + }, + expectDone: true, + }, + { + name: "TrafficRouting reconcile Progressing, and create canary ingress", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getTrafficRouting: func() *v1alpha1.TrafficRouting { + obj := demoTR.DeepCopy() + obj.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseProgressing, + } + obj.Finalizers = []string{fmt.Sprintf("%s/rollout-test", v1alpha1.ProgressingRolloutFinalizerPrefix), + util.TrafficRoutingFinalizer} + return obj + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress, *v1alpha1.TrafficRouting) { + s1 := demoService.DeepCopy() + i1 := demoIngress.DeepCopy() + i2 := demoIngress.DeepCopy() + i2.Name = "echoserver-canary" + i2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + i2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "0" + tr := demoTR.DeepCopy() + tr.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseProgressing, + } + return []*corev1.Service{s1}, []*netv1.Ingress{i1, i2}, tr + }, + expectDone: false, + }, + { + name: "TrafficRouting reconcile Progressing, and set ingress headers, and return false", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + i1 := demoIngress.DeepCopy() + i2 := demoIngress.DeepCopy() + i2.Name = "echoserver-canary" + i2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + i2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "0" + return []*corev1.Service{s1}, []*netv1.Ingress{i1, i2} + }, + getTrafficRouting: func() *v1alpha1.TrafficRouting { + obj := demoTR.DeepCopy() + obj.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseProgressing, + } + obj.Finalizers = []string{fmt.Sprintf("%s/rollout-test", v1alpha1.ProgressingRolloutFinalizerPrefix), + util.TrafficRoutingFinalizer} + return obj + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress, *v1alpha1.TrafficRouting) { + s1 := demoService.DeepCopy() + i1 := demoIngress.DeepCopy() + i2 := demoIngress.DeepCopy() + i2.Name = "echoserver-canary" + i2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + i2.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" + i2.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456" + tr := demoTR.DeepCopy() + tr.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseProgressing, + } + return []*corev1.Service{s1}, []*netv1.Ingress{i1, i2}, tr + }, + expectDone: false, + }, + { + name: "TrafficRouting reconcile Progressing, and set ingress headers, and return true", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + i1 := demoIngress.DeepCopy() + i2 := demoIngress.DeepCopy() + i2.Name = "echoserver-canary" + i2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + i2.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" + i2.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456" + return []*corev1.Service{s1}, []*netv1.Ingress{i1, i2} + }, + getTrafficRouting: func() *v1alpha1.TrafficRouting { + obj := demoTR.DeepCopy() + obj.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseProgressing, + } + obj.Finalizers = []string{fmt.Sprintf("%s/rollout-test", v1alpha1.ProgressingRolloutFinalizerPrefix), + util.TrafficRoutingFinalizer} + return obj + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress, *v1alpha1.TrafficRouting) { + s1 := demoService.DeepCopy() + i1 := demoIngress.DeepCopy() + i2 := demoIngress.DeepCopy() + i2.Name = "echoserver-canary" + i2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + i2.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" + i2.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456" + tr := demoTR.DeepCopy() + tr.Status = v1alpha1.TrafficRoutingStatus{ + Phase: v1alpha1.TrafficRoutingPhaseProgressing, + } + return []*corev1.Service{s1}, []*netv1.Ingress{i1, i2}, tr + }, + expectDone: true, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + ss, ig := cs.getObj() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ig[0], ss[0], demoConf.DeepCopy()).Build() + if len(ss) == 2 { + _ = client.Create(context.TODO(), ss[1]) + } + if len(ig) == 2 { + _ = client.Create(context.TODO(), ig[1]) + } + tr := cs.getTrafficRouting() + _ = client.Create(context.TODO(), tr) + manager := TrafficRoutingReconciler{ + Client: client, + Scheme: scheme, + trafficRoutingManager: trafficrouting.NewTrafficRoutingManager(client), + } + result, err := manager.Reconcile(context.TODO(), ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: tr.Namespace, + Name: tr.Name, + }, + }) + if err != nil { + t.Fatalf("TrafficRouting Reconcile failed: %s", err) + } + if cs.expectDone != (result.RequeueAfter == 0) { + t.Fatalf("TrafficRouting Reconcile expect(%v), but get(%v)", cs.expectDone, result.RequeueAfter) + } + ss, ig, tr = cs.expectObj() + checkObjEqual(client, t, tr) + for _, obj := range ss { + checkObjEqual(client, t, obj) + } + for _, obj := range ig { + checkObjEqual(client, t, obj) + } + }) + } +} + +func checkObjEqual(c client.WithWatch, t *testing.T, expect client.Object) { + gvk := expect.GetObjectKind().GroupVersionKind() + obj := getEmptyObject(gvk) + err := c.Get(context.TODO(), client.ObjectKey{Namespace: expect.GetNamespace(), Name: expect.GetName()}, obj) + if err != nil { + t.Fatalf("get object failed: %s", err.Error()) + } + switch gvk.Kind { + case "Service": + s1 := obj.(*corev1.Service) + s2 := expect.(*corev1.Service) + if !reflect.DeepEqual(s1.Spec, s2.Spec) { + t.Fatalf("expect(%s), but get object(%s)", util.DumpJSON(s2.Spec), util.DumpJSON(s1.Spec)) + } + case "Ingress": + s1 := obj.(*netv1.Ingress) + s2 := expect.(*netv1.Ingress) + if !reflect.DeepEqual(s1.Spec, s2.Spec) || !reflect.DeepEqual(s1.Annotations, s2.Annotations) { + t.Fatalf("expect(%s), but get object(%s)", util.DumpJSON(s2), util.DumpJSON(s1)) + } + + case "TrafficRouting": + s1 := obj.(*v1alpha1.TrafficRouting) + s1.Status.Message = "" + s2 := expect.(*v1alpha1.TrafficRouting) + if !reflect.DeepEqual(s1.Status, s2.Status) { + t.Fatalf("expect(%s), but get object(%s)", util.DumpJSON(s2), util.DumpJSON(s1)) + } + } +} + +func getEmptyObject(gvk schema.GroupVersionKind) client.Object { + switch gvk.Kind { + case "Service": + return &corev1.Service{} + case "Ingress": + return &netv1.Ingress{} + case "TrafficRouting": + return &v1alpha1.TrafficRouting{} + } + return nil +} diff --git a/pkg/trafficrouting/manager.go b/pkg/trafficrouting/manager.go index 4b1c3157..d11e14e0 100644 --- a/pkg/trafficrouting/manager.go +++ b/pkg/trafficrouting/manager.go @@ -37,9 +37,27 @@ import ( var ( defaultGracePeriodSeconds int32 = 3 - rolloutControllerKind = v1alpha1.SchemeGroupVersion.WithKind("Rollout") ) +type TrafficRoutingContext struct { + // only for log info + Key string + Namespace string + ObjectRef []v1alpha1.TrafficRoutingRef + Strategy v1alpha1.TrafficRoutingStrategy + // OnlyTrafficRouting + OnlyTrafficRouting bool + OwnerRef metav1.OwnerReference + // workload.RevisionLabelKey + RevisionLabelKey string + // status.CanaryStatus.StableRevision + StableRevision string + // status.CanaryStatus.PodTemplateHash + CanaryRevision string + // newStatus.canaryStatus.LastUpdateTime + LastUpdateTime *metav1.Time +} + // Manager responsible for adjusting network resources // such as Service, Ingress, Gateway API, etc., to achieve traffic grayscale. type Manager struct { @@ -52,48 +70,44 @@ func NewTrafficRoutingManager(c client.Client) *Manager { // InitializeTrafficRouting determine if the network resources(service & ingress & gateway api) exist. // If it is Ingress, init method will create the canary ingress resources, and set weight=0. -func (m *Manager) InitializeTrafficRouting(c *util.RolloutContext) error { - if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { +func (m *Manager) InitializeTrafficRouting(c *TrafficRoutingContext) error { + if len(c.ObjectRef) == 0 { return nil } - sService := c.Rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service + objectRef := c.ObjectRef[0] + sService := objectRef.Service // check service service := &corev1.Service{} - if err := m.Get(context.TODO(), types.NamespacedName{Namespace: c.Rollout.Namespace, Name: sService}, service); err != nil { + if err := m.Get(context.TODO(), types.NamespacedName{Namespace: c.Namespace, Name: sService}, service); err != nil { return err } - cService := fmt.Sprintf("%s-canary", sService) + cService := getCanaryServiceName(sService, c.OnlyTrafficRouting) // new network provider, ingress or gateway - trController, err := newNetworkProvider(m.Client, c.Rollout, c.NewStatus, sService, cService) + trController, err := newNetworkProvider(m.Client, c, sService, cService) if err != nil { - klog.Errorf("rollout(%s/%s) newNetworkProvider failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + klog.Errorf("%s newNetworkProvider failed: %s", c.Key, err.Error()) return err } return trController.Initialize(context.TODO()) } -func (m *Manager) DoTrafficRouting(c *util.RolloutContext) (bool, error) { - if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { +func (m *Manager) DoTrafficRouting(c *TrafficRoutingContext) (bool, error) { + if len(c.ObjectRef) == 0 { return true, nil } - trafficRouting := c.Rollout.Spec.Strategy.Canary.TrafficRoutings[0] + trafficRouting := c.ObjectRef[0] if trafficRouting.GracePeriodSeconds <= 0 { trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds } - canaryStatus := c.NewStatus.CanaryStatus - currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] - if currentStep.Weight == nil && len(currentStep.Matches) == 0 { + if c.Strategy.Weight == nil && len(c.Strategy.Matches) == 0 { return true, nil } - if canaryStatus.StableRevision == "" || canaryStatus.PodTemplateHash == "" { - klog.Warningf("rollout(%s/%s) stableRevision or podTemplateHash can not be empty, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) - return false, nil - } + //fetch stable service stableService := &corev1.Service{} - err := m.Get(context.TODO(), client.ObjectKey{Namespace: c.Rollout.Namespace, Name: trafficRouting.Service}, stableService) + err := m.Get(context.TODO(), client.ObjectKey{Namespace: c.Namespace, Name: trafficRouting.Service}, stableService) if err != nil { - klog.Errorf("rollout(%s/%s) get stable service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service, err.Error()) + klog.Errorf("%s get stable service(%s) failed: %s", c.Key, trafficRouting.Service, err.Error()) // not found, wait a moment, retry if errors.IsNotFound(err) { return false, nil @@ -101,93 +115,99 @@ func (m *Manager) DoTrafficRouting(c *util.RolloutContext) (bool, error) { return false, err } // canary service name - canaryServiceName := fmt.Sprintf("%s-canary", trafficRouting.Service) - // fetch canary service + canaryServiceName := getCanaryServiceName(trafficRouting.Service, c.OnlyTrafficRouting) canaryService := &corev1.Service{} - err = m.Get(context.TODO(), client.ObjectKey{Namespace: c.Rollout.Namespace, Name: canaryServiceName}, canaryService) - if err != nil && !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) get canary service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, canaryServiceName, err.Error()) - return false, err - } else if errors.IsNotFound(err) { - canaryService, err = m.createCanaryService(c, canaryServiceName, *stableService.Spec.DeepCopy()) - if err != nil { + canaryService.Namespace = stableService.Namespace + canaryService.Name = canaryServiceName + // end-to-end canary deployment scenario(a -> b -> c), if only b or c is released, + //and a is not released in this scenario, then the canary service is not needed. + if !c.OnlyTrafficRouting { + if c.StableRevision == "" || c.CanaryRevision == "" { + klog.Warningf("%s stableRevision or podTemplateHash can not be empty, and wait a moment", c.Key) + return false, nil + } + // fetch canary service + err = m.Get(context.TODO(), client.ObjectKey{Namespace: c.Namespace, Name: canaryServiceName}, canaryService) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("%s get canary service(%s) failed: %s", c.Key, canaryServiceName, err.Error()) return false, err + } else if errors.IsNotFound(err) { + canaryService, err = m.createCanaryService(c, canaryServiceName, *stableService.Spec.DeepCopy()) + if err != nil { + return false, err + } } - } - // patch canary service only selector the canary pods - if canaryService.Spec.Selector[c.Workload.RevisionLabelKey] != canaryStatus.PodTemplateHash { - body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, c.Workload.RevisionLabelKey, canaryStatus.PodTemplateHash) - if err = m.Patch(context.TODO(), canaryService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { - klog.Errorf("rollout(%s/%s) patch canary service(%s) selector failed: %s", c.Rollout.Namespace, c.Rollout.Name, canaryService.Name, err.Error()) - return false, err + // patch canary service to only select the canary pods + if canaryService.Spec.Selector[c.RevisionLabelKey] != c.CanaryRevision { + body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, c.RevisionLabelKey, c.CanaryRevision) + if err = m.Patch(context.TODO(), canaryService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { + klog.Errorf("%s patch canary service(%s) selector failed: %s", c.Key, canaryService.Name, err.Error()) + return false, err + } + // update canary service time, and wait 3 seconds, just to be safe + c.LastUpdateTime = &metav1.Time{Time: time.Now()} + klog.Infof("%s patch canary service(%s) selector(%s=%s) success", + c.Key, canaryService.Name, c.RevisionLabelKey, c.CanaryRevision) } - // update canary service time, and wait 3 seconds, just to be safe - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - klog.Infof("rollout(%s/%s) patch canary service(%s) selector(%s=%s) success", - c.Rollout.Namespace, c.Rollout.Name, canaryService.Name, c.Workload.RevisionLabelKey, canaryStatus.PodTemplateHash) - } - // patch stable service only selector the stable pods - if stableService.Spec.Selector[c.Workload.RevisionLabelKey] != canaryStatus.StableRevision { - body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, c.Workload.RevisionLabelKey, canaryStatus.StableRevision) - if err = m.Patch(context.TODO(), stableService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { - klog.Errorf("rollout(%s/%s) patch stable service(%s) selector failed: %s", c.Rollout.Namespace, c.Rollout.Name, stableService.Name, err.Error()) - return false, err + // patch stable service to only select the stable pods + if stableService.Spec.Selector[c.RevisionLabelKey] != c.StableRevision { + body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, c.RevisionLabelKey, c.StableRevision) + if err = m.Patch(context.TODO(), stableService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { + klog.Errorf("%s patch stable service(%s) selector failed: %s", c.Key, stableService.Name, err.Error()) + return false, err + } + // update stable service time, and wait 3 seconds, just to be safe + c.LastUpdateTime = &metav1.Time{Time: time.Now()} + klog.Infof("add %s stable service(%s) selector(%s=%s) success", + c.Key, stableService.Name, c.RevisionLabelKey, c.StableRevision) + return false, nil + } + // After modify stable service configuration, give the network provider 3 seconds to react + if verifyTime := c.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { + klog.Infof("%s update service selector, and wait 3 seconds", c.Key) + return false, nil } - // update stable service time, and wait 3 seconds, just to be safe - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - klog.Infof("add rollout(%s/%s) stable service(%s) selector(%s=%s) success", - c.Rollout.Namespace, c.Rollout.Name, stableService.Name, c.Workload.RevisionLabelKey, canaryStatus.StableRevision) - return false, nil - } - // After modify stable service configuration, give the network provider 3 seconds to react - if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { - klog.Infof("rollout(%s/%s) update service selector, and wait 3 seconds", c.Rollout.Namespace, c.Rollout.Name) - return false, nil } // new network provider, ingress or gateway - trController, err := newNetworkProvider(m.Client, c.Rollout, c.NewStatus, stableService.Name, canaryService.Name) + trController, err := newNetworkProvider(m.Client, c, stableService.Name, canaryService.Name) if err != nil { - klog.Errorf("rollout(%s/%s) newNetworkProvider failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + klog.Errorf("%s newNetworkProvider failed: %s", c.Key, err.Error()) return false, err } - cStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] - steps := len(c.Rollout.Spec.Strategy.Canary.Steps) - cond := util.GetRolloutCondition(*c.NewStatus, v1alpha1.RolloutConditionProgressing) - cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and doing traffic routing", canaryStatus.CurrentStepIndex, steps) - verify, err := trController.EnsureRoutes(context.TODO(), cStep.Weight, cStep.Matches) + verify, err := trController.EnsureRoutes(context.TODO(), &c.Strategy) if err != nil { return false, err } else if !verify { - klog.Infof("rollout(%s/%s) is doing step(%d) trafficRouting(%s)", c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex, util.DumpJSON(cStep)) + klog.Infof("%s is doing trafficRouting(%s), and wait a moment", c.Key, util.DumpJSON(c.Strategy)) return false, nil } - klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%s) success", c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex, util.DumpJSON(cStep)) + klog.Infof("%s do trafficRouting(%s) success", c.Key, util.DumpJSON(c.Strategy)) return true, nil } -func (m *Manager) FinalisingTrafficRouting(c *util.RolloutContext, onlyRestoreStableService bool) (bool, error) { - if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { +func (m *Manager) FinalisingTrafficRouting(c *TrafficRoutingContext, onlyRestoreStableService bool) (bool, error) { + if len(c.ObjectRef) == 0 { return true, nil } - trafficRouting := c.Rollout.Spec.Strategy.Canary.TrafficRoutings[0] + trafficRouting := c.ObjectRef[0] if trafficRouting.GracePeriodSeconds <= 0 { trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds } - cServiceName := fmt.Sprintf("%s-canary", trafficRouting.Service) - trController, err := newNetworkProvider(m.Client, c.Rollout, c.NewStatus, trafficRouting.Service, cServiceName) + cServiceName := getCanaryServiceName(trafficRouting.Service, c.OnlyTrafficRouting) + trController, err := newNetworkProvider(m.Client, c, trafficRouting.Service, cServiceName) if err != nil { - klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + klog.Errorf("%s newTrafficRoutingController failed: %s", c.Key, err.Error()) return false, err } - cService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: c.Rollout.Namespace, Name: cServiceName}} + cService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: c.Namespace, Name: cServiceName}} // if canary svc has been already cleaned up, just return if err = m.Get(context.TODO(), client.ObjectKeyFromObject(cService), cService); err != nil { if !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) get canary service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, cServiceName, err.Error()) + klog.Errorf("%s get canary service(%s) failed: %s", c.Key, cServiceName, err.Error()) return false, err } // In rollout failure case, no canary-service will be created, this step ensures that the canary-ingress can be deleted in a time. @@ -197,10 +217,7 @@ func (m *Manager) FinalisingTrafficRouting(c *util.RolloutContext, onlyRestoreSt return true, nil } - if c.NewStatus.CanaryStatus == nil { - c.NewStatus.CanaryStatus = &v1alpha1.CanaryStatus{} - } - klog.Infof("rollout(%s/%s) start finalising traffic routing", c.Rollout.Namespace, c.Rollout.Name) + klog.Infof("%s start finalising traffic routing", c.Key) // remove stable service the pod revision selector, so stable service will be selector all version pods. verify, err := m.restoreStableService(c) if err != nil || !verify { @@ -210,17 +227,18 @@ func (m *Manager) FinalisingTrafficRouting(c *util.RolloutContext, onlyRestoreSt } // First route 100% traffic to stable service - verify, err = trController.EnsureRoutes(context.TODO(), utilpointer.Int32(0), nil) + c.Strategy.Weight = utilpointer.Int32(0) + verify, err = trController.EnsureRoutes(context.TODO(), &c.Strategy) if err != nil { return false, err } else if !verify { - c.NewStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + c.LastUpdateTime = &metav1.Time{Time: time.Now()} return false, nil } - if c.NewStatus.CanaryStatus.LastUpdateTime != nil { + if c.LastUpdateTime != nil { // After restore the stable service configuration, give network provider 3 seconds to react - if verifyTime := c.NewStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { - klog.Infof("rollout(%s/%s) route 100% traffic to stable service, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) + if verifyTime := c.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { + klog.Infof("%s route 100% traffic to stable service, and wait a moment", c.Key) return false, nil } } @@ -232,29 +250,29 @@ func (m *Manager) FinalisingTrafficRouting(c *util.RolloutContext, onlyRestoreSt // remove canary service err = m.Delete(context.TODO(), cService) if err != nil && !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) remove canary service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, cService.Name, err.Error()) + klog.Errorf("%s remove canary service(%s) failed: %s", c.Key, cService.Name, err.Error()) return false, err } - klog.Infof("rollout(%s/%s) remove canary service(%s) success", c.Rollout.Namespace, c.Rollout.Name, cService.Name) + klog.Infof("%s remove canary service(%s) success", c.Key, cService.Name) return true, nil } -func newNetworkProvider(c client.Client, rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutStatus, sService, cService string) (network.NetworkProvider, error) { - trafficRouting := rollout.Spec.Strategy.Canary.TrafficRoutings[0] +func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, cService string) (network.NetworkProvider, error) { + trafficRouting := con.ObjectRef[0] if trafficRouting.Ingress != nil { return ingress.NewIngressTrafficRouting(c, ingress.Config{ - RolloutName: rollout.Name, - RolloutNs: rollout.Namespace, + Key: con.Key, + Namespace: con.Namespace, CanaryService: cService, StableService: sService, TrafficConf: trafficRouting.Ingress, - OwnerRef: *metav1.NewControllerRef(rollout, rolloutControllerKind), + OwnerRef: con.OwnerRef, }) } if trafficRouting.Gateway != nil { return gateway.NewGatewayTrafficRouting(c, gateway.Config{ - RolloutName: rollout.Name, - RolloutNs: rollout.Namespace, + Key: con.Key, + Namespace: con.Namespace, CanaryService: cService, StableService: sService, TrafficConf: trafficRouting.Gateway, @@ -263,12 +281,12 @@ func newNetworkProvider(c client.Client, rollout *v1alpha1.Rollout, newStatus *v return nil, fmt.Errorf("TrafficRouting current only support Ingress or Gateway API") } -func (m *Manager) createCanaryService(c *util.RolloutContext, cService string, spec corev1.ServiceSpec) (*corev1.Service, error) { +func (m *Manager) createCanaryService(c *TrafficRoutingContext, cService string, spec corev1.ServiceSpec) (*corev1.Service, error) { canaryService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Namespace: c.Rollout.Namespace, + Namespace: c.Namespace, Name: cService, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(c.Rollout, rolloutControllerKind)}, + OwnerReferences: []metav1.OwnerReference{c.OwnerRef}, }, Spec: spec, } @@ -280,50 +298,57 @@ func (m *Manager) createCanaryService(c *util.RolloutContext, cService string, s canaryService.Spec.IPFamilyPolicy = nil canaryService.Spec.IPFamilies = nil canaryService.Spec.LoadBalancerIP = "" - canaryService.Spec.Selector[c.Workload.RevisionLabelKey] = c.NewStatus.CanaryStatus.PodTemplateHash + canaryService.Spec.Selector[c.RevisionLabelKey] = c.CanaryRevision err := m.Create(context.TODO(), canaryService) if err != nil && !errors.IsAlreadyExists(err) { - klog.Errorf("rollout(%s/%s) create canary service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, cService, err.Error()) + klog.Errorf("%s create canary service(%s) failed: %s", c.Key, cService, err.Error()) return nil, err } - klog.Infof("rollout(%s/%s) create canary service(%s) success", c.Rollout.Namespace, c.Rollout.Name, util.DumpJSON(canaryService)) + klog.Infof("%s create canary service(%s) success", c.Key, util.DumpJSON(canaryService)) return canaryService, nil } // remove stable service the pod revision selector, so stable service will be selector all version pods. -func (m *Manager) restoreStableService(c *util.RolloutContext) (bool, error) { - if c.Workload == nil { - return true, nil +func (m *Manager) restoreStableService(c *TrafficRoutingContext) (bool, error) { + trafficRouting := c.ObjectRef[0] + if trafficRouting.GracePeriodSeconds <= 0 { + trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds } - trafficRouting := c.Rollout.Spec.Strategy.Canary.TrafficRoutings[0] //fetch stable service stableService := &corev1.Service{} - err := m.Get(context.TODO(), client.ObjectKey{Namespace: c.Rollout.Namespace, Name: trafficRouting.Service}, stableService) + err := m.Get(context.TODO(), client.ObjectKey{Namespace: c.Namespace, Name: trafficRouting.Service}, stableService) if err != nil { if errors.IsNotFound(err) { return true, nil } - klog.Errorf("rollout(%s/%s) get stable service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service, err.Error()) + klog.Errorf("%s get stable service(%s) failed: %s", c.Key, trafficRouting.Service, err.Error()) return false, err } - if stableService.Spec.Selector[c.Workload.RevisionLabelKey] != "" { - body := fmt.Sprintf(`{"spec":{"selector":{"%s":null}}}`, c.Workload.RevisionLabelKey) + if stableService.Spec.Selector[c.RevisionLabelKey] != "" { + body := fmt.Sprintf(`{"spec":{"selector":{"%s":null}}}`, c.RevisionLabelKey) if err = m.Patch(context.TODO(), stableService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { - klog.Errorf("rollout(%s/%s) patch stable service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service, err.Error()) + klog.Errorf("%s patch stable service(%s) failed: %s", c.Key, trafficRouting.Service, err.Error()) return false, err } - klog.Infof("remove rollout(%s/%s) stable service(%s) pod revision selector, and wait a moment", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service) - c.NewStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + klog.Infof("remove %s stable service(%s) pod revision selector, and wait a moment", c.Key, trafficRouting.Service) + c.LastUpdateTime = &metav1.Time{Time: time.Now()} return false, nil } - if c.NewStatus.CanaryStatus.LastUpdateTime == nil { + if c.LastUpdateTime == nil { return true, nil } // After restore the stable service configuration, give network provider 3 seconds to react - if verifyTime := c.NewStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { - klog.Infof("rollout(%s/%s) restoring stable service(%s), and wait a moment", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service) + if verifyTime := c.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { + klog.Infof("%s restoring stable service(%s), and wait a moment", c.Key, trafficRouting.Service) return false, nil } - klog.Infof("rollout(%s/%s) doFinalising stable service(%s) success", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service) + klog.Infof("%s doFinalising stable service(%s) success", c.Key, trafficRouting.Service) return true, nil } + +func getCanaryServiceName(sService string, onlyTrafficRouting bool) string { + if onlyTrafficRouting { + return sService + } + return fmt.Sprintf("%s-canary", sService) +} diff --git a/pkg/trafficrouting/manager_test.go b/pkg/trafficrouting/manager_test.go index 77e3cf2e..94fa9c43 100644 --- a/pkg/trafficrouting/manager_test.go +++ b/pkg/trafficrouting/manager_test.go @@ -123,23 +123,31 @@ var ( Canary: &v1alpha1.CanaryStrategy{ Steps: []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(5), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(5), + }, Replicas: &intstr.IntOrString{IntVal: 1}, }, { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Replicas: &intstr.IntOrString{IntVal: 2}, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, Replicas: &intstr.IntOrString{IntVal: 6}, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Replicas: &intstr.IntOrString{IntVal: 10}, }, }, - TrafficRoutings: []*v1alpha1.TrafficRouting{ + TrafficRoutings: []v1alpha1.TrafficRoutingRef{ { Service: "echoserver", Ingress: &v1alpha1.IngressTrafficRouting{ @@ -283,7 +291,13 @@ func TestDoTrafficRouting(t *testing.T) { s2 := demoService.DeepCopy() s2.Name = "echoserver-canary" s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" - return []*corev1.Service{s1, s2}, []*netv1.Ingress{demoIngress.DeepCopy()} + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "0" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} }, getRollout: func() (*v1alpha1.Rollout, *util.Workload) { obj := demoRollout.DeepCopy() @@ -385,6 +399,10 @@ func TestDoTrafficRouting(t *testing.T) { for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { + if cs.name != "DoTrafficRouting test3" { + return + } + fmt.Println("start DoTrafficRouting test3") ss, ig := cs.getObj() client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ig[0], ss[0], demoConf.DeepCopy()).Build() if len(ss) == 2 { @@ -393,9 +411,20 @@ func TestDoTrafficRouting(t *testing.T) { if len(ig) == 2 { _ = client.Create(context.TODO(), ig[1]) } - c := &util.RolloutContext{} - c.Rollout, c.Workload = cs.getRollout() - c.NewStatus = c.Rollout.Status.DeepCopy() + rollout, workload := cs.getRollout() + newStatus := rollout.Status.DeepCopy() + currentStep := rollout.Spec.Strategy.Canary.Steps[newStatus.CanaryStatus.CurrentStepIndex-1] + c := &TrafficRoutingContext{ + Key: fmt.Sprintf("Rollout(%s/%s)", rollout.Namespace, rollout.Name), + Namespace: rollout.Namespace, + ObjectRef: rollout.Spec.Strategy.Canary.TrafficRoutings, + Strategy: currentStep.TrafficRoutingStrategy, + OwnerRef: *metav1.NewControllerRef(rollout, v1alpha1.SchemeGroupVersion.WithKind("Rollout")), + RevisionLabelKey: workload.RevisionLabelKey, + StableRevision: newStatus.CanaryStatus.StableRevision, + CanaryRevision: newStatus.CanaryStatus.PodTemplateHash, + LastUpdateTime: newStatus.CanaryStatus.LastUpdateTime, + } manager := NewTrafficRoutingManager(client) err := manager.InitializeTrafficRouting(c) if err != nil { @@ -621,9 +650,20 @@ func TestFinalisingTrafficRouting(t *testing.T) { if len(ig) == 2 { _ = client.Create(context.TODO(), ig[1]) } - c := &util.RolloutContext{} - c.Rollout, c.Workload = cs.getRollout() - c.NewStatus = c.Rollout.Status.DeepCopy() + rollout, workload := cs.getRollout() + newStatus := rollout.Status.DeepCopy() + currentStep := rollout.Spec.Strategy.Canary.Steps[newStatus.CanaryStatus.CurrentStepIndex-1] + c := &TrafficRoutingContext{ + Key: fmt.Sprintf("Rollout(%s/%s)", rollout.Namespace, rollout.Name), + Namespace: rollout.Namespace, + ObjectRef: rollout.Spec.Strategy.Canary.TrafficRoutings, + Strategy: currentStep.TrafficRoutingStrategy, + OwnerRef: *metav1.NewControllerRef(rollout, v1alpha1.SchemeGroupVersion.WithKind("Rollout")), + RevisionLabelKey: workload.RevisionLabelKey, + StableRevision: newStatus.CanaryStatus.StableRevision, + CanaryRevision: newStatus.CanaryStatus.PodTemplateHash, + LastUpdateTime: newStatus.CanaryStatus.LastUpdateTime, + } manager := NewTrafficRoutingManager(client) done, err := manager.FinalisingTrafficRouting(c, cs.onlyRestoreStableService) if err != nil { diff --git a/pkg/trafficrouting/network/gateway/gateway.go b/pkg/trafficrouting/network/gateway/gateway.go index 827beb7b..29392d50 100644 --- a/pkg/trafficrouting/network/gateway/gateway.go +++ b/pkg/trafficrouting/network/gateway/gateway.go @@ -29,8 +29,9 @@ import ( ) type Config struct { - RolloutName string - RolloutNs string + // only for log info + Key string + Namespace string CanaryService string StableService string TrafficConf *rolloutv1alpha1.GatewayTrafficRouting @@ -52,17 +53,20 @@ func NewGatewayTrafficRouting(client client.Client, conf Config) (network.Networ func (r *gatewayController) Initialize(ctx context.Context) error { route := &gatewayv1alpha2.HTTPRoute{} - return r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, route) + return r.Get(ctx, types.NamespacedName{Namespace: r.conf.Namespace, Name: *r.conf.TrafficConf.HTTPRouteName}, route) } -func (r *gatewayController) EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error) { +func (r *gatewayController) EnsureRoutes(ctx context.Context, strategy *rolloutv1alpha1.TrafficRoutingStrategy) (bool, error) { + weight := strategy.Weight + matches := strategy.Matches + // headerModifier := strategy.RequestHeaderModifier var httpRoute gatewayv1alpha2.HTTPRoute - err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, &httpRoute) + err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.Namespace, Name: *r.conf.TrafficConf.HTTPRouteName}, &httpRoute) if err != nil { return false, err } // desired route - desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, weight, matches) + desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, weight, matches, nil) if reflect.DeepEqual(httpRoute.Spec.Rules, desiredRule) { return true, nil } @@ -76,25 +80,25 @@ func (r *gatewayController) EnsureRoutes(ctx context.Context, weight *int32, mat routeClone.Spec.Rules = desiredRule return r.Client.Update(context.TODO(), routeClone) }); err != nil { - klog.Errorf("update rollout(%s/%s) httpRoute(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, httpRoute.Name, err.Error()) + klog.Errorf("update %s httpRoute(%s) failed: %s", r.conf.Key, httpRoute.Name, err.Error()) return false, err } - klog.Infof("rollout(%s/%s) set HTTPRoute(name:%s weight:%d) success", r.conf.RolloutNs, r.conf.RolloutName, *r.conf.TrafficConf.HTTPRouteName, *weight) + klog.Infof("%s set HTTPRoute(name:%s weight:%d) success", r.conf.Key, *r.conf.TrafficConf.HTTPRouteName, *weight) return false, nil } func (r *gatewayController) Finalise(ctx context.Context) error { httpRoute := &gatewayv1alpha2.HTTPRoute{} - err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, httpRoute) + err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.Namespace, Name: *r.conf.TrafficConf.HTTPRouteName}, httpRoute) if err != nil { if errors.IsNotFound(err) { return nil } - klog.Errorf("rollout(%s/%s) get HTTPRoute failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + klog.Errorf("%s get HTTPRoute failed: %s", r.conf.Key, err.Error()) return err } // desired rule - desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, utilpointer.Int32(-1), nil) + desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, utilpointer.Int32(-1), nil, nil) if reflect.DeepEqual(httpRoute.Spec.Rules, desiredRule) { return nil } @@ -107,14 +111,15 @@ func (r *gatewayController) Finalise(ctx context.Context) error { routeClone.Spec.Rules = desiredRule return r.Client.Update(context.TODO(), routeClone) }); err != nil { - klog.Errorf("update rollout(%s/%s) httpRoute(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, httpRoute.Name, err.Error()) + klog.Errorf("update %s httpRoute(%s) failed: %s", r.conf.Key, httpRoute.Name, err.Error()) return err } - klog.Infof("rollout(%s/%s) TrafficRouting Finalise success", r.conf.RolloutNs, r.conf.RolloutName) + klog.Infof("%s TrafficRouting Finalise success", r.conf.Key) return nil } -func (r *gatewayController) buildDesiredHTTPRoute(rules []gatewayv1alpha2.HTTPRouteRule, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) []gatewayv1alpha2.HTTPRouteRule { +func (r *gatewayController) buildDesiredHTTPRoute(rules []gatewayv1alpha2.HTTPRouteRule, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch, + rh *gatewayv1alpha2.HTTPRequestHeaderFilter) []gatewayv1alpha2.HTTPRouteRule { var desired []gatewayv1alpha2.HTTPRouteRule // Only when finalize method parameter weight=-1, // then we need to remove the canary route policy and restore to the original configuration diff --git a/pkg/trafficrouting/network/gateway/gateway_test.go b/pkg/trafficrouting/network/gateway/gateway_test.go index 0eeeaeb5..5e867319 100644 --- a/pkg/trafficrouting/network/gateway/gateway_test.go +++ b/pkg/trafficrouting/network/gateway/gateway_test.go @@ -443,7 +443,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { } conf := Config{ - RolloutName: "rollout-demo", + Key: "rollout-demo", CanaryService: "store-svc-canary", StableService: "store-svc", } @@ -452,7 +452,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { t.Run(cs.name, func(t *testing.T) { controller := &gatewayController{conf: conf} weight, matches := cs.getRoutes() - current := controller.buildDesiredHTTPRoute(cs.getRouteRules(), weight, matches) + current := controller.buildDesiredHTTPRoute(cs.getRouteRules(), weight, matches, nil) desired := cs.desiredRules() if !reflect.DeepEqual(current, desired) { t.Fatalf("expect: %v, but get %v", util.DumpJSON(desired), util.DumpJSON(current)) diff --git a/pkg/trafficrouting/network/ingress/ingress.go b/pkg/trafficrouting/network/ingress/ingress.go index a7c4484d..e54b0779 100644 --- a/pkg/trafficrouting/network/ingress/ingress.go +++ b/pkg/trafficrouting/network/ingress/ingress.go @@ -50,8 +50,9 @@ type ingressController struct { } type Config struct { - RolloutName string - RolloutNs string + // only for log info + Key string + Namespace string CanaryService string StableService string TrafficConf *rolloutv1alpha1.IngressTrafficRouting @@ -73,50 +74,51 @@ func NewIngressTrafficRouting(client client.Client, conf Config) (network.Networ return r, nil } -// Initialize verify the existence of the ingress resource and generate the canary ingress +// Initialize only determine if the network resources(ingress & gateway api) exist. +// If error is nil, then the network resources exist. func (r *ingressController) Initialize(ctx context.Context) error { ingress := &netv1.Ingress{} - err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.TrafficConf.Name}, ingress) - if err != nil { - return err - } - canaryIngress := &netv1.Ingress{} - err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.canaryIngressName}, canaryIngress) - if err != nil && !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.canaryIngressName, err.Error()) - return err - } else if err == nil { - return nil - } - - // build and create canary ingress - canaryIngress = r.buildCanaryIngress(ingress) - canaryIngress.Annotations, err = r.executeLuaForCanary(canaryIngress.Annotations, utilpointer.Int32(0), nil) - if err != nil { - klog.Errorf("rollout(%s/%s) execute lua failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) - return err - } - if err = r.Create(ctx, canaryIngress); err != nil { - klog.Errorf("rollout(%s/%s) create canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) - return err - } - klog.Infof("rollout(%s/%s) create canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, util.DumpJSON(canaryIngress)) - return nil + return r.Get(ctx, types.NamespacedName{Namespace: r.conf.Namespace, Name: r.conf.TrafficConf.Name}, ingress) } -func (r *ingressController) EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error) { +func (r *ingressController) EnsureRoutes(ctx context.Context, strategy *rolloutv1alpha1.TrafficRoutingStrategy) (bool, error) { + weight := strategy.Weight + matches := strategy.Matches + // headerModifier := strategy.RequestHeaderModifier + canaryIngress := &netv1.Ingress{} - err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: defaultCanaryIngressName(r.conf.TrafficConf.Name)}, canaryIngress) - if err != nil { - if weight != nil && *weight == 0 && errors.IsNotFound(err) { + err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.Namespace, Name: defaultCanaryIngressName(r.conf.TrafficConf.Name)}, canaryIngress) + if errors.IsNotFound(err) { + // finalizer scenario, canary ingress maybe not found + if weight != nil && *weight == 0 { return true, nil } - klog.Errorf("rollout(%s/%s) get canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + // create canary ingress + ingress := &netv1.Ingress{} + err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.Namespace, Name: r.conf.TrafficConf.Name}, ingress) + if err != nil { + return false, err + } + // build and create canary ingress + canaryIngress = r.buildCanaryIngress(ingress) + canaryIngress.Annotations, err = r.executeLuaForCanary(canaryIngress.Annotations, utilpointer.Int32(0), nil) + if err != nil { + klog.Errorf("%s execute lua failed: %s", r.conf.Key, err.Error()) + return false, err + } + if err = r.Create(ctx, canaryIngress); err != nil { + klog.Errorf("%s create canary ingress failed: %s", r.conf.Key, err.Error()) + return false, err + } + klog.Infof("%s create canary ingress(%s) success", r.conf.Key, util.DumpJSON(canaryIngress)) + return false, nil + } else if err != nil { + klog.Errorf("%s get canary ingress failed: %s", r.conf.Key, err.Error()) return false, err } newAnnotations, err := r.executeLuaForCanary(canaryIngress.Annotations, weight, matches) if err != nil { - klog.Errorf("rollout(%s/%s) execute lua failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + klog.Errorf("%s execute lua failed: %s", r.conf.Key, err.Error()) return false, err } if reflect.DeepEqual(canaryIngress.Annotations, newAnnotations) { @@ -126,23 +128,23 @@ func (r *ingressController) EnsureRoutes(ctx context.Context, weight *int32, mat byte2, _ := json.Marshal(metav1.ObjectMeta{Annotations: newAnnotations}) patch, err := jsonpatch.CreateMergePatch(byte1, byte2) if err != nil { - klog.Errorf("rollout(%s/%s) create merge patch failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + klog.Errorf("%s create merge patch failed: %s", r.conf.Key, err.Error()) return false, err } body := fmt.Sprintf(`{"metadata":%s}`, string(patch)) if err = r.Patch(ctx, canaryIngress, client.RawPatch(types.MergePatchType, []byte(body))); err != nil { - klog.Errorf("rollout(%s/%s) set canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error()) + klog.Errorf("%s set canary ingress(%s) failed: %s", r.conf.Key, canaryIngress.Name, err.Error()) return false, err } - klog.Infof("rollout(%s/%s) set canary ingress(%s) annotations(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, util.DumpJSON(newAnnotations)) + klog.Infof("%s set canary ingress(%s) annotations(%s) success", r.conf.Key, canaryIngress.Name, util.DumpJSON(newAnnotations)) return false, nil } func (r *ingressController) Finalise(ctx context.Context) error { canaryIngress := &netv1.Ingress{} - err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.canaryIngressName}, canaryIngress) + err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.Namespace, Name: r.canaryIngressName}, canaryIngress) if err != nil && !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.canaryIngressName, err.Error()) + klog.Errorf("%s get canary ingress(%s) failed: %s", r.conf.Key, r.canaryIngressName, err.Error()) return err } if errors.IsNotFound(err) || !canaryIngress.DeletionTimestamp.IsZero() { @@ -150,10 +152,10 @@ func (r *ingressController) Finalise(ctx context.Context) error { } // immediate delete canary ingress if err = r.Delete(ctx, canaryIngress); err != nil { - klog.Errorf("rollout(%s/%s) remove canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error()) + klog.Errorf("%s remove canary ingress(%s) failed: %s", r.conf.Key, canaryIngress.Name, err.Error()) return err } - klog.Infof("rollout(%s/%s) remove canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name) + klog.Infof("%s remove canary ingress(%s) success", r.conf.Key, canaryIngress.Name) return nil } @@ -215,6 +217,7 @@ func defaultCanaryIngressName(name string) string { } func (r *ingressController) executeLuaForCanary(annotations map[string]string, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (map[string]string, error) { + if weight == nil { // the lua script does not have a pointer type, // so we need to pass weight=-1 to indicate the case where weight is nil. @@ -225,12 +228,14 @@ func (r *ingressController) executeLuaForCanary(annotations map[string]string, w Weight string Matches []rolloutv1alpha1.HttpRouteMatch CanaryService string + //todo, RequestHeaderModifier *gatewayv1alpha2.HTTPRequestHeaderFilter } data := &LuaData{ Annotations: annotations, Weight: fmt.Sprintf("%d", *weight), Matches: matches, CanaryService: r.conf.CanaryService, + // RequestHeaderModifier: headerModifier, } unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data) if err != nil { diff --git a/pkg/trafficrouting/network/ingress/ingress_test.go b/pkg/trafficrouting/network/ingress/ingress_test.go index 96c87828..d7c9d77f 100644 --- a/pkg/trafficrouting/network/ingress/ingress_test.go +++ b/pkg/trafficrouting/network/ingress/ingress_test.go @@ -44,6 +44,12 @@ var ( }, Data: map[string]string{ fmt.Sprintf("%s.nginx", configuration.LuaTrafficRoutingIngressTypePrefix): ` + function split(input, delimiter) + local arr = {} + string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end) + return arr + end + annotations = obj.annotations annotations["nginx.ingress.kubernetes.io/canary"] = "true" annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil @@ -55,6 +61,14 @@ var ( then annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight end + if ( obj.requestHeaderModifier ) + then + local str = '' + for _,header in ipairs(obj.requestHeaderModifier.set) do + str = str..string.format("%s %s\n", header.name, header.value) + end + annotations["mse.ingress.kubernetes.io/request-header-control-update"] = str + end if ( not obj.matches ) then return annotations @@ -240,10 +254,9 @@ func init() { func TestInitialize(t *testing.T) { cases := []struct { - name string - getConfigmap func() *corev1.ConfigMap - getIngress func() []*netv1.Ingress - expectIngress func() *netv1.Ingress + name string + getConfigmap func() *corev1.ConfigMap + getIngress func() []*netv1.Ingress }{ { name: "init test1", @@ -253,21 +266,11 @@ func TestInitialize(t *testing.T) { getIngress: func() []*netv1.Ingress { return []*netv1.Ingress{demoIngress.DeepCopy()} }, - expectIngress: func() *netv1.Ingress { - expect := demoIngress.DeepCopy() - expect.Name = "echoserver-canary" - expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" - expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0" - expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1] - expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" - expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" - return expect - }, }, } config := Config{ - RolloutName: "rollout-demo", + Key: "rollout-demo", StableService: "echoserver", CanaryService: "echoserver-canary", TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{ @@ -291,17 +294,6 @@ func TestInitialize(t *testing.T) { t.Fatalf("Initialize failed: %s", err.Error()) return } - canaryIngress := &netv1.Ingress{} - err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress) - if err != nil { - t.Fatalf("Get canary ingress failed: %s", err.Error()) - return - } - expect := cs.expectIngress() - if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) || - !reflect.DeepEqual(canaryIngress.Spec, expect.Spec) { - t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress)) - } }) } } @@ -311,7 +303,7 @@ func TestEnsureRoutes(t *testing.T) { name string getConfigmap func() *corev1.ConfigMap getIngress func() []*netv1.Ingress - getRoutes func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) + getRoutes func() *rolloutsv1alpha1.CanaryStep expectIngress func() *netv1.Ingress ingressType string }{ @@ -330,25 +322,42 @@ func TestEnsureRoutes(t *testing.T) { canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" return []*netv1.Ingress{demoIngress.DeepCopy(), canary} }, - getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { - return nil, []rolloutsv1alpha1.HttpRouteMatch{ - // header - { - Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + getRoutes: func() *rolloutsv1alpha1.CanaryStep { + return &rolloutsv1alpha1.CanaryStep{ + TrafficRoutingStrategy: rolloutsv1alpha1.TrafficRoutingStrategy{ + Weight: nil, + Matches: []rolloutsv1alpha1.HttpRouteMatch{ + // header { - Name: "user_id", - Value: "123456", + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123456", + }, + }, }, - }, - }, - // cookies - { - Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + // cookies { - Name: "canary-by-cookie", - Value: "demo", + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "canary-by-cookie", + Value: "demo", + }, + }, }, }, + /*RequestHeaderModifier: &gatewayv1alpha2.HTTPRequestHeaderFilter{ + Set: []gatewayv1alpha2.HTTPHeader{ + { + Name: "gray", + Value: "blue", + }, + { + Name: "gray", + Value: "green", + }, + }, + },*/ }, } }, @@ -359,6 +368,7 @@ func TestEnsureRoutes(t *testing.T) { expect.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo" expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456" + //expect.Annotations["mse.ingress.kubernetes.io/request-header-control-update"] = "gray blue\ngray green\n" expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1] expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" @@ -382,8 +392,12 @@ func TestEnsureRoutes(t *testing.T) { canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" return []*netv1.Ingress{demoIngress.DeepCopy(), canary} }, - getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { - return utilpointer.Int32(40), nil + getRoutes: func() *rolloutsv1alpha1.CanaryStep { + return &rolloutsv1alpha1.CanaryStep{ + TrafficRoutingStrategy: rolloutsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, + } }, expectIngress: func() *netv1.Ingress { expect := demoIngress.DeepCopy() @@ -413,16 +427,20 @@ func TestEnsureRoutes(t *testing.T) { canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" return []*netv1.Ingress{demoIngress.DeepCopy(), canary} }, - getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { + getRoutes: func() *rolloutsv1alpha1.CanaryStep { iType := gatewayv1alpha2.HeaderMatchRegularExpression - return nil, []rolloutsv1alpha1.HttpRouteMatch{ - // header - { - Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + return &rolloutsv1alpha1.CanaryStep{ + TrafficRoutingStrategy: rolloutsv1alpha1.TrafficRoutingStrategy{ + Matches: []rolloutsv1alpha1.HttpRouteMatch{ + // header { - Name: "user_id", - Value: "123*", - Type: &iType, + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123*", + Type: &iType, + }, + }, }, }, }, @@ -457,22 +475,26 @@ func TestEnsureRoutes(t *testing.T) { canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" return []*netv1.Ingress{demoIngress.DeepCopy(), canary} }, - getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { - return nil, []rolloutsv1alpha1.HttpRouteMatch{ - // header - { - Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + getRoutes: func() *rolloutsv1alpha1.CanaryStep { + return &rolloutsv1alpha1.CanaryStep{ + TrafficRoutingStrategy: rolloutsv1alpha1.TrafficRoutingStrategy{ + Matches: []rolloutsv1alpha1.HttpRouteMatch{ + // header { - Name: "Cookie", - Value: "demo1=value1;demo2=value2", - }, - { - Name: "SourceIp", - Value: "192.168.0.0/16;172.16.0.0/16", - }, - { - Name: "headername", - Value: "headervalue1;headervalue2", + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "Cookie", + Value: "demo1=value1;demo2=value2", + }, + { + Name: "SourceIp", + Value: "192.168.0.0/16;172.16.0.0/16", + }, + { + Name: "headername", + Value: "headervalue1;headervalue2", + }, + }, }, }, }, @@ -493,7 +515,7 @@ func TestEnsureRoutes(t *testing.T) { } config := Config{ - RolloutName: "rollout-demo", + Key: "rollout-demo", StableService: "echoserver", CanaryService: "echoserver-canary", TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{ @@ -513,8 +535,8 @@ func TestEnsureRoutes(t *testing.T) { t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error()) return } - weight, matches := cs.getRoutes() - _, err = controller.EnsureRoutes(context.TODO(), weight, matches) + step := cs.getRoutes() + _, err = controller.EnsureRoutes(context.TODO(), &step.TrafficRoutingStrategy) if err != nil { t.Fatalf("EnsureRoutes failed: %s", err.Error()) return @@ -528,7 +550,7 @@ func TestEnsureRoutes(t *testing.T) { expect := cs.expectIngress() if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) || !reflect.DeepEqual(canaryIngress.Spec, expect.Spec) { - t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress)) + t.Fatalf("but get(%s)", util.DumpJSON(canaryIngress)) } }) } @@ -563,7 +585,7 @@ func TestFinalise(t *testing.T) { } config := Config{ - RolloutName: "rollout-demo", + Key: "rollout-demo", StableService: "echoserver", CanaryService: "echoserver-canary", TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{ diff --git a/pkg/trafficrouting/network/interface.go b/pkg/trafficrouting/network/interface.go index 887ef1a5..8cf0faf5 100644 --- a/pkg/trafficrouting/network/interface.go +++ b/pkg/trafficrouting/network/interface.go @@ -24,8 +24,8 @@ import ( // NetworkProvider common function across all TrafficRouting implementation type NetworkProvider interface { - // Initialize determine if the network resources(ingress & gateway api) exist. - // If it is Ingress, init method will create the canary ingress resources, and set weight=0. + // Initialize only determine if the network resources(ingress & gateway api) exist. + // If error is nil, then the network resources exist. Initialize(ctx context.Context) error // EnsureRoutes check and set canary weight and matches. // weight indicates percentage of traffic to canary service, and range of values[0,100] @@ -33,7 +33,7 @@ type NetworkProvider interface { // 1. check if canary has been set desired weight. // 2. If not, set canary desired weight // When the first set weight is returned false, mainly to give the provider some time to process, only when again ensure, will return true - EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error) + EnsureRoutes(ctx context.Context, strategy *rolloutv1alpha1.TrafficRoutingStrategy) (bool, error) // Finalise will do some cleanup work after the canary rollout complete, such as delete canary ingress. // Finalise is called with a 3-second delay after completing the canary. Finalise(ctx context.Context) error diff --git a/pkg/util/constant.go b/pkg/util/constant.go index 23fbd3c1..02ff7226 100644 --- a/pkg/util/constant.go +++ b/pkg/util/constant.go @@ -16,6 +16,12 @@ limitations under the License. package util +import ( + "fmt" + + "github.com/openkruise/rollouts/api/v1alpha1" +) + // For Rollout and BatchRelease const ( // BatchReleaseControlAnnotation is controller info about batchRelease when rollout @@ -41,6 +47,10 @@ const ( DeploymentRevisionAnnotation = "deployment.kubernetes.io/revision" ) +const ( + TrafficRoutingFinalizer = "rollouts.kruise.io/trafficrouting" +) + // For Pods const ( // NoNeedUpdatePodLabel will be patched to pod when rollback in batches if the pods no need to rollback @@ -57,6 +67,7 @@ const ( CloneSetType WorkloadType = "cloneset" DeploymentType WorkloadType = "deployment" StatefulSetType WorkloadType = "statefulset" + DaemonSetType WorkloadType = "daemonset" AddFinalizerOpType FinalizerOpType = "Add" RemoveFinalizerOpType FinalizerOpType = "Remove" @@ -65,3 +76,7 @@ const ( type WorkloadType string type FinalizerOpType string + +func ProgressingRolloutFinalizer(name string) string { + return fmt.Sprintf("%s/%s", v1alpha1.ProgressingRolloutFinalizerPrefix, name) +} diff --git a/pkg/util/context.go b/pkg/util/context.go deleted file mode 100644 index 4fb7b69a..00000000 --- a/pkg/util/context.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "time" - - "github.com/openkruise/rollouts/api/v1alpha1" -) - -type RolloutContext struct { - Rollout *v1alpha1.Rollout - NewStatus *v1alpha1.RolloutStatus - // related workload - Workload *Workload - // reconcile RequeueAfter recheckTime - RecheckTime *time.Time - // wait stable workload pods ready - WaitReady bool -} diff --git a/pkg/util/controller_finder.go b/pkg/util/controller_finder.go index a4ea6588..dd15519a 100644 --- a/pkg/util/controller_finder.go +++ b/pkg/util/controller_finder.go @@ -37,6 +37,7 @@ import ( // Workload is used to return (controller, scale, selector) fields from the // controller finder functions. type Workload struct { + metav1.TypeMeta metav1.ObjectMeta // replicas @@ -49,6 +50,8 @@ type Workload struct { PodTemplateHash string // Revision hash key RevisionLabelKey string + // label selector + Selector *metav1.LabelSelector // Is it in rollback phase IsInRollback bool @@ -159,6 +162,7 @@ func (r *ControllerFinder) getKruiseCloneSet(namespace string, ref *rolloutv1alp StableRevision: cloneSet.Status.CurrentRevision[strings.LastIndex(cloneSet.Status.CurrentRevision, "-")+1:], CanaryRevision: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], ObjectMeta: cloneSet.ObjectMeta, + TypeMeta: cloneSet.TypeMeta, Replicas: *cloneSet.Spec.Replicas, PodTemplateHash: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], IsStatusConsistent: true, @@ -200,6 +204,7 @@ func (r *ControllerFinder) getKruiseDaemonSet(namespace string, ref *rolloutv1al //StableRevision: daemonSet.Status.CurrentRevision[strings.LastIndex(cloneSet.Status.CurrentRevision, "-")+1:], CanaryRevision: daemonSet.Status.DaemonSetHash[strings.LastIndex(daemonSet.Status.DaemonSetHash, "-")+1:], ObjectMeta: daemonSet.ObjectMeta, + TypeMeta: daemonSet.TypeMeta, Replicas: daemonSet.Status.DesiredNumberScheduled, PodTemplateHash: daemonSet.Status.DaemonSetHash[strings.LastIndex(daemonSet.Status.DaemonSetHash, "-")+1:], IsStatusConsistent: true, @@ -246,6 +251,7 @@ func (r *ControllerFinder) getAdvancedDeployment(namespace string, ref *rolloutv StableRevision: stableRevision, CanaryRevision: ComputeHash(&deployment.Spec.Template, nil), ObjectMeta: deployment.ObjectMeta, + TypeMeta: deployment.TypeMeta, Replicas: *deployment.Spec.Replicas, IsStatusConsistent: true, } @@ -299,11 +305,13 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1alpha1. workload := &Workload{ ObjectMeta: stable.ObjectMeta, + TypeMeta: stable.TypeMeta, Replicas: *stable.Spec.Replicas, IsStatusConsistent: true, StableRevision: stableRs.Labels[apps.DefaultDeploymentUniqueLabelKey], CanaryRevision: ComputeHash(&stable.Spec.Template, nil), RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey, + Selector: stable.Spec.Selector, } // not in rollout progressing @@ -357,6 +365,7 @@ func (r *ControllerFinder) getStatefulSetLikeWorkload(namespace string, ref *rol StableRevision: workloadInfo.Status.StableRevision, CanaryRevision: workloadInfo.Status.UpdateRevision, ObjectMeta: workloadInfo.ObjectMeta, + TypeMeta: workloadInfo.TypeMeta, Replicas: workloadInfo.Replicas, PodTemplateHash: workloadInfo.Status.UpdateRevision, IsStatusConsistent: true, diff --git a/pkg/util/lua_configuration.go b/pkg/util/lua_configuration.go index 83fe3be9..e24c6638 100644 --- a/pkg/util/lua_configuration.go +++ b/pkg/util/lua_configuration.go @@ -32,7 +32,7 @@ func init() { luaConfigurationList = map[string]string{} _ = filepath.Walk("./lua_configuration", func(path string, f os.FileInfo, err error) error { if err != nil { - klog.Errorf("filepath walk ./lua_configuration failed: %s", err.Error()) + klog.Warningf("filepath walk ./lua_configuration failed: %s", err.Error()) return err } if f.IsDir() { diff --git a/pkg/util/parse_utils.go b/pkg/util/parse_utils.go index 9745ee12..fc379900 100644 --- a/pkg/util/parse_utils.go +++ b/pkg/util/parse_utils.go @@ -42,6 +42,7 @@ func ParseWorkload(object client.Object) *WorkloadInfo { return &WorkloadInfo{ LogKey: fmt.Sprintf("%s (%s)", key, gvk), ObjectMeta: *GetMetadata(object), + TypeMeta: *GetTypeMeta(object), Replicas: GetReplicas(object), Status: *ParseWorkloadStatus(object), } @@ -332,6 +333,27 @@ func GetMetadata(object client.Object) *metav1.ObjectMeta { } } +// GetTypeMeta can parse the whole TypeMeta field from client workload object +func GetTypeMeta(object client.Object) *metav1.TypeMeta { + switch o := object.(type) { + case *apps.Deployment: + return &o.TypeMeta + case *appsv1alpha1.CloneSet: + return &o.TypeMeta + case *apps.StatefulSet: + return &o.TypeMeta + case *appsv1beta1.StatefulSet: + return &o.TypeMeta + case *appsv1alpha1.DaemonSet: + return &o.TypeMeta + case *unstructured.Unstructured: + gvk := object.GetObjectKind().GroupVersionKind() + return &metav1.TypeMeta{APIVersion: gvk.GroupVersion().String(), Kind: gvk.Kind} + default: + panic("unsupported workload type to ParseSelector function") + } +} + // parseReplicasFromUnstructured parses replicas from unstructured workload object func parseReplicasFromUnstructured(object *unstructured.Unstructured) int32 { replicas := int32(1) diff --git a/pkg/util/workloads_utils.go b/pkg/util/workloads_utils.go index 0162a0ca..493f8907 100644 --- a/pkg/util/workloads_utils.go +++ b/pkg/util/workloads_utils.go @@ -71,6 +71,7 @@ type WorkloadStatus struct { } type WorkloadInfo struct { + metav1.TypeMeta metav1.ObjectMeta LogKey string Replicas int32 diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler.go b/pkg/webhook/rollout/validating/rollout_create_update_handler.go index 13d86e43..b9f90b54 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler.go @@ -67,11 +67,15 @@ func (h *RolloutCreateUpdateHandler) Handle(ctx context.Context, req admission.R if err := h.Decoder.Decode(req, obj); err != nil { return admission.Errored(http.StatusBadRequest, err) } + errList := h.validateRollout(obj) + if len(errList) != 0 { + return admission.Errored(http.StatusUnprocessableEntity, errList.ToAggregate()) + } oldObj := &appsv1alpha1.Rollout{} if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, oldObj); err != nil { return admission.Errored(http.StatusBadRequest, err) } - errList := h.validateRolloutUpdate(oldObj, obj) + errList = h.validateRolloutUpdate(oldObj, obj) if len(errList) != 0 { return admission.Errored(http.StatusUnprocessableEntity, errList.ToAggregate()) } @@ -199,11 +203,7 @@ func validateRolloutSpecCanaryStrategy(canary *appsv1alpha1.CanaryStrategy, fldP return errList } -func validateRolloutSpecCanaryTraffic(traffic *appsv1alpha1.TrafficRouting, fldPath *field.Path) field.ErrorList { - if traffic == nil { - return field.ErrorList{field.Invalid(fldPath, nil, "Canary.TrafficRoutings cannot be empty")} - } - +func validateRolloutSpecCanaryTraffic(traffic appsv1alpha1.TrafficRoutingRef, fldPath *field.Path) field.ErrorList { errList := field.ErrorList{} if len(traffic.Service) == 0 { errList = append(errList, field.Invalid(fldPath.Child("Service"), traffic.Service, "TrafficRouting.Service cannot be empty")) diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go index a2f65521..1c7c76ce 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go @@ -56,30 +56,42 @@ var ( Canary: &appsv1alpha1.CanaryStrategy{ Steps: []appsv1alpha1.CanaryStep{ { - Weight: utilpointer.Int32Ptr(10), + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(10), + }, Replicas: &intstr.IntOrString{Type: intstr.Int, IntVal: int32(1)}, Pause: appsv1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32Ptr(10), + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(10), + }, Replicas: &intstr.IntOrString{Type: intstr.Int, IntVal: int32(3)}, Pause: appsv1alpha1.RolloutPause{Duration: utilpointer.Int32(1 * 24 * 60 * 60)}, }, { - Weight: utilpointer.Int32Ptr(30), - Pause: appsv1alpha1.RolloutPause{Duration: utilpointer.Int32(7 * 24 * 60 * 60)}, + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(30), + }, + Pause: appsv1alpha1.RolloutPause{Duration: utilpointer.Int32(7 * 24 * 60 * 60)}, }, { - Weight: utilpointer.Int32Ptr(100), + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, }, { - Weight: utilpointer.Int32Ptr(101), + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(101), + }, }, { - Weight: utilpointer.Int32Ptr(200), + TrafficRoutingStrategy: appsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(200), + }, }, }, - TrafficRoutings: []*appsv1alpha1.TrafficRouting{ + TrafficRoutings: []appsv1alpha1.TrafficRoutingRef{ { Service: "service-demo", Ingress: &appsv1alpha1.IngressTrafficRouting{ diff --git a/pkg/webhook/workload/mutating/workload_update_handler_test.go b/pkg/webhook/workload/mutating/workload_update_handler_test.go index 8975136a..53930b3d 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler_test.go +++ b/pkg/webhook/workload/mutating/workload_update_handler_test.go @@ -376,7 +376,7 @@ func TestHandlerDeployment(t *testing.T) { getRollout: func() *appsv1alpha1.Rollout { demo := rolloutDemo.DeepCopy() demo.Spec.Strategy.Canary = &appsv1alpha1.CanaryStrategy{ - TrafficRoutings: []*appsv1alpha1.TrafficRouting{ + TrafficRoutings: []appsv1alpha1.TrafficRoutingRef{ { Service: "echoserver", Ingress: &appsv1alpha1.IngressTrafficRouting{ @@ -550,8 +550,7 @@ func TestHandlerDeployment(t *testing.T) { } delete(newObj.Labels, appsv1alpha1.DeploymentStableRevisionLabel) if !reflect.DeepEqual(newObj, cs.expectObj()) { - by, _ := json.Marshal(newObj) - t.Fatalf("handlerDeployment failed, and new(%s)", string(by)) + t.Fatalf("handlerDeployment failed, and expect(%s) new(%s)", util.DumpJSON(cs.expectObj()), util.DumpJSON(newObj)) } }) } diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index c03404bd..fdb06439 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -245,6 +245,8 @@ var _ = SIGDescribe("Rollout", func() { Eventually(func() bool { daemon := &appsv1alpha1.DaemonSet{} Expect(GetObject(daemonset.Name, daemon)).NotTo(HaveOccurred()) + klog.Infof("DaemonSet Generation(%d) ObservedGeneration(%d) DesiredNumberScheduled(%d) UpdatedNumberScheduled(%d) NumberReady(%d)", + daemon.Generation, daemon.Status.ObservedGeneration, daemon.Status.DesiredNumberScheduled, daemon.Status.UpdatedNumberScheduled, daemon.Status.NumberReady) return daemon.Status.ObservedGeneration == daemon.Generation && daemon.Status.DesiredNumberScheduled == daemon.Status.UpdatedNumberScheduled && daemon.Status.DesiredNumberScheduled == daemon.Status.NumberReady }, 5*time.Minute, time.Second).Should(BeTrue()) } @@ -415,19 +417,29 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, }, { - Weight: utilpointer.Int32(40), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, }, { - Weight: utilpointer.Int32(80), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(80), + }, }, { - Weight: utilpointer.Int32(90), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(90), + }, }, } CreateObject(rollout) @@ -561,13 +573,17 @@ var _ = SIGDescribe("Rollout", func() { replicas := intstr.FromInt(2) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Replicas: &replicas, Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, } CreateObject(rollout) @@ -996,23 +1012,31 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { - Weight: utilpointer.Int32(40), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, @@ -1129,23 +1153,31 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { - Weight: utilpointer.Int32(40), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, @@ -1258,31 +1290,41 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { - Weight: utilpointer.Int32(40), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { - Weight: utilpointer.Int32(80), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(80), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, @@ -1389,19 +1431,25 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, @@ -1450,7 +1498,6 @@ var _ = SIGDescribe("Rollout", func() { Expect(k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.Rollout{}, client.InNamespace(namespace), client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) WaitRolloutNotFound(rollout.Name) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - fmt.Println(util.DumpJSON(workload)) workload.Spec.Paused = false UpdateDeployment(workload) By("Update deployment paused=false") @@ -1485,19 +1532,25 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, @@ -1631,15 +1684,21 @@ var _ = SIGDescribe("Rollout", func() { // update rollout step configuration rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(10), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(10), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(30), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(30), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, @@ -1732,21 +1791,23 @@ var _ = SIGDescribe("Rollout", func() { replica2 := intstr.FromInt(2) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Matches: []v1alpha1.HttpRouteMatch{ - { - Headers: []gatewayv1alpha2.HTTPHeaderMatch{ - { - Type: &headerType, - Name: "user_id", - Value: "123456", + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Matches: []v1alpha1.HttpRouteMatch{ + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Type: &headerType, + Name: "user_id", + Value: "123456", + }, }, }, - }, - { - Headers: []gatewayv1alpha2.HTTPHeaderMatch{ - { - Name: "canary-by-cookie", - Value: "demo", + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "canary-by-cookie", + Value: "demo", + }, }, }, }, @@ -1755,7 +1816,9 @@ var _ = SIGDescribe("Rollout", func() { Replicas: &replica1, }, { - Weight: utilpointer.Int32(30), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(30), + }, Replicas: &replica2, Pause: v1alpha1.RolloutPause{}, }, @@ -1894,20 +1957,22 @@ var _ = SIGDescribe("Rollout", func() { replica2 := intstr.FromInt(2) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Matches: []v1alpha1.HttpRouteMatch{ - { - Headers: []gatewayv1alpha2.HTTPHeaderMatch{ - { - Name: "Cookie", - Value: "demo1=value1;demo2=value2", - }, - { - Name: "SourceIp", - Value: "192.168.0.0/16;172.16.0.0/16", - }, - { - Name: "headername", - Value: "headervalue1;headervalue2", + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Matches: []v1alpha1.HttpRouteMatch{ + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "Cookie", + Value: "demo1=value1;demo2=value2", + }, + { + Name: "SourceIp", + Value: "192.168.0.0/16;172.16.0.0/16", + }, + { + Name: "headername", + Value: "headervalue1;headervalue2", + }, }, }, }, @@ -1916,7 +1981,9 @@ var _ = SIGDescribe("Rollout", func() { Replicas: &replica1, }, { - Weight: utilpointer.Int32(30), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(30), + }, Replicas: &replica2, Pause: v1alpha1.RolloutPause{}, }, @@ -2062,20 +2129,22 @@ var _ = SIGDescribe("Rollout", func() { } rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Matches: []v1alpha1.HttpRouteMatch{ - { - Headers: []gatewayv1alpha2.HTTPHeaderMatch{ - { - Name: "Cookie", - Value: "demo1=value1;demo2=value2", - }, - { - Name: "SourceIp", - Value: "192.168.0.0/16;172.16.0.0/16", - }, - { - Name: "headername", - Value: "headervalue1;headervalue2", + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Matches: []v1alpha1.HttpRouteMatch{ + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "Cookie", + Value: "demo1=value1;demo2=value2", + }, + { + Name: "SourceIp", + Value: "192.168.0.0/16;172.16.0.0/16", + }, + { + Name: "headername", + Value: "headervalue1;headervalue2", + }, }, }, }, @@ -2220,19 +2289,29 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/gateway/rollout-test.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, }, { - Weight: utilpointer.Int32(40), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, }, { - Weight: utilpointer.Int32(80), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(80), + }, }, { - Weight: utilpointer.Int32(90), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(90), + }, }, } CreateObject(rollout) @@ -2378,12 +2457,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, } rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ @@ -2873,7 +2956,7 @@ var _ = SIGDescribe("Rollout", func() { WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) }) - It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { + It("CloneSet V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { By("Creating Rollout...") rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -2987,12 +3070,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, } rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ @@ -3493,12 +3580,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, } rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ @@ -4069,23 +4160,31 @@ var _ = SIGDescribe("Rollout", func() { } rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { - Weight: utilpointer.Int32(40), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, @@ -4165,23 +4264,33 @@ var _ = SIGDescribe("Rollout", func() { } rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(40), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(80), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(80), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, @@ -4271,23 +4380,33 @@ var _ = SIGDescribe("Rollout", func() { } rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(40), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(80), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(80), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, @@ -4361,23 +4480,33 @@ var _ = SIGDescribe("Rollout", func() { } rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(40), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(40), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(80), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(80), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, @@ -4451,20 +4580,28 @@ var _ = SIGDescribe("Rollout", func() { FailureThreshold: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, Steps: []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(10), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(10), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(30), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(30), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, + Pause: v1alpha1.RolloutPause{}, }, }, } @@ -4524,12 +4661,16 @@ var _ = SIGDescribe("Rollout", func() { } rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, } rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ @@ -4682,16 +4823,22 @@ var _ = SIGDescribe("Rollout", func() { rollout.Spec.Strategy.Canary.TrafficRoutings = nil rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), - Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, + Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, }, } rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ @@ -4788,16 +4935,22 @@ var _ = SIGDescribe("Rollout", func() { rollout.Spec.Strategy.Canary.TrafficRoutings = nil rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), - Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, + Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, }, } rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ @@ -4871,16 +5024,22 @@ var _ = SIGDescribe("Rollout", func() { rollout.Spec.Strategy.Canary.TrafficRoutings = nil rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), - Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, + Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, }, } rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ @@ -4942,16 +5101,22 @@ var _ = SIGDescribe("Rollout", func() { rollout.Spec.Strategy.Canary.TrafficRoutings = nil rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { - Weight: utilpointer.Int32(20), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(20), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(60), - Pause: v1alpha1.RolloutPause{}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(60), + }, + Pause: v1alpha1.RolloutPause{}, }, { - Weight: utilpointer.Int32(100), - Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, + TrafficRoutingStrategy: v1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(100), + }, + Pause: v1alpha1.RolloutPause{Duration: utilpointer.Int32(0)}, }, } rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ @@ -5285,7 +5450,7 @@ var _ = SIGDescribe("Rollout", func() { By("check rollout status & paused success") // v1 -> v2, start rollout action - By("Update deployment env NODE_NAME from(version1) -> to(version2)") + By("Update DaemonSet env NODE_NAME from(version1) -> to(version2)") newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) workload.Spec.Template.Spec.Containers[0].Env = newEnvs UpdateDaemonSet(workload) @@ -5297,16 +5462,12 @@ var _ = SIGDescribe("Rollout", func() { Expect(workload.Status.UpdatedNumberScheduled).Should(BeNumerically("==", 1)) Expect(workload.Status.NumberReady).Should(BeNumerically("==", workload.Status.DesiredNumberScheduled)) Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Paused).Should(BeFalse()) - By("check deployment status & paused success") - + By("check DaemonSet status & paused success") // delete rollout + By("Delete rollout crd, and wait DaemonSet ready") Expect(k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.Rollout{}, client.InNamespace(namespace), client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) WaitRolloutNotFound(rollout.Name) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - fmt.Println(util.DumpJSON(workload)) - workload.Spec.UpdateStrategy.RollingUpdate.Paused = utilpointer.Bool(false) - UpdateDaemonSet(workload) - By("Update deployment paused=false") WaitDaemonSetAllPodsReady(workload) // check daemonset