diff --git a/api/v1alpha1/trafficrouting_types.go b/api/v1alpha1/trafficrouting_types.go index 0386a877..e87cc815 100644 --- a/api/v1alpha1/trafficrouting_types.go +++ b/api/v1alpha1/trafficrouting_types.go @@ -35,7 +35,12 @@ type TrafficRoutingRef struct { Ingress *IngressTrafficRouting `json:"ingress,omitempty"` // Gateway holds Gateway specific configuration to route traffic // Gateway configuration only supports >= v0.4.0 (v1alpha2). - Gateway *GatewayTrafficRouting `json:"gateway,omitempty"` + Gateway *GatewayTrafficRouting `json:"gateway,omitempty"` + NetworkRefs *[]NetworkRef `json:"networkRefs,omitempty"` + // +optional + //+kubebuilder:default=true + // create a new canary service or just use the stable service + CreateCanaryService bool `json:"createCanaryService,omitempty"` } // IngressTrafficRouting configuration for ingress controller to control traffic routing @@ -149,6 +154,12 @@ type TrafficRoutingList struct { Items []TrafficRouting `json:"items"` } +type NetworkRef struct { + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` +} + func init() { SchemeBuilder.Register(&TrafficRouting{}, &TrafficRoutingList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 29d6019a..fa718e40 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -397,6 +397,21 @@ func (in *NameAndSpecData) DeepCopy() *NameAndSpecData { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkRef) DeepCopyInto(out *NetworkRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkRef. +func (in *NetworkRef) DeepCopy() *NetworkRef { + if in == nil { + return nil + } + out := new(NetworkRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectRef) DeepCopyInto(out *ObjectRef) { *out = *in @@ -901,6 +916,15 @@ func (in *TrafficRoutingRef) DeepCopyInto(out *TrafficRoutingRef) { *out = new(GatewayTrafficRouting) (*in).DeepCopyInto(*out) } + if in.NetworkRefs != nil { + in, out := &in.NetworkRefs, &out.NetworkRefs + *out = new([]NetworkRef) + if **in != nil { + in, out := *in, *out + *out = make([]NetworkRef, len(*in)) + copy(*out, *in) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingRef. diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index 5e189d67..b1c42b34 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -340,6 +340,11 @@ spec: for supported service meshes to enable more fine-grained traffic routing properties: + createCanaryService: + default: true + description: create a new canary service or just use + the stable service + type: boolean gateway: description: Gateway holds Gateway specific configuration to route traffic Gateway configuration only supports @@ -373,6 +378,17 @@ spec: required: - name type: object + networkRefs: + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + type: object + type: array service: description: Service holds the name of a service which selects pods with stable version and don't select diff --git a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml index 68775c5f..e715bf87 100644 --- a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml +++ b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml @@ -54,6 +54,11 @@ spec: for supported service meshes to enable more fine-grained traffic routing properties: + createCanaryService: + default: true + description: create a new canary service or just use the stable + service + type: boolean gateway: description: Gateway holds Gateway specific configuration to route traffic Gateway configuration only supports >= v0.4.0 @@ -85,6 +90,17 @@ spec: required: - name type: object + networkRefs: + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + type: object + type: array service: description: Service holds the name of a service which selects pods with stable version and don't select any pods with canary diff --git a/lua_configuration/lua.go b/lua_configuration/lua.go new file mode 100644 index 00000000..cbe5cc9a --- /dev/null +++ b/lua_configuration/lua.go @@ -0,0 +1,205 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util/luamanager" + lua "github.com/yuin/gopher-lua" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + utilpointer "k8s.io/utils/pointer" +) + +type LuaData struct { + Data Data + CanaryWeight int32 + StableWeight int32 + Matches []v1alpha1.HttpRouteMatch + CanaryService string + StableService string + PatchPodMetadata *v1alpha1.PatchPodTemplateMetadata +} + +type Data struct { + Spec interface{} `json:"spec,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +type TestCase struct { + Rollout *v1alpha1.Rollout `json:"rollout,omitempty"` + Original *unstructured.Unstructured `json:"original,omitempty"` + Expected []*unstructured.Unstructured `json:"expected,omitempty"` +} + +// convert testdata to lua object for debugging +func main() { + err := PathWalk() + if err != nil { + fmt.Println(err) + } +} + +func PathWalk() error { + err := filepath.Walk("./", func(path string, f os.FileInfo, err error) error { + if !strings.Contains(path, "trafficRouting.lua") { + return nil + } + if err != nil { + return fmt.Errorf("failed to walk path: %s", err.Error()) + } + dir := filepath.Dir(path) + err = filepath.Walk(filepath.Join(dir, "testdata"), func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" { + fmt.Printf("--- walking path: %s ---\n", path) + err = ObjectToTable(path) + if err != nil { + return fmt.Errorf("failed to convert object to table: %s", err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk path: %s", err.Error()) + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk path: %s", err) + } + return nil +} + +// convert a testcase object to lua table for debug +func ObjectToTable(path string) error { + luaManager := &luamanager.LuaManager{} + dir, file := filepath.Split(path) + testCase, err := getLuaTestCase(path) + if err != nil { + return fmt.Errorf("failed to get lua testcase: %s", err) + } + uList := make(map[string]interface{}) + rollout := testCase.Rollout + steps := rollout.Spec.Strategy.Canary.Steps + for i, step := range steps { + weight := step.TrafficRoutingStrategy.Weight + if step.TrafficRoutingStrategy.Weight == nil { + weight = utilpointer.Int32(-1) + } + var canaryService string + stableService := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service + if rollout.Spec.Strategy.Canary.TrafficRoutings[0].CreateCanaryService { + canaryService = fmt.Sprintf("%s-canary", stableService) + } else { + canaryService = stableService + } + data := &LuaData{ + Data: Data{ + Labels: testCase.Original.GetLabels(), + Annotations: testCase.Original.GetAnnotations(), + Spec: testCase.Original.Object["spec"], + }, + Matches: step.TrafficRoutingStrategy.Matches, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + CanaryService: canaryService, + StableService: stableService, + PatchPodMetadata: rollout.Spec.Strategy.Canary.PatchPodTemplateMetadata, + } + uList[fmt.Sprintf("step_%d", i)] = data + } + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&uList) + if err != nil { + return fmt.Errorf("failed to convert to unstructured: %s", err) + } + u := &unstructured.Unstructured{Object: unObj} + script := ` + function serialize(obj, isKey) + local lua = "" + local t = type(obj) + if t == "number" then + lua = lua .. obj + elseif t == "boolean" then + lua = lua .. tostring(obj) + elseif t == "string" then + if isKey then + lua = lua .. string.format("%s", obj) + else + lua = lua .. string.format("%q", obj) + end + elseif t == "table" then + lua = lua .. "{" + for k, v in pairs(obj) do + if type(k) == "string" then + lua = lua .. serialize(k, true) .. "=" .. serialize(v, false) .. "," + else + lua = lua .. serialize(v, false) .. "," + end + end + local metatable = getmetatable(obj) + if metatable ~= nil and type(metatable.__index) == "table" then + for k, v in pairs(metatable.__index) do + if type(k) == "string" then + lua = lua .. serialize(k, true) .. "=" .. serialize(v, false) .. "," + else + lua = lua .. serialize(v, false) .. "," + end + end + end + lua = lua .. "}" + elseif t == "nil" then + return nil + else + error("can not serialize a " .. t .. " type.") + end + return lua + end + + function table2string(tablevalue) + local stringtable = "steps=" .. serialize(tablevalue) + print(stringtable) + return stringtable + end + return table2string(obj) + ` + l, err := luaManager.RunLuaScript(u, script) + returnValue := l.Get(-1) + if returnValue.Type() == lua.LTString { + filePath := fmt.Sprintf("%s%s_obj.lua", dir, strings.Split(file, ".")[0]) + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("failed to open file: %s", err) + } + defer file.Close() + v := returnValue.String() + _, err = io.WriteString(file, v) + if err != nil { + return fmt.Errorf("failed to WriteString %s", err) + } + } + if err != nil { + return fmt.Errorf("failed to run lua script: %s", err) + } + return nil +} + +func getLuaTestCase(path string) (*TestCase, error) { + yamlFile, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + luaTestCase := &TestCase{} + err = yaml.Unmarshal(yamlFile, luaTestCase) + if err != nil { + return nil, err + } + return luaTestCase, nil +} diff --git a/pkg/controller/rollout/rollout_canary.go b/pkg/controller/rollout/rollout_canary.go index 9cd18e13..1093acab 100644 --- a/pkg/controller/rollout/rollout_canary.go +++ b/pkg/controller/rollout/rollout_canary.go @@ -264,6 +264,7 @@ func (m *canaryReleaseManager) doCanaryFinalising(c *RolloutContext) (bool, erro } else if !done { return false, nil } + m.trafficRoutingManager.RemoveTrafficRoutingController(tr) klog.Infof("rollout(%s/%s) doCanaryFinalising success", c.Rollout.Namespace, c.Rollout.Name) return true, nil } diff --git a/pkg/controller/rollout/rollout_progressing.go b/pkg/controller/rollout/rollout_progressing.go index 9635d6d0..6d14220c 100644 --- a/pkg/controller/rollout/rollout_progressing.go +++ b/pkg/controller/rollout/rollout_progressing.go @@ -466,5 +466,6 @@ func newTrafficRoutingContext(c *RolloutContext) *trafficrouting.TrafficRoutingC StableRevision: c.NewStatus.CanaryStatus.StableRevision, CanaryRevision: c.NewStatus.CanaryStatus.PodTemplateHash, LastUpdateTime: c.NewStatus.CanaryStatus.LastUpdateTime, + PatchPodMetadata: c.Rollout.Spec.Strategy.Canary.PatchPodTemplateMetadata, } } diff --git a/pkg/controller/trafficrouting/trafficrouting_controller.go b/pkg/controller/trafficrouting/trafficrouting_controller.go index eb62d3c2..163376fb 100644 --- a/pkg/controller/trafficrouting/trafficrouting_controller.go +++ b/pkg/controller/trafficrouting/trafficrouting_controller.go @@ -215,11 +215,10 @@ func (r *TrafficRoutingReconciler) SetupWithManager(mgr ctrl.Manager) error { func newTrafficRoutingContext(tr *v1alpha1.TrafficRouting) *trafficrouting.TrafficRoutingContext { return &trafficrouting.TrafficRoutingContext{ - Key: fmt.Sprintf("TrafficRouting(%s/%s)", tr.Namespace, tr.Name), - Namespace: tr.Namespace, - ObjectRef: tr.Spec.ObjectRef, - Strategy: tr.Spec.Strategy, - OwnerRef: *metav1.NewControllerRef(tr, trControllerKind), - OnlyTrafficRouting: true, + Key: fmt.Sprintf("TrafficRouting(%s/%s)", tr.Namespace, tr.Name), + Namespace: tr.Namespace, + ObjectRef: tr.Spec.ObjectRef, + Strategy: tr.Spec.Strategy, + OwnerRef: *metav1.NewControllerRef(tr, trControllerKind), } } diff --git a/pkg/controller/trafficrouting/trafficrouting_controller_test.go b/pkg/controller/trafficrouting/trafficrouting_controller_test.go index 6485a3f2..668e9520 100644 --- a/pkg/controller/trafficrouting/trafficrouting_controller_test.go +++ b/pkg/controller/trafficrouting/trafficrouting_controller_test.go @@ -116,7 +116,8 @@ var ( Spec: v1alpha1.TrafficRoutingSpec{ ObjectRef: []v1alpha1.TrafficRoutingRef{ { - Service: "echoserver", + Service: "echoserver", + CreateCanaryService: false, Ingress: &v1alpha1.IngressTrafficRouting{ Name: "echoserver", }, @@ -379,6 +380,7 @@ func TestTrafficRoutingTest(t *testing.T) { for _, obj := range ig { checkObjEqual(client, t, obj) } + manager.trafficRoutingManager.RemoveTrafficRoutingController(newTrafficRoutingContext(tr)) }) } } diff --git a/pkg/trafficrouting/manager.go b/pkg/trafficrouting/manager.go index 390b4cd0..be130566 100644 --- a/pkg/trafficrouting/manager.go +++ b/pkg/trafficrouting/manager.go @@ -19,10 +19,12 @@ package trafficrouting import ( "context" "fmt" + "sync" "time" "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/trafficrouting/network" + "github.com/openkruise/rollouts/pkg/trafficrouting/network/custom" "github.com/openkruise/rollouts/pkg/trafficrouting/network/gateway" "github.com/openkruise/rollouts/pkg/trafficrouting/network/ingress" "github.com/openkruise/rollouts/pkg/util" @@ -37,6 +39,7 @@ import ( var ( defaultGracePeriodSeconds int32 = 3 + ControllerMap sync.Map ) type TrafficRoutingContext struct { @@ -45,9 +48,7 @@ type TrafficRoutingContext struct { Namespace string ObjectRef []v1alpha1.TrafficRoutingRef Strategy v1alpha1.TrafficRoutingStrategy - // OnlyTrafficRouting - OnlyTrafficRouting bool - OwnerRef metav1.OwnerReference + OwnerRef metav1.OwnerReference // workload.RevisionLabelKey RevisionLabelKey string // status.CanaryStatus.StableRevision @@ -55,7 +56,8 @@ type TrafficRoutingContext struct { // status.CanaryStatus.PodTemplateHash CanaryRevision string // newStatus.canaryStatus.LastUpdateTime - LastUpdateTime *metav1.Time + LastUpdateTime *metav1.Time + PatchPodMetadata *v1alpha1.PatchPodTemplateMetadata } // Manager responsible for adjusting network resources @@ -81,14 +83,20 @@ func (m *Manager) InitializeTrafficRouting(c *TrafficRoutingContext) error { if err := m.Get(context.TODO(), types.NamespacedName{Namespace: c.Namespace, Name: sService}, service); err != nil { return err } - cService := getCanaryServiceName(sService, c.OnlyTrafficRouting) - // new network provider, ingress or gateway + cService := getCanaryServiceName(sService, objectRef.CreateCanaryService) + // new network provider + key := fmt.Sprintf("%s.%s", c.Key, sService) trController, err := newNetworkProvider(m.Client, c, sService, cService) if err != nil { klog.Errorf("%s newNetworkProvider failed: %s", c.Key, err.Error()) return err } - return trController.Initialize(context.TODO()) + err = trController.Initialize(context.TODO()) + if err != nil { + return err + } + ControllerMap.Store(key, trController) + return nil } func (m *Manager) DoTrafficRouting(c *TrafficRoutingContext) (bool, error) { @@ -115,13 +123,13 @@ func (m *Manager) DoTrafficRouting(c *TrafficRoutingContext) (bool, error) { return false, err } // canary service name - canaryServiceName := getCanaryServiceName(trafficRouting.Service, c.OnlyTrafficRouting) + canaryServiceName := getCanaryServiceName(trafficRouting.Service, trafficRouting.CreateCanaryService) canaryService := &corev1.Service{} canaryService.Namespace = stableService.Namespace canaryService.Name = canaryServiceName // end-to-end canary deployment scenario(a -> b -> c), if only b or c is released, //and a is not released in this scenario, then the canary service is not needed. - if !c.OnlyTrafficRouting { + if trafficRouting.CreateCanaryService { if c.StableRevision == "" || c.CanaryRevision == "" { klog.Warningf("%s stableRevision or podTemplateHash can not be empty, and wait a moment", c.Key) return false, nil @@ -170,11 +178,11 @@ func (m *Manager) DoTrafficRouting(c *TrafficRoutingContext) (bool, error) { } } - // new network provider, ingress or gateway - trController, err := newNetworkProvider(m.Client, c, stableService.Name, canaryService.Name) + // new network provider + key := fmt.Sprintf("%s.%s", c.Key, trafficRouting.Service) + trController, err := m.getController(key, c) if err != nil { - klog.Errorf("%s newNetworkProvider failed: %s", c.Key, err.Error()) - return false, err + klog.Errorf("failed to get trafficRouting controller: %s", err) } verify, err := trController.EnsureRoutes(context.TODO(), &c.Strategy) if err != nil { @@ -188,6 +196,7 @@ func (m *Manager) DoTrafficRouting(c *TrafficRoutingContext) (bool, error) { } func (m *Manager) FinalisingTrafficRouting(c *TrafficRoutingContext, onlyRestoreStableService bool) (bool, error) { + var err error if len(c.ObjectRef) == 0 { return true, nil } @@ -196,13 +205,12 @@ func (m *Manager) FinalisingTrafficRouting(c *TrafficRoutingContext, onlyRestore trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds } - cServiceName := getCanaryServiceName(trafficRouting.Service, c.OnlyTrafficRouting) - trController, err := newNetworkProvider(m.Client, c, trafficRouting.Service, cServiceName) + cServiceName := getCanaryServiceName(trafficRouting.Service, trafficRouting.CreateCanaryService) + key := fmt.Sprintf("%s.%s", c.Key, trafficRouting.Service) + trController, err := m.getController(key, c) if err != nil { - klog.Errorf("%s newTrafficRoutingController failed: %s", c.Key, err.Error()) - return false, err + klog.Errorf("failed to get trafficRouting controller: %s", err) } - cService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: c.Namespace, Name: cServiceName}} // if canary svc has been already cleaned up, just return if err = m.Get(context.TODO(), client.ObjectKeyFromObject(cService), cService); err != nil { @@ -249,7 +257,7 @@ func (m *Manager) FinalisingTrafficRouting(c *TrafficRoutingContext, onlyRestore } // end to end deployment, don't remove the canary service; // because canary service is stable service - if !c.OnlyTrafficRouting { + if trafficRouting.CreateCanaryService { // remove canary service err = m.Delete(context.TODO(), cService) if err != nil && !errors.IsNotFound(err) { @@ -261,8 +269,47 @@ func (m *Manager) FinalisingTrafficRouting(c *TrafficRoutingContext, onlyRestore return true, nil } +// remove controller stored in controllerMap +func (m *Manager) RemoveTrafficRoutingController(c *TrafficRoutingContext) { + if c.ObjectRef != nil { + trafficRouting := c.ObjectRef[0] + key := fmt.Sprintf("%s.%s", c.Key, trafficRouting.Service) + ControllerMap.Delete(key) + } +} + +func (m *Manager) getController(key string, c *TrafficRoutingContext) (network.NetworkProvider, error) { + val, ok := ControllerMap.Load(key) + var trController network.NetworkProvider + var err error + trafficRouting := c.ObjectRef[0] + cServiceName := getCanaryServiceName(trafficRouting.Service, trafficRouting.CreateCanaryService) + if !ok { + trController, err = newNetworkProvider(m.Client, c, trafficRouting.Service, cServiceName) + if err != nil { + klog.Errorf("%s newTrafficRoutingController failed: %s", c.Key, err.Error()) + return trController, err + } + ControllerMap.Store(key, trController) + } else { + trController = val.(network.NetworkProvider) + } + return trController, nil +} + func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, cService string) (network.NetworkProvider, error) { trafficRouting := con.ObjectRef[0] + if trafficRouting.NetworkRefs != nil { + return custom.NewCustomController(c, custom.Config{ + RolloutName: con.Key, + RolloutNs: con.Namespace, + CanaryService: cService, + StableService: sService, + TrafficConf: *trafficRouting.NetworkRefs, + OwnerRef: con.OwnerRef, + PatchPodMetadata: con.PatchPodMetadata, + }) + } if trafficRouting.Ingress != nil { return ingress.NewIngressTrafficRouting(c, ingress.Config{ Key: con.Key, @@ -355,9 +402,9 @@ func (m *Manager) restoreStableService(c *TrafficRoutingContext) (bool, error) { return true, nil } -func getCanaryServiceName(sService string, onlyTrafficRouting bool) string { - if onlyTrafficRouting { - return sService +func getCanaryServiceName(sService string, createCanaryService bool) string { + if createCanaryService { + return fmt.Sprintf("%s-canary", sService) } - return fmt.Sprintf("%s-canary", sService) + return sService } diff --git a/pkg/trafficrouting/manager_test.go b/pkg/trafficrouting/manager_test.go index 720f767a..411e9597 100644 --- a/pkg/trafficrouting/manager_test.go +++ b/pkg/trafficrouting/manager_test.go @@ -662,6 +662,7 @@ func TestFinalisingTrafficRouting(t *testing.T) { } manager := NewTrafficRoutingManager(client) done, err := manager.FinalisingTrafficRouting(c, cs.onlyRestoreStableService) + manager.RemoveTrafficRoutingController(c) if err != nil { t.Fatalf("DoTrafficRouting failed: %s", err) } diff --git a/pkg/trafficrouting/network/custom/custom.go b/pkg/trafficrouting/network/custom/custom.go new file mode 100644 index 00000000..2e979883 --- /dev/null +++ b/pkg/trafficrouting/network/custom/custom.go @@ -0,0 +1,310 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting/network" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + "github.com/openkruise/rollouts/pkg/util/luamanager" + lua "github.com/yuin/gopher-lua" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + OriginalSpecAnnotation = "rollouts.kruise.io/origin-spec-configuration" + LuaConfigMap = "kruise-rollout-configuration" +) + +type Data struct { + Spec interface{} `json:"spec,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +type customController struct { + client.Client + conf Config + luaManager *luamanager.LuaManager + luaScript map[string]string +} + +type Config struct { + RolloutName string + RolloutNs string + CanaryService string + StableService string + // network providers need to be created + TrafficConf []rolloutv1alpha1.NetworkRef + OwnerRef metav1.OwnerReference + PatchPodMetadata *rolloutv1alpha1.PatchPodTemplateMetadata +} + +func NewCustomController(client client.Client, conf Config) (network.NetworkProvider, error) { + r := &customController{ + Client: client, + conf: conf, + luaManager: &luamanager.LuaManager{}, + luaScript: map[string]string{}, + } + return r, nil +} + +func (r *customController) Initialize(ctx context.Context) error { + for _, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + return err + } + // check if lua script exists + if _, ok := r.luaScript[ref.Kind]; !ok { + script := r.getLuaScript(ctx, ref) + if script == "" { + return fmt.Errorf("failed to get lua script for %s", ref.Kind) + } + r.luaScript[ref.Kind] = script + } + if err := r.storeObject(obj); err != nil { + klog.Errorf("failed to store object: %s/%s", ref.Kind, ref.Name) + return err + } + } + return nil +} + +func (r *customController) EnsureRoutes(ctx context.Context, strategy *rolloutv1alpha1.TrafficRoutingStrategy) (bool, error) { + var err error + var done = true + for _, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + return false, err + } + specStr := obj.GetAnnotations()[OriginalSpecAnnotation] + if specStr == "" { + continue + } + var oSpec Data + _ = json.Unmarshal([]byte(specStr), &oSpec) + luaScript, ok := r.luaScript[ref.Kind] + // in case when rollout finalising the controller restart unexpectedly and the Initialize step is skipped + if !ok { + script := r.getLuaScript(ctx, ref) + if script == "" { + return false, fmt.Errorf("failed to get lua script for %s", ref.Kind) + } + r.luaScript[ref.Kind] = script + luaScript = script + } + nSpec, err := r.executeLuaForCanary(oSpec, strategy, luaScript) + if err != nil { + return false, err + } + if cmpAndSetObject(nSpec, obj) { + continue + } + if err = r.Update(context.TODO(), obj); err != nil { + return false, err + } + done = false + } + return done, nil +} + +func (r *customController) Finalise(ctx context.Context) error { + for _, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + if errors.IsNotFound(err) { + continue + } + return err + } + if err := r.restoreObject(obj); err != nil { + klog.Errorf("failed to restore object: %s/%s", ref.Kind, ref.Name) + return err + } + } + return nil +} + +// store spec of an object in OriginalSpecAnnotation +func (r *customController) storeObject(obj *unstructured.Unstructured) error { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + labels := obj.GetLabels() + oSpec := annotations[OriginalSpecAnnotation] + delete(annotations, OriginalSpecAnnotation) + data := Data{ + Spec: obj.Object["spec"], + Labels: labels, + Annotations: annotations, + } + cSpec := util.DumpJSON(data) + if oSpec == cSpec { + return nil + } + annotations[OriginalSpecAnnotation] = cSpec + obj.SetAnnotations(annotations) + if err := r.Update(context.TODO(), obj); err != nil { + return err + } + return nil +} + +// restore an object from spec stored in OriginalSpecAnnotation +func (r *customController) restoreObject(obj *unstructured.Unstructured) error { + annotations := obj.GetAnnotations() + if annotations == nil || annotations[OriginalSpecAnnotation] == "" { + return nil + } + specStr := annotations[OriginalSpecAnnotation] + var oSpec Data + _ = json.Unmarshal([]byte(specStr), &oSpec) + obj.Object["spec"] = oSpec.Spec + obj.SetAnnotations(oSpec.Annotations) + obj.SetLabels(oSpec.Labels) + if err := r.Update(context.TODO(), obj); err != nil { + return err + } + return nil +} + +func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) { + weight := strategy.Weight + matches := strategy.Matches + if weight == nil { + // the lua script does not have a pointer type, + // so we need to pass weight=-1 to indicate the case where weight is nil. + weight = utilpointer.Int32(-1) + } + type LuaData struct { + Data Data + CanaryWeight int32 + StableWeight int32 + Matches []rolloutv1alpha1.HttpRouteMatch + CanaryService string + StableService string + PatchPodMetadata *rolloutv1alpha1.PatchPodTemplateMetadata + } + data := &LuaData{ + Data: spec, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + Matches: matches, + CanaryService: r.conf.CanaryService, + StableService: r.conf.StableService, + PatchPodMetadata: r.conf.PatchPodMetadata, + } + + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data) + if err != nil { + return Data{}, err + } + u := &unstructured.Unstructured{Object: unObj} + l, err := r.luaManager.RunLuaScript(u, luaScript) + if err != nil { + return Data{}, err + } + returnValue := l.Get(-1) + if returnValue.Type() == lua.LTTable { + jsonBytes, err := luamanager.Encode(returnValue) + if err != nil { + return Data{}, err + } + var obj Data + err = json.Unmarshal(jsonBytes, &obj) + if err != nil { + return Data{}, err + } + return obj, nil + } + return Data{}, fmt.Errorf("expect table output from Lua script, not %s", returnValue.Type().String()) +} + +func (r *customController) getLuaScript(ctx context.Context, ref rolloutv1alpha1.NetworkRef) string { + // get local lua script + // luaScript.Provider: CRDGroupt/Kind + group := strings.Split(ref.APIVersion, "/")[0] + key := fmt.Sprintf("lua_configuration/%s/trafficRouting.lua", fmt.Sprintf("%s/%s", group, ref.Kind)) + script := util.GetLuaConfigurationContent(key) + if script != "" { + return script + } + + // if lua script is not found locally, then try ConfigMap + nameSpace := util.GetRolloutNamespace() // kruise-rollout + name := LuaConfigMap + configMap := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Namespace: nameSpace, Name: name}, configMap) + if err != nil { + klog.Errorf("failed to get configMap %s/%s", nameSpace, name) + } else { + // in format like "lua.traffic.routing.ingress.aliyun-alb" + key = fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingCustomTypePrefix, ref.Kind, group) + if script, ok := configMap.Data[key]; ok { + return script + } else if !ok { + klog.Errorf("expected script %s not found in ConfigMap", key) + } + } + return "" +} + +// compare and update obj, return if the obj is updated +func cmpAndSetObject(data Data, obj *unstructured.Unstructured) bool { + spec := data.Spec + annotations := data.Annotations + if annotations == nil { + annotations = make(map[string]string) + } + annotations[OriginalSpecAnnotation] = obj.GetAnnotations()[OriginalSpecAnnotation] + labels := data.Labels + if util.DumpJSON(obj.Object["spec"]) == util.DumpJSON(spec) && + reflect.DeepEqual(obj.GetAnnotations(), annotations) && + reflect.DeepEqual(obj.GetLabels(), labels) { + return true + } + obj.Object["spec"] = spec + obj.SetAnnotations(annotations) + obj.SetLabels(labels) + return false +} diff --git a/pkg/trafficrouting/network/custom/custom_test.go b/pkg/trafficrouting/network/custom/custom_test.go new file mode 100644 index 00000000..c9dfd109 --- /dev/null +++ b/pkg/trafficrouting/network/custom/custom_test.go @@ -0,0 +1,348 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "testing" + + rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + scheme *runtime.Scheme + networkDemo = ` + { + "apiVersion": "networking.istio.io/v1alpha3", + "kind": "VirtualService", + "metadata": { + "name": "echoserver", + "annotations": { + "virtual": "test" + } + }, + "spec": { + "hosts": [ + "echoserver.example.com" + ], + "http": [ + { + "route": [ + { + "destination": { + "host": "echoserver", + } + } + ] + } + ] + } + } + ` +) + +func init() { + scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = rolloutsv1alpha1.AddToScheme(scheme) +} + +func TestInitialize(t *testing.T) { + cases := []struct { + name string + getUnstructured func() *unstructured.Unstructured + getConfig func() Config + getConfigMap func() *corev1.ConfigMap + expectUnstructured func() *unstructured.Unstructured + }{ + { + name: "test1, find lua script locally", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.NetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + getConfigMap: func() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: LuaConfigMap, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript", + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + }, + { + name: "test2, find lua script in ConfigMap", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + u.SetAPIVersion("networking.test.io/v1alpha3") + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.NetworkRef{ + { + APIVersion: "networking.test.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + getConfigMap: func() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: LuaConfigMap, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.test.io"): "ExpectedLuaScript", + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + u.SetAPIVersion("networking.test.io/v1alpha3") + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + if err := fakeCli.Create(context.TODO(), cs.getConfigMap()); err != nil { + klog.Errorf(err.Error()) + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + err = c.Initialize(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + } + checkEqual(fakeCli, t, cs.expectUnstructured()) + }) + } +} + +func checkEqual(cli client.Client, t *testing.T, expect *unstructured.Unstructured) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(expect.GetAPIVersion()) + obj.SetKind(expect.GetKind()) + if err := cli.Get(context.TODO(), types.NamespacedName{Namespace: expect.GetNamespace(), Name: expect.GetName()}, obj); err != nil { + t.Fatalf("Get object failed: %s", err.Error()) + } + if !reflect.DeepEqual(obj.GetAnnotations(), expect.GetAnnotations()) { + fmt.Println(util.DumpJSON(obj.GetAnnotations()), util.DumpJSON(expect.GetAnnotations())) + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.GetAnnotations()), util.DumpJSON(obj.GetAnnotations())) + } + if util.DumpJSON(expect.Object["spec"]) != util.DumpJSON(obj.Object["spec"]) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.Object["spec"]), util.DumpJSON(obj.Object["spec"])) + } +} + +func TestEnsureRoutes(t *testing.T) { + cases := []struct { + name string + getLua func() map[string]string + getRoutes func() *rolloutsv1alpha1.TrafficRoutingStrategy + getUnstructured func() *unstructured.Unstructured + expectInfo func() (bool, *unstructured.Unstructured) + }{ + { + name: "test1", + getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy { + return &rolloutsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(5), + } + }, + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + expectInfo: func() (bool, *unstructured.Unstructured) { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}},"weight":95},{"destination":{"host":"echoserver-canary","port":{"number":80}},"weight":5}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + return false, u + }, + }, + } + config := Config{ + RolloutName: "rollout-demo", + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.NetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + c, _ := NewCustomController(fakeCli, config) + strategy := cs.getRoutes() + expect1, expect2 := cs.expectInfo() + c.Initialize(context.TODO()) + done, err := c.EnsureRoutes(context.TODO(), strategy) + if err != nil { + t.Fatalf("EnsureRoutes failed: %s", err.Error()) + } else if done != expect1 { + t.Fatalf("expect(%v), but get(%v)", expect1, done) + } + checkEqual(fakeCli, t, expect2) + }) + } +} + +func TestFinalise(t *testing.T) { + cases := []struct { + name string + getUnstructured func() *unstructured.Unstructured + getConfig func() Config + expectUnstructured func() *unstructured.Unstructured + }{ + { + name: "test1", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.NetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + return u + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + err = c.Finalise(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + } + checkEqual(fakeCli, t, cs.expectUnstructured()) + }) + } +} diff --git a/pkg/util/configuration/configuration.go b/pkg/util/configuration/configuration.go index 86282f5c..9bca9a79 100644 --- a/pkg/util/configuration/configuration.go +++ b/pkg/util/configuration/configuration.go @@ -31,6 +31,7 @@ const ( RolloutConfigurationName = "kruise-rollout-configuration" LuaTrafficRoutingIngressTypePrefix = "lua.traffic.routing.ingress" + LuaTrafficRoutingCustomTypePrefix = "lua.traffic.routing" ) func GetTrafficRoutingIngressLuaScript(client client.Client, iType string) (string, error) { diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler.go b/pkg/webhook/rollout/validating/rollout_create_update_handler.go index b9f90b54..81c4b829 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler.go @@ -209,8 +209,8 @@ func validateRolloutSpecCanaryTraffic(traffic appsv1alpha1.TrafficRoutingRef, fl errList = append(errList, field.Invalid(fldPath.Child("Service"), traffic.Service, "TrafficRouting.Service cannot be empty")) } - if traffic.Gateway == nil && traffic.Ingress == nil { - errList = append(errList, field.Invalid(fldPath.Child("TrafficRoutings"), traffic.Ingress, "TrafficRoutings must set the gateway or ingress")) + if traffic.Gateway == nil && traffic.Ingress == nil && traffic.NetworkRefs == nil { + errList = append(errList, field.Invalid(fldPath.Child("TrafficRoutings"), traffic.Ingress, "TrafficRoutings are not set")) } if traffic.Ingress != nil { diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index c76fab28..d63eb70c 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -19,6 +19,7 @@ package e2e import ( "context" "fmt" + "reflect" "sort" "strings" "time" @@ -34,6 +35,7 @@ import ( netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/util/retry" @@ -5725,6 +5727,51 @@ var _ = SIGDescribe("Rollout", func() { Expect(rollout1.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) }) }) + + KruiseDescribe("Custom network provider tests", func() { + It("Istio VirtualService test", func() { + index1 := &v1.ConfigMap{} + index2 := &v1.ConfigMap{} + Expect(ReadYamlToObject("./test_data/custom/index1.yaml", index1)).ToNot(HaveOccurred()) + Expect(ReadYamlToObject("./test_data/custom/index2.yaml", index2)).ToNot(HaveOccurred()) + CreateObject(index1) + CreateObject(index2) + + svc := &v1.Service{} + Expect(ReadYamlToObject("./test_data/custom/service.yaml", svc)).ToNot(HaveOccurred()) + CreateObject(svc) + + app := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/custom/appv1.yaml", app)).ToNot(HaveOccurred()) + CreateObject(app) + WaitDeploymentAllPodsReady(app) + + virtualService := &unstructured.Unstructured{} + Expect(ReadYamlToObject("./test_data/custom/virtualService.yaml", virtualService)).ToNot(HaveOccurred()) + CreateObject(virtualService) + expectVirtualService := &unstructured.Unstructured{} + + rollout := &v1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/custom/rollout.yaml", rollout)).ToNot(HaveOccurred()) + CreateObject(rollout) + + By("changing app version from v1 -> v2") + Expect(GetObject(app.Name, app)).NotTo(HaveOccurred()) + app.Spec.Template.Spec.Volumes[0].ConfigMap.Name = "nginx-configmap2" + UpdateDeployment(app) + WaitRolloutCanaryStepPaused(rollout.Name, 1) + Expect(GetObject(virtualService.GetName(), virtualService)).ToNot(HaveOccurred()) + Expect(ReadYamlToObject("./test_data/custom/expectVirtualService.yaml", expectVirtualService)).ToNot(HaveOccurred()) + Expect(reflect.DeepEqual(virtualService.Object["spec"], expectVirtualService.Object["spec"])).To(BeTrue()) + + By("resume rollout") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + Expect(GetObject(virtualService.GetName(), virtualService)).ToNot(HaveOccurred()) + Expect(ReadYamlToObject("./test_data/custom/expectVirtualService2.yaml", expectVirtualService)).ToNot(HaveOccurred()) + Expect(reflect.DeepEqual(virtualService.Object["spec"], expectVirtualService.Object["spec"])).To(BeTrue()) + }) + }) }) func mergeEnvVar(original []v1.EnvVar, add v1.EnvVar) []v1.EnvVar {