From 6e57766cf3e74481cec1669e0f6e355a70c2433e Mon Sep 17 00:00:00 2001 From: Kuromesi Date: Fri, 1 Sep 2023 09:24:28 +0800 Subject: [PATCH] add support for custom network providers Signed-off-by: Kuromesi --- api/v1alpha1/trafficrouting_types.go | 8 + api/v1alpha1/zz_generated.deepcopy.go | 24 ++ .../bases/rollouts.kruise.io_rollouts.yaml | 13 + .../rollouts.kruise.io_trafficroutings.yaml | 13 + pkg/trafficrouting/manager.go | 11 + pkg/trafficrouting/network/custom/custom.go | 315 ++++++++++++++++ .../network/custom/custom_test.go | 348 ++++++++++++++++++ pkg/util/configuration/configuration.go | 1 + .../rollout_create_update_handler.go | 4 +- 9 files changed, 735 insertions(+), 2 deletions(-) create mode 100644 pkg/trafficrouting/network/custom/custom.go create mode 100644 pkg/trafficrouting/network/custom/custom_test.go diff --git a/api/v1alpha1/trafficrouting_types.go b/api/v1alpha1/trafficrouting_types.go index 0386a877..589a5a90 100644 --- a/api/v1alpha1/trafficrouting_types.go +++ b/api/v1alpha1/trafficrouting_types.go @@ -36,6 +36,8 @@ type TrafficRoutingRef struct { // Gateway holds Gateway specific configuration to route traffic // Gateway configuration only supports >= v0.4.0 (v1alpha2). Gateway *GatewayTrafficRouting `json:"gateway,omitempty"` + // CustomNetworkRefs hold a list of custom providers to route traffic + CustomNetworkRefs *[]CustomNetworkRef `json:"customNetworkRefs,omitempty"` } // IngressTrafficRouting configuration for ingress controller to control traffic routing @@ -149,6 +151,12 @@ type TrafficRoutingList struct { Items []TrafficRouting `json:"items"` } +type CustomNetworkRef struct { + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` +} + func init() { SchemeBuilder.Register(&TrafficRouting{}, &TrafficRoutingList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 29d6019a..863553e5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -256,6 +256,21 @@ func (in *CanaryStrategy) DeepCopy() *CanaryStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomNetworkRef) DeepCopyInto(out *CustomNetworkRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomNetworkRef. +func (in *CustomNetworkRef) DeepCopy() *CustomNetworkRef { + if in == nil { + return nil + } + out := new(CustomNetworkRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentExtraStatus) DeepCopyInto(out *DeploymentExtraStatus) { *out = *in @@ -901,6 +916,15 @@ func (in *TrafficRoutingRef) DeepCopyInto(out *TrafficRoutingRef) { *out = new(GatewayTrafficRouting) (*in).DeepCopyInto(*out) } + if in.CustomNetworkRefs != nil { + in, out := &in.CustomNetworkRefs, &out.CustomNetworkRefs + *out = new([]CustomNetworkRef) + if **in != nil { + in, out := *in, *out + *out = make([]CustomNetworkRef, len(*in)) + copy(*out, *in) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingRef. diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index 5e189d67..e78296cb 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -340,6 +340,19 @@ spec: for supported service meshes to enable more fine-grained traffic routing properties: + customNetworkRefs: + description: CustomNetworkRefs hold a list of custom + providers to route traffic + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + type: object + type: array gateway: description: Gateway holds Gateway specific configuration to route traffic Gateway configuration only supports diff --git a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml index 68775c5f..f6d32c5f 100644 --- a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml +++ b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml @@ -54,6 +54,19 @@ spec: for supported service meshes to enable more fine-grained traffic routing properties: + customNetworkRefs: + description: CustomNetworkRefs hold a list of custom providers + to route traffic + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + type: object + type: array gateway: description: Gateway holds Gateway specific configuration to route traffic Gateway configuration only supports >= v0.4.0 diff --git a/pkg/trafficrouting/manager.go b/pkg/trafficrouting/manager.go index 390b4cd0..5a12f39f 100644 --- a/pkg/trafficrouting/manager.go +++ b/pkg/trafficrouting/manager.go @@ -23,6 +23,7 @@ import ( "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/trafficrouting/network" + "github.com/openkruise/rollouts/pkg/trafficrouting/network/custom" "github.com/openkruise/rollouts/pkg/trafficrouting/network/gateway" "github.com/openkruise/rollouts/pkg/trafficrouting/network/ingress" "github.com/openkruise/rollouts/pkg/util" @@ -263,6 +264,16 @@ func (m *Manager) FinalisingTrafficRouting(c *TrafficRoutingContext, onlyRestore func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, cService string) (network.NetworkProvider, error) { trafficRouting := con.ObjectRef[0] + if trafficRouting.CustomNetworkRefs != nil { + return custom.NewCustomController(c, custom.Config{ + RolloutName: con.Key, + RolloutNs: con.Namespace, + CanaryService: cService, + StableService: sService, + TrafficConf: *trafficRouting.CustomNetworkRefs, + OwnerRef: con.OwnerRef, + }) + } if trafficRouting.Ingress != nil { return ingress.NewIngressTrafficRouting(c, ingress.Config{ Key: con.Key, diff --git a/pkg/trafficrouting/network/custom/custom.go b/pkg/trafficrouting/network/custom/custom.go new file mode 100644 index 00000000..7ac428c2 --- /dev/null +++ b/pkg/trafficrouting/network/custom/custom.go @@ -0,0 +1,315 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + + "github.com/openkruise/rollouts/api/v1alpha1" + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting/network" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + "github.com/openkruise/rollouts/pkg/util/luamanager" + lua "github.com/yuin/gopher-lua" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + OriginalSpecAnnotation = "rollouts.kruise.io/origin-spec-configuration" + LuaConfigMap = "kruise-rollout-configuration" +) + +type Data struct { + Spec interface{} `json:"spec,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +type customController struct { + client.Client + conf Config + luaManager *luamanager.LuaManager +} + +type Config struct { + RolloutName string + RolloutNs string + CanaryService string + StableService string + // network providers need to be created + TrafficConf []rolloutv1alpha1.CustomNetworkRef + OwnerRef metav1.OwnerReference +} + +func NewCustomController(client client.Client, conf Config) (network.NetworkProvider, error) { + r := &customController{ + Client: client, + conf: conf, + luaManager: &luamanager.LuaManager{}, + } + return r, nil +} + +func (r *customController) Initialize(ctx context.Context) error { + for _, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + klog.Errorf("failed to get custom network provider %s/%s", ref.Kind, ref.Name) + return err + } + // check if lua script exists + _, err := r.getLuaScript(ctx, ref) + if err != nil { + klog.Errorf("failed to get lua script for custom network provider %s: %s", ref.Kind, err.Error()) + return err + } + if err := r.storeObject(obj); err != nil { + klog.Errorf("failed to store custom network provider %s/%s", ref.Kind, ref.Name) + return err + } + } + return nil +} + +func (r *customController) EnsureRoutes(ctx context.Context, strategy *rolloutv1alpha1.TrafficRoutingStrategy) (bool, error) { + var err error + var done = true + for _, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + return false, err + } + specStr := obj.GetAnnotations()[OriginalSpecAnnotation] + if specStr == "" { + continue + } + var oSpec Data + _ = json.Unmarshal([]byte(specStr), &oSpec) + luaScript, err := r.getLuaScript(ctx, ref) + if err != nil { + klog.Errorf("failed to get lua script for %s", ref.Kind) + return false, err + } + nSpec, err := r.executeLuaForCanary(oSpec, strategy, luaScript) + if err != nil { + return false, err + } + if cmpAndSetObject(nSpec, obj) { + continue + } + if err = r.Update(context.TODO(), obj); err != nil { + klog.Errorf("failed to update custom network provider") + return false, err + } + klog.Infof("update custom network provider %s/%s success") + done = false + } + return done, nil +} + +func (r *customController) Finalise(ctx context.Context) error { + for _, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + if errors.IsNotFound(err) { + klog.Infof("custom network provider %s/%s not found when finalising", ref.Kind, ref.Name) + continue + } + return err + } + // when one failed how to proceed? + if err := r.restoreObject(obj); err != nil { + klog.Errorf("failed to restore object: %s/%s", ref.Kind, ref.Name) + return err + } + } + return nil +} + +// store spec of an object in OriginalSpecAnnotation +func (r *customController) storeObject(obj *unstructured.Unstructured) error { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + labels := obj.GetLabels() + oSpec := annotations[OriginalSpecAnnotation] + delete(annotations, OriginalSpecAnnotation) + data := Data{ + Spec: obj.Object["spec"], + Labels: labels, + Annotations: annotations, + } + cSpec := util.DumpJSON(data) + if oSpec == cSpec { + return nil + } + annotations[OriginalSpecAnnotation] = cSpec + obj.SetAnnotations(annotations) + if err := r.Update(context.TODO(), obj); err != nil { + klog.Errorf("failed to store custom network provider %s/%s", obj.GetKind(), obj.GetName()) + return err + } + klog.Infof("store custom network provider %s/%s success", obj.GetKind(), obj.GetName()) + return nil +} + +// restore an object from spec stored in OriginalSpecAnnotation +func (r *customController) restoreObject(obj *unstructured.Unstructured) error { + annotations := obj.GetAnnotations() + if annotations == nil || annotations[OriginalSpecAnnotation] == "" { + return nil + } + specStr := annotations[OriginalSpecAnnotation] + var oSpec Data + _ = json.Unmarshal([]byte(specStr), &oSpec) + obj.Object["spec"] = oSpec.Spec + obj.SetAnnotations(oSpec.Annotations) + obj.SetLabels(oSpec.Labels) + if err := r.Update(context.TODO(), obj); err != nil { + klog.Errorf("failed to restore custom network provider %s/%s", obj.GetKind(), obj.GetName()) + return err + } + klog.Infof("restore custom network provider %s/%s success", obj.GetKind(), obj.GetName()) + return nil +} + +func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) { + weight := strategy.Weight + matches := strategy.Matches + rollout := &v1alpha1.Rollout{} + if err := r.Get(context.TODO(), types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.RolloutName}, rollout); err != nil { + klog.Errorf("failed to get rollout/%s when execute custom network provider lua script", r.conf.RolloutName) + return Data{}, err + } + if weight == nil { + // the lua script does not have a pointer type, + // so we need to pass weight=-1 to indicate the case where weight is nil. + weight = utilpointer.Int32(-1) + } + type LuaData struct { + Data Data + CanaryWeight int32 + StableWeight int32 + Matches []rolloutv1alpha1.HttpRouteMatch + CanaryService string + StableService string + PatchPodMetadata *rolloutv1alpha1.PatchPodTemplateMetadata + } + data := &LuaData{ + Data: spec, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + Matches: matches, + CanaryService: r.conf.CanaryService, + StableService: r.conf.StableService, + PatchPodMetadata: rollout.Spec.Strategy.Canary.PatchPodTemplateMetadata, + } + + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data) + if err != nil { + return Data{}, err + } + u := &unstructured.Unstructured{Object: unObj} + l, err := r.luaManager.RunLuaScript(u, luaScript) + if err != nil { + return Data{}, err + } + returnValue := l.Get(-1) + if returnValue.Type() == lua.LTTable { + jsonBytes, err := luamanager.Encode(returnValue) + if err != nil { + return Data{}, err + } + var obj Data + err = json.Unmarshal(jsonBytes, &obj) + if err != nil { + return Data{}, err + } + return obj, nil + } + return Data{}, fmt.Errorf("expect table output from Lua script, not %s", returnValue.Type().String()) +} + +func (r *customController) getLuaScript(ctx context.Context, ref rolloutv1alpha1.CustomNetworkRef) (string, error) { + // get local lua script + // luaScript.Provider: CRDGroupt/Kind + group := strings.Split(ref.APIVersion, "/")[0] + key := fmt.Sprintf("lua_configuration/%s/trafficRouting.lua", fmt.Sprintf("%s/%s", group, ref.Kind)) + script := util.GetLuaConfigurationContent(key) + if script != "" { + return script, nil + } + + // if lua script is not found locally, then try ConfigMap + nameSpace := util.GetRolloutNamespace() // kruise-rollout + name := LuaConfigMap + configMap := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Namespace: nameSpace, Name: name}, configMap) + if err != nil { + return "", fmt.Errorf("failed to get configMap %s/%s", nameSpace, name) + } else { + // in format like "lua.traffic.routing.ingress.aliyun-alb" + key = fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingCustomTypePrefix, ref.Kind, group) + if script, ok := configMap.Data[key]; ok { + return script, nil + } else if !ok { + return "", fmt.Errorf("expected script of %s not found in ConfigMap", key) + } + } + return "", nil +} + +// compare and update obj, return if the obj is updated +func cmpAndSetObject(data Data, obj *unstructured.Unstructured) bool { + spec := data.Spec + annotations := data.Annotations + if annotations == nil { + annotations = make(map[string]string) + } + annotations[OriginalSpecAnnotation] = obj.GetAnnotations()[OriginalSpecAnnotation] + labels := data.Labels + if util.DumpJSON(obj.Object["spec"]) == util.DumpJSON(spec) && + reflect.DeepEqual(obj.GetAnnotations(), annotations) && + reflect.DeepEqual(obj.GetLabels(), labels) { + return true + } + obj.Object["spec"] = spec + obj.SetAnnotations(annotations) + obj.SetLabels(labels) + return false +} diff --git a/pkg/trafficrouting/network/custom/custom_test.go b/pkg/trafficrouting/network/custom/custom_test.go new file mode 100644 index 00000000..46134be2 --- /dev/null +++ b/pkg/trafficrouting/network/custom/custom_test.go @@ -0,0 +1,348 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "testing" + + rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + scheme *runtime.Scheme + networkDemo = ` + { + "apiVersion": "networking.istio.io/v1alpha3", + "kind": "VirtualService", + "metadata": { + "name": "echoserver", + "annotations": { + "virtual": "test" + } + }, + "spec": { + "hosts": [ + "echoserver.example.com" + ], + "http": [ + { + "route": [ + { + "destination": { + "host": "echoserver", + } + } + ] + } + ] + } + } + ` +) + +func init() { + scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = rolloutsv1alpha1.AddToScheme(scheme) +} + +func TestInitialize(t *testing.T) { + cases := []struct { + name string + getUnstructured func() *unstructured.Unstructured + getConfig func() Config + getConfigMap func() *corev1.ConfigMap + expectUnstructured func() *unstructured.Unstructured + }{ + { + name: "test1, find lua script locally", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + getConfigMap: func() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: LuaConfigMap, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript", + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + }, + { + name: "test2, find lua script in ConfigMap", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + u.SetAPIVersion("networking.test.io/v1alpha3") + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.test.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + getConfigMap: func() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: LuaConfigMap, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.test.io"): "ExpectedLuaScript", + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + u.SetAPIVersion("networking.test.io/v1alpha3") + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + if err := fakeCli.Create(context.TODO(), cs.getConfigMap()); err != nil { + klog.Errorf(err.Error()) + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + err = c.Initialize(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + } + checkEqual(fakeCli, t, cs.expectUnstructured()) + }) + } +} + +func checkEqual(cli client.Client, t *testing.T, expect *unstructured.Unstructured) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(expect.GetAPIVersion()) + obj.SetKind(expect.GetKind()) + if err := cli.Get(context.TODO(), types.NamespacedName{Namespace: expect.GetNamespace(), Name: expect.GetName()}, obj); err != nil { + t.Fatalf("Get object failed: %s", err.Error()) + } + if !reflect.DeepEqual(obj.GetAnnotations(), expect.GetAnnotations()) { + fmt.Println(util.DumpJSON(obj.GetAnnotations()), util.DumpJSON(expect.GetAnnotations())) + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.GetAnnotations()), util.DumpJSON(obj.GetAnnotations())) + } + if util.DumpJSON(expect.Object["spec"]) != util.DumpJSON(obj.Object["spec"]) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.Object["spec"]), util.DumpJSON(obj.Object["spec"])) + } +} + +func TestEnsureRoutes(t *testing.T) { + cases := []struct { + name string + getLua func() map[string]string + getRoutes func() *rolloutsv1alpha1.TrafficRoutingStrategy + getUnstructured func() *unstructured.Unstructured + expectInfo func() (bool, *unstructured.Unstructured) + }{ + { + name: "test1", + getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy { + return &rolloutsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(5), + } + }, + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + expectInfo: func() (bool, *unstructured.Unstructured) { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}},"weight":95},{"destination":{"host":"echoserver-canary","port":{"number":80}},"weight":5}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + return false, u + }, + }, + } + config := Config{ + RolloutName: "rollout-demo", + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + c, _ := NewCustomController(fakeCli, config) + strategy := cs.getRoutes() + expect1, expect2 := cs.expectInfo() + c.Initialize(context.TODO()) + done, err := c.EnsureRoutes(context.TODO(), strategy) + if err != nil { + t.Fatalf("EnsureRoutes failed: %s", err.Error()) + } else if done != expect1 { + t.Fatalf("expect(%v), but get(%v)", expect1, done) + } + checkEqual(fakeCli, t, expect2) + }) + } +} + +func TestFinalise(t *testing.T) { + cases := []struct { + name string + getUnstructured func() *unstructured.Unstructured + getConfig func() Config + expectUnstructured func() *unstructured.Unstructured + }{ + { + name: "test1", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + return u + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + err = c.Finalise(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + } + checkEqual(fakeCli, t, cs.expectUnstructured()) + }) + } +} diff --git a/pkg/util/configuration/configuration.go b/pkg/util/configuration/configuration.go index 86282f5c..9bca9a79 100644 --- a/pkg/util/configuration/configuration.go +++ b/pkg/util/configuration/configuration.go @@ -31,6 +31,7 @@ const ( RolloutConfigurationName = "kruise-rollout-configuration" LuaTrafficRoutingIngressTypePrefix = "lua.traffic.routing.ingress" + LuaTrafficRoutingCustomTypePrefix = "lua.traffic.routing" ) func GetTrafficRoutingIngressLuaScript(client client.Client, iType string) (string, error) { diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler.go b/pkg/webhook/rollout/validating/rollout_create_update_handler.go index b9f90b54..bf3fa6a8 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler.go @@ -209,8 +209,8 @@ func validateRolloutSpecCanaryTraffic(traffic appsv1alpha1.TrafficRoutingRef, fl errList = append(errList, field.Invalid(fldPath.Child("Service"), traffic.Service, "TrafficRouting.Service cannot be empty")) } - if traffic.Gateway == nil && traffic.Ingress == nil { - errList = append(errList, field.Invalid(fldPath.Child("TrafficRoutings"), traffic.Ingress, "TrafficRoutings must set the gateway or ingress")) + if traffic.Gateway == nil && traffic.Ingress == nil && traffic.CustomNetworkRefs == nil { + errList = append(errList, field.Invalid(fldPath.Child("TrafficRoutings"), traffic.Ingress, "TrafficRoutings are not set")) } if traffic.Ingress != nil {