diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index f4894c75..b1c42b34 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -341,7 +341,7 @@ spec: traffic routing properties: createCanaryService: - default: false + default: true description: create a new canary service or just use the stable service type: boolean diff --git a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml index 66eb0e89..e715bf87 100644 --- a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml +++ b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml @@ -55,7 +55,7 @@ spec: routing properties: createCanaryService: - default: false + default: true description: create a new canary service or just use the stable service type: boolean diff --git a/pkg/controller/trafficrouting/trafficrouting_controller_test.go b/pkg/controller/trafficrouting/trafficrouting_controller_test.go index 6485a3f2..668e9520 100644 --- a/pkg/controller/trafficrouting/trafficrouting_controller_test.go +++ b/pkg/controller/trafficrouting/trafficrouting_controller_test.go @@ -116,7 +116,8 @@ var ( Spec: v1alpha1.TrafficRoutingSpec{ ObjectRef: []v1alpha1.TrafficRoutingRef{ { - Service: "echoserver", + Service: "echoserver", + CreateCanaryService: false, Ingress: &v1alpha1.IngressTrafficRouting{ Name: "echoserver", }, @@ -379,6 +380,7 @@ func TestTrafficRoutingTest(t *testing.T) { for _, obj := range ig { checkObjEqual(client, t, obj) } + manager.trafficRoutingManager.RemoveTrafficRoutingController(newTrafficRoutingContext(tr)) }) } } diff --git a/pkg/trafficrouting/manager_test.go b/pkg/trafficrouting/manager_test.go index 94fa9c43..cb664400 100644 --- a/pkg/trafficrouting/manager_test.go +++ b/pkg/trafficrouting/manager_test.go @@ -666,6 +666,7 @@ func TestFinalisingTrafficRouting(t *testing.T) { } manager := NewTrafficRoutingManager(client) done, err := manager.FinalisingTrafficRouting(c, cs.onlyRestoreStableService) + manager.RemoveTrafficRoutingController(c) if err != nil { t.Fatalf("DoTrafficRouting failed: %s", err) } diff --git a/pkg/trafficrouting/network/custom/custom.go b/pkg/trafficrouting/network/custom/custom.go index 073d39a9..1e3904b4 100644 --- a/pkg/trafficrouting/network/custom/custom.go +++ b/pkg/trafficrouting/network/custom/custom.go @@ -188,10 +188,8 @@ func (r *customController) storeObject(obj *unstructured.Unstructured) error { } annotations[OriginalSpecAnnotation] = cSpec obj.SetAnnotations(annotations) - if err := r.Update(context.TODO(), obj); err != nil { - return err - } - return nil + err := r.Update(context.TODO(), obj) + return err } // restore an object from spec stored in OriginalSpecAnnotation @@ -206,10 +204,8 @@ func (r *customController) restoreObject(obj *unstructured.Unstructured) error { obj.Object["spec"] = oSpec.Spec obj.SetAnnotations(oSpec.Annotations) obj.SetLabels(oSpec.Labels) - if err := r.Update(context.TODO(), obj); err != nil { - return err - } - return nil + err := r.Update(context.TODO(), obj) + return err } func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) { diff --git a/pkg/trafficrouting/network/custom/custom_test.go b/pkg/trafficrouting/network/custom/custom_test.go new file mode 100644 index 00000000..c9dfd109 --- /dev/null +++ b/pkg/trafficrouting/network/custom/custom_test.go @@ -0,0 +1,348 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "testing" + + rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + scheme *runtime.Scheme + networkDemo = ` + { + "apiVersion": "networking.istio.io/v1alpha3", + "kind": "VirtualService", + "metadata": { + "name": "echoserver", + "annotations": { + "virtual": "test" + } + }, + "spec": { + "hosts": [ + "echoserver.example.com" + ], + "http": [ + { + "route": [ + { + "destination": { + "host": "echoserver", + } + } + ] + } + ] + } + } + ` +) + +func init() { + scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = rolloutsv1alpha1.AddToScheme(scheme) +} + +func TestInitialize(t *testing.T) { + cases := []struct { + name string + getUnstructured func() *unstructured.Unstructured + getConfig func() Config + getConfigMap func() *corev1.ConfigMap + expectUnstructured func() *unstructured.Unstructured + }{ + { + name: "test1, find lua script locally", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.NetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + getConfigMap: func() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: LuaConfigMap, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript", + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + }, + { + name: "test2, find lua script in ConfigMap", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + u.SetAPIVersion("networking.test.io/v1alpha3") + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.NetworkRef{ + { + APIVersion: "networking.test.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + getConfigMap: func() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: LuaConfigMap, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.test.io"): "ExpectedLuaScript", + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + u.SetAPIVersion("networking.test.io/v1alpha3") + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + if err := fakeCli.Create(context.TODO(), cs.getConfigMap()); err != nil { + klog.Errorf(err.Error()) + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + err = c.Initialize(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + } + checkEqual(fakeCli, t, cs.expectUnstructured()) + }) + } +} + +func checkEqual(cli client.Client, t *testing.T, expect *unstructured.Unstructured) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(expect.GetAPIVersion()) + obj.SetKind(expect.GetKind()) + if err := cli.Get(context.TODO(), types.NamespacedName{Namespace: expect.GetNamespace(), Name: expect.GetName()}, obj); err != nil { + t.Fatalf("Get object failed: %s", err.Error()) + } + if !reflect.DeepEqual(obj.GetAnnotations(), expect.GetAnnotations()) { + fmt.Println(util.DumpJSON(obj.GetAnnotations()), util.DumpJSON(expect.GetAnnotations())) + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.GetAnnotations()), util.DumpJSON(obj.GetAnnotations())) + } + if util.DumpJSON(expect.Object["spec"]) != util.DumpJSON(obj.Object["spec"]) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.Object["spec"]), util.DumpJSON(obj.Object["spec"])) + } +} + +func TestEnsureRoutes(t *testing.T) { + cases := []struct { + name string + getLua func() map[string]string + getRoutes func() *rolloutsv1alpha1.TrafficRoutingStrategy + getUnstructured func() *unstructured.Unstructured + expectInfo func() (bool, *unstructured.Unstructured) + }{ + { + name: "test1", + getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy { + return &rolloutsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(5), + } + }, + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + return u + }, + expectInfo: func() (bool, *unstructured.Unstructured) { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}},"weight":95},{"destination":{"host":"echoserver-canary","port":{"number":80}},"weight":5}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + return false, u + }, + }, + } + config := Config{ + RolloutName: "rollout-demo", + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.NetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + c, _ := NewCustomController(fakeCli, config) + strategy := cs.getRoutes() + expect1, expect2 := cs.expectInfo() + c.Initialize(context.TODO()) + done, err := c.EnsureRoutes(context.TODO(), strategy) + if err != nil { + t.Fatalf("EnsureRoutes failed: %s", err.Error()) + } else if done != expect1 { + t.Fatalf("expect(%v), but get(%v)", expect1, done) + } + checkEqual(fakeCli, t, expect2) + }) + } +} + +func TestFinalise(t *testing.T) { + cases := []struct { + name string + getUnstructured func() *unstructured.Unstructured + getConfig func() Config + expectUnstructured func() *unstructured.Unstructured + }{ + { + name: "test1", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.NetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(networkDemo)) + return u + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + err = c.Finalise(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + } + checkEqual(fakeCli, t, cs.expectUnstructured()) + }) + } +} diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index 69d90967..af0cdbd3 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -19,6 +19,7 @@ package e2e import ( "context" "fmt" + "reflect" "sort" "strings" "time" @@ -34,6 +35,7 @@ import ( netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/util/retry" @@ -5582,6 +5584,51 @@ var _ = SIGDescribe("Rollout", func() { Expect(rollout1.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) }) }) + + KruiseDescribe("Custom network provider tests", func() { + It("Istio VirtualService test", func() { + index1 := &v1.ConfigMap{} + index2 := &v1.ConfigMap{} + Expect(ReadYamlToObject("./test_data/custom/index1.yaml", index1)).ToNot(HaveOccurred()) + Expect(ReadYamlToObject("./test_data/custom/index2.yaml", index2)).ToNot(HaveOccurred()) + CreateObject(index1) + CreateObject(index2) + + svc := &v1.Service{} + Expect(ReadYamlToObject("./test_data/custom/service.yaml", svc)).ToNot(HaveOccurred()) + CreateObject(svc) + + app := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/custom/appv1.yaml", app)).ToNot(HaveOccurred()) + CreateObject(app) + WaitDeploymentAllPodsReady(app) + + virtualService := &unstructured.Unstructured{} + Expect(ReadYamlToObject("./test_data/custom/virtualService.yaml", virtualService)).ToNot(HaveOccurred()) + CreateObject(virtualService) + expectVirtualService := &unstructured.Unstructured{} + + rollout := &v1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/custom/rollout.yaml", rollout)).ToNot(HaveOccurred()) + CreateObject(rollout) + + By("changing app version from v1 -> v2") + Expect(GetObject(app.Name, app)).NotTo(HaveOccurred()) + app.Spec.Template.Spec.Volumes[0].ConfigMap.Name = "nginx-configmap2" + UpdateDeployment(app) + WaitRolloutCanaryStepPaused(rollout.Name, 1) + Expect(GetObject(virtualService.GetName(), virtualService)).ToNot(HaveOccurred()) + Expect(ReadYamlToObject("./test_data/custom/expectVirtualService.yaml", expectVirtualService)).ToNot(HaveOccurred()) + Expect(reflect.DeepEqual(virtualService.Object["spec"], expectVirtualService.Object["spec"])).To(BeTrue()) + + By("resume rollout") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + Expect(GetObject(virtualService.GetName(), virtualService)).ToNot(HaveOccurred()) + Expect(ReadYamlToObject("./test_data/custom/expectVirtualService2.yaml", expectVirtualService)).ToNot(HaveOccurred()) + Expect(reflect.DeepEqual(virtualService.Object["spec"], expectVirtualService.Object["spec"])).To(BeTrue()) + }) + }) }) func mergeEnvVar(original []v1.EnvVar, add v1.EnvVar) []v1.EnvVar { diff --git a/test/e2e/test_data/custom/appv1.yaml b/test/e2e/test_data/custom/appv1.yaml new file mode 100644 index 00000000..ca4ac78e --- /dev/null +++ b/test/e2e/test_data/custom/appv1.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + namespace: demo +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumeMounts: + - name: html-volume + mountPath: /usr/share/nginx/html + volumes: + - name: html-volume + configMap: + name: nginx-configmap1 \ No newline at end of file diff --git a/test/e2e/test_data/custom/expectVirtualService.yaml b/test/e2e/test_data/custom/expectVirtualService.yaml new file mode 100644 index 00000000..f22346ca --- /dev/null +++ b/test/e2e/test_data/custom/expectVirtualService.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: nginx-vs + namespace: demo +spec: + hosts: + - "*" + gateways: + - nginx-gateway + http: + - match: + - headers: + user-agent: + exact: pc + name: + regex: .*demo + route: + - destination: + host: nginx-service + weight: 80 + - destination: + host: nginx-service-canary + weight: 20 + - route: + - destination: + host: nginx-service diff --git a/test/e2e/test_data/custom/expectVirtualService2.yaml b/test/e2e/test_data/custom/expectVirtualService2.yaml new file mode 100644 index 00000000..506d659b --- /dev/null +++ b/test/e2e/test_data/custom/expectVirtualService2.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: nginx-vs + namespace: demo +spec: + hosts: + - "*" + gateways: + - nginx-gateway + http: + - route: + - destination: + host: nginx-service + weight: 50 + - destination: + host: nginx-service-canary + weight: 50 \ No newline at end of file diff --git a/test/e2e/test_data/custom/index1.yaml b/test/e2e/test_data/custom/index1.yaml new file mode 100644 index 00000000..82d5e49f --- /dev/null +++ b/test/e2e/test_data/custom/index1.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-configmap1 + namespace: demo +data: + index.html: | +

Hello from nginx-v1

\ No newline at end of file diff --git a/test/e2e/test_data/custom/index2.yaml b/test/e2e/test_data/custom/index2.yaml new file mode 100644 index 00000000..86f7a0f0 --- /dev/null +++ b/test/e2e/test_data/custom/index2.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-configmap2 + namespace: demo +data: + index.html: | +

Hello from nginx-v2

diff --git a/test/e2e/test_data/custom/rollout.yaml b/test/e2e/test_data/custom/rollout.yaml new file mode 100644 index 00000000..adaa2a40 --- /dev/null +++ b/test/e2e/test_data/custom/rollout.yaml @@ -0,0 +1,33 @@ +apiVersion: rollouts.kruise.io/v1alpha1 +kind: Rollout +metadata: + name: rollouts-demo + namespace: demo + annotations: + rollouts.kruise.io/rolling-style: canary +spec: + disabled: false + objectRef: + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-deployment + strategy: + canary: + steps: + - weight: 20 + matches: + - headers: + - type: Exact + name: user-agent + value: pc + - type: RegularExpression + name: name + value: ".*demo" + - weight: 50 + trafficRoutings: + - service: nginx-service + networkRefs: + - apiVersion: networking.istio.io/v1alpha3 + kind: VirtualService + name: nginx-vs \ No newline at end of file diff --git a/test/e2e/test_data/custom/service.yaml b/test/e2e/test_data/custom/service.yaml new file mode 100644 index 00000000..b931be32 --- /dev/null +++ b/test/e2e/test_data/custom/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-service + namespace: demo +spec: + selector: + app: nginx + ports: + - protocol: TCP + port: 80 + targetPort: 80 + name: http \ No newline at end of file diff --git a/test/e2e/test_data/custom/virtualService.yaml b/test/e2e/test_data/custom/virtualService.yaml new file mode 100644 index 00000000..d519abba --- /dev/null +++ b/test/e2e/test_data/custom/virtualService.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: nginx-vs + namespace: demo +spec: + hosts: + - "*" + gateways: + - nginx-gateway + http: + - route: + - destination: + host: nginx-service