From 0133c2dff26f3156b06421d70ee2a71733dd6132 Mon Sep 17 00:00:00 2001 From: wulemao <1194736083@qq.com> Date: Thu, 26 Sep 2024 11:54:27 +0800 Subject: [PATCH] add e2e for migration rollback feature Signed-off-by: wulemao <1194736083@qq.com> --- test/e2e/framework/work.go | 44 ++++++++++++ ...test.go => migration_and_rollback_test.go} | 68 ++++++++++++++++--- 2 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 test/e2e/framework/work.go rename test/e2e/{seamless_migration_test.go => migration_and_rollback_test.go} (79%) diff --git a/test/e2e/framework/work.go b/test/e2e/framework/work.go new file mode 100644 index 000000000000..d2f31a03f491 --- /dev/null +++ b/test/e2e/framework/work.go @@ -0,0 +1,44 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "fmt" + + "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// WaitForWorkToDisappear waiting for work to disappear util timeout +func WaitForWorkToDisappear(client karmada.Interface, namespace, name string) { + klog.Infof("Waiting for work(%s/%s) to disappear", namespace, name) + gomega.Eventually(func() error { + _, err := client.WorkV1alpha1().Works(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil { + return fmt.Errorf("work(%s/%s) still exist", namespace, name) + } + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get work(%s/%s), err: %w", namespace, name, err) + } + return nil + }, pollTimeout, pollInterval).ShouldNot(gomega.HaveOccurred()) +} diff --git a/test/e2e/seamless_migration_test.go b/test/e2e/migration_and_rollback_test.go similarity index 79% rename from test/e2e/seamless_migration_test.go rename to test/e2e/migration_and_rollback_test.go index 180627cc660c..6ddf650f70f8 100644 --- a/test/e2e/seamless_migration_test.go +++ b/test/e2e/migration_and_rollback_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "k8s.io/utils/ptr" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" @@ -38,7 +39,7 @@ import ( "github.com/karmada-io/karmada/test/helper" ) -var _ = ginkgo.Describe("Seamless migration testing", func() { +var _ = ginkgo.Describe("Seamless migration and rollback testing", func() { var member1 string var member1Client kubernetes.Interface @@ -51,7 +52,7 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { ginkgo.Context("Test migrate namespaced resource: Deployment", func() { var deployment *appsv1.Deployment var propagationPolicy *policyv1alpha1.PropagationPolicy - var bindingName string + var bindingName, workName, workNamespace string ginkgo.BeforeEach(func() { deployment = helper.NewDeployment(testNamespace, deploymentNamePrefix+rand.String(RandomStrLength)) @@ -65,6 +66,8 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}}, }) bindingName = names.GenerateBindingName(deployment.Kind, deployment.Name) + workName = names.GenerateWorkName(deployment.Kind, deployment.Name, deployment.Namespace) + workNamespace = names.GenerateExecutionSpaceName(member1) }) ginkgo.BeforeEach(func() { @@ -76,8 +79,8 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { framework.CreatePropagationPolicy(karmadaClient, propagationPolicy) ginkgo.DeferCleanup(func() { - // Delete Deployment in karmada control plane - framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + // Delete Deployment in member cluster + framework.RemoveDeployment(member1Client, deployment.Namespace, deployment.Name) // Delete PropagationPolicy in karmada control plane framework.RemovePropagationPolicy(karmadaClient, propagationPolicy.Namespace, propagationPolicy.Name) @@ -103,9 +106,10 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { }, pollTimeout, pollInterval).Should(gomega.Equal(true)) }) - // Step 2, Update PropagationPolicy in karmada control plane with conflictResolution=Overwrite + // Step 2, Update PropagationPolicy in karmada control plane with conflictResolution=Overwrite and preserveResourcesOnDeletion=true ginkgo.By(fmt.Sprintf("Update PropagationPolicy %s in karmada control plane with conflictResolution=Overwrite", propagationPolicy.Name), func() { propagationPolicy.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite + propagationPolicy.Spec.PreserveResourcesOnDeletion = ptr.To[bool](true) framework.UpdatePropagationPolicyWithSpec(karmadaClient, propagationPolicy.Namespace, propagationPolicy.Name, propagationPolicy.Spec) }) @@ -122,6 +126,19 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { gomega.Expect(items[0].Applied).Should(gomega.BeTrue()) gomega.Expect(items[0].Health).Should(gomega.Equal(workv1alpha2.ResourceHealthy)) }) + + // Step 4, Delete resource template and check whether member cluster resource is preserved + ginkgo.By("Delete resource template and check whether member cluster resource is preserved", func() { + // Delete Deployment in karmada control plane + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + + // Wait for work deleted + framework.WaitForWorkToDisappear(karmadaClient, workNamespace, workName) + + // Check member cluster resource is preserved + framework.WaitDeploymentPresentOnClusterFitWith(member1, deployment.Namespace, deployment.Name, + func(*appsv1.Deployment) bool { return true }) + }) }) }) @@ -129,7 +146,7 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { var clusterRoleName string var clusterRole *rbacv1.ClusterRole var cpp *policyv1alpha1.ClusterPropagationPolicy - var bindingName string + var bindingName, workName, workNamespace string ginkgo.BeforeEach(func() { clusterRoleName = clusterRoleNamePrefix + rand.String(RandomStrLength) @@ -151,7 +168,10 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}}, }) cpp.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite + cpp.Spec.PreserveResourcesOnDeletion = ptr.To[bool](true) bindingName = names.GenerateBindingName(clusterRole.Kind, clusterRole.Name) + workName = names.GenerateWorkName(clusterRole.Kind, clusterRole.Name, clusterRole.Namespace) + workNamespace = names.GenerateExecutionSpaceName(member1) }) ginkgo.BeforeEach(func() { @@ -163,8 +183,8 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { framework.CreateClusterPropagationPolicy(karmadaClient, cpp) ginkgo.DeferCleanup(func() { - // Delete ClusterRole in karmada control plane - framework.RemoveClusterRole(kubeClient, clusterRoleName) + // Delete ClusterRole in member cluster + framework.RemoveClusterRole(member1Client, clusterRoleName) // Delete ClusterPropagationPolicy in karmada control plane framework.RemoveClusterPropagationPolicy(karmadaClient, cpp.Name) @@ -187,6 +207,17 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { return e1 == nil && e2 == nil && len(binding.Status.AggregatedStatus) > 0 && binding.Status.AggregatedStatus[0].Applied }, pollTimeout, pollInterval).Should(gomega.Equal(true)) }) + + ginkgo.By("Delete resource template and check whether member cluster resource is preserved", func() { + // Delete ClusterRole in karmada control plane + framework.RemoveClusterRole(kubeClient, clusterRole.Name) + + // Wait for work deleted + framework.WaitForWorkToDisappear(karmadaClient, workNamespace, workName) + + // Check member cluster resource is preserved + framework.WaitClusterRolePresentOnClusterFitWith(member1, clusterRole.Name, func(*rbacv1.ClusterRole) bool { return true }) + }) }) }) @@ -194,7 +225,7 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { var serviceName string var service *corev1.Service var pp *policyv1alpha1.PropagationPolicy - var bindingName string + var bindingName, workName, workNamespace string ginkgo.BeforeEach(func() { serviceName = serviceNamePrefix + rand.String(RandomStrLength) @@ -209,7 +240,10 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}}, }) pp.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite + pp.Spec.PreserveResourcesOnDeletion = ptr.To[bool](true) bindingName = names.GenerateBindingName(service.Kind, service.Name) + workName = names.GenerateWorkName(service.Kind, service.Name, service.Namespace) + workNamespace = names.GenerateExecutionSpaceName(member1) }) ginkgo.BeforeEach(func() { @@ -221,8 +255,8 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { framework.CreatePropagationPolicy(karmadaClient, pp) ginkgo.DeferCleanup(func() { - // Delete Service in karmada control plane - framework.RemoveService(kubeClient, testNamespace, serviceName) + // Delete Service in member cluster + framework.RemoveService(member1Client, testNamespace, serviceName) // Delete PropagationPolicy in karmada control plane framework.RemovePropagationPolicy(karmadaClient, testNamespace, pp.Name) @@ -245,6 +279,18 @@ var _ = ginkgo.Describe("Seamless migration testing", func() { return e1 == nil && e2 == nil && len(binding.Status.AggregatedStatus) > 0 && binding.Status.AggregatedStatus[0].Applied }, pollTimeout, pollInterval).Should(gomega.Equal(true)) }) + + ginkgo.By("Delete resource template and check whether member cluster resource is preserved", func() { + // Delete Service in karmada control plane + framework.RemoveService(kubeClient, service.Namespace, service.Name) + + // Wait for work deleted + framework.WaitForWorkToDisappear(karmadaClient, workNamespace, workName) + + // Check member cluster resource is preserved + framework.WaitServicePresentOnClusterFitWith(member1, service.Namespace, service.Name, + func(*corev1.Service) bool { return true }) + }) }) }) })