From 14d0a32784f35999e322ea7134b732d0350e260b Mon Sep 17 00:00:00 2001 From: Rory Z <16801068+Rory-Z@users.noreply.github.com> Date: Mon, 24 Jul 2023 23:23:41 +0800 Subject: [PATCH] test(v2alpha2): rebalance cover emqx 4 --- e2e/v2alpha2/e2e_rebalance_test.go | 266 ++++++++++++++++++++++++++--- e2e/v2alpha2/suite_test.go | 11 ++ 2 files changed, 254 insertions(+), 23 deletions(-) diff --git a/e2e/v2alpha2/e2e_rebalance_test.go b/e2e/v2alpha2/e2e_rebalance_test.go index a90b5406a..36d21f680 100644 --- a/e2e/v2alpha2/e2e_rebalance_test.go +++ b/e2e/v2alpha2/e2e_rebalance_test.go @@ -4,12 +4,14 @@ import ( "context" "fmt" + appsv1beta4 "github.com/emqx/emqx-operator/apis/apps/v1beta4" appsv2alpha2 "github.com/emqx/emqx-operator/apis/apps/v2alpha2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/utils/pointer" // "k8s.io/apimachinery/pkg/types" @@ -18,6 +20,10 @@ import ( ) var rebalance = appsv2alpha2.Rebalance{ + TypeMeta: metav1.TypeMeta{ + Kind: "Rebalance", + APIVersion: appsv2alpha2.GroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ Name: "rebalance", }, @@ -35,7 +41,7 @@ var rebalance = appsv2alpha2.Rebalance{ }, } -var _ = Describe("Emqx v2alpha2 Rebalance Test", Label("rebalance"), func() { +var _ = Describe("EMQX 5 Rebalance Test", func() { var instance *appsv2alpha2.EMQX var r *appsv2alpha2.Rebalance BeforeEach(func() { @@ -96,8 +102,8 @@ var _ = Describe("Emqx v2alpha2 Rebalance Test", Label("rebalance"), func() { HaveField("Status", corev1.ConditionTrue), )), HaveField("Conditions", ContainElements( - HaveField("Message", fmt.Sprintf("EMQX %s is not found", r.Spec.InstanceName)), - )), + HaveField("Message", ContainSubstring(fmt.Sprintf("%s is not found", r.Spec.InstanceName)))), + ), )) }) }) @@ -142,25 +148,25 @@ var _ = Describe("Emqx v2alpha2 Rebalance Test", Label("rebalance"), func() { }) AfterEach(func() { - // By("Deleting EMQX CR", func() { - // // delete emqx cr - // Eventually(func() error { - // return k8sClient.Delete(context.TODO(), instance) - // }, timeout, interval).Should(Succeed()) - // Eventually(func() bool { - // return k8sErrors.IsNotFound(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(instance), instance)) - // }).Should(BeTrue()) - // }) - - // // delete rebalance cr - // By("Deleting Rebalance CR", func() { - // Eventually(func() error { - // return k8sClient.Delete(context.TODO(), r) - // }, timeout, interval).Should(Succeed()) - // Eventually(func() bool { - // return k8sErrors.IsNotFound(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r)) - // }).Should(BeTrue()) - // }) + By("Deleting EMQX CR", func() { + // delete emqx cr + Eventually(func() error { + return k8sClient.Delete(context.TODO(), instance) + }, timeout, interval).Should(Succeed()) + Eventually(func() bool { + return k8sErrors.IsNotFound(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(instance), instance)) + }).Should(BeTrue()) + }) + + // delete rebalance cr + By("Deleting Rebalance CR", func() { + Eventually(func() error { + return k8sClient.Delete(context.TODO(), r) + }, timeout, interval).Should(Succeed()) + Eventually(func() bool { + return k8sErrors.IsNotFound(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r)) + }).Should(BeTrue()) + }) }) It("Check rebalance status", func() { @@ -190,7 +196,221 @@ var _ = Describe("Emqx v2alpha2 Rebalance Test", Label("rebalance"), func() { And( HaveField("Type", appsv2alpha2.RebalanceConditionFailed), HaveField("Status", corev1.ConditionTrue), - HaveField("Message", "Failed to start rebalance: request api failed: 400 Bad Request"), + HaveField("Message", ContainSubstring("Failed to start rebalance")), + ), + )), + )) + }) + + By("Mock rebalance is in progress", func() { + // mock rebalance processing + r.Status.Phase = appsv2alpha2.RebalancePhaseProcessing + r.Status.Conditions = []appsv2alpha2.RebalanceCondition{} + Expect(k8sClient.Status().Update(context.TODO(), r)).Should(Succeed()) + + // update annotations for target reconciler + Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r)).Should(Succeed()) + r.Annotations = map[string]string{"test": "e2e"} + Expect(k8sClient.Update(context.TODO(), r)).Should(Succeed()) + }) + + By("Rebalance should completed", func() { + Eventually(func() appsv2alpha2.RebalanceStatus { + _ = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r) + return r.Status + }, timeout, interval).Should(And( + HaveField("Phase", appsv2alpha2.RebalancePhaseCompleted), + HaveField("RebalanceStates", BeNil()), + HaveField("Conditions", ContainElements( + HaveField("Type", appsv2alpha2.RebalanceConditionCompleted), + )), + HaveField("Conditions", ContainElements( + HaveField("Status", corev1.ConditionTrue), + )), + )) + }) + }) + }) +}) + +var _ = Describe("EMQX 4 Rebalance Test", Label("rebalance"), func() { + var instance *appsv1beta4.EmqxEnterprise + var r *appsv2alpha2.Rebalance + BeforeEach(func() { + instance = &appsv1beta4.EmqxEnterprise{ + TypeMeta: metav1.TypeMeta{ + Kind: "EmqxEnterprise", + APIVersion: "apps.emqx.io/v1beta4", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "emqx-ee", + Namespace: "e2e-test-v2alpha2" + "-" + rand.String(5), + Labels: map[string]string{ + "test": "e2e", + }, + }, + Spec: appsv1beta4.EmqxEnterpriseSpec{ + Replicas: pointer.Int32(1), + ClusterDomain: "cluster.local", + Template: appsv1beta4.EmqxTemplate{ + Spec: appsv1beta4.EmqxTemplateSpec{ + EmqxContainer: appsv1beta4.EmqxContainer{ + Image: appsv1beta4.EmqxImage{ + Repository: "emqx/emqx-ee", + Version: "4.4.18", + }, + }, + }, + }, + }, + } + instance.Default() + }) + + Context("EMQX is not found", func() { + BeforeEach(func() { + r = rebalance.DeepCopy() + r.Namespace = instance.GetNamespace() + r.Spec.InstanceKind = instance.GroupVersionKind().Kind + r.Spec.InstanceName = "no-exist" + + By("Creating namespace", func() { + // create namespace + Eventually(func() bool { + err := k8sClient.Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.GetNamespace(), + Labels: map[string]string{ + "test": "e2e", + }, + }, + }) + return err == nil || k8sErrors.IsAlreadyExists(err) + }).Should(BeTrue()) + }) + + By("Creating Rebalance CR", func() { + Expect(k8sClient.Create(context.TODO(), r)).Should(Succeed()) + }) + }) + + AfterEach(func() { + By("Deleting Rebalance CR, can be successful", func() { + Eventually(func() error { + return k8sClient.Delete(context.TODO(), r) + }, timeout, interval).Should(Succeed()) + Eventually(func() bool { + return k8sErrors.IsNotFound(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r)) + }).Should(BeTrue()) + }) + }) + + It("Rebalance will failed, because the EMQX is not found", func() { + Eventually(func() appsv2alpha2.RebalanceStatus { + _ = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r) + return r.Status + }, timeout, interval).Should(And( + HaveField("Phase", appsv2alpha2.RebalancePhaseFailed), + HaveField("RebalanceStates", BeNil()), + HaveField("Conditions", ContainElements( + HaveField("Type", appsv2alpha2.RebalanceConditionFailed), + )), + HaveField("Conditions", ContainElements( + HaveField("Status", corev1.ConditionTrue), + )), + HaveField("Conditions", ContainElements( + HaveField("Message", ContainSubstring(fmt.Sprintf("%s is not found", r.Spec.InstanceName)))), + ), + )) + }) + }) + + Context("EMQX is exist", func() { + BeforeEach(func() { + r = rebalance.DeepCopy() + + By("Creating namespace", func() { + // create namespace + Eventually(func() bool { + err := k8sClient.Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.GetNamespace(), + Labels: map[string]string{ + "test": "e2e", + }, + }, + }) + return err == nil || k8sErrors.IsAlreadyExists(err) + }).Should(BeTrue()) + }) + + By("Creating EMQX CR", func() { + // create EMQX CR + Expect(instance.ValidateCreate()).Should(Succeed()) + Expect(k8sClient.Create(context.TODO(), instance)).Should(Succeed()) + + // check EMQX CR if created successfully + Eventually(func() *appsv1beta4.EmqxEnterprise { + _ = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(instance), instance) + return instance + }).WithTimeout(timeout).WithPolling(interval).Should( + WithTransform(func(instance *appsv1beta4.EmqxEnterprise) bool { + return instance.Status.IsConditionTrue(appsv1beta4.ConditionRunning) + }, BeTrue()), + ) + }) + }) + + AfterEach(func() { + By("Deleting EMQX CR", func() { + // delete emqx cr + Eventually(func() error { + return k8sClient.Delete(context.TODO(), instance) + }, timeout, interval).Should(Succeed()) + Eventually(func() bool { + return k8sErrors.IsNotFound(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(instance), instance)) + }).Should(BeTrue()) + }) + + // delete rebalance cr + By("Deleting Rebalance CR", func() { + Eventually(func() error { + return k8sClient.Delete(context.TODO(), r) + }, timeout, interval).Should(Succeed()) + Eventually(func() bool { + return k8sErrors.IsNotFound(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r)) + }).Should(BeTrue()) + }) + }) + + It("Check rebalance status", func() { + By("Create rebalance", func() { + r.Namespace = instance.GetNamespace() + r.Spec.InstanceName = instance.GetName() + r.Spec.InstanceKind = "EmqxEnterprise" + + Expect(k8sClient.Create(context.TODO(), r)).Should(Succeed()) + }) + + By("Rebalance should have finalizer", func() { + Eventually(func() []string { + _ = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r) + return r.GetFinalizers() + }, timeout, interval).Should(ContainElements("apps.emqx.io/finalizer")) + }) + + By("Rebalance will failed, because the EMQX is nothing to balance", func() { + Eventually(func() appsv2alpha2.RebalanceStatus { + _ = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(r), r) + return r.Status + }, timeout, interval).Should(And( + HaveField("Phase", appsv2alpha2.RebalancePhaseFailed), + HaveField("RebalanceStates", BeNil()), + HaveField("Conditions", ContainElements( + And( + HaveField("Type", appsv2alpha2.RebalanceConditionFailed), + HaveField("Status", corev1.ConditionTrue), + HaveField("Message", ContainSubstring("Failed to start rebalance")), ), )), )) diff --git a/e2e/v2alpha2/suite_test.go b/e2e/v2alpha2/suite_test.go index a52335687..7fcd6380d 100644 --- a/e2e/v2alpha2/suite_test.go +++ b/e2e/v2alpha2/suite_test.go @@ -36,7 +36,10 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + appsv1beta4 "github.com/emqx/emqx-operator/apis/apps/v1beta4" appsv2alpha2 "github.com/emqx/emqx-operator/apis/apps/v2alpha2" + + appscontrollersv1beta4 "github.com/emqx/emqx-operator/controllers/apps/v1beta4" appscontrollersv2alpha2 "github.com/emqx/emqx-operator/controllers/apps/v2alpha2" //+kubebuilder:scaffold:imports ) @@ -90,6 +93,9 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) + err = appsv1beta4.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = appsv2alpha2.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) @@ -105,6 +111,11 @@ var _ = BeforeSuite(func() { }) Expect(err).ToNot(HaveOccurred()) + err = (&appscontrollersv1beta4.EmqxEnterpriseReconciler{ + EmqxReconciler: appscontrollersv1beta4.NewEmqxReconciler(k8sManager), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + err = appscontrollersv2alpha2.NewEMQXReconciler(k8sManager).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred())