Skip to content

Commit

Permalink
chore: fix test noise
Browse files Browse the repository at this point in the history 
Signed-off-by: Rory Z <[email protected]>
  • Loading branch information
@Rory-Z
Rory-Z committed Jan 7, 2024
1 parent f7aa9ec commit 104d3b4
Show file tree
Hide file tree
Showing 8 changed files with 200 additions and 155 deletions.
2 changes: 1 addition & 1 deletion RELEASE.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ EMQX Operator 2.2.7 has been released.

+ `apps.emqx.io/v2beta1 EMQX`.

+ Sometimes the updated statefulSet / replicaSet will not be ready, because the EMQX node can not be started. Then we will roll back EMQX CR spec, the EMQX operator controller will create a new statefulSet / replicaSet. But the new statefulSet / replicaSet will be the same as the previous one, so we didn't need to create it, just change the EMQX status.
+ Sometimes the updated statefulSet / replicaSet will not be ready, because the EMQX node can not be started. Then we will roll back EMQX CR spec, the EMQX operator controller will create a new statefulSet / replicaSet. But the new statefulSet / replicaSet will be the same as the previous one, so we didn't need to create it, just change the EMQX status.

## How to install/upgrade EMQX Operator 💡

Expand Down
41 changes: 20 additions & 21 deletions e2e/v1beta4/e2e_base_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ limitations under the License.
package v1beta4

import (
"context"
"fmt"

appsv1beta4 "github.com/emqx/emqx-operator/apis/apps/v1beta4"
Expand Down Expand Up @@ -224,7 +223,7 @@ var _ = Describe("Base E2E Test", Label("base"), func() {
By("create EMQX Plugin")
plugin.Labels = emqx.GetLabels()
plugin.Namespace = emqx.GetNamespace()
Expect(k8sClient.Create(context.TODO(), plugin)).Should(Succeed())
Expect(k8sClient.Create(ctx, plugin)).Should(Succeed())

By("check EMQX CR status")
Expect(emqx.GetStatus().GetReplicas()).Should(Equal(int32(1)))
Expand All @@ -235,7 +234,7 @@ var _ = Describe("Base E2E Test", Label("base"), func() {

By("check pod annotations")
sts := &appsv1.StatefulSet{}
Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(emqx), sts)).Should(Succeed())
Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(emqx), sts)).Should(Succeed())
Expect(sts.Spec.Template.Annotations).Should(HaveKey(handler.ManageContainersAnnotation))

By("checking the EMQX Custom Resource's EndpointSlice", func() {
Expand All @@ -246,7 +245,7 @@ var _ = Describe("Base E2E Test", Label("base"), func() {
Eventually(func() []string {
list := appsv1beta4.EmqxPluginList{}
_ = k8sClient.List(
context.Background(),
ctx,
&list,
client.InNamespace(emqx.GetNamespace()),
client.MatchingLabels(emqx.GetLabels()),
Expand All @@ -265,7 +264,7 @@ var _ = Describe("Base E2E Test", Label("base"), func() {
Eventually(func() map[string]string {
cm := &corev1.ConfigMap{}
_ = k8sClient.Get(
context.Background(),
ctx,
types.NamespacedName{
Name: fmt.Sprintf("%s-%s", emqx.GetName(), "plugins-config"),
Namespace: emqx.GetNamespace(),
Expand All @@ -278,7 +277,7 @@ var _ = Describe("Base E2E Test", Label("base"), func() {
Eventually(func() []corev1.ServicePort {
svc := &corev1.Service{}
_ = k8sClient.Get(
context.Background(),
ctx,
types.NamespacedName{
Name: fmt.Sprintf("%s-%s", emqx.GetName(), "headless"),
Namespace: emqx.GetNamespace(),
Expand All @@ -296,7 +295,7 @@ var _ = Describe("Base E2E Test", Label("base"), func() {
Eventually(func() error {
p := &appsv1beta4.EmqxPlugin{}
err := k8sClient.Get(
context.Background(),
ctx,
types.NamespacedName{
Name: plugin.GetName(),
Namespace: plugin.GetNamespace(),
Expand All @@ -310,7 +309,7 @@ var _ = Describe("Base E2E Test", Label("base"), func() {
p.Spec.Config["lwm2m.bind.udp.2"] = "0.0.0.0:5696"
p.Spec.Config["lwm2m.bind.dtls.1"] = "0.0.0.0:5697"
p.Spec.Config["lwm2m.bind.dtls.2"] = "0.0.0.0:5698"
return k8sClient.Update(context.Background(), p)
return k8sClient.Update(ctx, p)
}, timeout, interval).Should(Succeed())

pluginPorts = []corev1.ServicePort{
Expand Down Expand Up @@ -356,18 +355,18 @@ func createEmqx(emqx appsv1beta4.Emqx) {
emqx.SetNamespace(emqx.GetNamespace() + "-" + rand.String(5))
emqx.Default()
Expect(emqx.ValidateCreate()).Should(Succeed())
Expect(k8sClient.Create(context.TODO(), &corev1.Namespace{
Expect(k8sClient.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: emqx.GetNamespace(),
Labels: map[string]string{
"test": "e2e",
},
},
})).Should(Succeed())
Expect(k8sClient.Create(context.TODO(), emqx)).Should(Succeed())
Expect(k8sClient.Create(ctx, emqx)).Should(Succeed())
Eventually(func() bool {
_ = k8sClient.Get(
context.TODO(),
ctx,
types.NamespacedName{
Name: emqx.GetName(),
Namespace: emqx.GetNamespace(),
Expand All @@ -382,25 +381,25 @@ func deleteEmqx(emqx appsv1beta4.Emqx) {
finalizer := "apps.emqx.io/finalizer"
plugins := &appsv1beta4.EmqxPluginList{}
_ = k8sClient.List(
context.Background(),
ctx,
plugins,
client.InNamespace("default"),
)
for _, plugin := range plugins.Items {
controllerutil.RemoveFinalizer(&plugin, finalizer)
Expect(k8sClient.Update(context.Background(), &plugin)).Should(Succeed())
Expect(k8sClient.Delete(context.Background(), &plugin)).Should(Succeed())
Expect(k8sClient.Update(ctx, &plugin)).Should(Succeed())
Expect(k8sClient.Delete(ctx, &plugin)).Should(Succeed())
}
Expect(k8sClient.Delete(context.TODO(), emqx)).Should(Succeed())
Expect(k8sClient.Delete(ctx, emqx)).Should(Succeed())

Expect(k8sClient.Delete(context.TODO(), &corev1.Namespace{
Expect(k8sClient.Delete(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: emqx.GetNamespace(),
},
})).Should(Succeed())

Eventually(func() bool {
err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: emqx.GetNamespace()}, &corev1.Namespace{})
err := k8sClient.Get(ctx, types.NamespacedName{Name: emqx.GetNamespace()}, &corev1.Namespace{})
return k8sErrors.IsNotFound(err)
}, timeout, interval).Should(BeTrue())
}
Expand All @@ -409,7 +408,7 @@ func checkService(emqx appsv1beta4.Emqx, ports, pluginPorts []corev1.ServicePort
Eventually(func() []corev1.ServicePort {
svc := &corev1.Service{}
_ = k8sClient.Get(
context.Background(),
ctx,
types.NamespacedName{
Name: emqx.GetName(),
Namespace: emqx.GetNamespace(),
Expand All @@ -423,7 +422,7 @@ func checkService(emqx appsv1beta4.Emqx, ports, pluginPorts []corev1.ServicePort
func checkPodAndEndpointsAndEndpointSlices(emqx appsv1beta4.Emqx, ports, pluginPorts []corev1.ServicePort, headlessPort corev1.ServicePort, count int) {
podList := &corev1.PodList{}
Eventually(func() []corev1.Pod {
_ = k8sClient.List(context.TODO(), podList,
_ = k8sClient.List(ctx, podList,
client.InNamespace(emqx.GetNamespace()),
client.MatchingLabels(emqx.GetSpec().GetTemplate().Labels),
)
Expand Down Expand Up @@ -494,7 +493,7 @@ func checkPodAndEndpointsAndEndpointSlices(emqx appsv1beta4.Emqx, ports, pluginP

Eventually(func() *corev1.Endpoints {
ep := &corev1.Endpoints{}
_ = k8sClient.Get(context.TODO(), types.NamespacedName{Name: emqx.GetSpec().GetServiceTemplate().Name, Namespace: emqx.GetSpec().GetServiceTemplate().Namespace}, ep)
_ = k8sClient.Get(ctx, types.NamespacedName{Name: emqx.GetSpec().GetServiceTemplate().Name, Namespace: emqx.GetSpec().GetServiceTemplate().Namespace}, ep)
return ep
}, timeout, interval).Should(HaveField("Subsets",
And(
Expand All @@ -511,7 +510,7 @@ func checkPodAndEndpointsAndEndpointSlices(emqx appsv1beta4.Emqx, ports, pluginP
Eventually(func() []discoveryv1.EndpointSlice {
list := &discoveryv1.EndpointSliceList{}
_ = k8sClient.List(
context.TODO(), list,
ctx, list,
client.InNamespace(emqx.GetNamespace()),
client.MatchingLabels(
map[string]string{
Expand Down
21 changes: 10 additions & 11 deletions e2e/v1beta4/e2e_blure_green_upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ limitations under the License.
package v1beta4

import (
"context"
"fmt"
"sort"

Expand Down Expand Up @@ -57,7 +56,7 @@ var _ = Describe("Blue Green Update Test", Label("blue"), func() {
existedStsList = &appsv1.StatefulSetList{}
Eventually(func() []appsv1.StatefulSet {
_ = k8sClient.List(
context.TODO(),
ctx,
existedStsList,
client.InNamespace(emqx.GetNamespace()),
client.MatchingLabels(emqx.GetLabels()),
Expand All @@ -69,7 +68,7 @@ var _ = Describe("Blue Green Update Test", Label("blue"), func() {
Eventually(func() string {
// Wait sts ready
_ = k8sClient.Get(
context.TODO(),
ctx,
types.NamespacedName{
Name: sts.GetName(),
Namespace: sts.GetNamespace(),
Expand All @@ -82,7 +81,7 @@ var _ = Describe("Blue Green Update Test", Label("blue"), func() {
By("check CR status before update")
Eventually(func() appsv1beta4.EmqxEnterpriseStatus {
ee := &appsv1beta4.EmqxEnterprise{}
_ = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(emqx), ee)
_ = k8sClient.Get(ctx, client.ObjectKeyFromObject(emqx), ee)
return ee.Status
}, timeout, interval).Should(And(
HaveField("Conditions", HaveExactElements(
Expand All @@ -104,20 +103,20 @@ var _ = Describe("Blue Green Update Test", Label("blue"), func() {
})

By("update EMQX CR")
Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(emqx), emqx)).Should(Succeed())
Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(emqx), emqx)).Should(Succeed())
emqx.Spec.Template.Spec.Volumes = append(emqx.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "test-blue-green-update",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
})
Expect(k8sClient.Update(context.Background(), emqx)).Should(Succeed())
Expect(k8sClient.Update(ctx, emqx)).Should(Succeed())

By("wait create new sts")
existedStsList = &appsv1.StatefulSetList{}
Eventually(func() []appsv1.StatefulSet {
_ = k8sClient.List(
context.TODO(),
ctx,
existedStsList,
client.InNamespace(emqx.GetNamespace()),
client.MatchingLabels(emqx.GetLabels()),
Expand All @@ -136,7 +135,7 @@ var _ = Describe("Blue Green Update Test", Label("blue"), func() {
Eventually(func() string {
// Wait sts ready
_ = k8sClient.Get(
context.TODO(),
ctx,
types.NamespacedName{
Name: newSts.GetName(),
Namespace: newSts.GetNamespace(),
Expand All @@ -149,7 +148,7 @@ var _ = Describe("Blue Green Update Test", Label("blue"), func() {
By("check CR status in blue-green updating")
Eventually(func() appsv1beta4.EmqxEnterpriseStatus {
ee := &appsv1beta4.EmqxEnterprise{}
_ = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(emqx), ee)
_ = k8sClient.Get(ctx, client.ObjectKeyFromObject(emqx), ee)
return ee.Status
}, timeout, interval).Should(And(
HaveField("Conditions", HaveExactElements(
Expand Down Expand Up @@ -178,7 +177,7 @@ var _ = Describe("Blue Green Update Test", Label("blue"), func() {
By("check CR status after update")
Eventually(func() appsv1beta4.EmqxEnterpriseStatus {
ee := &appsv1beta4.EmqxEnterprise{}
_ = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(emqx), ee)
_ = k8sClient.Get(ctx, client.ObjectKeyFromObject(emqx), ee)
return ee.Status
}, timeout, interval).Should(And(
HaveField("Conditions", HaveExactElements(
Expand All @@ -203,7 +202,7 @@ var _ = Describe("Blue Green Update Test", Label("blue"), func() {
Eventually(func() []corev1.Pod {
podList := &corev1.PodList{}
_ = k8sClient.List(
context.TODO(),
ctx,
podList,
client.InNamespace(sts.GetNamespace()),
client.MatchingLabels(map[string]string{
Expand Down
14 changes: 6 additions & 8 deletions e2e/v1beta4/e2e_upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ limitations under the License.
package v1beta4

import (
"context"

appsv1beta4 "github.com/emqx/emqx-operator/apis/apps/v1beta4"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
Expand All @@ -36,7 +34,7 @@ var _ = Describe("Upgrade Test", Label("upgrade"), func() {
By("wait sts ready", func() {
Eventually(func() []appsv1.StatefulSet {
list := &appsv1.StatefulSetList{}
_ = k8sClient.List(context.TODO(), list,
_ = k8sClient.List(ctx, list,
client.InNamespace(emqx.GetNamespace()),
client.MatchingLabels(emqx.GetLabels()),
)
Expand All @@ -60,21 +58,21 @@ var _ = Describe("Upgrade Test", Label("upgrade"), func() {
switch emqx.(type) {
case *appsv1beta4.EmqxBroker:
obj := &appsv1beta4.EmqxBroker{}
Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(emqx), obj)).Should(Succeed())
Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(emqx), obj)).Should(Succeed())
obj.Spec.Template.Spec.EmqxContainer.Image.Version = "4.4.17"
Expect(k8sClient.Update(context.TODO(), obj)).Should(Succeed())
Expect(k8sClient.Update(ctx, obj)).Should(Succeed())
case *appsv1beta4.EmqxEnterprise:
obj := &appsv1beta4.EmqxEnterprise{}
Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(emqx), obj)).Should(Succeed())
Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(emqx), obj)).Should(Succeed())
obj.Spec.Template.Spec.EmqxContainer.Image.Version = "4.4.17"
Expect(k8sClient.Update(context.TODO(), obj)).Should(Succeed())
Expect(k8sClient.Update(ctx, obj)).Should(Succeed())
}
})

By("wait sts ready", func() {
Eventually(func() []appsv1.StatefulSet {
list := &appsv1.StatefulSetList{}
_ = k8sClient.List(context.TODO(), list,
_ = k8sClient.List(ctx, list,
client.InNamespace(emqx.GetNamespace()),
client.MatchingLabels(emqx.GetLabels()),
)
Expand Down
5 changes: 4 additions & 1 deletion e2e/v1beta4/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package v1beta4

import (
"context"
"os"
"path/filepath"
"testing"
Expand All @@ -42,8 +43,9 @@ import (

// var cfg *rest.Config
var timeout, interval time.Duration
var k8sClient client.Client
var testEnv *envtest.Environment
var k8sClient client.Client
var ctx context.Context

func TestAPIs(t *testing.T) {
RegisterFailHandler(Fail)
Expand All @@ -60,6 +62,7 @@ func TestAPIs(t *testing.T) {
var _ = BeforeSuite(func() {
timeout = time.Minute * 3
interval = time.Millisecond * 250
ctx = context.Background()

Expect(os.Setenv("USE_EXISTING_CLUSTER", "true")).To(Succeed())

Expand Down
Loading

0 comments on commit 104d3b4

Please sign in to comment.