test: add e2e test cases for helm chart install

.github/workflows/e2e-test.yaml

@@ -0,0 +1,38 @@
+name: Run E2E Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - name: Login to Docker Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.DOCKER_REGISTRY }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Install Dependencies
+        run: |
+          go install github.com/onsi/ginkgo/v2/[email protected]
+          go install sigs.k8s.io/[email protected]
+          sudo cp ~/go/bin/ginkgo /usr/local/bin
+          sudo cp ~/go/bin/kind /usr/local/bin
+          curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
+          ls -l
+          sudo cp ./kustomize /usr/local/bin
+          # pull docker images
+          docker pull api7ee.azurecr.io/api7-dashboard:2.13.2302
+          docker pull api7ee.azurecr.io/api7-gateway:2.13.2302
+      - name: Run E2E Test Suites
+        working-directory: ./
+        run: |
+          make e2e-test

Makefile

@@ -0,0 +1,23 @@
+KIND_CLUSTER_NAME ?= api7-test
+KUBECONFIG ?= /tmp/$(KIND_CLUSTER_NAME).kubeconfig
+
+IMAGE_REGISTRY ?= "api7ee.azurecr.io"
+DASHBOARD_IMAGE_REPOSITORY ?= "api7-dashboard"
+DASHBOARD_IMAGE_TAG ?= "2.13.2302"
+GATEWAY_IMAGE_REPOSITORY ?= "api7-gateway"
+GATEWAY_IMAGE_TAG ?= "2.13.2302"
+
+.PHONY: help
+help:  ## Display this help message.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+.PHONY: kind-up
+kind-up: ## Create the Kubernetes cluster by kind
+	@KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./test/scripts/kind.sh
+	@kind get kubeconfig --name ${KIND_CLUSTER_NAME} > ${KUBECONFIG}
+
+.PHONY: e2e-test
+e2e-test: kind-up ## Run e2e test cases
+	@kind load docker-image --nodes $(KIND_CLUSTER_NAME)-worker --name $(KIND_CLUSTER_NAME) $(IMAGE_REGISTRY)/$(DASHBOARD_IMAGE_REPOSITORY):$(DASHBOARD_IMAGE_TAG)
+	@kind load docker-image --nodes $(KIND_CLUSTER_NAME)-worker --name $(KIND_CLUSTER_NAME) $(IMAGE_REGISTRY)/$(GATEWAY_IMAGE_REPOSITORY):$(GATEWAY_IMAGE_TAG)
+	@cd test/e2e && go env -w GOFLAGS="-mod=mod" && ginkgo -r $(CLI_TEST_TAGS)

test/e2e/alarm_test.go

@@ -0,0 +1,11 @@
+package e2e
+
+import (
+	"github.com/onsi/ginkgo/v2"
+)
+
+var _ = ginkgo.Describe("Alarm Test", func() {
+	ginkgo.Specify("prepare test data", func() {
+		// TODO: implement
+	})
+})

test/e2e/e2e_test.go

@@ -0,0 +1,44 @@
+package e2e
+
+import (
+	"testing"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	"github.com/api7/api7-helm-chart/test/framework"
+	"github.com/api7/api7-helm-chart/test/setup"
+)
+
+// TestHelmChart is the entry point for the E2E test.
+func TestHelmChart(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	f := framework.GetGlobalFramework()
+
+	ginkgo.BeforeSuite(func() {
+		// deploy Dashboard
+		err := setup.DeployDashboard(f)
+		gomega.Expect(err).Should(gomega.BeNil())
+		err = setup.EnsureDashboardReady(f)
+		gomega.Expect(err).Should(gomega.BeNil())
+		setup.CreateTunnelForDashboard(f)
+
+		// admin login
+		token := setup.LoginDashboard(f)
+		// import license
+		setup.ImportLicense(f, token)
+		// create default cluster
+		setup.CreateDefaultCluster(f, token)
+		// create default workspace
+		setup.CreateDefaultWorkspace(f, token)
+	})
+
+	ginkgo.AfterSuite(func() {
+		err := setup.UninstallDashboard(f)
+		gomega.Expect(err).Should(gomega.BeNil())
+
+		f.CleanUp()
+	})
+
+	ginkgo.RunSpecs(t, "E2E test suites")
+}

test/e2e/testdata/certs/license

@@ -0,0 +1 @@
+A-4GfX06HWgo99GtG21scvK6LFGmqoru7NGKze_LY4ZngNgaxLJdjl6AZAUR7jC7e9iVlKoYYyc8QjJHVBnb3SULMzqQt6qAaMlSgbomKmyaiXOn8oE3_bcaz-Vx5kYBfR5Rs9TTu6aHk-6TWSbig3aOTQo4ZkvHbQBdgu-GPFciJz029hVedC4ue-eGFGCj7rIyJ5jE837WnTvQtcfMiHK8cDU-DYDVwdGszIqFCyFrj9Q0AK-7_Ji-VwzG89h_D2sCA8raFCJh9ghwRyhJaKdGoPG1AktQ70Y_gqhViFEyWUeKboDVhVPAkyf4yE9Jzr1uap3ghf45Xi8eoEmNbbYcugPf2DTIyRyyCNqzz-QexFKWP19KyihCbhA3E6k0Q2njyZJTK9HwpWv6K8mfQ41cxBksPrQyfWfmTP7M0IKZyACqoVt3tbF-w17MoOmd_mCNBKZz7oVjwXfTekSyi9J0nLRUBbqww2oGXz5THYRJ2j1xFx8KFiGKe3GS2O9IaMX3qeYVSVl0i-V_YztIa-J3gemEen13QQlKQTfnMnK4i4QU9SqJ1sN4gU7UWFchVDTLDhJcAuyMSqfnbgzddzZxTEIPkAeR-P5RPJNVjeDMIP5H7IgW6N0rjzkXWGsPjEr8n1TeOhTW0UZodj-F-NPEsLtt9-qhL6zMV0OJa8A

test/framework/framework.go

@@ -0,0 +1,139 @@
+package framework
+
+import (
+	"bufio"
+	"context"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+var (
+	_globalFramework *Framework
+	_globalOnce      sync.Once
+)
+
+// Framework is the framework for e2e tests.
+type Framework struct {
+	GomegaT *gomega.GomegaWithT
+	GinkgoT ginkgo.GinkgoTInterface
+
+	Render *rendering
+	config *Config
+
+	clientset   *kubernetes.Clientset
+	clientCfg   *restclient.Config
+	KubectlOpts *k8s.KubectlOptions
+	tunnels     map[string]*k8s.Tunnel
+	tunnelMutex sync.Mutex
+}
+
+type rendering struct {
+	Config *Config
+}
+
+// Config return the config of framework.
+type Config struct {
+	Namespace  string `json:"namespace"`
+	KubeConfig string `json:"kubeconfig"`
+}
+
+var config = &Config{}
+
+// GetGlobalFramework get a global framework with default settings.
+func GetGlobalFramework() *Framework {
+	if _globalFramework == nil {
+		_globalOnce.Do(func() {
+			_globalFramework = InitGlobalFramework(config)
+		})
+	}
+
+	return _globalFramework
+}
+
+// InitGlobalFramework init a global framework with default settings.
+func InitGlobalFramework(config *Config) *Framework {
+	f := &Framework{
+		GomegaT: gomega.NewWithT(ginkgo.GinkgoT(4)),
+		GinkgoT: ginkgo.GinkgoT(),
+		Render: &rendering{
+			Config: config,
+		},
+		config:  config,
+		tunnels: make(map[string]*k8s.Tunnel),
+	}
+	f.config.Namespace = getNamespace()
+
+	// use the current context in kubeconfig
+	cfg, err := clientcmd.BuildConfigFromFlags("", getKubeconfig())
+	f.clientCfg = cfg
+	f.GomegaT.Expect(err).ShouldNot(gomega.HaveOccurred(), "creating kubeconfig")
+
+	f.KubectlOpts = k8s.NewKubectlOptions("", "", f.config.Namespace)
+	// create the clientset
+	clientset, err := kubernetes.NewForConfig(cfg)
+	f.GomegaT.Expect(err).ShouldNot(gomega.HaveOccurred(), "creating clientset")
+
+	f.clientset = clientset
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err = f.clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: config.Namespace,
+		},
+	}, metav1.CreateOptions{})
+	f.GomegaT.Expect(err).ShouldNot(gomega.HaveOccurred(), "creating namespace")
+
+	ginkgo.AfterEach(f.dumpErrorLogs)
+	return f
+}
+
+// CloseTunnels close tunnels for the framework.
+func (f *Framework) CloseTunnels() {
+	for _, tunnel := range f.tunnels {
+		tunnel.Close()
+	}
+}
+
+// CleanUp clean up the resources that create by framework.
+func (f *Framework) CleanUp() {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	f.CloseTunnels()
+	err := f.clientset.CoreV1().Namespaces().Delete(ctx, f.config.Namespace, metav1.DeleteOptions{})
+	f.GomegaT.Expect(err).ShouldNot(gomega.HaveOccurred(), "deleting namespace")
+}
+
+// dumpErrorLogs dump the error logs of the test.
+func (f *Framework) dumpErrorLogs() {
+	if ginkgo.CurrentSpecReport().Failed() {
+		ginkgo.GinkgoWriter.Println("dump component logs")
+		logs := f.LogPodBySelector("logdump=True", -1, true)
+		scanner := bufio.NewScanner(strings.NewReader(logs))
+		for scanner.Scan() {
+			line := scanner.Text()
+			ginkgo.GinkgoWriter.Println(line)
+		}
+	}
+}
+
+// GetPodEndpoint returns an url.URL object which contains accessible
+// the pod listen address.
+func (f *Framework) GetPodEndpoint(name string) *url.URL {
+	return &url.URL{
+		Scheme: "http",
+		Host:   f.tunnels[name].Endpoint(),
+	}
+}

test/framework/helm.go

@@ -0,0 +1,53 @@
+package framework
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+func (f *Framework) GetWorkingDir() (string, error) {
+	pwd, err := os.Getwd()
+	if err != nil {
+		return "", fmt.Errorf("failed to get working directory: %v", err)
+	}
+
+	return filepath.Abs(pwd + "/../../")
+}
+
+func (f *Framework) InstallChart(dir, name string, args ...string) error {
+	pwd, err := f.GetWorkingDir()
+	if err != nil {
+		return err
+	}
+
+	// helm dependency update
+	cmd := exec.Command("helm", "dependency", "update", ".")
+	cmd.Dir = pwd + "/charts/" + dir
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to update dependency error: %v, output: %s", err, string(output))
+	}
+
+	// install chart
+	args = append([]string{"install", name, ".", "-n", f.config.Namespace}, args...)
+	cmd = exec.Command("helm", args...)
+	cmd.Dir = pwd + "/charts/" + dir
+	output, err = cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to install %s error: %v, output: %s", name, err, string(output))
+	}
+
+	return nil
+}
+
+func (f *Framework) UninstallChart(name string) error {
+	cmd := exec.Command("helm", "uninstall", name, "-n", f.config.Namespace)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to uninstall %s release: %v, output: %s", name, err, string(output))
+	}
+
+	return nil
+}