test: CLI E2E tests with template operator

.github/workflows/test-e2e-create-module.yml

@@ -17,7 +17,7 @@ jobs:
   e2e:
     strategy:
       matrix:
-        e2e-test: [ "create_module_kubebuilder_project", "create_module_module_config"]
+        e2e-test: [ "test-kubebuilder-module-creation", "test-moduleconfig-module-creation"]
     name: "Run E2E tests"
     runs-on: ubuntu-latest
     env:
@@ -33,7 +33,7 @@ jobs:
           go-version-file: 'go.mod'
           cache-dependency-path: 'go.sum'
       - name: Build Kyma CLI
-        run: | 
+        run: |
           make resolve validate build-linux
           chmod +x ./bin/kyma-linux
           ls -la ./bin
@@ -54,7 +54,7 @@ jobs:
         run: |
           k3d registry create oci.localhost --port 5001
       - name: Run create module with kubebuilder-project
-        if: ${{ matrix.e2e-test == 'create_module_kubebuilder_project' }}
+        if: ${{ matrix.e2e-test == 'test-kubebuilder-module-creation' }}
         run: |
           kyma alpha create module \
           --name kyma-project.io/module/template-operator \
@@ -67,7 +67,7 @@ jobs:
           --sec-scanners-config ./template-operator/sec-scanners-config.yaml
           echo "MODULE_TEMPLATE_PATH=/tmp/kubebuilder-template.yaml" >> "$GITHUB_ENV"
       - name: Run create module with module-config
-        if: ${{ matrix.e2e-test == 'create_module_module_config' }}
+        if: ${{ matrix.e2e-test == 'test-moduleconfig-module-creation' }}
         run: |
           cd ./template-operator
           make build-manifests
@@ -83,4 +83,4 @@ jobs:
       - name: Verify module template
         run: |
           echo $MODULE_TEMPLATE_PATH
-          make -C tests/e2e ${{ matrix.e2e-test }}
+          make -C tests/e2e test-module-creation

.github/workflows/test-smoke.yml → .github/workflows/test-e2e.yml

@@ -1,4 +1,4 @@
-name: TestSuite Smoke
+name: TestSuite E2E
 
 on:
   push:
@@ -14,20 +14,22 @@ on:
       - 'go.sum'
       - '**.go'
 jobs:
-  cli-deploy:
-    name: "kyma deploy"
+  e2e-tests:
+    name: "Run E2E tests"
     runs-on: ubuntu-latest
     env:
-      LIFECYCLE_MANAGER: ${{ github.repository }}
       K3D_VERSION: v5.4.7
     steps:
-      - name: Checkout
+      - name: Checkout Kyma CLI
         uses: actions/checkout@v3
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
           cache-dependency-path: 'go.sum'
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v3
+        id: install
       - name: Build Kyma CLI
         run: | 
           make resolve validate build-linux
@@ -48,5 +50,5 @@ jobs:
             --name kyma
       - name: Update Kubeconfigs
         run: k3d kubeconfig merge -a -d
-      - name: Run kyma deploy
-        run: kyma --ci alpha deploy
+      - name: Run E2E tests
+        run: make -C tests/e2e test-module-enabling-disabling

Makefile

@@ -67,7 +67,7 @@ docs:
 
 .PHONY: test
 test:
-	go test -race -coverprofile=cover.out ./...
+	go test `go list ./... | grep -v /tests/e2e` -race -coverprofile=cover.out
 	@echo "Total test coverage: $$(go tool cover -func=cover.out | grep total | awk '{print $$3}')"
 	@rm cover.out
 

go.mod

@@ -28,6 +28,8 @@ require (
 	github.com/kyma-project/hydroform/provision v0.0.0-20230831071441-f3501c89bace
 	github.com/kyma-project/lifecycle-manager v0.0.0-20230911065458-6926c58bcd43
 	github.com/mandelsoft/vfs v0.0.0-20230714093241-d557f163aecd
+	github.com/onsi/ginkgo/v2 v2.12.1
+	github.com/onsi/gomega v1.27.10
 	github.com/open-component-model/ocm v0.3.0-rc.1
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0-rc4
@@ -130,6 +132,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-migrate/migrate/v4 v4.15.1 // indirect
@@ -142,6 +145,7 @@ require (
 	github.com/google/go-github/v45 v45.2.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.1 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
@@ -198,7 +202,6 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
-	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/opencontainers/runc v1.1.5 // indirect
 	github.com/otiai10/copy v1.9.0 // indirect
 	github.com/panjf2000/ants/v2 v2.7.1 // indirect

go.sum

@@ -1344,8 +1344,8 @@ github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI=
-github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ=
+github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA=
+github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=

tests/e2e/Makefile

@@ -31,10 +31,10 @@ help: ## Display this help.
 
 ##@ E2E Tests
 
-.PHONY: create_module_kubebuilder_project
-create_module_kubebuilder_project:
-	go test --tags=create_module_kubebuilder_project
+.PHONY: test-module-creation
+test-module-creation:
+	go test ./create_module -run "Test_ModuleTemplate"
 
-.PHONY: create_module_module_config
-create_module_module_config:
-	go test --tags=create_module_module_config
+.PHONY: test-module-enabling-disabling
+test-module-enabling-disabling:
+	go test -ginkgo.v -ginkgo.focus "Kyma CLI deploy, enable and disable commands usage"

tests/e2e/kyma_create_module_test.go → tests/e2e/create_module/kyma_create_module_test.go

@@ -1,6 +1,4 @@
-//go:build create_module_kubebuilder_project || create_module_module_config
-
-package e2e_test
+package create_module_test
 
 import (
 	"os"
@@ -17,8 +15,6 @@ import (
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/cpi"
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/genericocireg"
 	ocmOCIReg "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ocireg"
-	"gopkg.in/yaml.v3"
-
 	"github.com/stretchr/testify/assert"
 )
 
@@ -66,25 +62,17 @@ func Test_ModuleTemplate(t *testing.T) {
 	sourceAccessSpec, err := ocm.DefaultContext().AccessSpecForSpec(source.Access)
 	assert.Nil(t, err)
 	githubAccessSpec, ok := sourceAccessSpec.(*github.AccessSpec)
+	assert.Equal(t, ok, true)
 	assert.Equal(t, githubAccessSpec.Type, github.Type)
 	assert.Contains(t, testRepoURL, githubAccessSpec.RepoURL)
 
 	// test security scan labels
 	secScanLabels := descriptor.Sources[0].Labels
-
-	var devBranch string
-	yaml.Unmarshal(secScanLabels[1].Value, &devBranch)
-	assert.Equal(t, "main", devBranch)
-
-	var rcTag string
-	yaml.Unmarshal(secScanLabels[2].Value, &rcTag)
-	assert.Equal(t, "0.5.0", rcTag)
-
-	var language string
-	yaml.Unmarshal(secScanLabels[3].Value, &language)
-	assert.Equal(t, "golang-mod", language)
-
-	var exclude string
-	yaml.Unmarshal(secScanLabels[4].Value, &exclude)
-	assert.Equal(t, "**/test/**,**/*_test.go", exclude)
+	assert.Equal(t, map[string]string{
+		"git.kyma-project.io/ref":                  "refs/heads/main",
+		"scan.security.kyma-project.io/dev-branch": "main",
+		"scan.security.kyma-project.io/rc-tag":     "0.5.0",
+		"scan.security.kyma-project.io/language":   "golang-mod",
+		"scan.security.kyma-project.io/exclude":    "**/test/**,**/*_test.go",
+	}, e2e.Flatten(secScanLabels))
 }

tests/e2e/deploy_enable_disable_test.go

@@ -0,0 +1,129 @@
+package e2e_test
+
+import (
+	"github.com/kyma-project/cli/internal/cli"
+	. "github.com/kyma-project/cli/tests/e2e"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Kyma CLI deploy, enable and disable commands usage", Ordered, func() {
+	kcpSystemNamespace := "kcp-system"
+	deployments := map[string]string{
+		"template-operator-controller-manager": "template-operator-system",
+		"sample-redis-deployment":              "manifest-redis",
+	}
+
+	Context("Given a Kyma Cluster", func() {
+		It("When `kyma alpha deploy` command is executed", func() {
+			Expect(ExecuteKymaDeployCommand()).To(Succeed())
+
+			By("Then the Kyma CR is in a ready state")
+			Eventually(KymaCRIsInReadyState).
+				WithContext(ctx).
+				WithArguments(k8sClient, cli.KymaNameDefault, cli.KymaNamespaceDefault).
+				Should(BeTrue())
+
+			By("And the Lifecycle Manager is in a ready state")
+			Eventually(DeploymentIsReady).
+				WithContext(ctx).
+				WithArguments(k8sClient, "lifecycle-manager-controller-manager", kcpSystemNamespace).
+				Should(BeTrue())
+		})
+	})
+
+	Context("Given a valid Template Operator module template", func() {
+		It("When a Template Operator module is applied", func() {
+			Expect(ApplyModuleTemplate("module_templates/moduletemplate_template_operator_regular.yaml")).
+				To(Succeed())
+
+			By("And the Template Operator gets enabled")
+			Expect(EnableKymaModuleWithReadyState("template-operator")).To(Succeed())
+			Eventually(KymaContainsModuleInReadyState).
+				WithContext(ctx).
+				WithArguments(k8sClient, cli.KymaNameDefault, cli.KymaNamespaceDefault, "template-operator").
+				Should(BeTrue())
+		})
+
+		It("Then Template Operator resources are deployed in the cluster", func() {
+			Eventually(ModuleResourcesAreReady).
+				WithContext(ctx).
+				WithArguments(k8sClient, "samples.operator.kyma-project.io", deployments).
+				Should(BeTrue())
+		})
+
+		It("And the Template Operator's CR state is ready", func() {
+			Eventually(CRIsReady).
+				WithContext(ctx).
+				WithArguments("sample", "sample-yaml", "kyma-system").
+				Should(BeTrue())
+		})
+	})
+
+	Context("Given a Template Operator module in a ready state", func() {
+		It("When `kyma disable module` command is execute", func() {
+			Expect(DisableModuleOnKyma("template-operator")).To(Succeed())
+
+			Eventually(KymaCRIsInReadyState).
+				WithContext(ctx).
+				WithArguments(k8sClient, cli.KymaNameDefault, cli.KymaNamespaceDefault).
+				Should(BeTrue())
+		})
+
+		It("Then the Template Operator's resources are removed from the cluster", func() {
+			Eventually(ModuleResourcesAreReady).
+				WithContext(ctx).
+				WithArguments(k8sClient, "samples.operator.kyma-project.io", deployments).
+				Should(BeFalse())
+		})
+	})
+
+	Context("Given a warning state Template Operator module template", func() {
+		It("When a Template Operator module is applied", func() {
+			Expect(ApplyModuleTemplate(
+				"module_templates/moduletemplate_template_operator_regular_warning.yaml")).
+				To(Succeed())
+		})
+
+		It("And the Template Operator enable command invoked", func() {
+			Expect(EnableKymaModuleWithWarningState("template-operator")).To(Succeed())
+
+			Eventually(KymaContainsModuleInWarningState).
+				WithContext(ctx).
+				WithArguments(k8sClient, cli.KymaNameDefault, cli.KymaNamespaceDefault, "template-operator").
+				Should(BeTrue())
+		})
+
+		It("Then the Template Operator's resources are deployed in the cluster", func() {
+			Eventually(ModuleResourcesAreReady).
+				WithContext(ctx).
+				WithArguments(k8sClient, "samples.operator.kyma-project.io", deployments).
+				Should(BeTrue())
+		})
+
+		It("And the Template Operator's CR state is in a warning state", func() {
+			Eventually(CRIsInWarningState).
+				WithContext(ctx).
+				WithArguments("sample", "sample-yaml", "kyma-system").
+				Should(BeTrue())
+		})
+	})
+
+	Context("Given a Template Operator module in a warning state", func() {
+		It("When `kyma disable module` command is executed", func() {
+			Expect(DisableModuleOnKyma("template-operator")).To(Succeed())
+
+			Eventually(KymaCRIsInReadyState).
+				WithContext(ctx).
+				WithArguments(k8sClient, cli.KymaNameDefault, cli.KymaNamespaceDefault).
+				Should(BeTrue())
+		})
+
+		It("Then Template Operator's resources are removed from the cluster", func() {
+			Eventually(ModuleResourcesAreReady).
+				WithContext(ctx).
+				WithArguments(k8sClient, "samples.operator.kyma-project.io", deployments).
+				Should(BeFalse())
+		})
+	})
+})

tests/e2e/e2e_suite_test.go

@@ -0,0 +1,58 @@
+package e2e_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/kyma-project/lifecycle-manager/api/v1beta2"
+	"github.com/kyma-project/lifecycle-manager/pkg/log"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"go.uber.org/zap/zapcore"
+	v1extensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+const (
+	timeout  = 60 * time.Second
+	interval = 1 * time.Second
+)
+
+var (
+	ctx       context.Context
+	cancel    context.CancelFunc
+	k8sClient client.Client
+)
+
+func TestE2e(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "E2e Suite")
+}
+
+var _ = BeforeSuite(func() {
+	ctx, cancel = context.WithCancel(context.TODO())
+	logf.SetLogger(log.ConfigLogger(9, zapcore.AddSync(GinkgoWriter)))
+
+	SetDefaultEventuallyPollingInterval(interval)
+	SetDefaultEventuallyTimeout(timeout)
+
+	kubeConfig := ctrl.GetConfigOrDie()
+	Expect(kubeConfig).NotTo(BeNil())
+	var err error
+	Expect(v1beta2.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
+	Expect(v1extensions.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
+
+	k8sClient, err = client.New(kubeConfig, client.Options{Scheme: scheme.Scheme})
+	Expect(err).NotTo(HaveOccurred())
+	go func() {
+		defer GinkgoRecover()
+	}()
+})
+
+var _ = AfterSuite(func() {
+	cancel()
+})