From e85ca3e990a9e1e96e5b03483189e8bcaddca2d5 Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Sun, 16 Jun 2024 21:01:59 -0400 Subject: [PATCH] allow KinD clusters to be deleted or retained Modifies the KinD Fixture to make it more stable and more flexible. The default behaviour is for the KinD Fixture to delete the KinD cluster only if the KinD cluster existed before the Fixture is started. Similarly, if the KinD Cluster did *not* exist before the Fixture is started, the Fixture deletes the KinD Cluster when it is stopped. This commit adds a WithDeleteOnStop() and WithRetainOnStop() modifier to the KinD Fixture creation allowing users to indicate if they want to change that default behaviour. In addition to the above enhancements, this commit makes our testing more stable by adding some KinD cluster cleanup to our Makefile as well as serializing execution of longer-running, resource-intensive tests. Issue #13 Signed-off-by: Jay Pipes --- .github/workflows/gate-tests.yml | 4 +- Makefile | 38 ++++++++++++- eval_test.go | 70 +++++------------------ fixtures/kind/kind.go | 97 ++++++++++++++++++++++++++++++-- fixtures/kind/kind_test.go | 22 +++++++- go.mod | 2 +- go.sum | 4 +- placement_test.go | 51 +++++++++++++++++ testutil/kind.go | 13 +++++ 9 files changed, 231 insertions(+), 70 deletions(-) create mode 100644 placement_test.go create mode 100644 testutil/kind.go diff --git a/.github/workflows/gate-tests.yml b/.github/workflows/gate-tests.yml index 19bf432..f3f129a 100644 --- a/.github/workflows/gate-tests.yml +++ b/.github/workflows/gate-tests.yml @@ -38,7 +38,7 @@ jobs: - name: run non-Kind tests env: SKIP_KIND: 1 - run: go test -v ./... + run: make test test-all: strategy: matrix: @@ -69,4 +69,4 @@ jobs: with: cluster_name: kind - name: run all tests - run: go test -v ./... + run: make test-all diff --git a/Makefile b/Makefile index ce0acc9..5a63fd6 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,44 @@ VERSION ?= $(shell git describe --tags --always --dirty) +CLUSTERS = $(shell kind get clusters) .PHONY: test +kind-install: +ifeq (, $(shell command -v kind)) + go install sigs.k8s.io/kind@v0.23.0 +endif + # We clear the test cache because some of the tests require an out-of-band KinD # cluster to run against and we want to re-run tests against that KinD cluster # instead of from cached unit test results. -test: +clear-test-cache: + @echo -n "clearing Go test cache ... " @go clean -testcache - @go test -v ./... + @echo "ok." + +kind-clear-clusters: +ifneq (, $(shell command -v kind)) +ifneq (, $(CLUSTERS)) + @echo -n "clearing KinD clusters ... " + @for c in $(CLUSTERS); do kind delete cluster -q --name $$c; done + @echo "ok." +endif +endif + +kind-create-cluster: +ifneq (, $(shell command -v kind)) + @echo -n "creating 'kind' cluster ... " + @kind create cluster -q + @echo "ok." + @sleep 5 +endif + +test: clear-test-cache kind-clear-clusters kind-create-cluster test-kind-simple + +test-kind-simple: + @go test -v ./parse_test.go + @go test -v ./eval_test.go + +test-all: test kind-clear-clusters + @go test -v ./fixtures/kind/kind_test.go + @go test -v ./placement_test.go diff --git a/eval_test.go b/eval_test.go index dbad7ea..3e95c65 100644 --- a/eval_test.go +++ b/eval_test.go @@ -5,21 +5,19 @@ package kube_test import ( - "bufio" - "bytes" - "fmt" - "os" "path/filepath" "testing" "github.com/gdt-dev/gdt" gdtcontext "github.com/gdt-dev/gdt/context" - kindfix "github.com/gdt-dev/kube/fixtures/kind" "github.com/stretchr/testify/require" + + kindfix "github.com/gdt-dev/kube/fixtures/kind" + "github.com/gdt-dev/kube/testutil" ) func TestListPodsEmpty(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "list-pods-empty.yaml") @@ -36,7 +34,7 @@ func TestListPodsEmpty(t *testing.T) { } func TestGetPodNotFound(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "get-pod-not-found.yaml") @@ -53,7 +51,7 @@ func TestGetPodNotFound(t *testing.T) { } func TestCreateUnknownResource(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "create-unknown-resource.yaml") @@ -70,7 +68,7 @@ func TestCreateUnknownResource(t *testing.T) { } func TestDeleteResourceNotFound(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "delete-resource-not-found.yaml") @@ -87,7 +85,7 @@ func TestDeleteResourceNotFound(t *testing.T) { } func TestDeleteUnknownResource(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "delete-unknown-resource.yaml") @@ -104,7 +102,7 @@ func TestDeleteUnknownResource(t *testing.T) { } func TestPodCreateGetDelete(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "create-get-delete-pod.yaml") @@ -121,7 +119,7 @@ func TestPodCreateGetDelete(t *testing.T) { } func TestMatches(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "matches.yaml") @@ -138,7 +136,7 @@ func TestMatches(t *testing.T) { } func TestConditions(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "conditions.yaml") @@ -155,7 +153,7 @@ func TestConditions(t *testing.T) { } func TestJSON(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "json.yaml") @@ -172,7 +170,7 @@ func TestJSON(t *testing.T) { } func TestApply(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "apply-deployment.yaml") @@ -189,7 +187,7 @@ func TestApply(t *testing.T) { } func TestEnvvarSubstitution(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) t.Setenv("pod_name", "foo") @@ -208,7 +206,7 @@ func TestEnvvarSubstitution(t *testing.T) { } func TestWithLabels(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "list-pods-with-labels.yaml") @@ -223,41 +221,3 @@ func TestWithLabels(t *testing.T) { err = s.Run(ctx, t) require.Nil(err) } - -func TestPlacementSpread(t *testing.T) { - skipIfNoKind(t) - require := require.New(t) - - fp := filepath.Join("testdata", "placement-spread.yaml") - - s, err := gdt.From(fp) - require.Nil(err) - require.NotNil(s) - - kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml") - - var b bytes.Buffer - w := bufio.NewWriter(&b) - ctx := gdtcontext.New(gdtcontext.WithDebug(w)) - - ctx = gdtcontext.RegisterFixture( - ctx, "kind-three-workers-three-zones", - kindfix.New( - kindfix.WithClusterName("kind-three-workers-three-zones"), - kindfix.WithConfigPath(kindCfgPath), - ), - ) - - err = s.Run(ctx, t) - require.Nil(err) - - w.Flush() - fmt.Println(b.String()) -} - -func skipIfNoKind(t *testing.T) { - _, found := os.LookupEnv("SKIP_KIND") - if found { - t.Skipf("skipping KinD-requiring test") - } -} diff --git a/fixtures/kind/kind.go b/fixtures/kind/kind.go index 03cf1f6..1982eda 100644 --- a/fixtures/kind/kind.go +++ b/fixtures/kind/kind.go @@ -5,8 +5,11 @@ package kind import ( + "context" "strings" + gdtcontext "github.com/gdt-dev/gdt/context" + "github.com/gdt-dev/gdt/debug" gdttypes "github.com/gdt-dev/gdt/types" "github.com/samber/lo" "sigs.k8s.io/kind/pkg/cluster" @@ -20,9 +23,29 @@ import ( type KindFixture struct { // provider is the KinD cluster provider provider *cluster.Provider - // cfgStr contains the stringified KUBECONFIG that KinD returns in its - // Provider.KubeConfig() call - cfgStr string + // deleteOnStop indicates that the KinD cluster should be deleted when + // the fixture is stopped. Fixtures are stopped when test scenarios + // utilizing the fixture have executed all their test steps. + // + // By default, KinD clusters that were already running when the fixture was + // started are not deleted. This is to prevent the deletion of KinD + // clusters that were in use outside of a gdt-kube execution. To override + // this behaviour and always delete the KinD cluster on stop, use the + // WithDeleteOnStop() modifier. + deleteOnStop bool + // retainOnStop indicates that the KinD cluster should *not* be deleted + // when the fixture is stopped. Fixtures are stopped when test scenarios + // utilizing the fixture have executed all their test steps. + // + // By default, KinD clusters that were *not* already running when the fixture was + // started are deleted when the fixture stops. This is to clean up KinD + // clusters that were created and used by the gdt-kube execution. To override + // this behaviour and always retain the KinD cluster on stop, use the + // WithRetainOnStop() modifier. + retainOnStop bool + // runningBeforeStart is true when the KinD cluster was already running + // when the fixture was started. + runningBeforeStart bool // ClusterName is the name of the KinD cluster. If not specified, gdt will // use the default cluster name that KinD uses, which is just "kind" ClusterName string @@ -34,20 +57,35 @@ type KindFixture struct { ConfigPath string } -func (f *KindFixture) Start() { +func (f *KindFixture) Start(ctx context.Context) { + ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.start") + defer func() { + ctx = gdtcontext.PopTrace(ctx) + }() if f.ClusterName == "" { f.ClusterName = kindconst.DefaultClusterName } if f.isRunning() { + debug.Println(ctx, "cluster %s already running", f.ClusterName) + f.runningBeforeStart = true return } opts := []cluster.CreateOption{} if f.ConfigPath != "" { + debug.Println( + ctx, "using custom kind config %s for cluster %s", + f.ConfigPath, f.ClusterName, + ) opts = append(opts, cluster.CreateWithConfigFile(f.ConfigPath)) } if err := f.provider.Create(f.ClusterName, opts...); err != nil { panic(err) } + debug.Println(ctx, "cluster %s successfully created", f.ClusterName) + if !f.retainOnStop { + f.deleteOnStop = true + debug.Println(ctx, "cluster %s will be deleted on stop", f.ClusterName) + } } func (f *KindFixture) isRunning() bool { @@ -61,7 +99,26 @@ func (f *KindFixture) isRunning() bool { return lo.Contains(clusterNames, f.ClusterName) } -func (f *KindFixture) Stop() {} +func (f *KindFixture) Stop(ctx context.Context) { + ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.stop") + defer func() { + ctx = gdtcontext.PopTrace(ctx) + }() + if !f.isRunning() { + debug.Println(ctx, "cluster %s not running", f.ClusterName) + return + } + if f.runningBeforeStart && !f.deleteOnStop { + debug.Println(ctx, "cluster %s was running before start and deleteOnStop=false so not deleting", f.ClusterName) + return + } + if f.deleteOnStop { + if err := f.provider.Delete(f.ClusterName, ""); err != nil { + panic(err) + } + debug.Println(ctx, "cluster %s successfully deleted", f.ClusterName) + } +} func (f *KindFixture) HasState(key string) bool { lkey := strings.ToLower(key) @@ -119,6 +176,36 @@ func WithConfigPath(path string) KindFixtureModifier { } } +// WithDeleteOnStop instructs gdt-kube to always delete the KinD cluster when +// the fixture stops. Fixtures are stopped when test scenarios utilizing the +// fixture have executed all their test steps. +// +// By default, KinD clusters that were already running when the fixture was +// started are not deleted. This is to prevent the deletion of KinD +// clusters that were in use outside of a gdt-kube execution. To override +// this behaviour and always delete the KinD cluster on stop, use the +// WithDeleteOnStop() modifier. +func WithDeleteOnStop() KindFixtureModifier { + return func(f *KindFixture) { + f.deleteOnStop = true + } +} + +// WithRetainOnStop instructs gdt-kube that the KinD cluster should *not* be +// deleted when the fixture is stopped. Fixtures are stopped when test +// scenarios utilizing the fixture have executed all their test steps. +// +// By default, KinD clusters that were *not* already running when the fixture +// was started are deleted when the fixture stops. This is to clean up KinD +// clusters that were created and used by the gdt-kube execution. To override +// this behaviour and always retain the KinD cluster on stop, use the +// WithRetainOnStop() modifier. +func WithRetainOnStop() KindFixtureModifier { + return func(f *KindFixture) { + f.retainOnStop = true + } +} + // New returns a fixture that exposes Kubernetes configuration/context // information about a KinD cluster. If no such KinD cluster exists, one will // be created. If the KinD cluster is created, it is destroyed at the end of diff --git a/fixtures/kind/kind_test.go b/fixtures/kind/kind_test.go index 556c0a0..3778bba 100644 --- a/fixtures/kind/kind_test.go +++ b/fixtures/kind/kind_test.go @@ -5,6 +5,9 @@ package kind_test import ( + "bufio" + "bytes" + "fmt" "os" "path/filepath" "testing" @@ -25,10 +28,19 @@ func TestDefaultSingleControlPlane(t *testing.T) { require.Nil(err) require.NotNil(s) - ctx := gdtcontext.New() - ctx = gdtcontext.RegisterFixture(ctx, "kind", kindfix.New()) + var b bytes.Buffer + w := bufio.NewWriter(&b) + ctx := gdtcontext.New(gdtcontext.WithDebug(w)) + ctx = gdtcontext.RegisterFixture( + ctx, "kind", + kindfix.New( + kindfix.WithDeleteOnStop(), + ), + ) err = s.Run(ctx, t) + w.Flush() + fmt.Println(b.String()) require.Nil(err) } @@ -44,7 +56,9 @@ func TestOneControlPlaneOneWorker(t *testing.T) { kindCfgPath := filepath.Join("testdata", "kind-config-one-cp-one-worker.yaml") - ctx := gdtcontext.New() + var b bytes.Buffer + w := bufio.NewWriter(&b) + ctx := gdtcontext.New(gdtcontext.WithDebug(w)) ctx = gdtcontext.RegisterFixture( ctx, "kind-one-cp-one-worker", kindfix.New( @@ -54,6 +68,8 @@ func TestOneControlPlaneOneWorker(t *testing.T) { ) err = s.Run(ctx, t) + w.Flush() + fmt.Println(b.String()) require.Nil(err) } diff --git a/go.mod b/go.mod index 104af8d..38d2b71 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/cenkalti/backoff/v4 v4.2.1 - github.com/gdt-dev/gdt v1.5.0 + github.com/gdt-dev/gdt v1.6.2 github.com/samber/lo v1.38.1 github.com/stretchr/testify v1.8.4 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index ce6ea22..803368e 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/gdt-dev/gdt v1.5.0 h1:HnLbiU8zXKK73SCGgm0R68dMvfry/J+pACHDODty48Y= -github.com/gdt-dev/gdt v1.5.0/go.mod h1:qkAfKZpEIYy4ymXcDvcZpfxgVvRDQTpSqeU/ze/EobU= +github.com/gdt-dev/gdt v1.6.2 h1:ZHugSKIpMdO8hLQ1pGMPs0xGrLwQIutliDFfTjCkdys= +github.com/gdt-dev/gdt v1.6.2/go.mod h1:qkAfKZpEIYy4ymXcDvcZpfxgVvRDQTpSqeU/ze/EobU= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= diff --git a/placement_test.go b/placement_test.go new file mode 100644 index 0000000..977d4e3 --- /dev/null +++ b/placement_test.go @@ -0,0 +1,51 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. + +package kube_test + +import ( + "bufio" + "bytes" + "fmt" + "path/filepath" + "testing" + + "github.com/gdt-dev/gdt" + gdtcontext "github.com/gdt-dev/gdt/context" + "github.com/stretchr/testify/require" + + kindfix "github.com/gdt-dev/kube/fixtures/kind" + "github.com/gdt-dev/kube/testutil" +) + +func TestPlacementSpread(t *testing.T) { + testutil.SkipIfNoKind(t) + require := require.New(t) + + fp := filepath.Join("testdata", "placement-spread.yaml") + + s, err := gdt.From(fp) + require.Nil(err) + require.NotNil(s) + + kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml") + + var b bytes.Buffer + w := bufio.NewWriter(&b) + ctx := gdtcontext.New(gdtcontext.WithDebug(w)) + + ctx = gdtcontext.RegisterFixture( + ctx, "kind-three-workers-three-zones", + kindfix.New( + kindfix.WithClusterName("kind-three-workers-three-zones"), + kindfix.WithConfigPath(kindCfgPath), + ), + ) + + err = s.Run(ctx, t) + require.Nil(err) + + w.Flush() + fmt.Println(b.String()) +} diff --git a/testutil/kind.go b/testutil/kind.go new file mode 100644 index 0000000..db488ff --- /dev/null +++ b/testutil/kind.go @@ -0,0 +1,13 @@ +package testutil + +import ( + "os" + "testing" +) + +func SkipIfNoKind(t *testing.T) { + _, found := os.LookupEnv("SKIP_KIND") + if found { + t.Skipf("skipping KinD-requiring test") + } +}