Skip to content

Commit

Permalink
allow KinD clusters to be deleted or retained
Browse files Browse the repository at this point in the history
Modifies the KinD Fixture to make it more stable and more flexible.

The default behaviour is for the KinD Fixture to delete the KinD cluster
only if the KinD cluster existed before the Fixture is started.
Similarly, if the KinD Cluster did *not* exist before the Fixture is
started, the Fixture deletes the KinD Cluster when it is stopped.

This commit adds a WithDeleteOnStop() and WithRetainOnStop() modifier to
the KinD Fixture creation allowing users to indicate if they want to
change that default behaviour.

In addition to the above enhancements, this commit makes our testing
more stable by adding some KinD cluster cleanup to our Makefile as well
as serializing execution of longer-running, resource-intensive tests.

Issue #13

Signed-off-by: Jay Pipes <[email protected]>
  • Loading branch information
jaypipes committed Jun 17, 2024
1 parent 5e64948 commit e85ca3e
Show file tree
Hide file tree
Showing 9 changed files with 231 additions and 70 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/gate-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
- name: run non-Kind tests
env:
SKIP_KIND: 1
run: go test -v ./...
run: make test
test-all:
strategy:
matrix:
Expand Down Expand Up @@ -69,4 +69,4 @@ jobs:
with:
cluster_name: kind
- name: run all tests
run: go test -v ./...
run: make test-all
38 changes: 36 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,10 +1,44 @@
VERSION ?= $(shell git describe --tags --always --dirty)
CLUSTERS = $(shell kind get clusters)

.PHONY: test

kind-install:
ifeq (, $(shell command -v kind))
go install sigs.k8s.io/[email protected]
endif

# We clear the test cache because some of the tests require an out-of-band KinD
# cluster to run against and we want to re-run tests against that KinD cluster
# instead of from cached unit test results.
test:
clear-test-cache:
@echo -n "clearing Go test cache ... "
@go clean -testcache
@go test -v ./...
@echo "ok."

kind-clear-clusters:
ifneq (, $(shell command -v kind))
ifneq (, $(CLUSTERS))
@echo -n "clearing KinD clusters ... "
@for c in $(CLUSTERS); do kind delete cluster -q --name $$c; done
@echo "ok."
endif
endif

kind-create-cluster:
ifneq (, $(shell command -v kind))
@echo -n "creating 'kind' cluster ... "
@kind create cluster -q
@echo "ok."
@sleep 5
endif

test: clear-test-cache kind-clear-clusters kind-create-cluster test-kind-simple

test-kind-simple:
@go test -v ./parse_test.go
@go test -v ./eval_test.go

test-all: test kind-clear-clusters
@go test -v ./fixtures/kind/kind_test.go
@go test -v ./placement_test.go
70 changes: 15 additions & 55 deletions eval_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,19 @@
package kube_test

import (
"bufio"
"bytes"
"fmt"
"os"
"path/filepath"
"testing"

"github.com/gdt-dev/gdt"
gdtcontext "github.com/gdt-dev/gdt/context"
kindfix "github.com/gdt-dev/kube/fixtures/kind"
"github.com/stretchr/testify/require"

kindfix "github.com/gdt-dev/kube/fixtures/kind"
"github.com/gdt-dev/kube/testutil"
)

func TestListPodsEmpty(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "list-pods-empty.yaml")
Expand All @@ -36,7 +34,7 @@ func TestListPodsEmpty(t *testing.T) {
}

func TestGetPodNotFound(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "get-pod-not-found.yaml")
Expand All @@ -53,7 +51,7 @@ func TestGetPodNotFound(t *testing.T) {
}

func TestCreateUnknownResource(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "create-unknown-resource.yaml")
Expand All @@ -70,7 +68,7 @@ func TestCreateUnknownResource(t *testing.T) {
}

func TestDeleteResourceNotFound(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "delete-resource-not-found.yaml")
Expand All @@ -87,7 +85,7 @@ func TestDeleteResourceNotFound(t *testing.T) {
}

func TestDeleteUnknownResource(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "delete-unknown-resource.yaml")
Expand All @@ -104,7 +102,7 @@ func TestDeleteUnknownResource(t *testing.T) {
}

func TestPodCreateGetDelete(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "create-get-delete-pod.yaml")
Expand All @@ -121,7 +119,7 @@ func TestPodCreateGetDelete(t *testing.T) {
}

func TestMatches(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "matches.yaml")
Expand All @@ -138,7 +136,7 @@ func TestMatches(t *testing.T) {
}

func TestConditions(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "conditions.yaml")
Expand All @@ -155,7 +153,7 @@ func TestConditions(t *testing.T) {
}

func TestJSON(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "json.yaml")
Expand All @@ -172,7 +170,7 @@ func TestJSON(t *testing.T) {
}

func TestApply(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "apply-deployment.yaml")
Expand All @@ -189,7 +187,7 @@ func TestApply(t *testing.T) {
}

func TestEnvvarSubstitution(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

t.Setenv("pod_name", "foo")
Expand All @@ -208,7 +206,7 @@ func TestEnvvarSubstitution(t *testing.T) {
}

func TestWithLabels(t *testing.T) {
skipIfNoKind(t)
testutil.SkipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "list-pods-with-labels.yaml")
Expand All @@ -223,41 +221,3 @@ func TestWithLabels(t *testing.T) {
err = s.Run(ctx, t)
require.Nil(err)
}

func TestPlacementSpread(t *testing.T) {
skipIfNoKind(t)
require := require.New(t)

fp := filepath.Join("testdata", "placement-spread.yaml")

s, err := gdt.From(fp)
require.Nil(err)
require.NotNil(s)

kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml")

var b bytes.Buffer
w := bufio.NewWriter(&b)
ctx := gdtcontext.New(gdtcontext.WithDebug(w))

ctx = gdtcontext.RegisterFixture(
ctx, "kind-three-workers-three-zones",
kindfix.New(
kindfix.WithClusterName("kind-three-workers-three-zones"),
kindfix.WithConfigPath(kindCfgPath),
),
)

err = s.Run(ctx, t)
require.Nil(err)

w.Flush()
fmt.Println(b.String())
}

func skipIfNoKind(t *testing.T) {
_, found := os.LookupEnv("SKIP_KIND")
if found {
t.Skipf("skipping KinD-requiring test")
}
}
97 changes: 92 additions & 5 deletions fixtures/kind/kind.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@
package kind

import (
"context"
"strings"

gdtcontext "github.com/gdt-dev/gdt/context"
"github.com/gdt-dev/gdt/debug"
gdttypes "github.com/gdt-dev/gdt/types"
"github.com/samber/lo"
"sigs.k8s.io/kind/pkg/cluster"
Expand All @@ -20,9 +23,29 @@ import (
type KindFixture struct {
// provider is the KinD cluster provider
provider *cluster.Provider
// cfgStr contains the stringified KUBECONFIG that KinD returns in its
// Provider.KubeConfig() call
cfgStr string
// deleteOnStop indicates that the KinD cluster should be deleted when
// the fixture is stopped. Fixtures are stopped when test scenarios
// utilizing the fixture have executed all their test steps.
//
// By default, KinD clusters that were already running when the fixture was
// started are not deleted. This is to prevent the deletion of KinD
// clusters that were in use outside of a gdt-kube execution. To override
// this behaviour and always delete the KinD cluster on stop, use the
// WithDeleteOnStop() modifier.
deleteOnStop bool
// retainOnStop indicates that the KinD cluster should *not* be deleted
// when the fixture is stopped. Fixtures are stopped when test scenarios
// utilizing the fixture have executed all their test steps.
//
// By default, KinD clusters that were *not* already running when the fixture was
// started are deleted when the fixture stops. This is to clean up KinD
// clusters that were created and used by the gdt-kube execution. To override
// this behaviour and always retain the KinD cluster on stop, use the
// WithRetainOnStop() modifier.
retainOnStop bool
// runningBeforeStart is true when the KinD cluster was already running
// when the fixture was started.
runningBeforeStart bool
// ClusterName is the name of the KinD cluster. If not specified, gdt will
// use the default cluster name that KinD uses, which is just "kind"
ClusterName string
Expand All @@ -34,20 +57,35 @@ type KindFixture struct {
ConfigPath string
}

func (f *KindFixture) Start() {
func (f *KindFixture) Start(ctx context.Context) {
ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.start")
defer func() {
ctx = gdtcontext.PopTrace(ctx)
}()
if f.ClusterName == "" {
f.ClusterName = kindconst.DefaultClusterName
}
if f.isRunning() {
debug.Println(ctx, "cluster %s already running", f.ClusterName)
f.runningBeforeStart = true
return
}
opts := []cluster.CreateOption{}
if f.ConfigPath != "" {
debug.Println(
ctx, "using custom kind config %s for cluster %s",
f.ConfigPath, f.ClusterName,
)
opts = append(opts, cluster.CreateWithConfigFile(f.ConfigPath))
}
if err := f.provider.Create(f.ClusterName, opts...); err != nil {
panic(err)
}
debug.Println(ctx, "cluster %s successfully created", f.ClusterName)
if !f.retainOnStop {
f.deleteOnStop = true
debug.Println(ctx, "cluster %s will be deleted on stop", f.ClusterName)
}
}

func (f *KindFixture) isRunning() bool {
Expand All @@ -61,7 +99,26 @@ func (f *KindFixture) isRunning() bool {
return lo.Contains(clusterNames, f.ClusterName)
}

func (f *KindFixture) Stop() {}
func (f *KindFixture) Stop(ctx context.Context) {
ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.stop")
defer func() {
ctx = gdtcontext.PopTrace(ctx)
}()
if !f.isRunning() {
debug.Println(ctx, "cluster %s not running", f.ClusterName)
return
}
if f.runningBeforeStart && !f.deleteOnStop {
debug.Println(ctx, "cluster %s was running before start and deleteOnStop=false so not deleting", f.ClusterName)
return
}
if f.deleteOnStop {
if err := f.provider.Delete(f.ClusterName, ""); err != nil {
panic(err)
}
debug.Println(ctx, "cluster %s successfully deleted", f.ClusterName)
}
}

func (f *KindFixture) HasState(key string) bool {
lkey := strings.ToLower(key)
Expand Down Expand Up @@ -119,6 +176,36 @@ func WithConfigPath(path string) KindFixtureModifier {
}
}

// WithDeleteOnStop instructs gdt-kube to always delete the KinD cluster when
// the fixture stops. Fixtures are stopped when test scenarios utilizing the
// fixture have executed all their test steps.
//
// By default, KinD clusters that were already running when the fixture was
// started are not deleted. This is to prevent the deletion of KinD
// clusters that were in use outside of a gdt-kube execution. To override
// this behaviour and always delete the KinD cluster on stop, use the
// WithDeleteOnStop() modifier.
func WithDeleteOnStop() KindFixtureModifier {
return func(f *KindFixture) {
f.deleteOnStop = true
}
}

// WithRetainOnStop instructs gdt-kube that the KinD cluster should *not* be
// deleted when the fixture is stopped. Fixtures are stopped when test
// scenarios utilizing the fixture have executed all their test steps.
//
// By default, KinD clusters that were *not* already running when the fixture
// was started are deleted when the fixture stops. This is to clean up KinD
// clusters that were created and used by the gdt-kube execution. To override
// this behaviour and always retain the KinD cluster on stop, use the
// WithRetainOnStop() modifier.
func WithRetainOnStop() KindFixtureModifier {
return func(f *KindFixture) {
f.retainOnStop = true
}
}

// New returns a fixture that exposes Kubernetes configuration/context
// information about a KinD cluster. If no such KinD cluster exists, one will
// be created. If the KinD cluster is created, it is destroyed at the end of
Expand Down
Loading

0 comments on commit e85ca3e

Please sign in to comment.