allow KinD clusters to be deleted or retained

.github/workflows/gate-tests.yml

@@ -38,7 +38,7 @@ jobs:
      - name: run non-Kind tests
        env:
          SKIP_KIND: 1
-       run: go test -v ./...
+       run: make test
   test-all:
     strategy:
       matrix:
@@ -69,4 +69,4 @@ jobs:
        with:
          cluster_name: kind
      - name: run all tests
-       run: go test -v ./...
+       run: make test-all

Makefile

@@ -1,10 +1,44 @@
 VERSION ?= $(shell git describe --tags --always --dirty)
+CLUSTERS = $(shell kind get clusters)
 
 .PHONY: test
 
+kind-install:
+ifeq (, $(shell command -v kind))
+	go install sigs.k8s.io/[email protected]
+endif
+
 # We clear the test cache because some of the tests require an out-of-band KinD
 # cluster to run against and we want to re-run tests against that KinD cluster
 # instead of from cached unit test results.
-test:
+clear-test-cache:
+	@echo -n "clearing Go test cache ... "
 	@go clean -testcache
-	@go test -v ./...
+	@echo "ok."
+
+kind-clear-clusters:
+ifneq (, $(shell command -v kind))
+ifneq (, $(CLUSTERS))
+	@echo -n "clearing KinD clusters ... "
+	@for c in $(CLUSTERS); do kind delete cluster -q --name $$c; done
+	@echo "ok."
+endif
+endif
+
+kind-create-cluster:
+ifneq (, $(shell command -v kind))
+	@echo -n "creating 'kind' cluster ... "
+	@kind create cluster -q
+	@echo "ok."
+	@sleep 5
+endif
+
+test: clear-test-cache kind-clear-clusters kind-create-cluster test-kind-simple
+
+test-kind-simple:
+	@go test -v ./parse_test.go
+	@go test -v ./eval_test.go
+
+test-all: test kind-clear-clusters
+	@go test -v ./fixtures/kind/kind_test.go
+	@go test -v ./placement_test.go

eval_test.go

@@ -5,21 +5,19 @@
 package kube_test
 
 import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/gdt-dev/gdt"
 	gdtcontext "github.com/gdt-dev/gdt/context"
-	kindfix "github.com/gdt-dev/kube/fixtures/kind"
 	"github.com/stretchr/testify/require"
+
+	kindfix "github.com/gdt-dev/kube/fixtures/kind"
+	"github.com/gdt-dev/kube/testutil"
 )
 
 func TestListPodsEmpty(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "list-pods-empty.yaml")
@@ -36,7 +34,7 @@ func TestListPodsEmpty(t *testing.T) {
 }
 
 func TestGetPodNotFound(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "get-pod-not-found.yaml")
@@ -53,7 +51,7 @@ func TestGetPodNotFound(t *testing.T) {
 }
 
 func TestCreateUnknownResource(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "create-unknown-resource.yaml")
@@ -70,7 +68,7 @@ func TestCreateUnknownResource(t *testing.T) {
 }
 
 func TestDeleteResourceNotFound(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "delete-resource-not-found.yaml")
@@ -87,7 +85,7 @@ func TestDeleteResourceNotFound(t *testing.T) {
 }
 
 func TestDeleteUnknownResource(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "delete-unknown-resource.yaml")
@@ -104,7 +102,7 @@ func TestDeleteUnknownResource(t *testing.T) {
 }
 
 func TestPodCreateGetDelete(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "create-get-delete-pod.yaml")
@@ -121,7 +119,7 @@ func TestPodCreateGetDelete(t *testing.T) {
 }
 
 func TestMatches(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "matches.yaml")
@@ -138,7 +136,7 @@ func TestMatches(t *testing.T) {
 }
 
 func TestConditions(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "conditions.yaml")
@@ -155,7 +153,7 @@ func TestConditions(t *testing.T) {
 }
 
 func TestJSON(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "json.yaml")
@@ -172,7 +170,7 @@ func TestJSON(t *testing.T) {
 }
 
 func TestApply(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "apply-deployment.yaml")
@@ -189,7 +187,7 @@ func TestApply(t *testing.T) {
 }
 
 func TestEnvvarSubstitution(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	t.Setenv("pod_name", "foo")
@@ -208,7 +206,7 @@ func TestEnvvarSubstitution(t *testing.T) {
 }
 
 func TestWithLabels(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "list-pods-with-labels.yaml")
@@ -223,41 +221,3 @@ func TestWithLabels(t *testing.T) {
 	err = s.Run(ctx, t)
 	require.Nil(err)
 }
-
-func TestPlacementSpread(t *testing.T) {
-	skipIfNoKind(t)
-	require := require.New(t)
-
-	fp := filepath.Join("testdata", "placement-spread.yaml")
-
-	s, err := gdt.From(fp)
-	require.Nil(err)
-	require.NotNil(s)
-
-	kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml")
-
-	var b bytes.Buffer
-	w := bufio.NewWriter(&b)
-	ctx := gdtcontext.New(gdtcontext.WithDebug(w))
-
-	ctx = gdtcontext.RegisterFixture(
-		ctx, "kind-three-workers-three-zones",
-		kindfix.New(
-			kindfix.WithClusterName("kind-three-workers-three-zones"),
-			kindfix.WithConfigPath(kindCfgPath),
-		),
-	)
-
-	err = s.Run(ctx, t)
-	require.Nil(err)
-
-	w.Flush()
-	fmt.Println(b.String())
-}
-
-func skipIfNoKind(t *testing.T) {
-	_, found := os.LookupEnv("SKIP_KIND")
-	if found {
-		t.Skipf("skipping KinD-requiring test")
-	}
-}

fixtures/kind/kind.go

@@ -5,8 +5,11 @@
 package kind
 
 import (
+	"context"
 	"strings"
 
+	gdtcontext "github.com/gdt-dev/gdt/context"
+	"github.com/gdt-dev/gdt/debug"
 	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/samber/lo"
 	"sigs.k8s.io/kind/pkg/cluster"
@@ -20,9 +23,29 @@ import (
 type KindFixture struct {
 	// provider is the KinD cluster provider
 	provider *cluster.Provider
-	// cfgStr contains the stringified KUBECONFIG that KinD returns in its
-	// Provider.KubeConfig() call
-	cfgStr string
+	// deleteOnStop indicates that the KinD cluster should be deleted when
+	// the fixture is stopped. Fixtures are stopped when test scenarios
+	// utilizing the fixture have executed all their test steps.
+	//
+	// By default, KinD clusters that were already running when the fixture was
+	// started are not deleted. This is to prevent the deletion of KinD
+	// clusters that were in use outside of a gdt-kube execution. To override
+	// this behaviour and always delete the KinD cluster on stop, use the
+	// WithDeleteOnStop() modifier.
+	deleteOnStop bool
+	// retainOnStop indicates that the KinD cluster should *not* be deleted
+	// when the fixture is stopped. Fixtures are stopped when test scenarios
+	// utilizing the fixture have executed all their test steps.
+	//
+	// By default, KinD clusters that were *not* already running when the fixture was
+	// started are deleted when the fixture stops. This is to clean up KinD
+	// clusters that were created and used by the gdt-kube execution. To override
+	// this behaviour and always retain the KinD cluster on stop, use the
+	// WithRetainOnStop() modifier.
+	retainOnStop bool
+	// runningBeforeStart is true when the KinD cluster was already running
+	// when the fixture was started.
+	runningBeforeStart bool
 	// ClusterName is the name of the KinD cluster. If not specified, gdt will
 	// use the default cluster name that KinD uses, which is just "kind"
 	ClusterName string
@@ -34,20 +57,35 @@ type KindFixture struct {
 	ConfigPath string
 }
 
-func (f *KindFixture) Start() {
+func (f *KindFixture) Start(ctx context.Context) {
+	ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.start")
+	defer func() {
+		ctx = gdtcontext.PopTrace(ctx)
+	}()
 	if f.ClusterName == "" {
 		f.ClusterName = kindconst.DefaultClusterName
 	}
 	if f.isRunning() {
+		debug.Println(ctx, "cluster %s already running", f.ClusterName)
+		f.runningBeforeStart = true
 		return
 	}
 	opts := []cluster.CreateOption{}
 	if f.ConfigPath != "" {
+		debug.Println(
+			ctx, "using custom kind config %s for cluster %s",
+			f.ConfigPath, f.ClusterName,
+		)
 		opts = append(opts, cluster.CreateWithConfigFile(f.ConfigPath))
 	}
 	if err := f.provider.Create(f.ClusterName, opts...); err != nil {
 		panic(err)
 	}
+	debug.Println(ctx, "cluster %s successfully created", f.ClusterName)
+	if !f.retainOnStop {
+		f.deleteOnStop = true
+		debug.Println(ctx, "cluster %s will be deleted on stop", f.ClusterName)
+	}
 }
 
 func (f *KindFixture) isRunning() bool {
@@ -61,7 +99,26 @@ func (f *KindFixture) isRunning() bool {
 	return lo.Contains(clusterNames, f.ClusterName)
 }
 
-func (f *KindFixture) Stop() {}
+func (f *KindFixture) Stop(ctx context.Context) {
+	ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.stop")
+	defer func() {
+		ctx = gdtcontext.PopTrace(ctx)
+	}()
+	if !f.isRunning() {
+		debug.Println(ctx, "cluster %s not running", f.ClusterName)
+		return
+	}
+	if f.runningBeforeStart && !f.deleteOnStop {
+		debug.Println(ctx, "cluster %s was running before start and deleteOnStop=false so not deleting", f.ClusterName)
+		return
+	}
+	if f.deleteOnStop {
+		if err := f.provider.Delete(f.ClusterName, ""); err != nil {
+			panic(err)
+		}
+		debug.Println(ctx, "cluster %s successfully deleted", f.ClusterName)
+	}
+}
 
 func (f *KindFixture) HasState(key string) bool {
 	lkey := strings.ToLower(key)
@@ -119,6 +176,36 @@ func WithConfigPath(path string) KindFixtureModifier {
 	}
 }
 
+// WithDeleteOnStop instructs gdt-kube to always delete the KinD cluster when
+// the fixture stops. Fixtures are stopped when test scenarios utilizing the
+// fixture have executed all their test steps.
+//
+// By default, KinD clusters that were already running when the fixture was
+// started are not deleted. This is to prevent the deletion of KinD
+// clusters that were in use outside of a gdt-kube execution. To override
+// this behaviour and always delete the KinD cluster on stop, use the
+// WithDeleteOnStop() modifier.
+func WithDeleteOnStop() KindFixtureModifier {
+	return func(f *KindFixture) {
+		f.deleteOnStop = true
+	}
+}
+
+// WithRetainOnStop instructs gdt-kube that the KinD cluster should *not* be
+// deleted when the fixture is stopped. Fixtures are stopped when test
+// scenarios utilizing the fixture have executed all their test steps.
+//
+// By default, KinD clusters that were *not* already running when the fixture
+// was started are deleted when the fixture stops. This is to clean up KinD
+// clusters that were created and used by the gdt-kube execution. To override
+// this behaviour and always retain the KinD cluster on stop, use the
+// WithRetainOnStop() modifier.
+func WithRetainOnStop() KindFixtureModifier {
+	return func(f *KindFixture) {
+		f.retainOnStop = true
+	}
+}
+
 // New returns a fixture that exposes Kubernetes configuration/context
 // information about a KinD cluster. If no such KinD cluster exists, one will
 // be created. If the KinD cluster is created, it is destroyed at the end of