kubernetes-sigs · IrvingMg · Oct 17, 2024
diff --git a/cmd/experimental/kjobctl/Makefile b/cmd/experimental/kjobctl/Makefile
@@ -40,6 +40,12 @@ INTEGRATION_TARGET ?= $(PROJECT_DIR)/test/integration/...
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+# For local testing, we should allow user to use different kind cluster name
+# Default will delete default kind cluster
+KIND_CLUSTER_NAME ?= kind
+E2E_KIND_VERSION ?= kindest/node:v1.31.0
+K8S_VERSION = $(E2E_KIND_VERSION:kindest/node:v%=%)
+
 ##@ General
 
 # The help target prints out all targets with their descriptions organized
@@ -131,6 +137,12 @@ test-integration: gomod-download envtest ginkgo embeded-manifest ray-operator-cr
 	ENVTEST_K8S_VERSION=$(ENVTEST_K8S_VERSION) \
 	$(GINKGO) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --race --junit-report=junit-integration.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET)
 
+CREATE_KIND_CLUSTER ?= true
+.PHONY: test-e2e
+test-e2e: kind
+	@echo Running e2e for k8s ${K8S_VERSION}
+	E2E_KIND_VERSION=$(E2E_KIND_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" GINKGO_ARGS="$(GINKGO_ARGS)" ./hack/e2e-test.sh
+
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter
 	$(GOLANGCI_LINT) run
@@ -185,6 +197,7 @@ ENVTEST_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}
 GOLANGCI_LINT_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' github.com/golangci/golangci-lint)
 GOTESTSUM_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' gotest.tools/gotestsum)
 GINKGO_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' github.com/onsi/ginkgo/v2)
+KIND_VERSION ?= $(shell cd $(TOOLS_DIR); $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/kind)
 
 ## Tool Binaries
 KUBECTL ?= kubectl
@@ -223,6 +236,11 @@ gotestsum: ## Download gotestsum locally if necessary.
 ginkgo: ## Download ginkgo locally if necessary.
 	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo,$(GINKGO_VERSION))
 
+KIND = $(PROJECT_DIR)/bin/kind
+.PHONY: kind
+kind: ## Download kind locally if necessary.
+	@GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/kind@$(KIND_VERSION)
+
 ##@ External CRDs
 
 RAY_ROOT = $(shell $(GO_CMD) list -m -mod=readonly -f "{{.Dir}}" github.com/ray-project/kuberay/ray-operator)

diff --git a/cmd/experimental/kjobctl/go.mod b/cmd/experimental/kjobctl/go.mod
@@ -3,12 +3,14 @@ module sigs.k8s.io/kueue/cmd/experimental/kjobctl
 go 1.23
 
 require (
+	github.com/go-logr/logr v1.4.2
 	github.com/google/go-cmp v0.6.0
 	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/onsi/gomega v1.34.2
 	github.com/ray-project/kuberay/ray-operator v1.2.2
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
+	go.uber.org/zap v1.27.0
 	k8s.io/api v0.31.1
 	k8s.io/apimachinery v0.31.1
 	k8s.io/cli-runtime v0.31.1
@@ -36,7 +38,7 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
@@ -77,6 +79,7 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect

diff --git a/cmd/experimental/kjobctl/hack/e2e-test.sh b/cmd/experimental/kjobctl/hack/e2e-test.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SOURCE_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+ROOT_DIR="$SOURCE_DIR/.."
+
+export GINKGO="$ROOT_DIR"/bin/ginkgo
+export KIND="$ROOT_DIR"/bin/kind
+
+# $1 cluster name
+function cluster_create {
+    $KIND create cluster --name "$1" --image "$E2E_KIND_VERSION" --wait 1m -v 5  > "$ARTIFACTS/$1-create.log" 2>&1 \
+		||  { echo "unable to start the $1 cluster "; cat "$ARTIFACTS/$1-create.log" ; }
+	kubectl config use-context "kind-$1"
+    kubectl get nodes > "$ARTIFACTS/$1-nodes.log" || true
+    kubectl describe pods -n kube-system > "$ARTIFACTS/$1-system-pods.log" || true
+}
+
+# $1 - cluster name
+function cluster_cleanup {
+	kubectl config use-context "kind-$1"
+    $KIND export logs "$ARTIFACTS" --name "$1" || true
+    kubectl describe pods -n kueue-system > "$ARTIFACTS/$1-kueue-system-pods.log" || true
+    kubectl describe pods > "$ARTIFACTS/$1-default-pods.log" || true
+    $KIND delete cluster --name "$1"
+}
+
+function startup {
+    if [ "$CREATE_KIND_CLUSTER" == 'true' ]
+    then
+        if [ ! -d "$ARTIFACTS" ]; then
+            mkdir -p "$ARTIFACTS"
+        fi
+	cluster_create "$KIND_CLUSTER_NAME"
+    fi
+}
+
+function cleanup {
+    if [ "$CREATE_KIND_CLUSTER" == 'true' ]
+    then
+        if [ ! -d "$ARTIFACTS" ]; then
+            mkdir -p "$ARTIFACTS"
+        fi
+    uninstall_kjobctl
+	cluster_cleanup "$KIND_CLUSTER_NAME"
+    fi
+}
+
+function install_kjobctl {
+    cd $SOURCE_DIR/.. && make install
+}
+
+function uninstall_kjobctl {
+    cd $SOURCE_DIR/.. && make uninstall
+}
+
+trap cleanup EXIT
+startup
+install_kjobctl
+# shellcheck disable=SC2086
+$GINKGO $GINKGO_ARGS --junit-report=junit.xml --json-report=e2e.json --output-dir="$ARTIFACTS" -v ./test/e2e/...
diff --git a/cmd/experimental/kjobctl/hack/tools/go.mod b/cmd/experimental/kjobctl/hack/tools/go.mod
@@ -9,6 +9,7 @@ require (
 	k8s.io/code-generator v0.31.1
 	sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240813183042-b901db121e1f
 	sigs.k8s.io/controller-tools v0.16.4
+	sigs.k8s.io/kind v0.24.0
 	sigs.k8s.io/kustomize/kustomize/v5 v5.5.0
 )
 

diff --git a/cmd/experimental/kjobctl/hack/tools/go.sum b/cmd/experimental/kjobctl/hack/tools/go.sum
@@ -275,6 +275,8 @@ sigs.k8s.io/controller-tools v0.16.4 h1:VXmar78eDXbx1by/H09ikEq1hiq3bqInxuV3lMr3
 sigs.k8s.io/controller-tools v0.16.4/go.mod h1:kcsZyYMXiOFuBhofSPtkB90zTSxVRxVVyvtKQcx3q1A=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/kind v0.24.0 h1:g4y4eu0qa+SCeKESLpESgMmVFBebL0BDa6f777OIWrg=
+sigs.k8s.io/kind v0.24.0/go.mod h1:t7ueEpzPYJvHA8aeLtI52rtFftNgUYUaCwvxjk7phfw=
 sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo=
 sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U=
 sigs.k8s.io/kustomize/kustomize/v5 v5.5.0 h1:o1mtt6vpxsxDYaZKrw3BnEtc+pAjLz7UffnIvHNbvW0=

diff --git a/cmd/experimental/kjobctl/hack/tools/pinversion.go b/cmd/experimental/kjobctl/hack/tools/pinversion.go
@@ -24,9 +24,11 @@ import (
 	_ "gotest.tools/gotestsum/cmd"
 	_ "k8s.io/code-generator"
 	_ "sigs.k8s.io/controller-runtime/tools/setup-envtest/env"
+
 	// since verify will error when referencing a cmd package
 	// we need to reference individual dependencies used by it
 	_ "sigs.k8s.io/controller-tools/pkg/crd"
 	_ "sigs.k8s.io/controller-tools/pkg/genall/help/pretty"
+	_ "sigs.k8s.io/kind/pkg/cmd"
 	_ "sigs.k8s.io/kustomize/kustomize/v5/commands/edit/listbuiltin"
 )
diff --git a/cmd/experimental/kjobctl/test/e2e/e2e_test.go b/cmd/experimental/kjobctl/test/e2e/e2e_test.go
@@ -0,0 +1,53 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"os/exec"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util"
+)
+
+var _ = ginkgo.Describe("kjobctl", ginkgo.Ordered, func() {
+	var ns *corev1.Namespace
+
+	ginkgo.BeforeEach(func() {
+		ns = &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "e2e-",
+			},
+		}
+		gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed())
+	})
+	ginkgo.AfterEach(func() {
+		gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed())
+	})
+
+	ginkgo.It("Should print kjobctl information", func() {
+		cmd := exec.Command(kjobctlPath, "--help")
+		out, err := util.Run(cmd)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		info := util.GetNonEmptyLines(string(out))
+		gomega.Expect(info[0]).To(gomega.Equal("ML/AI/Batch Jobs Made Easy"))
+	})
+})
diff --git a/cmd/experimental/kjobctl/test/e2e/suite_test.go b/cmd/experimental/kjobctl/test/e2e/suite_test.go
@@ -0,0 +1,51 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util"
+)
+
+var (
+	kjobctlPath string
+	k8sClient   client.Client
+	ctx         context.Context
+)
+
+// Run e2e tests using the Ginkgo runner.
+func TestE2E(t *testing.T) {
+	suiteName := "End To End Suite"
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, suiteName)
+}
+
+var _ = ginkgo.BeforeSuite(func() {
+	dir, _ := util.GetProjectDir()
+	kjobctlPath = filepath.Join(dir, "bin", "kubectl-kjob")
+	k8sClient, _ = util.CreateClientUsingCluster("")
+	ctx = context.Background()
+
+	util.IsKjobInstalled()
+})