From 38733d5aa72cdf9f80cb7a231e4c6889decbe63d Mon Sep 17 00:00:00 2001 From: Wito Date: Fri, 25 Oct 2024 13:12:13 +0200 Subject: [PATCH] chore: reduce startup time and add skaffold (#5970) * chore: add skaffold * chore: add NATS retry * chore: add skaffold --- .dockerignore | 5 +- .gitignore | 2 + build/_local/agent-server.Dockerfile | 46 +++++++++++++ build/_local/testworkflow-init.Dockerfile | 63 +++++++++++++++++ build/_local/testworkflow-toolkit.Dockerfile | 71 ++++++++++++++++++++ build/_local/values.dev.tpl.yaml | 12 ++++ docker-bake.hcl | 46 +++++++++++++ go.mod | 1 + go.sum | 2 + pkg/event/bus/nats.go | 14 +++- pkg/utils/consts.go | 3 + skaffold.yaml | 58 ++++++++++++++++ 12 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 build/_local/agent-server.Dockerfile create mode 100644 build/_local/testworkflow-init.Dockerfile create mode 100644 build/_local/testworkflow-toolkit.Dockerfile create mode 100644 build/_local/values.dev.tpl.yaml create mode 100644 docker-bake.hcl create mode 100644 skaffold.yaml diff --git a/.dockerignore b/.dockerignore index 8da3bd1184d..9aa61bd961d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,12 @@ .git .gitignore .github -LICENSE README.md Makefile +build docs assets config choco -test \ No newline at end of file +test +skaffold.yaml diff --git a/.gitignore b/.gitignore index f8c639837d7..8dddeecf2be 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,5 @@ movie.mp4 /dist .vscode + +build/_local/values.dev.yaml diff --git a/build/_local/agent-server.Dockerfile b/build/_local/agent-server.Dockerfile new file mode 100644 index 00000000000..2dbcee1ce39 --- /dev/null +++ b/build/_local/agent-server.Dockerfile @@ -0,0 +1,46 @@ +################################### +## Build +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +RUN apk --no-cache --update add ca-certificates && (rm -rf /var/cache/apk/* || 0) + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/agent-server cmd/api-server/main.go + +################################### +## Debug +################################### +FROM golang:1.23-alpine AS debug + +ENV GOTRACEBACK=all +RUN go install github.com/go-delve/delve/cmd/dlv@v1.23.1 + +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=builder /app/build/_local/agent-server /testkube/ + +ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/agent-server"] + +################################### +## Distribution +################################### +FROM scratch AS dist + +COPY LICENSE /testkube/ +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=builder /app/build/_local/agent-server /testkube/ + +EXPOSE 8080 +ENTRYPOINT ["/testkube/agent-server"] diff --git a/build/_local/testworkflow-init.Dockerfile b/build/_local/testworkflow-init.Dockerfile new file mode 100644 index 00000000000..40ccb861f92 --- /dev/null +++ b/build/_local/testworkflow-init.Dockerfile @@ -0,0 +1,63 @@ +ARG BUSYBOX_IMAGE="busybox:1.36.1-musl" + +################################### +## Build testworkflow init +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go + +################################### +## Build testworkflow toolkit +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go + +################################### +## Debug +################################### +FROM golang:1.23-alpine AS debug + +ENV GOTRACEBACK=all +RUN go install github.com/go-delve/delve/cmd/dlv@v1.23.1 + +COPY --from=builder /app/build/_local/workflow-init /testkube/ + +ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/workflow-init"] + +################################### +## Distribution +################################### +FROM ${BUSYBOX_IMAGE} AS dist +RUN cp -rf /bin /.tktw-bin +COPY --from=builder /app/build/_local/workflow-init /testkube/init +USER 1001 +ENTRYPOINT ["/init"] + diff --git a/build/_local/testworkflow-toolkit.Dockerfile b/build/_local/testworkflow-toolkit.Dockerfile new file mode 100644 index 00000000000..99821a27beb --- /dev/null +++ b/build/_local/testworkflow-toolkit.Dockerfile @@ -0,0 +1,71 @@ +ARG BUSYBOX_IMAGE="busybox:1.36.1-musl" +ARG ALPINE_IMAGE="alpine:3.20.0" +FROM ${BUSYBOX_IMAGE} AS busybox + +################################### +## Build testworkflow-init +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-init + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go + +################################### +## Build testworkflow-toolkit +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-toolkit + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +RUN go install github.com/go-delve/delve/cmd/dlv@v1.23.1 + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-toolkit cmd/testworkflow-toolkit/main.go + +################################### +## Debug +################################### +FROM ${ALPINE_IMAGE} AS debug +RUN apk --no-cache add ca-certificates libssl3 git openssh-client +ENV GOTRACEBACK=all +COPY --from=builder-toolkit /go/bin/dlv / +COPY --from=busybox /bin /.tktw-bin +COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit +COPY --from=builder-init /app/build/_local/workflow-init /init +RUN adduser --disabled-password --home / --no-create-home --uid 1001 default +USER 1001 +ENTRYPOINT ["/dlv", "exec", "--headless", "--accept-multiclient", "--listen=:56300", "--api-version=2", "/toolkit"] + +################################### +## Distribution +################################### +FROM ${ALPINE_IMAGE} AS dist +RUN apk --no-cache add ca-certificates libssl3 git openssh-client +COPY --from=busybox /bin /.tktw-bin +COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit +COPY --from=builder-init /app/build/_local/workflow-init /init +RUN adduser --disabled-password --home / --no-create-home --uid 1001 default +USER 1001 +ENTRYPOINT ["/toolkit"] diff --git a/build/_local/values.dev.tpl.yaml b/build/_local/values.dev.tpl.yaml new file mode 100644 index 00000000000..fc89193a586 --- /dev/null +++ b/build/_local/values.dev.tpl.yaml @@ -0,0 +1,12 @@ +testkube-dashboard: + enabled: false +mongodb: + enabled: false +testkube-api: + cloud: + url: testkube-enterprise-api:8089 + tls: + enabled: false + existingSecret: + name: testkube-default-agent-token + key: agent-token \ No newline at end of file diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 00000000000..d02d45661ba --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,46 @@ +variable "GOCACHE" { + default = "/go/pkg" +} +variable "GOMODCACHE" { + default = "/root/.cache/go-build" +} + +group "default" { + targets = ["agent-server", "testworkflow-init", "testworkflow-toolkit"] +} + +target "agent-server-meta" {} +target "agent-server" { + inherits = ["agent-server-meta"] + context="." + dockerfile = "build/_local/agent-server.Dockerfile" + platforms = ["linux/arm64"] + args = { + GOCACHE = "${GOCACHE}" + GOMODCACHE = "${GOMODCACHE}" + } +} + +target "testworkflow-init-meta" {} +target "testworkflow-init" { + inherits = ["testworkflow-init-meta"] + context="." + dockerfile = "build/_local/testworkflow-init.Dockerfile" + platforms = ["linux/arm64"] + args = { + GOCACHE = "${GOCACHE}" + GOMODCACHE = "${GOMODCACHE}" + } +} + +target "testworkflow-toolkit-meta" {} +target "testworkflow-toolkit" { + inherits = ["testworkflow-toolkit-meta"] + context="." + dockerfile = "build/_local/testworkflow-toolkit.Dockerfile" + platforms = ["linux/arm64"] + args = { + GOCACHE = "${GOCACHE}" + GOMODCACHE = "${GOMODCACHE}" + } +} diff --git a/go.mod b/go.mod index cfb1e75d164..6b56b60c877 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ require ( github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/brotli v1.0.5 // indirect + github.com/avast/retry-go/v4 v4.6.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect diff --git a/go.sum b/go.sum index 954706a07c0..81122094b3f 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= +github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= +github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/pkg/event/bus/nats.go b/pkg/event/bus/nats.go index 79e61d1ebff..f652882cbb2 100644 --- a/pkg/event/bus/nats.go +++ b/pkg/event/bus/nats.go @@ -6,15 +6,19 @@ import ( "sync" "time" + "github.com/avast/retry-go/v4" "github.com/nats-io/nats.go" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/event/kind/common" "github.com/kubeshop/testkube/pkg/log" + "github.com/kubeshop/testkube/pkg/utils" ) var ( _ Bus = (*NATSBus)(nil) + + NATS_RETRY_ATTEMPTS uint = 20 ) const ( @@ -80,7 +84,15 @@ func NewNATSEncodedConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats. func NewNATSConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats.Conn, error) { opts = append(opts, optsFromConfig(cfg)...) - nc, err := nats.Connect(cfg.NatsURI, opts...) + log.DefaultLogger.Infoln("Connecting to NATS") + nc, err := retry.DoWithData( + func() (*nats.Conn, error) { + return nats.Connect(cfg.NatsURI, opts...) + }, + retry.DelayType(retry.FixedDelay), + retry.Delay(utils.DefaultRetryDelay), + retry.Attempts(NATS_RETRY_ATTEMPTS), + ) if err != nil { log.DefaultLogger.Fatalw("error connecting to nats", "error", err) return nil, err diff --git a/pkg/utils/consts.go b/pkg/utils/consts.go index 043bb8500c0..890c7cfac5b 100644 --- a/pkg/utils/consts.go +++ b/pkg/utils/consts.go @@ -1,6 +1,9 @@ package utils +import "time" + const ( // DefaultDockerRegistry is the default registry used when no registry is specified in the image name. DefaultDockerRegistry = "https://index.docker.io/v1/" + DefaultRetryDelay = time.Second * 3 ) diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 00000000000..0918d56d41c --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,58 @@ +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: testkube-agent +build: + local: + concurrency: 1 # for testing, should be possible to run all concurrently. + artifacts: + - image: docker.io/testkube-agent-server + context: . + custom: + buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set agent-server.tags="$IMAGE" --set agent-server.target="debug" agent-server + dependencies: + dockerfile: + path: build/_local/agent-server.Dockerfile + - image: docker.io/testworkflow-init + context: . + custom: + buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-init.tags="$IMAGE" testworkflow-init + dependencies: + dockerfile: + path: build/_local/testworkflow-init.Dockerfile + - image: docker.io/testworkflow-toolkit + context: . + custom: + buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-toolkit.tags="$IMAGE" --set testworkflow-toolkit.target="debug" testworkflow-toolkit + dependencies: + dockerfile: + path: build/_local/testworkflow-toolkit.Dockerfile +deploy: + helm: + # see https://skaffold.dev/docs/renderers/helm/#skaffoldyaml-configuration + releases: + - name: testkube-agent + repo: https://kubeshop.github.io/helm-charts + remoteChart: testkube + # Alternative: Local chart - useful for when you are actively making changes to the chart. +# chartPath: /Users/you/path/to/helm-charts/charts/testkube + upgradeOnChange: true +# skipBuildDependencies: true # This implies that you need to build dependencies yourself when you make local chart changes! + namespace: tk-dev + wait: true + createNamespace: true + valuesFiles: ['build/_local/values.dev.yaml'] # IMPORTANT: You will have to copy the values.dev.tpl.yaml template to get started! + setValueTemplates: + testkube-api.image.registry: '{{.IMAGE_DOMAIN_docker_io_testkube_agent_server}}' + testkube-api.image.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testkube_agent_server}}' + testkube-api.image.tag: '{{.IMAGE_TAG_docker_io_testkube_agent_server}}@{{.IMAGE_DIGEST_docker_io_testkube_agent_server}}' + testkube-api.imageTwInit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_init}}' + testkube-api.imageTwInit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_init}}' + testkube-api.imageTwInit.tag: '{{.IMAGE_TAG_docker_io_testworkflow_init}}@{{.IMAGE_DIGEST_docker_io_testworkflow_init}}' + testkube-api.imageTwToolkit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_toolkit}}' + testkube-api.imageTwToolkit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_toolkit}}' + testkube-api.imageTwToolkit.tag: '{{.IMAGE_TAG_docker_io_testworkflow_toolkit}}@{{.IMAGE_DIGEST_docker_io_testworkflow_toolkit}}' + flags: + upgrade: ["--no-hooks"] + statusCheckDeadlineSeconds: 300 + tolerateFailuresUntilDeadline: true