Skip to content

Commit

Permalink
chore: reduce startup time and add skaffold (#5970)
Browse files Browse the repository at this point in the history
* chore: add skaffold

* chore: add NATS retry

* chore: add skaffold
  • Loading branch information
WitoDelnat authored Oct 25, 2024
1 parent c08b9f8 commit 38733d5
Show file tree
Hide file tree
Showing 12 changed files with 320 additions and 3 deletions.
5 changes: 3 additions & 2 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
.git
.gitignore
.github
LICENSE
README.md
Makefile
build
docs
assets
config
choco
test
test
skaffold.yaml
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,5 @@ movie.mp4
/dist
.vscode


build/_local/values.dev.yaml
46 changes: 46 additions & 0 deletions build/_local/agent-server.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
###################################
## Build
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

RUN apk --no-cache --update add ca-certificates && (rm -rf /var/cache/apk/* || 0)

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/agent-server cmd/api-server/main.go

###################################
## Debug
###################################
FROM golang:1.23-alpine AS debug

ENV GOTRACEBACK=all
RUN go install github.com/go-delve/delve/cmd/[email protected]

COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=builder /app/build/_local/agent-server /testkube/

ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/agent-server"]

###################################
## Distribution
###################################
FROM scratch AS dist

COPY LICENSE /testkube/
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=builder /app/build/_local/agent-server /testkube/

EXPOSE 8080
ENTRYPOINT ["/testkube/agent-server"]
63 changes: 63 additions & 0 deletions build/_local/testworkflow-init.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
ARG BUSYBOX_IMAGE="busybox:1.36.1-musl"

###################################
## Build testworkflow init
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go

###################################
## Build testworkflow toolkit
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go

###################################
## Debug
###################################
FROM golang:1.23-alpine AS debug

ENV GOTRACEBACK=all
RUN go install github.com/go-delve/delve/cmd/[email protected]

COPY --from=builder /app/build/_local/workflow-init /testkube/

ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/workflow-init"]

###################################
## Distribution
###################################
FROM ${BUSYBOX_IMAGE} AS dist
RUN cp -rf /bin /.tktw-bin
COPY --from=builder /app/build/_local/workflow-init /testkube/init
USER 1001
ENTRYPOINT ["/init"]

71 changes: 71 additions & 0 deletions build/_local/testworkflow-toolkit.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
ARG BUSYBOX_IMAGE="busybox:1.36.1-musl"
ARG ALPINE_IMAGE="alpine:3.20.0"
FROM ${BUSYBOX_IMAGE} AS busybox

###################################
## Build testworkflow-init
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-init

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go

###################################
## Build testworkflow-toolkit
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-toolkit

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

RUN go install github.com/go-delve/delve/cmd/[email protected]

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-toolkit cmd/testworkflow-toolkit/main.go

###################################
## Debug
###################################
FROM ${ALPINE_IMAGE} AS debug
RUN apk --no-cache add ca-certificates libssl3 git openssh-client
ENV GOTRACEBACK=all
COPY --from=builder-toolkit /go/bin/dlv /
COPY --from=busybox /bin /.tktw-bin
COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit
COPY --from=builder-init /app/build/_local/workflow-init /init
RUN adduser --disabled-password --home / --no-create-home --uid 1001 default
USER 1001
ENTRYPOINT ["/dlv", "exec", "--headless", "--accept-multiclient", "--listen=:56300", "--api-version=2", "/toolkit"]

###################################
## Distribution
###################################
FROM ${ALPINE_IMAGE} AS dist
RUN apk --no-cache add ca-certificates libssl3 git openssh-client
COPY --from=busybox /bin /.tktw-bin
COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit
COPY --from=builder-init /app/build/_local/workflow-init /init
RUN adduser --disabled-password --home / --no-create-home --uid 1001 default
USER 1001
ENTRYPOINT ["/toolkit"]
12 changes: 12 additions & 0 deletions build/_local/values.dev.tpl.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
testkube-dashboard:
enabled: false
mongodb:
enabled: false
testkube-api:
cloud:
url: testkube-enterprise-api:8089
tls:
enabled: false
existingSecret:
name: testkube-default-agent-token
key: agent-token
46 changes: 46 additions & 0 deletions docker-bake.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
variable "GOCACHE" {
default = "/go/pkg"
}
variable "GOMODCACHE" {
default = "/root/.cache/go-build"
}

group "default" {
targets = ["agent-server", "testworkflow-init", "testworkflow-toolkit"]
}

target "agent-server-meta" {}
target "agent-server" {
inherits = ["agent-server-meta"]
context="."
dockerfile = "build/_local/agent-server.Dockerfile"
platforms = ["linux/arm64"]
args = {
GOCACHE = "${GOCACHE}"
GOMODCACHE = "${GOMODCACHE}"
}
}

target "testworkflow-init-meta" {}
target "testworkflow-init" {
inherits = ["testworkflow-init-meta"]
context="."
dockerfile = "build/_local/testworkflow-init.Dockerfile"
platforms = ["linux/arm64"]
args = {
GOCACHE = "${GOCACHE}"
GOMODCACHE = "${GOMODCACHE}"
}
}

target "testworkflow-toolkit-meta" {}
target "testworkflow-toolkit" {
inherits = ["testworkflow-toolkit-meta"]
context="."
dockerfile = "build/_local/testworkflow-toolkit.Dockerfile"
platforms = ["linux/arm64"]
args = {
GOCACHE = "${GOCACHE}"
GOMODCACHE = "${GOMODCACHE}"
}
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ require (
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/avast/retry-go/v4 v4.6.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down
14 changes: 13 additions & 1 deletion pkg/event/bus/nats.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,19 @@ import (
"sync"
"time"

"github.com/avast/retry-go/v4"
"github.com/nats-io/nats.go"

"github.com/kubeshop/testkube/pkg/api/v1/testkube"
"github.com/kubeshop/testkube/pkg/event/kind/common"
"github.com/kubeshop/testkube/pkg/log"
"github.com/kubeshop/testkube/pkg/utils"
)

var (
_ Bus = (*NATSBus)(nil)

NATS_RETRY_ATTEMPTS uint = 20
)

const (
Expand Down Expand Up @@ -80,7 +84,15 @@ func NewNATSEncodedConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats.
func NewNATSConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats.Conn, error) {
opts = append(opts, optsFromConfig(cfg)...)

nc, err := nats.Connect(cfg.NatsURI, opts...)
log.DefaultLogger.Infoln("Connecting to NATS")
nc, err := retry.DoWithData(
func() (*nats.Conn, error) {
return nats.Connect(cfg.NatsURI, opts...)
},
retry.DelayType(retry.FixedDelay),
retry.Delay(utils.DefaultRetryDelay),
retry.Attempts(NATS_RETRY_ATTEMPTS),
)
if err != nil {
log.DefaultLogger.Fatalw("error connecting to nats", "error", err)
return nil, err
Expand Down
3 changes: 3 additions & 0 deletions pkg/utils/consts.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package utils

import "time"

const (
// DefaultDockerRegistry is the default registry used when no registry is specified in the image name.
DefaultDockerRegistry = "https://index.docker.io/v1/"
DefaultRetryDelay = time.Second * 3
)
58 changes: 58 additions & 0 deletions skaffold.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
apiVersion: skaffold/v4beta11
kind: Config
metadata:
name: testkube-agent
build:
local:
concurrency: 1 # for testing, should be possible to run all concurrently.
artifacts:
- image: docker.io/testkube-agent-server
context: .
custom:
buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set agent-server.tags="$IMAGE" --set agent-server.target="debug" agent-server
dependencies:
dockerfile:
path: build/_local/agent-server.Dockerfile
- image: docker.io/testworkflow-init
context: .
custom:
buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-init.tags="$IMAGE" testworkflow-init
dependencies:
dockerfile:
path: build/_local/testworkflow-init.Dockerfile
- image: docker.io/testworkflow-toolkit
context: .
custom:
buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-toolkit.tags="$IMAGE" --set testworkflow-toolkit.target="debug" testworkflow-toolkit
dependencies:
dockerfile:
path: build/_local/testworkflow-toolkit.Dockerfile
deploy:
helm:
# see https://skaffold.dev/docs/renderers/helm/#skaffoldyaml-configuration
releases:
- name: testkube-agent
repo: https://kubeshop.github.io/helm-charts
remoteChart: testkube
# Alternative: Local chart - useful for when you are actively making changes to the chart.
# chartPath: /Users/you/path/to/helm-charts/charts/testkube
upgradeOnChange: true
# skipBuildDependencies: true # This implies that you need to build dependencies yourself when you make local chart changes!
namespace: tk-dev
wait: true
createNamespace: true
valuesFiles: ['build/_local/values.dev.yaml'] # IMPORTANT: You will have to copy the values.dev.tpl.yaml template to get started!
setValueTemplates:
testkube-api.image.registry: '{{.IMAGE_DOMAIN_docker_io_testkube_agent_server}}'
testkube-api.image.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testkube_agent_server}}'
testkube-api.image.tag: '{{.IMAGE_TAG_docker_io_testkube_agent_server}}@{{.IMAGE_DIGEST_docker_io_testkube_agent_server}}'
testkube-api.imageTwInit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_init}}'
testkube-api.imageTwInit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_init}}'
testkube-api.imageTwInit.tag: '{{.IMAGE_TAG_docker_io_testworkflow_init}}@{{.IMAGE_DIGEST_docker_io_testworkflow_init}}'
testkube-api.imageTwToolkit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_toolkit}}'
testkube-api.imageTwToolkit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_toolkit}}'
testkube-api.imageTwToolkit.tag: '{{.IMAGE_TAG_docker_io_testworkflow_toolkit}}@{{.IMAGE_DIGEST_docker_io_testworkflow_toolkit}}'
flags:
upgrade: ["--no-hooks"]
statusCheckDeadlineSeconds: 300
tolerateFailuresUntilDeadline: true

0 comments on commit 38733d5

Please sign in to comment.