chore: Release v2.0.7 (#5650)

* feat: [TKC-2194] improve workflow execution telemetry (#5648)

* TKC-2161: skip tls verification when saving logs if STORAGE_SKIP_VERIFY is true (#5627)

* build(deps): bump anchore/sbom-action from 0.16.0 to 0.16.1 (#5649)

Bumps [anchore/sbom-action](https://github.com/anchore/sbom-action) from 0.16.0 to 0.16.1.
- [Release notes](https://github.com/anchore/sbom-action/releases)
- [Commits](anchore/sbom-action@v0.16.0...v0.16.1)

---
updated-dependencies:
- dependency-name: anchore/sbom-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* feat(performance): improve Log Processing performance (#5647)

* feat(performance): buffer the logs sent from the container, to avoid sending message for each line
* feat(performance): batch Test Workflow's result updates
* fix(testworkflows): handle getting long container logs after the log rotation happens
   @see {@link https://stackoverflow.com/a/68673451}
* feat(testworkflows): optimize reading timestamp from Kubernetes logs
* feat(testworkflows): optimize buffering logs
* feat(testworkflows): use native channel instead of heavier Channels for WatchInstrumentedPod
* feat(testworkflows): increase buffer size for logs buffering

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Povilas Versockas <[email protected]>
Co-authored-by: Dejan Zele Pejchev <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/docker-build-api-executors-tag.yaml

@@ -19,7 +19,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up Docker Buildx
         id: buildx
@@ -94,7 +94,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up Docker Buildx
         id: buildx
@@ -178,7 +178,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -249,7 +249,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -317,7 +317,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -442,7 +442,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -510,7 +510,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -669,7 +669,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -741,7 +741,7 @@ jobs:
         uses: docker/setup-qemu-action@v3
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up Docker Buildx
         id: buildx
@@ -790,7 +790,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -863,7 +863,7 @@ jobs:
           fetch-depth: 0
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

.github/workflows/release-dev.yaml

@@ -131,7 +131,7 @@ jobs:
         with:
           fetch-depth: 0
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
       - name: Download Artifacts for Linux
         uses: actions/download-artifact@master
         with:

.github/workflows/release.yaml

@@ -121,7 +121,7 @@ jobs:
         with:
           fetch-depth: 0
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].0
+      - uses: anchore/sbom-action/[email protected].1
       - name: Download Artifacts for Linux
         uses: actions/download-artifact@master
         with:

cmd/api-server/main.go

@@ -273,7 +273,11 @@ func main() {
 		configRepository = cloudconfig.NewCloudResultRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey)
 		// Pro edition only (tcl protected code)
 		testWorkflowResultsRepository = cloudtestworkflow.NewCloudRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey)
-		testWorkflowOutputRepository = cloudtestworkflow.NewCloudOutputRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey)
+		var opts []cloudtestworkflow.Option
+		if cfg.StorageSkipVerify {
+			opts = append(opts, cloudtestworkflow.WithSkipVerify())
+		}
+		testWorkflowOutputRepository = cloudtestworkflow.NewCloudOutputRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey, opts...)
 		triggerLeaseBackend = triggers.NewAcquireAlwaysLeaseBackend()
 		artifactStorage = cloudartifacts.NewCloudArtifactsStorage(grpcClient, grpcConn, cfg.TestkubeProAPIKey)
 	} else {

pkg/cloud/data/testworkflow/output.go

@@ -3,6 +3,7 @@ package testworkflow
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"io"
 	"net/http"
 
@@ -18,11 +19,25 @@ import (
 var _ testworkflow.OutputRepository = (*CloudOutputRepository)(nil)
 
 type CloudOutputRepository struct {
-	executor executor.Executor
+	executor   executor.Executor
+	httpClient *http.Client
 }
 
-func NewCloudOutputRepository(client cloud.TestKubeCloudAPIClient, grpcConn *grpc.ClientConn, apiKey string) *CloudOutputRepository {
-	return &CloudOutputRepository{executor: executor.NewCloudGRPCExecutor(client, grpcConn, apiKey)}
+type Option func(*CloudOutputRepository)
+
+func WithSkipVerify() Option {
+	return func(r *CloudOutputRepository) {
+		transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+		r.httpClient.Transport = transport
+	}
+}
+
+func NewCloudOutputRepository(client cloud.TestKubeCloudAPIClient, grpcConn *grpc.ClientConn, apiKey string, opts ...Option) *CloudOutputRepository {
+	r := &CloudOutputRepository{executor: executor.NewCloudGRPCExecutor(client, grpcConn, apiKey), httpClient: http.DefaultClient}
+	for _, opt := range opts {
+		opt(r)
+	}
+	return r
 }
 
 // PresignSaveLog builds presigned storage URL to save the output in Cloud
@@ -59,7 +74,7 @@ func (r *CloudOutputRepository) SaveLog(ctx context.Context, id, workflowName st
 	if err != nil {
 		return err
 	}
-	res, err := http.DefaultClient.Do(req)
+	res, err := r.httpClient.Do(req)
 	if err != nil {
 		return errors.Wrap(err, "failed to save file in cloud storage")
 	}
@@ -79,7 +94,7 @@ func (r *CloudOutputRepository) ReadLog(ctx context.Context, id, workflowName st
 	if err != nil {
 		return nil, err
 	}
-	res, err := http.DefaultClient.Do(req)
+	res, err := r.httpClient.Do(req)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get file from cloud storage")
 	}

pkg/telemetry/payload.go

@@ -34,9 +34,16 @@ type Params struct {
 	ErrorType                  string     `json:"error_type,omitempty"`
 	ErrorStackTrace            string     `json:"error_stacktrace,omitempty"`
 	TestWorkflowSteps          int32      `json:"test_workflow_steps,omitempty"`
+	TestWorkflowExecuteCount   int32      `json:"test_workflow_execute_count,omitempty"`
+	TestWorkflowParallelUsed   bool       `json:"test_workflow_parallel_used,omitempty"`
+	TestWorkflowMatrixUsed     bool       `json:"test_workflow_matrix_used,omitempty"`
+	TestWorkflowServicesUsed   bool       `json:"test_workflow_services_used,omitempty"`
+	TestWorkflowIsSample       bool       `json:"test_workflow_is_sample,omitempty"`
+	TestWorkflowTemplates      []string   `json:"testWorkflowTemplates"`
+	TestWorkflowImages         []string   `json:"testWorkflowImages"`
 	TestWorkflowTemplateUsed   bool       `json:"test_workflow_template_used,omitempty"`
-	TestWorkflowImage          string     `json:"test_workflow_image,omitempty"`
 	TestWorkflowArtifactUsed   bool       `json:"test_workflow_artifact_used,omitempty"`
+	TestWorkflowImage          string     `json:"test_workflow_image,omitempty"`
 	TestWorkflowKubeshopGitURI bool       `json:"test_workflow_kubeshop_git_uri,omitempty"`
 	License                    string     `json:"license,omitempty"`
 	Step                       string     `json:"step,omitempty"`
@@ -84,9 +91,16 @@ type RunContext struct {
 
 type WorkflowParams struct {
 	TestWorkflowSteps          int32
-	TestWorkflowTemplateUsed   bool
+	TestWorkflowExecuteCount   int32
 	TestWorkflowImage          string
 	TestWorkflowArtifactUsed   bool
+	TestWorkflowParallelUsed   bool
+	TestWorkflowMatrixUsed     bool
+	TestWorkflowServicesUsed   bool
+	TestWorkflowTemplateUsed   bool
+	TestWorkflowIsSample       bool
+	TestWorkflowTemplates      []string
+	TestWorkflowImages         []string
 	TestWorkflowKubeshopGitURI bool
 }
 
@@ -290,7 +304,14 @@ func NewRunWorkflowPayload(name, clusterType string, params RunWorkflowParams) P
 					ClusterType:                clusterType,
 					Context:                    getAgentContext(),
 					TestWorkflowSteps:          params.TestWorkflowSteps,
+					TestWorkflowExecuteCount:   params.TestWorkflowExecuteCount,
+					TestWorkflowParallelUsed:   params.TestWorkflowParallelUsed,
 					TestWorkflowTemplateUsed:   params.TestWorkflowTemplateUsed,
+					TestWorkflowMatrixUsed:     params.TestWorkflowMatrixUsed,
+					TestWorkflowServicesUsed:   params.TestWorkflowServicesUsed,
+					TestWorkflowIsSample:       params.TestWorkflowIsSample,
+					TestWorkflowTemplates:      params.TestWorkflowTemplates,
+					TestWorkflowImages:         params.TestWorkflowImages,
 					TestWorkflowImage:          params.TestWorkflowImage,
 					TestWorkflowArtifactUsed:   params.TestWorkflowArtifactUsed,
 					TestWorkflowKubeshopGitURI: params.TestWorkflowKubeshopGitURI,

pkg/testworkflows/testworkflowcontroller/channel.go

@@ -20,6 +20,7 @@ type Channel[T any] interface {
 	Channel() <-chan ChannelMessage[T]
 	Close()
 	Done() <-chan struct{}
+	CtxErr() error
 }
 
 type channel[T any] struct {
@@ -168,3 +169,7 @@ func (c *channel[T]) Close() {
 func (c *channel[T]) Done() <-chan struct{} {
 	return c.ctx.Done()
 }
+
+func (c *channel[T]) CtxErr() error {
+	return c.ctx.Err()
+}

pkg/testworkflows/testworkflowcontroller/controller.go

@@ -212,7 +212,7 @@ func (c *controller) StopController() {
 }
 
 func (c *controller) Watch(parentCtx context.Context) <-chan ChannelMessage[Notification] {
-	w, err := WatchInstrumentedPod(parentCtx, c.clientSet, c.signature, c.scheduledAt, c.pod, c.podEvents, WatchInstrumentedPodOptions{
+	ch, err := WatchInstrumentedPod(parentCtx, c.clientSet, c.signature, c.scheduledAt, c.pod, c.podEvents, WatchInstrumentedPodOptions{
 		JobEvents: c.jobEvents,
 		Job:       c.job,
 	})
@@ -222,7 +222,7 @@ func (c *controller) Watch(parentCtx context.Context) <-chan ChannelMessage[Noti
 		v.Close()
 		return v.Channel()
 	}
-	return w.Channel()
+	return ch
 }
 
 // TODO: Make it actually light
@@ -281,15 +281,15 @@ func (c *controller) Logs(parentCtx context.Context, follow bool) io.Reader {
 		case <-c.podEvents.Peek(parentCtx):
 		case <-alignTimeoutCh:
 		}
-		w, err := WatchInstrumentedPod(parentCtx, c.clientSet, c.signature, c.scheduledAt, c.pod, c.podEvents, WatchInstrumentedPodOptions{
+		ch, err := WatchInstrumentedPod(parentCtx, c.clientSet, c.signature, c.scheduledAt, c.pod, c.podEvents, WatchInstrumentedPodOptions{
 			JobEvents: c.jobEvents,
 			Job:       c.job,
 			Follow:    common.Ptr(follow),
 		})
 		if err != nil {
 			return
 		}
-		for v := range w.Channel() {
+		for v := range ch {
 			if v.Error == nil && v.Value.Log != "" && !v.Value.Temporary {
 				if ref != v.Value.Ref && v.Value.Ref != "" {
 					ref = v.Value.Ref