From 10f8ed7c96496d73b160da833ab0a348d7cfe720 Mon Sep 17 00:00:00 2001
From: Povilas Versockas
Date: Wed, 29 May 2024 15:26:11 +0300
Subject: [PATCH] feat: [TKC-2063] Delete testworkflow output when workflow is
deleted (#5509)
---
cmd/api-server/main.go | 2 +-
pkg/tcl/apitcl/v1/testworkflows.go | 4 ++
.../data/testworkflow/execution_models.go | 7 +++
pkg/tcl/cloudtcl/data/testworkflow/output.go | 6 +++
.../repositorytcl/testworkflow/interface.go | 3 ++
.../testworkflow/minio_output_repository.go | 43 ++++++++++++++++---
.../testworkflow/mock_output_repository.go | 14 ++++++
7 files changed, 73 insertions(+), 6 deletions(-)
diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go
index 5c8ec8b4a39..733e18c7943 100644
--- a/cmd/api-server/main.go
+++ b/cmd/api-server/main.go
@@ -298,7 +298,7 @@ func main() {
}
storageClient = minioClient
// Pro edition only (tcl protected code)
- testWorkflowOutputRepository = testworkflow.NewMinioOutputRepository(storageClient, cfg.LogsBucket)
+ testWorkflowOutputRepository = testworkflow.NewMinioOutputRepository(storageClient, db.Collection(testworkflow.CollectionName), cfg.LogsBucket)
artifactStorage = minio.NewMinIOArtifactClient(storageClient)
// init storage
isMinioStorage := cfg.LogsStorage == "minio"
diff --git a/pkg/tcl/apitcl/v1/testworkflows.go b/pkg/tcl/apitcl/v1/testworkflows.go
index 1c4703d8d0e..4420b71c57c 100644
--- a/pkg/tcl/apitcl/v1/testworkflows.go
+++ b/pkg/tcl/apitcl/v1/testworkflows.go
@@ -69,6 +69,10 @@ func (s *apiTCL) DeleteTestWorkflowHandler() fiber.Handler {
}
skipExecutions := c.Query("skipDeleteExecutions", "")
if skipExecutions != "true" {
+ err = s.TestWorkflowOutput.DeleteOutputByTestWorkflow(context.Background(), name)
+ if err != nil {
+ return s.ClientError(c, "deleting executions output", err)
+ }
err = s.TestWorkflowResults.DeleteByTestWorkflow(context.Background(), name)
if err != nil {
return s.ClientError(c, "deleting executions", err)
diff --git a/pkg/tcl/cloudtcl/data/testworkflow/execution_models.go b/pkg/tcl/cloudtcl/data/testworkflow/execution_models.go
index e457fb1d9e6..5d72a96be04 100644
--- a/pkg/tcl/cloudtcl/data/testworkflow/execution_models.go
+++ b/pkg/tcl/cloudtcl/data/testworkflow/execution_models.go
@@ -114,6 +114,13 @@ type ExecutionDeleteByWorkflowRequest struct {
type ExecutionDeleteByWorkflowResponse struct {
}
+type ExecutionDeleteOutputByWorkflowRequest struct {
+ WorkflowName string `json:"workflowName"`
+}
+
+type ExecutionDeleteOutputByWorkflowResponse struct {
+}
+
type ExecutionDeleteAllRequest struct {
}
diff --git a/pkg/tcl/cloudtcl/data/testworkflow/output.go b/pkg/tcl/cloudtcl/data/testworkflow/output.go
index 94ffdceff91..40122b87176 100644
--- a/pkg/tcl/cloudtcl/data/testworkflow/output.go
+++ b/pkg/tcl/cloudtcl/data/testworkflow/output.go
@@ -105,3 +105,9 @@ func (r *CloudOutputRepository) HasLog(ctx context.Context, id, workflowName str
}
return pass(r.executor, ctx, req, process)
}
+
+// DeleteByTestWorkflow deletes execution results by workflow
+func (r *CloudOutputRepository) DeleteOutputByTestWorkflow(ctx context.Context, workflowName string) (err error) {
+ req := ExecutionDeleteOutputByWorkflowRequest{WorkflowName: workflowName}
+ return passNoContent(r.executor, ctx, req)
+}
diff --git a/pkg/tcl/repositorytcl/testworkflow/interface.go b/pkg/tcl/repositorytcl/testworkflow/interface.go
index 635868c5128..265e8742047 100644
--- a/pkg/tcl/repositorytcl/testworkflow/interface.go
+++ b/pkg/tcl/repositorytcl/testworkflow/interface.go
@@ -86,4 +86,7 @@ type OutputRepository interface {
ReadLog(ctx context.Context, id, workflowName string) (io.Reader, error)
// HasLog checks if there is an output in Minio
HasLog(ctx context.Context, id, workflowName string) (bool, error)
+
+ // DeleteOutputByTestWorkflow deletes execution output by test workflow
+ DeleteOutputByTestWorkflow(ctx context.Context, testWorkflowName string) error
}
diff --git a/pkg/tcl/repositorytcl/testworkflow/minio_output_repository.go b/pkg/tcl/repositorytcl/testworkflow/minio_output_repository.go
index 5802dc4b0e1..12fc71cefb3 100644
--- a/pkg/tcl/repositorytcl/testworkflow/minio_output_repository.go
+++ b/pkg/tcl/repositorytcl/testworkflow/minio_output_repository.go
@@ -13,8 +13,12 @@ import (
"io"
"time"
+ "github.com/kubeshop/testkube/pkg/api/v1/testkube"
"github.com/kubeshop/testkube/pkg/log"
"github.com/kubeshop/testkube/pkg/storage"
+
+ "go.mongodb.org/mongo-driver/bson"
+ "go.mongodb.org/mongo-driver/mongo"
)
var _ OutputRepository = (*MinioRepository)(nil)
@@ -22,15 +26,17 @@ var _ OutputRepository = (*MinioRepository)(nil)
const bucketFolder = "testworkflows"
type MinioRepository struct {
- storage storage.Client
- bucket string
+ storage storage.Client
+ executionCollection *mongo.Collection
+ bucket string
}
-func NewMinioOutputRepository(storageClient storage.Client, bucket string) *MinioRepository {
+func NewMinioOutputRepository(storageClient storage.Client, executionCollection *mongo.Collection, bucket string) *MinioRepository {
log.DefaultLogger.Debugw("creating minio workflow output repository", "bucket", bucket)
return &MinioRepository{
- storage: storageClient,
- bucket: bucket,
+ storage: storageClient,
+ executionCollection: executionCollection,
+ bucket: bucket,
}
}
@@ -66,3 +72,30 @@ func (m *MinioRepository) HasLog(ctx context.Context, id, workflowName string) (
}
return true, nil
}
+
+func (m *MinioRepository) DeleteOutputByTestWorkflow(ctx context.Context, testWorkflowName string) error {
+ log.DefaultLogger.Debugw("deleting output by testWorkflowName", "testWorkflowName", testWorkflowName)
+ var executions []testkube.TestWorkflowExecution
+ //TODO
+ cursor, err := m.executionCollection.Find(ctx, bson.M{"testworkflow.name": testWorkflowName})
+ if err != nil {
+ return err
+ }
+ err = cursor.All(ctx, &executions)
+ if err != nil {
+ return err
+ }
+ for _, execution := range executions {
+ log.DefaultLogger.Debugw("deleting output for execution", "execution", execution)
+ err = m.DeleteOutput(ctx, execution.Id)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (m *MinioRepository) DeleteOutput(ctx context.Context, id string) error {
+ log.DefaultLogger.Debugw("deleting test workflow output", "id", id)
+ return m.storage.DeleteFileFromBucket(ctx, m.bucket, bucketFolder, id)
+}
diff --git a/pkg/tcl/repositorytcl/testworkflow/mock_output_repository.go b/pkg/tcl/repositorytcl/testworkflow/mock_output_repository.go
index bc01a3f0fbb..15a8ab433be 100644
--- a/pkg/tcl/repositorytcl/testworkflow/mock_output_repository.go
+++ b/pkg/tcl/repositorytcl/testworkflow/mock_output_repository.go
@@ -35,6 +35,20 @@ func (m *MockOutputRepository) EXPECT() *MockOutputRepositoryMockRecorder {
return m.recorder
}
+// DeleteOutputByTestWorkflow mocks base method.
+func (m *MockOutputRepository) DeleteOutputByTestWorkflow(arg0 context.Context, arg1 string) error {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "DeleteOutputByTestWorkflow", arg0, arg1)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// DeleteOutputByTestWorkflow indicates an expected call of DeleteOutputByTestWorkflow.
+func (mr *MockOutputRepositoryMockRecorder) DeleteOutputByTestWorkflow(arg0, arg1 interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOutputByTestWorkflow", reflect.TypeOf((*MockOutputRepository)(nil).DeleteOutputByTestWorkflow), arg0, arg1)
+}
+
// HasLog mocks base method.
func (m *MockOutputRepository) HasLog(arg0 context.Context, arg1, arg2 string) (bool, error) {
m.ctrl.T.Helper()