From 10f8ed7c96496d73b160da833ab0a348d7cfe720 Mon Sep 17 00:00:00 2001 From: Povilas Versockas Date: Wed, 29 May 2024 15:26:11 +0300 Subject: [PATCH] feat: [TKC-2063] Delete testworkflow output when workflow is deleted (#5509) --- cmd/api-server/main.go | 2 +- pkg/tcl/apitcl/v1/testworkflows.go | 4 ++ .../data/testworkflow/execution_models.go | 7 +++ pkg/tcl/cloudtcl/data/testworkflow/output.go | 6 +++ .../repositorytcl/testworkflow/interface.go | 3 ++ .../testworkflow/minio_output_repository.go | 43 ++++++++++++++++--- .../testworkflow/mock_output_repository.go | 14 ++++++ 7 files changed, 73 insertions(+), 6 deletions(-) diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go index 5c8ec8b4a39..733e18c7943 100644 --- a/cmd/api-server/main.go +++ b/cmd/api-server/main.go @@ -298,7 +298,7 @@ func main() { } storageClient = minioClient // Pro edition only (tcl protected code) - testWorkflowOutputRepository = testworkflow.NewMinioOutputRepository(storageClient, cfg.LogsBucket) + testWorkflowOutputRepository = testworkflow.NewMinioOutputRepository(storageClient, db.Collection(testworkflow.CollectionName), cfg.LogsBucket) artifactStorage = minio.NewMinIOArtifactClient(storageClient) // init storage isMinioStorage := cfg.LogsStorage == "minio" diff --git a/pkg/tcl/apitcl/v1/testworkflows.go b/pkg/tcl/apitcl/v1/testworkflows.go index 1c4703d8d0e..4420b71c57c 100644 --- a/pkg/tcl/apitcl/v1/testworkflows.go +++ b/pkg/tcl/apitcl/v1/testworkflows.go @@ -69,6 +69,10 @@ func (s *apiTCL) DeleteTestWorkflowHandler() fiber.Handler { } skipExecutions := c.Query("skipDeleteExecutions", "") if skipExecutions != "true" { + err = s.TestWorkflowOutput.DeleteOutputByTestWorkflow(context.Background(), name) + if err != nil { + return s.ClientError(c, "deleting executions output", err) + } err = s.TestWorkflowResults.DeleteByTestWorkflow(context.Background(), name) if err != nil { return s.ClientError(c, "deleting executions", err) diff --git a/pkg/tcl/cloudtcl/data/testworkflow/execution_models.go b/pkg/tcl/cloudtcl/data/testworkflow/execution_models.go index e457fb1d9e6..5d72a96be04 100644 --- a/pkg/tcl/cloudtcl/data/testworkflow/execution_models.go +++ b/pkg/tcl/cloudtcl/data/testworkflow/execution_models.go @@ -114,6 +114,13 @@ type ExecutionDeleteByWorkflowRequest struct { type ExecutionDeleteByWorkflowResponse struct { } +type ExecutionDeleteOutputByWorkflowRequest struct { + WorkflowName string `json:"workflowName"` +} + +type ExecutionDeleteOutputByWorkflowResponse struct { +} + type ExecutionDeleteAllRequest struct { } diff --git a/pkg/tcl/cloudtcl/data/testworkflow/output.go b/pkg/tcl/cloudtcl/data/testworkflow/output.go index 94ffdceff91..40122b87176 100644 --- a/pkg/tcl/cloudtcl/data/testworkflow/output.go +++ b/pkg/tcl/cloudtcl/data/testworkflow/output.go @@ -105,3 +105,9 @@ func (r *CloudOutputRepository) HasLog(ctx context.Context, id, workflowName str } return pass(r.executor, ctx, req, process) } + +// DeleteByTestWorkflow deletes execution results by workflow +func (r *CloudOutputRepository) DeleteOutputByTestWorkflow(ctx context.Context, workflowName string) (err error) { + req := ExecutionDeleteOutputByWorkflowRequest{WorkflowName: workflowName} + return passNoContent(r.executor, ctx, req) +} diff --git a/pkg/tcl/repositorytcl/testworkflow/interface.go b/pkg/tcl/repositorytcl/testworkflow/interface.go index 635868c5128..265e8742047 100644 --- a/pkg/tcl/repositorytcl/testworkflow/interface.go +++ b/pkg/tcl/repositorytcl/testworkflow/interface.go @@ -86,4 +86,7 @@ type OutputRepository interface { ReadLog(ctx context.Context, id, workflowName string) (io.Reader, error) // HasLog checks if there is an output in Minio HasLog(ctx context.Context, id, workflowName string) (bool, error) + + // DeleteOutputByTestWorkflow deletes execution output by test workflow + DeleteOutputByTestWorkflow(ctx context.Context, testWorkflowName string) error } diff --git a/pkg/tcl/repositorytcl/testworkflow/minio_output_repository.go b/pkg/tcl/repositorytcl/testworkflow/minio_output_repository.go index 5802dc4b0e1..12fc71cefb3 100644 --- a/pkg/tcl/repositorytcl/testworkflow/minio_output_repository.go +++ b/pkg/tcl/repositorytcl/testworkflow/minio_output_repository.go @@ -13,8 +13,12 @@ import ( "io" "time" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/log" "github.com/kubeshop/testkube/pkg/storage" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" ) var _ OutputRepository = (*MinioRepository)(nil) @@ -22,15 +26,17 @@ var _ OutputRepository = (*MinioRepository)(nil) const bucketFolder = "testworkflows" type MinioRepository struct { - storage storage.Client - bucket string + storage storage.Client + executionCollection *mongo.Collection + bucket string } -func NewMinioOutputRepository(storageClient storage.Client, bucket string) *MinioRepository { +func NewMinioOutputRepository(storageClient storage.Client, executionCollection *mongo.Collection, bucket string) *MinioRepository { log.DefaultLogger.Debugw("creating minio workflow output repository", "bucket", bucket) return &MinioRepository{ - storage: storageClient, - bucket: bucket, + storage: storageClient, + executionCollection: executionCollection, + bucket: bucket, } } @@ -66,3 +72,30 @@ func (m *MinioRepository) HasLog(ctx context.Context, id, workflowName string) ( } return true, nil } + +func (m *MinioRepository) DeleteOutputByTestWorkflow(ctx context.Context, testWorkflowName string) error { + log.DefaultLogger.Debugw("deleting output by testWorkflowName", "testWorkflowName", testWorkflowName) + var executions []testkube.TestWorkflowExecution + //TODO + cursor, err := m.executionCollection.Find(ctx, bson.M{"testworkflow.name": testWorkflowName}) + if err != nil { + return err + } + err = cursor.All(ctx, &executions) + if err != nil { + return err + } + for _, execution := range executions { + log.DefaultLogger.Debugw("deleting output for execution", "execution", execution) + err = m.DeleteOutput(ctx, execution.Id) + if err != nil { + return err + } + } + return nil +} + +func (m *MinioRepository) DeleteOutput(ctx context.Context, id string) error { + log.DefaultLogger.Debugw("deleting test workflow output", "id", id) + return m.storage.DeleteFileFromBucket(ctx, m.bucket, bucketFolder, id) +} diff --git a/pkg/tcl/repositorytcl/testworkflow/mock_output_repository.go b/pkg/tcl/repositorytcl/testworkflow/mock_output_repository.go index bc01a3f0fbb..15a8ab433be 100644 --- a/pkg/tcl/repositorytcl/testworkflow/mock_output_repository.go +++ b/pkg/tcl/repositorytcl/testworkflow/mock_output_repository.go @@ -35,6 +35,20 @@ func (m *MockOutputRepository) EXPECT() *MockOutputRepositoryMockRecorder { return m.recorder } +// DeleteOutputByTestWorkflow mocks base method. +func (m *MockOutputRepository) DeleteOutputByTestWorkflow(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOutputByTestWorkflow", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteOutputByTestWorkflow indicates an expected call of DeleteOutputByTestWorkflow. +func (mr *MockOutputRepositoryMockRecorder) DeleteOutputByTestWorkflow(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOutputByTestWorkflow", reflect.TypeOf((*MockOutputRepository)(nil).DeleteOutputByTestWorkflow), arg0, arg1) +} + // HasLog mocks base method. func (m *MockOutputRepository) HasLog(arg0 context.Context, arg1, arg2 string) (bool, error) { m.ctrl.T.Helper()