diff --git a/cmd/kubectl-testkube/commands/testworkflows/run.go b/cmd/kubectl-testkube/commands/testworkflows/run.go index afc648b2214..a9a562142c8 100644 --- a/cmd/kubectl-testkube/commands/testworkflows/run.go +++ b/cmd/kubectl-testkube/commands/testworkflows/run.go @@ -13,7 +13,7 @@ import ( "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/render" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testworkflows/renderer" - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" apiclientv1 "github.com/kubeshop/testkube/pkg/api/v1/client" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" @@ -304,7 +304,7 @@ func printRawLogLines(logs []byte, steps []testkube.TestWorkflowSignature, resul line = line[getTimestampLength(line)+1:] } - start := data.StartHintRe.FindStringSubmatch(line) + start := instructions.StartHintRe.FindStringSubmatch(line) if len(start) == 0 { line += "\x07" fmt.Println(line) diff --git a/cmd/tcl/testworkflow-toolkit/commands/execute.go b/cmd/tcl/testworkflow-toolkit/commands/execute.go index b146ae54621..fd398f8fc46 100644 --- a/cmd/tcl/testworkflow-toolkit/commands/execute.go +++ b/cmd/tcl/testworkflow-toolkit/commands/execute.go @@ -22,6 +22,7 @@ import ( commontcl "github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/common" "github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/spawn" "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/env" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/transfer" "github.com/kubeshop/testkube/internal/common" @@ -92,7 +93,7 @@ func buildTestExecution(test testworkflowsv1.StepExecuteTest, async bool) (func( return } - data.PrintOutput(env.Ref(), "test-start", &testExecutionDetails{ + instructions.PrintOutput(env.Ref(), "test-start", &testExecutionDetails{ Id: exec.Id, Name: exec.Name, TestName: exec.TestName, @@ -126,7 +127,7 @@ func buildTestExecution(test testworkflowsv1.StepExecuteTest, async bool) (func( break loop } if prevStatus != status { - data.PrintOutput(env.Ref(), "test-status", &executionResult{Id: exec.Id, Status: string(status)}) + instructions.PrintOutput(env.Ref(), "test-status", &executionResult{Id: exec.Id, Status: string(status)}) } prevStatus = status } @@ -140,7 +141,7 @@ func buildTestExecution(test testworkflowsv1.StepExecuteTest, async bool) (func( color = ui.Red } - data.PrintOutput(env.Ref(), "test-end", &executionResult{Id: exec.Id, Status: string(status)}) + instructions.PrintOutput(env.Ref(), "test-end", &executionResult{Id: exec.Id, Status: string(status)}) fmt.Printf("%s • %s\n", color(execName), string(status)) return }, nil @@ -161,7 +162,7 @@ func buildWorkflowExecution(workflow testworkflowsv1.StepExecuteWorkflow, async return } - data.PrintOutput(env.Ref(), "testworkflow-start", &testWorkflowExecutionDetails{ + instructions.PrintOutput(env.Ref(), "testworkflow-start", &testWorkflowExecutionDetails{ Id: exec.Id, Name: exec.Name, TestWorkflowName: exec.Workflow.Name, @@ -195,7 +196,7 @@ func buildWorkflowExecution(workflow testworkflowsv1.StepExecuteWorkflow, async break loop } if prevStatus != status { - data.PrintOutput(env.Ref(), "testworkflow-status", &executionResult{Id: exec.Id, Status: string(status)}) + instructions.PrintOutput(env.Ref(), "testworkflow-status", &executionResult{Id: exec.Id, Status: string(status)}) } prevStatus = status } @@ -209,7 +210,7 @@ func buildWorkflowExecution(workflow testworkflowsv1.StepExecuteWorkflow, async color = ui.Red } - data.PrintOutput(env.Ref(), "testworkflow-end", &executionResult{Id: exec.Id, Status: string(status)}) + instructions.PrintOutput(env.Ref(), "testworkflow-end", &executionResult{Id: exec.Id, Status: string(status)}) fmt.Printf("%s • %s\n", color(execName), string(status)) return }, nil diff --git a/cmd/tcl/testworkflow-toolkit/commands/kill.go b/cmd/tcl/testworkflow-toolkit/commands/kill.go index b96edbd95a4..753a2b67148 100644 --- a/cmd/tcl/testworkflow-toolkit/commands/kill.go +++ b/cmd/tcl/testworkflow-toolkit/commands/kill.go @@ -19,6 +19,7 @@ import ( commontcl "github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/common" "github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/spawn" "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/artifacts" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/env" "github.com/kubeshop/testkube/pkg/expressions" @@ -73,7 +74,7 @@ func NewKillCmd() *cobra.Command { Register("index", index). RegisterAccessorExt(func(name string) (interface{}, bool, error) { if name == "count" { - expr, err := expressions.CompileAndResolve(fmt.Sprintf("len(services.%s)", service)) + expr, err := expressions.CompileAndResolve(fmt.Sprintf("len(%s)", data.ServicesPrefix+service)) return expr, true, err } return nil, false, nil @@ -104,7 +105,7 @@ func NewKillCmd() *cobra.Command { logsFilePath, err := spawn.SaveLogs(context.Background(), clientSet, storage, env.Namespace(), id, service+"/", index) if err == nil { - data.PrintOutput(env.Ref(), "service", ServiceInfo{Group: groupRef, Name: service, Index: index, Logs: storage.FullPath(logsFilePath)}) + instructions.PrintOutput(env.Ref(), "service", ServiceInfo{Group: groupRef, Name: service, Index: index, Logs: storage.FullPath(logsFilePath)}) log("saved logs") } else { log("warning", "problem saving the logs", err.Error()) diff --git a/cmd/tcl/testworkflow-toolkit/commands/parallel.go b/cmd/tcl/testworkflow-toolkit/commands/parallel.go index 1b5fbee7052..24405cf1565 100644 --- a/cmd/tcl/testworkflow-toolkit/commands/parallel.go +++ b/cmd/tcl/testworkflow-toolkit/commands/parallel.go @@ -23,7 +23,7 @@ import ( testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" commontcl "github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/common" "github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/spawn" - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/artifacts" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/env" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/transfer" @@ -31,9 +31,9 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/expressions" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowcontroller" - "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/presets" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" "github.com/kubeshop/testkube/pkg/ui" ) @@ -163,7 +163,7 @@ func NewParallelCmd() *cobra.Command { // Send initial output for index := range specs { - data.PrintOutput(env.Ref(), "parallel", ParallelStatus{ + instructions.PrintOutput(env.Ref(), "parallel", ParallelStatus{ Index: index, Description: descriptions[index], }) @@ -200,7 +200,7 @@ func NewParallelCmd() *cobra.Command { } // Compute the bundle instructions - sig := testworkflowprocessor.MapSignatureListToInternal(bundle.Signature) + sig := stage.MapSignatureListToInternal(bundle.Signature) namespace := bundle.Job.Namespace if namespace == "" { namespace = env.Namespace() @@ -228,7 +228,7 @@ func NewParallelCmd() *cobra.Command { if shouldSaveLogs { logsFilePath, err := spawn.SaveLogs(context.Background(), clientSet, storage, namespace, id, "", index) if err == nil { - data.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Logs: storage.FullPath(logsFilePath)}) + instructions.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Logs: storage.FullPath(logsFilePath)}) log("saved logs") } else { log("warning", "problem saving the logs", err.Error()) @@ -246,7 +246,7 @@ func NewParallelCmd() *cobra.Command { }() // Inform about the step structure - data.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Signature: sig}) + instructions.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Signature: sig}) // Control the execution // TODO: Consider aggregated controller to limit number of watchers @@ -291,11 +291,11 @@ func NewParallelCmd() *cobra.Command { prevStep = v.Current prevStatus = v.Status if v.Result.IsFinished() { - data.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Status: v.Status, Result: v.Result}) + instructions.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Status: v.Status, Result: v.Result}) ctxCancel() return v.Result.IsPassed() } else { - data.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Status: v.Status, Current: v.Current}) + instructions.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Status: v.Status, Current: v.Current}) } } } diff --git a/cmd/tcl/testworkflow-toolkit/commands/services.go b/cmd/tcl/testworkflow-toolkit/commands/services.go index 55041796379..07ce4f40e38 100644 --- a/cmd/tcl/testworkflow-toolkit/commands/services.go +++ b/cmd/tcl/testworkflow-toolkit/commands/services.go @@ -25,6 +25,7 @@ import ( commontcl "github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/common" "github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/spawn" "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/env" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/transfer" "github.com/kubeshop/testkube/internal/common" @@ -111,7 +112,7 @@ func NewServicesCmd() *cobra.Command { } // Initialize empty array of details for each of the services - data.PrintHintDetails(env.Ref(), fmt.Sprintf("services.%s", name), []ServiceState{}) + instructions.PrintHintDetails(env.Ref(), data.ServicesPrefix+name, []ServiceState{}) } // Analyze instances to run @@ -189,12 +190,12 @@ func NewServicesCmd() *cobra.Command { for i := range svcInstances { state[name][i].Description = svcInstances[i].Description } - data.PrintHintDetails(env.Ref(), fmt.Sprintf("services.%s", name), state) + instructions.PrintHintDetails(env.Ref(), data.ServicesPrefix+name, state) } // Inform about each service instance for _, instance := range instances { - data.PrintOutput(env.Ref(), "service", ServiceInfo{ + instructions.PrintOutput(env.Ref(), "service", ServiceInfo{ Group: groupRef, Index: instance.Index, Name: instance.Name, @@ -274,7 +275,8 @@ func NewServicesCmd() *cobra.Command { if namespace == "" { namespace = env.Namespace() } - mainRef := bundle.Job.Spec.Template.Spec.Containers[0].Name + + mainRef := bundle.Actions().GetLastRef() // Deploy the resources // TODO: Avoid using Job @@ -329,7 +331,7 @@ func NewServicesCmd() *cobra.Command { state[instance.Name][index].Ip = v.PodIP log(fmt.Sprintf("assigned to %s IP", ui.LightBlue(v.PodIP))) info.Status = ServiceStatusRunning - data.PrintOutput(env.Ref(), "service", info) + instructions.PrintOutput(env.Ref(), "service", info) } if v.Current == mainRef { @@ -349,7 +351,7 @@ func NewServicesCmd() *cobra.Command { if !started { info.Status = ServiceStatusFailed log("container failed") - data.PrintOutput(env.Ref(), "service", info) + instructions.PrintOutput(env.Ref(), "service", info) return false } @@ -377,7 +379,7 @@ func NewServicesCmd() *cobra.Command { log("container ready") info.Status = ServiceStatusReady } - data.PrintOutput(env.Ref(), "service", info) + instructions.PrintOutput(env.Ref(), "service", info) return ready } @@ -387,7 +389,7 @@ func NewServicesCmd() *cobra.Command { // Inform about the services state for k := range state { - data.PrintHintDetails(env.Ref(), fmt.Sprintf("services.%s", k), state[k]) + instructions.PrintHintDetails(env.Ref(), data.ServicesPrefix+k, state[k]) } // Notify the results diff --git a/cmd/tcl/testworkflow-toolkit/spawn/utils.go b/cmd/tcl/testworkflow-toolkit/spawn/utils.go index 6d6db65b967..92d4812f4ed 100644 --- a/cmd/tcl/testworkflow-toolkit/spawn/utils.go +++ b/cmd/tcl/testworkflow-toolkit/spawn/utils.go @@ -33,8 +33,8 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/expressions" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowcontroller" - "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" ) func MapDynamicListToStringList(list []interface{}) []string { @@ -174,7 +174,7 @@ func ProcessFetch(transferSrv transfer.Server, fetch []testworkflowsv1.StepParal Env: []corev1.EnvVar{ {Name: "TK_NS", Value: env.Namespace()}, {Name: "TK_REF", Value: env.Ref()}, - testworkflowprocessor.BypassToolkitCheck, + stage.BypassToolkitCheck, }, Args: &result, }, diff --git a/cmd/testworkflow-init/commands/run.go b/cmd/testworkflow-init/commands/run.go new file mode 100644 index 00000000000..9afd0ca66d4 --- /dev/null +++ b/cmd/testworkflow-init/commands/run.go @@ -0,0 +1,79 @@ +package commands + +import ( + "slices" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" + "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/orchestration" + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" + "github.com/kubeshop/testkube/pkg/expressions" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" +) + +func Run(run lite.ActionExecute, container lite.LiteActionContainer) { + machine := data.GetInternalTestWorkflowMachine() + state := data.GetState() + step := state.GetStep(run.Ref) + + // Abandon executing if the step was finished before + if step.IsFinished() { + return + } + + // Obtain command to run + command := make([]string, 0) + if container.Config.Command != nil { + command = slices.Clone(*container.Config.Command) + } + if container.Config.Args != nil { + command = append(command, *container.Config.Args...) + } + + // Ensure the command is not empty + if len(command) == 0 { + output.ExitErrorf(data.CodeInputError, "command is required") + } + + // Resolve the command to run + for i := range command { + value, err := expressions.CompileAndResolveTemplate(command[i], machine, expressions.FinalizerFail) + if err != nil { + output.ExitErrorf(data.CodeInternal, "failed to compute argument '%d': %s", i, err.Error()) + } + command[i], _ = value.Static().StringValue() + } + + // Run the operation + execution := orchestration.Executions.Create(command[0], command[1:]) + result, err := execution.Run() + if err != nil { + output.ExitErrorf(data.CodeInternal, "failed to execute: %v", err) + } + + // Initialize local state + var status data.StepStatus + + success := result.ExitCode == 0 + + // Compute the result + if run.Negative { + success = !success + } + if result.Aborted { + status = data.StepStatusAborted + } else if success { + status = data.StepStatusPassed + } else { + status = data.StepStatusFailed + } + + // Abandon saving execution data if the step has been finished before + if step.IsFinished() { + return + } + + // Notify about the status + step.SetStatus(status).SetExitCode(result.ExitCode) + orchestration.FinishExecution(step, constants.ExecutionResult{ExitCode: result.ExitCode, Iteration: int(step.Iteration)}) +} diff --git a/cmd/testworkflow-init/commands/setup.go b/cmd/testworkflow-init/commands/setup.go new file mode 100644 index 00000000000..ee78ded348e --- /dev/null +++ b/cmd/testworkflow-init/commands/setup.go @@ -0,0 +1,76 @@ +package commands + +import ( + "os" + "os/exec" + "strings" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" + "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" + "github.com/kubeshop/testkube/pkg/version" +) + +func Setup(config lite.ActionSetup) error { + stdout := output.Std + stdoutUnsafe := stdout.Direct() + + // Copy the init process + stdoutUnsafe.Print("Configuring init process...") + if config.CopyInit { + err := exec.Command("cp", "/init", data.InitPath).Run() + if err != nil { + stdoutUnsafe.Error(" error\n") + stdoutUnsafe.Errorf(" failed to copy the /init process: %s\n", err.Error()) + return err + } + stdoutUnsafe.Print(" done\n") + } else { + stdoutUnsafe.Print(" skipped\n") + } + + // Copy the shell and useful libraries + stdoutUnsafe.Print("Configuring shell...") + if config.CopyBinaries { + // Use `cp` on the whole directory, as it has plenty of files, which lead to the same FS block. + // Copying individual files will lead to high FS usage + err := exec.Command("cp", "-rf", "/bin", data.InternalBinPath).Run() + if err != nil { + stdoutUnsafe.Error(" error\n") + stdoutUnsafe.Errorf(" failed to copy the binaries: %s\n", err.Error()) + return err + } + stdoutUnsafe.Print(" done\n") + } else { + stdoutUnsafe.Print(" skipped\n") + } + + // Expose debugging Pod information + stdoutUnsafe.Output(data.InitStepName, "pod", map[string]string{ + "name": os.Getenv(constants.EnvPodName), + "nodeName": os.Getenv(constants.EnvNodeName), + "namespace": os.Getenv(constants.EnvNamespaceName), + "serviceAccountName": os.Getenv(constants.EnvServiceAccountName), + "agent": version.Version, + "toolkit": stripCommonImagePrefix(os.Getenv("TESTKUBE_TW_TOOLKIT_IMAGE"), "testkube-tw-toolkit"), + "init": stripCommonImagePrefix(os.Getenv("TESTKUBE_TW_INIT_IMAGE"), "testkube-tw-init"), + }) + + return nil +} + +func stripCommonImagePrefix(image, common string) string { + if !strings.HasPrefix(image, "docker.io/") { + return image + } + image = image[10:] + if !strings.HasPrefix(image, "kubeshop/") { + return image + } + image = image[9:] + if !strings.HasPrefix(image, common+":") { + return image + } + return image[len(common)+1:] +} diff --git a/cmd/testworkflow-init/constants/commands.go b/cmd/testworkflow-init/constants/commands.go index c9823142a21..de826192651 100644 --- a/cmd/testworkflow-init/constants/commands.go +++ b/cmd/testworkflow-init/constants/commands.go @@ -1,26 +1,12 @@ package constants const ( - ArgSeparator = "--" - ArgInit = "-i" - ArgInitLong = "--init" - ArgCondition = "-c" - ArgConditionLong = "--cond" - ArgResult = "-r" - ArgResultLong = "--result" - ArgTimeout = "-t" - ArgTimeoutLong = "--timeout" - ArgComputeEnv = "-e" - ArgComputeEnvLong = "--env" - ArgNegative = "-n" - ArgNegativeLong = "--negative" - ArgPaused = "-p" - ArgPausedLong = "--pause" - ArgDebug = "--debug" - ArgWorkingDir = "-w" - ArgWorkingDirLong = "--workingDir" - ArgToolkit = "-k" - ArgToolkitLong = "--toolkit" - ArgRetryUntil = "--retryUntil" // TODO: Replace when multi-level retry will be there - ArgRetryCount = "--retryCount" // TODO: Replace when multi-level retry will be there + EnvGroupActions = "01" + EnvGroupDebug = "00" + + EnvNodeName = "TKI_N" + EnvPodName = "TKI_P" + EnvNamespaceName = "TKI_S" + EnvServiceAccountName = "TKI_A" + EnvActions = "TKI_I" ) diff --git a/cmd/testworkflow-init/constants/instructions.go b/cmd/testworkflow-init/constants/instructions.go index 64801dab63b..9a1e3fe44b0 100644 --- a/cmd/testworkflow-init/constants/instructions.go +++ b/cmd/testworkflow-init/constants/instructions.go @@ -3,8 +3,14 @@ package constants const ( InstructionStart = "start" InstructionEnd = "end" - InstructionStatus = "status" + InstructionExecution = "execution" InstructionPause = "pause" InstructionResume = "resume" InstructionIteration = "iteration" ) + +type ExecutionResult struct { + ExitCode uint8 `json:"code"` + Details string `json:"details,omitempty"` + Iteration int `json:"iteration,omitempty"` +} diff --git a/cmd/testworkflow-init/control/options.go b/cmd/testworkflow-init/control/options.go new file mode 100644 index 00000000000..6e1e8d0abc8 --- /dev/null +++ b/cmd/testworkflow-init/control/options.go @@ -0,0 +1,16 @@ +package control + +import "time" + +type ServerOptions struct { + HandlePause func(ts time.Time) error + HandleResume func(ts time.Time) error +} + +func (p ServerOptions) Pause(ts time.Time) error { + return p.HandlePause(ts) +} + +func (p ServerOptions) Resume(ts time.Time) error { + return p.HandleResume(ts) +} diff --git a/cmd/testworkflow-init/control/server.go b/cmd/testworkflow-init/control/server.go index 6292cd1ed8d..dd8ce52443f 100644 --- a/cmd/testworkflow-init/control/server.go +++ b/cmd/testworkflow-init/control/server.go @@ -6,26 +6,31 @@ import ( "net" "net/http" "time" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" ) type Pauseable interface { Pause(time.Time) error - Resume() error + Resume(ts time.Time) error } type server struct { - port int - step Pauseable + port int + target Pauseable } -func NewServer(port int, step Pauseable) *server { +func NewServer(port int, target Pauseable) *server { return &server{ - port: port, - step: step, + port: port, + target: target, } } func (s *server) handler() *http.ServeMux { + stdout := output.Std + stdoutUnsafe := stdout.Direct() + mux := http.NewServeMux() // TODO: Consider "shell" command too for debugging? mux.HandleFunc("/pause", func(w http.ResponseWriter, r *http.Request) { @@ -33,8 +38,8 @@ func (s *server) handler() *http.ServeMux { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - if err := s.step.Pause(time.Now()); err != nil { - fmt.Printf("Warning: failed to pause: %s\n", err.Error()) + if err := s.target.Pause(time.Now()); err != nil { + stdoutUnsafe.Warnf("warn: failed to pause: %s\n", err.Error()) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -45,8 +50,8 @@ func (s *server) handler() *http.ServeMux { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - if err := s.step.Resume(); err != nil { - fmt.Printf("Warning: failed to resume: %s\n", err.Error()) + if err := s.target.Resume(time.Now()); err != nil { + stdoutUnsafe.Warnf("warn: failed to resume: %s\n", err.Error()) http.Error(w, err.Error(), http.StatusInternalServerError) return } diff --git a/cmd/testworkflow-init/data/config.go b/cmd/testworkflow-init/data/config.go deleted file mode 100644 index be1b9fecb19..00000000000 --- a/cmd/testworkflow-init/data/config.go +++ /dev/null @@ -1,25 +0,0 @@ -package data - -import ( - "os" -) - -type config struct { - Negative bool - Debug bool - RetryCount int - RetryUntil string - - Resulting []Rule -} - -var Config = &config{ - Debug: os.Getenv("DEBUG") == "1", -} - -func LoadConfig(config map[string]string) { - Config.Debug = getBool(config, "debug", Config.Debug) - Config.RetryCount = getInt(config, "retryCount", 0) - Config.RetryUntil = getStr(config, "retryUntil", "self.passed") - Config.Negative = getBool(config, "negative", false) -} diff --git a/cmd/testworkflow-init/data/constants.go b/cmd/testworkflow-init/data/constants.go new file mode 100644 index 00000000000..69ee95e0a5f --- /dev/null +++ b/cmd/testworkflow-init/data/constants.go @@ -0,0 +1,54 @@ +package data + +import "path/filepath" + +const ( + InitStepName = "tktw-init" + InternalPath = "/.tktw" + TerminationLogPath = "/dev/termination-log" +) + +var ( + InternalBinPath = filepath.Join(InternalPath, "bin") + InitPath = filepath.Join(InternalPath, "init") + StatePath = filepath.Join(InternalPath, "state") +) + +type StepStatus string + +const ( + StepStatusPassed StepStatus = "passed" + StepStatusTimeout StepStatus = "timeout" + StepStatusFailed StepStatus = "failed" + StepStatusAborted StepStatus = "aborted" + StepStatusSkipped StepStatus = "skipped" +) + +func (s StepStatus) Code() string { + return string(s)[0:1] +} + +func StepStatusFromCode(code string) StepStatus { + if len(code) != 1 { + return StepStatusAborted + } + switch code[0] { + case StepStatusPassed[0]: + return StepStatusPassed + case StepStatusTimeout[0]: + return StepStatusTimeout + case StepStatusFailed[0]: + return StepStatusFailed + case StepStatusAborted[0]: + return StepStatusAborted + case StepStatusSkipped[0]: + return StepStatusSkipped + } + return StepStatusAborted +} + +const ( + CodeAborted uint8 = 137 + CodeInputError uint8 = 155 + CodeInternal uint8 = 190 +) diff --git a/cmd/testworkflow-init/data/expressions.go b/cmd/testworkflow-init/data/expressions.go index 4b9d5a07d8c..39a8a97661c 100644 --- a/cmd/testworkflow-init/data/expressions.go +++ b/cmd/testworkflow-init/data/expressions.go @@ -1,12 +1,25 @@ package data import ( + "fmt" "os" "strings" + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" "github.com/kubeshop/testkube/pkg/expressions" ) +const ( + OutputKey = "output" + OutputPrefix = OutputKey + "." + ServicesKey = "services" + ServicesPrefix = ServicesKey + "." + EnvKey = "env" + EnvPrefix = EnvKey + "." + RefKey = "_ref" + StatusKey = "status" +) + var aliases = map[string]string{ "always": `true`, "never": `false`, @@ -17,20 +30,20 @@ var aliases = map[string]string{ "self.error": `self.failed`, "self.success": `self.passed`, - "passed": `!status`, - "failed": `bool(status) && status != "skipped"`, + "passed": fmt.Sprintf(`%s == "passed"`, StatusKey), + "failed": fmt.Sprintf(`%s != "passed" && %s != "skipped"`, StatusKey, StatusKey), - "self.passed": `!self.status`, - "self.failed": `bool(self.status) && self.status != "skipped"`, + "self.passed": `self.status == "passed"`, + "self.failed": `self.status != "passed" && self.status != "skipped"`, } var LocalMachine = expressions.NewMachine(). - Register("status", expressions.MustCompile("self.status")) + Register(StatusKey, expressions.MustCompile("self.status")) var RefMachine = expressions.NewMachine(). RegisterAccessor(func(name string) (interface{}, bool) { - if name == "_ref" { - return Step.Ref, true + if name == RefKey { + return GetState().CurrentRef, true } return nil, false }) @@ -52,34 +65,48 @@ var AliasMachine = expressions.NewMachine(). var StateMachine = expressions.NewMachine(). RegisterAccessor(func(name string) (interface{}, bool) { if name == "status" { - return State.GetStatus(), true + currentStatus := GetState().CurrentStatus + expr, err := expressions.EvalExpression(currentStatus, RefStatusMachine, AliasMachine) + if err != nil { + output.ExitErrorf(CodeInternal, "current status is invalid: %s: %v\n", currentStatus, err.Error()) + } + if passed, _ := expr.BoolValue(); passed { + return string(StepStatusPassed), true + } + return string(StepStatusFailed), true } else if name == "self.status" { - return State.GetSelfStatus(), true + state := GetState() + step := state.GetStep(state.CurrentRef) + if step.Status == nil { + return nil, false + } + return string(*step.Status), true } return nil, false }). RegisterAccessorExt(func(name string) (interface{}, bool, error) { - if strings.HasPrefix(name, "output.") { - return State.GetOutput(name[7:]) + if strings.HasPrefix(name, OutputPrefix) { + return GetState().GetOutput(name[len(OutputPrefix):]) } return nil, false, nil }). RegisterAccessorExt(func(name string) (interface{}, bool, error) { - if strings.HasPrefix(name, "services.") { - return State.GetOutput(name) + if strings.HasPrefix(name, ServicesPrefix) { + // TODO TODO TODO TODO + return GetState().GetOutput(name) } return nil, false, nil }) var EnvMachine = expressions.NewMachine(). RegisterAccessor(func(name string) (interface{}, bool) { - if strings.HasPrefix(name, "env.") { - return os.Getenv(name[4:]), true + if strings.HasPrefix(name, EnvPrefix) { + return os.Getenv(name[len(EnvPrefix):]), true } return nil, false }). RegisterAccessor(func(name string) (interface{}, bool) { - if name != "env" { + if name != EnvKey { return nil, false } env := make(map[string]string) @@ -92,13 +119,20 @@ var EnvMachine = expressions.NewMachine(). var RefSuccessMachine = expressions.NewMachine(). RegisterAccessor(func(ref string) (interface{}, bool) { - s := State.GetStep(ref) - return s.Status == StepStatusPassed || s.Status == StepStatusSkipped, s.HasStatus + s := GetState().GetStep(ref) + if s.Status == nil { + return nil, false + } + return *s.Status == StepStatusPassed || *s.Status == StepStatusSkipped, true }) var RefStatusMachine = expressions.NewMachine(). RegisterAccessor(func(ref string) (interface{}, bool) { - return string(State.GetStep(ref).Status), true + status := GetState().GetStep(ref).Status + if status == nil { + return nil, false + } + return string(*status), true }) func Template(tpl string, m ...expressions.Machine) (string, error) { @@ -110,11 +144,3 @@ func Expression(expr string, m ...expressions.Machine) (expressions.StaticValue, m = append(m, AliasMachine, GetBaseTestWorkflowMachine()) return expressions.EvalExpression(expr, m...) } - -func RefSuccessExpression(expr string) (expressions.StaticValue, error) { - return expressions.EvalExpression(expr, RefSuccessMachine) -} - -func RefStatusExpression(expr string) (expressions.StaticValue, error) { - return expressions.EvalExpression(expr, RefStatusMachine) -} diff --git a/cmd/testworkflow-init/data/global.go b/cmd/testworkflow-init/data/global.go index 6164c90d668..8c31ac8e5e0 100644 --- a/cmd/testworkflow-init/data/global.go +++ b/cmd/testworkflow-init/data/global.go @@ -15,6 +15,10 @@ func GetBaseTestWorkflowMachine() expressions.Machine { wd = "/" } fileMachine := libs.NewFsMachine(os.DirFS("/"), wd) - LoadState() + GetState() // load state return expressions.CombinedMachines(EnvMachine, StateMachine, fileMachine) } + +func GetInternalTestWorkflowMachine() expressions.Machine { + return expressions.CombinedMachines(RefSuccessMachine, AliasMachine, GetBaseTestWorkflowMachine()) +} diff --git a/cmd/testworkflow-init/data/outputProcessor.go b/cmd/testworkflow-init/data/outputProcessor.go index fb2f4cf1193..ac576846db7 100644 --- a/cmd/testworkflow-init/data/outputProcessor.go +++ b/cmd/testworkflow-init/data/outputProcessor.go @@ -4,19 +4,19 @@ import ( "bytes" "errors" "io" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" ) type outputProcessor struct { writer io.Writer - ref string closed bool lastLine []byte } -func NewOutputProcessor(ref string, writer io.Writer) io.WriteCloser { +func NewOutputProcessor(writer io.Writer) io.WriteCloser { return &outputProcessor{ writer: writer, - ref: ref, } } @@ -29,12 +29,12 @@ func (o *outputProcessor) Write(p []byte) (int, error) { lines := bytes.Split(append(o.lastLine, p...), []byte("\n")) o.lastLine = nil for i := range lines { - instruction, _, _ := DetectInstruction(lines[i]) + instruction, _, _ := instructions.DetectInstruction(lines[i]) if instruction == nil && i == len(lines)-1 { o.lastLine = lines[i] } if instruction != nil && instruction.Value != nil { - State.SetOutput(instruction.Ref, instruction.Name, instruction.Value) + GetState().SetOutput(instruction.Ref, instruction.Name, instruction.Value) } } diff --git a/cmd/testworkflow-init/data/state.go b/cmd/testworkflow-init/data/state.go index 1d0ee8fed26..7cff8f56ac0 100644 --- a/cmd/testworkflow-init/data/state.go +++ b/cmd/testworkflow-init/data/state.go @@ -2,39 +2,34 @@ package data import ( "bytes" - "encoding/gob" "encoding/json" "fmt" "os" - "path/filepath" + "slices" + "strings" "sync" - "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" "github.com/kubeshop/testkube/pkg/expressions" -) - -const ( - defaultInternalPath = "/.tktw" - defaultTerminationLogPath = "/dev/termination-log" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" ) type state struct { - Status TestWorkflowStatus `json:"status"` - Steps map[string]*StepInfo `json:"steps"` - Output map[string]string `json:"output"` -} + Actions [][]lite.LiteAction `json:"a,omitempty"` + CurrentGroupIndex int `json:"g,omitempty"` -var State = &state{ - Steps: map[string]*StepInfo{}, - Output: map[string]string{}, + CurrentRef string `json:"c,omitempty"` + CurrentStatus string `json:"s,omitempty"` + Output map[string]string `json:"o,omitempty"` + Steps map[string]*StepData `json:"S,omitempty"` } -func (s *state) GetStep(ref string) *StepInfo { - _, ok := State.Steps[ref] - if !ok { - State.Steps[ref] = &StepInfo{Ref: ref} +func (s *state) GetActions(groupIndex int) []lite.LiteAction { + if groupIndex < 0 || groupIndex >= len(s.Actions) { + panic("unknown actions group") } - return State.Steps[ref] + s.CurrentGroupIndex = groupIndex + return s.Actions[groupIndex] } func (s *state) GetOutput(name string) (expressions.Expression, bool, error) { @@ -54,37 +49,54 @@ func (s *state) SetOutput(ref, name string, value interface{}) { if err == nil { s.Output[name] = string(v) } else { - fmt.Printf("Warning: couldn't save '%s' (%s) output: %s\n", name, ref, err.Error()) + output.Std.Warnf("warn: couldn't save '%s' (%s) output: %s\n", name, ref, err.Error()) } } -func (s *state) GetSelfStatus() string { - if Step.Executed { - return string(Step.Status) +func (s *state) GetStep(ref string) *StepData { + if s.Steps[ref] == nil { + s.Steps[ref] = &StepData{Ref: ref} } - v := s.GetStep(Step.Ref) - if v.Status != StepStatusPassed { - return string(v.Status) + if s.Steps[ref].Condition == "" { + s.Steps[ref].Condition = "passed" } - return string(Step.Status) + return s.Steps[ref] } -func (s *state) GetStatus() string { - if Step.Executed { - return string(Step.Status) - } - if Step.InitStatus == "" { - return string(s.Status) +func (s *state) getSubSteps(ref string, visited *map[*StepData]struct{}) { + // Ignore already visited node + if _, ok := (*visited)[s.Steps[ref]]; ok { + return } - v, err := RefStatusExpression(Step.InitStatus) - if err != nil { - return string(s.Status) + + // Append the node + (*visited)[s.Steps[ref]] = struct{}{} + + // Visit its children + for _, sub := range s.Steps { + if slices.Contains(sub.Parents, ref) { + s.getSubSteps(sub.Ref, visited) + } } - str, _ := v.Static().StringValue() - if str == "" { - return string(s.Status) +} + +func (s *state) GetSubSteps(ref string) []*StepData { + visited := map[*StepData]struct{}{} + s.getSubSteps(ref, &visited) + result := make([]*StepData, 0, len(visited)) + for r := range visited { + result = append(result, r) } - return str + return result +} + +func (s *state) SetCurrentStatus(expression string) { + s.CurrentStatus = expression +} + +var currentState = &state{ + Output: map[string]string{}, + Steps: map[string]*StepData{}, } func readState(filePath string) { @@ -98,7 +110,7 @@ func readState(filePath string) { if len(b) == 0 { return } - err = gob.NewDecoder(bytes.NewBuffer(b)).Decode(&State) + err = json.NewDecoder(bytes.NewBuffer(b)).Decode(¤tState) if err != nil { panic(err) } @@ -106,7 +118,7 @@ func readState(filePath string) { func persistState(filePath string) { b := bytes.Buffer{} - err := gob.NewEncoder(&b).Encode(State) + err := json.NewEncoder(&b).Encode(currentState) if err != nil { panic(err) } @@ -117,67 +129,53 @@ func persistState(filePath string) { } } -func recomputeStatuses() { - // Read current status - status := StepStatus(State.GetSelfStatus()) - - // Update own status - State.GetStep(Step.Ref).SetStatus(status) - - // Update expected failure statuses - Iterate(Config.Resulting, func(r Rule) bool { - v, err := RefSuccessExpression(r.Expr) - if err != nil { - return false +func persistTerminationLog() { + // Read the state + s := GetState() + + // Get list of statuses + actions := s.GetActions(s.CurrentGroupIndex) + statuses := make([]string, 0) + for i := range actions { + ref := "" + if actions[i].Type() == lite.ActionTypeEnd { + ref = *actions[i].End } - vv, _ := v.Static().BoolValue() - if !vv { - for _, ref := range r.Refs { - if ref == "" { - State.Status = TestWorkflowStatusFailed - } else { - State.GetStep(ref).SetStatus(StepStatusFailed) - } - } + if actions[i].Type() == lite.ActionTypeSetup { + ref = InitStepName } - return true - }) -} + if ref == "" { + continue + } + step := s.GetStep(ref) + if step.Status == nil { + statuses = append(statuses, fmt.Sprintf("%s,%d", StepStatusAborted, CodeAborted)) + } else { + statuses = append(statuses, fmt.Sprintf("%s,%d", (*step.Status).Code(), step.ExitCode)) + } + } -func persistStatus(filePath string) { - // Persist container termination result - res := fmt.Sprintf(`%s,%d`, State.GetStep(Step.Ref).Status, Step.ExitCode) - err := os.WriteFile(filePath, []byte(res), 0755) + // Write the termination log + err := os.WriteFile(TerminationLogPath, []byte(strings.Join(statuses, "/")), 0) if err != nil { - panic(err) + output.UnsafeExitErrorf(CodeInternal, "failed to save the termination log: %s", err.Error()) } } var loadStateMu sync.Mutex var loadedState bool -func LoadState() { +func GetState() *state { defer loadStateMu.Unlock() loadStateMu.Lock() if !loadedState { - readState(filepath.Join(defaultInternalPath, "state")) + readState(StatePath) loadedState = true } + return currentState } -func Finish() { - // Persist step information and shared data - recomputeStatuses() - persistStatus(defaultTerminationLogPath) - persistState(filepath.Join(defaultInternalPath, "state")) - - // Kill the sub-process - Step.Kill() - - // Emit end hint to allow exporting the timestamp - PrintHint(Step.Ref, constants.InstructionEnd) - - // The init process needs to finish with zero exit code, - // to continue with the next container. - os.Exit(0) +func SaveState() { + persistState(StatePath) + persistTerminationLog() } diff --git a/cmd/testworkflow-init/data/step.go b/cmd/testworkflow-init/data/step.go deleted file mode 100644 index 5802f349783..00000000000 --- a/cmd/testworkflow-init/data/step.go +++ /dev/null @@ -1,227 +0,0 @@ -package data - -import ( - "fmt" - "os" - "os/exec" - "sync" - "sync/atomic" - "time" - - "github.com/pkg/errors" - gopsutil "github.com/shirou/gopsutil/v3/process" - - "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" -) - -var Step = &step{} - -type step struct { - Ref string - Status StepStatus - ExitCode uint8 - Executed bool - InitStatus string - - paused atomic.Bool - pausedNs atomic.Int64 - pausedStart time.Time - cmd *exec.Cmd - runMu sync.Mutex - cmdMu sync.Mutex - pauseMu sync.Mutex -} - -// TODO: Obfuscate Stdout/Stderr streams -func (s *step) Run(negative bool, cmd string, args ...string) { - // Avoid multiple runs at once - s.runMu.Lock() - defer s.runMu.Unlock() - - // Wait until not paused - s.pauseMu.Lock() - - // Prepare the command - s.cmdMu.Lock() - s.cmd = exec.Command(cmd, args...) - out := NewOutputProcessor(s.Ref, os.Stdout) - s.cmd.Stdout = out - s.cmd.Stderr = os.Stderr - s.cmd.Stdin = os.Stdin - - // Initialize local state - var success bool - var exitCode uint8 - - // Run the command - err := s.cmd.Start() - if err == nil { - s.pauseMu.Unlock() - s.cmdMu.Unlock() - success, exitCode = getProcessStatus(s.cmd.Wait()) - } else { - s.pauseMu.Unlock() - s.cmdMu.Unlock() - success, exitCode = getProcessStatus(err) - } - - s.ExitCode = exitCode - if negative { - success = !success - } - if success { - s.Status = StepStatusPassed - } else { - s.Status = StepStatusFailed - } - - // Clean up - s.cmdMu.Lock() - s.cmd = nil - s.cmdMu.Unlock() -} - -func (s *step) Took(since time.Time) time.Duration { - now := time.Now() - if s.paused.Load() { - now = s.pausedStart - } - if !now.After(since) { - return 0 - } - return now.Sub(since) - time.Duration(s.pausedNs.Load()) -} - -func (s *step) Kill() { - s.cmdMu.Lock() - if s.cmd != nil && s.cmd.Process != nil { - _ = s.cmd.Process.Kill() - } - s.cmdMu.Unlock() -} - -func (s *step) Pause(t time.Time) (err error) { - // Lock running - swapped := s.paused.CompareAndSwap(false, true) - if !swapped { - return nil - } - s.pauseMu.Lock() - - // Save the information about current pause time - s.pausedStart = time.Now() - - // Pause already started application - s.cmdMu.Lock() - if s.cmd != nil && s.cmd.Process != nil { - ps, totalFailure, err2 := processes() - if err2 != nil && totalFailure { - err = err2 - } else { - err = each(int32(s.cmd.Process.Pid), ps, func(p *gopsutil.Process) error { - return p.Suspend() - }) - } - } - s.cmdMu.Unlock() - - // Display output - PrintHintDetails(s.Ref, constants.InstructionPause, t.Format(constants.PreciseTimeFormat)) - return err -} - -func (s *step) Resume() (err error) { - // Unlock running - swapped := s.paused.CompareAndSwap(true, false) - if !swapped { - return nil - } - - // Finish current pause period - s.pausedNs.Add(time.Now().Sub(s.pausedStart).Nanoseconds()) - - // Resume started application - s.cmdMu.Lock() - if s.cmd != nil && s.cmd.Process != nil { - ps, totalFailure, err2 := processes() - if err2 != nil && totalFailure { - err = err2 - } else { - err = each(int32(s.cmd.Process.Pid), ps, func(p *gopsutil.Process) error { - return p.Resume() - }) - } - } - s.cmdMu.Unlock() - s.pauseMu.Unlock() - - // Display output - PrintHintDetails(s.Ref, constants.InstructionResume, time.Now().Format(constants.PreciseTimeFormat)) - return err -} - -func processes() (map[int32]int32, bool, error) { - // Get list of processes - list, err := gopsutil.Processes() - if err != nil { - return nil, true, errors.Wrapf(err, "failed to list processes") - } - ownPid := os.Getpid() - - // Get parent process for each process - r := map[int32]int32{} - var errs []error - for _, p := range list { - if p.Pid == int32(ownPid) { - continue - } - r[p.Pid], err = p.Ppid() - if err != nil { - errs = append(errs, err) - } - } - - // Return info - if len(errs) > 0 { - err = errors.Wrapf(errs[0], "failed to load %d/%d processes", len(errs), len(r)) - } - return r, len(errs) == len(r), err -} - -func each(pid int32, pidToPpid map[int32]int32, fn func(*gopsutil.Process) error) error { - if _, ok := pidToPpid[pid]; !ok { - return fmt.Errorf("process %d: not found", pid) - } - - // Run operation for the process - err := fn(&gopsutil.Process{Pid: pid}) - if err != nil { - return errors.Wrapf(err, "process %d: failed to perform", pid) - } - - // Run operation for all the children recursively - for p, ppid := range pidToPpid { - if ppid == pid { - err = each(p, pidToPpid, fn) - if err != nil { - return errors.Wrapf(err, "process %d: children", pid) - } - } - } - - return nil -} - -func getProcessStatus(err error) (bool, uint8) { - if err == nil { - return true, 0 - } - if e, ok := err.(*exec.ExitError); ok { - if e.ProcessState != nil { - return false, uint8(e.ProcessState.ExitCode()) - } - return false, 1 - } - fmt.Println(err.Error()) - return false, 1 -} diff --git a/cmd/testworkflow-init/data/stepData.go b/cmd/testworkflow-init/data/stepData.go new file mode 100644 index 00000000000..7b7371b0558 --- /dev/null +++ b/cmd/testworkflow-init/data/stepData.go @@ -0,0 +1,170 @@ +package data + +import ( + "errors" + "slices" + "sync" + "time" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" +) + +type RetryPolicy struct { + Count int32 `json:"count,omitempty"` + Until string `json:"until,omitempty" expr:"expression"` +} + +type StepData struct { + Ref string `json:"_,omitempty"` + ExitCode uint8 `json:"e,omitempty"` + Status *StepStatus `json:"s,omitempty"` + StartedAt *time.Time `json:"S,omitempty"` + Condition string `json:"c,omitempty"` + Parents []string `json:"p,omitempty"` + Timeout *time.Duration `json:"t,omitempty"` + PausedOnStart bool `json:"P,omitempty"` + Retry RetryPolicy `json:"r,omitempty"` + Result string `json:"R,omitempty"` + Iteration int32 `json:"i,omitempty"` + + // Pausing + PausedNs int64 `json:"n,omitempty"` + PausedStart *time.Time `json:"N,omitempty"` + paused bool + pauseMu sync.Mutex +} + +func (s *StepData) IsFinished() bool { + return s.Status != nil +} + +func (s *StepData) IsStarted() bool { + return s.StartedAt != nil +} + +func (s *StepData) ResolveCondition() (bool, error) { + if s.Condition == "" { + return false, errors.New("missing condition expression") + } + expr, err := Expression(s.Condition, RefSuccessMachine) + if err != nil { + return false, err + } + return expr.Static().BoolValue() +} + +func (s *StepData) ResolveResult() (StepStatus, error) { + if s.Result == "" { + return StepStatusAborted, errors.New("missing result expression") + } + expr, err := Expression(s.Result, RefSuccessMachine) + if err != nil { + return StepStatusAborted, err + } + success, err := expr.Static().BoolValue() + if err != nil { + return StepStatusAborted, err + } + if success { + return StepStatusPassed, nil + } + return StepStatusFailed, nil +} + +func (s *StepData) SetExitCode(exitCode uint8) *StepData { + s.ExitCode = exitCode + return s +} + +func (s *StepData) SetCondition(expression string) *StepData { + s.Condition = expression + return s +} + +func (s *StepData) SetParents(parents []string) *StepData { + parents = slices.Clone(parents) + slices.Reverse(parents) + s.Parents = parents + return s +} + +func (s *StepData) SetPausedOnStart(pause bool) *StepData { + s.PausedOnStart = pause + return s +} + +func (s *StepData) SetTimeout(timeout string) *StepData { + if timeout == "" { + s.Timeout = nil + } + duration, err := time.ParseDuration(timeout) + if err != nil { + output.ExitErrorf(CodeInputError, "invalid timeout duration: %s: %s", timeout, err.Error()) + } + s.Timeout = &duration + return s +} + +func (s *StepData) SetResult(expression string) *StepData { + s.Result = expression + return s +} + +func (s *StepData) SetRetryPolicy(policy RetryPolicy) *StepData { + s.Retry = policy + return s +} + +func (s *StepData) SetStatus(status StepStatus) *StepData { + s.Status = &status + return s +} + +func (s *StepData) RegisterPauseStart(ts time.Time) bool { + s.pauseMu.Lock() + defer s.pauseMu.Unlock() + + if s.paused { + return false + } + s.paused = true + start := ts + s.PausedStart = &start + return true +} + +func (s *StepData) RegisterPauseEnd(ts time.Time) bool { + s.pauseMu.Lock() + defer s.pauseMu.Unlock() + + if !s.paused { + return false + } + s.paused = false + took := ts.Sub(*s.PausedStart) + s.PausedStart = nil + s.PausedNs += took.Nanoseconds() + return true +} + +func (s *StepData) Took(ts time.Time) time.Duration { + s.pauseMu.Lock() + defer s.pauseMu.Unlock() + + if s.StartedAt == nil { + return 0 + } + + if s.PausedStart != nil { + return ts.Sub(*s.PausedStart) - time.Duration(s.PausedNs) + } + return ts.Sub(*s.StartedAt) - time.Duration(s.PausedNs) +} + +func (s *StepData) TimeLeft(ts time.Time) *time.Duration { + if s.Timeout == nil || s.StartedAt == nil { + return nil + } + left := *s.Timeout - s.Took(ts) + return &left +} diff --git a/cmd/testworkflow-init/data/types.go b/cmd/testworkflow-init/data/types.go deleted file mode 100644 index cd215511518..00000000000 --- a/cmd/testworkflow-init/data/types.go +++ /dev/null @@ -1,99 +0,0 @@ -package data - -import ( - "strings" - "time" - - "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" -) - -type TestWorkflowStatus string - -const ( - TestWorkflowStatusPassed TestWorkflowStatus = "" - TestWorkflowStatusFailed TestWorkflowStatus = "failed" - TestWorkflowStatusAborted TestWorkflowStatus = "aborted" -) - -type StepStatus string - -const ( - StepStatusPassed StepStatus = "" - StepStatusTimeout StepStatus = "timeout" - StepStatusFailed StepStatus = "failed" - StepStatusAborted StepStatus = "aborted" - StepStatusSkipped StepStatus = "skipped" -) - -type Rule struct { - Expr string - Refs []string -} - -type Timeout struct { - Ref string - Duration string -} - -type StepInfo struct { - Ref string `json:"ref"` - Status StepStatus `json:"status"` - HasStatus bool `json:"hasStatus"` - StartTime time.Time `json:"startTime"` - TimeoutAt time.Time `json:"timeoutAt"` - Iteration uint64 `json:"iteration"` -} - -func (s *StepInfo) Start(t time.Time) { - if s.StartTime.IsZero() { - s.StartTime = t - s.Iteration = 1 - PrintHint(s.Ref, constants.InstructionStart) - } -} - -func (s *StepInfo) Next() { - if s.StartTime.IsZero() { - s.Start(time.Now()) - } else { - s.Iteration++ - PrintHintDetails(s.Ref, constants.InstructionIteration, s.Iteration) - } -} - -func (s *StepInfo) Skip(t time.Time) { - if s.Status != StepStatusSkipped { - s.StartTime = t - s.Iteration = 0 - s.SetStatus(StepStatusSkipped) - } -} - -func (s *StepInfo) SetTimeoutDuration(t time.Time, duration string) error { - if !s.TimeoutAt.IsZero() { - return nil - } - s.Start(t) - v, err := Template(duration) - if err != nil { - return err - } - d, err := time.ParseDuration(strings.ReplaceAll(v, " ", "")) - if err != nil { - return err - } - s.TimeoutAt = s.StartTime.Add(d) - return nil -} - -func (s *StepInfo) SetStatus(status StepStatus) { - if !s.HasStatus || s.Status == StepStatusPassed { - s.Status = status - s.HasStatus = true - if status == StepStatusPassed { - PrintHintDetails(s.Ref, constants.InstructionStatus, "passed") - } else { - PrintHintDetails(s.Ref, constants.InstructionStatus, status) - } - } -} diff --git a/cmd/testworkflow-init/data/utils.go b/cmd/testworkflow-init/data/utils.go deleted file mode 100644 index 86957f65fa7..00000000000 --- a/cmd/testworkflow-init/data/utils.go +++ /dev/null @@ -1,53 +0,0 @@ -package data - -import ( - "fmt" - "os" - "strconv" - "strings" -) - -func getStr(config map[string]string, key string, defaultValue string) string { - val, ok := config[key] - if !ok { - return defaultValue - } - return val -} - -func getInt(config map[string]string, key string, defaultValue int) int { - str := getStr(config, key, "") - if str == "" { - return defaultValue - } - val, err := strconv.Atoi(str) - if err != nil { - fmt.Printf("invalid '%s' provided: '%s': %v\n", key, str, err) - os.Exit(155) - } - return val -} - -func getBool(config map[string]string, key string, defaultValue bool) bool { - str := getStr(config, key, "") - if str == "" { - return defaultValue - } - return strings.ToLower(str) == "true" || str == "1" -} - -// Iterate over all items, all the time, until no more is done -func Iterate[T any](v []T, fn func(T) bool) { - result := v - for { - l := len(result) - for i := 0; i < len(result); i++ { - if fn(result[i]) { - result = append(result[0:i], result[i+1:]...) - } - } - if len(result) == l { - return - } - } -} diff --git a/cmd/testworkflow-init/data/emit.go b/cmd/testworkflow-init/instructions/emit.go similarity index 90% rename from cmd/testworkflow-init/data/emit.go rename to cmd/testworkflow-init/instructions/emit.go index 911ed287b68..5f67c182d24 100644 --- a/cmd/testworkflow-init/data/emit.go +++ b/cmd/testworkflow-init/instructions/emit.go @@ -1,4 +1,4 @@ -package data +package instructions import ( "encoding/json" @@ -88,9 +88,20 @@ func PrintHintDetails(ref string, name string, value interface{}) { fmt.Print(SprintHintDetails(ref, name, value)) } +func MayBeInstruction(line []byte) bool { + if len(line) >= len(InstructionPrefix) { + for i := 0; i < len(InstructionPrefix); i++ { + if line[i] != InstructionPrefix[i] { + return false + } + } + } + return true +} + func DetectInstruction(line []byte) (*Instruction, bool, error) { // Fast check to avoid regexes - if len(line) < 4 || string(line[0:len(InstructionPrefix)]) != InstructionPrefix || string(line[:len(InstructionPrefix)]) != InstructionPrefix { + if len(line) < 4 || !MayBeInstruction(line) { return nil, false, nil } // Parse the line diff --git a/cmd/testworkflow-init/main.go b/cmd/testworkflow-init/main.go index bfb00838efd..6d37c36437f 100644 --- a/cmd/testworkflow-init/main.go +++ b/cmd/testworkflow-init/main.go @@ -1,265 +1,382 @@ package main import ( - "fmt" + "errors" "os" "os/signal" - "path/filepath" "slices" - "strings" + "strconv" "syscall" "time" - "github.com/kballard/go-shellquote" + "github.com/gookit/color" + "github.com/kubeshop/testkube/cmd/testworkflow-init/commands" "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" "github.com/kubeshop/testkube/cmd/testworkflow-init/control" "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/obfuscator" + "github.com/kubeshop/testkube/cmd/testworkflow-init/orchestration" "github.com/kubeshop/testkube/cmd/testworkflow-init/output" - "github.com/kubeshop/testkube/cmd/testworkflow-init/run" + "github.com/kubeshop/testkube/pkg/expressions" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" +) + +const ( + SensitiveMask = "****" + SensitiveVisibleLastChars = 2 + SensitiveMinimumLength = 4 ) func main() { - if len(os.Args) < 2 { - output.Failf(output.CodeInputError, "missing step reference") - } - data.Step.Ref = os.Args[1] - - now := time.Now() - - // Load shared state - data.LoadState() - - // Initialize space for parsing args - config := map[string]string{} - computed := []string(nil) - conditions := []data.Rule(nil) - resulting := []data.Rule(nil) - timeouts := []data.Timeout(nil) - paused := false - toolkit := false - args := []string(nil) - - // Read arguments into the base data - for i := 2; i < len(os.Args); i += 2 { - if i+1 == len(os.Args) { - break - } - switch os.Args[i] { - case constants.ArgSeparator: - args = os.Args[i+1:] - i = len(os.Args) - case constants.ArgInit, constants.ArgInitLong: - data.Step.InitStatus = os.Args[i+1] - case constants.ArgCondition, constants.ArgConditionLong: - v := strings.SplitN(os.Args[i+1], "=", 2) - refs := strings.Split(v[0], ",") - if len(v) == 2 { - conditions = append(conditions, data.Rule{Expr: v[1], Refs: refs}) - } else { - conditions = append(conditions, data.Rule{Expr: "true", Refs: refs}) - } - case constants.ArgResult, constants.ArgResultLong: - v := strings.SplitN(os.Args[i+1], "=", 2) - refs := strings.Split(v[0], ",") - if len(v) == 2 { - resulting = append(resulting, data.Rule{Expr: v[1], Refs: refs}) - } else { - resulting = append(resulting, data.Rule{Expr: "true", Refs: refs}) - } - case constants.ArgTimeout, constants.ArgTimeoutLong: - v := strings.SplitN(os.Args[i+1], "=", 2) - if len(v) == 2 { - timeouts = append(timeouts, data.Timeout{Ref: v[0], Duration: v[1]}) - } else { - timeouts = append(timeouts, data.Timeout{Ref: v[0], Duration: ""}) - } - case constants.ArgComputeEnv, constants.ArgComputeEnvLong: - computed = append(computed, strings.Split(os.Args[i+1], ",")...) - case constants.ArgPaused, constants.ArgPausedLong: - paused = true - i-- - case constants.ArgNegative, constants.ArgNegativeLong: - config["negative"] = os.Args[i+1] - case constants.ArgWorkingDir, constants.ArgWorkingDirLong: - wd, err := filepath.Abs(os.Args[i+1]) - if err == nil { - _ = os.MkdirAll(wd, 0755) - err = os.Chdir(wd) - } else { - _ = os.MkdirAll(wd, 0755) - err = os.Chdir(os.Args[i+1]) - } - if err != nil { - fmt.Printf("warning: error using %s as working director: %s\n", os.Args[i+1], err.Error()) - } - case constants.ArgRetryCount: - config["retryCount"] = os.Args[i+1] - case constants.ArgRetryUntil: - config["retryUntil"] = os.Args[i+1] - case constants.ArgToolkit, constants.ArgToolkitLong: - toolkit = true - i-- - case constants.ArgDebug: - config["debug"] = os.Args[i+1] - default: - output.Failf(output.CodeInputError, "unknown parameter: %s", os.Args[i]) + // Force colors + color.ForceColor() + + // Configure standard output + stdout := output.Std + stdoutUnsafe := stdout.Direct() + + // Configure sensitive data obfuscation + stdout.SetSensitiveReplacer(obfuscator.ShowLastCharacters(SensitiveMask, SensitiveVisibleLastChars)) + orchestration.Setup.SetSensitiveWordMinimumLength(SensitiveMinimumLength) + + // Prepare empty state file if it doesn't exist + _, err := os.Stat(data.StatePath) + if errors.Is(err, os.ErrNotExist) { + stdout.Hint(data.InitStepName, constants.InstructionStart) + stdoutUnsafe.Print("Creating state...") + err := os.WriteFile(data.StatePath, nil, 0777) + if err != nil { + stdoutUnsafe.Error(" error\n") + output.ExitErrorf(data.CodeInternal, "failed to create state file: %s", err.Error()) } + os.Chmod(data.StatePath, 0777) + stdoutUnsafe.Print(" done\n") + } else if err != nil { + stdout.Hint(data.InitStepName, constants.InstructionStart) + stdoutUnsafe.Print("Accessing state...") + stdoutUnsafe.Error(" error\n") + output.ExitErrorf(data.CodeInternal, "cannot access state file: %s", err.Error()) } - // Clean up unnecessary variables for non-toolkit containers - if !toolkit { - _ = os.Unsetenv("TK_REF") + // Store the instructions in the state if they are provided + orchestration.Setup.UseBaseEnv() + stdout.SetSensitiveWords(orchestration.Setup.GetSensitiveWords()) + actionGroups := orchestration.Setup.GetActionGroups() + if actionGroups != nil { + stdoutUnsafe.Print("Initializing state...") + data.GetState().Actions = actionGroups + stdoutUnsafe.Print(" done\n") + + // Release the memory + actionGroups = nil } - // Configure PWD variable, to make it similar to shell environment variables - if os.Getenv("PWD") == "" { - cwd, err := os.Getwd() - if err == nil { - _ = os.Setenv("PWD", cwd) - } + // Distribute the details + currentContainer := lite.LiteActionContainer{} + + // Ensure there is a group index provided + if len(os.Args) != 2 { + output.ExitErrorf(data.CodeInternal, "invalid arguments provided - expected only one") } - // Compute environment variables - for _, name := range computed { - initial := os.Getenv(name) - value, err := data.Template(initial) - if err != nil { - output.Failf(output.CodeInputError, `resolving "%s" environment variable: %s: %s`, name, initial, err.Error()) - } - _ = os.Setenv(name, value) + // Determine group index to run + groupIndex, err := strconv.ParseInt(os.Args[1], 10, 32) + if err != nil { + output.ExitErrorf(data.CodeInputError, "invalid run group passed: %s", err.Error()) } - // Compute conditional steps - ignore errors initially, as the may be dependent on themselves - data.Iterate(conditions, func(c data.Rule) bool { - expr, err := data.Expression(c.Expr) - if err != nil { - return false - } - v, _ := expr.BoolValue() - if !v { - for _, r := range c.Refs { - data.State.GetStep(r).Skip(now) - } - } - return true - }) + // Handle aborting + stopSignal := make(chan os.Signal, 1) + signal.Notify(stopSignal, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-stopSignal + stdoutUnsafe.Print("The task was aborted.\n") + orchestration.Executions.Abort() + }() - // Fail invalid conditional steps - for _, c := range conditions { - _, err := data.Expression(c.Expr) + // Read the current state + state := data.GetState() + + // Run the control server + handlePause := func(ts time.Time, step *data.StepData) error { + if step.PausedStart != nil { + return nil + } + err = orchestration.Executions.Pause() if err != nil { - output.Failf(output.CodeInputError, "broken condition for refs: %s: %s: %s", strings.Join(c.Refs, ", "), c.Expr, err.Error()) + stdoutUnsafe.Warnf("warn: pause: %s\n", err.Error()) } - } - - // Start all acknowledged steps - for _, f := range resulting { - for _, r := range f.Refs { - if r != "" { - data.State.GetStep(r).Start(now) - } + orchestration.Pause(step, *step.StartedAt) + for _, parentRef := range step.Parents { + parent := state.GetStep(parentRef) + orchestration.Pause(parent, *step.StartedAt) } + return err } - for _, t := range timeouts { - if t.Ref != "" { - data.State.GetStep(t.Ref).Start(now) + handleResume := func(ts time.Time, step *data.StepData) error { + if step.PausedStart == nil { + return nil } - } - data.State.GetStep(data.Step.Ref).Start(now) - - // Register timeouts - for _, t := range timeouts { - err := data.State.GetStep(t.Ref).SetTimeoutDuration(now, t.Duration) + err = orchestration.Executions.Resume() if err != nil { - output.Failf(output.CodeInputError, "broken timeout for ref: %s: %s: %s", t.Ref, t.Duration, err.Error()) + stdoutUnsafe.Warnf("warn: resume: %s\n", err.Error()) } + orchestration.Resume(step, ts) + for _, parentRef := range step.Parents { + parent := state.GetStep(parentRef) + orchestration.Resume(parent, ts) + } + return err + } + controlSrv := control.NewServer(constants.ControlServerPort, control.ServerOptions{ + HandlePause: func(ts time.Time) error { + return handlePause(ts, state.GetStep(state.CurrentRef)) + }, + HandleResume: func(ts time.Time) error { + return handleResume(ts, state.GetStep(state.CurrentRef)) + }, + }) + _, err = controlSrv.Listen() + if err != nil { + output.ExitErrorf(data.CodeInternal, "Failed to start control server at port %d: %s\n", constants.ControlServerPort, err.Error()) } - // Save the resulting conditions - data.Config.Resulting = resulting + // Keep a list of paused steps for execution + delayedPauses := make([]string, 0) - // Don't call further if the step is already skipped - if data.State.GetStep(data.Step.Ref).Status == data.StepStatusSkipped { - if data.Config.Debug { - fmt.Printf("Skipped.\n") - } - data.Finish() - } + // Interpret the operations + for _, action := range state.GetActions(int(groupIndex)) { + switch action.Type() { + case lite.ActionTypeDeclare: + state.GetStep(action.Declare.Ref). + SetCondition(action.Declare.Condition). + SetParents(action.Declare.Parents) - // Handle pausing - if paused { - data.Step.Pause(now) - } + case lite.ActionTypePause: + state.GetStep(action.Pause.Ref). + SetPausedOnStart(true) - // Load the rest of the configuration - var err error - for k, v := range config { - config[k], err = data.Template(v) - if err != nil { - output.Failf(output.CodeInputError, `resolving "%s" param: %s: %s`, k, v, err.Error()) - } - } - data.LoadConfig(config) + case lite.ActionTypeResult: + state.GetStep(action.Result.Ref). + SetResult(action.Result.Value) - // Compute templates in the cmd/args - original := slices.Clone(args) - for i := range args { - args[i], err = data.Template(args[i]) - if err != nil { - output.Failf(output.CodeInputError, `resolving command: %s: %s`, shellquote.Join(original...), err.Error()) - } - } + case lite.ActionTypeTimeout: + state.GetStep(action.Timeout.Ref). + SetTimeout(action.Timeout.Timeout) - // Fail when there is nothing to run - if len(args) == 0 { - output.Failf(output.CodeNoCommand, "missing command to run") - } + case lite.ActionTypeRetry: + state.GetStep(action.Retry.Ref). + SetRetryPolicy(data.RetryPolicy{Count: action.Retry.Count, Until: action.Retry.Until}) - // Handle aborting - stopSignal := make(chan os.Signal, 1) - signal.Notify(stopSignal, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-stopSignal - fmt.Println("The task was aborted.") - data.Step.Status = data.StepStatusAborted - data.Step.ExitCode = output.CodeAborted - data.Finish() - }() + case lite.ActionTypeContainerTransition: + orchestration.Setup.SetConfig(action.Container.Config) + orchestration.Setup.AdvanceEnv() + stdout.SetSensitiveWords(orchestration.Setup.GetSensitiveWords()) + currentContainer = *action.Container + + case lite.ActionTypeCurrentStatus: + state.SetCurrentStatus(*action.CurrentStatus) + + case lite.ActionTypeStart: + if *action.Start == "" { + continue + } + step := state.GetStep(*action.Start) + orchestration.Start(step) + + // Determine if the step should be skipped + executable, err := step.ResolveCondition() + if err != nil { + output.ExitErrorf(data.CodeInternal, "failed to determine condition of '%s' step: %s: %v", *action.Start, step.Condition, err.Error()) + } + if !executable { + step.SetStatus(data.StepStatusSkipped) + + // Skip all the children + for _, v := range state.Steps { + if slices.Contains(v.Parents, step.Ref) { + v.SetStatus(data.StepStatusSkipped) + } + } + } + + // Delay the pause until next children execution + if !step.IsFinished() && step.PausedOnStart { + delayedPauses = append(delayedPauses, state.CurrentRef) + } + + case lite.ActionTypeEnd: + if *action.End == "" { + continue + } + step := state.GetStep(*action.End) + if step.Status == nil { + status, err := step.ResolveResult() + if err != nil { + output.ExitErrorf(data.CodeInternal, "failed to determine result of '%s' step: %s: %v", *action.End, step.Result, err.Error()) + } + step.SetStatus(status) + } + orchestration.End(step) - // Handle timeouts. - // Ignores time when the step was paused. - for _, t := range timeouts { - go func(ref string) { - start := now - timeout := data.State.GetStep(ref).TimeoutAt.Sub(start) + case lite.ActionTypeSetup: + orchestration.Setup.UseEnv(constants.EnvGroupDebug) + stdout.SetSensitiveWords(orchestration.Setup.GetSensitiveWords()) + step := state.GetStep(data.InitStepName) + err := commands.Setup(*action.Setup) + if err == nil { + step.SetStatus(data.StepStatusPassed) + } else { + step.SetStatus(data.StepStatusFailed) + } + orchestration.End(step) + if err != nil { + os.Exit(1) + } + + case lite.ActionTypeExecute: + // Ignore running when the step is already resolved (= skipped) + step := state.GetStep(action.Execute.Ref) + if step.IsFinished() { + continue + } + + // Ignore when it is aborted + if orchestration.Executions.IsAborted() { + step.SetStatus(data.StepStatusAborted) + continue + } + + // Configure the environment + orchestration.Setup.UseCurrentEnv() + if !action.Execute.Toolkit { + _ = os.Unsetenv("TK_REF") + } + + // List all the parents + leaf := []*data.StepData{step} + for i := range step.Parents { + leaf = append(leaf, state.GetStep(step.Parents[i])) + } + + // Compute the pause + paused := make([]string, 0) + if slices.Contains(delayedPauses, action.Execute.Ref) { + paused = append(paused, action.Execute.Ref) + } + for _, parentRef := range step.Parents { + if slices.Contains(delayedPauses, parentRef) { + paused = append(paused, parentRef) + } + } + + // Pause + if len(paused) > 0 { + delayedPauses = nil + _ = handlePause(*step.StartedAt, step) + } + + // Configure timeout finalizer + finalizeTimeout := func() { + // Check timed out steps in leaf + timedOut := orchestration.GetTimedOut(leaf...) + if timedOut == nil { + return + } + + // Iterate over timed out step + for _, r := range timedOut { + r.SetStatus(data.StepStatusTimeout) + sub := state.GetSubSteps(r.Ref) + for i := range sub { + if sub[i].IsFinished() { + continue + } + if sub[i].IsStarted() { + sub[i].SetStatus(data.StepStatusTimeout) + } else { + sub[i].SetStatus(data.StepStatusSkipped) + } + } + stdoutUnsafe.Println("Timed out.") + } + _ = orchestration.Executions.Kill() + + return + } + + // Handle immediate timeout + finalizeTimeout() + + // Avoid execution if it's finished + if step.IsFinished() { + continue + } + + // Iterate retries for { - time.Sleep(timeout) - took := data.Step.Took(start) - if took < timeout { - timeout -= took - continue + // Reset the status + step.Status = nil + + // Ignore when it is aborted + if orchestration.Executions.IsAborted() { + step.SetStatus(data.StepStatusAborted) + break } - fmt.Printf("Timed out.\n") - data.State.GetStep(ref).SetStatus(data.StepStatusTimeout) - data.Step.Status = data.StepStatusTimeout - data.Step.ExitCode = output.CodeTimeout - data.Finish() + + // Register timeouts + stopTimeoutWatcher := orchestration.WatchTimeout(finalizeTimeout, leaf...) + + // Run the command + commands.Run(*action.Execute, currentContainer) + + // Stop timer listener + stopTimeoutWatcher() + + // Ensure there won't be any hanging processes after the command is executed + _ = orchestration.Executions.Kill() + + // TODO: Handle retry policy in tree independently + // Verify if there may be any other iteration + if step.Iteration >= step.Retry.Count { + break + } + + // Verify if the retry condition is matching + until := step.Retry.Until + if until == "" { + until = "passed" + } + expr, err := expressions.CompileAndResolve(until, data.LocalMachine, data.GetInternalTestWorkflowMachine(), expressions.FinalizerFail) + if err != nil { + stdout.Printf("failed to execute retry condition: %s: %s\n", until, err.Error()) + break + } + shouldStop, _ := expr.Static().BoolValue() + if shouldStop { + break + } + + // Continue with the next iteration + step.Iteration++ + stdout.HintDetails(step.Ref, constants.InstructionIteration, step.Iteration) + stdoutUnsafe.Printf("\nExit code: %d • Retrying: attempt #%d (of %d):\n", step.ExitCode, step.Iteration, step.Retry.Count) } - }(t.Ref) - } + } - // Run the control server - controlSrv := control.NewServer(constants.ControlServerPort, data.Step) - _, err = controlSrv.Listen() - if err != nil { - fmt.Printf("Failed to start control server at port %d: %s\n", constants.ControlServerPort, err.Error()) - os.Exit(int(output.CodeInternal)) + // Save the status after each action + data.SaveState() } - // Start the task - data.Step.Executed = true - run.Run(args[0], args[1:]) + // Ensure the latest state is saved + data.SaveState() - os.Exit(0) + // Stop the container after all the instructions are interpret + _ = orchestration.Executions.Kill() + if orchestration.Executions.IsAborted() { + os.Exit(int(data.CodeAborted)) + } else { + os.Exit(0) + } } diff --git a/cmd/testworkflow-init/obfuscator/obfuscator.go b/cmd/testworkflow-init/obfuscator/obfuscator.go new file mode 100644 index 00000000000..5f5b1b3dab2 --- /dev/null +++ b/cmd/testworkflow-init/obfuscator/obfuscator.go @@ -0,0 +1,171 @@ +package obfuscator + +import ( + "io" + "sync" + "unsafe" +) + +type obfuscator struct { + dst io.Writer + replacer func([]byte) []byte + + rootNode *SearchTree + currentNode *SearchTree + currentBuffer []byte + currentPosition int + currentEnd int + + writeMu sync.Mutex +} + +func New(dst io.Writer, replacer func([]byte) []byte, words []string) *obfuscator { + // Build radix-tree for checking the words + rootNode := NewSearchTree() + for _, word := range words { + rootNode.Append(unsafe.Slice(unsafe.StringData(word), len(word))) + } + + return &obfuscator{ + dst: dst, + replacer: replacer, + rootNode: rootNode, + currentNode: rootNode, + currentEnd: -1, + } +} + +func (s *obfuscator) SetSensitiveReplacer(replacer func([]byte) []byte) { + s.replacer = replacer +} + +func (s *obfuscator) SetSensitiveWords(words []string) { + rootNode := NewSearchTree() + for _, word := range words { + rootNode.Append(unsafe.Slice(unsafe.StringData(word), len(word))) + } + + s.rootNode = rootNode + s.currentPosition = 0 + s.currentEnd = -1 + s.currentNode = s.rootNode +} + +func (s *obfuscator) resetBuffer() { + s.currentBuffer = nil + s.currentPosition = 0 + s.currentEnd = -1 + s.currentNode = s.rootNode +} + +func (s *obfuscator) Write(p []byte) (n int, err error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + + // Read data from the previous cycle + if s.currentBuffer != nil { + p = append(s.currentBuffer, p...) + } + currentPosition := s.currentPosition + currentStart := currentPosition + currentEnd := s.currentEnd + currentNode := s.currentNode + s.resetBuffer() + + var nn int + for currentPosition < len(p) { + end, depth, mayContinue, current := currentNode.Hits(p, currentPosition) + + // Continue with next characters when there was no hit + if end == -1 && !mayContinue { + currentPosition++ + continue + } + + // Flush the non-sensitive contents if there is a potential hit + if currentPosition != currentStart { + currentStart = 0 + nn, err = s.dst.Write(p[:currentPosition]) + n += nn + if err != nil { + return + } + + // Calibrate data without the non-sensitive content + p = p[currentPosition:] + depth -= currentPosition + if end != -1 { + end -= currentPosition + } + currentPosition = 0 + } + + // Adjust the current end character + if end != -1 { + currentEnd = end + } + + // The sensitive word may still not be finished in this buffer + if mayContinue { + // Buffer data + s.currentBuffer = p + s.currentNode = current + s.currentPosition = depth + s.currentEnd = currentEnd + + // End a call + return n + len(p), nil + } + + // Flush the acknowledged sensitive data + replacement := s.replacer(p[:currentEnd]) + nn, err = s.dst.Write(replacement) + nn += currentEnd - len(replacement) + n += nn + p = p[currentEnd:] + currentEnd = -1 + currentPosition = 0 + currentNode = s.rootNode + if err != nil { + return n, err + } + } + + // Write the rest of data + if len(p) > 0 { + nn, err = s.dst.Write(p) + return n + nn, err + } + return n, nil +} + +func (s *obfuscator) Flush() error { + for s.currentBuffer != nil { + // Flush all if there is no smaller sensitive chunk + if s.currentEnd == -1 { + left := s.currentBuffer + s.resetBuffer() + + _, err := s.dst.Write(left) + if err != nil { + return err + } + return nil + } + + // Flush the next sensitive part + _, err := s.dst.Write(s.replacer(s.currentBuffer[:s.currentEnd])) + if err != nil { + return err + } + + // Write the remaining part + left := s.currentBuffer[s.currentEnd:] + s.resetBuffer() + _, err = s.Write(left) + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/testworkflow-init/obfuscator/obfuscator_test.go b/cmd/testworkflow-init/obfuscator/obfuscator_test.go new file mode 100644 index 00000000000..ee84b7fa71b --- /dev/null +++ b/cmd/testworkflow-init/obfuscator/obfuscator_test.go @@ -0,0 +1,136 @@ +package obfuscator + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestObfuscator_Full(t *testing.T) { + buf := &bytes.Buffer{} + passthrough := New(buf, FullReplace("*****"), []string{ + "sensitive", + "scope", + "testKube", + }) + + _, _ = passthrough.Write([]byte("there is some sensitive content in scope of testkube")) + + result, err := io.ReadAll(buf) + + assert.NoError(t, err) + assert.Equal(t, "there is some ***** content in ***** of testkube", string(result)) +} + +func TestObfuscator_End(t *testing.T) { + buf := &bytes.Buffer{} + passthrough := New(buf, FullReplace("*****"), []string{ + "sensitive", + "scope", + "testKube", + }) + + _, _ = passthrough.Write([]byte("there is some sensitive")) + + result, err := io.ReadAll(buf) + + assert.NoError(t, err) + assert.Equal(t, "there is some *****", string(result)) +} + +func TestObfuscator_Partial(t *testing.T) { + buf := &bytes.Buffer{} + passthrough := New(buf, FullReplace("*****"), []string{ + "sensitive", + "scope", + }) + + _, _ = passthrough.Write([]byte("there is some sensitiv")) + _, _ = passthrough.Write([]byte("e content in scope of testkube")) + + result, err := io.ReadAll(buf) + + assert.NoError(t, err) + assert.Equal(t, "there is some ***** content in ***** of testkube", string(result)) +} + +func TestObfuscator_FlushLowerHit(t *testing.T) { + buf := &bytes.Buffer{} + passthrough := New(buf, FullReplace("*****"), []string{ + "sensitive", + "sens", + }) + + _, _ = passthrough.Write([]byte("sensitiv")) + passthrough.Flush() + + result, err := io.ReadAll(buf) + + assert.NoError(t, err) + assert.Equal(t, "*****itiv", string(result)) +} + +func TestObfuscator_FlushNoHit(t *testing.T) { + buf := &bytes.Buffer{} + passthrough := New(buf, FullReplace("*****"), []string{ + "sensitive", + }) + + _, _ = passthrough.Write([]byte("sensitiv")) + passthrough.Flush() + + result, err := io.ReadAll(buf) + + assert.NoError(t, err) + assert.Equal(t, "sensitiv", string(result)) +} + +func TestObfuscator_FlushDoubleHit(t *testing.T) { + buf := &bytes.Buffer{} + passthrough := New(buf, FullReplace("*****"), []string{ + "sensitive", + "sens", + "tiv", + }) + + _, _ = passthrough.Write([]byte("sensitiv")) + passthrough.Flush() + + result, err := io.ReadAll(buf) + + assert.NoError(t, err) + assert.Equal(t, "*****i*****", string(result)) +} + +func TestObfuscator_Order(t *testing.T) { + buf := &bytes.Buffer{} + passthrough := New(buf, FullReplace("*****"), []string{ + "sensitive", + "sens", + }) + + _, _ = passthrough.Write([]byte("there is some sensitive content in scope of testkube")) + + result, err := io.ReadAll(buf) + + assert.NoError(t, err) + assert.Equal(t, "there is some ***** content in scope of testkube", string(result)) +} + +func TestObfuscator_Multiple(t *testing.T) { + buf := &bytes.Buffer{} + passthrough := New(buf, FullReplace("*****"), []string{ + "hello world", + "hello", + "blah", + }) + + _, _ = passthrough.Write([]byte("hello world there hello hahahaha helblah in there blah")) + + result, err := io.ReadAll(buf) + + assert.NoError(t, err) + assert.Equal(t, "***** there ***** hahahaha hel***** in there *****", string(result)) +} diff --git a/cmd/testworkflow-init/obfuscator/replacers.go b/cmd/testworkflow-init/obfuscator/replacers.go new file mode 100644 index 00000000000..6db85dc00a3 --- /dev/null +++ b/cmd/testworkflow-init/obfuscator/replacers.go @@ -0,0 +1,18 @@ +package obfuscator + +func FullReplace(value string) func([]byte) []byte { + replacement := []byte(value) + return func(_ []byte) []byte { + return replacement + } +} + +func ShowLastCharacters(prefix string, visibleChars int) func([]byte) []byte { + replacement := []byte(prefix) + return func(v []byte) []byte { + if len(v) <= visibleChars { + return v + } + return append(replacement, v[len(v)-visibleChars:]...) + } +} diff --git a/cmd/testworkflow-init/obfuscator/replacers_test.go b/cmd/testworkflow-init/obfuscator/replacers_test.go new file mode 100644 index 00000000000..37a3a56805e --- /dev/null +++ b/cmd/testworkflow-init/obfuscator/replacers_test.go @@ -0,0 +1,19 @@ +package obfuscator + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFullReplace(t *testing.T) { + assert.Equal(t, []byte("abc"), FullReplace("abc")([]byte("def"))) +} + +func TestLastCharacters(t *testing.T) { + assert.Equal(t, []byte("***"), ShowLastCharacters("***", 0)([]byte("def"))) + assert.Equal(t, []byte("***f"), ShowLastCharacters("***", 1)([]byte("def"))) + assert.Equal(t, []byte("***ef"), ShowLastCharacters("***", 2)([]byte("def"))) + assert.Equal(t, []byte("def"), ShowLastCharacters("***", 3)([]byte("def"))) + assert.Equal(t, []byte("def"), ShowLastCharacters("***", 4)([]byte("def"))) +} diff --git a/cmd/testworkflow-init/obfuscator/searchtree.go b/cmd/testworkflow-init/obfuscator/searchtree.go new file mode 100644 index 00000000000..ede5239a8e1 --- /dev/null +++ b/cmd/testworkflow-init/obfuscator/searchtree.go @@ -0,0 +1,52 @@ +package obfuscator + +type SearchTree struct { + tree map[byte]*SearchTree + last bool +} + +func NewSearchTree() *SearchTree { + return &SearchTree{tree: map[byte]*SearchTree{}} +} + +func (r *SearchTree) HasChildren() bool { + return len(r.tree) != 0 +} + +func (r *SearchTree) Append(word []byte) { + node := r + for i := 0; i < len(word); i++ { + if _, ok := node.tree[word[i]]; !ok { + node.tree[word[i]] = NewSearchTree() + } + node = node.tree[word[i]] + } + node.last = true +} + +func (r *SearchTree) Hits(b []byte, index int) (int, int, bool, *SearchTree) { + // It may continue, unless some byte is not found + end := -1 + mayContinue := true + current := r + + // Go in depth + for ; index < len(b); index++ { + // Go into next byte + if v, ok := current.tree[b[index]]; ok { + current = v + if current.last { + end = index + 1 + } + continue + } + + // Continuation not found + mayContinue = false + break + } + if mayContinue && !current.HasChildren() { + mayContinue = false + } + return end, index, mayContinue, current +} diff --git a/cmd/testworkflow-init/orchestration/control.go b/cmd/testworkflow-init/orchestration/control.go new file mode 100644 index 00000000000..a2c5e84dda1 --- /dev/null +++ b/cmd/testworkflow-init/orchestration/control.go @@ -0,0 +1,41 @@ +package orchestration + +import ( + "encoding/json" + "time" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" + "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" +) + +func Start(step *data.StepData) { + state := data.GetState() + state.CurrentRef = step.Ref + startedAt := time.Now() + step.StartedAt = &startedAt + instructions.PrintHint(step.Ref, constants.InstructionStart) +} + +func Pause(step *data.StepData, ts time.Time) { + step.RegisterPauseStart(ts) + instructions.PrintHintDetails(step.Ref, constants.InstructionPause, ts.UTC().Format(constants.PreciseTimeFormat)) +} + +func Resume(step *data.StepData, ts time.Time) { + step.RegisterPauseEnd(ts) + instructions.PrintHintDetails(step.Ref, constants.InstructionResume, ts.UTC().Format(constants.PreciseTimeFormat)) +} + +func FinishExecution(step *data.StepData, result constants.ExecutionResult) { + instructions.PrintHintDetails(step.Ref, constants.InstructionExecution, result) +} + +func End(step *data.StepData) { + if !step.IsFinished() { + v, e := json.Marshal(step) + output.ExitErrorf(data.CodeInternal, "cannot mark unfinished step as finished: %s, %v", string(v), e) + } + instructions.PrintHintDetails(step.Ref, constants.InstructionEnd, *step.Status) +} diff --git a/cmd/testworkflow-init/orchestration/executions.go b/cmd/testworkflow-init/orchestration/executions.go new file mode 100644 index 00000000000..741cd086bca --- /dev/null +++ b/cmd/testworkflow-init/orchestration/executions.go @@ -0,0 +1,206 @@ +package orchestration + +import ( + "io" + "os" + "os/exec" + "sync" + "sync/atomic" + + "github.com/pkg/errors" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" +) + +var ( + Executions = newExecutionGroup(data.NewOutputProcessor(output.Std), output.Std) +) + +type executionResult struct { + ExitCode uint8 + Aborted bool +} + +type executionGroup struct { + aborted atomic.Bool + outStream io.Writer + errStream io.Writer + + executions []*execution + executionsMu sync.Mutex + + paused atomic.Bool + pauseMu sync.Mutex +} + +func newExecutionGroup(outStream io.Writer, errStream io.Writer) *executionGroup { + return &executionGroup{ + outStream: outStream, + errStream: errStream, + } +} + +func (e *executionGroup) Create(cmd string, args []string) *execution { + // Instantiate the execution + ex := &execution{group: e} + ex.cmd = exec.Command(cmd, args...) + ex.cmd.Stdout = e.outStream + ex.cmd.Stderr = e.errStream + + // Append to the list TODO: delete that after finish + e.executionsMu.Lock() + e.executions = append(e.executions, ex) + e.executionsMu.Unlock() + + return ex +} + +func (e *executionGroup) Pause() (err error) { + // Lock running + swapped := e.paused.CompareAndSwap(false, true) + if !swapped { + return nil + } + e.pauseMu.Lock() + + // Lock the executions state + e.executionsMu.Lock() + defer e.executionsMu.Unlock() + + // Retrieve all started processes + ps, totalFailure, err := processes() + if totalFailure { + return errors.Wrap(err, "failed to pause: failed to list processes") + } + if err != nil { + output.Std.Direct().Warnf("warn: failed to pause: failed to list some processes: %v\n", err) + } + + // Ignore the init process, to not suspend it accidentally + ps.VirtualizePath(int32(os.Getpid())) + err = ps.Suspend() + return errors.Wrap(err, "failed to pause") +} + +func (e *executionGroup) Resume() (err error) { + // Lock running + swapped := e.paused.CompareAndSwap(true, false) + if !swapped { + return nil + } + defer e.pauseMu.Unlock() + + // Lock the executions state + e.executionsMu.Lock() + defer e.executionsMu.Unlock() + + // Retrieve all started processes + ps, totalFailure, err := processes() + if totalFailure { + return errors.Wrap(err, "failed to resume: failed to list processes") + } + if err != nil { + output.Std.Direct().Warnf("warn: failed to resume: failed to list some processes: %v\n", err) + } + + // Ignore the init process, to not suspend it accidentally + ps.VirtualizePath(int32(os.Getpid())) + err = ps.Resume() + return errors.Wrap(err, "failed to resume") +} + +func (e *executionGroup) Kill() (err error) { + // Lock the executions state + e.executionsMu.Lock() + defer e.executionsMu.Unlock() + + // Retrieve all started processes + ps, totalFailure, err := processes() + if totalFailure { + return errors.Wrap(err, "failed to kill: failed to list processes") + } + if err != nil { + output.Std.Direct().Warnf("warn: failed to kill: failed to list some processes: %v\n", err.Error()) + } + + // Ignore the init process, to not suspend it accidentally + ps.VirtualizePath(int32(os.Getpid())) + err = ps.Kill() + return errors.Wrap(err, "failed to kill") +} + +func (e *executionGroup) Abort() { + e.aborted.Store(true) + _ = e.Kill() +} + +func (e *executionGroup) IsAborted() bool { + return e.aborted.Load() +} + +type execution struct { + cmd *exec.Cmd + cmdMu sync.Mutex + group *executionGroup +} + +func (e *execution) Run() (*executionResult, error) { + // Immediately fail when aborted + if e.group.aborted.Load() { + return &executionResult{Aborted: true, ExitCode: data.CodeAborted}, nil + } + + // Ensure it's not paused + e.group.pauseMu.Lock() + + // Ensure the command is not running multiple times + e.cmdMu.Lock() + + // Immediately fail when aborted + if e.group.aborted.Load() { + e.group.pauseMu.Unlock() + e.cmdMu.Unlock() + return &executionResult{Aborted: true, ExitCode: data.CodeAborted}, nil + } + + // Initialize local state + var exitCode uint8 + + // Run the command + err := e.cmd.Start() + if err == nil { + e.group.pauseMu.Unlock() + e.cmdMu.Unlock() + _, exitCode = getProcessStatus(e.cmd.Wait()) + } else { + e.group.pauseMu.Unlock() + e.cmdMu.Unlock() + _, exitCode = getProcessStatus(err) + } + + // Clean up + e.cmdMu.Lock() + e.cmd = nil + e.cmdMu.Unlock() + + // Fail when aborted + if e.group.aborted.Load() { + return &executionResult{Aborted: true, ExitCode: data.CodeAborted}, nil + } + + return &executionResult{ExitCode: exitCode}, nil +} + +func getProcessStatus(err error) (bool, uint8) { + if err == nil { + return true, 0 + } + if e, ok := err.(*exec.ExitError); ok { + if e.ProcessState != nil { + return false, uint8(e.ProcessState.ExitCode()) + } + return false, 1 + } + return false, 1 +} diff --git a/cmd/testworkflow-init/orchestration/processes.go b/cmd/testworkflow-init/orchestration/processes.go new file mode 100644 index 00000000000..559d849f174 --- /dev/null +++ b/cmd/testworkflow-init/orchestration/processes.go @@ -0,0 +1,164 @@ +package orchestration + +import ( + errors2 "errors" + + "github.com/pkg/errors" + gopsutil "github.com/shirou/gopsutil/v3/process" +) + +type processNode struct { + pid int32 + nodes map[*processNode]struct{} +} + +func (p *processNode) Find(pid int32) []*processNode { + if p.pid == pid { + return []*processNode{p} + } + // Try to find directly + for n := range p.nodes { + if n.pid == pid { + return append([]*processNode{p}, n) + } + } + + // Try to find in the children + for n := range p.nodes { + found := n.Find(pid) + if found != nil { + return append([]*processNode{p}, found...) + } + } + + return nil +} + +func (p *processNode) VirtualizePath(pid int32) { + path := p.Find(pid) + if path == nil { + return + } + + // Cannot virtualize itself + if len(path) == 1 { + return + } + + // Virtualize recursively + for i := 1; i < len(path); i++ { + delete(path[0].nodes, path[i]) + for node := range path[i].nodes { + path[0].nodes[node] = struct{}{} + } + } +} + +// Suspend all the processes in group, starting from top +func (p *processNode) Suspend() error { + errs := make([]error, 0) + if p.pid != -1 { + err := (&gopsutil.Process{Pid: p.pid}).Suspend() + if err != nil { + errs = append(errs, err) + } + } + for node := range p.nodes { + err := node.Suspend() + if err != nil { + errs = append(errs, err) + } + } + if len(errs) == 0 { + return nil + } + if p.pid == -1 { + return errors.Wrap(errors2.Join(errs...), "suspending processes") + } + return errors.Wrapf(errors2.Join(errs...), "suspending process %d", p.pid) +} + +// Resume all the processes in group, starting from bottom +func (p *processNode) Resume() error { + errs := make([]error, 0) + for node := range p.nodes { + err := node.Resume() + if err != nil { + errs = append(errs, err) + } + } + if p.pid != -1 { + err := (&gopsutil.Process{Pid: p.pid}).Resume() + if err != nil { + errs = append(errs, err) + } + } + if len(errs) == 0 { + return nil + } + if p.pid == -1 { + return errors.Wrap(errors2.Join(errs...), "suspending processes") + } + return errors.Wrapf(errors2.Join(errs...), "suspending process %d", p.pid) +} + +// Kill all the processes in group, starting from top +func (p *processNode) Kill() error { + errs := make([]error, 0) + if p.pid != -1 { + return errors.Wrap((&gopsutil.Process{Pid: p.pid}).Kill(), "killing processes") + } + for node := range p.nodes { + err := node.Kill() + if err != nil { + errs = append(errs, err) + } + } + if len(errs) == 0 { + return nil + } + return errors.Wrapf(errors2.Join(errs...), "killing process %d", p.pid) +} + +func processes() (*processNode, bool, error) { + // Get list of processes + list, err := gopsutil.Processes() + if err != nil { + return nil, true, errors.Wrapf(err, "failed to list processes") + } + + // Put all the processes in the map + detached := map[int32]struct{}{} + r := map[int32]*processNode{} + var errs []error + for _, p := range list { + detached[p.Pid] = struct{}{} + r[p.Pid] = &processNode{pid: p.Pid, nodes: map[*processNode]struct{}{}} + } + + // Create tree of processes + for _, p := range list { + ppid, err := p.Ppid() + if err != nil { + errs = append(errs, err) + continue + } + if r[ppid] == nil || ppid == p.Pid { + continue + } + r[ppid].nodes[r[p.Pid]] = struct{}{} + delete(detached, p.Pid) + } + + // Make virtual root of detached processes + root := &processNode{pid: -1, nodes: make(map[*processNode]struct{}, len(detached))} + for pid := range detached { + root.nodes[r[pid]] = struct{}{} + } + + // Return info + if len(errs) > 0 { + err = errors.Wrapf(errs[0], "failed to load %d/%d processes", len(errs), len(r)) + } + return root, len(errs) == len(r), err +} diff --git a/cmd/testworkflow-init/orchestration/setup.go b/cmd/testworkflow-init/orchestration/setup.go new file mode 100644 index 00000000000..a92a1843449 --- /dev/null +++ b/cmd/testworkflow-init/orchestration/setup.go @@ -0,0 +1,237 @@ +package orchestration + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" + "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/output" + "github.com/kubeshop/testkube/pkg/expressions" + "github.com/kubeshop/testkube/pkg/expressions/libs" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" +) + +var ( + scopedRegex = regexp.MustCompile(`^_(00|01|\d|[1-9]\d*)(C)?(S?)_`) + Setup = newSetup() + defaultWorkingDir = getWorkingDir() + commonSensitiveVariables = []string{ + "TK_C_KEY", // Cloud API key + "TK_OS_ACCESSKEY", // Object Storage Access Key + "TK_OS_SECRETKEY", // Object Storage Secret Key + "TK_OS_TOKEN", // Object Storage Token + "TK_GIT_USERNAME", // Git Username + "TK_GIT_TOKEN", // Git Token + "TK_SSH_KEY", // Git SSH Key + } +) + +func getWorkingDir() string { + wd, _ := os.Getwd() + if wd == "" { + return "/" + } + return wd +} + +type setup struct { + envBase map[string]string + envGroups map[string]map[string]string + envGroupsComputed map[string]map[string]struct{} + envGroupsSensitive map[string]map[string]struct{} + envCurrentGroup int + envSelectedGroup string + minSensitiveWordLength int +} + +func newSetup() *setup { + c := &setup{ + envBase: map[string]string{}, + envGroups: map[string]map[string]string{}, + envGroupsComputed: map[string]map[string]struct{}{}, + envGroupsSensitive: map[string]map[string]struct{}{}, + envCurrentGroup: -1, + minSensitiveWordLength: 1, + } + c.initialize() + return c +} + +func (c *setup) initialize() { + // Iterate over the environment variables to group them + for _, item := range os.Environ() { + match := scopedRegex.FindStringSubmatch(item) + key, value, _ := strings.Cut(item, "=") + if match == nil { + c.envBase[key] = value + continue + } + + if c.envGroups[match[1]] == nil { + c.envGroups[match[1]] = map[string]string{} + c.envGroupsComputed[match[1]] = map[string]struct{}{} + c.envGroupsSensitive[match[1]] = map[string]struct{}{} + } + c.envGroups[match[1]][key[len(match[0]):]] = value + if match[2] == "C" { + c.envGroupsComputed[match[1]][key[len(match[0]):]] = struct{}{} + } + if match[3] == "S" { + c.envGroupsSensitive[match[1]][key[len(match[0]):]] = struct{}{} + } + os.Unsetenv(key) + } +} + +func (c *setup) UseBaseEnv() { + os.Clearenv() + for k, v := range c.envBase { + os.Setenv(k, v) + } +} + +func (c *setup) SetSensitiveWordMinimumLength(length int) { + if length > 0 { + c.minSensitiveWordLength = length + } else { + c.minSensitiveWordLength = 1 + } +} + +func (c *setup) GetSensitiveWords() []string { + words := make([]string, 0) + for _, name := range commonSensitiveVariables { + value := os.Getenv(name) + if len(value) < c.minSensitiveWordLength { + continue + } + words = append(words, value) + } + for k := range c.envBase { + value := os.Getenv(k) + if len(value) < c.minSensitiveWordLength { + continue + } + if _, ok := c.envGroupsSensitive[c.envSelectedGroup][k]; ok { + words = append(words, value) + } + } + for k := range c.envGroups[c.envSelectedGroup] { + value := os.Getenv(k) + if len(value) < c.minSensitiveWordLength { + continue + } + if _, ok := c.envGroupsSensitive[c.envSelectedGroup][k]; ok { + words = append(words, value) + } + } + return words +} + +func (c *setup) GetActionGroups() (actions [][]lite.LiteAction) { + serialized := c.envGroups[constants.EnvGroupActions][constants.EnvActions] + if serialized == "" { + return + } + err := json.Unmarshal([]byte(serialized), &actions) + if err != nil { + panic(fmt.Sprintf("failed to read the actions from Pod: %s", err.Error())) + } + return actions +} + +func (c *setup) UseEnv(group string) { + c.UseBaseEnv() + c.envSelectedGroup = group + + envTemplates := map[string]string{} + envResolutions := map[string]expressions.Expression{} + for k, v := range c.envGroups[group] { + if _, ok := c.envGroupsComputed[group][k]; ok { + envTemplates[k] = v + } else { + os.Setenv(k, v) + } + } + + // Configure PWD variable, to make it similar to shell environment variables + cwd := getWorkingDir() + if os.Getenv("PWD") == "" { + os.Setenv("PWD", cwd) + } + + // Ensure the built-in binaries are available + if os.Getenv("PATH") == "" { + os.Setenv("PATH", data.InternalBinPath) + } else { + os.Setenv("PATH", fmt.Sprintf("%s:%s", os.Getenv("PATH"), data.InternalBinPath)) + } + + // Compute dynamic environment variables + addonMachine := expressions.CombinedMachines(data.RefSuccessMachine, data.AliasMachine, data.StateMachine, libs.NewFsMachine(os.DirFS("/"), cwd)) + localEnvMachine := expressions.NewMachine(). + RegisterAccessorExt(func(accessorName string) (interface{}, bool, error) { + if !strings.HasPrefix(accessorName, "env.") { + return nil, false, nil + } + name := accessorName[4:] + if v, ok := envResolutions[name]; ok { + return v, true, nil + } else if _, ok := envTemplates[name]; ok { + result, err := expressions.CompileAndResolveTemplate(envTemplates[name], addonMachine) + if err != nil { + envResolutions[name] = result + } + return result, true, err + } + return os.Getenv(name), true, nil + }) + for name, expr := range envTemplates { + value, err := expressions.CompileAndResolveTemplate(expr, localEnvMachine, addonMachine, expressions.FinalizerFail) + if err != nil { + output.ExitErrorf(data.CodeInputError, "failed to compute '%s' environment variable: %s", name, err.Error()) + } + str, _ := value.Static().StringValue() + os.Setenv(name, str) + } +} + +func (c *setup) UseCurrentEnv() { + c.UseEnv(fmt.Sprintf("%d", c.envCurrentGroup)) +} + +func (c *setup) AdvanceEnv() { + c.envCurrentGroup++ + c.UseCurrentEnv() +} + +func (c *setup) SetWorkingDir(workingDir string) { + _ = os.Chdir(defaultWorkingDir) + if workingDir == "" { + return + } + wd, err := filepath.Abs(workingDir) + if err != nil { + wd = workingDir + _ = os.MkdirAll(wd, 0755) + } else { + err = os.MkdirAll(wd, 0755) + } + + if err != nil { + output.Std.Direct().Warnf("warn: error using %s as working directory: %s\n", workingDir, err.Error()) + } +} + +func (c *setup) SetConfig(config lite.LiteContainerConfig) { + if config.WorkingDir == nil || *config.WorkingDir == "" { + c.SetWorkingDir("") + } else { + c.SetWorkingDir(*config.WorkingDir) + } +} diff --git a/cmd/testworkflow-init/orchestration/timeout.go b/cmd/testworkflow-init/orchestration/timeout.go new file mode 100644 index 00000000000..b4d500c4d91 --- /dev/null +++ b/cmd/testworkflow-init/orchestration/timeout.go @@ -0,0 +1,78 @@ +package orchestration + +import ( + "context" + "sync" + "sync/atomic" + "time" +) + +type Timeoutable interface { + TimeLeft(ts time.Time) *time.Duration + IsFinished() bool +} + +func GetTimedOut[T Timeoutable](objects ...T) (result []T) { + now := time.Now() + for _, t := range objects { + // Check if that is still timeoutable + left := t.TimeLeft(now) + if left != nil && !t.IsFinished() && *left <= 0 { + result = append(result, t) + } + } + return result +} + +func WatchTimeout[T Timeoutable](handler func(), objects ...T) func() { + ctx, ctxCancel := context.WithCancel(context.Background()) + var fired atomic.Bool + run := func() { + swapped := fired.CompareAndSwap(false, true) + if swapped { + handler() + } + } + + // Set up timers for all timeoutable objects + var wg sync.WaitGroup + wg.Add(len(objects)) + for i := range objects { + go func(t T) { + defer wg.Done() + for { + // Check if that is still timeoutable + left := t.TimeLeft(time.Now()) + if left == nil || t.IsFinished() || ctx.Err() != nil { + return + } + + // Fire the handler if it timed out + if *left <= 0 { + run() + return + } + + // Wait until time is finished, or we are no longer waiting + timer := time.NewTimer(*left) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return + case <-timer.C: + } + } + }(objects[i]) + } + + // Cancel if all timeouts are done + go func() { + wg.Wait() + ctxCancel() + }() + + // Allow to cancel externally + return ctxCancel +} diff --git a/cmd/testworkflow-init/output/constants.go b/cmd/testworkflow-init/output/constants.go deleted file mode 100644 index 59c5f1dcaae..00000000000 --- a/cmd/testworkflow-init/output/constants.go +++ /dev/null @@ -1,9 +0,0 @@ -package output - -const ( - CodeTimeout uint8 = 124 - CodeAborted uint8 = 137 - CodeInputError uint8 = 155 - CodeNoCommand uint8 = 189 - CodeInternal uint8 = 190 -) diff --git a/cmd/testworkflow-init/output/fail.go b/cmd/testworkflow-init/output/fail.go new file mode 100644 index 00000000000..0f77fa52216 --- /dev/null +++ b/cmd/testworkflow-init/output/fail.go @@ -0,0 +1,19 @@ +package output + +import "os" + +func ExitErrorf(exitCode uint8, message string, args ...interface{}) { + // Print message + Std.Printf(message+"\n", args...) + + // Exit + os.Exit(int(exitCode)) +} + +func UnsafeExitErrorf(exitCode uint8, message string, args ...interface{}) { + // Print message + Std.Direct().Printf(message+"\n", args...) + + // Exit + os.Exit(int(exitCode)) +} diff --git a/cmd/testworkflow-init/output/output.go b/cmd/testworkflow-init/output/output.go deleted file mode 100644 index c5193847efc..00000000000 --- a/cmd/testworkflow-init/output/output.go +++ /dev/null @@ -1,19 +0,0 @@ -package output - -import ( - "fmt" - "os" - - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" -) - -func Failf(exitCode uint8, message string, args ...interface{}) { - // Print message - fmt.Printf(message+"\n", args...) - - // Kill the sub-process - data.Step.Kill() - - // Exit - os.Exit(int(exitCode)) -} diff --git a/cmd/testworkflow-init/output/printer.go b/cmd/testworkflow-init/output/printer.go new file mode 100644 index 00000000000..ce1d1c5ab0e --- /dev/null +++ b/cmd/testworkflow-init/output/printer.go @@ -0,0 +1,90 @@ +package output + +import ( + "fmt" + "io" + "unsafe" + + "github.com/gookit/color" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" +) + +type FlushWriter interface { + io.Writer + Flush() +} + +type printer struct { + through io.Writer + direct io.Writer +} + +// Write sends bytes, sanitizing it +func (s *printer) Write(p []byte) (n int, err error) { + return s.through.Write(p) +} + +// Printf sends a formatted string via stream, sanitizing it +func (s *printer) Printf(format string, args ...interface{}) { + _, _ = fmt.Fprintf(s.through, format, args...) +} + +// Print sends a bare string via stream, sanitizing it +func (s *printer) Print(message string) { + _, _ = s.through.Write(unsafe.Slice(unsafe.StringData(message), len(message))) +} + +// Println sends a bare string via stream, sanitizing it +func (s *printer) Println(message string) { + buf := make([]byte, len(message)+1) + copy(buf, unsafe.Slice(unsafe.StringData(message), len(message))) + buf[len(buf)-1] = '\n' + _, _ = s.through.Write(buf) +} + +func (s *printer) printfColor(color color.Color, format string, args ...interface{}) { + s.Printf(color.Render(format), args...) +} + +func (s *printer) printColor(color color.Color, message string) { + s.printfColor(color, "%s", message) +} + +// Errorf sends a formatted string via stream, sanitizing it +func (s *printer) Errorf(format string, args ...interface{}) { + s.printfColor(color.FgRed, format, args...) +} + +// Error sends a bare string via stream, sanitizing it +func (s *printer) Error(message string) { + s.printColor(color.FgRed, message) +} + +// Warnf sends a formatted string via stream, sanitizing it +func (s *printer) Warnf(format string, args ...interface{}) { + s.printfColor(color.FgYellow, format, args...) +} + +// Warn sends a bare string via stream, sanitizing it +func (s *printer) Warn(message string) { + s.printColor(color.FgYellow, message) +} + +// Hint sends a hint via stream, bypassing the output sanitization +func (s *printer) Hint(ref, name string) { + hint := instructions.SprintHint(ref, name) + _, _ = s.direct.Write(unsafe.Slice(unsafe.StringData(hint), len(hint))) +} + +// HintDetails sends a hint via stream, bypassing the output sanitization +func (s *printer) HintDetails(ref, name string, value interface{}) { + hint := instructions.SprintHintDetails(ref, name, value) + _, _ = s.direct.Write(unsafe.Slice(unsafe.StringData(hint), len(hint))) +} + +// Output sends an instruction via stream, bypassing the output sanitization +func (s *printer) Output(ref, name string, value interface{}) { + output := instructions.SprintOutput(ref, name, value) + _, _ = s.direct.Write(unsafe.Slice(unsafe.StringData(output), len(output))) +} diff --git a/cmd/testworkflow-init/output/stream.go b/cmd/testworkflow-init/output/stream.go new file mode 100644 index 00000000000..6079a87d9e5 --- /dev/null +++ b/cmd/testworkflow-init/output/stream.go @@ -0,0 +1,53 @@ +package output + +import ( + "io" + "os" + + "github.com/kubeshop/testkube/cmd/testworkflow-init/obfuscator" +) + +var ( + Std = NewStream(os.Stdout) +) + +type ObfuscatorLike interface { + SetSensitiveWords([]string) + SetSensitiveReplacer(func([]byte) []byte) +} + +type stream struct { + *printer + + direct *stream +} + +func NewStream(dst io.Writer) *stream { + s := &stream{} + s.printer = &printer{direct: dst} + s.printer.through = obfuscator.New(dst, obfuscator.FullReplace("*****"), nil) + s.direct = &stream{printer: &printer{direct: s.printer.direct, through: s.printer.direct}} + return s +} + +func (s *stream) Direct() *stream { + return s.direct +} + +func (s *stream) SetSensitiveWords(words []string) { + if v, ok := s.printer.through.(ObfuscatorLike); ok { + v.SetSensitiveWords(words) + } +} + +func (s *stream) SetSensitiveReplacer(replacer func(value []byte) []byte) { + if v, ok := s.printer.through.(ObfuscatorLike); ok { + v.SetSensitiveReplacer(replacer) + } +} + +func (s *stream) Flush() { + if v, ok := s.printer.through.(FlushWriter); ok { + v.Flush() + } +} diff --git a/cmd/testworkflow-init/run/run.go b/cmd/testworkflow-init/run/run.go deleted file mode 100644 index d32ac3e0849..00000000000 --- a/cmd/testworkflow-init/run/run.go +++ /dev/null @@ -1,55 +0,0 @@ -package run - -import ( - "fmt" - "os" - - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" -) - -const ( - defaultBinPath = "/.tktw/bin" -) - -func execute(cmd string, args ...string) { - data.Step.Run(data.Config.Negative, cmd, args...) - - if data.Config.Negative { - fmt.Printf("Expected to fail: finished with exit code %d.\n", data.Step.ExitCode) - } else if data.Config.Debug { - fmt.Printf("Exit code: %d.\n", data.Step.ExitCode) - } -} - -func Run(cmd string, args []string) { - // Ensure the built-in binaries are available - if os.Getenv("PATH") == "" { - _ = os.Setenv("PATH", defaultBinPath) - } else { - _ = os.Setenv("PATH", fmt.Sprintf("%s:%s", os.Getenv("PATH"), defaultBinPath)) - } - - // Instantiate the command and run - execute(cmd, args...) - - // Retry if it's expected - // TODO: Support nested retries - step := data.State.GetStep(data.Step.Ref) - for step.Iteration <= uint64(data.Config.RetryCount) { - expr, err := data.Expression(data.Config.RetryUntil, data.LocalMachine) - if err != nil { - fmt.Printf("Failed to execute retry condition: %s: %s\n", data.Config.RetryUntil, err.Error()) - data.Finish() - } - v, _ := expr.BoolValue() - if v { - break - } - step.Next() - fmt.Printf("\nExit code: %d • Retrying: attempt #%d (of %d):\n", data.Step.ExitCode, step.Iteration-1, data.Config.RetryCount) - execute(cmd, args...) - } - - // Finish - data.Finish() -} diff --git a/pkg/expressions/propertyaccessor.go b/pkg/expressions/propertyaccessor.go index ca1322719e0..bd49fcb1629 100644 --- a/pkg/expressions/propertyaccessor.go +++ b/pkg/expressions/propertyaccessor.go @@ -33,6 +33,9 @@ func (s *propertyAccessor) Template() string { } func (s *propertyAccessor) SafeResolve(m ...Machine) (v Expression, changed bool, err error) { + if s.value == nil { + return nil, false, errors.New("no parent to access") + } if s.value.Static() == nil { s.value, changed, err = s.value.SafeResolve(m...) if !changed || err != nil || s.value.Static() == nil { diff --git a/pkg/tcl/testworkflowstcl/testworkflowprocessor/operations.go b/pkg/tcl/testworkflowstcl/testworkflowprocessor/operations.go index 80a8a5a44dc..d10a179774e 100644 --- a/pkg/tcl/testworkflowstcl/testworkflowprocessor/operations.go +++ b/pkg/tcl/testworkflowstcl/testworkflowprocessor/operations.go @@ -22,15 +22,16 @@ import ( "github.com/kubeshop/testkube/pkg/expressions" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowresolver" ) -func ProcessExecute(_ testworkflowprocessor.InternalProcessor, layer testworkflowprocessor.Intermediate, container testworkflowprocessor.Container, step testworkflowsv1.Step) (testworkflowprocessor.Stage, error) { +func ProcessExecute(_ testworkflowprocessor.InternalProcessor, layer testworkflowprocessor.Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Execute == nil { return nil, nil } container = container.CreateChild() - stage := testworkflowprocessor.NewContainerStage(layer.NextRef(), container) + stage := stage.NewContainerStage(layer.NextRef(), container) stage.SetRetryPolicy(step.Retry) hasWorkflows := len(step.Execute.Workflows) > 0 hasTests := len(step.Execute.Tests) > 0 @@ -82,12 +83,12 @@ func ProcessExecute(_ testworkflowprocessor.InternalProcessor, layer testworkflo return stage, nil } -func ProcessParallel(_ testworkflowprocessor.InternalProcessor, layer testworkflowprocessor.Intermediate, container testworkflowprocessor.Container, step testworkflowsv1.Step) (testworkflowprocessor.Stage, error) { +func ProcessParallel(_ testworkflowprocessor.InternalProcessor, layer testworkflowprocessor.Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Parallel == nil { return nil, nil } - stage := testworkflowprocessor.NewContainerStage(layer.NextRef(), container.CreateChild()) + stage := stage.NewContainerStage(layer.NextRef(), container.CreateChild()) stage.SetCategory("Run in parallel") // Inherit container defaults @@ -111,7 +112,7 @@ func ProcessParallel(_ testworkflowprocessor.InternalProcessor, layer testworkfl return stage, nil } -func ProcessServicesStart(_ testworkflowprocessor.InternalProcessor, layer testworkflowprocessor.Intermediate, container testworkflowprocessor.Container, step testworkflowsv1.Step) (testworkflowprocessor.Stage, error) { +func ProcessServicesStart(_ testworkflowprocessor.InternalProcessor, layer testworkflowprocessor.Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if len(step.Services) == 0 { return nil, nil } @@ -120,7 +121,7 @@ func ProcessServicesStart(_ testworkflowprocessor.InternalProcessor, layer testw podsRef := layer.NextRef() container.AppendEnv(corev1.EnvVar{Name: "TK_SVC_REF", Value: podsRef}) - stage := testworkflowprocessor.NewContainerStage(layer.NextRef(), container.CreateChild()) + stage := stage.NewContainerStage(layer.NextRef(), container.CreateChild()) stage.SetCategory("Start services") stage.Container(). @@ -143,12 +144,12 @@ func ProcessServicesStart(_ testworkflowprocessor.InternalProcessor, layer testw return stage, nil } -func ProcessServicesStop(_ testworkflowprocessor.InternalProcessor, layer testworkflowprocessor.Intermediate, container testworkflowprocessor.Container, step testworkflowsv1.Step) (testworkflowprocessor.Stage, error) { +func ProcessServicesStop(_ testworkflowprocessor.InternalProcessor, layer testworkflowprocessor.Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if len(step.Services) == 0 { return nil, nil } - stage := testworkflowprocessor.NewContainerStage(layer.NextRef(), container.CreateChild()) + stage := stage.NewContainerStage(layer.NextRef(), container.CreateChild()) stage.SetCondition("always") // TODO: actually, it's enough to do it when "Start services" is not skipped stage.SetOptional(true) stage.SetCategory("Stop services") diff --git a/pkg/testworkflows/testworkflowcontroller/controller.go b/pkg/testworkflows/testworkflowcontroller/controller.go index ef7b2cbc52c..93cbf05de4e 100644 --- a/pkg/testworkflows/testworkflowcontroller/controller.go +++ b/pkg/testworkflows/testworkflowcontroller/controller.go @@ -11,11 +11,11 @@ import ( "k8s.io/client-go/kubernetes" initconstants "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/pkg/api/v1/testkube" - "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" ) const ( @@ -75,7 +75,7 @@ func New(parentCtx context.Context, clientSet kubernetes.Interface, namespace, i // Ensure the main Job exists in the cluster, // and obtain the signature - var sig []testworkflowprocessor.Signature + var sig []stage.Signature var err error select { case j, ok := <-job.PeekMessage(ctx): @@ -88,7 +88,7 @@ func New(parentCtx context.Context, clientSet kubernetes.Interface, namespace, i ctxCancel() return nil, j.Error } - sig, err = testworkflowprocessor.GetSignatureFromJSON([]byte(j.Value.Annotations[constants.SignatureAnnotationName])) + sig, err = stage.GetSignatureFromJSON([]byte(j.Value.Annotations[constants.SignatureAnnotationName])) if err != nil { ctxCancel() return nil, errors.Wrap(err, "invalid job signature") @@ -133,7 +133,7 @@ type controller struct { id string namespace string scheduledAt time.Time - signature []testworkflowprocessor.Signature + signature []stage.Signature clientSet kubernetes.Interface ctx context.Context ctxCancel context.CancelFunc @@ -231,7 +231,7 @@ func (c *controller) WatchLightweight(parentCtx context.Context) <-chan Lightwei prevNodeName := "" prevPodIP := "" prevStatus := testkube.QUEUED_TestWorkflowStatus - sig := testworkflowprocessor.MapSignatureListToInternal(c.signature) + sig := stage.MapSignatureListToInternal(c.signature) ch := make(chan LightweightNotification) go func() { defer close(ch) @@ -293,7 +293,7 @@ func (c *controller) Logs(parentCtx context.Context, follow bool) io.Reader { if v.Error == nil && v.Value.Log != "" && !v.Value.Temporary { if ref != v.Value.Ref && v.Value.Ref != "" { ref = v.Value.Ref - _, _ = writer.Write([]byte(data.SprintHint(ref, initconstants.InstructionStart))) + _, _ = writer.Write([]byte(instructions.SprintHint(ref, initconstants.InstructionStart))) } _, _ = writer.Write([]byte(v.Value.Log)) } diff --git a/pkg/testworkflows/testworkflowcontroller/logs.go b/pkg/testworkflows/testworkflowcontroller/logs.go index b72b8b36807..ea04f770fe1 100644 --- a/pkg/testworkflows/testworkflowcontroller/logs.go +++ b/pkg/testworkflows/testworkflowcontroller/logs.go @@ -15,7 +15,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/pkg/log" "github.com/kubeshop/testkube/pkg/utils" @@ -29,15 +29,15 @@ const ( type Comment struct { Time time.Time - Hint *data.Instruction - Output *data.Instruction + Hint *instructions.Instruction + Output *instructions.Instruction } type ContainerLog struct { Time time.Time Log []byte - Hint *data.Instruction - Output *data.Instruction + Hint *instructions.Instruction + Output *instructions.Instruction } // getContainerLogsStream is getting logs stream, and tries to reinitialize the stream on EOF. @@ -120,7 +120,6 @@ func WatchContainerLogs(parentCtx context.Context, clientSet kubernetes.Interfac // Create logs stream request stream, err := getContainerLogsStream(ctx, clientSet, namespace, podName, containerName, follow, pod, since) - hadAnyContent := false if err == io.EOF { return } else if err != nil { @@ -220,90 +219,125 @@ func WatchContainerLogs(parentCtx context.Context, clientSet kubernetes.Interfac // Parse and return the logs reader := bufio.NewReaderSize(stream, FlushBufferSize) + readerAnyContent := false tsReader := newTimestampReader() - isNewLine := false - isStarted := false + lastTs := time.Now() + + hasNewLine := false + for { - var prepend []byte + // --- Step 1: READING TIMESTAMP // Read next timestamp err = tsReader.Read(reader) - if err == nil { - // Strip older logs - SinceTime in Kubernetes logs is ignoring milliseconds precision - if since != nil && since.After(tsReader.ts) { - _, _ = utils.ReadLongLine(reader) + + // Ignore too old logs. SinceTime in Kubernetes is precise only to seconds + if err == nil && !readerAnyContent { + if since != nil && !since.After(tsReader.ts) { + isPrefix := false + for isPrefix && err != nil { + _, isPrefix, err = reader.ReadLine() + } continue } - hadAnyContent = true - } else if err == io.EOF { - if !hadAnyContent { - return - } - // Reinitialize logs stream - since = common.Ptr(tsReader.ts.Add(1)) + } + + // Save information about the last timestamp + if err == nil { + lastTs = tsReader.ts + } + + // If the stream is finished, + // either the logfile has been rotated, or the container actually finished. + // Assume that only if there was EOF without any logs since, the container is done. + if err == io.EOF && !readerAnyContent { + return + } + + // If there was EOF, and we are not sure if container is done, + // reinitialize the stream from the time we have finished. + if err == io.EOF { + since = common.Ptr(lastTs.Add(1)) stream, err = getContainerLogsStream(ctx, clientSet, namespace, podName, containerName, follow, pod, since) if err != nil { return } reader.Reset(stream) - hadAnyContent = false + readerAnyContent = false continue - } else { - // Edge case: Kubernetes may send critical errors without timestamp (like ionotify) - if len(tsReader.Prefix()) > 0 { - prepend = bytes.Clone(tsReader.Prefix()) - } - flushLogBuffer() - w.Error(err) } - // Check for the next part - line, err := utils.ReadLongLine(reader) - if len(prepend) > 0 { - line = append(prepend, line...) + // Edge case: Kubernetes may send critical errors without timestamp (like ionotify) + if errors.Is(err, ErrInvalidTimestamp) && len(tsReader.Prefix()) > 0 { + appendLog(lastTs, []byte(tsReader.Format(lastTs)), []byte(" "), tsReader.Prefix()) + rest, _ := utils.ReadLongLine(reader) + appendLog(lastTs, rest, []byte("\n")) + hasNewLine = false + continue } - // Process the received line - if !isNewLine && len(line) == 0 { - isNewLine = true - } else if len(line) > 0 { - hadComment := false - instruction, isHint, err := data.DetectInstruction(line) - if err == nil && instruction != nil { - isNewLine = false - hadComment = true - log := ContainerLog{Time: tsReader.ts} - if isHint { - log.Hint = instruction - } else { - log.Output = instruction - } - flushLogBuffer() - w.Send(log) + // Push information about any other error + if err != nil { + w.Error(err) + continue + } + + // --- Step 2: READING THE BEGINNING OF THE LINE + + line, isPrefix, err := reader.ReadLine() + + // Between instructions there may be empty line that should be just ignored + if !isPrefix && len(line) == 0 { + if hasNewLine { + appendLog(lastTs, []byte("\n")) } + continue + } - // Append as regular log if expected - if !hadComment { - if !isStarted { - appendLog(tsReader.ts, tsReader.Prefix(), line) - isStarted = true - } else if isNewLine { - appendLog(tsReader.ts, []byte("\n"), tsReader.Prefix(), line) - } - isNewLine = true + // Fast-track: we know this line won't be an instruction + if !instructions.MayBeInstruction(line) { + if hasNewLine { + appendLog(lastTs, []byte("\n")) + } + appendLog(lastTs, tsReader.Prefix(), line) + for isPrefix && err == nil { + line, isPrefix, err = reader.ReadLine() + appendLog(lastTs, line) } - } else if isStarted { - appendLog(tsReader.ts, []byte("\n"), tsReader.Prefix()) + hasNewLine = true + continue } - // Handle the error - if err != nil { - if err != io.EOF { - flushLogBuffer() - w.Error(err) + // --- Step 3: FINISH READING THE LINE AND EXPORT DATA + + // Ensure we read the whole line to buffer to validate if it is instruction + for isPrefix && err == nil { + var currentLine []byte + currentLine, isPrefix, err = reader.ReadLine() + line = append(line, currentLine...) + } + + // Detect instruction + instruction, isHint, err := instructions.DetectInstruction(line) + if err == nil && instruction != nil { + item := ContainerLog{Time: lastTs} + if isHint { + item.Hint = instruction + } else { + item.Output = instruction } - return + flushLogBuffer() + w.Send(item) + hasNewLine = false + continue } + + // Print line if it's not an instruction + if hasNewLine { + appendLog(lastTs, []byte("\n")) + } + appendLog(lastTs, tsReader.Prefix(), line) + hasNewLine = true } }() @@ -374,6 +408,13 @@ func (t *timestampReader) read(reader *bufio.Reader) error { return nil } +func (t *timestampReader) Format(ts time.Time) string { + if t.utc == nil || *t.utc { + return ts.Format(KubernetesLogTimeFormat) + } + return ts.Format(KubernetesTimezoneLogTimeFormat) +} + // readUTC is optimized operation for reading the UTC timestamp (Z). func (t *timestampReader) readUTC(reader *bufio.Reader) error { // Read the possible timestamp slice diff --git a/pkg/testworkflows/testworkflowcontroller/notification.go b/pkg/testworkflows/testworkflowcontroller/notification.go index 03e838b45f1..e5e24581dad 100644 --- a/pkg/testworkflows/testworkflowcontroller/notification.go +++ b/pkg/testworkflows/testworkflowcontroller/notification.go @@ -4,7 +4,7 @@ import ( "encoding/json" "time" - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/log" ) @@ -14,7 +14,7 @@ type Notification struct { Result *testkube.TestWorkflowResult `json:"result,omitempty"` Ref string `json:"ref,omitempty"` Log string `json:"log,omitempty"` - Output *data.Instruction `json:"output,omitempty"` + Output *instructions.Instruction `json:"output,omitempty"` Temporary bool `json:"temporary,omitempty"` } @@ -29,7 +29,7 @@ func (n *Notification) ToInternal() testkube.TestWorkflowExecutionNotification { } } -func InstructionToInternal(instruction *data.Instruction) *testkube.TestWorkflowOutput { +func InstructionToInternal(instruction *instructions.Instruction) *testkube.TestWorkflowOutput { if instruction == nil { return nil } diff --git a/pkg/testworkflows/testworkflowcontroller/notifier.go b/pkg/testworkflows/testworkflowcontroller/notifier.go index bb86d9cdb23..eef9767e65b 100644 --- a/pkg/testworkflows/testworkflowcontroller/notifier.go +++ b/pkg/testworkflows/testworkflowcontroller/notifier.go @@ -6,10 +6,10 @@ import ( "sync" "time" - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/pkg/api/v1/testkube" - "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" "github.com/kubeshop/testkube/pkg/ui" ) @@ -259,7 +259,7 @@ func (n *notifier) Start(ref string, ts time.Time) { } } -func (n *notifier) Output(ref string, ts time.Time, output *data.Instruction) { +func (n *notifier) Output(ref string, ts time.Time, output *instructions.Instruction) { if ref == InitContainerName { ref = "" } @@ -287,7 +287,7 @@ func (n *notifier) UpdateStepStatus(ref string, status testkube.TestWorkflowStep n.emit() } -func (n *notifier) finishInit(status ContainerResult) { +func (n *notifier) finishInit(status ContainerResultStep) { if n.result.Initialization.FinishedAt.Equal(status.FinishedAt) && n.result.Initialization.Status != nil && *n.result.Initialization.Status == status.Status { return } @@ -298,7 +298,26 @@ func (n *notifier) finishInit(status ContainerResult) { n.emit() } -func (n *notifier) FinishStep(ref string, status ContainerResult) { +func (n *notifier) IsAnyAborted() bool { + if n.result.Initialization.Status != nil && *n.result.Initialization.Status == testkube.ABORTED_TestWorkflowStepStatus { + return true + } + for _, s := range n.result.Steps { + if s.Status != nil && *s.Status == testkube.ABORTED_TestWorkflowStepStatus { + return true + } + } + return false +} + +func (n *notifier) IsFinished(ref string) bool { + if ref == InitContainerName { + return !n.result.Initialization.FinishedAt.IsZero() + } + return !n.result.Steps[ref].FinishedAt.IsZero() +} + +func (n *notifier) FinishStep(ref string, status ContainerResultStep) { if ref == InitContainerName { n.finishInit(status) return @@ -338,7 +357,7 @@ func (n *notifier) GetStepResult(ref string) testkube.TestWorkflowStepResult { return n.result.Steps[ref] } -func newNotifier(ctx context.Context, signature []testworkflowprocessor.Signature, scheduledAt time.Time) *notifier { +func newNotifier(ctx context.Context, signature []stage.Signature, scheduledAt time.Time) *notifier { // Initialize the zero result sig := make([]testkube.TestWorkflowSignature, len(signature)) for i, s := range signature { @@ -350,7 +369,7 @@ func newNotifier(ctx context.Context, signature []testworkflowprocessor.Signatur Initialization: &testkube.TestWorkflowStepResult{ Status: common.Ptr(testkube.QUEUED_TestWorkflowStepStatus), }, - Steps: testworkflowprocessor.MapSignatureListToStepResults(signature), + Steps: stage.MapSignatureListToStepResults(signature), } result.Recompute(sig, scheduledAt) diff --git a/pkg/testworkflows/testworkflowcontroller/podstate.go b/pkg/testworkflows/testworkflowcontroller/podstate.go index d547c874266..ee1a267a5b0 100644 --- a/pkg/testworkflows/testworkflowcontroller/podstate.go +++ b/pkg/testworkflows/testworkflowcontroller/podstate.go @@ -2,7 +2,9 @@ package testworkflowcontroller import ( "context" + "regexp" "slices" + "strconv" "strings" "sync" "time" @@ -11,7 +13,9 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + "github.com/kubeshop/testkube/cmd/testworkflow-init/data" "github.com/kubeshop/testkube/internal/common" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" ) @@ -333,7 +337,39 @@ func (p *podState) containerResult(name string) (ContainerResult, error) { } return UnknownContainerResult, ErrNotTerminatedYet } - return GetContainerResult(*status), nil + + result := ContainerResult{ + ExitCode: int(status.State.Terminated.ExitCode), + FinishedAt: status.State.Terminated.FinishedAt.Time, + } + + // Workaround - GKE sends SIGKILL after the container is already terminated, + // and the pod gets stuck then. + if status.State.Terminated.Reason != "Completed" { + result.Details = status.State.Terminated.Reason + } + + re := regexp.MustCompile(`^([^,]),(0|[1-9]\d*)$`) + for _, message := range strings.Split(status.State.Terminated.Message, "/") { + match := re.FindStringSubmatch(message) + if match == nil { + result.Steps = append(result.Steps, ContainerResultStep{ + Status: testkube.ABORTED_TestWorkflowStepStatus, + FinishedAt: result.FinishedAt, + ExitCode: -1, + }) + } else { + exitCode, _ := strconv.Atoi(match[2]) + result.Steps = append(result.Steps, ContainerResultStep{ + Status: testkube.TestWorkflowStepStatus(data.StepStatusFromCode(match[1])), + Details: result.Details, + FinishedAt: result.FinishedAt, + ExitCode: exitCode, + }) + } + } + + return result, nil } func (p *podState) ContainerResult(name string) (ContainerResult, error) { diff --git a/pkg/testworkflows/testworkflowcontroller/utils.go b/pkg/testworkflows/testworkflowcontroller/utils.go index 77bb22bc4ef..bfe7fd2a6ee 100644 --- a/pkg/testworkflows/testworkflowcontroller/utils.go +++ b/pkg/testworkflows/testworkflowcontroller/utils.go @@ -2,7 +2,6 @@ package testworkflowcontroller import ( "regexp" - "strconv" "time" batchv1 "k8s.io/api/batch/v1" @@ -34,42 +33,20 @@ func IsJobDone(job *batchv1.Job) bool { return (job.Status.Active == 0 && (job.Status.Succeeded > 0 || job.Status.Failed > 0)) || job.ObjectMeta.DeletionTimestamp != nil } -type ContainerResult struct { +type ContainerResultStep struct { Status testkube.TestWorkflowStepStatus + ExitCode int + Details string + FinishedAt time.Time +} + +type ContainerResult struct { + Steps []ContainerResultStep Details string ExitCode int FinishedAt time.Time } var UnknownContainerResult = ContainerResult{ - Status: testkube.ABORTED_TestWorkflowStepStatus, ExitCode: -1, } - -func GetContainerResult(c corev1.ContainerStatus) ContainerResult { - if c.State.Waiting != nil { - return ContainerResult{Status: testkube.QUEUED_TestWorkflowStepStatus, ExitCode: -1} - } - if c.State.Running != nil { - return ContainerResult{Status: testkube.RUNNING_TestWorkflowStepStatus, ExitCode: -1} - } - re := regexp.MustCompile(`^([^,]*),(0|[1-9]\d*)$`) - - // Workaround - GKE sends SIGKILL after the container is already terminated, - // and the pod gets stuck then. - if c.State.Terminated.Reason != "Completed" { - return ContainerResult{Status: testkube.ABORTED_TestWorkflowStepStatus, Details: c.State.Terminated.Reason, ExitCode: -1, FinishedAt: c.State.Terminated.FinishedAt.Time} - } - - msg := c.State.Terminated.Message - match := re.FindStringSubmatch(msg) - if match == nil { - return ContainerResult{Status: testkube.ABORTED_TestWorkflowStepStatus, ExitCode: -1, FinishedAt: c.State.Terminated.FinishedAt.Time} - } - status := testkube.TestWorkflowStepStatus(match[1]) - exitCode, _ := strconv.Atoi(match[2]) - if status == "" { - status = testkube.PASSED_TestWorkflowStepStatus - } - return ContainerResult{Status: status, ExitCode: exitCode, FinishedAt: c.State.Terminated.FinishedAt.Time} -} diff --git a/pkg/testworkflows/testworkflowcontroller/watchinstrumentedpod.go b/pkg/testworkflows/testworkflowcontroller/watchinstrumentedpod.go index 1ae190e2f08..4d49f3e8875 100644 --- a/pkg/testworkflows/testworkflowcontroller/watchinstrumentedpod.go +++ b/pkg/testworkflows/testworkflowcontroller/watchinstrumentedpod.go @@ -2,7 +2,9 @@ package testworkflowcontroller import ( "context" + "encoding/json" "fmt" + "strconv" "time" "github.com/pkg/errors" @@ -13,7 +15,9 @@ import ( "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/pkg/api/v1/testkube" - "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" + constants2 "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" ) const ( @@ -27,7 +31,7 @@ type WatchInstrumentedPodOptions struct { Follow *bool } -func WatchInstrumentedPod(parentCtx context.Context, clientSet kubernetes.Interface, signature []testworkflowprocessor.Signature, scheduledAt time.Time, pod Channel[*corev1.Pod], podEvents Channel[*corev1.Event], opts WatchInstrumentedPodOptions) (<-chan ChannelMessage[Notification], error) { +func WatchInstrumentedPod(parentCtx context.Context, clientSet kubernetes.Interface, signature []stage.Signature, scheduledAt time.Time, pod Channel[*corev1.Pod], podEvents Channel[*corev1.Event], opts WatchInstrumentedPodOptions) (<-chan ChannelMessage[Notification], error) { // Avoid missing data if pod == nil { return nil, errors.New("pod watcher is required") @@ -69,54 +73,105 @@ func WatchInstrumentedPod(parentCtx context.Context, clientSet kubernetes.Interf // Load the namespace information podObj := <-pod.Peek(ctx) + // Load the references + var refs, endRefs [][]string + var actions actiontypes.ActionGroups + err := json.Unmarshal([]byte(podObj.Annotations[constants2.SpecAnnotationName]), &actions) + if err != nil { + s.Error(fmt.Errorf("invalid instructions: %v", err)) + return + } + refs = make([][]string, len(actions)) + endRefs = make([][]string, len(actions)) + for i := range actions { + for j := range actions[i] { + if actions[i][j].Setup != nil { + refs[i] = append(refs[i], InitContainerName) + endRefs[i] = append(endRefs[i], InitContainerName) + } + if actions[i][j].Start != nil && *actions[i][j].Start != "" { + refs[i] = append(refs[i], *actions[i][j].Start) + } + if actions[i][j].End != nil && *actions[i][j].End != "" { + endRefs[i] = append(endRefs[i], *actions[i][j].End) + } + } + } + // For each container: lastTs := s.result.Initialization.FinishedAt for _, container := range append(podObj.Spec.InitContainers, podObj.Spec.Containers...) { // Ignore non-standard TestWorkflow containers - ref := container.Name - if _, ok := s.result.Steps[ref]; !(ok || ref == InitContainerName) { + number, err := strconv.Atoi(container.Name) + if err != nil || number > len(refs) { continue } + index := number - 1 + containerName := container.Name + initialRef := refs[index][0] // Update queue time - s.Queue(ref, lastTs) + s.Queue(initialRef, lastTs) // Watch the container events - for v := range state.PreStart(ref) { + for v := range state.PreStart(containerName) { if v.Value.Queued != nil { - s.Queue(ref, state.QueuedAt(ref)) + s.Queue(initialRef, state.QueuedAt(containerName)) } else if v.Value.Started != nil { - s.Queue(ref, state.QueuedAt(ref)) - s.Start(ref, state.StartedAt(ref)) + s.Queue(initialRef, state.QueuedAt(containerName)) + s.Start(initialRef, state.StartedAt(containerName)) } else if v.Value.Event != nil { ts := maxTime(v.Value.Event.CreationTimestamp.Time, v.Value.Event.FirstTimestamp.Time, v.Value.Event.LastTimestamp.Time) - s.Event(ref, ts, v.Value.Event.Type, v.Value.Event.Reason, v.Value.Event.Message) + s.Event(initialRef, ts, v.Value.Event.Type, v.Value.Event.Reason, v.Value.Event.Message) } } // Ensure the queue/start time has been saved - if s.GetStepResult(ref).QueuedAt.IsZero() || s.GetStepResult(ref).StartedAt.IsZero() { - s.Error(fmt.Errorf("missing information about scheduled '%s' container", ref)) + if s.GetStepResult(initialRef).QueuedAt.IsZero() || s.GetStepResult(initialRef).StartedAt.IsZero() { + s.Error(fmt.Errorf("missing information about scheduled '%s' step in '%s' container", initialRef, container.Name)) return } // Watch the container logs - follow := common.ResolvePtr(opts.Follow, true) && !state.IsFinished(ref) - for v := range WatchContainerLogs(ctx, clientSet, podObj.Namespace, podObj.Name, ref, follow, 10, pod).Channel() { + follow := common.ResolvePtr(opts.Follow, true) && !state.IsFinished(containerName) + aborted := false + lastStarted := initialRef + executionStatuses := map[string]constants.ExecutionResult{} + for v := range WatchContainerLogs(ctx, clientSet, podObj.Namespace, podObj.Name, containerName, follow, 10, pod).Channel() { if v.Error != nil { s.Error(v.Error) } else if v.Value.Output != nil { s.Output(v.Value.Output.Ref, v.Value.Time, v.Value.Output) } else if v.Value.Hint != nil { + if v.Value.Hint.Ref == constants2.RootOperationName { + continue + } switch v.Value.Hint.Name { case constants.InstructionStart: - s.Start(ref, v.Value.Time) - case constants.InstructionStatus: + lastStarted = v.Value.Hint.Ref + s.Start(v.Value.Hint.Ref, v.Value.Time) + case constants.InstructionEnd: status := testkube.TestWorkflowStepStatus(v.Value.Hint.Value.(string)) if status == "" { status = testkube.PASSED_TestWorkflowStepStatus } - s.UpdateStepStatus(ref, status) + s.FinishStep(v.Value.Hint.Ref, ContainerResultStep{ + Status: status, + Details: executionStatuses[v.Value.Hint.Ref].Details, + ExitCode: int(executionStatuses[v.Value.Hint.Ref].ExitCode), + FinishedAt: v.Value.Time, + }) + + // Escape when the job was aborted + if status == testkube.ABORTED_TestWorkflowStepStatus { + aborted = true + break + } + case constants.InstructionExecution: + serialized, _ := json.Marshal(v.Value.Hint.Value) + var executionResult constants.ExecutionResult + _ = json.Unmarshal(serialized, &executionResult) + executionStatuses[v.Value.Hint.Ref] = executionResult case constants.InstructionPause: ts, _ := v.Value.Hint.Value.(string) start, err := time.Parse(constants.PreciseTimeFormat, ts) @@ -124,7 +179,7 @@ func WatchInstrumentedPod(parentCtx context.Context, clientSet kubernetes.Interf start = v.Value.Time s.Error(fmt.Errorf("invalid timestamp provided with pausing instruction: %v", v.Value.Hint.Value)) } - s.Pause(ref, start) + s.Pause(v.Value.Hint.Ref, start) case constants.InstructionResume: ts, _ := v.Value.Hint.Value.(string) end, err := time.Parse(constants.PreciseTimeFormat, ts) @@ -132,42 +187,86 @@ func WatchInstrumentedPod(parentCtx context.Context, clientSet kubernetes.Interf end = v.Value.Time s.Error(fmt.Errorf("invalid timestamp provided with resuming instruction: %v", v.Value.Hint.Value)) } - s.Resume(ref, end) + s.Resume(v.Value.Hint.Ref, end) } } else { - s.Raw(ref, v.Value.Time, string(v.Value.Log), false) + s.Raw(lastStarted, v.Value.Time, string(v.Value.Log), false) } } - // Get the final result - if follow { - <-state.Finished(ref) + if aborted { + // Don't wait for any other statuses if we already know that some task has been aborted + } else if follow { + <-state.Finished(container.Name) } else { select { - case <-state.Finished(ref): + case <-state.Finished(container.Name): case <-time.After(IdleTimeout): return } } - status, err := state.ContainerResult(ref) - if err != nil { - s.Error(err) - break + + // Fall back results to the termination log + if !aborted { + result, err := state.ContainerResult(container.Name) + if err != nil { + s.Error(err) + break + } + + for i, ref := range endRefs[index] { + // Ignore tree root hints + if ref == "root" { + continue + } + status := ContainerResultStep{ + Status: testkube.ABORTED_TestWorkflowStepStatus, + ExitCode: -1, + Details: "", + FinishedAt: result.FinishedAt, + } + if len(result.Steps) > i { + status = result.Steps[i] + } + if !s.IsFinished(ref) { + s.FinishStep(ref, status) + } + } } - s.FinishStep(ref, status) // Update the last timestamp - lastTs = s.GetLastTimestamp(ref) + lastTs = s.GetLastTimestamp(lastStarted) // Break the function if the step has been aborted. // Breaking only to the loop is not enough, // because due to GKE bug, the Job is still pending, // so it will get stuck there. - if status.Status == testkube.ABORTED_TestWorkflowStepStatus { - if status.Details == "" { - status.Details = "Manual" + if s.IsAnyAborted() { + reason := s.result.Steps[lastStarted].ErrorMessage + message := "Aborted" + if reason == "" { + message = fmt.Sprintf("\n%s Aborted", s.GetLastTimestamp(lastStarted).Format(KubernetesLogTimeFormat)) + } else { + message = fmt.Sprintf("\n%s Aborted (%s)", s.GetLastTimestamp(lastStarted).Format(KubernetesLogTimeFormat), reason) + } + s.Raw(lastStarted, s.GetLastTimestamp(lastStarted), message, false) + + // Mark all not started steps as skipped + for ref := range s.result.Steps { + if !s.IsFinished(ref) { + status := testkube.SKIPPED_TestWorkflowStepStatus + if s.result.Steps[ref].Status != nil && *s.result.Steps[ref].Status == testkube.ABORTED_TestWorkflowStepStatus { + status = testkube.ABORTED_TestWorkflowStepStatus + } + s.FinishStep(ref, ContainerResultStep{ + Status: status, + ExitCode: -1, + Details: "The execution was aborted before.", + FinishedAt: lastTs, + }) + } } - s.Raw(ref, s.GetLastTimestamp(ref), fmt.Sprintf("\n%s Aborted (%s)", s.GetLastTimestamp(ref).Format(KubernetesLogTimeFormat), status.Details), false) + break } } diff --git a/pkg/testworkflows/testworkflowexecutor/executor.go b/pkg/testworkflows/testworkflowexecutor/executor.go index b30f359c6d6..6904d92cec7 100644 --- a/pkg/testworkflows/testworkflowexecutor/executor.go +++ b/pkg/testworkflows/testworkflowexecutor/executor.go @@ -18,7 +18,7 @@ import ( testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" initconstants "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/cmd/testworkflow-init/instructions" v1 "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/pkg/api/v1/testkube" @@ -32,6 +32,7 @@ import ( "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowcontroller" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowresolver" ) @@ -121,7 +122,7 @@ func (e *executor) handleFatalError(execution *testkube.TestWorkflowExecution, e if ts.IsZero() { ts = time.Now() if isAborted || isTimeout { - ts = ts.Truncate(testworkflowcontroller.DefaultInitTimeout) + ts = ts.Add(-1 * testworkflowcontroller.DefaultInitTimeout) } } @@ -234,7 +235,7 @@ func (e *executor) Control(ctx context.Context, testWorkflow *testworkflowsv1.Te } else if !v.Value.Temporary { if ref != v.Value.Ref && v.Value.Ref != "" { ref = v.Value.Ref - _, err := writer.Write([]byte(data.SprintHint(ref, initconstants.InstructionStart))) + _, err := writer.Write([]byte(instructions.SprintHint(ref, initconstants.InstructionStart))) if err != nil { log.DefaultLogger.Error(errors.Wrap(err, "saving log output signature")) } @@ -504,14 +505,14 @@ func (e *executor) Execute(ctx context.Context, workflow testworkflowsv1.TestWor Number: number, ScheduledAt: now, StatusAt: now, - Signature: testworkflowprocessor.MapSignatureListToInternal(bundle.Signature), + Signature: stage.MapSignatureListToInternal(bundle.Signature), Result: &testkube.TestWorkflowResult{ Status: common.Ptr(testkube.QUEUED_TestWorkflowStatus), PredictedStatus: common.Ptr(testkube.PASSED_TestWorkflowStatus), Initialization: &testkube.TestWorkflowStepResult{ Status: common.Ptr(testkube.QUEUED_TestWorkflowStepStatus), }, - Steps: testworkflowprocessor.MapSignatureListToStepResults(bundle.Signature), + Steps: stage.MapSignatureListToStepResults(bundle.Signature), }, Output: []testkube.TestWorkflowOutput{}, Workflow: testworkflowmappers.MapKubeToAPI(initialWorkflow), diff --git a/pkg/testworkflows/testworkflowprocessor/action/actiontypes/action.go b/pkg/testworkflows/testworkflowprocessor/action/actiontypes/action.go new file mode 100644 index 00000000000..e54038ba251 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/actiontypes/action.go @@ -0,0 +1,56 @@ +package actiontypes + +import ( + "encoding/json" + "fmt" + + testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" +) + +type ActionContainer struct { + Ref string `json:"r"` + Config testworkflowsv1.ContainerConfig `json:"c"` +} + +type Action struct { + CurrentStatus *string `json:"s,omitempty"` + Start *string `json:"S,omitempty"` + End *string `json:"E,omitempty"` + Setup *lite.ActionSetup `json:"_,omitempty"` + Declare *lite.ActionDeclare `json:"d,omitempty"` + Result *lite.ActionResult `json:"r,omitempty"` + Container *ActionContainer `json:"c,omitempty"` + Execute *lite.ActionExecute `json:"e,omitempty"` + Timeout *lite.ActionTimeout `json:"t,omitempty"` + Pause *lite.ActionPause `json:"p,omitempty"` + Retry *lite.ActionRetry `json:"R,omitempty"` +} + +func (a *Action) Type() lite.ActionType { + if a.Declare != nil { + return lite.ActionTypeDeclare + } else if a.Pause != nil { + return lite.ActionTypePause + } else if a.Result != nil { + return lite.ActionTypeResult + } else if a.Timeout != nil { + return lite.ActionTypeTimeout + } else if a.Retry != nil { + return lite.ActionTypeRetry + } else if a.Container != nil { + return lite.ActionTypeContainerTransition + } else if a.CurrentStatus != nil { + return lite.ActionTypeCurrentStatus + } else if a.Start != nil { + return lite.ActionTypeStart + } else if a.End != nil { + return lite.ActionTypeEnd + } else if a.Setup != nil { + return lite.ActionTypeSetup + } else if a.Execute != nil { + return lite.ActionTypeExecute + } + v, e := json.Marshal(a) + panic(fmt.Sprintf("unknown action type: %s, %v", string(v), e)) +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite/action.go b/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite/action.go new file mode 100644 index 00000000000..04f763da5f2 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite/action.go @@ -0,0 +1,118 @@ +package lite + +import ( + "encoding/json" + "fmt" +) + +type ActionResult struct { + Ref string `json:"r"` + Value string `json:"v"` +} + +type ActionDeclare struct { + Condition string `json:"c"` + Ref string `json:"r"` + Parents []string `json:"p,omitempty"` +} + +type ActionExecute struct { + Ref string `json:"r"` + Negative bool `json:"n,omitempty"` + Toolkit bool `json:"t,omitempty"` +} + +type ActionPause struct { + Ref string `json:"r"` +} + +type ActionTimeout struct { + Ref string `json:"r"` + Timeout string `json:"t"` +} + +type ActionRetry struct { + Ref string `json:"r"` + Count int32 `json:"c,omitempty"` + Until string `json:"u,omitempty"` +} + +type ActionSetup struct { + CopyInit bool `json:"i,omitempty"` + CopyBinaries bool `json:"b,omitempty"` +} + +type ActionType string + +const ( + // Declarations + ActionTypeDeclare ActionType = "declare" + ActionTypePause ActionType = "pause" + ActionTypeResult ActionType = "result" + ActionTypeTimeout ActionType = "timeout" + ActionTypeRetry ActionType = "retry" + + // Operations + ActionTypeContainerTransition ActionType = "container" + ActionTypeCurrentStatus ActionType = "status" + ActionTypeStart ActionType = "start" + ActionTypeEnd ActionType = "end" + ActionTypeSetup ActionType = "setup" + ActionTypeExecute ActionType = "execute" +) + +type LiteContainerConfig struct { + Command *[]string `json:"command,omitempty"` + Args *[]string `json:"args,omitempty"` + WorkingDir *string `json:"workingDir,omitempty"` +} + +type LiteActionContainer struct { + Config LiteContainerConfig `json:"c"` +} + +// LiteAction is lightweight version of Action, +// that is intended to use directly in the Init Process. +// It's not including original ContainerConfig, +// as it requires additional 40MB of structs in the binary. +type LiteAction struct { + CurrentStatus *string `json:"s,omitempty"` + Start *string `json:"S,omitempty"` + End *string `json:"E,omitempty"` + Setup *ActionSetup `json:"_,omitempty"` + Declare *ActionDeclare `json:"d,omitempty"` + Result *ActionResult `json:"r,omitempty"` + Container *LiteActionContainer `json:"c,omitempty"` + Execute *ActionExecute `json:"e,omitempty"` + Timeout *ActionTimeout `json:"t,omitempty"` + Pause *ActionPause `json:"p,omitempty"` + Retry *ActionRetry `json:"R,omitempty"` +} + +func (a *LiteAction) Type() ActionType { + if a.Declare != nil { + return ActionTypeDeclare + } else if a.Pause != nil { + return ActionTypePause + } else if a.Result != nil { + return ActionTypeResult + } else if a.Timeout != nil { + return ActionTypeTimeout + } else if a.Retry != nil { + return ActionTypeRetry + } else if a.Container != nil { + return ActionTypeContainerTransition + } else if a.CurrentStatus != nil { + return ActionTypeCurrentStatus + } else if a.Start != nil { + return ActionTypeStart + } else if a.End != nil { + return ActionTypeEnd + } else if a.Setup != nil { + return ActionTypeSetup + } else if a.Execute != nil { + return ActionTypeExecute + } + v, e := json.Marshal(a) + panic(fmt.Sprintf("unknown action type: %s, %v", string(v), e)) +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite/utils.go b/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite/utils.go new file mode 100644 index 00000000000..0bfbb492f8e --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite/utils.go @@ -0,0 +1,53 @@ +package lite + +type LiteActionList []LiteAction + +func NewLiteActionList() LiteActionList { + return nil +} + +func (a LiteActionList) Setup(copyInit, copyBinaries bool) LiteActionList { + return append(a, LiteAction{Setup: &ActionSetup{CopyInit: copyInit, CopyBinaries: copyBinaries}}) +} + +func (a LiteActionList) Declare(ref string, condition string, parents ...string) LiteActionList { + return append(a, LiteAction{Declare: &ActionDeclare{Ref: ref, Condition: condition, Parents: parents}}) +} + +func (a LiteActionList) Start(ref string) LiteActionList { + return append(a, LiteAction{Start: &ref}) +} + +func (a LiteActionList) End(ref string) LiteActionList { + return append(a, LiteAction{End: &ref}) +} + +func (a LiteActionList) Pause(ref string) LiteActionList { + return append(a, LiteAction{Pause: &ActionPause{Ref: ref}}) +} + +func (a LiteActionList) CurrentStatus(expression string) LiteActionList { + return append(a, LiteAction{CurrentStatus: &expression}) +} + +func (a LiteActionList) Result(ref, expression string) LiteActionList { + return append(a, LiteAction{Result: &ActionResult{Ref: ref, Value: expression}}) +} + +func (a LiteActionList) Execute(ref string, negative bool) LiteActionList { + return append(a, LiteAction{Execute: &ActionExecute{Ref: ref, Negative: negative}}) +} + +func (a LiteActionList) MutateContainer(config LiteContainerConfig) LiteActionList { + return append(a, LiteAction{Container: &LiteActionContainer{Config: config}}) +} + +type LiteActionGroups []LiteActionList + +func (a LiteActionGroups) Append(fn func(list LiteActionList) LiteActionList) LiteActionGroups { + return append(a, fn(NewLiteActionList())) +} + +func NewLiteActionGroups() LiteActionGroups { + return nil +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/actiontypes/utils.go b/pkg/testworkflows/testworkflowprocessor/action/actiontypes/utils.go new file mode 100644 index 00000000000..f46b8a054d1 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/actiontypes/utils.go @@ -0,0 +1,113 @@ +package actiontypes + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + + testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + "github.com/kubeshop/testkube/cmd/testworkflow-init/data" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" +) + +func EnvName(group string, computed bool, sensitive bool, name string) string { + suffix := "" + if computed { + suffix = "C" + } + if sensitive { + suffix += "S" + } + return fmt.Sprintf("_%s%s_%s", group, suffix, name) +} + +func EnvVar(group string, computed, sensitive bool, name, value string) corev1.EnvVar { + return corev1.EnvVar{ + Name: EnvName(group, computed, sensitive, name), + Value: value, + } +} + +func EnvVarFrom(group string, computed, sensitive bool, name string, value corev1.EnvVarSource) corev1.EnvVar { + return corev1.EnvVar{ + Name: EnvName(group, computed, sensitive, name), + ValueFrom: &value, + } +} + +type ActionList []Action + +func NewActionList() ActionList { + return nil +} + +func (a ActionList) Setup(copyInit, copyBinaries bool) ActionList { + return append(a, Action{Setup: &lite.ActionSetup{CopyInit: copyInit, CopyBinaries: copyBinaries}}) +} + +func (a ActionList) Declare(ref string, condition string, parents ...string) ActionList { + return append(a, Action{Declare: &lite.ActionDeclare{Ref: ref, Condition: condition, Parents: parents}}) +} + +func (a ActionList) Start(ref string) ActionList { + return append(a, Action{Start: &ref}) +} + +func (a ActionList) End(ref string) ActionList { + return append(a, Action{End: &ref}) +} + +func (a ActionList) Pause(ref string) ActionList { + return append(a, Action{Pause: &lite.ActionPause{Ref: ref}}) +} + +func (a ActionList) CurrentStatus(expression string) ActionList { + return append(a, Action{CurrentStatus: &expression}) +} + +func (a ActionList) Result(ref, expression string) ActionList { + return append(a, Action{Result: &lite.ActionResult{Ref: ref, Value: expression}}) +} + +func (a ActionList) Execute(ref string, negative bool) ActionList { + return append(a, Action{Execute: &lite.ActionExecute{Ref: ref, Negative: negative}}) +} + +func (a ActionList) MutateContainer(ref string, config testworkflowsv1.ContainerConfig) ActionList { + return append(a, Action{Container: &ActionContainer{Ref: ref, Config: config}}) +} + +func (a ActionList) GetLastRef() string { + for i := len(a) - 1; i >= 0; i-- { + switch a[i].Type() { + case lite.ActionTypeStart: + return *a[i].Start + case lite.ActionTypeSetup: + return data.InitStepName + } + } + return "" +} + +type ActionGroups []ActionList + +func (a ActionGroups) Append(fn func(list ActionList) ActionList) ActionGroups { + return append(a, fn(NewActionList())) +} + +func NewActionGroups() ActionGroups { + return nil +} + +func (a ActionGroups) GetLastRef() (ref string) { + for i := len(a) - 1; i >= 0; i-- { + + for j := len(a[i]) - 1; j >= 0; j-- { + ref = a[i].GetLastRef() + if ref != "" { + return + } + } + } + return +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/containerize.go b/pkg/testworkflows/testworkflowprocessor/action/containerize.go new file mode 100644 index 00000000000..6f56eecd6a3 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/containerize.go @@ -0,0 +1,147 @@ +package action + +import ( + "fmt" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + + testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + constants2 "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + stage2 "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" +) + +func CreateContainer(groupId int, defaultContainer stage2.Container, actions []actiontypes.Action) (cr corev1.Container, actionsCleanup []actiontypes.Action, err error) { + actions = slices.Clone(actions) + actionsCleanup = actions + + // Find the container configurations and executable/setup steps + var setup *actiontypes.Action + executable := map[string]bool{} + containerConfigs := make([]*actiontypes.Action, 0) + for i := range actions { + if actions[i].Container != nil { + containerConfigs = append(containerConfigs, &actions[i]) + } else if actions[i].Setup != nil { + setup = &actions[i] + } else if actions[i].Execute != nil { + executable[actions[i].Execute.Ref] = true + } + } + + // Find the highest priority container configuration + var bestContainerConfig *actiontypes.Action + for i := range containerConfigs { + if executable[containerConfigs[i].Container.Ref] { + bestContainerConfig = containerConfigs[i] + break + } + } + if bestContainerConfig == nil && len(containerConfigs) > 0 { + bestContainerConfig = containerConfigs[len(containerConfigs)-1] + } + if bestContainerConfig == nil { + bestContainerConfig = &actiontypes.Action{Container: &actiontypes.ActionContainer{Config: defaultContainer.ToContainerConfig()}} + } + + // Build the cr base + // TODO: Handle the case when there are multiple exclusive execution configurations + // TODO: Handle a case when that configuration should join multiple configurations (i.e. envs/volumeMounts) + if len(containerConfigs) > 0 { + cr, err = stage2.NewContainer().ApplyCR(&bestContainerConfig.Container.Config).ToKubernetesTemplate() + if err != nil { + return corev1.Container{}, nil, err + } + + // Combine environment variables from each execution + cr.Env = nil + cr.EnvFrom = nil + for i := range containerConfigs { + // TODO: Avoid having multiple copies of the same environment variable + for _, e := range containerConfigs[i].Container.Config.Env { + newEnv := *e.DeepCopy() + computed := strings.Contains(newEnv.Value, "{{") + sensitive := newEnv.ValueFrom != nil && newEnv.ValueFrom.SecretKeyRef != nil + newEnv.Name = actiontypes.EnvName(fmt.Sprintf("%d", i), computed, sensitive, e.Name) + cr.Env = append(cr.Env, newEnv) + } + for _, e := range containerConfigs[i].Container.Config.EnvFrom { + newEnvFrom := *e.DeepCopy() + sensitive := newEnvFrom.SecretRef != nil + newEnvFrom.Prefix = actiontypes.EnvName(fmt.Sprintf("%d", i), false, sensitive, e.Prefix) + cr.EnvFrom = append(cr.EnvFrom, newEnvFrom) + } + } + // TODO: Combine the rest + } + + // Set up a default image when not specified + if cr.Image == "" { + cr.Image = constants.DefaultInitImage + cr.ImagePullPolicy = corev1.PullIfNotPresent + } else if cr.ImagePullPolicy == "" { + cr.ImagePullPolicy = corev1.PullIfNotPresent + } + + // Provide the data required for setup step + if setup != nil { + cr.Env = append(cr.Env, + corev1.EnvVar{Name: fmt.Sprintf("_%s_%s", constants2.EnvGroupDebug, constants2.EnvNodeName), ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }}, + corev1.EnvVar{Name: fmt.Sprintf("_%s_%s", constants2.EnvGroupDebug, constants2.EnvPodName), ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }}, + corev1.EnvVar{Name: fmt.Sprintf("_%s_%s", constants2.EnvGroupDebug, constants2.EnvNamespaceName), ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }}, + corev1.EnvVar{Name: fmt.Sprintf("_%s_%s", constants2.EnvGroupDebug, constants2.EnvServiceAccountName), ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.serviceAccountName"}, + }}, + corev1.EnvVar{Name: fmt.Sprintf("_%s_%s", constants2.EnvGroupActions, constants2.EnvActions), ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: constants.SpecAnnotationFieldPath}, + }}) + + // Apply basic mounts, so there is a state provided + for _, volumeMount := range defaultContainer.VolumeMounts() { + if !slices.ContainsFunc(cr.VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.Name == volumeMount.Name + }) { + cr.VolumeMounts = append(cr.VolumeMounts, volumeMount) + } + } + } + + // Avoid using /.tktw/init if there is Init Process Image - use /init then + initPath := constants.DefaultInitPath + if cr.Image == constants.DefaultInitImage { + initPath = "/init" + } + + // Point the Init Process to the proper group + cr.Name = fmt.Sprintf("%d", groupId+1) + cr.Command = []string{initPath, fmt.Sprintf("%d", groupId)} + cr.Args = nil + + // Clean up the executions + for i := range containerConfigs { + // TODO: Clean it up + newConfig := testworkflowsv1.ContainerConfig{} + if executable[containerConfigs[i].Container.Ref] { + newConfig.Command = containerConfigs[i].Container.Config.Command + newConfig.Args = containerConfigs[i].Container.Config.Args + } + newConfig.WorkingDir = containerConfigs[i].Container.Config.WorkingDir + // TODO: expose more? + + containerConfigs[i].Container = &actiontypes.ActionContainer{ + Ref: containerConfigs[i].Container.Ref, + Config: newConfig, + } + } + + return +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/group.go b/pkg/testworkflows/testworkflowprocessor/action/group.go new file mode 100644 index 00000000000..955e9ee1f54 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/group.go @@ -0,0 +1,66 @@ +package action + +import ( + "slices" + + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" +) + +func Group(actions []actiontypes.Action) (groups [][]actiontypes.Action) { + // Detect "start" and "execute" instructions + startIndexes := make([]int, 0) + startInstructions := make(map[string]int) + containerInstructions := make(map[string]int) + executeInstructions := make(map[string]int) + executeIndexes := make([]int, 0) + for i := range actions { + if actions[i].Start != nil { + startInstructions[*actions[i].Start] = i + startIndexes = append(startIndexes, i) + } else if actions[i].Execute != nil { + executeInstructions[actions[i].Execute.Ref] = i + executeIndexes = append(executeIndexes, i) + } else if actions[i].Container != nil { + containerInstructions[actions[i].Container.Ref] = i + } else if actions[i].Setup != nil { + executeIndexes = append(executeIndexes, i) + } + } + + // Start from end, to fill as much as it's possible + slices.Reverse(executeIndexes) + slices.Reverse(startIndexes) + + // Fast-track when there is only a single instruction to execute + if len(executeIndexes) <= 1 { + return [][]actiontypes.Action{actions} + } + + // Basic behavior: split based on each execute instruction + for _, executeIndex := range executeIndexes { + if actions[executeIndex].Setup != nil { + groups = append([][]actiontypes.Action{actions[executeIndex:]}, groups...) + actions = actions[:executeIndex] + continue + } + ref := actions[executeIndex].Execute.Ref + startIndex := startInstructions[ref] + if containerIndex, ok := containerInstructions[ref]; ok && containerIndex < startIndex { + startIndex = containerIndex + } + + // TODO: Combine multiple operations in a single container if it's possible + + groups = append([][]actiontypes.Action{actions[startIndex:]}, groups...) + actions = actions[:startIndex] + } + if len(actions) > 0 { + groups[0] = append(actions, groups[0]...) + } + + // TODO: Behavior: allow selected Toolkit actions to be executed in the same container + // TODO: Behavior: split based on the image used (use all mounts and variables altogether) + // TODO: Behavior: split based on the image used (isolate variables) + + return groups +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/group_test.go b/pkg/testworkflows/testworkflowprocessor/action/group_test.go new file mode 100644 index 00000000000..464ec6a2291 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/group_test.go @@ -0,0 +1,59 @@ +package action + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + "github.com/kubeshop/testkube/internal/common" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" +) + +func TestGroup_Basic(t *testing.T) { + input := actiontypes.NewActionList(). + // Configure + Setup(true, true). + Declare("init", "true"). + Declare("step1", "false"). + Declare("step2", "true", "init"). + Declare("step3", "true", "init"). + Result("init", "step2 && step3"). + Result("", "init"). + Start(""). + CurrentStatus("true"). + Start("init"). + // Step 1 is never running + CurrentStatus("init"). + Start("step1"). + CurrentStatus("init"). + + // Run step 2 + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + CurrentStatus("init"). + + // Run step 3 + MutateContainer("step3", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step3"). + Execute("step3", false). + End("step3"). + End("init"). + End("") + + want := [][]actiontypes.Action{ + input[:13], // ends before containerConfig("step2") + input[13:18], // ends before containerConfig("step3") + input[18:], + } + got := Group(input) + assert.Equal(t, want, got) +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/optimize.go b/pkg/testworkflows/testworkflowprocessor/action/optimize.go new file mode 100644 index 00000000000..d7b24fc3c5d --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/optimize.go @@ -0,0 +1,324 @@ +package action + +import ( + "fmt" + "reflect" + "regexp" + "strings" + + "k8s.io/apimachinery/pkg/util/rand" + + "github.com/kubeshop/testkube/internal/common" + "github.com/kubeshop/testkube/pkg/expressions" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" +) + +func optimize(actions []actiontypes.Action) ([]actiontypes.Action, error) { + // Detect all the step references + refs := make(map[string]struct{}) + executableRefs := make(map[string]struct{}) + for i := range actions { + if actions[i].Result != nil { + refs[actions[i].Result.Ref] = struct{}{} + } + + if actions[i].Execute != nil { + refs[actions[i].Execute.Ref] = struct{}{} + executableRefs[actions[i].Execute.Ref] = struct{}{} + } + if actions[i].End != nil { + refs[*actions[i].End] = struct{}{} + executableRefs[*actions[i].End] = struct{}{} + } + } + + // Delete empty `container` declarations + for i := 0; i < len(actions); i++ { + if actions[i].Container != nil && reflect.ValueOf(actions[i].Container.Config).IsZero() { + actions = append(actions[0:i], actions[i+1:]...) + i-- + } + } + + // Wrap all the references with boolean function, and simplify values + refReplacements := make(map[string]string) + refResults := make(map[string]string) + wrapStartRef := expressions.NewMachine().RegisterAccessor(func(name string) (interface{}, bool) { + if _, ok := executableRefs[name]; !ok { + return nil, false + } + if _, ok := refReplacements[name]; !ok { + hashStart := rand.String(10) + hashEnd := rand.String(10) + refReplacements[name] = fmt.Sprintf("_%s_%s_%s_", hashStart, name, hashEnd) + refResults[refReplacements[name]] = fmt.Sprintf("bool(%s)", name) + } + return expressions.MustCompile(refReplacements[name]), true + }) + wrapEndRef := expressions.NewMachine().RegisterAccessor(func(name string) (interface{}, bool) { + if result, ok := refResults[name]; ok { + return expressions.MustCompile(result), true + } + return nil, false + }) + for i := range actions { + if actions[i].CurrentStatus != nil { + actions[i].CurrentStatus = common.Ptr(simplifyExpression(*actions[i].CurrentStatus, wrapStartRef)) + actions[i].CurrentStatus = common.Ptr(simplifyExpression(*actions[i].CurrentStatus, wrapEndRef)) + } + if actions[i].Declare != nil { + actions[i].Declare.Condition = simplifyExpression(actions[i].Declare.Condition, wrapStartRef) + actions[i].Declare.Condition = simplifyExpression(actions[i].Declare.Condition, wrapEndRef) + } + if actions[i].Result != nil { + actions[i].Result.Value = simplifyExpression(actions[i].Result.Value, wrapStartRef) + actions[i].Result.Value = simplifyExpression(actions[i].Result.Value, wrapEndRef) + } + } + + // Detect immediately skipped steps + skipped := make(map[string]struct{}) + for i := range actions { + if actions[i].Declare != nil { + v, err := expressions.EvalExpressionPartial(actions[i].Declare.Condition) + if err == nil && v.Static() != nil { + b, err := v.Static().BoolValue() + if err == nil && !b { + skipped[actions[i].Declare.Ref] = struct{}{} + } + } + } + } + + // List all the results + results := make(map[string]expressions.Expression) + conditions := make(map[string]expressions.Expression) + for i := range actions { + if actions[i].Result != nil { + var err error + refs[actions[i].Result.Ref] = struct{}{} + results[actions[i].Result.Ref], err = expressions.EvalExpressionPartial(actions[i].Result.Value) + if err != nil { + return nil, err + } + } + + if actions[i].Declare != nil { + var err error + conditions[actions[i].Declare.Ref], err = expressions.EvalExpressionPartial(actions[i].Declare.Condition) + if err != nil { + return nil, err + } + } + + if actions[i].Execute != nil { + refs[actions[i].Execute.Ref] = struct{}{} + } + } + + // Pre-resolve conditions + currentStatus := expressions.MustCompile("true") + executed := make(map[string]struct{}) + for i := range actions { + // Update current status + if actions[i].CurrentStatus != nil { + var err error + currentStatus, err = expressions.Compile(*actions[i].CurrentStatus) + if err != nil { + return nil, err + } + } + + // Mark step as executed + if actions[i].Execute != nil { + executed[actions[i].Execute.Ref] = struct{}{} + } else if actions[i].End != nil { + executed[*actions[i].End] = struct{}{} + } + + // Simplify the condition + if actions[i].Declare != nil { + // TODO: Handle `never` and other aliases + machine := expressions.NewMachine().RegisterAccessor(func(name string) (interface{}, bool) { + if name == "passed" || name == "success" { + return currentStatus, true + } else if name == "failed" || name == "error" { + return expressions.MustCompile("!passed"), true + } else if _, ok := skipped[name]; ok { + return true, true + } else if v, ok := results[name]; ok { + // Ignore steps that didn't execute yet + if _, ok := executed[name]; !ok { + return true, true + } + // Do not go deeper if the result is not determined yet + if v.Static() == nil { + return nil, false + } + c, ok2 := conditions[name] + if ok2 { + return expressions.MustCompile(fmt.Sprintf(`(%s) && (%s)`, c.String(), v.String())), true + } + return v, true + } else if _, ok := refs[name]; ok { + // Ignore steps that didn't execute yet + if _, ok := executed[name]; !ok { + return true, true + } + return nil, false + } + return nil, false + }) + actions[i].Declare.Condition = simplifyExpression(actions[i].Declare.Condition, machine) + conditions[actions[i].Declare.Ref] = expressions.MustCompile(actions[i].Declare.Condition) + for _, parentRef := range actions[i].Declare.Parents { + if _, ok := skipped[parentRef]; ok { + actions[i].Declare.Condition = "false" + break + } + } + } + } + + // Avoid unnecessary casting to boolean + uncastRegex := regexp.MustCompile(`bool\([^)]+\)`) + uncastBoolRefs := func(expr string) string { + return uncastRegex.ReplaceAllStringFunc(expr, func(s string) string { + ref := s[5 : len(s)-1] + if _, ok := refs[ref]; ok { + return ref + } + return s + }) + } + for i := range actions { + if actions[i].CurrentStatus != nil { + actions[i].CurrentStatus = common.Ptr(uncastBoolRefs(*actions[i].CurrentStatus)) + } + if actions[i].Declare != nil { + actions[i].Declare.Condition = uncastBoolRefs(actions[i].Declare.Condition) + } + if actions[i].Result != nil { + actions[i].Result.Value = uncastBoolRefs(actions[i].Result.Value) + } + } + + // Detect immediately skipped steps + skipped = make(map[string]struct{}) + for i := range actions { + if actions[i].Declare != nil { + v, err := expressions.EvalExpressionPartial(actions[i].Declare.Condition) + if err == nil && v.Static() != nil { + b, err := v.Static().BoolValue() + if err == nil && !b { + skipped[actions[i].Declare.Ref] = struct{}{} + } + } + } + } + + // Avoid executing skipped steps (Execute, Timeout, Retry, Result & End) + for i := 0; i < len(actions); i++ { + if actions[i].Execute != nil { + if _, ok := skipped[actions[i].Execute.Ref]; ok { + actions = append(actions[:i], actions[i+1:]...) + i-- + } + } + if actions[i].Result != nil { + if _, ok := skipped[actions[i].Result.Ref]; ok { + actions = append(actions[:i], actions[i+1:]...) + i-- + } + } + if actions[i].Timeout != nil { + if _, ok := skipped[actions[i].Timeout.Ref]; ok { + actions = append(actions[:i], actions[i+1:]...) + i-- + } + } + if actions[i].Retry != nil { + if _, ok := skipped[actions[i].Retry.Ref]; ok { + actions = append(actions[:i], actions[i+1:]...) + i-- + } + } + if actions[i].Pause != nil { + if _, ok := skipped[actions[i].Pause.Ref]; ok { + actions = append(actions[:i], actions[i+1:]...) + i-- + } + } + if actions[i].Container != nil { + if _, ok := skipped[actions[i].Container.Ref]; ok { + actions = append(actions[:i], actions[i+1:]...) + i-- + } + } + } + + // Ignore parents for already statically skipped conditions + for i := range actions { + if actions[i].Declare != nil { + if _, ok := skipped[actions[i].Declare.Ref]; ok { + actions[i].Declare.Parents = nil + } + } + } + + // TODO: Avoid using /.tktw/toolkit if there is Toolkit image + + // Avoid using /.tktw/bin/sh when it is internal image used, with binaries in /bin + for i := range actions { + if actions[i].Type() != lite.ActionTypeContainerTransition { + continue + } + if actions[i].Container.Config.Image != constants.DefaultInitImage && actions[i].Container.Config.Image != constants.DefaultToolkitImage { + continue + } + if actions[i].Container.Config.Command != nil && len(*actions[i].Container.Config.Command) > 0 && strings.HasPrefix((*actions[i].Container.Config.Command)[0], constants.InternalBinPath+"/") { + (*actions[i].Container.Config.Command)[0] = "/bin" + (*actions[i].Container.Config.Command)[0][len(constants.InternalBinPath):] + } + } + + // Avoid copying init process and common binaries, when it is not necessary + copyInit := false + copyBinaries := false + for i := range actions { + if actions[i].Type() == lite.ActionTypeContainerTransition { + if actions[i].Container.Config.Image != constants.DefaultInitImage { + copyInit = true + if actions[i].Container.Config.Image != constants.DefaultToolkitImage { + copyBinaries = true + } + } + } + } + for i := range actions { + if actions[i].Type() == lite.ActionTypeSetup { + actions[i].Setup.CopyInit = copyInit + actions[i].Setup.CopyBinaries = copyBinaries + } + } + + // Get rid of skipped steps from initial statuses and results + skipMachine := expressions.NewMachine(). + RegisterAccessor(func(name string) (interface{}, bool) { + if _, ok := skipped[name]; ok { + return true, true + } + return nil, false + }) + for i := range actions { + if actions[i].CurrentStatus != nil { + actions[i].CurrentStatus = common.Ptr(simplifyExpression(*actions[i].CurrentStatus, skipMachine)) + } + if actions[i].Result != nil { + actions[i].Result.Value = simplifyExpression(actions[i].Result.Value, skipMachine) + } + } + + return actions, nil +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/process.go b/pkg/testworkflows/testworkflowprocessor/action/process.go new file mode 100644 index 00000000000..521b3c6a8b1 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/process.go @@ -0,0 +1,152 @@ +package action + +import ( + "fmt" + "strings" + + "github.com/pkg/errors" + + "github.com/kubeshop/testkube/internal/common" + "github.com/kubeshop/testkube/pkg/expressions" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" + stage2 "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" +) + +func process(currentStatus string, parents []string, stage stage2.Stage, machines ...expressions.Machine) (actions []actiontypes.Action, err error) { + // Store the init status + actions = append(actions, actiontypes.Action{ + CurrentStatus: common.Ptr(currentStatus), + }) + + // Compute the skip condition + condition := stage.Condition() + if condition == "" || condition == "null" { + condition = "passed" + } + actions = append(actions, actiontypes.Action{ + Declare: &lite.ActionDeclare{Ref: stage.Ref(), Condition: condition, Parents: parents}, + }) + + // Configure the container for action + var containerConfig stage2.Container + if group, ok := stage.(stage2.GroupStage); ok { + containerConfig = group.ContainerDefaults() + } else { + containerConfig = stage.(stage2.ContainerStage).Container() + } + if containerConfig != nil { + c := containerConfig.Detach() + err = c.Resolve(machines...) + if err != nil { + return nil, err + } + actions = append(actions, actiontypes.Action{ + Container: &actiontypes.ActionContainer{Ref: stage.Ref(), Config: c.ToContainerConfig()}, + }) + } + + // Mark the current operation as started + actions = append(actions, actiontypes.Action{ + Start: common.Ptr(stage.Ref()), + }) + + // Store the timeout information + if stage.Timeout() != "" { + actions = append(actions, actiontypes.Action{ + Timeout: &lite.ActionTimeout{Ref: stage.Ref(), Timeout: stage.Timeout()}, + }) + } + + // Store the retry condition + if stage.RetryPolicy().Count != 0 { + actions = append(actions, actiontypes.Action{ + Retry: &lite.ActionRetry{Ref: stage.Ref(), Count: stage.RetryPolicy().Count, Until: stage.RetryPolicy().Until}, + }) + } + + // Handle pause + if stage.Paused() { + actions = append(actions, actiontypes.Action{ + Pause: &lite.ActionPause{Ref: stage.Ref()}, + }) + } + + // Handle executable action + if exec, ok := stage.(stage2.ContainerStage); ok { + actions = append(actions, actiontypes.Action{ + Execute: &lite.ActionExecute{ + Ref: exec.Ref(), + Negative: exec.Negative(), + Toolkit: exec.IsToolkit(), + }, + }) + } + + // Handle group + if group, ok := stage.(stage2.GroupStage); ok { + // Build initial status for children + if currentStatus == "true" { + currentStatus = stage.Ref() + } else { + currentStatus = fmt.Sprintf("%s && %s", stage.Ref(), currentStatus) + } + parents = append(parents, group.Ref()) + + // Handle children + refs := make([]string, 0) + for _, ch := range group.Children() { + sub, err := process(currentStatus, parents, ch, machines...) + if err != nil { + return nil, errors.Wrap(err, "processing group children") + } + if !ch.Optional() { + currentStatus = fmt.Sprintf("%s && %s", ch.Ref(), currentStatus) + refs = append(refs, ch.Ref()) + } + actions = append(actions, sub...) + } + + // Handle results + result := "true" + if group.Negative() { + result = "false" + } + if len(refs) > 0 { + if group.Negative() { + result = strings.Join(common.MapSlice(refs, func(ref string) string { + return "!" + ref + }), "||") + } else { + result = strings.Join(refs, "&&") + } + } + actions = append(actions, actiontypes.Action{Result: &lite.ActionResult{Ref: group.Ref(), Value: result}}) + } + + // Mark the current operation as finished + actions = append(actions, actiontypes.Action{ + End: common.Ptr(stage.Ref()), + }) + + return +} + +func Process(root stage2.Stage, machines ...expressions.Machine) ([]actiontypes.Action, error) { + actions, err := process("true", nil, root, machines...) + if err != nil { + return nil, err + } + actions = append([]actiontypes.Action{{Setup: &lite.ActionSetup{CopyInit: true, CopyBinaries: true}}, {Start: common.Ptr("")}}, actions...) + actions = append(actions, actiontypes.Action{Result: &lite.ActionResult{Ref: "", Value: root.Ref()}}, actiontypes.Action{End: common.Ptr("")}) + + // Optimize until simplest list of operations + for { + prevLength := len(actions) + actions, err = optimize(actions) + if err != nil || len(actions) == prevLength { + sort(actions) + return actions, errors.Wrap(err, "processing operations") + } + } +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/process_test.go b/pkg/testworkflows/testworkflowprocessor/action/process_test.go new file mode 100644 index 00000000000..38ccf374531 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/process_test.go @@ -0,0 +1,651 @@ +package action + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + "github.com/kubeshop/testkube/internal/common" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" +) + +func TestProcess_BasicSteps(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + root.Add(stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b"))) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("step1", "true", "init"). + Declare("step2", "step1", "init"). + + // Declare group resolutions + Result("init", "step1&&step2"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("init"). + MutateContainer("step1", testworkflowsv1.ContainerConfig{ + Image: "image:1.2.3", + Command: common.Ptr([]string{"a", "b"}), + }). + Start("step1"). + Execute("step1", false). + End("step1"). + + // Run the step 2 + CurrentStatus("step1&&init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_Grouping(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + root.Add(stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b"))) + group1 := stage.NewGroupStage("group1", false) + group1.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + group1.Add(stage.NewContainerStage("step3", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + root.Add(group1) + root.Add(stage.NewContainerStage("step4", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("step1", "true", "init"). + Declare("group1", "step1", "init"). + Declare("step2", "step1", "init", "group1"). + Declare("step3", "step2&&step1", "init", "group1"). + Declare("step4", "group1&&step1", "init"). + + // Declare group resolutions + Result("group1", "step2&&step3"). + Result("init", "step1&&group1&&step4"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("init"). + MutateContainer("step1", testworkflowsv1.ContainerConfig{ + Image: "image:1.2.3", + Command: common.Ptr([]string{"a", "b"}), + }). + Start("step1"). + Execute("step1", false). + End("step1"). + + // Start the group 1 + CurrentStatus("step1&&init"). + Start("group1"). + + // Run the step 2 + CurrentStatus("group1&&step1&&init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Run the step 2 + CurrentStatus("step2&&group1&&step1&&init"). + MutateContainer("step3", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step3"). + Execute("step3", false). + End("step3"). + + // End the group 1 + End("group1"). + + // Run the step 4 + CurrentStatus("group1&&step1&&init"). + MutateContainer("step4", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step4"). + Execute("step4", false). + End("step4"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_Pause(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + step1 := stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b")) + step1.SetPaused(true) + root.Add(step1) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("step1", "true", "init"). + Declare("step2", "step1", "init"). + + // Declare information about potential pauses + Pause("step1"). + + // Declare group resolutions + Result("init", "step1&&step2"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("init"). + MutateContainer("step1", testworkflowsv1.ContainerConfig{ + Image: "image:1.2.3", + Command: common.Ptr([]string{"a", "b"}), + }). + Start("step1"). + Execute("step1", false). + End("step1"). + + // Run the step 2 + CurrentStatus("step1&&init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_NegativeStep(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + step1 := stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b")) + step1.SetNegative(true) + root.Add(step1) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("step1", "true", "init"). + Declare("step2", "step1", "init"). + + // Declare group resolutions + Result("init", "step1&&step2"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("init"). + MutateContainer("step1", testworkflowsv1.ContainerConfig{ + Image: "image:1.2.3", + Command: common.Ptr([]string{"a", "b"}), + }). + Start("step1"). + Execute("step1", true). + End("step1"). + + // Run the step 2 + CurrentStatus("step1&&init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_NegativeGroup(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + root.SetNegative(true) + root.Add(stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b"))) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("step1", "true", "init"). + Declare("step2", "step1", "init"). + + // Declare group resolutions + Result("init", "!step1||!step2"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("init"). + MutateContainer("step1", testworkflowsv1.ContainerConfig{ + Image: "image:1.2.3", + Command: common.Ptr([]string{"a", "b"}), + }). + Start("step1"). + Execute("step1", false). + End("step1"). + + // Run the step 2 + CurrentStatus("step1&&init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_OptionalStep(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + step1 := stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b")) + step1.SetOptional(true) + root.Add(step1) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("step1", "true", "init"). + Declare("step2", "true", "init"). // because step1 is optional + + // Declare group resolutions + Result("init", "step2"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("init"). + MutateContainer("step1", testworkflowsv1.ContainerConfig{ + Image: "image:1.2.3", + Command: common.Ptr([]string{"a", "b"}), + }). + Start("step1"). + Execute("step1", false). + End("step1"). + + // Run the step 2 + CurrentStatus("init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_OptionalGroup(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + group := stage.NewGroupStage("inner", false) + group.SetOptional(true) + group.Add(stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b"))) + group.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + root.Add(group) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("inner", "true", "init"). + Declare("step1", "true", "init", "inner"). + Declare("step2", "step1", "init", "inner"). + + // Declare group resolutions + Result("inner", "step1&&step2"). + Result("init", "true"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + CurrentStatus("init"). + Start("inner"). + + // Run the step 1 + CurrentStatus("inner&&init"). + MutateContainer("step1", testworkflowsv1.ContainerConfig{ + Image: "image:1.2.3", + Command: common.Ptr([]string{"a", "b"}), + }). + Start("step1"). + Execute("step1", false). + End("step1"). + + // Run the step 2 + CurrentStatus("step1&&inner&&init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Finish + End("inner"). + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_IgnoreExecutionOfStaticSkip(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + step1 := stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b")) + step1.SetCondition("false") + root.Add(step1) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("step1", "false"). + Declare("step2", "true", "init"). // because step1 is skipped + + // Declare group resolutions + Result("init", "step2"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("init"). + Start("step1"). // don't execute as it is skipped + End("step1"). + + // Run the step 2 + CurrentStatus("init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_IgnoreExecutionOfStaticSkipGroup(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + root.SetCondition("false") + root.Add(stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b"))) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(false, false). // don't copy as there is nothing to do + + // Declare stage conditions + Declare("init", "false"). + Declare("step1", "false"). + Declare("step2", "false"). + + // Declare group resolutions + Result("", "true"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("true"). + Start("step1"). // don't execute as it is skipped + End("step1"). + + // Run the step 2 + CurrentStatus("true"). + Start("step2"). // don't execute as it is skipped + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_IgnoreExecutionOfStaticSkipGroup_Pause(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + root.SetCondition("false") + root.SetPaused(true) + root.Add(stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b"))) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(false, false). // don't copy as there is nothing to do + + // Declare stage conditions + Declare("init", "false"). + Declare("step1", "false"). + Declare("step2", "false"). + + //Pause("init"). // ignored as it's not executed + + // Declare group resolutions + Result("", "true"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("true"). + Start("step1"). // don't execute as it is skipped + End("step1"). + + // Run the step 2 + CurrentStatus("true"). + Start("step2"). // don't execute as it is skipped + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} + +func TestProcess_IgnoreExecutionOfStaticSkip_PauseGroup(t *testing.T) { + // Build the structure + root := stage.NewGroupStage("init", false) + root.SetPaused(true) + step1 := stage.NewContainerStage("step1", stage.NewContainer().SetImage("image:1.2.3").SetCommand("a", "b")) + step1.SetCondition("false") + root.Add(step1) + root.Add(stage.NewContainerStage("step2", stage.NewContainer().SetImage("image:3.2.1").SetCommand("c", "d"))) + + // Build the expectations + want := actiontypes.NewActionList(). + // Configure + Setup(true, true). + + // Declare stage conditions + Declare("init", "true"). + Declare("step1", "false"). + Declare("step2", "true", "init"). // because step1 is skipped + + // Declare information about potential pauses + Pause("init"). + + // Declare group resolutions + Result("init", "step2"). + Result("", "init"). + + // Initialize + Start(""). + CurrentStatus("true"). + Start("init"). + + // Run the step 1 + CurrentStatus("init"). + Start("step1"). // don't execute as it is skipped + End("step1"). + + // Run the step 2 + CurrentStatus("init"). + MutateContainer("step2", testworkflowsv1.ContainerConfig{ + Image: "image:3.2.1", + Command: common.Ptr([]string{"c", "d"}), + }). + Start("step2"). + Execute("step2", false). + End("step2"). + + // Finish + End("init"). + End("") + + // Assert + got, err := Process(root) + assert.NoError(t, err) + assert.Equal(t, want, actiontypes.ActionList(got)) +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/sort.go b/pkg/testworkflows/testworkflowprocessor/action/sort.go new file mode 100644 index 00000000000..78869bcebca --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/sort.go @@ -0,0 +1,75 @@ +package action + +import ( + "slices" + + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" +) + +func sort(actions []actiontypes.Action) { + // Move retry policies to top + slices.SortStableFunc(actions, func(a actiontypes.Action, b actiontypes.Action) int { + if (a.Retry == nil) == (b.Retry == nil) { + return 0 + } + if a.Retry == nil { + return 1 + } + return -1 + }) + + // Move timeouts to top + slices.SortStableFunc(actions, func(a actiontypes.Action, b actiontypes.Action) int { + if (a.Timeout == nil) == (b.Timeout == nil) { + return 0 + } + if a.Timeout == nil { + return 1 + } + return -1 + }) + + // Move results to top + slices.SortStableFunc(actions, func(a actiontypes.Action, b actiontypes.Action) int { + if (a.Result == nil) == (b.Result == nil) { + return 0 + } + if a.Result == nil { + return 1 + } + return -1 + }) + + // Move pause information to top + slices.SortStableFunc(actions, func(a actiontypes.Action, b actiontypes.Action) int { + if (a.Pause == nil) == (b.Pause == nil) { + return 0 + } + if a.Pause == nil { + return 1 + } + return -1 + }) + + // Move declarations to top + slices.SortStableFunc(actions, func(a actiontypes.Action, b actiontypes.Action) int { + if (a.Declare == nil) == (b.Declare == nil) { + return 0 + } + if a.Declare == nil { + return 1 + } + return -1 + }) + + // Move setup to top + slices.SortStableFunc(actions, func(a actiontypes.Action, b actiontypes.Action) int { + if (a.Setup == nil) == (b.Setup == nil) { + return 0 + } + if a.Setup == nil { + return 1 + } + return -1 + }) +} diff --git a/pkg/testworkflows/testworkflowprocessor/action/utils.go b/pkg/testworkflows/testworkflowprocessor/action/utils.go new file mode 100644 index 00000000000..6532947eee6 --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/action/utils.go @@ -0,0 +1,11 @@ +package action + +import "github.com/kubeshop/testkube/pkg/expressions" + +func simplifyExpression(expr string, machines ...expressions.Machine) string { + v, err := expressions.EvalExpressionPartial(expr, machines...) + if err == nil { + return v.String() + } + return expr +} diff --git a/pkg/testworkflows/testworkflowprocessor/bundle.go b/pkg/testworkflows/testworkflowprocessor/bundle.go index 69e3d9e8ea0..2000a8fad61 100644 --- a/pkg/testworkflows/testworkflowprocessor/bundle.go +++ b/pkg/testworkflows/testworkflowprocessor/bundle.go @@ -2,19 +2,35 @@ package testworkflowprocessor import ( "context" + "encoding/json" "github.com/pkg/errors" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" ) type Bundle struct { Secrets []corev1.Secret ConfigMaps []corev1.ConfigMap Job batchv1.Job - Signature []Signature + Signature []stage.Signature +} + +func (b *Bundle) Actions() (actions actiontypes.ActionGroups) { + _ = json.Unmarshal([]byte(b.Job.Spec.Template.Annotations[constants.SpecAnnotationName]), &actions) + return +} + +func (b *Bundle) LiteActions() (actions lite.LiteActionGroups) { + _ = json.Unmarshal([]byte(b.Job.Spec.Template.Annotations[constants.SpecAnnotationName]), &actions) + return } func (b *Bundle) Deploy(ctx context.Context, clientSet kubernetes.Interface, namespace string) (err error) { diff --git a/pkg/testworkflows/testworkflowprocessor/constants/constants.go b/pkg/testworkflows/testworkflowprocessor/constants/constants.go index 70e6a3b3da8..a0fc38c86cd 100644 --- a/pkg/testworkflows/testworkflowprocessor/constants/constants.go +++ b/pkg/testworkflows/testworkflowprocessor/constants/constants.go @@ -5,12 +5,10 @@ import ( "fmt" "os" "path/filepath" - "strings" corev1 "k8s.io/api/core/v1" testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" - "github.com/kubeshop/testkube/cmd/testworkflow-init/data" "github.com/kubeshop/testkube/pkg/version" ) @@ -23,8 +21,11 @@ const ( RootResourceIdLabelName = "testworkflowid-root" GroupIdLabelName = "testworkflowid-group" SignatureAnnotationName = "testworkflows.testkube.io/signature" + SpecAnnotationName = "testworkflows.testkube.io/spec" + SpecAnnotationFieldPath = "metadata.annotations['" + SpecAnnotationName + "']" RFC3339Millis = "2006-01-02T15:04:05.000Z07:00" OpenSourceOperationErrorMessage = "operation is not available when running the Testkube Agent in the standalone mode" + RootOperationName = "root" ) var ( @@ -35,32 +36,6 @@ var ( DefaultTransferDirPath = filepath.Join(DefaultInternalPath, "transfer") DefaultTmpDirPath = filepath.Join(DefaultInternalPath, "tmp") DefaultTransferPort = 60433 - InitScript = strings.TrimSpace(strings.NewReplacer( - "", InternalBinPath, - "", DefaultInitPath, - "", DefaultStatePath, - "", DefaultTerminationLogPath, - "", strings.ReplaceAll(data.SprintOutput("tktw-init", "pod", map[string]string{ - "name": "$TK_DEBUG_POD", - "nodeName": "$TK_DEBUG_NODE", - "namespace": "$TK_DEBUG_NS", - "serviceAccountName": "$TK_DEBUG_SVC", - "agent": version.Version, - "toolkit": stripCommonImagePrefix(getToolkitImage(), "testkube-tw-toolkit"), - "init": stripCommonImagePrefix(getInitImage(), "testkube-tw-init"), - }), "\"", "\\\""), - ).Replace(` -set -e -trap '[ $? -eq 0 ] && exit 0 || echo -n "failed,1" > && exit 1' EXIT -echo "Configuring state..." -touch && chmod 777 -echo "Configuring init process..." -cp /init -echo "Configuring shell..." -cp -rf /bin /.tktw/bin -echo -n "" -echo -n ',0' > && echo 'Done.' && exit 0 - `)) DefaultShellHeader = "set -e\n" DefaultContainerConfig = testworkflowsv1.ContainerConfig{ Image: DefaultInitImage, @@ -75,21 +50,6 @@ echo -n ',0' > && echo 'Done.' && exit 0 ErrOpenSourceServicesOperationIsNotAvailable = errors.New(`"services" ` + OpenSourceOperationErrorMessage) ) -func stripCommonImagePrefix(image, common string) string { - if !strings.HasPrefix(image, "docker.io/") { - return image - } - image = image[10:] - if !strings.HasPrefix(image, "kubeshop/") { - return image - } - image = image[9:] - if !strings.HasPrefix(image, common+":") { - return image - } - return image[len(common)+1:] -} - func getInitImage() string { img := os.Getenv("TESTKUBE_TW_INIT_IMAGE") if img == "" { diff --git a/pkg/testworkflows/testworkflowprocessor/initprocess.go b/pkg/testworkflows/testworkflowprocessor/initprocess.go deleted file mode 100644 index 51bdf8d91ce..00000000000 --- a/pkg/testworkflows/testworkflowprocessor/initprocess.go +++ /dev/null @@ -1,231 +0,0 @@ -package testworkflowprocessor - -import ( - "errors" - "fmt" - "maps" - "strconv" - "strings" - - testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" - "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" - "github.com/kubeshop/testkube/internal/common" - "github.com/kubeshop/testkube/pkg/expressions" - constants2 "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" -) - -type initProcess struct { - ref string - workingDir string - init []string - params []string - retry map[string]testworkflowsv1.RetryPolicy - paused bool - command []string - args []string - envs []string - results []string - conditions map[string][]string - negative bool - toolkit bool - errors []error -} - -func NewInitProcess() *initProcess { - return &initProcess{ - conditions: map[string][]string{}, - retry: map[string]testworkflowsv1.RetryPolicy{}, - } -} - -func (p *initProcess) Error() error { - if len(p.errors) == 0 { - return nil - } - return errors.Join(p.errors...) -} - -func (p *initProcess) SetRef(ref string) *initProcess { - p.ref = ref - return p -} - -func (p *initProcess) Command() []string { - args := p.params - - // TODO: Support nested retries - policy, ok := p.retry[p.ref] - if ok { - args = append(args, constants.ArgRetryCount, strconv.Itoa(int(policy.Count)), constants.ArgRetryUntil, expressions.Escape(policy.Until)) - } - if p.negative { - args = append(args, constants.ArgNegative, "true") - } - if len(p.init) > 0 { - args = append(args, constants.ArgInit, strings.Join(p.init, "&&")) - } - if len(p.envs) > 0 { - args = append(args, constants.ArgComputeEnv, strings.Join(p.envs, ",")) - } - if len(p.conditions) > 0 { - for k, v := range p.conditions { - args = append(args, constants.ArgCondition, fmt.Sprintf("%s=%s", strings.Join(common.UniqueSlice(v), ","), k)) - } - } - for _, r := range p.results { - args = append(args, constants.ArgResult, r) - } - if p.paused { - args = append(args, constants.ArgPaused) - } - if p.workingDir != "" { - args = append(args, constants.ArgWorkingDir, p.workingDir) - } - if p.toolkit { - args = append(args, constants.ArgToolkit) - } - return append([]string{constants2.DefaultInitPath, p.ref}, append(args, constants.ArgSeparator)...) -} - -func (p *initProcess) Args() []string { - args := make([]string, 0) - if len(p.command) > 0 { - args = p.command - } - if len(p.command) > 0 || len(p.args) > 0 { - args = append(args, p.args...) - } - return args -} - -func (p *initProcess) param(args ...string) *initProcess { - p.params = append(p.params, args...) - return p -} - -func (p *initProcess) compile(expr ...string) []string { - for i, e := range expr { - res, err := expressions.Compile(e) - if err == nil { - expr[i] = res.String() - } else { - p.errors = append(p.errors, fmt.Errorf("resolving expression: %s: %s", expr[i], err.Error())) - } - } - return expr -} - -func (p *initProcess) SetCommand(command ...string) *initProcess { - p.command = command - return p -} - -func (p *initProcess) SetArgs(args ...string) *initProcess { - p.args = args - return p -} - -func (p *initProcess) SetToolkit(toolkit bool) *initProcess { - p.toolkit = toolkit - return p -} - -func (p *initProcess) AddTimeout(duration string, refs ...string) *initProcess { - return p.param(constants.ArgTimeout, fmt.Sprintf("%s=%s", strings.Join(refs, ","), duration)) -} - -func (p *initProcess) SetInitialStatus(expr ...string) *initProcess { - p.init = nil - for _, v := range p.compile(expr...) { - p.init = append(p.init, v) - } - return p -} - -func (p *initProcess) PrependInitialStatus(expr ...string) *initProcess { - init := []string(nil) - for _, v := range p.compile(expr...) { - init = append(init, v) - } - p.init = append(init, p.init...) - return p -} - -func (p *initProcess) AddComputedEnvs(names ...string) *initProcess { - p.envs = append(p.envs, names...) - return p -} - -func (p *initProcess) SetNegative(negative bool) *initProcess { - p.negative = negative - return p -} - -func (p *initProcess) AddResult(condition string, refs ...string) *initProcess { - if len(refs) == 0 || condition == "" { - return p - } - p.results = append(p.results, fmt.Sprintf("%s=%s", strings.Join(refs, ","), p.compile(condition)[0])) - return p -} - -func (p *initProcess) ResetResults() *initProcess { - p.results = nil - return p -} - -func (p *initProcess) AddCondition(condition string, refs ...string) *initProcess { - if len(refs) == 0 || condition == "" { - return p - } - expr := p.compile(condition)[0] - p.conditions[expr] = append(p.conditions[expr], refs...) - return p -} - -func (p *initProcess) ResetCondition() *initProcess { - p.conditions = make(map[string][]string) - return p -} - -func (p *initProcess) AddRetryPolicy(policy testworkflowsv1.RetryPolicy, ref string) *initProcess { - if policy.Count <= 0 { - delete(p.retry, ref) - return p - } - until := policy.Until - if until == "" { - until = "passed" - } - p.retry[ref] = testworkflowsv1.RetryPolicy{Count: policy.Count, Until: until} - return p -} - -func (p *initProcess) SetPaused(paused bool) *initProcess { - p.paused = paused - return p -} - -func (p *initProcess) SetWorkingDir(workingDir string) *initProcess { - p.workingDir = workingDir - return p -} - -func (p *initProcess) Children(ref string) *initProcess { - return &initProcess{ - ref: ref, - params: p.params, - retry: maps.Clone(p.retry), - paused: p.paused, - command: p.command, - args: p.args, - workingDir: p.workingDir, - init: p.init, - envs: p.envs, - results: p.results, - conditions: maps.Clone(p.conditions), - negative: p.negative, - toolkit: p.toolkit, - errors: p.errors, - } -} diff --git a/pkg/testworkflows/testworkflowprocessor/intermediate.go b/pkg/testworkflows/testworkflowprocessor/intermediate.go index 508060d9deb..b92b268338a 100644 --- a/pkg/testworkflows/testworkflowprocessor/intermediate.go +++ b/pkg/testworkflows/testworkflowprocessor/intermediate.go @@ -6,6 +6,7 @@ import ( corev1 "k8s.io/api/core/v1" testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowresolver" ) @@ -13,7 +14,7 @@ import ( type Intermediate interface { RefCounter - ContainerDefaults() Container + ContainerDefaults() stage.Container PodConfig() testworkflowsv1.PodConfig JobConfig() testworkflowsv1.JobConfig @@ -38,8 +39,8 @@ type intermediate struct { RefCounter // Routine - Root GroupStage `expr:"include"` - Container Container `expr:"include"` + Root stage.GroupStage `expr:"include"` + Container stage.Container `expr:"include"` // Job & Pod resources & data Pod testworkflowsv1.PodConfig `expr:"include"` @@ -57,12 +58,12 @@ func NewIntermediate() Intermediate { ref := NewRefCounter() return &intermediate{ RefCounter: ref, - Root: NewGroupStage("", true), - Container: NewContainer(), + Root: stage.NewGroupStage("", true), + Container: stage.NewContainer(), Files: NewConfigMapFiles(fmt.Sprintf("{{resource.id}}-%s", ref.NextRef()), nil)} } -func (s *intermediate) ContainerDefaults() Container { +func (s *intermediate) ContainerDefaults() stage.Container { return s.Container } diff --git a/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go b/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go index 194cac7c281..8f571dc3e7a 100644 --- a/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go +++ b/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go @@ -5,10 +5,13 @@ package testworkflowprocessor import ( - reflect "reflect" + "reflect" + + "github.com/golang/mock/gomock" - gomock "github.com/golang/mock/gomock" v1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" + v10 "k8s.io/api/core/v1" ) @@ -164,10 +167,10 @@ func (mr *MockIntermediateMockRecorder) ConfigMaps() *gomock.Call { } // ContainerDefaults mocks base method. -func (m *MockIntermediate) ContainerDefaults() Container { +func (m *MockIntermediate) ContainerDefaults() stage.Container { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ContainerDefaults") - ret0, _ := ret[0].(Container) + ret0, _ := ret[0].(stage.Container) return ret0 } diff --git a/pkg/testworkflows/testworkflowprocessor/mock_internalprocessor.go b/pkg/testworkflows/testworkflowprocessor/mock_internalprocessor.go index e96e71df282..bde47d2ad08 100644 --- a/pkg/testworkflows/testworkflowprocessor/mock_internalprocessor.go +++ b/pkg/testworkflows/testworkflowprocessor/mock_internalprocessor.go @@ -5,10 +5,12 @@ package testworkflowprocessor import ( - reflect "reflect" + "reflect" + + "github.com/golang/mock/gomock" - gomock "github.com/golang/mock/gomock" v1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" ) // MockInternalProcessor is a mock of InternalProcessor interface. @@ -35,10 +37,10 @@ func (m *MockInternalProcessor) EXPECT() *MockInternalProcessorMockRecorder { } // Process mocks base method. -func (m *MockInternalProcessor) Process(arg0 Intermediate, arg1 Container, arg2 v1.Step) (Stage, error) { +func (m *MockInternalProcessor) Process(arg0 Intermediate, arg1 stage.Container, arg2 v1.Step) (stage.Stage, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Process", arg0, arg1, arg2) - ret0, _ := ret[0].(Stage) + ret0, _ := ret[0].(stage.Stage) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/pkg/testworkflows/testworkflowprocessor/mock_processor.go b/pkg/testworkflows/testworkflowprocessor/mock_processor.go index 7952368b044..e80c4d7791a 100644 --- a/pkg/testworkflows/testworkflowprocessor/mock_processor.go +++ b/pkg/testworkflows/testworkflowprocessor/mock_processor.go @@ -5,12 +5,14 @@ package testworkflowprocessor import ( - context "context" - reflect "reflect" + "context" + "reflect" + + "github.com/golang/mock/gomock" - gomock "github.com/golang/mock/gomock" v1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" - expressions "github.com/kubeshop/testkube/pkg/expressions" + "github.com/kubeshop/testkube/pkg/expressions" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" ) // MockProcessor is a mock of Processor interface. @@ -57,7 +59,7 @@ func (mr *MockProcessorMockRecorder) Bundle(arg0, arg1 interface{}, arg2 ...inte } // Register mocks base method. -func (m *MockProcessor) Register(arg0 func(InternalProcessor, Intermediate, Container, v1.Step) (Stage, error)) Processor { +func (m *MockProcessor) Register(arg0 func(InternalProcessor, Intermediate, stage.Container, v1.Step) (stage.Stage, error)) Processor { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Register", arg0) ret0, _ := ret[0].(Processor) diff --git a/pkg/testworkflows/testworkflowprocessor/operations.go b/pkg/testworkflows/testworkflowprocessor/operations.go index b25927e3cff..a140f2db986 100644 --- a/pkg/testworkflows/testworkflowprocessor/operations.go +++ b/pkg/testworkflows/testworkflowprocessor/operations.go @@ -10,9 +10,10 @@ import ( testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" ) -func ProcessDelay(_ InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { +func ProcessDelay(_ InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Delay == "" { return nil, nil } @@ -23,28 +24,28 @@ func ProcessDelay(_ InternalProcessor, layer Intermediate, container Container, shell := container.CreateChild(). SetCommand("sleep"). SetArgs(fmt.Sprintf("%g", t.Seconds())) - stage := NewContainerStage(layer.NextRef(), shell) + stage := stage.NewContainerStage(layer.NextRef(), shell) stage.SetCategory(fmt.Sprintf("Delay: %s", step.Delay)) return stage, nil } -func ProcessShellCommand(_ InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { +func ProcessShellCommand(_ InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Shell == "" { return nil, nil } shell := container.CreateChild().SetCommand(constants.DefaultShellPath).SetArgs("-c", constants.DefaultShellHeader+step.Shell) - stage := NewContainerStage(layer.NextRef(), shell) + stage := stage.NewContainerStage(layer.NextRef(), shell) stage.SetCategory("Run shell command") stage.SetRetryPolicy(step.Retry) return stage, nil } -func ProcessRunCommand(_ InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { +func ProcessRunCommand(_ InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Run == nil { return nil, nil } container = container.CreateChild().ApplyCR(&step.Run.ContainerConfig) - stage := NewContainerStage(layer.NextRef(), container) + stage := stage.NewContainerStage(layer.NextRef(), container) stage.SetRetryPolicy(step.Retry) stage.SetCategory("Run") if step.Run.Shell != nil { @@ -57,8 +58,8 @@ func ProcessRunCommand(_ InternalProcessor, layer Intermediate, container Contai return stage, nil } -func ProcessNestedSetupSteps(p InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { - group := NewGroupStage(layer.NextRef(), true) +func ProcessNestedSetupSteps(p InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { + group := stage.NewGroupStage(layer.NextRef(), true) for _, n := range step.Setup { stage, err := p.Process(layer, container.CreateChild(), n) if err != nil { @@ -69,8 +70,8 @@ func ProcessNestedSetupSteps(p InternalProcessor, layer Intermediate, container return group, nil } -func ProcessNestedSteps(p InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { - group := NewGroupStage(layer.NextRef(), true) +func ProcessNestedSteps(p InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { + group := stage.NewGroupStage(layer.NextRef(), true) for _, n := range step.Steps { stage, err := p.Process(layer, container.CreateChild(), n) if err != nil { @@ -81,7 +82,7 @@ func ProcessNestedSteps(p InternalProcessor, layer Intermediate, container Conta return group, nil } -func ProcessContentFiles(_ InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { +func ProcessContentFiles(_ InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Content == nil { return nil, nil } @@ -151,13 +152,13 @@ func ProcessContentFiles(_ InternalProcessor, layer Intermediate, container Cont return nil, nil } -func ProcessContentGit(_ InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { +func ProcessContentGit(_ InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Content == nil || step.Content.Git == nil { return nil, nil } selfContainer := container.CreateChild() - stage := NewContainerStage(layer.NextRef(), selfContainer) + stage := stage.NewContainerStage(layer.NextRef(), selfContainer) stage.SetRetryPolicy(step.Retry) stage.SetCategory("Clone Git repository") @@ -230,13 +231,13 @@ func ProcessContentGit(_ InternalProcessor, layer Intermediate, container Contai return stage, nil } -func ProcessContentTarball(_ InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { +func ProcessContentTarball(_ InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Content == nil || len(step.Content.Tarball) == 0 { return nil, nil } selfContainer := container.CreateChild() - stage := NewContainerStage(layer.NextRef(), selfContainer) + stage := stage.NewContainerStage(layer.NextRef(), selfContainer) stage.SetRetryPolicy(step.Retry) stage.SetCategory("Fetch tarball") @@ -269,7 +270,7 @@ func ProcessContentTarball(_ InternalProcessor, layer Intermediate, container Co return stage, nil } -func ProcessArtifacts(_ InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { +func ProcessArtifacts(_ InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Artifacts == nil { return nil, nil } @@ -280,7 +281,7 @@ func ProcessArtifacts(_ InternalProcessor, layer Intermediate, container Contain selfContainer := container.CreateChild(). ApplyCR(&testworkflowsv1.ContainerConfig{WorkingDir: step.Artifacts.WorkingDir}) - stage := NewContainerStage(layer.NextRef(), selfContainer) + stage := stage.NewContainerStage(layer.NextRef(), selfContainer) stage.SetRetryPolicy(step.Retry) stage.SetCondition("always") stage.SetCategory("Upload artifacts") @@ -301,21 +302,21 @@ func ProcessArtifacts(_ InternalProcessor, layer Intermediate, container Contain return stage, nil } -func StubExecute(_ InternalProcessor, _ Intermediate, _ Container, step testworkflowsv1.Step) (Stage, error) { +func StubExecute(_ InternalProcessor, _ Intermediate, _ stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Execute != nil { return nil, constants.ErrOpenSourceExecuteOperationIsNotAvailable } return nil, nil } -func StubParallel(_ InternalProcessor, _ Intermediate, _ Container, step testworkflowsv1.Step) (Stage, error) { +func StubParallel(_ InternalProcessor, _ Intermediate, _ stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if step.Parallel != nil { return nil, constants.ErrOpenSourceParallelOperationIsNotAvailable } return nil, nil } -func StubServices(_ InternalProcessor, _ Intermediate, _ Container, step testworkflowsv1.Step) (Stage, error) { +func StubServices(_ InternalProcessor, _ Intermediate, _ stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { if len(step.Services) != 0 { return nil, constants.ErrOpenSourceServicesOperationIsNotAvailable } diff --git a/pkg/testworkflows/testworkflowprocessor/presets/processor_test.go b/pkg/testworkflows/testworkflowprocessor/presets/processor_test.go index 11699575e7b..7a09ffef685 100644 --- a/pkg/testworkflows/testworkflowprocessor/presets/processor_test.go +++ b/pkg/testworkflows/testworkflowprocessor/presets/processor_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -12,16 +13,34 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" + constants2 "github.com/kubeshop/testkube/cmd/testworkflow-init/constants" "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/pkg/expressions" "github.com/kubeshop/testkube/pkg/imageinspector" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" ) +const ( + dummyUserId = 1234 + dummyGroupId = 4321 +) + +var ( + dummyEntrypoint = []string{"/dummy-entrypoint", "entrypoint-arg"} + dummyCmd = []string{"/dummy-cmd", "cmd-arg"} +) + type dummyInspector struct{} func (*dummyInspector) Inspect(ctx context.Context, registry, image string, pullPolicy corev1.PullPolicy, pullSecretNames []string) (*imageinspector.Info, error) { - return &imageinspector.Info{}, nil + return &imageinspector.Info{ + Entrypoint: dummyEntrypoint, + Cmd: dummyCmd, + User: dummyUserId, + Group: dummyGroupId, + }, nil } func (*dummyInspector) ResolveName(registry, image string) string { @@ -34,22 +53,48 @@ var ( execMachine = expressions.NewMachine(). Register("resource.root", "dummy-id"). Register("resource.id", "dummy-id-abc") - initEnvs = []corev1.EnvVar{ - {Name: "TK_DEBUG_NODE", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }}, - {Name: "TK_DEBUG_POD", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, - }}, - {Name: "TK_DEBUG_NS", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }}, - {Name: "TK_DEBUG_SVC", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.serviceAccountName"}, - }}, - } + envActions = actiontypes.EnvVarFrom(constants2.EnvGroupActions, false, false, constants2.EnvActions, corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: constants.SpecAnnotationFieldPath}, + }) + envDebugNode = actiontypes.EnvVarFrom(constants2.EnvGroupDebug, false, false, constants2.EnvNodeName, corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }) + envDebugPod = actiontypes.EnvVarFrom(constants2.EnvGroupDebug, false, false, constants2.EnvPodName, corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }) + envDebugNamespace = actiontypes.EnvVarFrom(constants2.EnvGroupDebug, false, false, constants2.EnvNamespaceName, corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }) + envDebugServiceAccount = actiontypes.EnvVarFrom(constants2.EnvGroupDebug, false, false, constants2.EnvServiceAccountName, corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.serviceAccountName"}, + }) ) +func env(index int, computed bool, name, value string) corev1.EnvVar { + return actiontypes.EnvVar(fmt.Sprintf("%d", index), computed, false, name, value) +} + +func cmd(values ...string) *[]string { + return &values +} + +func cmdShell(shell string) *[]string { + args := []string{"-c", "set -e\n" + shell} + return &args +} + +func and(values ...string) string { + return strings.Join(values, "&&") +} + +func getSpec(actions actiontypes.ActionGroups) string { + v, err := json.Marshal(actions) + if err != nil { + panic(err) + } + return string(v) +} + func TestProcessEmpty(t *testing.T) { wf := &testworkflowsv1.TestWorkflow{} @@ -77,6 +122,32 @@ func TestProcessBasic(t *testing.T) { volumes := res.Job.Spec.Template.Spec.Volumes volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts + wantActions := actiontypes.NewActionGroups(). + Append(func(list actiontypes.ActionList) actiontypes.ActionList { + return list. + Setup(false, false). + Declare(constants.RootOperationName, "true"). + Declare(sig[0].Ref(), "true", constants.RootOperationName). + Result(constants.RootOperationName, sig[0].Ref()). + Result("", constants.RootOperationName). + Start(""). + CurrentStatus("true"). + Start(constants.RootOperationName). + CurrentStatus(constants.RootOperationName) + }). + Append(func(list actiontypes.ActionList) actiontypes.ActionList { + return list. + MutateContainer(sig[0].Ref(), testworkflowsv1.ContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test"), + }). + Start(sig[0].Ref()). + Execute(sig[0].Ref(), false). + End(sig[0].Ref()). + End(constants.RootOperationName). + End("") + }) + want := batchv1.Job{ TypeMeta: metav1.TypeMeta{Kind: "Job", APIVersion: "batch/v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -97,7 +168,9 @@ func TestProcessBasic(t *testing.T) { constants.ResourceIdLabelName: "dummy-id-abc", constants.RootResourceIdLabelName: "dummy-id", }, - Annotations: map[string]string(nil), + Annotations: map[string]string{ + constants.SpecAnnotationName: getSpec(wantActions), + }, }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, @@ -105,13 +178,18 @@ func TestProcessBasic(t *testing.T) { Volumes: volumes, InitContainers: []corev1.Container{ { - Name: "tktw-init", + Name: "1", Image: constants.DefaultInitImage, ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, + Command: []string{"/init", "0"}, + Env: []corev1.EnvVar{ + envDebugNode, + envDebugPod, + envDebugNamespace, + envDebugServiceAccount, + envActions, + }, + VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, @@ -119,21 +197,13 @@ func TestProcessBasic(t *testing.T) { }, Containers: []corev1.Container{ { - Name: sig[0].Ref(), - ImagePullPolicy: "", + Name: "2", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[0].Ref()), - "-r", fmt.Sprintf("=%s", sig[0].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "1"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -158,6 +228,133 @@ func TestProcessBasic(t *testing.T) { assert.True(t, volumeMounts[1].Name == volumes[1].Name) } +func TestProcessShellWithNonStandardImage(t *testing.T) { + wf := &testworkflowsv1.TestWorkflow{ + Spec: testworkflowsv1.TestWorkflowSpec{ + Steps: []testworkflowsv1.Step{ + { + StepDefaults: testworkflowsv1.StepDefaults{Container: &testworkflowsv1.ContainerConfig{Image: "custom:1.2.3"}}, + StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test"}, + }, + }, + }, + } + + res, err := proc.Bundle(context.Background(), wf, execMachine) + assert.NoError(t, err) + + sig := res.Signature + sigSerialized, _ := json.Marshal(sig) + + volumes := res.Job.Spec.Template.Spec.Volumes + volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts + + wantActions := actiontypes.NewActionGroups(). + Append(func(list actiontypes.ActionList) actiontypes.ActionList { + return list. + Setup(true, true). + Declare(constants.RootOperationName, "true"). + Declare(sig[0].Ref(), "true", constants.RootOperationName). + Result(constants.RootOperationName, sig[0].Ref()). + Result("", constants.RootOperationName). + Start(""). + CurrentStatus("true"). + Start(constants.RootOperationName). + CurrentStatus(constants.RootOperationName) + }). + Append(func(list actiontypes.ActionList) actiontypes.ActionList { + return list. + MutateContainer(sig[0].Ref(), testworkflowsv1.ContainerConfig{ + Command: cmd("/.tktw/bin/sh"), + Args: cmdShell("shell-test"), + }). + Start(sig[0].Ref()). + Execute(sig[0].Ref(), false). + End(sig[0].Ref()). + End(constants.RootOperationName). + End("") + }) + + want := batchv1.Job{ + TypeMeta: metav1.TypeMeta{Kind: "Job", APIVersion: "batch/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-id-abc", + Labels: map[string]string{ + constants.ResourceIdLabelName: "dummy-id-abc", + constants.RootResourceIdLabelName: "dummy-id", + }, + Annotations: map[string]string{ + constants.SignatureAnnotationName: string(sigSerialized), + }, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: common.Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.ResourceIdLabelName: "dummy-id-abc", + constants.RootResourceIdLabelName: "dummy-id", + }, + Annotations: map[string]string{ + constants.SpecAnnotationName: getSpec(wantActions), + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + EnableServiceLinks: common.Ptr(false), + Volumes: volumes, + InitContainers: []corev1.Container{ + { + Name: "1", + Image: constants.DefaultInitImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "0"}, + Env: []corev1.EnvVar{ + envDebugNode, + envDebugPod, + envDebugNamespace, + envDebugServiceAccount, + envActions, + }, + VolumeMounts: volumeMounts, + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: common.Ptr(int64(dummyGroupId)), + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "2", + Image: "custom:1.2.3", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/.tktw/init", "1"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), + }, + VolumeMounts: volumeMounts, + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: common.Ptr(int64(dummyGroupId)), + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: common.Ptr(int64(dummyGroupId)), + }, + }, + }, + }, + } + + assert.Equal(t, want, res.Job) + + assert.Equal(t, 2, len(volumeMounts)) + assert.Equal(t, 2, len(volumes)) + assert.Equal(t, constants.DefaultInternalPath, volumeMounts[0].MountPath) + assert.Equal(t, constants.DefaultDataPath, volumeMounts[1].MountPath) + assert.True(t, volumeMounts[0].Name == volumes[0].Name) + assert.True(t, volumeMounts[1].Name == volumes[1].Name) +} + func TestProcessBasicEnvReference(t *testing.T) { wf := &testworkflowsv1.TestWorkflow{ Spec: testworkflowsv1.TestWorkflowSpec{ @@ -185,19 +382,50 @@ func TestProcessBasicEnvReference(t *testing.T) { volumes := res.Job.Spec.Template.Spec.Volumes volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts - want := corev1.PodSpec{ + wantActions := lite.NewLiteActionGroups(). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + Setup(false, false). + Declare(constants.RootOperationName, "true"). + Declare(sig[0].Ref(), "true", constants.RootOperationName). + Result(constants.RootOperationName, sig[0].Ref()). + Result("", constants.RootOperationName). + Start(""). + CurrentStatus("true"). + Start(constants.RootOperationName). + CurrentStatus(constants.RootOperationName) + }). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + MutateContainer(lite.LiteContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test"), + }). + Start(sig[0].Ref()). + Execute(sig[0].Ref(), false). + End(sig[0].Ref()). + End(constants.RootOperationName). + End("") + }) + + wantPod := corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, EnableServiceLinks: common.Ptr(false), Volumes: volumes, InitContainers: []corev1.Container{ { - Name: "tktw-init", + Name: "1", Image: constants.DefaultInitImage, ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, + Command: []string{"/init", "0"}, + Env: []corev1.EnvVar{ + envDebugNode, + envDebugPod, + envDebugNamespace, + envDebugServiceAccount, + envActions, + }, + VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, @@ -205,29 +433,18 @@ func TestProcessBasicEnvReference(t *testing.T) { }, Containers: []corev1.Container{ { - Name: sig[0].Ref(), - ImagePullPolicy: "", + Name: "2", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-e", "UNDETERMINED,NEXT", - "-c", fmt.Sprintf("%s=passed", sig[0].Ref()), - "-r", fmt.Sprintf("=%s", sig[0].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "1"}, Env: []corev1.EnvVar{ - {Name: "CI", Value: "1"}, - {Name: "ZERO", Value: "foo"}, - {Name: "UNDETERMINED", Value: "{{call(abc)}}xxx"}, - {Name: "INPUT", Value: "foobar"}, - {Name: "NEXT", Value: "foo{{env.UNDETERMINED}}foofoobarbar"}, - {Name: "LAST", Value: "foofoobarbar"}, + env(0, false, "CI", "1"), + env(0, false, "ZERO", "foo"), + env(0, true, "UNDETERMINED", "{{call(abc)}}xxx"), + env(0, false, "INPUT", "foobar"), + env(0, true, "NEXT", "foo{{env.UNDETERMINED}}foofoobarbar"), + env(0, false, "LAST", "foofoobarbar"), }, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -240,7 +457,8 @@ func TestProcessBasicEnvReference(t *testing.T) { } assert.NoError(t, err) - assert.Equal(t, want, res.Job.Spec.Template.Spec) + assert.Equal(t, wantPod, res.Job.Spec.Template.Spec) + assert.Equal(t, wantActions, res.LiteActions()) } func TestProcessMultipleSteps(t *testing.T) { @@ -259,39 +477,74 @@ func TestProcessMultipleSteps(t *testing.T) { volumes := res.Job.Spec.Template.Spec.Volumes volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts + wantActions := lite.NewLiteActionGroups(). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + Setup(false, false). + Declare(constants.RootOperationName, "true"). + Declare(sig[0].Ref(), "true", constants.RootOperationName). + Declare(sig[1].Ref(), sig[0].Ref(), constants.RootOperationName). + Result(constants.RootOperationName, and(sig[0].Ref(), sig[1].Ref())). + Result("", constants.RootOperationName). + Start(""). + CurrentStatus("true"). + Start(constants.RootOperationName). + CurrentStatus(constants.RootOperationName) + }). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + MutateContainer(lite.LiteContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test"), + }). + Start(sig[0].Ref()). + Execute(sig[0].Ref(), false). + End(sig[0].Ref()). + CurrentStatus(and(sig[0].Ref(), constants.RootOperationName)) + }). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + MutateContainer(lite.LiteContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test-2"), + }). + Start(sig[1].Ref()). + Execute(sig[1].Ref(), false). + End(sig[1].Ref()). + End(constants.RootOperationName). + End("") + }) + want := corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, EnableServiceLinks: common.Ptr(false), Volumes: volumes, InitContainers: []corev1.Container{ { - Name: "tktw-init", + Name: "1", Image: constants.DefaultInitImage, ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, + Command: []string{"/init", "0"}, + Env: []corev1.EnvVar{ + envDebugNode, + envDebugPod, + envDebugNamespace, + envDebugServiceAccount, + envActions, + }, + VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, }, { - Name: sig[0].Ref(), - ImagePullPolicy: "", + Name: "2", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s,%s=passed", sig[0].Ref(), sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[1].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "1"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -300,21 +553,13 @@ func TestProcessMultipleSteps(t *testing.T) { }, Containers: []corev1.Container{ { - Name: sig[1].Ref(), - ImagePullPolicy: "", + Name: "3", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[1].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "2"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-2"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -328,6 +573,7 @@ func TestProcessMultipleSteps(t *testing.T) { assert.NoError(t, err) assert.Equal(t, want, res.Job.Spec.Template.Spec) + assert.Equal(t, wantActions, res.LiteActions()) } func TestProcessNestedSteps(t *testing.T) { @@ -353,85 +599,129 @@ func TestProcessNestedSteps(t *testing.T) { volumes := res.Job.Spec.Template.Spec.Volumes volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts + wantActions := lite.NewLiteActionGroups(). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + Setup(false, false). + Declare(constants.RootOperationName, "true"). + Declare(sig[0].Ref(), "true", constants.RootOperationName). + Declare(sig[1].Ref(), sig[0].Ref(), constants.RootOperationName). + Declare(sig[1].Children()[0].Ref(), sig[0].Ref(), constants.RootOperationName, sig[1].Ref()). + Declare(sig[1].Children()[1].Ref(), and(sig[1].Children()[0].Ref(), sig[0].Ref()), constants.RootOperationName, sig[1].Ref()). + Declare(sig[2].Ref(), and(sig[1].Ref(), sig[0].Ref()), constants.RootOperationName). + Result(sig[1].Ref(), and(sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref())). + Result(constants.RootOperationName, and(sig[0].Ref(), sig[1].Ref(), sig[2].Ref())). + Result("", constants.RootOperationName). + Start(""). + CurrentStatus("true"). + Start(constants.RootOperationName). + CurrentStatus(constants.RootOperationName) + }). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + MutateContainer(lite.LiteContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test"), + }). + Start(sig[0].Ref()). + Execute(sig[0].Ref(), false). + End(sig[0].Ref()). + CurrentStatus(and(sig[0].Ref(), constants.RootOperationName)). + Start(sig[1].Ref()). + CurrentStatus(and(sig[1].Ref(), sig[0].Ref(), constants.RootOperationName)) + }). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + MutateContainer(lite.LiteContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test-2"), + }). + Start(sig[1].Children()[0].Ref()). + Execute(sig[1].Children()[0].Ref(), false). + End(sig[1].Children()[0].Ref()). + CurrentStatus(and(sig[1].Children()[0].Ref(), sig[1].Ref(), sig[0].Ref(), constants.RootOperationName)) + }). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + MutateContainer(lite.LiteContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test-3"), + }). + Start(sig[1].Children()[1].Ref()). + Execute(sig[1].Children()[1].Ref(), false). + End(sig[1].Children()[1].Ref()). + End(sig[1].Ref()). + CurrentStatus(and(sig[1].Ref(), sig[0].Ref(), constants.RootOperationName)) + }). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + MutateContainer(lite.LiteContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test-4"), + }). + Start(sig[2].Ref()). + Execute(sig[2].Ref(), false). + End(sig[2].Ref()). + End(constants.RootOperationName). + End("") + }) + want := corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, EnableServiceLinks: common.Ptr(false), Volumes: volumes, InitContainers: []corev1.Container{ { - Name: "tktw-init", + Name: "1", Image: constants.DefaultInitImage, ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, + Command: []string{"/init", "0"}, + Env: []corev1.EnvVar{ + envDebugNode, + envDebugPod, + envDebugNamespace, + envDebugServiceAccount, + envActions, + }, + VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, }, { - Name: sig[0].Ref(), - ImagePullPolicy: "", + Name: "2", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s,%s,%s,%s=passed", sig[0].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref(), sig[2].Ref()), - "-r", fmt.Sprintf("=%s&&%s&&%s", sig[0].Ref(), sig[1].Ref(), sig[2].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "1"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, }, { - Name: sig[1].Children()[0].Ref(), - ImagePullPolicy: "", + Name: "3", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Children()[0].Ref(), - "-i", fmt.Sprintf("%s", sig[1].Ref()), - "-c", fmt.Sprintf("%s,%s,%s=passed", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s&&%s", sig[0].Ref(), sig[1].Ref(), sig[2].Ref()), - "-r", fmt.Sprintf("%s=%s&&%s", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "2"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-2"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, }, { - Name: sig[1].Children()[1].Ref(), - ImagePullPolicy: "", + Name: "4", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Children()[1].Ref(), - "-i", fmt.Sprintf("%s", sig[1].Ref()), - "-c", fmt.Sprintf("%s=passed", sig[1].Children()[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s&&%s", sig[0].Ref(), sig[1].Ref(), sig[2].Ref()), - "-r", fmt.Sprintf("%s=%s&&%s", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "3"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-3"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -440,21 +730,13 @@ func TestProcessNestedSteps(t *testing.T) { }, Containers: []corev1.Container{ { - Name: sig[2].Ref(), - ImagePullPolicy: "", + Name: "5", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[2].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[2].Ref()), - "-r", fmt.Sprintf("=%s&&%s&&%s", sig[0].Ref(), sig[1].Ref(), sig[2].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "4"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-4"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -467,32 +749,35 @@ func TestProcessNestedSteps(t *testing.T) { } assert.NoError(t, err) + assert.Equal(t, wantActions, res.LiteActions()) assert.Equal(t, want, res.Job.Spec.Template.Spec) } -func TestProcessOptionalSteps(t *testing.T) { +func TestProcessLocalContent(t *testing.T) { wf := &testworkflowsv1.TestWorkflow{ Spec: testworkflowsv1.TestWorkflowSpec{ Steps: []testworkflowsv1.Step{ - {StepMeta: testworkflowsv1.StepMeta{Name: "A"}, StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test"}}, - { - StepMeta: testworkflowsv1.StepMeta{Name: "B"}, - StepControl: testworkflowsv1.StepControl{Optional: true}, - Steps: []testworkflowsv1.Step{ - {StepMeta: testworkflowsv1.StepMeta{Name: "C"}, StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-2"}}, - {StepMeta: testworkflowsv1.StepMeta{Name: "D"}, StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-3"}}, + {StepOperations: testworkflowsv1.StepOperations{ + Shell: "shell-test", + }, StepSource: testworkflowsv1.StepSource{ + Content: &testworkflowsv1.Content{ + Files: []testworkflowsv1.ContentFile{{ + Path: "/some/path", + Content: `some-{{"{{"}}content`, + }}, }, - }, - {StepMeta: testworkflowsv1.StepMeta{Name: "E"}, StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-4"}}, + }}, + {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-2"}}, }, }, } res, err := proc.Bundle(context.Background(), wf, execMachine) - sig := res.Signature + assert.NoError(t, err) volumes := res.Job.Spec.Template.Spec.Volumes volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts + volumeMountsWithContent := res.Job.Spec.Template.Spec.InitContainers[1].VolumeMounts want := corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, @@ -500,78 +785,31 @@ func TestProcessOptionalSteps(t *testing.T) { Volumes: volumes, InitContainers: []corev1.Container{ { - Name: "tktw-init", + Name: "1", Image: constants.DefaultInitImage, ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - { - Name: sig[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s,%s,%s,%s=passed", sig[0].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref(), sig[2].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[2].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - { - Name: sig[1].Children()[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Children()[0].Ref(), - "-i", fmt.Sprintf("%s", sig[1].Ref()), - "-c", fmt.Sprintf("%s,%s,%s=passed", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref()), - "-r", fmt.Sprintf("%s=%s&&%s", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref()), - "--", + Command: []string{"/init", "0"}, + Env: []corev1.EnvVar{ + envDebugNode, + envDebugPod, + envDebugNamespace, + envDebugServiceAccount, + envActions, }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-2"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, }, { - Name: sig[1].Children()[1].Ref(), - ImagePullPolicy: "", + Name: "2", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Children()[1].Ref(), - "-i", fmt.Sprintf("%s", sig[1].Ref()), - "-c", fmt.Sprintf("%s=passed", sig[1].Children()[1].Ref()), - "-r", fmt.Sprintf("%s=%s&&%s", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "1"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-3"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, + VolumeMounts: volumeMountsWithContent, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, @@ -579,21 +817,13 @@ func TestProcessOptionalSteps(t *testing.T) { }, Containers: []corev1.Container{ { - Name: sig[2].Ref(), - ImagePullPolicy: "", + Name: "3", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[2].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[2].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[2].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "2"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-4"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -605,30 +835,37 @@ func TestProcessOptionalSteps(t *testing.T) { }, } - assert.NoError(t, err) assert.Equal(t, want, res.Job.Spec.Template.Spec) + assert.Equal(t, 2, len(volumeMounts)) + assert.Equal(t, 3, len(volumeMountsWithContent)) + assert.Equal(t, volumeMounts, volumeMountsWithContent[:2]) + assert.Equal(t, "/some/path", volumeMountsWithContent[2].MountPath) + assert.Equal(t, 1, len(res.ConfigMaps)) + assert.Equal(t, volumeMountsWithContent[2].Name, volumes[2].Name) + assert.Equal(t, volumes[2].ConfigMap.Name, res.ConfigMaps[0].Name) + assert.Equal(t, "some-{{content", res.ConfigMaps[0].Data[volumeMountsWithContent[2].SubPath]) } -func TestProcessNegativeSteps(t *testing.T) { +func TestProcessGlobalContent(t *testing.T) { wf := &testworkflowsv1.TestWorkflow{ Spec: testworkflowsv1.TestWorkflowSpec{ - Steps: []testworkflowsv1.Step{ - {StepMeta: testworkflowsv1.StepMeta{Name: "A"}, StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test"}}, - { - StepMeta: testworkflowsv1.StepMeta{Name: "B"}, - StepControl: testworkflowsv1.StepControl{Negative: true}, - Steps: []testworkflowsv1.Step{ - {StepMeta: testworkflowsv1.StepMeta{Name: "C"}, StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-2"}}, - {StepMeta: testworkflowsv1.StepMeta{Name: "D"}, StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-3"}}, - }, + TestWorkflowSpecBase: testworkflowsv1.TestWorkflowSpecBase{ + Content: &testworkflowsv1.Content{ + Files: []testworkflowsv1.ContentFile{{ + Path: "/some/path", + Content: `some-{{"{{"}}content`, + }}, }, - {StepMeta: testworkflowsv1.StepMeta{Name: "E"}, StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-4"}}, + }, + Steps: []testworkflowsv1.Step{ + {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test"}}, + {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-2"}}, }, }, } res, err := proc.Bundle(context.Background(), wf, execMachine) - sig := res.Signature + assert.NoError(t, err) volumes := res.Job.Spec.Template.Spec.Volumes volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts @@ -639,81 +876,30 @@ func TestProcessNegativeSteps(t *testing.T) { Volumes: volumes, InitContainers: []corev1.Container{ { - Name: "tktw-init", + Name: "1", Image: constants.DefaultInitImage, ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - { - Name: sig[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s,%s,%s,%s=passed", sig[0].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref(), sig[2].Ref()), - "-r", fmt.Sprintf("=%s&&%s&&%s", sig[0].Ref(), sig[1].Ref(), sig[2].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - { - Name: sig[1].Children()[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Children()[0].Ref(), - "-i", fmt.Sprintf("%s.v", sig[1].Ref()), - "-c", fmt.Sprintf("%s,%s,%s,%s.v=passed", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref(), sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s&&%s", sig[0].Ref(), sig[1].Ref(), sig[2].Ref()), - "-r", fmt.Sprintf("%s=!%s.v", sig[1].Ref(), sig[1].Ref()), - "-r", fmt.Sprintf("%s.v=%s&&%s", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref()), - "--", + Command: []string{"/init", "0"}, + Env: []corev1.EnvVar{ + envDebugNode, + envDebugPod, + envDebugNamespace, + envDebugServiceAccount, + envActions, }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-2"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), }, }, { - Name: sig[1].Children()[1].Ref(), - ImagePullPolicy: "", + Name: "2", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Children()[1].Ref(), - "-i", fmt.Sprintf("%s.v", sig[1].Ref()), - "-c", fmt.Sprintf("%s=passed", sig[1].Children()[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s&&%s", sig[0].Ref(), sig[1].Ref(), sig[2].Ref()), - "-r", fmt.Sprintf("%s=!%s.v", sig[1].Ref(), sig[1].Ref()), - "-r", fmt.Sprintf("%s.v=%s&&%s", sig[1].Ref(), sig[1].Children()[0].Ref(), sig[1].Children()[1].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "1"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-3"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -722,21 +908,13 @@ func TestProcessNegativeSteps(t *testing.T) { }, Containers: []corev1.Container{ { - Name: sig[2].Ref(), - ImagePullPolicy: "", + Name: "3", Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[2].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[2].Ref()), - "-r", fmt.Sprintf("=%s&&%s&&%s", sig[0].Ref(), sig[1].Ref(), sig[2].Ref()), - "--", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/init", "2"}, + Env: []corev1.EnvVar{ + env(0, false, "CI", "1"), }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-4"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, VolumeMounts: volumeMounts, SecurityContext: &corev1.SecurityContext{ RunAsGroup: common.Ptr(constants.DefaultFsGroup), @@ -748,16 +926,41 @@ func TestProcessNegativeSteps(t *testing.T) { }, } - assert.NoError(t, err) assert.Equal(t, want, res.Job.Spec.Template.Spec) + assert.Equal(t, 3, len(volumeMounts)) + assert.Equal(t, "/some/path", volumeMounts[2].MountPath) + assert.Equal(t, 1, len(res.ConfigMaps)) + assert.Equal(t, volumeMounts[2].Name, volumes[2].Name) + assert.Equal(t, volumes[2].ConfigMap.Name, res.ConfigMaps[0].Name) + assert.Equal(t, "some-{{content", res.ConfigMaps[0].Data[volumeMounts[2].SubPath]) } -func TestProcessNegativeContainerStep(t *testing.T) { +func TestProcessEscapedAnnotations(t *testing.T) { wf := &testworkflowsv1.TestWorkflow{ Spec: testworkflowsv1.TestWorkflowSpec{ + TestWorkflowSpecBase: testworkflowsv1.TestWorkflowSpecBase{ + Pod: &testworkflowsv1.PodConfig{ + Annotations: map[string]string{ + "vault.hashicorp.com/agent-inject-template-database-config.txt": `{{"{{"}}- with secret "internal/data/database/config" -}}{{"{{"}} .Data.data.username }}@{{"{{"}} .Data.data.password }}{{"{{"}}- end -}}`, + }, + }, + }, Steps: []testworkflowsv1.Step{ - {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test"}}, - {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-2"}, StepControl: testworkflowsv1.StepControl{Negative: true}}, + {StepOperations: testworkflowsv1.StepOperations{Run: &testworkflowsv1.StepRun{Shell: common.Ptr("shell-test")}}}, + }, + }, + } + + res, err := proc.Bundle(context.Background(), wf, execMachine) + assert.NoError(t, err) + assert.Equal(t, `{{- with secret "internal/data/database/config" -}}{{ .Data.data.username }}@{{ .Data.data.password }}{{- end -}}`, res.Job.Spec.Template.Annotations["vault.hashicorp.com/agent-inject-template-database-config.txt"]) +} + +func TestProcessShell(t *testing.T) { + wf := &testworkflowsv1.TestWorkflow{ + Spec: testworkflowsv1.TestWorkflowSpec{ + Steps: []testworkflowsv1.Step{ + {StepOperations: testworkflowsv1.StepOperations{Run: &testworkflowsv1.StepRun{Shell: common.Ptr("shell-test")}}}, }, }, } @@ -765,579 +968,32 @@ func TestProcessNegativeContainerStep(t *testing.T) { res, err := proc.Bundle(context.Background(), wf, execMachine) sig := res.Signature - volumes := res.Job.Spec.Template.Spec.Volumes - volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts + want := lite.NewLiteActionGroups(). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + Setup(false, false). + Declare(constants.RootOperationName, "true"). + Declare(sig[0].Ref(), "true", constants.RootOperationName). + Result(constants.RootOperationName, sig[0].Ref()). + Result("", constants.RootOperationName). + Start(""). + CurrentStatus("true"). + Start(constants.RootOperationName). + CurrentStatus(constants.RootOperationName) + }). + Append(func(list lite.LiteActionList) lite.LiteActionList { + return list. + MutateContainer(lite.LiteContainerConfig{ + Command: cmd("/bin/sh"), + Args: cmdShell("shell-test"), + }). + Start(sig[0].Ref()). + Execute(sig[0].Ref(), false). + End(sig[0].Ref()). + End(constants.RootOperationName). + End("") + }) - want := corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - EnableServiceLinks: common.Ptr(false), - Volumes: volumes, - InitContainers: []corev1.Container{ - { - Name: "tktw-init", - Image: constants.DefaultInitImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - { - Name: sig[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s,%s=passed", sig[0].Ref(), sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[1].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - Containers: []corev1.Container{ - { - Name: sig[1].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Ref(), - "-n", "true", - "-c", fmt.Sprintf("%s=passed", sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[1].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-2"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: common.Ptr(constants.DefaultFsGroup), - }, - } - - assert.NoError(t, err) - assert.Equal(t, want, res.Job.Spec.Template.Spec) -} - -func TestProcessOptionalContainerStep(t *testing.T) { - wf := &testworkflowsv1.TestWorkflow{ - Spec: testworkflowsv1.TestWorkflowSpec{ - Steps: []testworkflowsv1.Step{ - {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test"}}, - {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-2"}, StepControl: testworkflowsv1.StepControl{Optional: true}}, - }, - }, - } - - res, err := proc.Bundle(context.Background(), wf, execMachine) - sig := res.Signature - - volumes := res.Job.Spec.Template.Spec.Volumes - volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts - - want := corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - EnableServiceLinks: common.Ptr(false), - Volumes: volumes, - InitContainers: []corev1.Container{ - { - Name: "tktw-init", - Image: constants.DefaultInitImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - { - Name: sig[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s,%s=passed", sig[0].Ref(), sig[1].Ref()), - "-r", fmt.Sprintf("=%s", sig[0].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - Containers: []corev1.Container{ - { - Name: sig[1].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[1].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-2"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: common.Ptr(constants.DefaultFsGroup), - }, - } - - assert.NoError(t, err) - assert.Equal(t, want, res.Job.Spec.Template.Spec) -} - -func TestProcessLocalContent(t *testing.T) { - wf := &testworkflowsv1.TestWorkflow{ - Spec: testworkflowsv1.TestWorkflowSpec{ - Steps: []testworkflowsv1.Step{ - {StepOperations: testworkflowsv1.StepOperations{ - Shell: "shell-test", - }, StepSource: testworkflowsv1.StepSource{ - Content: &testworkflowsv1.Content{ - Files: []testworkflowsv1.ContentFile{{ - Path: "/some/path", - Content: `some-{{"{{"}}content`, - }}, - }, - }}, - {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-2"}}, - }, - }, - } - - res, err := proc.Bundle(context.Background(), wf, execMachine) - assert.NoError(t, err) - - sig := res.Signature - - volumes := res.Job.Spec.Template.Spec.Volumes - volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts - volumeMountsWithContent := res.Job.Spec.Template.Spec.InitContainers[1].VolumeMounts - - want := corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - EnableServiceLinks: common.Ptr(false), - Volumes: volumes, - InitContainers: []corev1.Container{ - { - Name: "tktw-init", - Image: constants.DefaultInitImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - { - Name: sig[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s,%s=passed", sig[0].Ref(), sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[1].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMountsWithContent, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - Containers: []corev1.Container{ - { - Name: sig[1].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[1].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-2"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: common.Ptr(constants.DefaultFsGroup), - }, - } - - assert.Equal(t, want, res.Job.Spec.Template.Spec) - assert.Equal(t, 2, len(volumeMounts)) - assert.Equal(t, 3, len(volumeMountsWithContent)) - assert.Equal(t, volumeMounts, volumeMountsWithContent[:2]) - assert.Equal(t, "/some/path", volumeMountsWithContent[2].MountPath) - assert.Equal(t, 1, len(res.ConfigMaps)) - assert.Equal(t, volumeMountsWithContent[2].Name, volumes[2].Name) - assert.Equal(t, volumes[2].ConfigMap.Name, res.ConfigMaps[0].Name) - assert.Equal(t, "some-{{content", res.ConfigMaps[0].Data[volumeMountsWithContent[2].SubPath]) -} - -func TestProcessGlobalContent(t *testing.T) { - wf := &testworkflowsv1.TestWorkflow{ - Spec: testworkflowsv1.TestWorkflowSpec{ - TestWorkflowSpecBase: testworkflowsv1.TestWorkflowSpecBase{ - Content: &testworkflowsv1.Content{ - Files: []testworkflowsv1.ContentFile{{ - Path: "/some/path", - Content: `some-{{"{{"}}content`, - }}, - }, - }, - Steps: []testworkflowsv1.Step{ - {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test"}}, - {StepOperations: testworkflowsv1.StepOperations{Shell: "shell-test-2"}}, - }, - }, - } - - res, err := proc.Bundle(context.Background(), wf, execMachine) - assert.NoError(t, err) - - sig := res.Signature - - volumes := res.Job.Spec.Template.Spec.Volumes - volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts - - want := corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - EnableServiceLinks: common.Ptr(false), - Volumes: volumes, - InitContainers: []corev1.Container{ - { - Name: "tktw-init", - Image: constants.DefaultInitImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - { - Name: sig[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s,%s=passed", sig[0].Ref(), sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[1].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - Containers: []corev1.Container{ - { - Name: sig[1].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[1].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[1].Ref()), - "-r", fmt.Sprintf("=%s&&%s", sig[0].Ref(), sig[1].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test-2"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: common.Ptr(constants.DefaultFsGroup), - }, - } - - assert.Equal(t, want, res.Job.Spec.Template.Spec) - assert.Equal(t, 3, len(volumeMounts)) - assert.Equal(t, "/some/path", volumeMounts[2].MountPath) - assert.Equal(t, 1, len(res.ConfigMaps)) - assert.Equal(t, volumeMounts[2].Name, volumes[2].Name) - assert.Equal(t, volumes[2].ConfigMap.Name, res.ConfigMaps[0].Name) - assert.Equal(t, "some-{{content", res.ConfigMaps[0].Data[volumeMounts[2].SubPath]) -} - -func TestProcessRunShell(t *testing.T) { - wf := &testworkflowsv1.TestWorkflow{ - Spec: testworkflowsv1.TestWorkflowSpec{ - Steps: []testworkflowsv1.Step{ - {StepOperations: testworkflowsv1.StepOperations{Run: &testworkflowsv1.StepRun{Shell: common.Ptr("shell-test")}}}, - }, - }, - } - - res, err := proc.Bundle(context.Background(), wf, execMachine) - assert.NoError(t, err) - - sig := res.Signature - sigSerialized, _ := json.Marshal(sig) - - volumes := res.Job.Spec.Template.Spec.Volumes - volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts - - want := batchv1.Job{ - TypeMeta: metav1.TypeMeta{Kind: "Job", APIVersion: "batch/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "dummy-id-abc", - Labels: map[string]string{ - constants.RootResourceIdLabelName: "dummy-id", - constants.ResourceIdLabelName: "dummy-id-abc", - }, - Annotations: map[string]string{ - constants.SignatureAnnotationName: string(sigSerialized), - }, - }, - Spec: batchv1.JobSpec{ - BackoffLimit: common.Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - constants.RootResourceIdLabelName: "dummy-id", - constants.ResourceIdLabelName: "dummy-id-abc", - }, - Annotations: map[string]string(nil), - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - EnableServiceLinks: common.Ptr(false), - Volumes: volumes, - InitContainers: []corev1.Container{ - { - Name: "tktw-init", - Image: constants.DefaultInitImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - Containers: []corev1.Container{ - { - Name: sig[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[0].Ref()), - "-r", fmt.Sprintf("=%s", sig[0].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - }, - } - - assert.Equal(t, want, res.Job) - - assert.Equal(t, 2, len(volumeMounts)) - assert.Equal(t, 2, len(volumes)) - assert.Equal(t, constants.DefaultInternalPath, volumeMounts[0].MountPath) - assert.Equal(t, constants.DefaultDataPath, volumeMounts[1].MountPath) - assert.True(t, volumeMounts[0].Name == volumes[0].Name) - assert.True(t, volumeMounts[1].Name == volumes[1].Name) -} - -func TestProcessEscapedAnnotations(t *testing.T) { - wf := &testworkflowsv1.TestWorkflow{ - Spec: testworkflowsv1.TestWorkflowSpec{ - TestWorkflowSpecBase: testworkflowsv1.TestWorkflowSpecBase{ - Pod: &testworkflowsv1.PodConfig{ - Annotations: map[string]string{ - "vault.hashicorp.com/agent-inject-template-database-config.txt": `{{"{{"}}- with secret "internal/data/database/config" -}}{{"{{"}} .Data.data.username }}@{{"{{"}} .Data.data.password }}{{"{{"}}- end -}}`, - }, - }, - }, - Steps: []testworkflowsv1.Step{ - {StepOperations: testworkflowsv1.StepOperations{Run: &testworkflowsv1.StepRun{Shell: common.Ptr("shell-test")}}}, - }, - }, - } - - res, err := proc.Bundle(context.Background(), wf, execMachine) assert.NoError(t, err) - - sig := res.Signature - sigSerialized, _ := json.Marshal(sig) - - volumes := res.Job.Spec.Template.Spec.Volumes - volumeMounts := res.Job.Spec.Template.Spec.InitContainers[0].VolumeMounts - - want := batchv1.Job{ - TypeMeta: metav1.TypeMeta{Kind: "Job", APIVersion: "batch/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "dummy-id-abc", - Labels: map[string]string{ - constants.RootResourceIdLabelName: "dummy-id", - constants.ResourceIdLabelName: "dummy-id-abc", - }, - Annotations: map[string]string{ - constants.SignatureAnnotationName: string(sigSerialized), - }, - }, - Spec: batchv1.JobSpec{ - BackoffLimit: common.Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - constants.RootResourceIdLabelName: "dummy-id", - constants.ResourceIdLabelName: "dummy-id-abc", - }, - Annotations: map[string]string{ - "vault.hashicorp.com/agent-inject-template-database-config.txt": `{{- with secret "internal/data/database/config" -}}{{ .Data.data.username }}@{{ .Data.data.password }}{{- end -}}`, - }, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - EnableServiceLinks: common.Ptr(false), - Volumes: volumes, - InitContainers: []corev1.Container{ - { - Name: "tktw-init", - Image: constants.DefaultInitImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - Env: initEnvs, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - Containers: []corev1.Container{ - { - Name: sig[0].Ref(), - ImagePullPolicy: "", - Image: constants.DefaultInitImage, - Command: []string{ - "/.tktw/init", - sig[0].Ref(), - "-c", fmt.Sprintf("%s=passed", sig[0].Ref()), - "-r", fmt.Sprintf("=%s", sig[0].Ref()), - "--", - }, - Args: []string{constants.DefaultShellPath, "-c", constants.DefaultShellHeader + "shell-test"}, - WorkingDir: "", - EnvFrom: []corev1.EnvFromSource(nil), - Env: []corev1.EnvVar{{Name: "CI", Value: "1"}}, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: common.Ptr(constants.DefaultFsGroup), - }, - }, - }, - }, - } - - assert.Equal(t, want, res.Job) - - assert.Equal(t, 2, len(volumeMounts)) - assert.Equal(t, 2, len(volumes)) - assert.Equal(t, constants.DefaultInternalPath, volumeMounts[0].MountPath) - assert.Equal(t, constants.DefaultDataPath, volumeMounts[1].MountPath) - assert.True(t, volumeMounts[0].Name == volumes[0].Name) - assert.True(t, volumeMounts[1].Name == volumes[1].Name) + assert.Equal(t, want, res.LiteActions()) } diff --git a/pkg/testworkflows/testworkflowprocessor/processor.go b/pkg/testworkflows/testworkflowprocessor/processor.go index 998cef68800..89a78cf53d2 100644 --- a/pkg/testworkflows/testworkflowprocessor/processor.go +++ b/pkg/testworkflows/testworkflowprocessor/processor.go @@ -16,7 +16,10 @@ import ( "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/pkg/expressions" "github.com/kubeshop/testkube/pkg/imageinspector" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" ) //go:generate mockgen -destination=./mock_processor.go -package=testworkflowprocessor "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" Processor @@ -27,10 +30,10 @@ type Processor interface { //go:generate mockgen -destination=./mock_internalprocessor.go -package=testworkflowprocessor "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" InternalProcessor type InternalProcessor interface { - Process(layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) + Process(layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) } -type Operation = func(processor InternalProcessor, layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) +type Operation = func(processor InternalProcessor, layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) type processor struct { inspector imageinspector.Inspector @@ -46,7 +49,7 @@ func (p *processor) Register(operation Operation) Processor { return p } -func (p *processor) process(layer Intermediate, container Container, step testworkflowsv1.Step, ref string) (Stage, error) { +func (p *processor) process(layer Intermediate, container stage.Container, step testworkflowsv1.Step, ref string) (stage.Stage, error) { // Configure defaults if step.WorkingDir != nil { container.SetWorkingDir(*step.WorkingDir) @@ -54,7 +57,7 @@ func (p *processor) process(layer Intermediate, container Container, step testwo container.ApplyCR(step.Container) // Build an initial group for the inner items - self := NewGroupStage(ref, false) + self := stage.NewGroupStage(ref, false) self.SetName(step.Name) self.SetOptional(step.Optional).SetNegative(step.Negative).SetTimeout(step.Timeout).SetPaused(step.Paused) if step.Condition != "" { @@ -74,7 +77,7 @@ func (p *processor) process(layer Intermediate, container Container, step testwo // Add virtual pause step in case no other is there if self.HasPause() && len(self.Children()) == 0 { - pause := NewContainerStage(self.Ref()+"pause", container.CreateChild(). + pause := stage.NewContainerStage(self.Ref()+"pause", container.CreateChild(). SetCommand(constants.DefaultShellPath). SetArgs("-c", "exit 0")) pause.SetCategory("Wait for continue") @@ -84,7 +87,7 @@ func (p *processor) process(layer Intermediate, container Container, step testwo return self, nil } -func (p *processor) Process(layer Intermediate, container Container, step testworkflowsv1.Step) (Stage, error) { +func (p *processor) Process(layer Intermediate, container stage.Container, step testworkflowsv1.Step) (stage.Stage, error) { return p.process(layer, container, step, layer.NextRef()) } @@ -123,7 +126,7 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo if err != nil { return nil, errors.Wrap(err, "error while simplifying workflow instructions") } - root, err := p.process(layer, layer.ContainerDefaults(), rootStep, "") + root, err := p.process(layer, layer.ContainerDefaults(), rootStep, constants.RootOperationName) if err != nil { return nil, errors.Wrap(err, "processing error") } @@ -210,7 +213,7 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo // Adjust the security context in case it's a single container besides the Testkube' containers // TODO: Consider flag argument, that would be used only for services? containerStages := root.ContainerStages() - var otherContainers []ContainerStage + var otherContainers []stage.ContainerStage for _, c := range containerStages { if c.Container().Image() != constants.DefaultInitImage && c.Container().Image() != constants.DefaultToolkitImage { otherContainers = append(otherContainers, c) @@ -244,10 +247,21 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo } // Build list of the containers - containers, err := buildKubernetesContainers(root, NewInitProcess().SetRef(root.Ref()), fsGroup, machines...) + actions, err := action.Process(root, machines...) if err != nil { - return nil, errors.Wrap(err, "building Kubernetes containers") + return nil, errors.Wrap(err, "analyzing Kubernetes container operations") + } + actionGroups := action.Group(actions) + containers := make([]corev1.Container, len(actionGroups)) + for i := range actionGroups { + var bareActions []actiontypes.Action + containers[i], bareActions, err = action.CreateContainer(i, layer.ContainerDefaults(), actionGroups[i]) + actionGroups[i] = bareActions + if err != nil { + return nil, errors.Wrap(err, "building Kubernetes containers") + } } + for i := range containers { err = expressions.FinalizeForce(&containers[i].EnvFrom, machines...) if err != nil { @@ -272,6 +286,19 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo containers[i].VolumeMounts[j].MountPath = filepath.Clean(filepath.Join(workingDir, containers[i].VolumeMounts[j].MountPath)) } } + + // Avoid having working directory set up, so we have the default one + containers[i].WorkingDir = "" + + // Ensure the cr will have proper access to FS + if fsGroup != nil { + if containers[i].SecurityContext == nil { + containers[i].SecurityContext = &corev1.SecurityContext{} + } + if containers[i].SecurityContext.RunAsGroup == nil { + containers[i].SecurityContext.RunAsGroup = fsGroup + } + } } // Build pod template @@ -312,42 +339,7 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo }, } AnnotateControlledBy(&podSpec, resourceRoot.Template(), resourceId.Template()) - - defaultResources, defaultResourcesErr := MapResourcesToKubernetesResources(common.Ptr(layer.ContainerDefaults().Resources())) - if defaultResourcesErr != nil { - return nil, defaultResourcesErr - } - initContainer := corev1.Container{ - Name: "tktw-init", - Image: constants.DefaultInitImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-c"}, - Args: []string{constants.InitScript}, - VolumeMounts: layer.ContainerDefaults().VolumeMounts(), - Resources: defaultResources, - Env: []corev1.EnvVar{ - {Name: "TK_DEBUG_NODE", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }}, - {Name: "TK_DEBUG_POD", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, - }}, - {Name: "TK_DEBUG_NS", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }}, - {Name: "TK_DEBUG_SVC", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.serviceAccountName"}, - }}, - }, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: fsGroup, - }, - } - err = expressions.FinalizeForce(&initContainer, machines...) - if err != nil { - return nil, errors.Wrap(err, "finalizing container's resources") - } - podSpec.Spec.InitContainers = append([]corev1.Container{initContainer}, containers[:len(containers)-1]...) + podSpec.Spec.InitContainers = containers[:len(containers)-1] podSpec.Spec.Containers = containers[len(containers)-1:] // Build job spec @@ -383,6 +375,16 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo }) jobSpec.Annotations = jobAnnotations + // Build running instructions + // TODO: Get rid of the unnecessary ContainerConfig parts + actionGroupsSerialized, _ := json.Marshal(actionGroups) + podAnnotations := make(map[string]string) + maps.Copy(podAnnotations, jobSpec.Spec.Template.Annotations) + maps.Copy(podAnnotations, map[string]string{ + constants.SpecAnnotationName: string(actionGroupsSerialized), + }) + jobSpec.Spec.Template.Annotations = podAnnotations + // Build bundle bundle = &Bundle{ ConfigMaps: configMaps, diff --git a/pkg/testworkflows/testworkflowprocessor/container.go b/pkg/testworkflows/testworkflowprocessor/stage/container.go similarity index 91% rename from pkg/testworkflows/testworkflowprocessor/container.go rename to pkg/testworkflows/testworkflowprocessor/stage/container.go index 31cd8a2488b..81a4752b9c1 100644 --- a/pkg/testworkflows/testworkflowprocessor/container.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/container.go @@ -1,4 +1,4 @@ -package testworkflowprocessor +package stage import ( "maps" @@ -16,8 +16,9 @@ import ( ) type container struct { - parent *container - Cr testworkflowsv1.ContainerConfig `expr:"include"` + parent *container + toolkit bool + Cr testworkflowsv1.ContainerConfig `expr:"include"` } type ContainerComposition interface { @@ -47,6 +48,7 @@ type ContainerAccessors interface { HasVolumeAt(path string) bool ToContainerConfig() testworkflowsv1.ContainerConfig + IsToolkit() bool } type ContainerMutations[T any] interface { @@ -67,7 +69,7 @@ type ContainerMutations[T any] interface { EnableToolkit(ref string) T } -//go:generate mockgen -destination=./mock_container.go -package=testworkflowprocessor "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" Container +//go:generate mockgen -destination=./mock_container.go -package=stage "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" Container type Container interface { ContainerComposition ContainerAccessors @@ -94,7 +96,7 @@ func (c *container) Root() Container { if c.parent == nil { return c } - return c.parent.Parent() + return c.parent.Root() } func (c *container) Parent() Container { @@ -102,7 +104,7 @@ func (c *container) Parent() Container { } func (c *container) CreateChild() Container { - return &container{parent: c} + return &container{parent: c, toolkit: c.toolkit} } // Getters @@ -309,24 +311,41 @@ func (c *container) ToContainerConfig() testworkflowsv1.ContainerConfig { for i := range volumeMounts { volumeMounts[i] = *volumeMounts[i].DeepCopy() } + workingDir := common.Ptr(c.WorkingDir()) + if *workingDir == "" { + workingDir = nil + } + resources := &testworkflowsv1.Resources{ + Requests: maps.Clone(c.Resources().Requests), + Limits: maps.Clone(c.Resources().Limits), + } + if len(resources.Requests) == 0 && len(resources.Limits) == 0 { + resources = nil + } + args := common.Ptr(slices.Clone(c.Args())) + if *args == nil { + args = nil + } + command := common.Ptr(slices.Clone(c.Command())) + if *command == nil { + command = nil + } return testworkflowsv1.ContainerConfig{ - WorkingDir: common.Ptr(c.WorkingDir()), + WorkingDir: workingDir, Image: c.Image(), ImagePullPolicy: c.ImagePullPolicy(), Env: env, EnvFrom: envFrom, - Command: common.Ptr(slices.Clone(c.Command())), - Args: common.Ptr(slices.Clone(c.Args())), - Resources: &testworkflowsv1.Resources{ - Requests: maps.Clone(c.Resources().Requests), - Limits: maps.Clone(c.Resources().Limits), - }, + Command: command, + Args: args, + Resources: resources, SecurityContext: c.SecurityContext().DeepCopy(), VolumeMounts: volumeMounts, } } func (c *container) Detach() Container { + c.toolkit = c.IsToolkit() c.Cr = c.ToContainerConfig() c.parent = nil return c @@ -394,8 +413,18 @@ func (c *container) ApplyImageData(image *imageinspector.Info, resolvedImageName return nil } +func (c *container) IsToolkit() bool { + return c.toolkit || (c.parent != nil && c.parent.IsToolkit()) || slices.Contains(c.Cr.Env, BypassToolkitCheck) +} + +func (c *container) MarkAsToolkit() Container { + c.toolkit = true + return c +} + func (c *container) EnableToolkit(ref string) Container { return c. + MarkAsToolkit(). AppendEnv(corev1.EnvVar{ Name: "TK_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}}, diff --git a/pkg/testworkflows/testworkflowprocessor/containerstage.go b/pkg/testworkflows/testworkflowprocessor/stage/containerstage.go similarity index 85% rename from pkg/testworkflows/testworkflowprocessor/containerstage.go rename to pkg/testworkflows/testworkflowprocessor/stage/containerstage.go index 3644ab7ba0d..0013879ed2a 100644 --- a/pkg/testworkflows/testworkflowprocessor/containerstage.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/containerstage.go @@ -1,12 +1,19 @@ -package testworkflowprocessor +package stage import ( "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/rand" "github.com/kubeshop/testkube/pkg/expressions" "github.com/kubeshop/testkube/pkg/imageinspector" ) +var BypassToolkitCheck = corev1.EnvVar{ + Name: "TK_TC_SECURITY", + Value: rand.String(20), +} + type containerStage struct { stageMetadata stageLifecycle @@ -16,6 +23,7 @@ type containerStage struct { type ContainerStage interface { Stage Container() Container + IsToolkit() bool } func NewContainerStage(ref string, container Container) ContainerStage { @@ -72,3 +80,7 @@ func (s *containerStage) Container() Container { func (s *containerStage) HasPause() bool { return s.paused } + +func (s *containerStage) IsToolkit() bool { + return s.container.IsToolkit() +} diff --git a/pkg/testworkflows/testworkflowprocessor/groupstage.go b/pkg/testworkflows/testworkflowprocessor/stage/groupstage.go similarity index 91% rename from pkg/testworkflows/testworkflowprocessor/groupstage.go rename to pkg/testworkflows/testworkflowprocessor/stage/groupstage.go index b8aef9d8a3b..4b805928f05 100644 --- a/pkg/testworkflows/testworkflowprocessor/groupstage.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/groupstage.go @@ -1,4 +1,4 @@ -package testworkflowprocessor +package stage import ( "maps" @@ -12,12 +12,15 @@ import ( type groupStage struct { stageMetadata stageLifecycle - children []Stage - virtual bool + containerDefaults Container + children []Stage + virtual bool } type GroupStage interface { Stage + SetContainerDefaults(c Container) + ContainerDefaults() Container Children() []Stage RecursiveChildren() []Stage Add(stages ...Stage) GroupStage @@ -30,6 +33,14 @@ func NewGroupStage(ref string, virtual bool) GroupStage { } } +func (s *groupStage) SetContainerDefaults(c Container) { + s.containerDefaults = c +} + +func (s *groupStage) ContainerDefaults() Container { + return s.containerDefaults +} + func (s *groupStage) Len() int { count := 0 for _, ch := range s.Children() { diff --git a/pkg/testworkflows/testworkflowprocessor/mock_container.go b/pkg/testworkflows/testworkflowprocessor/stage/mock_container.go similarity index 96% rename from pkg/testworkflows/testworkflowprocessor/mock_container.go rename to pkg/testworkflows/testworkflowprocessor/stage/mock_container.go index 6a66f9aefbd..5dcb2c2cf24 100644 --- a/pkg/testworkflows/testworkflowprocessor/mock_container.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/mock_container.go @@ -1,8 +1,8 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor (interfaces: Container) +// Source: github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage (interfaces: Container) -// Package testworkflowprocessor is a generated GoMock package. -package testworkflowprocessor +// Package stage is a generated GoMock package. +package stage import ( reflect "reflect" @@ -273,6 +273,20 @@ func (mr *MockContainerMockRecorder) ImagePullPolicy() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImagePullPolicy", reflect.TypeOf((*MockContainer)(nil).ImagePullPolicy)) } +// IsToolkit mocks base method. +func (m *MockContainer) IsToolkit() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsToolkit") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsToolkit indicates an expected call of IsToolkit. +func (mr *MockContainerMockRecorder) IsToolkit() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsToolkit", reflect.TypeOf((*MockContainer)(nil).IsToolkit)) +} + // Parent mocks base method. func (m *MockContainer) Parent() Container { m.ctrl.T.Helper() diff --git a/pkg/testworkflows/testworkflowprocessor/mock_stage.go b/pkg/testworkflows/testworkflowprocessor/stage/mock_stage.go similarity index 97% rename from pkg/testworkflows/testworkflowprocessor/mock_stage.go rename to pkg/testworkflows/testworkflowprocessor/stage/mock_stage.go index 6f33191ea6a..d28031d6760 100644 --- a/pkg/testworkflows/testworkflowprocessor/mock_stage.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/mock_stage.go @@ -1,16 +1,17 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor (interfaces: Stage) +// Source: github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage (interfaces: Stage) -// Package testworkflowprocessor is a generated GoMock package. -package testworkflowprocessor +// Package stage is a generated GoMock package. +package stage import ( - reflect "reflect" + "reflect" + + "github.com/golang/mock/gomock" - gomock "github.com/golang/mock/gomock" v1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" - expressions "github.com/kubeshop/testkube/pkg/expressions" - imageinspector "github.com/kubeshop/testkube/pkg/imageinspector" + "github.com/kubeshop/testkube/pkg/expressions" + "github.com/kubeshop/testkube/pkg/imageinspector" ) // MockStage is a mock of Stage interface. diff --git a/pkg/testworkflows/testworkflowprocessor/signature.go b/pkg/testworkflows/testworkflowprocessor/stage/signature.go similarity index 99% rename from pkg/testworkflows/testworkflowprocessor/signature.go rename to pkg/testworkflows/testworkflowprocessor/stage/signature.go index b320363429e..c1b31a4e4a9 100644 --- a/pkg/testworkflows/testworkflowprocessor/signature.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/signature.go @@ -1,4 +1,4 @@ -package testworkflowprocessor +package stage import ( "encoding/json" diff --git a/pkg/testworkflows/testworkflowprocessor/stage.go b/pkg/testworkflows/testworkflowprocessor/stage/stage.go similarity index 70% rename from pkg/testworkflows/testworkflowprocessor/stage.go rename to pkg/testworkflows/testworkflowprocessor/stage/stage.go index b1d60203716..dfc3a4de215 100644 --- a/pkg/testworkflows/testworkflowprocessor/stage.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/stage.go @@ -1,11 +1,11 @@ -package testworkflowprocessor +package stage import ( "github.com/kubeshop/testkube/pkg/expressions" "github.com/kubeshop/testkube/pkg/imageinspector" ) -//go:generate mockgen -destination=./mock_stage.go -package=testworkflowprocessor "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" Stage +//go:generate mockgen -destination=./mock_stage.go -package=stage "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" Stage type Stage interface { StageMetadata StageLifecycle diff --git a/pkg/testworkflows/testworkflowprocessor/stagelifecycle.go b/pkg/testworkflows/testworkflowprocessor/stage/stagelifecycle.go similarity index 98% rename from pkg/testworkflows/testworkflowprocessor/stagelifecycle.go rename to pkg/testworkflows/testworkflowprocessor/stage/stagelifecycle.go index 924f96b7fb6..c2560cc3a3a 100644 --- a/pkg/testworkflows/testworkflowprocessor/stagelifecycle.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/stagelifecycle.go @@ -1,4 +1,4 @@ -package testworkflowprocessor +package stage import ( "strings" diff --git a/pkg/testworkflows/testworkflowprocessor/stagemetadata.go b/pkg/testworkflows/testworkflowprocessor/stage/stagemetadata.go similarity index 95% rename from pkg/testworkflows/testworkflowprocessor/stagemetadata.go rename to pkg/testworkflows/testworkflowprocessor/stage/stagemetadata.go index 404ff2bbb57..b0a8fada018 100644 --- a/pkg/testworkflows/testworkflowprocessor/stagemetadata.go +++ b/pkg/testworkflows/testworkflowprocessor/stage/stagemetadata.go @@ -1,4 +1,4 @@ -package testworkflowprocessor +package stage type StageMetadata interface { Ref() string diff --git a/pkg/testworkflows/testworkflowprocessor/stage/utils.go b/pkg/testworkflows/testworkflowprocessor/stage/utils.go new file mode 100644 index 00000000000..c5327005b6d --- /dev/null +++ b/pkg/testworkflows/testworkflowprocessor/stage/utils.go @@ -0,0 +1,36 @@ +package stage + +import ( + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + quantity "k8s.io/apimachinery/pkg/api/resource" + + testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" +) + +func MapResourcesToKubernetesResources(resources *testworkflowsv1.Resources) (corev1.ResourceRequirements, error) { + result := corev1.ResourceRequirements{} + if resources != nil { + if len(resources.Requests) > 0 { + result.Requests = make(corev1.ResourceList) + } + if len(resources.Limits) > 0 { + result.Limits = make(corev1.ResourceList) + } + for k, v := range resources.Requests { + var err error + result.Requests[k], err = quantity.ParseQuantity(v.String()) + if err != nil { + return corev1.ResourceRequirements{}, errors.Wrap(err, "parsing resources") + } + } + for k, v := range resources.Limits { + var err error + result.Limits[k], err = quantity.ParseQuantity(v.String()) + if err != nil { + return corev1.ResourceRequirements{}, errors.Wrap(err, "parsing resources") + } + } + } + return result, nil +} diff --git a/pkg/testworkflows/testworkflowprocessor/utils.go b/pkg/testworkflows/testworkflowprocessor/utils.go index f899b9dbcca..a548e8a9017 100644 --- a/pkg/testworkflows/testworkflowprocessor/utils.go +++ b/pkg/testworkflows/testworkflowprocessor/utils.go @@ -1,54 +1,12 @@ package testworkflowprocessor import ( - "fmt" - "strings" - - "github.com/pkg/errors" batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - quantity "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" - "github.com/kubeshop/testkube/internal/common" - "github.com/kubeshop/testkube/pkg/expressions" - "github.com/kubeshop/testkube/pkg/rand" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" ) -var BypassToolkitCheck = corev1.EnvVar{ - Name: "TK_TC_SECURITY", - Value: rand.String(20), -} - -func MapResourcesToKubernetesResources(resources *testworkflowsv1.Resources) (corev1.ResourceRequirements, error) { - result := corev1.ResourceRequirements{} - if resources != nil { - if len(resources.Requests) > 0 { - result.Requests = make(corev1.ResourceList) - } - if len(resources.Limits) > 0 { - result.Limits = make(corev1.ResourceList) - } - for k, v := range resources.Requests { - var err error - result.Requests[k], err = quantity.ParseQuantity(v.String()) - if err != nil { - return corev1.ResourceRequirements{}, errors.Wrap(err, "parsing resources") - } - } - for k, v := range resources.Limits { - var err error - result.Limits[k], err = quantity.ParseQuantity(v.String()) - if err != nil { - return corev1.ResourceRequirements{}, errors.Wrap(err, "parsing resources") - } - } - } - return result, nil -} - func AnnotateControlledBy(obj metav1.Object, rootId, id string) { labels := obj.GetLabels() if labels == nil { @@ -77,128 +35,3 @@ func AnnotateGroupId(obj metav1.Object, id string) { AnnotateGroupId(&v.Spec.Template, id) } } - -func getRef(stage Stage) string { - return stage.Ref() -} - -func isNotOptional(stage Stage) bool { - return !stage.Optional() -} - -func buildKubernetesContainers(stage Stage, init *initProcess, fsGroup *int64, machines ...expressions.Machine) (containers []corev1.Container, err error) { - if stage.Paused() { - init.SetPaused(stage.Paused()) - } - if stage.Timeout() != "" { - init.AddTimeout(stage.Timeout(), stage.Ref()) - } - if stage.Ref() != "" { - init.AddCondition(stage.Condition(), stage.Ref()) - } - init.AddRetryPolicy(stage.RetryPolicy(), stage.Ref()) - - group, ok := stage.(GroupStage) - if ok { - recursiveRefs := common.MapSlice(group.RecursiveChildren(), getRef) - directRefResults := common.MapSlice(common.FilterSlice(group.Children(), isNotOptional), getRef) - - init.AddCondition(stage.Condition(), recursiveRefs...) - - if group.Negative() { - // Create virtual layer that will be put down into actual negative step - init.SetRef(stage.Ref() + ".v") - init.AddCondition(stage.Condition(), stage.Ref()+".v") - init.PrependInitialStatus(stage.Ref() + ".v") - init.AddResult("!"+stage.Ref()+".v", stage.Ref()) - } else if stage.Ref() != "" { - init.PrependInitialStatus(stage.Ref()) - } - - if group.Optional() { - init.ResetResults() - } - - if group.Negative() { - init.AddResult(strings.Join(directRefResults, "&&"), stage.Ref()+".v") - } else { - init.AddResult(strings.Join(directRefResults, "&&"), stage.Ref()) - } - - for i, ch := range group.Children() { - // Condition should be executed only in the first leaf - if i == 1 { - init.ResetCondition().SetPaused(false) - } - // Pass down to another group or container - sub, serr := buildKubernetesContainers(ch, init.Children(ch.Ref()), fsGroup, machines...) - if serr != nil { - return nil, fmt.Errorf("%s: %s: resolving children: %s", stage.Ref(), stage.Name(), serr.Error()) - } - containers = append(containers, sub...) - } - return - } - c, ok := stage.(ContainerStage) - if !ok { - return nil, fmt.Errorf("%s: %s: stage that is neither container nor group", stage.Ref(), stage.Name()) - } - err = c.Container().Detach().Resolve(machines...) - if err != nil { - return nil, fmt.Errorf("%s: %s: resolving container: %s", stage.Ref(), stage.Name(), err.Error()) - } - - cr, err := c.Container().ToKubernetesTemplate() - if err != nil { - return nil, fmt.Errorf("%s: %s: building container template: %s", stage.Ref(), stage.Name(), err.Error()) - } - cr.Name = c.Ref() - - if c.Optional() { - init.ResetResults() - } - - bypass := false - refEnvVar := "" - for _, e := range cr.Env { - if e.Name == BypassToolkitCheck.Name && e.Value == BypassToolkitCheck.Value { - bypass = true - } - if e.Name == "TK_REF" { - refEnvVar = e.Value - } - } - - init. - SetNegative(c.Negative()). - AddRetryPolicy(c.RetryPolicy(), c.Ref()). - SetCommand(cr.Command...). - SetArgs(cr.Args...). - SetWorkingDir(cr.WorkingDir). - SetToolkit(bypass || (cr.Image == constants.DefaultToolkitImage && c.Ref() == refEnvVar)) - - for _, env := range cr.Env { - if strings.Contains(env.Value, "{{") { - init.AddComputedEnvs(env.Name) - } - } - - if init.Error() != nil { - return nil, init.Error() - } - - cr.Command = init.Command() - cr.Args = init.Args() - cr.WorkingDir = "" - - // Ensure the container will have proper access to FS - if cr.SecurityContext == nil { - cr.SecurityContext = &corev1.SecurityContext{} - } - if cr.SecurityContext.RunAsGroup == nil { - cr.SecurityContext.RunAsGroup = fsGroup - } - - containers = []corev1.Container{cr} - return -} diff --git a/pkg/testworkflows/testworkflowresolver/dedupe.go b/pkg/testworkflows/testworkflowresolver/dedupe.go index 6767474f71a..43a245412c7 100644 --- a/pkg/testworkflows/testworkflowresolver/dedupe.go +++ b/pkg/testworkflows/testworkflowresolver/dedupe.go @@ -22,5 +22,8 @@ func DedupeEnvVars(envs []corev1.EnvVar) []corev1.EnvVar { result = append([]corev1.EnvVar{envs[i]}, result...) } } + if len(result) == 0 { + return nil + } return result }