diff --git a/cmd/main.go b/cmd/main.go index 128bc150..1d48b1d3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -150,6 +150,10 @@ func main() { Usage: "dry run does not create chaos, only logs planned chaos commands", EnvVar: "DRY-RUN", }, + cli.BoolFlag{ + Name: "skip-error", + Usage: "skip chaos command error and retry to execute the command on next interval tick", + }, } if err := app.Run(os.Args); err != nil { diff --git a/pkg/chaos/command.go b/pkg/chaos/command.go index b60e5bce..14ab2b10 100644 --- a/pkg/chaos/command.go +++ b/pkg/chaos/command.go @@ -52,7 +52,7 @@ func GetNamesOrPattern(c *cli.Context) ([]string, string) { } // RunChaosCommand run chaos command in go routine -func RunChaosCommand(topContext context.Context, command Command, intervalStr string, random bool) error { +func RunChaosCommand(topContext context.Context, command Command, intervalStr string, random, skipError bool) error { // parse interval interval, err := util.GetIntervalValue(intervalStr) if err != nil { @@ -75,7 +75,10 @@ func RunChaosCommand(topContext context.Context, command Command, intervalStr st for { // run chaos function if err := command.Run(ctx, random); err != nil { - return err + if !skipError { + return err + } + log.WithError(err).Warn("skipping error") } // wait for next timer tick or cancel select { diff --git a/pkg/chaos/docker/cmd/kill.go b/pkg/chaos/docker/cmd/kill.go index 78a60ee0..09446d65 100644 --- a/pkg/chaos/docker/cmd/kill.go +++ b/pkg/chaos/docker/cmd/kill.go @@ -46,6 +46,8 @@ func (cmd *killContext) kill(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get interval interval := c.GlobalString("interval") // get names or pattern @@ -60,5 +62,5 @@ func (cmd *killContext) kill(c *cli.Context) error { return err } // run kill command - return chaos.RunChaosCommand(cmd.context, killCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, killCommand, interval, random, skipError) } diff --git a/pkg/chaos/docker/cmd/pause.go b/pkg/chaos/docker/cmd/pause.go index ad943450..c02033a4 100644 --- a/pkg/chaos/docker/cmd/pause.go +++ b/pkg/chaos/docker/cmd/pause.go @@ -43,6 +43,8 @@ func (cmd *pauseContext) pause(c *cli.Context) error { random := c.GlobalBool("random") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get labels labels := c.GlobalStringSlice("label") // get global chaos interval @@ -59,5 +61,5 @@ func (cmd *pauseContext) pause(c *cli.Context) error { return err } // run pause command - return chaos.RunChaosCommand(cmd.context, pauseCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, pauseCommand, interval, random, skipError) } diff --git a/pkg/chaos/docker/cmd/remove.go b/pkg/chaos/docker/cmd/remove.go index fad9910b..c4371062 100644 --- a/pkg/chaos/docker/cmd/remove.go +++ b/pkg/chaos/docker/cmd/remove.go @@ -53,6 +53,8 @@ func (cmd *removeContext) remove(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get interval interval := c.GlobalString("interval") // get names or pattern @@ -71,5 +73,5 @@ func (cmd *removeContext) remove(c *cli.Context) error { return err } // run remove command - return chaos.RunChaosCommand(cmd.context, removeCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, removeCommand, interval, random, skipError) } diff --git a/pkg/chaos/docker/cmd/stop.go b/pkg/chaos/docker/cmd/stop.go index 2bb71c39..b1a6339c 100644 --- a/pkg/chaos/docker/cmd/stop.go +++ b/pkg/chaos/docker/cmd/stop.go @@ -55,6 +55,8 @@ func (cmd *stopContext) stop(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get global chaos interval interval := c.GlobalString("interval") // get wait time @@ -73,5 +75,5 @@ func (cmd *stopContext) stop(c *cli.Context) error { return err } // run stop command - return chaos.RunChaosCommand(cmd.context, stopCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, stopCommand, interval, random, skipError) } diff --git a/pkg/chaos/netem/cmd/corrupt.go b/pkg/chaos/netem/cmd/corrupt.go index 90074c43..d6c70978 100644 --- a/pkg/chaos/netem/cmd/corrupt.go +++ b/pkg/chaos/netem/cmd/corrupt.go @@ -46,6 +46,8 @@ func (cmd *corruptContext) corrupt(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get names or pattern names, pattern := chaos.GetNamesOrPattern(c) // get global chaos interval @@ -79,5 +81,5 @@ func (cmd *corruptContext) corrupt(c *cli.Context) error { return err } // run netem command - return chaos.RunChaosCommand(cmd.context, corruptCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, corruptCommand, interval, random, skipError) } diff --git a/pkg/chaos/netem/cmd/delay.go b/pkg/chaos/netem/cmd/delay.go index 4c482b7b..5767c912 100644 --- a/pkg/chaos/netem/cmd/delay.go +++ b/pkg/chaos/netem/cmd/delay.go @@ -56,6 +56,8 @@ func (cmd *delayContext) delay(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get names or pattern names, pattern := chaos.GetNamesOrPattern(c) // get global chaos interval @@ -93,5 +95,5 @@ func (cmd *delayContext) delay(c *cli.Context) error { return err } // run netem delay command - return chaos.RunChaosCommand(cmd.context, delayCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, delayCommand, interval, random, skipError) } diff --git a/pkg/chaos/netem/cmd/duplicate.go b/pkg/chaos/netem/cmd/duplicate.go index 378ff46e..559367f4 100644 --- a/pkg/chaos/netem/cmd/duplicate.go +++ b/pkg/chaos/netem/cmd/duplicate.go @@ -46,6 +46,8 @@ func (cmd *duplicateContext) duplicate(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get names or pattern names, pattern := chaos.GetNamesOrPattern(c) // get global chaos interval @@ -79,5 +81,5 @@ func (cmd *duplicateContext) duplicate(c *cli.Context) error { return err } // run netem command - return chaos.RunChaosCommand(cmd.context, duplicateCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, duplicateCommand, interval, random, skipError) } diff --git a/pkg/chaos/netem/cmd/loss.go b/pkg/chaos/netem/cmd/loss.go index 82a5dcc0..92a7fbe0 100644 --- a/pkg/chaos/netem/cmd/loss.go +++ b/pkg/chaos/netem/cmd/loss.go @@ -46,6 +46,8 @@ func (cmd *lossContext) loss(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get names or pattern names, pattern := chaos.GetNamesOrPattern(c) // get global chaos interval @@ -79,5 +81,5 @@ func (cmd *lossContext) loss(c *cli.Context) error { return err } // run netem command - return chaos.RunChaosCommand(cmd.context, lossCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, lossCommand, interval, random, skipError) } diff --git a/pkg/chaos/netem/cmd/loss_ge.go b/pkg/chaos/netem/cmd/loss_ge.go index e47f9e1d..7f29ecf3 100644 --- a/pkg/chaos/netem/cmd/loss_ge.go +++ b/pkg/chaos/netem/cmd/loss_ge.go @@ -57,6 +57,8 @@ func (cmd *lossGEContext) lossGE(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get names or pattern names, pattern := chaos.GetNamesOrPattern(c) // get global chaos interval @@ -94,5 +96,5 @@ func (cmd *lossGEContext) lossGE(c *cli.Context) error { return err } // run netem command - return chaos.RunChaosCommand(cmd.context, lossGECommand, interval, random) + return chaos.RunChaosCommand(cmd.context, lossGECommand, interval, random, skipError) } diff --git a/pkg/chaos/netem/cmd/loss_state.go b/pkg/chaos/netem/cmd/loss_state.go index 2825b76f..a1a49b29 100644 --- a/pkg/chaos/netem/cmd/loss_state.go +++ b/pkg/chaos/netem/cmd/loss_state.go @@ -68,6 +68,8 @@ func (cmd *lossStateContext) lossState(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get names or pattern names, pattern := chaos.GetNamesOrPattern(c) // get global chaos interval @@ -107,5 +109,5 @@ func (cmd *lossStateContext) lossState(c *cli.Context) error { return err } // run netem command - return chaos.RunChaosCommand(cmd.context, lossStateCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, lossStateCommand, interval, random, skipError) } diff --git a/pkg/chaos/netem/cmd/rate.go b/pkg/chaos/netem/cmd/rate.go index 08f6a0a2..0242e93b 100644 --- a/pkg/chaos/netem/cmd/rate.go +++ b/pkg/chaos/netem/cmd/rate.go @@ -56,6 +56,8 @@ func (cmd *rateContext) rate(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get names or pattern names, pattern := chaos.GetNamesOrPattern(c) // get global chaos interval @@ -93,5 +95,5 @@ func (cmd *rateContext) rate(c *cli.Context) error { return err } // run netem command - return chaos.RunChaosCommand(cmd.context, lossCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, lossCommand, interval, random, skipError) } diff --git a/pkg/chaos/stress/cmd/stress.go b/pkg/chaos/stress/cmd/stress.go index ee638095..88504996 100644 --- a/pkg/chaos/stress/cmd/stress.go +++ b/pkg/chaos/stress/cmd/stress.go @@ -54,6 +54,8 @@ func (cmd *stressContext) stress(c *cli.Context) error { labels := c.GlobalStringSlice("label") // get dry-run mode dryRun := c.GlobalBool("dry-run") + // get skip error flag + skipError := c.GlobalBool("skip-error") // get interval interval := c.GlobalString("interval") // get names or pattern @@ -74,5 +76,5 @@ func (cmd *stressContext) stress(c *cli.Context) error { return err } // run stress command - return chaos.RunChaosCommand(cmd.context, stressCommand, interval, random) + return chaos.RunChaosCommand(cmd.context, stressCommand, interval, random, skipError) }