can skip error in chaos command and continue on next interval tick #175

alexei-led · Sep 12, 2020 · bce2756 · bce2756
1 parent cb5e503
commit bce2756
Show file tree

Hide file tree

Showing 14 changed files with 45 additions and 14 deletions.
diff --git a/cmd/main.go b/cmd/main.go
@@ -150,6 +150,10 @@ func main() {
 			Usage:  "dry run does not create chaos, only logs planned chaos commands",
 			EnvVar: "DRY-RUN",
 		},
+		cli.BoolFlag{
+			Name:  "skip-error",
+			Usage: "skip chaos command error and retry to execute the command on next interval tick",
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {

diff --git a/pkg/chaos/command.go b/pkg/chaos/command.go
@@ -52,7 +52,7 @@ func GetNamesOrPattern(c *cli.Context) ([]string, string) {
 }
 
 // RunChaosCommand run chaos command in go routine
-func RunChaosCommand(topContext context.Context, command Command, intervalStr string, random bool) error {
+func RunChaosCommand(topContext context.Context, command Command, intervalStr string, random, skipError bool) error {
 	// parse interval
 	interval, err := util.GetIntervalValue(intervalStr)
 	if err != nil {
@@ -75,7 +75,10 @@ func RunChaosCommand(topContext context.Context, command Command, intervalStr st
 	for {
 		// run chaos function
 		if err := command.Run(ctx, random); err != nil {
-			return err
+			if !skipError {
+				return err
+			}
+			log.WithError(err).Warn("skipping error")
 		}
 		// wait for next timer tick or cancel
 		select {

diff --git a/pkg/chaos/docker/cmd/kill.go b/pkg/chaos/docker/cmd/kill.go
@@ -46,6 +46,8 @@ func (cmd *killContext) kill(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get interval
 	interval := c.GlobalString("interval")
 	// get names or pattern
@@ -60,5 +62,5 @@ func (cmd *killContext) kill(c *cli.Context) error {
 		return err
 	}
 	// run kill command
-	return chaos.RunChaosCommand(cmd.context, killCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, killCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/docker/cmd/pause.go b/pkg/chaos/docker/cmd/pause.go
@@ -43,6 +43,8 @@ func (cmd *pauseContext) pause(c *cli.Context) error {
 	random := c.GlobalBool("random")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get labels
 	labels := c.GlobalStringSlice("label")
 	// get global chaos interval
@@ -59,5 +61,5 @@ func (cmd *pauseContext) pause(c *cli.Context) error {
 		return err
 	}
 	// run pause command
-	return chaos.RunChaosCommand(cmd.context, pauseCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, pauseCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/docker/cmd/remove.go b/pkg/chaos/docker/cmd/remove.go
@@ -53,6 +53,8 @@ func (cmd *removeContext) remove(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get interval
 	interval := c.GlobalString("interval")
 	// get names or pattern
@@ -71,5 +73,5 @@ func (cmd *removeContext) remove(c *cli.Context) error {
 		return err
 	}
 	// run remove command
-	return chaos.RunChaosCommand(cmd.context, removeCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, removeCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/docker/cmd/stop.go b/pkg/chaos/docker/cmd/stop.go
@@ -55,6 +55,8 @@ func (cmd *stopContext) stop(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get global chaos interval
 	interval := c.GlobalString("interval")
 	// get wait time
@@ -73,5 +75,5 @@ func (cmd *stopContext) stop(c *cli.Context) error {
 		return err
 	}
 	// run stop command
-	return chaos.RunChaosCommand(cmd.context, stopCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, stopCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/netem/cmd/corrupt.go b/pkg/chaos/netem/cmd/corrupt.go
@@ -46,6 +46,8 @@ func (cmd *corruptContext) corrupt(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get names or pattern
 	names, pattern := chaos.GetNamesOrPattern(c)
 	// get global chaos interval
@@ -79,5 +81,5 @@ func (cmd *corruptContext) corrupt(c *cli.Context) error {
 		return err
 	}
 	// run netem command
-	return chaos.RunChaosCommand(cmd.context, corruptCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, corruptCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/netem/cmd/delay.go b/pkg/chaos/netem/cmd/delay.go
@@ -56,6 +56,8 @@ func (cmd *delayContext) delay(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get names or pattern
 	names, pattern := chaos.GetNamesOrPattern(c)
 	// get global chaos interval
@@ -93,5 +95,5 @@ func (cmd *delayContext) delay(c *cli.Context) error {
 		return err
 	}
 	// run netem delay command
-	return chaos.RunChaosCommand(cmd.context, delayCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, delayCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/netem/cmd/duplicate.go b/pkg/chaos/netem/cmd/duplicate.go
@@ -46,6 +46,8 @@ func (cmd *duplicateContext) duplicate(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get names or pattern
 	names, pattern := chaos.GetNamesOrPattern(c)
 	// get global chaos interval
@@ -79,5 +81,5 @@ func (cmd *duplicateContext) duplicate(c *cli.Context) error {
 		return err
 	}
 	// run netem command
-	return chaos.RunChaosCommand(cmd.context, duplicateCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, duplicateCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/netem/cmd/loss.go b/pkg/chaos/netem/cmd/loss.go
@@ -46,6 +46,8 @@ func (cmd *lossContext) loss(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get names or pattern
 	names, pattern := chaos.GetNamesOrPattern(c)
 	// get global chaos interval
@@ -79,5 +81,5 @@ func (cmd *lossContext) loss(c *cli.Context) error {
 		return err
 	}
 	// run netem command
-	return chaos.RunChaosCommand(cmd.context, lossCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, lossCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/netem/cmd/loss_ge.go b/pkg/chaos/netem/cmd/loss_ge.go
@@ -57,6 +57,8 @@ func (cmd *lossGEContext) lossGE(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get names or pattern
 	names, pattern := chaos.GetNamesOrPattern(c)
 	// get global chaos interval
@@ -94,5 +96,5 @@ func (cmd *lossGEContext) lossGE(c *cli.Context) error {
 		return err
 	}
 	// run netem command
-	return chaos.RunChaosCommand(cmd.context, lossGECommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, lossGECommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/netem/cmd/loss_state.go b/pkg/chaos/netem/cmd/loss_state.go
@@ -68,6 +68,8 @@ func (cmd *lossStateContext) lossState(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get names or pattern
 	names, pattern := chaos.GetNamesOrPattern(c)
 	// get global chaos interval
@@ -107,5 +109,5 @@ func (cmd *lossStateContext) lossState(c *cli.Context) error {
 		return err
 	}
 	// run netem command
-	return chaos.RunChaosCommand(cmd.context, lossStateCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, lossStateCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/netem/cmd/rate.go b/pkg/chaos/netem/cmd/rate.go
@@ -56,6 +56,8 @@ func (cmd *rateContext) rate(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get names or pattern
 	names, pattern := chaos.GetNamesOrPattern(c)
 	// get global chaos interval
@@ -93,5 +95,5 @@ func (cmd *rateContext) rate(c *cli.Context) error {
 		return err
 	}
 	// run netem command
-	return chaos.RunChaosCommand(cmd.context, lossCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, lossCommand, interval, random, skipError)
 }
diff --git a/pkg/chaos/stress/cmd/stress.go b/pkg/chaos/stress/cmd/stress.go
@@ -54,6 +54,8 @@ func (cmd *stressContext) stress(c *cli.Context) error {
 	labels := c.GlobalStringSlice("label")
 	// get dry-run mode
 	dryRun := c.GlobalBool("dry-run")
+	// get skip error flag
+	skipError := c.GlobalBool("skip-error")
 	// get interval
 	interval := c.GlobalString("interval")
 	// get names or pattern
@@ -74,5 +76,5 @@ func (cmd *stressContext) stress(c *cli.Context) error {
 		return err
 	}
 	// run stress command
-	return chaos.RunChaosCommand(cmd.context, stressCommand, interval, random)
+	return chaos.RunChaosCommand(cmd.context, stressCommand, interval, random, skipError)
 }