diff --git a/.github/workflows/havoc-lint.yml b/.github/workflows/havoc-lint.yml new file mode 100644 index 000000000..de73fbb0c --- /dev/null +++ b/.github/workflows/havoc-lint.yml @@ -0,0 +1,22 @@ +name: Havoc Lint +on: + push: +permissions: + contents: read +jobs: + golangci: + defaults: + run: + working-directory: havoc + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.20' + cache: true + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.51.2 diff --git a/.github/workflows/havoc-release.yml b/.github/workflows/havoc-release.yml new file mode 100644 index 000000000..708ce0683 --- /dev/null +++ b/.github/workflows/havoc-release.yml @@ -0,0 +1,37 @@ +on: + release: + types: [created] + +permissions: + contents: write + packages: write + +jobs: + release-linux-amd64: + defaults: + run: + working-directory: havoc + name: release linux/amd64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: wangyoucao577/go-release-action@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: linux + goarch: amd64 + project_path: cmd + release-linux-arm64: + defaults: + run: + working-directory: havoc + name: release linux/arm64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: wangyoucao577/go-release-action@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: darwin + goarch: arm64 + project_path: cmd diff --git a/.github/workflows/havoc-test-api.yml b/.github/workflows/havoc-test-api.yml new file mode 100644 index 000000000..412c30070 --- /dev/null +++ b/.github/workflows/havoc-test-api.yml @@ -0,0 +1,22 @@ +name: CLI tests +on: + push: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + defaults: + run: + working-directory: havoc + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + cache: true + - name: Run tests + run: | + make test diff --git a/.prettierignore b/.prettierignore index 5c08ba825..71b9e910e 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,3 +7,4 @@ k8s-test-runner/chart/**/*.yaml node_modules/ index.yaml wasp/** +havoc/testdata/** \ No newline at end of file diff --git a/havoc/.gitignore b/havoc/.gitignore new file mode 100644 index 000000000..be5b7c479 --- /dev/null +++ b/havoc/.gitignore @@ -0,0 +1,59 @@ +# IDE and environment +.idea/ +.vscode/ +.DS_STORE + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories +dist/ +vendor/ +node_modules/ +.yarn/ + +# Mercuy server configuration file +config.toml +config.*.toml + +# Personal/secret env vars +.envrc-personal + +# Other +tmp/ +*.log +*.swp +.air.toml +.DS_Store +output.txt + +# Default binary name with go build +main +test.sh + +# Default generated experiments +havoc-experiments +havoc-default +experiments-crib-core +testdata/experiments-test +havoc-monkey-temp-dir + +# General env vars config +.envrc + +# Dumps of configs +config_dump.toml +pods_dump.json +testdata/result/** +testdata/results/** +!testdata/results/.gitkeep diff --git a/havoc/LICENSE b/havoc/LICENSE new file mode 100644 index 000000000..98f60070f --- /dev/null +++ b/havoc/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 SmartContract ChainLink, Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/havoc/Makefile b/havoc/Makefile new file mode 100644 index 000000000..fb995d094 --- /dev/null +++ b/havoc/Makefile @@ -0,0 +1,24 @@ +.PHONY: test +test: + go test -v -count 1 `go list ./... | grep -v examples` -run TestSmoke + +.PHONY: test_race +test_race: + go test -v -race -count 1 `go list ./... | grep -v examples` -run TestSmoke + +.PHONY: test+cover +test_cover: + go test -v -coverprofile cover.out -count 1 `go list ./... | grep -v examples` -run "TestSmoke|TestAPI" + go tool cover -html cover.out + +.PHONY: install +install: + go install cmd/havoc.go + +.PHONE: build +build: + go build cmd/havoc.go + +.PHONY: lint +lint: + golangci-lint --color=always run -v diff --git a/havoc/README.md b/havoc/README.md new file mode 100644 index 000000000..26e0d6c56 --- /dev/null +++ b/havoc/README.md @@ -0,0 +1,171 @@ +## Havoc + +_DISCLAIMER_: This software is not even early Alpha, and still in development, use it on your own risk + +Havoc is a tool that introspects your k8s namespace and generates a `ChaosMesh` CRDs suite for you + +You can use havoc as a CLI to quickly test hypothesis or run it in "monkey" mode with your load tests and have Grafana annotations + +### How it works + +![img.png](img.png) + +Havoc generates groups of experiments based on your pods and labels found in namespace + +In order to test your namespace you need to label pods accordingly: + +- `havoc-component-group` is like `app.kubernetes.io/component` (see [recommendation](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/)) but should be set explicitly +- `havoc-network-group` in most cases match `havoc-component-group` but sometimes you need to break network even inside component group, ex. distributed databases + +Example: + +``` + havoc-component-group: node + havoc-network-group: nodes-1 +``` + +Every pod without a group will be marked as `no-group` and experiments will be assigned accordingly + +Single pod experiments: + +- PodFailure +- NetworkChaos (Pod latency) +- Stress (Memory) +- Stress (CPU) +- External service failure (Network partition) +- Blockchain specific experiments + +Group experiments: + +- Group failure +- Group latency +- Group CPU +- Group memory +- Group network partition +- OpenAPI based HTTP experiments + +You can generate default chaos suite by [configuring](havoc.toml) havoc then set `dir` param and add your custom experiments, then run monkey to test your services + +### Why use it? + +#### Without Havoc your workflow is + +- Inspect full rendered deployment of your namespace +- Figure out multiple groups of components you can select by various labels or annotations to form experiments +- If some components are not selectable - ask DevOps guys to change the manifests +- Create set of experiments for each chaos experiment type by hand or copy from other product chaos tests +- Calculate permutations of different groups and calculate composite experiments (network partitioning, latency) +- Create experiment for each API in every OpenAPI spec +- Compose huge ChaosMesh Workflow YAML that fails without proper validation errors if group has no match or label is invalid +- Run the load test, then manually run the chaos suite +- Check experiment logs to debug with kubectl +- Figure out which failures are caused by which experiments +- If you have more than one project, use some templating make experiments work for other projects + +#### With Havoc + +- Have a simple labelling convention for your namespaces, fill 5 vars in `TOML` config +- Run chaos testing with `havoc -c havoc.toml run ${namespace}` + +### Install + +Please use GitHub releases of this repo +Download latest [release](https://github.com/smartcontractkit/havoc/releases) + +You need `kubectl` to available on your machine + +You also need [ChaosMesh](https://chaos-mesh.org/) installed in your `k8s` cluster + +### Grafana integration + +Set env variables + +``` +HAVOC_LOG_LEVEL={warn,info,debug,trace} +GRAFANA_URL="..." +GRAFANA_TOKEN="..." +``` + +Set dashboard names in `havoc.toml` + +``` +[havoc.grafana] +# UIDs of dashboard which should be annotated with chaos experiments metadata +# You can also try to use name as you see it in the top bar of your dashboard but that's not guaranteed to match +dashboard_uids = ["WaspDebug", "e98b5451-12dc-4a8b-9576-2c0b67ddbd0c"] +``` + +### Manual usage + +Generate default experiments for your namespace + +``` +havoc -c havoc.toml generate [namespace] +``` + +Check this [section](havoc.toml) for `ignore_pods` and `ignore_group_labels`, default settings should be reasonable, however, you can tweak them + +This will create `havoc-experiments` dir, then you can choose from recommended experiments + +``` +havoc -c havoc.toml apply +``` + +You can also apply your experiment directly, using absolute or relative path to experiment file + +``` +havoc -c havoc.toml apply ${experiment_file_path} +``` + +### Monkey mode + +You can run havoc as an automated sequential or randomized suite + +``` +havoc -c havoc.toml run [namespace] +``` + +See `[havoc.monkey]` config [here](havoc.toml) + +### Programmatic usage + +See how you can use recommended experiments from code in [examples](examples) + +### Custom experiments + +Havoc is just a generator and a module that reads your `dir = $mydir` from config + +If you wish to add custom experiments written by hand create your custom directory and add experiments + +Experiments will be executed in lexicographic order, however, for custom experiments there are 2 simple rules: + +- directory names must be in + +``` + "external", + "failure", + "latency", + "cpu", + "memory", + "group-failure", + "group-latency", + "group-cpu", + "group-memory", + "group-partition", + "blockchain_rewind_head", + "http" +``` + +- `metadata.name` should be equal to your experiment filename + +When you are using `run` monkey command, if directory is not empty havoc won't automatically generate experiments, so you can extend generated experiments with your custom modifications + +### Developing + +We are using [nix](https://nixos.org/) + +Enter the shell + +``` +nix develop +``` diff --git a/havoc/cli.go b/havoc/cli.go new file mode 100644 index 000000000..d240820d9 --- /dev/null +++ b/havoc/cli.go @@ -0,0 +1,209 @@ +package havoc + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/c-bata/go-prompt" + "github.com/pkg/errors" + "github.com/urfave/cli/v2" +) + +const ( + DefaultCMDTimeout = "3m" + + ErrNoSelection = "no selection, exiting" + ErrInvalidNamespace = "first argument must be a valid k8s namespace" + ErrAutocompleteError = "autocomplete file walk errored" +) + +func experimentCompleter(dir string, expType string) (func(d prompt.Document) []prompt.Suggest, error) { + s := make([]prompt.Suggest, 0) + err := filepath.Walk( + fmt.Sprintf("%s/%s", dir, expType), + func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + s = append(s, prompt.Suggest{ + Text: info.Name(), + Description: info.Name(), + }) + return nil + }) + if err != nil { + return nil, err + } + return func(d prompt.Document) []prompt.Suggest { + return prompt.FilterHasPrefix(s, d.GetWordBeforeCursor(), true) + }, nil +} + +func experimentTypeCompleter(dir string) (func(d prompt.Document) []prompt.Suggest, error) { + s := make([]prompt.Suggest, 0) + err := filepath.Walk( + dir, + func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() && info.Name() != dir { + s = append(s, prompt.Suggest{ + Text: info.Name(), + Description: info.Name(), + }) + } + return nil + }) + if err != nil { + return nil, err + } + return func(d prompt.Document) []prompt.Suggest { + return prompt.FilterHasPrefix(s, d.GetWordBeforeCursor(), true) + }, nil +} + +func RunCLI(args []string) error { + app := &cli.App{ + EnableBashCompletion: true, + Name: "havoc", + Version: "v0.0.1", + Usage: "Automatic chaos experiments CLI", + UsageText: `Utility to generate and apply chaos experiments for a namespace`, + Before: func(cCtx *cli.Context) error { + InitDefaultLogging() + return nil + }, + Flags: []cli.Flag{ + &cli.StringFlag{Name: "config", Aliases: []string{"c"}}, + }, + Commands: []*cli.Command{ + { + Name: "generate", + HelpName: "generate", + Aliases: []string{"g"}, + Description: `generates chaos experiments: +havoc generate [namespace] +or use custom config +havoc -c havoc.toml generate [namespace] +you can also specify a directory where to put manifests +havoc -c havoc.toml -d custom_experiments [namespace] +`, + Action: func(cliCtx *cli.Context) error { + ns := cliCtx.Args().Get(0) + if ns == "" { + return errors.New(ErrInvalidNamespace) + } + cfg, err := ReadConfig(cliCtx.String("config")) + if err != nil { + return err + } + m, err := NewController(cfg) + if err != nil { + return err + } + return m.GenerateSpecs(ns) + }, + }, + { + Name: "apply", + HelpName: "apply", + Aliases: []string{"a"}, + Description: `applies an experiment from a file: +examples: +# selecting an experiment +havoc -c havoc.toml apply +# applying experiment directly with relative or abs path +havoc apply ${experiment_path} +`, + Action: func(cliCtx *cli.Context) error { + cfg, err := ReadConfig(cliCtx.String("config")) + if err != nil { + return err + } + m, err := NewController(cfg) + if err != nil { + return err + } + cc, err := experimentTypeCompleter(m.cfg.Havoc.Dir) + if err != nil { + return err + } + + var expPath string + + arg := cliCtx.Args().Get(0) + + if arg != "" { + expPath = arg + } else { + expType := prompt.Input("Choose experiment type >> ", cc) + if expType == "" { + return errors.New(ErrNoSelection) + } + c, err := experimentCompleter(m.cfg.Havoc.Dir, expType) + if err != nil { + return errors.Wrap(err, ErrAutocompleteError) + } + expName := prompt.Input("Choose experiment name >> ", c) + if expName == "" { + return errors.New(ErrNoSelection) + } + expPath = fmt.Sprintf("%s/%s/%s", cfg.Havoc.Dir, expType, expName) + } + + nexp, err := NewNamedExperiment(expPath) + if err != nil { + return err + } + + return m.ApplyAndAnnotate(nexp) + }, + }, + { + Name: "run", + HelpName: "run", + Aliases: []string{"r"}, + Description: `starts a chaos monkey +examples: +havoc run -c havoc.toml [namespace] +`, + Action: func(cliCtx *cli.Context) error { + ns := cliCtx.Args().Get(0) + cfgPath := cliCtx.String("config") + cfg, err := ReadConfig(cfgPath) + if err != nil { + return err + } + m, err := NewController(cfg) + if err != nil { + return err + } + if _, err := os.Stat(cfg.Havoc.Dir); err != nil { + L.Info(). + Str("Dir", cfg.Havoc.Dir). + Msg("Dir not found, generating specified experiments directory") + err = m.GenerateSpecs(ns) + if err != nil { + return err + } + L.Info(). + Str("Dir", cfg.Havoc.Dir). + Msg("Using existing experiments dir, skipping generation") + } else { + L.Info(). + Str("Dir", cfg.Havoc.Dir). + Msg("Using existing experiments dir, skipping generation") + } + return m.Run() + }, + }, + }, + } + return app.Run(args) +} diff --git a/havoc/cmd/havoc.go b/havoc/cmd/havoc.go new file mode 100644 index 000000000..eeb809d9f --- /dev/null +++ b/havoc/cmd/havoc.go @@ -0,0 +1,13 @@ +package main + +import ( + "os" + + "github.com/smartcontractkit/chainlink-testing-framework/havoc" +) + +func main() { + if err := havoc.RunCLI(os.Args); err != nil { + havoc.L.Fatal().Err(err).Send() + } +} diff --git a/havoc/config.go b/havoc/config.go new file mode 100644 index 000000000..2d0c33d4f --- /dev/null +++ b/havoc/config.go @@ -0,0 +1,350 @@ +package havoc + +import ( + "github.com/pelletier/go-toml/v2" + "github.com/pkg/errors" + "github.com/rs/zerolog" + "os" + "strings" +) + +const ( + ErrReadSethConfig = "failed to read TOML config for havoc" + ErrUnmarshalSethConfig = "failed to unmarshal TOML config for havoc" + + ErrFailureGroupIsNil = "failure group must be specified in config" + ErrLatencyGroupIsNil = "latency group must be specified in config" + ErrStressCPUGroupIsNil = "stress cpu group must be specified in config" + ErrStressMemoryGroupIsNil = "stress memory group must be specified in config" + ErrFormat = "format error" +) + +const ( + DefaultExperimentsDir = "havoc-experiments" + DefaultPodFailureDuration = "1m" + DefaultNetworkLatencyDuration = "1m" + DefaultNetworkPartitionDuration = "1m" + DefaultHTTPDuration = "1m" + DefaultNetworkPartitionLabel = "havoc-network-group" + DefaultComponentGroupLabelKey = "havoc-component-group" + DefaultStressMemoryDuration = "1m" + DefaultStressMemoryWorkers = 1 + DefaultStressMemoryAmount = "512MB" + DefaultStressCPUDuration = "1m" + DefaultStressCPUWorkers = 1 + DefaultStressCPULoad = 100 + DefaultNetworkLatency = "300ms" + DefaultMonkeyDuration = "24h" + DefaultMonkeyMode = "seq" + DefaultMonkeyCooldown = "30s" +) + +var ( + DefaultGroupPercentage = []string{"10", "20", "30"} + DefaultGroupFixed = []string{"1", "2", "3"} + DefaultNetworkPartitionGroupPercentage = []string{"100"} +) + +var ( + DefaultIgnoreGroupLabels = []string{ + "mainnet", + "release", + "intents.otterize.com", + "pod-template-hash", + "rollouts-pod-template-hash", + "chain.link/app", + "chain.link/cost-center", + "chain.link/env", + "chain.link/project", + "chain.link/team", + "app.kubernetes.io/part-of", + "app.kubernetes.io/managed-by", + "app.chain.link/product", + "app.kubernetes.io/version", + "app.chain.link/blockchain", + "app.kubernetes.io/instance", + "app.kubernetes.io/name", + } +) + +type Config struct { + Havoc *Havoc `toml:"havoc"` +} + +type Havoc struct { + Dir string `toml:"dir"` + ExperimentTypes []string `toml:"experiment_types"` + NamespaceLabelFilter string `toml:"namespace_label_filter"` + ComponentLabelKey string `toml:"component_label_key"` + IgnoredPods []string `toml:"ignore_pods"` + IgnoreGroupLabels []string `toml:"ignore_group_labels"` + Failure *Failure `toml:"failure"` + Latency *Latency `toml:"latency"` + NetworkPartition *NetworkPartition `toml:"network_partition"` + StressMemory *StressMemory `toml:"stress_memory"` + StressCPU *StressCPU `toml:"stress_cpu"` + ExternalTargets *ExternalTargets `toml:"external_targets"` + BlockchainRewindHead *BlockchainRewindHead `toml:"blockchain_rewind_head"` + OpenAPI *OpenAPI `toml:"openapi"` + Monkey *Monkey `toml:"monkey"` + Grafana *Grafana `toml:"grafana"` +} + +func dumpConfig(cfg *Config) { + if L.GetLevel() == zerolog.DebugLevel { + d, _ := toml.Marshal(cfg) + _ = os.WriteFile("config_dump.toml", d, os.ModePerm) + } +} + +func DefaultConfig() *Config { + return &Config{ + Havoc: &Havoc{ + Dir: DefaultExperimentsDir, + ExperimentTypes: RecommendedExperimentTypes, + ComponentLabelKey: DefaultComponentGroupLabelKey, + IgnoreGroupLabels: DefaultIgnoreGroupLabels, + Failure: &Failure{ + Duration: DefaultPodFailureDuration, + GroupFixed: DefaultGroupFixed, + }, + Latency: &Latency{ + Duration: DefaultNetworkLatencyDuration, + Latency: DefaultNetworkLatency, + GroupFixed: DefaultGroupFixed, + }, + StressMemory: &StressMemory{ + Duration: DefaultStressMemoryDuration, + Workers: DefaultStressMemoryWorkers, + Memory: DefaultStressMemoryAmount, + GroupFixed: DefaultGroupFixed, + }, + StressCPU: &StressCPU{ + Duration: DefaultStressCPUDuration, + Workers: DefaultStressCPUWorkers, + Load: DefaultStressCPULoad, + GroupFixed: DefaultGroupFixed, + }, + NetworkPartition: &NetworkPartition{ + Duration: DefaultNetworkPartitionDuration, + Label: DefaultNetworkPartitionLabel, + GroupPercentage: DefaultNetworkPartitionGroupPercentage, + }, + OpenAPI: &OpenAPI{ + Duration: DefaultHTTPDuration, + GroupFixed: DefaultGroupFixed, + }, + Monkey: &Monkey{ + Duration: DefaultMonkeyDuration, + Mode: DefaultMonkeyMode, + Cooldown: DefaultMonkeyCooldown, + }, + Grafana: &Grafana{ + URL: os.Getenv("GRAFANA_URL"), + Token: os.Getenv("GRAFANA_TOKEN"), + }, + }, + } +} + +func (c *Config) Validate() []error { + errs := make([]error, 0) + if c.Havoc.Dir == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "monkey.dir must not be empty")) + } + if c.Havoc.Failure == nil { + errs = append(errs, errors.New(ErrFailureGroupIsNil)) + } + if c.Havoc.Latency == nil { + errs = append(errs, errors.New(ErrLatencyGroupIsNil)) + } + if c.Havoc.StressCPU == nil { + errs = append(errs, errors.New(ErrStressCPUGroupIsNil)) + } + if c.Havoc.StressMemory == nil { + errs = append(errs, errors.New(ErrStressMemoryGroupIsNil)) + } + if c.Havoc.Failure != nil { + if c.Havoc.Failure.Duration == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "failure.duration must be in Go duration format, 1d2h3m0s")) + } + } + if c.Havoc.Latency != nil { + if c.Havoc.Latency.Duration == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "latency.duration must be in Go duration format, 1d2h3m0s")) + } + if c.Havoc.Latency.Latency == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "latency.latency must be in milliseconds format, ex.: 300ms")) + } + } + if c.Havoc.StressMemory != nil { + if c.Havoc.StressMemory.Workers <= 0 { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "stress_memory.workers must be set, ex.: \"4\"")) + } + if c.Havoc.StressMemory.Memory == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "stress_memory.memory must be set, ex.: \"256MB\" or \"25%\"")) + } + } + if c.Havoc.StressCPU != nil { + if c.Havoc.StressCPU.Workers <= 0 { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "stress_cpu.workers must be set, ex.: \"1\"")) + } + if c.Havoc.StressCPU.Load <= 0 { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "stress_cpu.load must be set, ex.: \"100\"")) + } + } + if c.Havoc.BlockchainRewindHead != nil { + if c.Havoc.BlockchainRewindHead.Duration == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "havoc.blockchain_rewind_head.duration must be set, ex.: \"30s\"")) + } + for _, bn := range c.Havoc.BlockchainRewindHead.NodesConfig { + if bn.ExecutorPodPrefix == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "havoc.blockchain_rewind_head.nodes.executor_pod_prefix must be set, ex.: \"geth\"")) + } + if bn.ExecutorContainerName == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "havoc.blockchain_rewind_head.nodes.executor_container_name must be set, ex.: \"geth-network\"")) + } + if bn.NodeInternalHTTPURL == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "havoc.blockchain_rewind_head.nodes.node_internal_http_url must be set, ex.: \"geth-1337:8544\"")) + } + if len(bn.Blocks) == 0 { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "havoc.blockchain_rewind_head.nodes.blocks must be set, ex.: \"10\"")) + } + } + } + if c.Havoc.Monkey != nil { + if c.Havoc.Monkey.Mode == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "monkey.mode must be either \"seq\" or \"rand\"")) + } + if c.Havoc.Monkey.Duration == "" { + errs = append(errs, errors.Wrap(errors.New(ErrFormat), "monkey.duration must be in Go duration format, 1d2h3m0s")) + } + } + return errs +} + +type Failure struct { + Duration string `toml:"duration"` + GroupPercentage []string `toml:"group_percentage"` + GroupFixed []string `toml:"group_fixed"` +} + +type Latency struct { + Duration string `toml:"duration"` + Latency string `toml:"latency"` + GroupPercentage []string `toml:"group_percentage"` + GroupFixed []string `toml:"group_fixed"` +} + +type NetworkPartition struct { + Duration string `toml:"duration"` + Label string `toml:"label"` + GroupPercentage []string `toml:"group_percentage"` + GroupFixed []string `toml:"group_fixed"` +} + +type StressMemory struct { + Duration string `toml:"duration"` + Workers int `toml:"workers"` + Memory string `toml:"memory"` + GroupPercentage []string `toml:"group_percentage"` + GroupFixed []string `toml:"group_fixed"` +} + +type StressCPU struct { + Duration string `toml:"duration"` + Workers int `toml:"workers"` + Load int `toml:"load"` + GroupPercentage []string `toml:"group_percentage"` + GroupFixed []string `toml:"group_fixed"` +} + +type ExternalTargets struct { + Duration string `toml:"duration"` + URLs []string `toml:"urls"` +} + +type BlockchainRewindHead struct { + Duration string `toml:"duration"` + NodesConfig []*BlockchainNodeConfig `toml:"nodes"` +} + +type BlockchainNodeConfig struct { + ExecutorPodPrefix string `toml:"executor_pod_prefix"` + ExecutorContainerName string `toml:"executor_container_name"` + NodeInternalHTTPURL string `toml:"node_internal_http_url"` + Blocks []int64 `toml:"blocks"` +} + +type OpenAPI struct { + Mapping map[string]*OpenApiSpecInfo `toml:"mapping"` + Duration string `toml:"duration"` + GroupPercentage []string `toml:"group_percentage"` + GroupFixed []string `toml:"group_fixed"` +} + +type OpenApiSpecInfo struct { + SpecToPortMappings []*SpecToPort `toml:"spec_to_port"` +} + +type SpecToPort struct { + Port int64 `toml:"port"` + Path string `toml:"path"` +} + +type Monkey struct { + Duration string `toml:"duration"` + Cooldown string `toml:"cooldown"` + Mode string `toml:"mode"` +} + +type Grafana struct { + URL string `toml:"grafana_url"` + Token string `toml:"grafana_token"` + DashboardUIDs []string `toml:"dashboard_uids"` +} + +func ReadConfig(path string) (*Config, error) { + cfg := DefaultConfig() + dumpConfig(cfg) + if path == "" { + L.Info().Msg("No config specified, using default configuration") + } else { + L.Debug(). + Str("Path", path). + Msg("Reading config from path") + d, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrap(err, ErrReadSethConfig) + } + err = toml.Unmarshal(d, &cfg) + if err != nil { + return nil, errors.Wrap(err, ErrUnmarshalSethConfig) + } + } + L.Debug(). + Interface("Config", cfg). + Msg("Configuration loaded") + cfg.Havoc.Grafana.URL = os.Getenv("GRAFANA_URL") + cfg.Havoc.Grafana.Token = os.Getenv("GRAFANA_TOKEN") + return cfg, nil +} + +// nolint +func sliceContains(target string, array []string) bool { + for _, element := range array { + if element == target { + return true + } + } + return false +} + +func sliceContainsSubString(target string, array []string) bool { + for _, element := range array { + if strings.Contains(target, element) { + return true + } + } + return false +} diff --git a/havoc/examples/experiments_test.go b/havoc/examples/experiments_test.go new file mode 100644 index 000000000..f82051af2 --- /dev/null +++ b/havoc/examples/experiments_test.go @@ -0,0 +1,41 @@ +package havoc_example + +import ( + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/havoc" + "github.com/stretchr/testify/require" + "testing" +) + +func createMonkey(t *testing.T, l zerolog.Logger, namespace string) *havoc.Controller { + havoc.SetGlobalLogger(l) + cfg, err := havoc.ReadConfig("config.toml") + require.NoError(t, err) + c, err := havoc.NewController(cfg) + err = c.GenerateSpecs(namespace) + require.NoError(t, err) + return c +} + +func TestMyLoad(t *testing.T) { + /* my testing logger */ + l := havoc.L + /* my load test preparation here */ + /* wrapping with chaos monkey */ + monkey := createMonkey(t, l, "my namespace, get it from config") + go monkey.Run() + /* my test runs and ends */ + errs := monkey.Stop() + require.Len(t, errs, 0) +} + +func TestCodeRun(t *testing.T) { + cfg, err := havoc.ReadConfig("../havoc.toml") + require.NoError(t, err) + c, err := havoc.NewController(cfg) + require.NoError(t, err) + nexp, err := havoc.NewNamedExperiment("../experiments-crib-core/failure/failure-app-node-1-bootstrap-69fb558d9-s7npw.yaml") + require.NoError(t, err) + err = c.ApplyAndAnnotate(nexp) + require.NoError(t, err) +} diff --git a/havoc/flake.lock b/havoc/flake.lock new file mode 100644 index 000000000..cc78f23f2 --- /dev/null +++ b/havoc/flake.lock @@ -0,0 +1,111 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "locked": { + "lastModified": 1644229661, + "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "foundry": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1704359589, + "narHash": "sha256-kBFwc8WgQq57TLtOuU7yRoos9S3/P6eZO28xNEqOmH4=", + "owner": "shazow", + "repo": "foundry.nix", + "rev": "c5090280d94328924eddca6939e82216183a9461", + "type": "github" + }, + "original": { + "owner": "shazow", + "ref": "monthly", + "repo": "foundry.nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1666753130, + "narHash": "sha256-Wff1dGPFSneXJLI2c0kkdWTgxnQ416KE6X4KnFkgPYQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f540aeda6f677354f1e7144ab04352f61aaa0118", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1704194953, + "narHash": "sha256-RtDKd8Mynhe5CFnVT8s0/0yqtWFMM9LmCzXv/YKxnq4=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "bd645e8668ec6612439a9ee7e71f7eac4099d4f6", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "foundry": "foundry", + "nixpkgs": "nixpkgs_2" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/havoc/flake.nix b/havoc/flake.nix new file mode 100644 index 000000000..b8cd5f9bd --- /dev/null +++ b/havoc/flake.nix @@ -0,0 +1,16 @@ +{ + description = "havoc"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + foundry.url = "github:shazow/foundry.nix/monthly"; # Use monthly branch for permanent releases + }; + + outputs = inputs@{ self, nixpkgs, flake-utils, foundry, ... }: + flake-utils.lib.eachDefaultSystem (system: + let pkgs = import nixpkgs { inherit system; overlays = [ foundry.overlay ]; }; + in rec { + devShell = pkgs.callPackage ./shell.nix {}; + }); +} diff --git a/havoc/generate.go b/havoc/generate.go new file mode 100644 index 000000000..bc036a8f8 --- /dev/null +++ b/havoc/generate.go @@ -0,0 +1,1113 @@ +package havoc + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/samber/lo" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "text/template" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + "gopkg.in/yaml.v3" +) + +const ( + ErrParsingTemplate = "failed to parse Go text template" + + ErrExperimentTimeout = "waiting for experiment to finish timed out" + ErrExperimentApply = "error applying experiment manifest" + ErrInvalidCustomKind = "invalid custom Kind of experiment" +) + +const ( + DebugContainerImage = "curlimages/curl:latest" +) + +var ( + RecommendedExperimentTypes = []string{ + ChaosTypeFailure, + ChaosTypeLatency, + ChaosTypeGroupFailure, + ChaosTypeGroupLatency, + ChaosTypeStressMemory, + ChaosTypeStressGroupMemory, + ChaosTypeStressCPU, + ChaosTypeStressGroupCPU, + ChaosTypePartitionGroup, + ChaosTypeHTTP, + //ChaosTypePartitionExternal, + } +) + +// MarshalTemplate Helper to marshal templates +func MarshalTemplate(jobSpec interface{}, name, templateString string) (string, error) { + var buf bytes.Buffer + tmpl, err := template.New(name).Parse(templateString) + if err != nil { + return "", errors.Wrap(err, ErrParsingTemplate) + } + err = tmpl.Execute(&buf, jobSpec) + if err != nil { + return "", err + } + return buf.String(), err +} + +type CommonExperimentMeta struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + } `yaml:"metadata"` +} + +type HTTPExperiment struct { + ExperimentName string + Metadata *Metadata + Namespace string + Mode string + ModeValue string + Selector string + PodName string + Port int64 + Target string + Path string + Method string + Abort bool + Duration string +} + +func (m HTTPExperiment) String() (string, error) { + tpl := ` +kind: HTTPChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: {{ .ExperimentName }} +spec: + mode: {{ .Mode }} + {{- if .ModeValue }} + value: '{{ .ModeValue }}' + {{- end }} + selector: + namespaces: + - {{ .Namespace }} + {{- if .Selector}} + labelSelectors: + {{ .Selector }} + {{- else}} + fieldSelectors: + metadata.name: {{ .PodName }} + {{- end}} + target: Request + port: {{ .Port }} + method: {{ .Method }} + path: {{ .Path }} + abort: {{ .Abort }} + duration: {{ .Duration }} +` + return MarshalTemplate( + m, + uuid.NewString(), + tpl, + ) +} + +type BlockchainRewindHeadExperiment struct { + ExperimentName string `yaml:"experimentName"` + Metadata *Metadata `yaml:"metadata"` + Namespace string `yaml:"namespace"` + PodName string `yaml:"podName"` + ExecutorPodPrefix string `yaml:"executorPodPrefix"` + ExecutorContainerName string `yaml:"executorContainerName"` + NodeInternalHTTPURL string `yaml:"nodeInternalHTTPURL"` + Blocks int64 `yaml:"blocks"` +} + +type Metadata struct { + Name string `json:"name"` + Labels map[string]string `json:"labels"` +} + +func (m BlockchainRewindHeadExperiment) String() (string, error) { + tpl := ` +kind: blockchain_rewind_head +name: {{ .ExperimentName }} +metadata: + name: {{ .Metadata.Name }} +podName: {{ .PodName }} +executorContainerName: {{ .ExecutorContainerName }} +nodeInternalHTTPURL: {{ .NodeInternalHTTPURL }} +namespace: {{ .Namespace }} +blocks: {{ .Blocks }} +` + return MarshalTemplate( + m, + uuid.NewString(), + tpl, + ) +} + +type NetworkChaosExperiment struct { + ExperimentName string + Mode string + ModeValue string + Namespace string + Duration string + Latency string + PodName string + Selector string +} + +func (m NetworkChaosExperiment) String() (string, error) { + tpl := ` +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: {{ .ExperimentName }} + namespace: {{ .Namespace }} +spec: + selector: + namespaces: + - {{ .Namespace }} + {{- if .Selector}} + labelSelectors: + {{ .Selector }} + {{- else}} + fieldSelectors: + metadata.name: {{ .PodName }} + {{- end}} + mode: {{ .Mode }} + {{- if .ModeValue }} + value: '{{ .ModeValue }}' + {{- end }} + action: delay + duration: {{ .Duration }} + delay: + latency: {{ .Latency }} + direction: from + target: + selector: + namespaces: + - {{ .Namespace }} + {{- if .Selector}} + labelSelectors: + {{ .Selector }} + {{- else}} + fieldSelectors: + metadata.name: {{ .PodName }} + {{- end}} + mode: {{ .Mode }} + {{- if .ModeValue }} + value: '{{ .ModeValue }}' + {{- end }} +` + return MarshalTemplate( + m, + uuid.NewString(), + tpl, + ) +} + +type NetworkChaosGroupPartitionExperiment struct { + ExperimentName string + ModeTo string + ModeToValue string + ModeFrom string + ModeFromValue string + Direction string + Namespace string + Duration string + SelectorFrom string + SelectorTo string +} + +func (m NetworkChaosGroupPartitionExperiment) String() (string, error) { + tpl := ` +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: {{ .ExperimentName }} + namespace: {{ .Namespace }} +spec: + selector: + namespaces: + - {{ .Namespace }} + labelSelectors: + {{ .SelectorFrom }} + action: partition + mode: {{ .ModeFrom }} + {{- if .ModeFromValue }} + value: '{{ .ModeFromValue }}' + {{- end }} + duration: {{ .Duration }} + direction: {{ .Direction }} + target: + mode: {{ .ModeTo }} + {{- if .ModeToValue }} + value: '{{ .ModeToValue }}' + {{- end }} + selector: + namespaces: + - {{ .Namespace }} + labelSelectors: + {{ .SelectorTo }} +` + return MarshalTemplate( + m, + uuid.NewString(), + tpl, + ) +} + +type NetworkChaosExternalPartitionExperiment struct { + ExperimentName string + Namespace string + Duration string + PodName string + ExternalURL string +} + +func (m NetworkChaosExternalPartitionExperiment) String() (string, error) { + tpl := ` +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: {{ .ExperimentName }} + namespace: {{ .Namespace }} +spec: + selector: + namespaces: + - {{ .Namespace }} + mode: all + action: partition + duration: {{ .Duration }} + direction: to + target: + selector: + namespaces: + - {{ .Namespace }} + mode: all + externalTargets: + - {{ .ExternalURL }} +` + return MarshalTemplate( + m, + uuid.NewString(), + tpl, + ) +} + +type PodFailureExperiment struct { + ExperimentName string + Mode string + ModeValue string + Namespace string + Duration string + PodName string + Selector string +} + +func (m PodFailureExperiment) String() (string, error) { + tpl := ` +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: {{ .ExperimentName }} + namespace: {{ .Namespace }} +spec: + action: pod-failure + mode: {{ .Mode }} + {{- if .ModeValue }} + value: '{{ .ModeValue }}' + {{- end }} + duration: {{ .Duration }} + selector: + {{- if .Selector}} + labelSelectors: + {{ .Selector }} + {{- else}} + fieldSelectors: + metadata.name: {{ .PodName }} + {{- end}} +` + return MarshalTemplate( + m, + uuid.NewString(), + tpl, + ) +} + +type PodStressCPUExperiment struct { + ExperimentName string + Mode string + ModeValue string + Namespace string + Workers int + Load int + Duration string + PodName string + Selector string +} + +func (m PodStressCPUExperiment) String() (string, error) { + tpl := ` +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: {{ .ExperimentName }} + namespace: {{ .Namespace }} +spec: + mode: {{ .Mode }} + {{- if .ModeValue }} + value: '{{ .ModeValue }}' + {{- end }} + duration: {{ .Duration }} + selector: + {{- if .Selector}} + labelSelectors: + {{ .Selector }} + {{- else}} + fieldSelectors: + metadata.name: {{ .PodName }} + {{- end}} + stressors: + cpu: + workers: {{ .Workers }} + load: {{ .Load }} +` + return MarshalTemplate( + m, + uuid.NewString(), + tpl, + ) +} + +type PodStressMemoryExperiment struct { + ExperimentName string + Mode string + ModeValue string + Namespace string + Workers int + Memory string + Duration string + PodName string + Selector string +} + +func (m PodStressMemoryExperiment) String() (string, error) { + tpl := ` +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: {{ .ExperimentName }} + namespace: {{ .Namespace }} +spec: + mode: {{ .Mode }} + {{- if .ModeValue }} + value: '{{ .ModeValue }}' + {{- end }} + duration: {{ .Duration }} + selector: + {{- if .Selector}} + labelSelectors: + {{ .Selector }} + {{- else}} + fieldSelectors: + metadata.name: {{ .PodName }} + {{- end}} + stressors: + memory: + workers: {{ .Workers }} + size: {{ .Memory }} +` + return MarshalTemplate( + m, + uuid.NewString(), + tpl, + ) +} + +type CRD struct { + Kind string `yaml:"kind"` + APIVersion string `yaml:"apiVersion"` + Metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + } `yaml:"metadata"` + Spec interface{} `yaml:"spec"` // Use interface{} if the spec can have various structures +} + +type NamedExperiment struct { + CRD + Name string + Path string + CRDBytes []byte +} + +func NewNamedExperiment(expPath string) (*NamedExperiment, error) { + data, err := os.ReadFile(expPath) + if err != nil { + return nil, err + } + + var exp CRD + err = yaml.Unmarshal(data, &exp) + if err != nil { + return nil, err + } + expName := exp.Metadata.Name + if expName == "" { + return nil, errors.Errorf("experiment metadata.name is empty") + } + + return &NamedExperiment{ + CRD: exp, + Name: expName, + Path: expPath, + CRDBytes: data, + }, nil +} + +func (m *Controller) readExistingExperimentTypes(dir string) ([]string, error) { + expTypes := make([]string, 0) + err := filepath.Walk( + dir, + func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() && info.Name() != dir { + expTypes = append(expTypes, info.Name()) + return nil + } + return err + }) + if err != nil { + return nil, err + } + sort.Slice(expTypes, func(i, j int) bool { + return expTypes[i] < expTypes[j] + }) + L.Info().Strs("Order", expTypes).Msg("Order of experiment dirs execution") + return expTypes, nil +} + +func (m *Controller) ReadExperimentsFromDir(expTypes []string, dir string) ([]*NamedExperiment, error) { + expData := make([]*NamedExperiment, 0) + for _, expType := range expTypes { + targetDir := fmt.Sprintf("%s/%s", dir, expType) + if _, err := os.Stat(targetDir); err != nil { + // it's okay, some experiments may be skipped due configuration + continue + } + err := filepath.Walk( + fmt.Sprintf("%s/%s", dir, expType), + func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + exp, err := NewNamedExperiment(path) + if err != nil { + return err + } + expData = append(expData, exp) + return err + }) + if err != nil { + return nil, err + } + } + return expData, nil +} + +// maybeFailAll is a special case where we've labelled component properly but +// there is only one component, no need to apply experiment to part of a group, so we set 100% +func maybeFailAll(e lo.Entry[string, int], origValue string) string { + if e.Value == 1 { + return "100" + } + return origValue +} + +func (m *Controller) generate( + namespace string, + oapiSpecs []*OAPISpecData, + allPodsInfo map[string][]*PodResponse, + podsInfo []*PodResponse, + groupLabels []lo.Entry[string, int], + netLabels [][]string, +) (*ChaosSpecs, error) { + allExperimentsByType := make(map[string]map[string]string) + for _, expType := range m.cfg.Havoc.ExperimentTypes { + experiments := make(map[string]string) + switch expType { + case ChaosTypeHTTP: + for _, entry := range groupLabels { + if _, ok := m.cfg.Havoc.OpenAPI.Mapping[m.groupValueFromLabelSelector(entry.Key)]; ok { + if err := m.generateOAPIExperiments(experiments, namespace, entry, oapiSpecs); err != nil { + return nil, err + } + } + } + case ChaosTypeBlockchainSetHead: + for _, p := range allPodsInfo { + for _, pi := range p { + for _, nodeCfg := range m.cfg.Havoc.BlockchainRewindHead.NodesConfig { + if strings.Contains(pi.Metadata.Name, nodeCfg.ExecutorPodPrefix) { + for _, b := range nodeCfg.Blocks { + name := fmt.Sprintf("%s-%s-%d", ChaosTypeBlockchainSetHead, pi.Metadata.Name, b) + experiment, err := BlockchainRewindHeadExperiment{ + ExperimentName: name, + Metadata: &Metadata{Name: name}, + Namespace: namespace, + NodeInternalHTTPURL: nodeCfg.NodeInternalHTTPURL, + PodName: pi.Metadata.Name, + ExecutorContainerName: nodeCfg.ExecutorContainerName, + Blocks: b, + }.String() + if err != nil { + return nil, err + } + shortName := fmt.Sprintf("%s-%d", pi.Metadata.Name, b) + experiments[shortName] = experiment + } + } + } + } + } + case ChaosTypePartitionExternal: + if m.cfg.Havoc.ExternalTargets == nil { + continue + } + for _, u := range m.cfg.Havoc.ExternalTargets.URLs { + nsAndURLHash := fmt.Sprintf("%s-%s", namespace, urlHash(u)) + experiment, err := NetworkChaosExternalPartitionExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypePartitionExternal, nsAndURLHash), + Duration: m.cfg.Havoc.ExternalTargets.Duration, + ExternalURL: fmt.Sprintf("'%s'", u), + }.String() + if err != nil { + return nil, err + } + experiments[nsAndURLHash] = experiment + } + case ChaosTypePartitionGroup: + for _, pair := range netLabels { + for _, groupModeValue := range m.cfg.Havoc.NetworkPartition.GroupPercentage { + sanitizedLabel := sanitizeLabel(fmt.Sprintf("%s-to-%s", pair[0], pair[1])) + sanitizedLabel = fmt.Sprintf("%s-%s-perc", sanitizedLabel, groupModeValue) + experiment, err := NetworkChaosGroupPartitionExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypePartitionGroup, sanitizedLabel), + Duration: m.cfg.Havoc.NetworkPartition.Duration, + ModeFrom: "fixed-percent", + ModeFromValue: groupModeValue, + ModeTo: "fixed-percent", + ModeToValue: groupModeValue, + Direction: "from", + SelectorFrom: pair[0], + SelectorTo: pair[1], + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + for _, groupModeValue := range m.cfg.Havoc.NetworkPartition.GroupFixed { + sanitizedLabel := sanitizeLabel(fmt.Sprintf("%s-to-%s", pair[0], pair[1])) + sanitizedLabel = fmt.Sprintf("%s-%s-fixed", sanitizedLabel, groupModeValue) + experiment, err := NetworkChaosGroupPartitionExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypePartitionGroup, sanitizedLabel), + Duration: m.cfg.Havoc.NetworkPartition.Duration, + ModeFrom: "fixed-percent", + ModeFromValue: groupModeValue, + ModeTo: "fixed-percent", + ModeToValue: groupModeValue, + Direction: "from", + SelectorFrom: pair[0], + SelectorTo: pair[1], + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + } + case ChaosTypeFailure: + for _, pi := range podsInfo { + experiment, err := PodFailureExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeFailure, pi.Metadata.Name), + Mode: "one", + Duration: m.cfg.Havoc.Failure.Duration, + PodName: pi.Metadata.Name, + }.String() + if err != nil { + return nil, err + } + experiments[pi.Metadata.Name] = experiment + } + case ChaosTypeLatency: + for _, pi := range podsInfo { + experiment, err := NetworkChaosExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeLatency, pi.Metadata.Name), + Mode: "one", + Duration: m.cfg.Havoc.Latency.Duration, + Latency: m.cfg.Havoc.Latency.Latency, + PodName: pi.Metadata.Name, + }.String() + if err != nil { + return nil, err + } + experiments[pi.Metadata.Name] = experiment + } + case ChaosTypeStressCPU: + for _, pi := range podsInfo { + experiment, err := PodStressCPUExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeStressCPU, pi.Metadata.Name), + Duration: m.cfg.Havoc.StressCPU.Duration, + Workers: m.cfg.Havoc.StressCPU.Workers, + Load: m.cfg.Havoc.StressCPU.Load, + Mode: "one", + PodName: pi.Metadata.Name, + }.String() + if err != nil { + return nil, err + } + experiments[pi.Metadata.Name] = experiment + } + case ChaosTypeStressMemory: + for _, pi := range podsInfo { + experiment, err := PodStressMemoryExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeStressMemory, pi.Metadata.Name), + Duration: m.cfg.Havoc.StressMemory.Duration, + Workers: m.cfg.Havoc.StressMemory.Workers, + Memory: m.cfg.Havoc.StressMemory.Memory, + Mode: "one", + PodName: pi.Metadata.Name, + }.String() + if err != nil { + return nil, err + } + experiments[pi.Metadata.Name] = experiment + } + case ChaosTypeStressGroupMemory: + for _, entry := range groupLabels { + for _, groupModeValue := range m.cfg.Havoc.StressMemory.GroupPercentage { + groupModeValue = maybeFailAll(entry, groupModeValue) + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedLabel = fmt.Sprintf("%s-%s-perc", sanitizedLabel, groupModeValue) + experiment, err := PodStressMemoryExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeStressGroupMemory, sanitizedLabel), + Duration: m.cfg.Havoc.StressMemory.Duration, + Workers: m.cfg.Havoc.StressMemory.Workers, + Memory: m.cfg.Havoc.StressMemory.Memory, + Mode: "fixed-percent", + ModeValue: groupModeValue, + Selector: entry.Key, + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + for _, groupModeValue := range m.cfg.Havoc.StressMemory.GroupFixed { + groupModeValue = maybeFailAll(entry, groupModeValue) + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedLabel = fmt.Sprintf("%s-%s-fixed", sanitizedLabel, groupModeValue) + experiment, err := PodStressMemoryExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeStressGroupMemory, sanitizedLabel), + Duration: m.cfg.Havoc.StressMemory.Duration, + Workers: m.cfg.Havoc.StressMemory.Workers, + Memory: m.cfg.Havoc.StressMemory.Memory, + Mode: "fixed", + ModeValue: groupModeValue, + Selector: entry.Key, + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + } + case ChaosTypeStressGroupCPU: + for _, entry := range groupLabels { + for _, groupModeValue := range m.cfg.Havoc.StressCPU.GroupPercentage { + groupModeValue = maybeFailAll(entry, groupModeValue) + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedLabel = fmt.Sprintf("%s-%s-perc", sanitizedLabel, groupModeValue) + experiment, err := PodStressCPUExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeStressGroupCPU, sanitizedLabel), + Duration: m.cfg.Havoc.StressCPU.Duration, + Workers: m.cfg.Havoc.StressCPU.Workers, + Load: m.cfg.Havoc.StressCPU.Load, + Mode: "fixed-percent", + ModeValue: groupModeValue, + Selector: entry.Key, + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + for _, groupModeValue := range m.cfg.Havoc.StressCPU.GroupFixed { + groupModeValue = maybeFailAll(entry, groupModeValue) + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedLabel = fmt.Sprintf("%s-%s-fixed", sanitizedLabel, groupModeValue) + experiment, err := PodStressCPUExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeStressGroupCPU, sanitizedLabel), + Duration: m.cfg.Havoc.StressCPU.Duration, + Workers: m.cfg.Havoc.StressCPU.Workers, + Load: m.cfg.Havoc.StressCPU.Load, + Mode: "fixed", + ModeValue: groupModeValue, + Selector: entry.Key, + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + } + case ChaosTypeGroupFailure: + for _, entry := range groupLabels { + for _, groupModeValue := range m.cfg.Havoc.Failure.GroupPercentage { + groupModeValue = maybeFailAll(entry, groupModeValue) + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedLabel = fmt.Sprintf("%s-%s-perc", sanitizedLabel, groupModeValue) + experiment, err := PodFailureExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeGroupFailure, sanitizedLabel), + Duration: m.cfg.Havoc.Failure.Duration, + Mode: "fixed-percent", + ModeValue: groupModeValue, + Selector: entry.Key, + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + for _, groupModeValue := range m.cfg.Havoc.Failure.GroupFixed { + groupModeValue = maybeFailAll(entry, groupModeValue) + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedLabel = fmt.Sprintf("%s-%s-fixed", sanitizedLabel, groupModeValue) + experiment, err := PodFailureExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeGroupFailure, sanitizedLabel), + Duration: m.cfg.Havoc.Failure.Duration, + Mode: "fixed", + ModeValue: groupModeValue, + Selector: entry.Key, + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + } + case ChaosTypeGroupLatency: + for _, entry := range groupLabels { + for _, groupModeValue := range m.cfg.Havoc.Latency.GroupPercentage { + groupModeValue = maybeFailAll(entry, groupModeValue) + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedLabel = fmt.Sprintf("%s-%s-perc", sanitizedLabel, groupModeValue) + experiment, err := NetworkChaosExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeGroupLatency, sanitizedLabel), + Mode: "fixed-percent", + ModeValue: groupModeValue, + Duration: m.cfg.Havoc.Latency.Duration, + Latency: m.cfg.Havoc.Latency.Latency, + Selector: entry.Key, + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + for _, groupModeValue := range m.cfg.Havoc.Latency.GroupFixed { + groupModeValue = maybeFailAll(entry, groupModeValue) + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedLabel = fmt.Sprintf("%s-%s-fixed", sanitizedLabel, groupModeValue) + experiment, err := NetworkChaosExperiment{ + Namespace: namespace, + ExperimentName: fmt.Sprintf("%s-%s", ChaosTypeGroupLatency, sanitizedLabel), + Mode: "fixed", + ModeValue: groupModeValue, + Duration: m.cfg.Havoc.Latency.Duration, + Latency: m.cfg.Havoc.Latency.Latency, + Selector: entry.Key, + }.String() + if err != nil { + return nil, err + } + experiments[sanitizedLabel] = experiment + } + } + } + allExperimentsByType[expType] = experiments + } + return &ChaosSpecs{ + ExperimentsByType: allExperimentsByType, + }, nil +} + +func urlHash(url string) string { + hasher := md5.New() + hasher.Write([]byte(url)) + hashBytes := hasher.Sum(nil) + return hex.EncodeToString(hashBytes) +} + +func sanitizeLabel(label string) string { + sanitizedLabel := strings.Replace(label, "'", "", -1) + sanitizedLabel = strings.Replace(sanitizedLabel, ": ", "-", -1) + sanitizedLabel = strings.Replace(sanitizedLabel, ".", "-", -1) + sanitizedLabel = strings.Replace(sanitizedLabel, "/", "-", -1) + return sanitizedLabel +} + +type EventJSONItemResponse struct { + APIVersion string `json:"apiVersion"` + Count int `json:"count"` + EventTime any `json:"eventTime"` + FirstTimestamp time.Time `json:"firstTimestamp"` + InvolvedObject struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Name string `json:"name"` + Namespace string `json:"namespace"` + ResourceVersion string `json:"resourceVersion"` + UID string `json:"uid"` + } `json:"involvedObject"` + Kind string `json:"kind"` + LastTimestamp time.Time `json:"lastTimestamp"` + Message string `json:"message"` + Metadata struct { + Annotations struct { + ChaosMeshOrgType string `json:"chaos-mesh.org/type"` + } `json:"annotations"` + CreationTimestamp time.Time `json:"creationTimestamp"` + Name string `json:"name"` + Namespace string `json:"namespace"` + ResourceVersion string `json:"resourceVersion"` + UID string `json:"uid"` + } `json:"metadata"` + Reason string `json:"reason"` + ReportingComponent string `json:"reportingComponent"` + ReportingInstance string `json:"reportingInstance"` + Source struct { + Component string `json:"component"` + } `json:"source"` + Type string `json:"type"` +} + +type EventsJSONResponse struct { + APIVersion string `json:"apiVersion"` + Items []*EventJSONItemResponse `json:"items"` + Kind string `json:"kind"` + Metadata struct { + ResourceVersion string `json:"resourceVersion"` + } `json:"metadata"` +} + +func eventsForLastMinutes(out string, timeOfApplication time.Time) error { + var d *EventsJSONResponse + if err := json.Unmarshal([]byte(out), &d); err != nil { + return err + } + L.Debug().Msg("Listing all experiment events") + for _, i := range d.Items { + if i.LastTimestamp.After(timeOfApplication) { + L.Info(). + Time("Time", i.LastTimestamp). + Str("Reason", i.Reason). + Str("Message", i.Message). + Send() + } + } + return nil +} + +func (m *Controller) ApplyExperiment(exp *NamedExperiment, wait bool) error { + timeOfApplication := time.Now() + var errDefer error + if exp.Kind == ChaosTypeBlockchainSetHead { + return m.ApplyCustomKindChaosFile(exp, ChaosTypeBlockchainSetHead, wait) + } + L.Info(). + Str("Dir", m.cfg.Havoc.Dir). + Str("Type", exp.Kind). + Str("Name", exp.Metadata.Name). + Msg("Applying experiment manifest") + fmt.Println(string(exp.CRDBytes)) + _, err := ExecCmd(fmt.Sprintf("kubectl apply -f %s", exp.Path)) + if err != nil { + return errors.Wrap(err, ErrExperimentApply) + } + if wait { + resourceType := ExperimentTypesToCRDNames[exp.Kind] + if resourceType == "" { + return errors.Errorf("%s resource not present in %+v list", exp.Kind, ExperimentTypesToCRDNames) + } + _, err = ExecCmd( + fmt.Sprintf("kubectl wait -n %s %s --field-selector=metadata.name=%s --for condition=AllRecovered=True --timeout %s", + exp.Metadata.Namespace, + resourceType, + exp.Metadata.Name, + DefaultCMDTimeout, + )) + if err != nil { + return errors.Wrap(err, ErrExperimentTimeout) + } + out, err := ExecCmd( + fmt.Sprintf("kubectl get -n %s events --field-selector involvedObject.name=%s -o json", + exp.Metadata.Namespace, + exp.Name, + )) + if err != nil { + return err + } + if err = eventsForLastMinutes(out, timeOfApplication); err != nil { + return err + } + _, err = ExecCmd(fmt.Sprintf("kubectl -n %s delete %s %s", exp.Metadata.Namespace, resourceType, exp.Name)) + if err != nil { + return err + } + L.Info().Msg("Chaos experiment successfully recovered") + } + return errDefer +} + +type CurrentBlockResponse struct { + Result string `json:"result"` +} + +func (m *Controller) ApplyCustomKindChaosFile(exp *NamedExperiment, chaosType string, wait bool) error { + switch chaosType { + case ChaosTypeBlockchainSetHead: + var rewind *BlockchainRewindHeadExperiment + data, err := os.ReadFile(exp.Path) + if err != nil { + return nil + } + if err := yaml.Unmarshal(data, &rewind); err != nil { + return err + } + L.Info(). + Str("Dir", m.cfg.Havoc.Dir). + Str("Type", chaosType). + Str("Name", exp.Name). + Msg("Applying custom experiment") + fmt.Println(string(exp.CRDBytes)) + lastBlkCommand := fmt.Sprintf(`kubectl -n %s -it debug %s --image=%s --target=%s -- curl -s -X POST -H Content-Type:application/json --data {"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":88} %s`, + rewind.Namespace, + rewind.PodName, + DebugContainerImage, + rewind.ExecutorContainerName, + rewind.NodeInternalHTTPURL, + ) + out, err := ExecCmd(lastBlkCommand) + if err != nil { + return err + } + msg, err := findJSONMsg(out) + if err != nil { + return err + } + var res *CurrentBlockResponse + if err := json.Unmarshal([]byte(msg), &res); err != nil { + return err + } + decimalLastBlock, err := strconv.ParseInt(res.Result[2:], 16, 64) + if err != nil { + return err + } + moveToBlock := decimalLastBlock - rewind.Blocks + moveToBlockHex := strconv.FormatInt(moveToBlock, 16) + setHeadCommand := fmt.Sprintf(`kubectl -n %s -it debug %s --image=%s --target=%s -- curl -s -X POST -H Content-Type:application/json --data {"jsonrpc":"2.0","method":"debug_setHead","params":["0x%s"],"id":5} %s`, + rewind.Namespace, + rewind.PodName, + DebugContainerImage, + rewind.ExecutorContainerName, + moveToBlockHex, + rewind.NodeInternalHTTPURL, + ) + _, err = ExecCmd(setHeadCommand) + if err != nil { + return err + } + default: + return errors.New(ErrInvalidCustomKind) + } + return nil +} + +func findJSONMsg(s string) (string, error) { + startIndex := strings.Index(s, "{") + endIndex := strings.LastIndex(s, "}") + if startIndex != -1 && endIndex != -1 { + substring := s[startIndex : endIndex+1] + L.Debug(). + Str("Message", substring). + Msg("JSON substring response") + return substring, nil + } else { + return "", errors.New("no JSON substring found in response") + } +} + +// GenerateSpecs generates specs from namespace, should be used programmatically in tests +func (m *Controller) GenerateSpecs(ns string) error { + podsInfo, err := m.GetPodsInfo(ns) + if err != nil { + return err + } + _, _, err = m.generateSpecs(ns, podsInfo) + return err +} + +func (m *Controller) generateSpecs(namespace string, podListResponse *PodsListResponse) (*ChaosSpecs, []*PodResponse, error) { + L.Trace(). + Interface("PodListResponse", podListResponse). + Msg("Found pods") + all, noGroup, componentLabels, networkLabels, err := m.processPodInfoLo(podListResponse) + if err != nil { + return nil, nil, err + } + L.Info().Msg("Processing OpenAPI specs") + specs, err := m.ParseOpenAPISpecs() + if err != nil { + return nil, nil, err + } + L.Info().Msg("Generating chaos experiments") + csp, err := m.generate(namespace, specs, all, noGroup, componentLabels, networkLabels) + if err != nil { + return nil, nil, err + } + return csp, noGroup, csp.Dump(m.cfg.Havoc.Dir) +} diff --git a/havoc/go.mod b/havoc/go.mod new file mode 100644 index 000000000..edbbfd04f --- /dev/null +++ b/havoc/go.mod @@ -0,0 +1,96 @@ +module github.com/smartcontractkit/chainlink-testing-framework/havoc + +go 1.21 + +require ( + github.com/c-bata/go-prompt v0.2.6 + github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a + github.com/getkin/kin-openapi v0.122.0 + github.com/go-resty/resty/v2 v2.11.0 + github.com/google/uuid v1.5.0 + github.com/pelletier/go-toml/v2 v2.1.1 + github.com/pkg/errors v0.9.1 + github.com/rs/zerolog v1.31.0 + github.com/samber/lo v1.39.0 + github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.1 + github.com/stretchr/testify v1.8.4 + github.com/urfave/cli/v2 v2.27.1 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.28.2 + k8s.io/client-go v0.28.2 +) + +require ( + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.10.1 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/grafana/grafana-foundation-sdk/go v0.0.0-20240326122733-6f96a993222b // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apiextensions-apiserver v0.28.1 // indirect + k8s.io/apimachinery v0.28.2 // indirect + k8s.io/component-base v0.28.2 // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/invopop/yaml v0.2.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mattn/go-tty v0.0.3 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/pkg/term v1.2.0-beta.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.15.0 // indirect + sigs.k8s.io/controller-runtime v0.16.2 +) + +exclude github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 diff --git a/havoc/go.sum b/havoc/go.sum new file mode 100644 index 000000000..a6e130297 --- /dev/null +++ b/havoc/go.sum @@ -0,0 +1,309 @@ +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= +github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= +github.com/c-bata/go-prompt v0.2.6 h1:POP+nrHE+DfLYx370bedwNhsqmpCUynWPxuHi0C5vZI= +github.com/c-bata/go-prompt v0.2.6/go.mod h1:/LMAke8wD2FsNu9EXNdHxNLbd9MedkPnCdfpU9wwHfY= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a h1:6Pg3a6j/41QDzH/oYcMLwwKsf3x/HXcu9W/dBaf2Hzs= +github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a/go.mod h1:x11iCbZV6hzzSQWMq610B6Wl5Lg1dhwqcVfeiWQQnQQ= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= +github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getkin/kin-openapi v0.122.0 h1:WB9Jbl0Hp/T79/JF9xlSW5Kl9uYdk/AWD0yAd9HOM10= +github.com/getkin/kin-openapi v0.122.0/go.mod h1:PCWw/lfBrJY4HcdqE3jj+QFkaFK8ABoqo7PvqVhXXqw= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/grafana-foundation-sdk/go v0.0.0-20240326122733-6f96a993222b h1:Msqs1nc2qWMxTriDCITKl58Td+7Md/RURmUmH7RXKns= +github.com/grafana/grafana-foundation-sdk/go v0.0.0-20240326122733-6f96a993222b/go.mod h1:WtWosval1KCZP9BGa42b8aVoJmVXSg0EvQXi9LDSVZQ= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= +github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= +github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/term v1.2.0-beta.2 h1:L3y/h2jkuBVFdWiJvNfYfKmzcCnILw7mJWm2JQuMppw= +github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.1 h1:1/r1wQZ4TOFpZ13w94r7amdF096Z96RuEnkOmrz1BGE= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.1/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= +github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= +k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= +k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw= +k8s.io/apiextensions-apiserver v0.28.1/go.mod h1:sVvrI+P4vxh2YBBcm8n2ThjNyzU4BQGilCQ/JAY5kGs= +k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= +k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= +k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= +k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= +k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E= +k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= +sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/havoc/havoc.go b/havoc/havoc.go new file mode 100644 index 000000000..e279e391a --- /dev/null +++ b/havoc/havoc.go @@ -0,0 +1,80 @@ +package havoc + +import ( + "fmt" + "os" + "strings" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +const ( + ChaosTypeBlockchainSetHead = "blockchain_rewind_head" + ChaosTypeFailure = "failure" + ChaosTypeGroupFailure = "group-failure" + ChaosTypeLatency = "latency" + ChaosTypeGroupLatency = "group-latency" + ChaosTypeStressMemory = "memory" + ChaosTypeStressGroupMemory = "group-memory" + ChaosTypeStressCPU = "cpu" + ChaosTypeStressGroupCPU = "group-cpu" + ChaosTypePartitionExternal = "external" + ChaosTypePartitionGroup = "group-partition" + ChaosTypeHTTP = "http" +) + +var ( + ExperimentTypesToCRDNames = map[string]string{ + "PodChaos": "podchaos.chaos-mesh.org", + "StressChaos": "stresschaos.chaos-mesh.org", + "NetworkChaos": "networkchaos.chaos-mesh.org", + "HTTPChaos": "httpchaos.chaos-mesh.org", + } +) + +var L zerolog.Logger + +func SetGlobalLogger(l zerolog.Logger) { + L = l.With().Str("Component", "havoc").Logger() +} + +func InitDefaultLogging() { + lvl, err := zerolog.ParseLevel(os.Getenv("HAVOC_LOG_LEVEL")) + if err != nil { + panic(err) + } + if lvl.String() == "" { + lvl = zerolog.InfoLevel + } + L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(lvl) +} + +type ChaosSpecs struct { + ExperimentsByType map[string]map[string]string +} + +func (m *ChaosSpecs) Dump(dir string) error { + if err := os.RemoveAll(dir); err != nil { + return err + } + if err := os.Mkdir(dir, os.ModePerm); err != nil { + return err + } + L.Info().Str("Dir", dir).Msg("Writing experiments to a dir") + for expType := range m.ExperimentsByType { + if len(m.ExperimentsByType[expType]) == 0 { + continue + } + if err := os.Mkdir(fmt.Sprintf("%s/%s", dir, expType), os.ModePerm); err != nil { + return err + } + for expName, expBody := range m.ExperimentsByType[expType] { + fname := strings.ToLower(fmt.Sprintf("%s/%s/%s-%s.yaml", dir, expType, expType, expName)) + if err := os.WriteFile(fname, []byte(expBody), os.ModePerm); err != nil { + return err + } + } + } + return nil +} diff --git a/havoc/havoc.toml b/havoc/havoc.toml new file mode 100644 index 000000000..7e9b86695 --- /dev/null +++ b/havoc/havoc.toml @@ -0,0 +1,141 @@ +[havoc] +# dir is a custom dir you can select, if null monkey will create a new dir +dir = "experiments-crib-core" +# if you have multiple products inside one namespace this can help to filter by label in k=v format +namespace_label_filter = "" +# pods with this prefix will be ignored when generating experiments +ignore_pods = ["-db-"] +# name of the key to select components in the namespace +component_label_key = "havoc-component-group" +# group labels containing these strings will be ignored when generating group experiments +ignore_group_labels = [ + "mainnet", + "release", + "intents.otterize.com", + "pod-template-hash", + "rollouts-pod-template-hash", + "chain.link/app", + "chain.link/cost-center", + "chain.link/env", + "chain.link/project", + "chain.link/team", + "app.kubernetes.io/part-of", + "app.kubernetes.io/managed-by", + "app.chain.link/product", + "app.kubernetes.io/version", + "app.chain.link/blockchain", + "app.kubernetes.io/instance", + "app.kubernetes.io/name", +] +# these are experiment types you'd like to generate +experiment_types = [ + "external", + "failure", + "latency", + "cpu", + "memory", + "group-failure", + "group-latency", + "group-cpu", + "group-memory", + "group-partition", + "blockchain_rewind_head", + "http" +] + +[havoc.failure] +# duration of a "failure" experiment +duration = "10s" +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_fixed = ["3", "2", "1"] + +[havoc.latency] +# duration of "latency" experiment +duration = "10s" +# constant latency to inject +latency = "300ms" +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_fixed = ["3", "2", "1"] + +[havoc.stress_memory] +# duration of "stress" experiment affecting pod memory +duration = "10s" +# amount of workers which occupies memory +workers = 1 +# total amount of memory occupied +memory = "512MB" +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_fixed = ["3", "2", "1"] + +[havoc.stress_cpu] +# duration of "stress" experiment affecting pod CPU +duration = "10s" +# amount of workers which occupies cpu +workers = 1 +# amount of CPU core utilization, 100 means 1 worker will consume 1 cpu, 2 workers + 100 load = 2 CPUs +load = 100 +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_fixed = ["3", "2", "1"] + +[havoc.network_partition] +# duration of "network partition" experiment affecting pod CPU +duration = "30s" +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_percentage = ["100"] +# a label to split pods for experiments +label = "havoc-network-group" + +[havoc.blockchain_rewind_head] +# duration of "blockchain" experiment +duration = "30s" + +[[havoc.blockchain_rewind_head.nodes]] +# label of executor pod +executor_pod_prefix = "geth-1337" +# executor container name +executor_container_name = "geth-network" +# blockchain node internal HTTP URL +node_internal_http_url = "geth-1337:8544" +# blocks to rewind from last +blocks = [30, 20, 10] + +[[havoc.blockchain_rewind_head.nodes]] +# label of executor pod +executor_pod_prefix = "geth-2337" +# executor container name +executor_container_name = "geth-network" +# blockchain node internal HTTP URL +node_internal_http_url = "geth-2337:8544" +# blocks to rewind from last +blocks = [30, 20, 10] + +[havoc.external_targets] +# duration of "external" experiment +duration = "10s" +# URL of external service that'd fail to resolve +urls = ["www.google.com"] + +# you can map OpenAPI 3.0.0 specifications to your component groups, let's say you have +# component_label_key = "havoc-component-group" and some pods having "havoc-component-group: node" +[havoc.openapi] +[havoc.openapi.mapping.node] +[[havoc.openapi.mapping.node.spec_to_port]] +# port on which your instances are exposing this API +port = 8080 +# path to OpenAPI 3.0.0 +path = "testdata/openapi_specs/petshop.yaml" + +[havoc.monkey] +# havoc monkey mode: +# seq - runs all experiments from all dirs sequentially one time +# rand - runs random experiments from all dirs +mode = "rand" +# duration of havoc monkey +duration = "3m" +# cooldown between experiments +cooldown = "5s" + +[havoc.grafana] +# UIDs of dashboard which should be annotated with chaos experiments metadata +# You can also try to use name as you see it in the top bar of your dashboard but that's not guaranteed to match +dashboard_uids = ["WaspDebug", "e98b5451-12dc-4a8b-9576-2c0b67ddbd0c"] diff --git a/havoc/havoc_test.go b/havoc/havoc_test.go new file mode 100644 index 000000000..2a55bbbf6 --- /dev/null +++ b/havoc/havoc_test.go @@ -0,0 +1,134 @@ +package havoc + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +var ( + // We are not testing with real k8s, namespace is just a placeholder that should match in snapshots/results + Namespace = "cl-cluster" + TestDataDir = "testdata" + SnapshotDir = filepath.Join(TestDataDir, "snapshot") + ResultsDir = filepath.Join(TestDataDir, "results") + DeploymentsDir = filepath.Join(TestDataDir, "deployments") + ConfigsDir = filepath.Join(TestDataDir, "configs") + OAPISpecs = filepath.Join(TestDataDir, "openapi_specs") +) + +var ( + AllExperimentTypes = []string{ + ChaosTypeFailure, + ChaosTypeLatency, + ChaosTypeGroupFailure, + ChaosTypeGroupLatency, + ChaosTypeStressMemory, + ChaosTypeStressGroupMemory, + ChaosTypeStressCPU, + ChaosTypeStressGroupCPU, + ChaosTypePartitionGroup, + ChaosTypeHTTP, + ChaosTypePartitionExternal, + ChaosTypeBlockchainSetHead, + } +) + +func init() { + InitDefaultLogging() +} + +func setup(t *testing.T, podsInfoPath string, configPath string, resultsDir string) (*Controller, *PodsListResponse) { + d, err := os.ReadFile(filepath.Join(DeploymentsDir, podsInfoPath)) + require.NoError(t, err) + var plr *PodsListResponse + err = json.Unmarshal(d, &plr) + require.NoError(t, err) + var cfg *Config + if configPath != "" { + cfg, err = ReadConfig(filepath.Join(ConfigsDir, configPath)) + require.NoError(t, err) + } else { + cfg = DefaultConfig() + cfg.Havoc.Dir = filepath.Join(ResultsDir, resultsDir) + } + m, err := NewController(cfg) + require.NoError(t, err) + return m, plr +} + +func TestSmokeParsingGenerating(t *testing.T) { + type test struct { + name string + podsDumpName string + configName string + snapshotDir string + resultsDir string + } + tests := []test{ + { + name: "can generate for 1 pod without groups", + podsDumpName: "deployment_single_pod.json", + configName: "", + snapshotDir: "single_pod", + resultsDir: "single_pod", + }, + { + name: "can generate for a component group", + podsDumpName: "deployment_single_group.json", + configName: "", + snapshotDir: "single_group", + resultsDir: "single_group", + }, + { + name: "standalone pods + component group + network group + blockchain experiments", + podsDumpName: "deployment_crib_block_rewind.json", + configName: "crib-all.toml", + snapshotDir: "all", + resultsDir: "all", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m, plr := setup(t, tc.podsDumpName, tc.configName, tc.resultsDir) + _, _, err := m.generateSpecs(Namespace, plr) + require.NoError(t, err) + snapshotData, err := m.ReadExperimentsFromDir(AllExperimentTypes, filepath.Join(SnapshotDir, tc.snapshotDir)) + require.NoError(t, err) + generatedData, err := m.ReadExperimentsFromDir(AllExperimentTypes, filepath.Join(ResultsDir, tc.resultsDir)) + require.NoError(t, err) + require.Equal(t, len(snapshotData), len(generatedData)) + for i := range snapshotData { + // Replace snapshot dir name to match it with expected results path + snapshotData[i].Path = strings.ReplaceAll(snapshotData[i].Path, SnapshotDir, ResultsDir) + require.Equal(t, snapshotData[i], generatedData[i]) + } + }) + } +} + +/* +These are just an easy way to enter debug with arbitrary config, or some tweaks, run it manually +*/ +func TestManualGenerate(t *testing.T) { + cfg, err := ReadConfig("havoc.toml") + require.NoError(t, err) + m, err := NewController(cfg) + require.NoError(t, err) + err = m.GenerateSpecs("cl-cluster") + require.NoError(t, err) +} + +func TestManualRun(t *testing.T) { + cfg, err := ReadConfig("havoc.toml") + require.NoError(t, err) + m, err := NewController(cfg) + require.NoError(t, err) + err = m.Run() + require.NoError(t, err) +} diff --git a/havoc/img.png b/havoc/img.png new file mode 100644 index 000000000..094d857ed Binary files /dev/null and b/havoc/img.png differ diff --git a/havoc/k8schaos/README.md b/havoc/k8schaos/README.md new file mode 100644 index 000000000..78d0acecd --- /dev/null +++ b/havoc/k8schaos/README.md @@ -0,0 +1,211 @@ +## k8schaos + +The `k8schaos` package is a Go library designed to facilitate chaos testing within Kubernetes environments using Chaos Mesh. It offers a structured way to define, execute, and manage chaos experiments as code, directly integrated into Go applications or testing suites. This package simplifies the creation and control of Chaos Mesh experiments, including network chaos, pod failures, and stress testing on Kubernetes clusters. + +### Features + +- **Chaos Object Management:** Easily create, update, pause, resume, and delete chaos experiments using Go structures and methods. +- **Lifecycle Hooks:** Utilize chaos listeners to hook into lifecycle events of chaos experiments, such as creation, start, pause, resume, and finish. +- **Support for Various Chaos Experiments:** Create and manage different types of chaos experiments like NetworkChaos, IOChaos, StressChaos, PodChaos, and HTTPChaos. +- **Chaos Experiment Status Monitoring:** Monitor and react to the status of chaos experiments programmatically. + +### Installation + +To use k8schaos in your project, ensure you have a Go environment setup. Then, install the package using go get: + +``` +go get -u github.com/smartcontractkit/havoc/k8schaos +``` + +Ensure your Kubernetes cluster is accessible and that you have Chaos Mesh installed and configured. + +### Monitoring and Observability in Chaos Experiments + +`k8schaos` enhances chaos experiment observability through structured logging and Grafana annotations, facilitated by implementing the ChaosListener interface. This approach allows for detailed monitoring, debugging, and visual representation of chaos experiments' impact. + +#### Structured Logging with ChaosLogger + +`ChaosLogger` leverages the zerolog library to provide structured, queryable logging of chaos events. It automatically logs key lifecycle events such as creation, start, pause, and termination of chaos experiments, including detailed contextual information. + +Instantiate `ChaosLogger` and register it as a listener to your chaos experiments: + +``` +logger := k8schaos.NewChaosLogger() +chaos.AddListener(logger) +``` + +### Default package logger + +k8schaos/logger.go contains default `Logger` instance for the package. + +#### Visual Monitoring with Grafana Annotations + +`SingleLineGrafanaAnnotator` is a `ChaosListener` that annotates Grafana dashboards with chaos experiment events. This visual representation helps correlate chaos events with their effects on system metrics and logs. + +Initialize `SingleLineGrafanaAnnotator` with your Grafana instance details and register it alongside `ChaosLogger`: + +``` +annotator := k8schaos.NewSingleLineGrafanaAnnotator( + "http://grafana-instance.com", + "grafana-access-token", + "dashboard-uid", +) +chaos.AddListener(annotator) +``` + +### Creating a Chaos Experiment + +To create a chaos experiment, define the chaos object options, initialize a chaos experiment with NewChaos, and then call Create to start the experiment. + +Here is an example of creating and starting a PodChaos experiment: + +``` +package main + +import ( + "context" + "github.com/smartcontractkit/havoc/k8schaos" + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" +) + +func main() { + // Initialize dependencies + client, err := k8schaos.NewChaosMeshClient() + if err != nil { + panic(err) + } + logger := k8schaos.NewChaosLogger() + annotator := k8schaos.NewSingleLineGrafanaAnnotator( + "http://grafana-instance.com", + "grafana-access-token", + "dashboard-uid", + ) + + // Define chaos experiment + podChaos := &v1alpha1.PodChaos{ /* PodChaos spec */ } + chaos, err := k8schaos.NewChaos(k8schaos.ChaosOpts{ + Object: podChaos, + Description: "Pod failure example", + DelayCreate: 5 * time.Second, + Client: client, + }) + if err != nil { + panic(err) + } + + // Register listeners + chaos.AddListener(logger) + chaos.AddListener(annotator) + + // Start chaos experiment + chaos.Create(context.Background()) + + // Manage chaos lifecycle... +} +``` + +### Test Example + +``` +func TestChaosDON(t *testing.T) { + testDuration := time.Minute * 60 + + // Load test config + cfg := &config.MercuryQAEnvChaos{} + + // Define chaos experiments and their schedule + + k8sClient, err := havoc.NewChaosMeshClient() + require.NoError(t, err) + + // Test 3.2: Disable 2 nodes simultaneously + + podFailureChaos4, err := k8s_chaos.MercuryPodChaosSchedule(k8s_chaos.MercuryScheduledPodChaosOpts{ + Name: "schedule-don-ocr-node-failure-4", + Description: "Disable 2 nodes (clc-ocr-mercury-arb-testnet-qa-nodes-3 and clc-ocr-mercury-arb-testnet-qa-nodes-4)", + DelayCreate: time.Minute * 0, + Duration: time.Minute * 20, + Namespace: cfg.ChaosNodeNamespace, + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.ChaosNodeNamespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Key: "app.kubernetes.io/instance", + Operator: "In", + Values: []string{ + "clc-ocr-mercury-arb-testnet-qa-nodes-3", + "clc-ocr-mercury-arb-testnet-qa-nodes-4", + }, + }, + }, + }, + }, + }, + Client: k8sClient, + }) + require.NoError(t, err) + + // Test 3.3: Disable 3 nodes simultaneously + + podFailureChaos5, err := k8s_chaos.MercuryPodChaosSchedule(k8s_chaos.MercuryScheduledPodChaosOpts{ + Name: "schedule-don-ocr-node-failure-5", + Description: "Disable 3 nodes (clc-ocr-mercury-arb-testnet-qa-nodes-3, clc-ocr-mercury-arb-testnet-qa-nodes-4 and clc-ocr-mercury-arb-testnet-qa-nodes-5)", + DelayCreate: time.Minute * 40, + Duration: time.Minute * 20, + Namespace: cfg.ChaosNodeNamespace, + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.ChaosNodeNamespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Key: "app.kubernetes.io/instance", + Operator: "In", + Values: []string{ + "clc-ocr-mercury-arb-testnet-qa-nodes-3", + "clc-ocr-mercury-arb-testnet-qa-nodes-4", + "clc-ocr-mercury-arb-testnet-qa-nodes-5", + }, + }, + }, + }, + }, + }, + Client: k8sClient, + }) + require.NoError(t, err) + + chaosList := []havoc.ChaosEntity{ + podFailureChaos4, + podFailureChaos5, + } + + for _, chaos := range chaosList { + chaos.AddListener(havoc.NewChaosLogger()) + chaos.AddListener(havoc.NewSingleLineGrafanaAnnotator(cfg.GrafanaURL, cfg.GrafanaToken, cfg.GrafanaDashboardUID)) + + // Fail the test if the chaos object already exists + exists, err := havoc.ChaosObjectExists(chaos.GetObject(), k8sClient) + require.NoError(t, err) + require.False(t, exists, "chaos object already exists: %s. Delete it before starting the test", chaos.GetChaosName()) + + chaos.Create(context.Background()) + } + + t.Cleanup(func() { + for _, chaos := range chaosList { + // Delete chaos object if it still exists + chaos.Delete(context.Background()) + } + }) + + // Simulate user activity/load for the duration of the chaos experiments + runUserLoad(t, cfg, testDuration) +} +``` diff --git a/havoc/k8schaos/chaos.go b/havoc/k8schaos/chaos.go new file mode 100644 index 000000000..7f3f15a03 --- /dev/null +++ b/havoc/k8schaos/chaos.go @@ -0,0 +1,615 @@ +package k8schaos + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/pkg/errors" + "github.com/rs/zerolog" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Chaos struct { + Object client.Object + Description string + DelayCreate time.Duration // Delay before creating the chaos object + Status ChaosStatus + Client client.Client + listeners []ChaosListener + cancelMonitor context.CancelFunc + startTime time.Time + endTime time.Time + logger *zerolog.Logger +} + +// ChaosStatus represents the status of a chaos experiment. +type ChaosStatus string + +// These constants define possible states of a chaos experiment. +const ( + StatusCreated ChaosStatus = "created" + StatusCreationFailed ChaosStatus = "creation_failed" + StatusRunning ChaosStatus = "running" + StatusPaused ChaosStatus = "paused" + StatusFinished ChaosStatus = "finished" + StatusDeleted ChaosStatus = "deleted" + StatusUnknown ChaosStatus = "unknown" // For any state that doesn't match the above +) + +type ChaosOpts struct { + Object client.Object + Description string + DelayCreate time.Duration + Client client.Client + Listeners []ChaosListener + Logger *zerolog.Logger +} + +func NewChaos(opts ChaosOpts) (*Chaos, error) { + if opts.Client == nil { + return nil, errors.New("client is required") + } + if opts.Object == nil { + return nil, errors.New("chaos object is required") + } + if opts.Logger == nil { + return nil, errors.New("logger is required") + } + + return &Chaos{ + Object: opts.Object, + Description: opts.Description, + DelayCreate: opts.DelayCreate, + Client: opts.Client, + listeners: opts.Listeners, + logger: opts.Logger, + }, nil +} + +// Create initiates a delayed creation of a chaos object, respecting context cancellation and deletion requests. +// It uses a timer based on `DelayCreate` and calls `create` method upon expiration unless preempted by deletion. +func (c *Chaos) Create(ctx context.Context) { + done := make(chan struct{}) + + // Create the timer with the delay to create the chaos object + timer := time.NewTimer(c.DelayCreate) + + go func() { + select { + case <-ctx.Done(): + // If the context is canceled, stop the timer and exit + if !timer.Stop() { + <-timer.C // If the timer already expired, drain the channel + } + close(done) // Signal that the operation was canceled + case <-timer.C: + // Timer expired, check if deletion was not requested + if c.Status != StatusDeleted { + c.createNow(ctx) + } + close(done) // Signal that the creation process is either done or skipped + } + }() +} + +func (c *Chaos) Update(ctx context.Context) error { + // Modify the resource + // For example, adding or updating an annotation + annotations := c.Object.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations["example.com/trigger-injection"] = "true" + c.Object.SetAnnotations(annotations) + + if err := c.Client.Update(ctx, c.Object); err != nil { + return errors.Wrap(err, "failed to update chaos object") + } + + return nil +} + +// createNow is a private method that encapsulates the chaos object creation logic. +func (c *Chaos) createNow(ctx context.Context) { + if err := c.Client.Create(ctx, c.Object); err != nil { + c.notifyListeners(string(StatusCreationFailed), err) + return + } + c.notifyListeners(string(StatusCreated), nil) + + // Create a cancellable context for monitorStatus + monitorCtx, cancel := context.WithCancel(ctx) + c.cancelMonitor = cancel + go c.monitorStatus(monitorCtx) +} + +func (c *Chaos) Pause(ctx context.Context) error { + err := c.updateChaosObject(ctx) + if err != nil { + return errors.Wrap(err, "could not update the chaos object") + } + + annotations := c.Object.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[v1alpha1.PauseAnnotationKey] = strconv.FormatBool(true) + c.Object.SetAnnotations(annotations) + + err = c.Client.Update(context.Background(), c.Object) + if err != nil { + return errors.Wrap(err, "could not update the annotation to set the chaos experiment into pause state") + } + + c.notifyListeners("paused", nil) + return nil +} + +func (c *Chaos) Resume(ctx context.Context) error { + // Implement resume logic here + c.notifyListeners("resumed", nil) + return nil +} + +func (c *Chaos) Delete(ctx context.Context) error { + // Cancel the monitoring goroutine + if c.cancelMonitor != nil { + c.cancelMonitor() + } + + // If the chaos was running or paused, update the status and notify listeners + if c.Status == StatusPaused || c.Status == StatusRunning { + err := c.updateChaosObject(ctx) + if err != nil { + return errors.Wrap(err, "could not update the chaos object") + } + c.Status = StatusFinished + c.endTime = time.Now() + c.notifyListeners("finished", nil) + } + + if err := c.Client.Delete(ctx, c.Object); err != nil { + return errors.Wrap(err, "failed to delete chaos object") + } + + c.Status = StatusDeleted + + c.logger.Info().Str("name", c.GetChaosName()).Msg("Chaos deleted") + + return nil +} + +func (c *Chaos) GetObject() client.Object { + return c.Object +} + +func (c *Chaos) GetChaosName() string { + return c.Object.GetName() +} + +func (c *Chaos) GetChaosDescription() string { + return c.Description +} + +func (c *Chaos) GetChaosTypeStr() string { + switch c.Object.(type) { + case *v1alpha1.NetworkChaos: + return "NetworkChaos" + case *v1alpha1.IOChaos: + return "IOChaos" + case *v1alpha1.StressChaos: + return "StressChaos" + case *v1alpha1.PodChaos: + return "PodChaos" + case *v1alpha1.HTTPChaos: + return "HTTPChaos" + default: + return "Unknown" + } +} + +func (c *Chaos) GetChaosSpec() interface{} { + switch spec := c.Object.(type) { + case *v1alpha1.NetworkChaos: + return spec.Spec + case *v1alpha1.IOChaos: + return spec.Spec + case *v1alpha1.StressChaos: + return spec.Spec + case *v1alpha1.PodChaos: + return spec.Spec + case *v1alpha1.HTTPChaos: + return spec.Spec + default: + return nil + } +} + +func (c *Chaos) GetChaosDuration() (time.Duration, error) { + var durationStr *string + switch spec := c.Object.(type) { + case *v1alpha1.NetworkChaos: + durationStr = spec.Spec.Duration + case *v1alpha1.IOChaos: + durationStr = spec.Spec.Duration + case *v1alpha1.StressChaos: + durationStr = spec.Spec.Duration + case *v1alpha1.PodChaos: + durationStr = spec.Spec.Duration + case *v1alpha1.HTTPChaos: + durationStr = spec.Spec.Duration + } + + if durationStr == nil { + return time.Duration(0), fmt.Errorf("could not get duration for chaos object: %v", c.Object) + } + duration, err := time.ParseDuration(*durationStr) + if err != nil { + return time.Duration(0), fmt.Errorf("could not parse duration: %w", err) + } + return duration, nil +} + +func (c *Chaos) GetChaosEvents() (*corev1.EventList, error) { + listOpts := []client.ListOption{ + client.InNamespace(c.Object.GetNamespace()), + client.MatchingFields{"involvedObject.name": c.Object.GetName(), "involvedObject.kind": c.GetChaosKind()}, + } + events := &corev1.EventList{} + if err := c.Client.List(context.Background(), events, listOpts...); err != nil { + return nil, fmt.Errorf("could not list chaos events: %w", err) + } + + return events, nil +} + +func (c *Chaos) GetChaosKind() string { + switch c.Object.(type) { + case *v1alpha1.NetworkChaos: + return "NetworkChaos" + case *v1alpha1.IOChaos: + return "IOChaos" + case *v1alpha1.StressChaos: + return "StressChaos" + case *v1alpha1.PodChaos: + return "PodChaos" + case *v1alpha1.HTTPChaos: + return "HTTPChaos" + default: + panic(fmt.Sprintf("could not get chaos kind for object: %v", c.Object)) + } +} + +func (c *Chaos) GetChaosStatus() (*v1alpha1.ChaosStatus, error) { + switch obj := c.Object.(type) { + case *v1alpha1.NetworkChaos: + return obj.GetStatus(), nil + case *v1alpha1.IOChaos: + return obj.GetStatus(), nil + case *v1alpha1.StressChaos: + return obj.GetStatus(), nil + case *v1alpha1.PodChaos: + return obj.GetStatus(), nil + case *v1alpha1.HTTPChaos: + return obj.GetStatus(), nil + default: + return nil, fmt.Errorf("could not get chaos status for %s", c.GetChaosKind()) + } +} + +func (c *Chaos) GetExperimentStatus() (v1alpha1.ExperimentStatus, error) { + switch obj := c.Object.(type) { + case *v1alpha1.NetworkChaos: + return obj.Status.Experiment, nil + case *v1alpha1.IOChaos: + return obj.Status.Experiment, nil + case *v1alpha1.StressChaos: + return obj.Status.Experiment, nil + case *v1alpha1.PodChaos: + return obj.Status.Experiment, nil + case *v1alpha1.HTTPChaos: + return obj.Status.Experiment, nil + default: + return v1alpha1.ExperimentStatus{}, fmt.Errorf("could not experiment status for object: %v", c.Object) + } +} + +func ChaosObjectExists(object client.Object, c client.Client) (bool, error) { + switch obj := object.(type) { + case *v1alpha1.NetworkChaos, *v1alpha1.IOChaos, *v1alpha1.StressChaos, *v1alpha1.PodChaos, *v1alpha1.HTTPChaos, *v1alpha1.Schedule: + err := c.Get(context.Background(), client.ObjectKeyFromObject(obj), obj) + if err != nil { + if client.IgnoreNotFound(err) == nil { + // If the error is NotFound, the object does not exist. + return false, nil + } + // For any other errors, return the error. + return false, err + } + // If there's no error, the object exists. + return true, nil + default: + return false, fmt.Errorf("unsupported chaos object type: %T", obj) + } +} + +func (c *Chaos) updateChaosObject(ctx context.Context) error { + switch obj := c.Object.(type) { + case *v1alpha1.NetworkChaos: + var objOut = &v1alpha1.NetworkChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get network chaos object") + } + c.Object = objOut + case *v1alpha1.IOChaos: + var objOut = &v1alpha1.IOChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get IO chaos object") + } + c.Object = objOut + case *v1alpha1.StressChaos: + var objOut = &v1alpha1.StressChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get stress chaos object") + } + c.Object = objOut + case *v1alpha1.PodChaos: + var objOut = &v1alpha1.PodChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get pod chaos object") + } + c.Object = objOut + case *v1alpha1.HTTPChaos: + var objOut = &v1alpha1.HTTPChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get HTTP chaos object") + } + c.Object = objOut + case *v1alpha1.Schedule: + var objOut = &v1alpha1.Schedule{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get schedule object") + } + c.Object = objOut + default: + return fmt.Errorf("unsupported chaos object type: %T", obj) + } + + return nil +} + +func isConditionTrue(status *v1alpha1.ChaosStatus, expectedCondition v1alpha1.ChaosCondition) bool { + if status == nil { + return false + } + + for _, condition := range status.Conditions { + if condition.Type == expectedCondition.Type { + return condition.Status == expectedCondition.Status + } + } + return false +} + +func (c *Chaos) AddListener(listener ChaosListener) { + c.listeners = append(c.listeners, listener) +} + +// GetStartTime returns the time when the chaos experiment started +func (c *Chaos) GetStartTime() time.Time { + return c.startTime +} + +// GetEndTime returns the time when the chaos experiment ended +func (c *Chaos) GetEndTime() time.Time { + return c.endTime +} + +// GetExpectedEndTime returns the time when the chaos experiment is expected to end +func (c *Chaos) GetExpectedEndTime() (time.Time, error) { + duration, err := c.GetChaosDuration() + if err != nil { + return time.Time{}, err + } + return c.startTime.Add(duration), nil +} + +type ChaosEventDetails struct { + Event string + Chaos *Chaos + Error error +} + +func (c *Chaos) notifyListeners(event string, err error) { + for _, listener := range c.listeners { + switch event { + case "created": + listener.OnChaosCreated(*c) + case string(StatusCreationFailed): + listener.OnChaosCreationFailed(*c, err) + case "started": + listener.OnChaosStarted(*c) + case "paused": + listener.OnChaosPaused(*c) + case "resumed": + listener.OnChaosStarted(*c) // Assuming "resumed" triggers "started" + case "finished": + listener.OnChaosEnded(*c) + case "unknown": + listener.OnChaosStatusUnknown(*c) + } + } +} + +func (c *Chaos) monitorStatus(ctx context.Context) { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + err := c.updateChaosObject(context.Background()) + if err != nil { + c.logger.Error().Err(err).Msg("failed to update chaos object") + continue + } + chaosStatus, err := c.GetChaosStatus() + if err != nil { + c.logger.Error().Err(err).Msg("failed to get chaos status") + continue + } + + var currentStatus ChaosStatus + + allRecovered := v1alpha1.ChaosCondition{ + Type: v1alpha1.ConditionAllRecovered, + Status: corev1.ConditionTrue, + } + allInjected := v1alpha1.ChaosCondition{ + Type: v1alpha1.ConditionAllInjected, + Status: corev1.ConditionTrue, + } + selected := v1alpha1.ChaosCondition{ + Type: v1alpha1.ConditionSelected, + Status: corev1.ConditionTrue, + } + paused := v1alpha1.ChaosCondition{ + Type: v1alpha1.ConditionPaused, + Status: corev1.ConditionTrue, + } + + if isConditionTrue(chaosStatus, selected) && isConditionTrue(chaosStatus, allInjected) { + currentStatus = StatusRunning + } else if isConditionTrue(chaosStatus, allRecovered) { + currentStatus = StatusFinished + } else if !isConditionTrue(chaosStatus, paused) && !isConditionTrue(chaosStatus, selected) { + currentStatus = StatusUnknown + } + + // If the status is unknown, always notify listeners + if currentStatus == StatusUnknown { + c.notifyListeners(string(StatusUnknown), nil) + continue + } + + // If the status has changed, update internal status and notify listeners + if c.Status != currentStatus { + c.Status = currentStatus + + switch c.Status { + case StatusCreated: + c.notifyListeners("created", nil) + case StatusRunning: + c.startTime = time.Now() + c.notifyListeners("started", nil) + case StatusPaused: + c.notifyListeners("paused", nil) + case StatusFinished: + c.endTime = time.Now() + c.notifyListeners("finished", nil) + // Delete the chaos object when it finishes + err := c.Delete(context.Background()) + if err != nil { + c.logger.Error().Err(err).Msg("failed to delete chaos object") + } + } + } + } + } +} + +type NetworkChaosOpts struct { + Name string + Description string + DelayCreate time.Duration + Delay *v1alpha1.DelaySpec + Loss *v1alpha1.LossSpec + NodeCount int + Duration time.Duration + Selector v1alpha1.PodSelectorSpec + K8sClient client.Client +} + +func (o *NetworkChaosOpts) Validate() error { + if o.Delay != nil { + latency, err := time.ParseDuration(o.Delay.Latency) + if err != nil { + return fmt.Errorf("invalid latency: %v", err) + } + if latency > 500*time.Millisecond { + return fmt.Errorf("duration should be less than 500ms") + } + } + if o.Loss != nil { + lossInt, err := strconv.Atoi(o.Loss.Loss) // Convert the string to an integer + if err != nil { + return fmt.Errorf("invalid loss value: %s", err) + } + if lossInt > 100 { + return fmt.Errorf("loss should be less than 100") + } + } + if o.Loss == nil && o.Delay == nil { + return fmt.Errorf("either delay or loss should be specified") + } + return nil + +} + +type PodChaosOpts struct { + Name string + Description string + DelayCreate time.Duration + NodeCount int + Duration time.Duration + Spec v1alpha1.PodChaosSpec + K8sClient client.Client +} + +type StressChaosOpts struct { + Name string + Description string + DelayCreate time.Duration + NodeCount int + Stressors *v1alpha1.Stressors + Duration time.Duration + Selector v1alpha1.PodSelectorSpec + K8sClient client.Client +} + +// NewChaosMeshClient initializes and returns a new Kubernetes client configured for Chaos Mesh +func NewChaosMeshClient() (client.Client, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) + config, err := kubeConfig.ClientConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to load kubeconfig") + } + + // Ensure the Chaos Mesh types are added to the scheme + if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { + return nil, errors.Wrap(err, "could not add the Chaos Mesh scheme") + } + + // Create a new client for the Chaos Mesh API + chaosClient, err := client.New(config, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return nil, errors.Wrap(err, "failed to create a client for Chaos Mesh") + } + + return chaosClient, nil +} diff --git a/havoc/k8schaos/chaos_entity.go b/havoc/k8schaos/chaos_entity.go new file mode 100644 index 000000000..a2f46110c --- /dev/null +++ b/havoc/k8schaos/chaos_entity.go @@ -0,0 +1,27 @@ +package k8schaos + +import ( + "context" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ChaosEntity is an interface that defines common behaviors for chaos management entities. +type ChaosEntity interface { + // Create initializes and submits the chaos object to Kubernetes. + Create(ctx context.Context) + // Delete removes the chaos object from Kubernetes. + Delete(ctx context.Context) error + // Registers a listener to receive updates about the chaos object's lifecycle. + AddListener(listener ChaosListener) + + GetObject() client.Object + GetChaosName() string + GetChaosDescription() string + GetChaosDuration() (time.Duration, error) + GetChaosSpec() interface{} + GetStartTime() time.Time + GetEndTime() time.Time + GetExpectedEndTime() (time.Time, error) +} diff --git a/havoc/k8schaos/chaos_helper.go b/havoc/k8schaos/chaos_helper.go new file mode 100644 index 000000000..7fbff6caf --- /dev/null +++ b/havoc/k8schaos/chaos_helper.go @@ -0,0 +1,44 @@ +package k8schaos + +import ( + "errors" + "time" +) + +// WaitForAllChaosRunning waits for all chaos experiments to be running +func WaitForAllChaosRunning(chaosObjects []*Chaos, timeoutDuration time.Duration) error { + timeout := time.NewTimer(timeoutDuration) + defer timeout.Stop() + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + runningStatus := make(map[*Chaos]bool) + for _, chaos := range chaosObjects { + runningStatus[chaos] = false + } + + for { + allRunning := true + + select { + case <-timeout.C: + return errors.New("timeout reached before all chaos experiments became running") + case <-ticker.C: + for chaos, isRunning := range runningStatus { + if !isRunning { // Only check if not already marked as running + if chaos.Status == StatusRunning { + runningStatus[chaos] = true + } else { + allRunning = false + } + } + } + + if allRunning { + return nil // All chaos objects are running, can exit + } + // Otherwise, continue the loop + } + } +} diff --git a/havoc/k8schaos/chaos_listener.go b/havoc/k8schaos/chaos_listener.go new file mode 100644 index 000000000..4f8ac44c1 --- /dev/null +++ b/havoc/k8schaos/chaos_listener.go @@ -0,0 +1,12 @@ +package k8schaos + +type ChaosListener interface { + OnChaosCreated(chaos Chaos) + OnChaosCreationFailed(chaos Chaos, reason error) + OnChaosStarted(chaos Chaos) + OnChaosPaused(chaos Chaos) + OnChaosEnded(chaos Chaos) // When the chaos is finished or deleted + OnChaosStatusUnknown(chaos Chaos) // When the chaos status is unknown + OnScheduleCreated(chaos Schedule) + OnScheduleDeleted(chaos Schedule) // When the chaos is finished or deleted +} diff --git a/havoc/k8schaos/console_logger.go b/havoc/k8schaos/console_logger.go new file mode 100644 index 000000000..9bee48d6b --- /dev/null +++ b/havoc/k8schaos/console_logger.go @@ -0,0 +1,140 @@ +package k8schaos + +import ( + "time" + + "github.com/rs/zerolog" +) + +type ChaosLogger struct { + logger zerolog.Logger +} + +func NewChaosLogger(logger zerolog.Logger) *ChaosLogger { + return &ChaosLogger{logger: logger} +} + +func (l ChaosLogger) OnChaosCreated(chaos Chaos) { + l.commonChaosLog("info", chaos).Msg("Chaos created") +} + +func (l ChaosLogger) OnChaosCreationFailed(chaos Chaos, reason error) { + l.commonChaosLog("error", chaos). + Err(reason). + Msg("Failed to create chaos object") +} + +func (l ChaosLogger) OnChaosStarted(chaos Chaos) { + experiment, _ := chaos.GetExperimentStatus() + + l.commonChaosLog("info", chaos). + Interface("spec", chaos.GetChaosSpec()). + Interface("records", experiment.Records). + Msg("Chaos started") +} + +func (l ChaosLogger) OnChaosPaused(chaos Chaos) { + l.commonChaosLog("info", chaos). + Msg("Chaos paused") +} + +func (l ChaosLogger) OnChaosEnded(chaos Chaos) { + l.commonChaosLog("info", chaos). + Msg("Chaos ended") +} + +func (l ChaosLogger) OnChaosDeleted(chaos Chaos) { + l.commonChaosLog("info", chaos). + Msg("Chaos deleted") +} + +type SimplifiedEvent struct { + LastTimestamp string + Type string + Message string +} + +func (l ChaosLogger) OnChaosStatusUnknown(chaos Chaos) { + status, _ := chaos.GetExperimentStatus() + events, _ := chaos.GetChaosEvents() + + // Create a slice to hold the simplified events + simplifiedEvents := make([]SimplifiedEvent, 0, len(events.Items)) + + // Iterate over the events and extract the required information + for _, event := range events.Items { + simplifiedEvents = append(simplifiedEvents, SimplifiedEvent{ + LastTimestamp: event.LastTimestamp.Time.Format(time.RFC3339), + Type: event.Type, + Message: event.Message, + }) + } + + l.commonChaosLog("error", chaos). + Interface("status", status). + Interface("events", simplifiedEvents). + Msg("Chaos status unknown") +} + +func (l ChaosLogger) OnScheduleCreated(schedule Schedule) { + duration, _ := schedule.GetChaosDuration() + + l.logger.Info(). + Str("logger", "chaos"). + Str("name", schedule.GetObject().GetName()). + Str("namespace", schedule.GetObject().GetNamespace()). + Str("description", schedule.GetChaosDescription()). + Str("duration", duration.String()). + Time("startTime", schedule.GetStartTime()). + Time("endTime", schedule.GetEndTime()). + Interface("spec", schedule.GetChaosSpec()). + Msg("Chaos schedule created") +} + +func (l ChaosLogger) OnScheduleDeleted(schedule Schedule) { + duration, _ := schedule.GetChaosDuration() + + l.logger.Info(). + Str("logger", "chaos"). + Str("name", schedule.GetObject().GetName()). + Str("namespace", schedule.GetObject().GetNamespace()). + Str("description", schedule.GetChaosDescription()). + Str("duration", duration.String()). + Time("startTime", schedule.GetStartTime()). + Time("endTime", schedule.GetEndTime()). + Interface("spec", schedule.GetChaosSpec()). + Msg("Chaos schedule deleted") +} + +func (l ChaosLogger) commonChaosLog(logLevel string, chaos Chaos) *zerolog.Event { + // Create a base event based on the dynamic log level + var event *zerolog.Event + switch logLevel { + case "debug": + event = l.logger.Debug() + case "info": + event = l.logger.Info() + case "warn": + event = l.logger.Warn() + case "error": + event = l.logger.Error() + case "fatal": + event = l.logger.Fatal() + case "panic": + event = l.logger.Panic() + default: + // Default to info level if an unknown level is provided + event = l.logger.Info() + } + + duration, _ := chaos.GetChaosDuration() + + return event. + Str("logger", "chaos"). + Str("name", chaos.GetObject().GetName()). + Str("namespace", chaos.GetObject().GetNamespace()). + Str("description", chaos.GetChaosDescription()). + Str("duration", duration.String()). + Time("startTime", chaos.GetStartTime()). + Time("endTime", chaos.GetEndTime()) +} diff --git a/havoc/k8schaos/logger.go b/havoc/k8schaos/logger.go new file mode 100644 index 000000000..58d9e18d5 --- /dev/null +++ b/havoc/k8schaos/logger.go @@ -0,0 +1,44 @@ +package k8schaos + +import ( + "os" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// Default logger +var Logger zerolog.Logger + +func init() { + // Default logger + Logger = CreateLogger(LoggerConfig{ + LogOutput: os.Getenv("CHAOS_LOG_OUTPUT"), + LogLevel: os.Getenv("CHAOS_LOG_LEVEL"), + LogType: "chaos", + }) +} + +type LoggerConfig struct { + LogOutput string // "json-console" for JSON output, empty or "console" for human-friendly console output + LogLevel string // Log level (e.g., "info", "debug", "error") + LogType string // Custom log type identifier +} + +// Create initializes a zerolog.Logger based on the specified configuration. +func CreateLogger(config LoggerConfig) zerolog.Logger { + // Parse the log level + lvl, err := zerolog.ParseLevel(config.LogLevel) + if err != nil { + panic(err) // Consider more graceful error handling based on your application's requirements + } + + switch config.LogOutput { + case "json-console": + // Configure for JSON console output + return zerolog.New(os.Stderr).Level(lvl).With().Timestamp().Str("type", config.LogType).Logger() + default: + // Configure for console (human-friendly) output + return log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "15:04:05"}).Level(lvl).With().Timestamp().Str("type", config.LogType).Logger() + } +} diff --git a/havoc/k8schaos/range_grafana_annotator.go b/havoc/k8schaos/range_grafana_annotator.go new file mode 100644 index 000000000..4d1658197 --- /dev/null +++ b/havoc/k8schaos/range_grafana_annotator.go @@ -0,0 +1,240 @@ +package k8schaos + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/grafana" +) + +type RangeGrafanaAnnotator struct { + client *grafana.Client + dashboardUID string + chaosMap map[string]int64 // Maps Chaos ID to Grafana Annotation ID + logger zerolog.Logger +} + +func NewRangeGrafanaAnnotator(grafanaURL, grafanaToken, dashboardUID string, logger zerolog.Logger) *RangeGrafanaAnnotator { + return &RangeGrafanaAnnotator{ + client: grafana.NewGrafanaClient(grafanaURL, grafanaToken), + dashboardUID: dashboardUID, + chaosMap: make(map[string]int64), + logger: logger, + } +} + +func (l RangeGrafanaAnnotator) OnChaosCreated(chaos Chaos) { +} + +func (l RangeGrafanaAnnotator) OnChaosStarted(chaos Chaos) { + experiment, _ := chaos.GetExperimentStatus() + duration, _ := chaos.GetChaosDuration() + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Started

", chaos.GetChaosTypeStr())) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.GetName())) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.GetStartTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", duration.String())) + + spec := chaos.GetChaosSpec() + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + + if len(experiment.Records) > 0 { + sb.WriteString("
") + sb.WriteString("
Records:
") + sb.WriteString("") + } + + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.GetStartTime()), + Text: sb.String(), + } + res, _, err := l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } + + l.chaosMap[chaos.GetChaosName()] = res.ID +} + +func (l RangeGrafanaAnnotator) OnChaosPaused(chaos Chaos) { +} + +func (l RangeGrafanaAnnotator) OnChaosEnded(chaos Chaos) { + annotationID, exists := l.chaosMap[chaos.GetChaosName()] + if !exists { + l.logger.Error().Msgf("No Grafana annotation ID found for Chaos: %s", chaos.GetChaosName()) + return + } + + experiment, _ := chaos.GetExperimentStatus() + duration, _ := chaos.GetChaosDuration() + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s

", chaos.GetChaosTypeStr())) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.GetName())) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.GetStartTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
End Time: %s
", chaos.GetEndTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", duration.String())) + + spec := chaos.GetChaosSpec() + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + + if len(experiment.Records) > 0 { + sb.WriteString("
") + sb.WriteString("
Records:
") + sb.WriteString("") + } + + sb.WriteString("") + + // Delete the temporary start annotation + _, err = l.client.DeleteAnnotation(annotationID) + if err != nil { + l.logger.Error().Msgf("could not delete temporary start annotation: %s", err) + } + delete(l.chaosMap, chaos.GetChaosName()) + + // Create the final annotation (time range) + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.GetStartTime()), + TimeEnd: Ptr[time.Time](chaos.GetEndTime()), + Text: sb.String(), + } + res, _, err := l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } + l.chaosMap[chaos.GetChaosName()] = res.ID +} + +func (l RangeGrafanaAnnotator) OnChaosStatusUnknown(chaos Chaos) { +} + +func (l RangeGrafanaAnnotator) OnScheduleCreated(chaos Schedule) { + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Schedule Created

", chaos.Object.Spec.Type)) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.ObjectMeta.Name)) + sb.WriteString(fmt.Sprintf("
Schedule: %s
", chaos.Object.Spec.Schedule)) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.startTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", chaos.Duration.String())) + + spec := chaos.Object.Spec.ScheduleItem + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.startTime), + Text: sb.String(), + } + res, _, err := l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } + + l.chaosMap[chaos.Object.GetName()] = res.ID +} + +func (l RangeGrafanaAnnotator) OnScheduleDeleted(chaos Schedule) { + annotationID, exists := l.chaosMap[chaos.Object.GetName()] + if !exists { + l.logger.Error().Msgf("No Grafana annotation ID found for Chaos: %s", chaos.Object.GetName()) + return + } + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Schedule

", chaos.Object.Spec.Type)) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.ObjectMeta.Name)) + sb.WriteString(fmt.Sprintf("
Schedule: %s
", chaos.Object.Spec.Schedule)) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.startTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
End Time: %s
", chaos.endTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", chaos.Duration.String())) + + spec := chaos.Object.Spec.ScheduleItem + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + sb.WriteString("") + + // Delete the temporary start annotation + _, err = l.client.DeleteAnnotation(annotationID) + if err != nil { + l.logger.Error().Msgf("could not delete temporary start annotation: %s", err) + } + delete(l.chaosMap, chaos.Object.GetName()) + + // Create the final annotation (time range) + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.startTime), + TimeEnd: Ptr[time.Time](chaos.endTime), + Text: sb.String(), + } + res, _, err := l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } + l.chaosMap[chaos.Object.GetName()] = res.ID +} diff --git a/havoc/k8schaos/schedule.go b/havoc/k8schaos/schedule.go new file mode 100644 index 000000000..f8cd543da --- /dev/null +++ b/havoc/k8schaos/schedule.go @@ -0,0 +1,225 @@ +package k8schaos + +import ( + "context" + "time" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/pkg/errors" + "github.com/rs/zerolog" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ScheduleStatus string + +const ( + ScheduleStatusCreated ScheduleStatus = "created" + ScheduleStatusDeleted ScheduleStatus = "deleted" + ScheduleStatusUnknown ScheduleStatus = "unknown" // For any state that doesn't match the above +) + +type Schedule struct { + Object *v1alpha1.Schedule + Description string + DelayCreate time.Duration // Delay before creating the chaos object + Duration time.Duration // Duration for which the chaos object should exist + Status ChaosStatus + Client client.Client + listeners []ChaosListener + cancelMonitor context.CancelFunc + startTime time.Time + endTime time.Time + logger *zerolog.Logger +} + +type ScheduleOpts struct { + Object *v1alpha1.Schedule + Description string + DelayCreate time.Duration + Duration time.Duration + Client client.Client + Listeners []ChaosListener + Logger *zerolog.Logger +} + +func NewSchedule(opts ScheduleOpts) (*Schedule, error) { + if opts.Client == nil { + return nil, errors.New("client is required") + } + if opts.Object == nil { + return nil, errors.New("chaos object is required") + } + if opts.Logger == nil { + return nil, errors.New("logger is required") + } + + return &Schedule{ + Object: opts.Object, + Description: opts.Description, + DelayCreate: opts.DelayCreate, + Duration: opts.Duration, + Client: opts.Client, + listeners: opts.Listeners, + logger: opts.Logger, + }, nil +} + +// Create initiates a delayed creation of a chaos object, respecting context cancellation and deletion requests. +// It uses a timer based on `DelayCreate` and calls `create` method upon expiration unless preempted by deletion. +func (s *Schedule) Create(ctx context.Context) { + done := make(chan struct{}) + + // Create the timer with the delay to create the chaos object + timer := time.NewTimer(s.DelayCreate) + + go func() { + select { + case <-ctx.Done(): + // If the context is canceled, stop the timer and exit + if !timer.Stop() { + <-timer.C // If the timer already expired, drain the channel + } + close(done) // Signal that the operation was canceled + case <-timer.C: + // Timer expired, check if deletion was not requested + if s.Status != StatusDeleted { + s.createNow(ctx) + } + close(done) // Signal that the creation process is either done or skipped + } + }() +} + +func (s *Schedule) Delete(ctx context.Context) error { + if err := s.Client.Delete(ctx, s.Object); err != nil { + return errors.Wrap(err, "failed to delete chaos object") + } + + // Cancel the monitoring goroutine + if s.cancelMonitor != nil { + s.cancelMonitor() + } + + s.endTime = time.Now() + s.Status = StatusDeleted + s.notifyListeners(string(ScheduleStatusDeleted)) + + return nil +} + +func (s *Schedule) AddListener(listener ChaosListener) { + s.listeners = append(s.listeners, listener) +} + +func (s *Schedule) GetObject() client.Object { + return s.Object +} + +func (s *Schedule) GetChaosName() string { + return s.Object.GetName() +} + +func (s *Schedule) GetChaosDescription() string { + return s.Description +} + +func (s *Schedule) GetChaosSpec() interface{} { + return s.Object.Spec.ScheduleItem +} + +func (s *Schedule) GetChaosDuration() (time.Duration, error) { + return s.Duration, nil +} + +func (s *Schedule) GetStartTime() time.Time { + return s.startTime +} + +func (s *Schedule) GetEndTime() time.Time { + return s.endTime +} + +func (s *Schedule) GetExpectedEndTime() (time.Time, error) { + duration, err := s.GetChaosDuration() + if err != nil { + return time.Time{}, err + } + return s.startTime.Add(duration), nil +} + +func (s *Schedule) createNow(ctx context.Context) { + if err := s.Client.Create(ctx, s.Object); err != nil { + Logger.Error().Err(err).Interface("chaos", s).Msg("failed to create chaos object") + return + } + s.startTime = time.Now() + s.Status = StatusCreated + s.notifyListeners(string(ScheduleStatusCreated)) + + // Create a cancellable context for monitorStatus + monitorCtx, cancel := context.WithCancel(ctx) + s.cancelMonitor = cancel + go s.monitorStatus(monitorCtx) + + // Start a deletion timer to delete the chaos object after the specified duration + done := make(chan struct{}) + deleteTimer := time.NewTimer(s.Duration) + go func() { + select { + case <-ctx.Done(): + // Context was canceled, ensure chaos object is deleted + if !deleteTimer.Stop() { + <-deleteTimer.C // Drain the timer if it already fired + } + err := s.Delete(context.Background()) + if err != nil { + s.logger.Error().Err(err).Msg("failed to delete chaos object") + } + close(done) + case <-deleteTimer.C: + // Duration elapsed, delete the chaos object + err := s.Delete(context.Background()) + if err != nil { + s.logger.Error().Err(err).Msg("failed to delete chaos object") + } + close(done) + } + }() +} + +func (s *Schedule) notifyListeners(event string) { + for _, listener := range s.listeners { + switch event { + case string(ScheduleStatusCreated): + listener.OnScheduleCreated(*s) + case string(ScheduleStatusDeleted): + listener.OnScheduleDeleted(*s) + } + } +} + +func (s *Schedule) monitorStatus(ctx context.Context) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + // Context canceled, stop monitoring + return + case <-ticker.C: + // Fetch the latest state of the Schedule object + var schedule v1alpha1.Schedule + if err := s.Client.Get(ctx, client.ObjectKey{ + Namespace: s.Object.GetNamespace(), + Name: s.Object.GetName(), + }, &schedule); err != nil { + Logger.Error().Err(err).Msg("Failed to get Schedule object") + continue + } + + // Log or process the schedule's current status + // Loggerog.Info().Interface("status", schedule.Status).Msg("Current Schedule Status") + } + } +} diff --git a/havoc/k8schaos/single_line_grafana_annotator.go b/havoc/k8schaos/single_line_grafana_annotator.go new file mode 100644 index 000000000..9afc91102 --- /dev/null +++ b/havoc/k8schaos/single_line_grafana_annotator.go @@ -0,0 +1,205 @@ +package k8schaos + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/grafana" +) + +type SingleLineGrafanaAnnotator struct { + client *grafana.Client + dashboardUID string + logger zerolog.Logger +} + +func NewSingleLineGrafanaAnnotator(grafanaURL, grafanaToken, dashboardUID string, logger zerolog.Logger) *SingleLineGrafanaAnnotator { + return &SingleLineGrafanaAnnotator{ + client: grafana.NewGrafanaClient(grafanaURL, grafanaToken), + dashboardUID: dashboardUID, + logger: logger, + } +} + +func (l SingleLineGrafanaAnnotator) OnChaosCreated(chaos Chaos) { +} + +func (l SingleLineGrafanaAnnotator) OnChaosCreationFailed(chaos Chaos, reason error) { +} + +func (l SingleLineGrafanaAnnotator) OnChaosStarted(chaos Chaos) { + experiment, _ := chaos.GetExperimentStatus() + duration, _ := chaos.GetChaosDuration() + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Started

", chaos.GetChaosTypeStr())) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.GetName())) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.GetStartTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", duration.String())) + + spec := chaos.GetChaosSpec() + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + + if len(experiment.Records) > 0 { + sb.WriteString("
") + sb.WriteString("
Records:
") + sb.WriteString("") + } + + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.GetStartTime()), + Text: sb.String(), + } + _, _, err = l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } +} + +func (l SingleLineGrafanaAnnotator) OnChaosPaused(chaos Chaos) { +} + +func (l SingleLineGrafanaAnnotator) OnChaosEnded(chaos Chaos) { + experiment, _ := chaos.GetExperimentStatus() + duration, _ := chaos.GetChaosDuration() + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Ended

", chaos.GetChaosTypeStr())) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.GetName())) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.GetStartTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
End Time: %s
", chaos.GetEndTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", duration.String())) + + spec := chaos.GetChaosSpec() + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + + if len(experiment.Records) > 0 { + sb.WriteString("
") + sb.WriteString("
Records:
") + sb.WriteString("") + } + + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.GetEndTime()), + Text: sb.String(), + } + _, _, err = l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } +} + +func (l SingleLineGrafanaAnnotator) OnChaosStatusUnknown(chaos Chaos) { +} + +func (l SingleLineGrafanaAnnotator) OnScheduleCreated(s Schedule) { + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Schedule Created

", s.Object.Spec.Type)) + sb.WriteString(fmt.Sprintf("
Name: %s
", s.Object.ObjectMeta.Name)) + sb.WriteString(fmt.Sprintf("
Schedule: %s
", s.Object.Spec.Schedule)) + if s.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", s.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", s.startTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", s.Duration.String())) + + spec := s.Object.Spec.ScheduleItem + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Schedule Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](s.startTime), + Text: sb.String(), + } + _, _, err = l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } +} + +func (l SingleLineGrafanaAnnotator) OnScheduleDeleted(s Schedule) { + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Schedule Ended

", s.Object.Spec.Type)) + sb.WriteString(fmt.Sprintf("
Name: %s
", s.Object.ObjectMeta.Name)) + sb.WriteString(fmt.Sprintf("
Schedule: %s
", s.Object.Spec.Schedule)) + if s.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", s.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", s.startTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
End Time: %s
", s.endTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", s.Duration.String())) + + spec := s.Object.Spec.ScheduleItem + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Schedule Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](s.endTime), + Text: sb.String(), + } + _, _, err = l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } +} diff --git a/havoc/k8schaos/utils.go b/havoc/k8schaos/utils.go new file mode 100644 index 000000000..c0d3b8ac4 --- /dev/null +++ b/havoc/k8schaos/utils.go @@ -0,0 +1,5 @@ +package k8schaos + +func Ptr[T any](value T) *T { + return &value +} diff --git a/havoc/monkey.go b/havoc/monkey.go new file mode 100644 index 000000000..b80229bc9 --- /dev/null +++ b/havoc/monkey.go @@ -0,0 +1,222 @@ +package havoc + +import ( + "context" + "fmt" + "math/rand" + "sync" + "time" + + "github.com/go-resty/resty/v2" + "github.com/pkg/errors" +) + +const ( + MonkeyModeSeq = "seq" + MonkeyModeRandom = "rand" + + ErrInvalidMode = "monkey mode is invalid, should be either \"seq\" or \"rand\"" +) + +type ExperimentAction struct { + Name string + ExperimentKind string + ExperimentSpec string + TimeStart int64 + TimeEnd int64 +} + +type ExperimentAnnotationBody struct { + DashboardUID string `json:"dashboardUID"` + Time int64 `json:"time"` + TimeEnd int64 `json:"timeEnd"` + Tags []string `json:"tags"` + Text string `json:"text"` +} + +type Controller struct { + cfg *Config + client *resty.Client + ctx context.Context + cancel context.CancelFunc + wg *sync.WaitGroup + errors []error + experimentActions []*ExperimentAction +} + +func NewController(cfg *Config) (*Controller, error) { + InitDefaultLogging() + if cfg == nil { + cfg = DefaultConfig() + dumpConfig(cfg) + } + c := resty.New() + c.SetBaseURL(cfg.Havoc.Grafana.URL) + c.SetAuthScheme("Bearer") + c.SetAuthToken(cfg.Havoc.Grafana.Token) + return &Controller{ + client: c, + cfg: cfg, + wg: &sync.WaitGroup{}, + errors: make([]error, 0), + experimentActions: make([]*ExperimentAction, 0), + }, nil +} + +// AnnotateExperiment sends annotation marker to Grafana dashboard +func (m *Controller) AnnotateExperiment(a *ExperimentAction) error { + if m.cfg.Havoc.Grafana.URL == "" || m.cfg.Havoc.Grafana.Token == "" { + L.Warn().Msg("Dashboards are not selected, experiment time wasn't annotated, please check README to enable Grafana integration") + return nil + } + for _, dashboardUID := range m.cfg.Havoc.Grafana.DashboardUIDs { + start := a.TimeStart * 1e3 + end := a.TimeEnd * 1e3 + specBody := fmt.Sprintf("
%s
", a.ExperimentSpec) + aa := &ExperimentAnnotationBody{ + DashboardUID: dashboardUID, + Time: start, + TimeEnd: end, + Tags: []string{"havoc", a.ExperimentKind}, + Text: fmt.Sprintf( + "File: %s\n%s", + a.Name, + specBody, + ), + } + _, err := m.client.R(). + SetBody(aa). + Post(fmt.Sprintf("%s/api/annotations", m.cfg.Havoc.Grafana.URL)) + if err != nil { + return err + } + L.Info(). + Str("DashboardUID", dashboardUID). + Str("Name", a.Name). + Int64("Start", a.TimeStart). + Int64("End", a.TimeEnd). + Msg("Annotated experiment") + } + return nil +} + +func (m *Controller) ApplyAndAnnotate(exp *NamedExperiment) error { + ea := &ExperimentAction{ + Name: exp.Name, + ExperimentKind: exp.Kind, + ExperimentSpec: string(exp.CRDBytes), + TimeStart: time.Now().Unix(), + } + if err := m.ApplyExperiment(exp, true); err != nil { + return err + } + ea.TimeEnd = time.Now().Unix() + return m.AnnotateExperiment(ea) +} + +func (m *Controller) Run() error { + L.Info().Msg("Starting chaos monkey") + dur, err := time.ParseDuration(m.cfg.Havoc.Monkey.Duration) + if err != nil { + return err + } + m.ctx, m.cancel = context.WithTimeout(context.Background(), dur) + defer m.cancel() + existingExperimentTypes, err := m.readExistingExperimentTypes(m.cfg.Havoc.Dir) + if err != nil { + m.errors = append(m.errors, err) + return err + } + + m.wg.Add(1) + switch m.cfg.Havoc.Monkey.Mode { + case MonkeyModeSeq: + for _, expType := range existingExperimentTypes { + experiments, err := m.ReadExperimentsFromDir([]string{expType}, m.cfg.Havoc.Dir) + if err != nil { + m.errors = append(m.errors, err) + return err + } + for _, exp := range experiments { + if err := m.ApplyAndAnnotate(exp); err != nil { + m.errors = append(m.errors, err) + return err + } + cdDuration, err := time.ParseDuration(m.cfg.Havoc.Monkey.Cooldown) + if err != nil { + m.errors = append(m.errors, err) + return err + } + select { + case <-m.ctx.Done(): + m.wg.Done() + L.Info().Msg("Monkey has finished by timeout") + return nil + default: + } + L.Info(). + Dur("Duration", cdDuration). + Msg("Cooldown between experiments") + time.Sleep(cdDuration) + } + } + L.Info().Msg("Monkey has finished all scheduled experiments") + m.wg.Done() + case MonkeyModeRandom: + allExperiments := make([]*NamedExperiment, 0) + r := rand.New(rand.NewSource(time.Now().Unix())) + for _, expType := range existingExperimentTypes { + experiments, err := m.ReadExperimentsFromDir([]string{expType}, m.cfg.Havoc.Dir) + if err != nil { + m.errors = append(m.errors, err) + return err + } + allExperiments = append(allExperiments, experiments...) + } + for { + select { + case <-m.ctx.Done(): + m.wg.Done() + L.Info().Msg("Monkey has finished by timeout") + return nil + default: + exp := pickExperiment(r, allExperiments) + if err := m.ApplyAndAnnotate(exp); err != nil { + m.errors = append(m.errors, err) + return err + } + cdDuration, err := time.ParseDuration(m.cfg.Havoc.Monkey.Cooldown) + if err != nil { + m.errors = append(m.errors, err) + return err + } + L.Info(). + Dur("Duration", cdDuration). + Msg("Cooldown between experiments") + time.Sleep(cdDuration) + } + } + default: + return errors.New(ErrInvalidMode) + } + return nil +} + +func (m *Controller) Stop() []error { + L.Info().Msg("Stopping chaos monkey") + m.cancel() + m.wg.Wait() + L.Info().Errs("Errors", m.errors).Msg("Chaos monkey stopped") + return m.errors +} + +func (m *Controller) Wait() []error { + L.Info().Msg("Waiting for chaos monkey to finish") + m.wg.Wait() + L.Info().Errs("Errors", m.errors).Msg("Chaos monkey finished") + return m.errors +} + +func pickExperiment(r *rand.Rand, s []*NamedExperiment) *NamedExperiment { + return s[r.Intn(len(s))] +} diff --git a/havoc/openapi.go b/havoc/openapi.go new file mode 100644 index 000000000..d5450bc22 --- /dev/null +++ b/havoc/openapi.go @@ -0,0 +1,140 @@ +package havoc + +import ( + "fmt" + "github.com/getkin/kin-openapi/openapi3" + "github.com/pkg/errors" + "github.com/samber/lo" + "regexp" + "strings" +) + +const ( + ErrParsingOpenAPISpec = "failed to parse OpenAPISpec" +) + +var ( + OpenAPIPathParam = regexp.MustCompile(`({.*})`) +) + +type OAPISpecData struct { + Port int64 + RawPaths []string + SpecData map[string]*openapi3.PathItem +} + +// ParseOpenAPISpecs parses OpenAPI spec methods +func (m *Controller) ParseOpenAPISpecs() ([]*OAPISpecData, error) { + data := make([]*OAPISpecData, 0) + for _, oapiData := range m.cfg.Havoc.OpenAPI.Mapping { + for _, p := range oapiData.SpecToPortMappings { + loader := openapi3.NewLoader() + doc, err := loader.LoadFromFile(p.Path) + if err != nil { + return nil, errors.Wrap(err, ErrParsingOpenAPISpec) + } + oa := &OAPISpecData{ + Port: p.Port, + RawPaths: make([]string, 0), + SpecData: doc.Paths.Map(), + } + for rawPath := range doc.Paths.Map() { + L.Info().Str("Path", rawPath).Msg("Found API path") + oa.RawPaths = append(oa.RawPaths, rawPath) + } + data = append(data, oa) + } + } + return data, nil +} + +// generateOAPIExperiments generates HTTP experiments for a component group (entry), for each method type +func (m *Controller) generateOAPIExperiments(experiments map[string]string, namespace string, entry lo.Entry[string, int], oapiSpecs []*OAPISpecData) error { + for _, apiSpec := range oapiSpecs { + for _, rawPath := range apiSpec.RawPaths { + pathData := apiSpec.SpecData[rawPath] + if pathData.Connect != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "CONNECT", apiSpec.Port); err != nil { + return err + } + } + if pathData.Delete != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "DELETE", apiSpec.Port); err != nil { + return err + } + } + if pathData.Get != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "GET", apiSpec.Port); err != nil { + return err + } + } + if pathData.Head != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "HEAD", apiSpec.Port); err != nil { + return err + } + } + if pathData.Options != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "OPTIONS", apiSpec.Port); err != nil { + return err + } + } + if pathData.Patch != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "PATCH", apiSpec.Port); err != nil { + return err + } + } + if pathData.Post != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "POST", apiSpec.Port); err != nil { + return err + } + } + if pathData.Put != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "PUT", apiSpec.Port); err != nil { + return err + } + } + if pathData.Trace != nil { + if err := m.generateHTTPExperiment(experiments, namespace, entry, rawPath, "TRACE", apiSpec.Port); err != nil { + return err + } + } + } + } + return nil +} + +func (m *Controller) generateHTTPExperiment( + experiments map[string]string, + namespace string, + entry lo.Entry[string, int], + rawPath string, + method string, + port int64, +) error { + sanitizedLabel := sanitizeLabel(entry.Key) + sanitizedRawPath := sanitizeLabel(rawPath) + sanitizedLabel = fmt.Sprintf("%s-%s-%s", sanitizedLabel, sanitizedRawPath, method) + experiment, err := HTTPExperiment{ + Namespace: namespace, + ExperimentName: strings.ToLower(fmt.Sprintf("%s-%s", ChaosTypeHTTP, sanitizedLabel)), + Duration: m.cfg.Havoc.StressCPU.Duration, + Mode: "all", + Selector: entry.Key, + Target: "Response", + Abort: true, + Path: pathToWildcardExpr(rawPath), + Method: method, + Port: port, + }.String() + if err != nil { + return err + } + experiments[sanitizedLabel] = experiment + return nil +} + +// pathToWildcardExpr transforms path params into wildcard expressions +// TODO: this need thorough testing though, since it can be much more complex +func pathToWildcardExpr(path string) string { + return string(OpenAPIPathParam.ReplaceAll([]byte(path), []byte("*"))) +} diff --git a/havoc/oscmd.go b/havoc/oscmd.go new file mode 100644 index 000000000..a168cadbb --- /dev/null +++ b/havoc/oscmd.go @@ -0,0 +1,39 @@ +package havoc + +import ( + "bytes" + "context" + "errors" + "os/exec" + "strings" +) + +func ExecCmd(command string) (string, error) { + L.Info().Interface("Command", command).Msg("Executing command") + c := strings.Split(command, " ") + cmd := exec.CommandContext(context.Background(), c[0], c[1:]...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitCode := exitErr.ExitCode() + L.Error(). + Int("Code", exitCode). + Msg("Command exited with status code") + L.Error(). + Str("Out", stdout.String()). + Str("Err", stderr.String()). + Msg("Command output") + } + } else { + L.Info().Msg("Command ran successfully") + L.Debug(). + Str("Out", stdout.String()). + Str("Err", stderr.String()). + Msg("Command output") + } + return stdout.String(), err +} diff --git a/havoc/parse.go b/havoc/parse.go new file mode 100644 index 000000000..7cf146a9e --- /dev/null +++ b/havoc/parse.go @@ -0,0 +1,181 @@ +package havoc + +import ( + "encoding/json" + "fmt" + "github.com/pkg/errors" + "github.com/rs/zerolog" + "github.com/samber/lo" + "os" + "sort" + "strings" +) + +const ( + ErrNoNamespace = "no namespace found" + ErrEmptyNamespace = "no pods found inside namespace, namespace is empty or check your filter" +) + +const ( + NoGroupKey = "no-group" +) + +type ManifestPart struct { + Kind string + Name string + FlattenedManifest map[string]interface{} +} + +// PodsListResponse pod list response from kubectl in JSON +type PodsListResponse struct { + Items []*PodResponse `json:"items"` +} + +// PodResponse pod info response from kubectl in JSON +type PodResponse struct { + Metadata struct { + Name string `json:"name"` + Labels map[string]string `json:"labels"` + } `json:"metadata"` +} + +type GroupInfo struct { + Label string + PodsAffected int +} + +// ActionablePodInfo info about pod and labels for which we can generate a chaos experiment +type ActionablePodInfo struct { + PodName string + Labels []string + HasGroup bool +} + +func uniquePairs(strings []string) [][]string { + var pairs [][]string + for i := 0; i < len(strings); i++ { + for j := i + 1; j < len(strings); j++ { + pair := []string{strings[i], strings[j]} + pairs = append(pairs, pair) + } + } + return pairs +} + +func (m *Controller) processPodInfoLo(plr *PodsListResponse) (map[string][]*PodResponse, []*PodResponse, []lo.Entry[string, int], [][]string, error) { + L.Info().Msg("Processing pods info") + // filtering + filteredPods := lo.Filter(plr.Items, func(item *PodResponse, index int) bool { + return !sliceContainsSubString(item.Metadata.Name, m.cfg.Havoc.IgnoredPods) + }) + labelsToAllow := append([]string{}, m.cfg.Havoc.ComponentLabelKey) + if m.hasNetworkExperiments() { + labelsToAllow = append(labelsToAllow, m.cfg.Havoc.NetworkPartition.Label) + } + for _, p := range filteredPods { + p.Metadata.Labels = lo.PickByKeys(p.Metadata.Labels, labelsToAllow) + } + if len(filteredPods) == 0 { + return nil, nil, nil, nil, errors.New(ErrEmptyNamespace) + } + // grouping + byComponent := lo.GroupBy(filteredPods, func(item *PodResponse) string { + key := m.cfg.Havoc.ComponentLabelKey + return m.labelSelector(key, item.Metadata.Labels[key]) + }) + var byPartition map[string][]*PodResponse + if m.hasNetworkExperiments() { + byPartition = lo.GroupBy(filteredPods, func(item *PodResponse) string { + key := m.cfg.Havoc.NetworkPartition.Label + return m.labelSelector(key, item.Metadata.Labels[key]) + }) + } + componentGroupInfo := lo.MapEntries(byComponent, func(key string, value []*PodResponse) (string, int) { + return key, len(value) + }) + componentGroupsInfo := lo.Reject(lo.Entries(componentGroupInfo), func(item lo.Entry[string, int], index int) bool { + return item.Key == NoGroupKey + }) + byPartition = lo.OmitByKeys(byPartition, []string{NoGroupKey}) + partKeys := lo.Keys(byPartition) + sort.Strings(partKeys) + networkGroupsInfo := uniquePairs(partKeys) + + m.printPartitions(byComponent, "Component groups found") + m.printPartitions(byPartition, "Network groups found") + return byComponent, byComponent[NoGroupKey], componentGroupsInfo, networkGroupsInfo, nil +} + +func (m *Controller) hasNetworkExperiments() bool { + if m.cfg.Havoc.NetworkPartition != nil && m.cfg.Havoc.NetworkPartition.Label != "" { + return true + } + return false +} + +func (m *Controller) printPartitions(parts map[string][]*PodResponse, msg string) { + for _, p := range parts { + for _, pp := range p { + L.Info(). + Str("Name", pp.Metadata.Name). + Interface("Labels", pp.Metadata.Labels). + Msg(msg) + } + } +} + +// labelSelector transforms selector to ChaosMesh CRD format +func (m *Controller) labelSelector(k, v string) string { + if v == "" { + return NoGroupKey + } else { + return fmt.Sprintf("'%s': '%s'", k, v) + } +} + +// groupValueFromLabelSelector returns just the selector value +func (m *Controller) groupValueFromLabelSelector(selector string) string { + val := strings.Split(selector, ": ")[1] + return strings.ReplaceAll(val, "'", "") +} + +// GetPodsInfo gets info about all the pods in the namespace +func (m *Controller) GetPodsInfo(namespace string) (*PodsListResponse, error) { + if _, err := ExecCmd(fmt.Sprintf("kubectl get ns %s", namespace)); err != nil { + return nil, errors.Wrap(errors.New(ErrNoNamespace), namespace) + } + var cmdBuilder strings.Builder + cmdBuilder.Write([]byte(fmt.Sprintf("kubectl get pods -n %s ", namespace))) + if m.cfg.Havoc.NamespaceLabelFilter != "" { + cmdBuilder.Write([]byte(fmt.Sprintf("-l %s ", m.cfg.Havoc.NamespaceLabelFilter))) + } + cmdBuilder.Write([]byte("-o json")) + out, err := ExecCmd(cmdBuilder.String()) + if err != nil { + return nil, err + } + if err := dumpPodInfo(out); err != nil { + return nil, err + } + var pr *PodsListResponse + if err := json.Unmarshal([]byte(out), &pr); err != nil { + return nil, err + } + return pr, nil +} + +func dumpPodInfo(out string) error { + if L.GetLevel() == zerolog.DebugLevel { + var plr *PodsListResponse + if err := json.Unmarshal([]byte(out), &plr); err != nil { + return err + } + d, err := json.Marshal(plr) + if err != nil { + return err + } + _ = os.WriteFile("pods_dump.json", d, os.ModePerm) + return nil + } + return nil +} diff --git a/havoc/shell.nix b/havoc/shell.nix new file mode 100644 index 000000000..1eaee585a --- /dev/null +++ b/havoc/shell.nix @@ -0,0 +1,18 @@ +{ stdenv, pkgs, lib }: + +pkgs.mkShell { + buildInputs = with pkgs; [ + go + gopls + delve + golangci-lint + gotools + jq + ]; + GOROOT="${pkgs.go}/share/go"; + + shellHook = '' + export PATH=$GOPATH/bin:$PATH + go install cmd/havoc.go + ''; +} diff --git a/havoc/testdata/configs/crib-all.toml b/havoc/testdata/configs/crib-all.toml new file mode 100644 index 000000000..c96ff1766 --- /dev/null +++ b/havoc/testdata/configs/crib-all.toml @@ -0,0 +1,126 @@ +[havoc] +# dir is a custom dir you can select, if null monkey will create a new dir +dir = "testdata/results/all" +# if you have multiple products inside one namespace this can help to filter by label in k=v format +namespace_label_filter = "" +# pods with this prefix will be ignored when generating experiments +ignore_pods = ["-db-"] +# name of the key to select components in the namespace +component_label_key = "havoc-component-group" +# group labels containing these strings will be ignored when generating group experiments +ignore_group_labels = [ + "mainnet", + "release", + "intents.otterize.com", + "pod-template-hash", + "rollouts-pod-template-hash", + "chain.link/app", + "chain.link/cost-center", + "chain.link/env", + "chain.link/project", + "chain.link/team", + "app.kubernetes.io/part-of", + "app.kubernetes.io/managed-by", + "app.chain.link/product", + "app.kubernetes.io/version", + "app.chain.link/blockchain", + "app.kubernetes.io/instance", + "app.kubernetes.io/name", +] +# these are experiment types you'd like to generate +experiment_types = [ + "external", + "failure", + "latency", + "cpu", + "memory", + "group-failure", + "group-latency", + "group-cpu", + "group-memory", + "group-partition", + "blockchain_rewind_head", +] +#experiment_types = ["group-partition"] + +[havoc.failure] +# duration of a "failure" experiment +duration = "10s" +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_fixed = ["3", "2", "1"] + +[havoc.latency] +# duration of "latency" experiment +duration = "10s" +# constant latency to inject +latency = "300ms" +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_fixed = ["3", "2", "1"] + +[havoc.stress_memory] +# duration of "stress" experiment affecting pod memory +duration = "10s" +# amount of workers which occupies memory +workers = 1 +# total amount of memory occupied +memory = "512MB" +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_fixed = ["3", "2", "1"] + +[havoc.stress_cpu] +# duration of "stress" experiment affecting pod CPU +duration = "10s" +# amount of workers which occupies cpu +workers = 1 +# amount of CPU core utilization, 100 means 1 worker will consume 1 cpu, 2 workers + 100 load = 2 CPUs +load = 100 +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_fixed = ["3", "2", "1"] + +[havoc.network_partition] +# duration of "network partition" experiment affecting pod CPU +duration = "30s" +# percentage of pods experiments affect in groups, see group-failure key and dir when generated +group_percentage = ["100"] +# a label to split pods for experiments +label = "havoc-network-group" + +[havoc.blockchain_rewind_head] +# duration of "blockchain" experiment +duration = "30s" + +[[havoc.blockchain_rewind_head.nodes]] +# label of executor pod +executor_pod_prefix = "geth-1337" +# executor container name +executor_container_name = "geth-network" +# blockchain node internal HTTP URL +node_internal_http_url = "geth-1337:8544" +# blocks to rewind from last +blocks = [30, 20, 10] + +[[havoc.blockchain_rewind_head.nodes]] +# label of executor pod +executor_pod_prefix = "geth-2337" +# executor container name +executor_container_name = "geth-network" +# blockchain node internal HTTP URL +node_internal_http_url = "geth-2337:8544" +# blocks to rewind from last +blocks = [30, 20, 10] + +[havoc.external_targets] +# duration of "external" experiment +duration = "10s" +# URL of external service that'd fail to resolve +urls = ["www.google.com"] + +[havoc.monkey] +# havoc monkey mode: +# seq - runs all experiments from all dirs sequentially one time +# rand - runs random experiments from all dirs +mode = "rand" +# duration of havoc monkey +duration = "3m" +# cooldown between experiments +cooldown = "10s" diff --git a/havoc/testdata/deployments/deployment_crib_1.json b/havoc/testdata/deployments/deployment_crib_1.json new file mode 100755 index 000000000..e868e385e --- /dev/null +++ b/havoc/testdata/deployments/deployment_crib_1.json @@ -0,0 +1,178 @@ +{ + "items": [ + { + "metadata": { + "name": "app-node-1-6bfbd56f65-brdtd", + "labels": { + "app": "app", + "instance": "node-1", + "intents.otterize.com/server": "app-node-1-cl-cluster-f3da74", + "network-partition-group": "network-group-2", + "pod-template-hash": "6bfbd56f65", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-1-db-dcd746f4c-2s9q7", + "labels": { + "app": "app-db", + "instance": "node-1-db", + "intents.otterize.com/server": "app-node-1-db-cl-cluster-162f92", + "pod-template-hash": "dcd746f4c", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-2-c989699d9-ng772", + "labels": { + "app": "app", + "instance": "node-2", + "intents.otterize.com/server": "app-node-2-cl-cluster-23bb66", + "network-partition-group": "network-group-2", + "pod-template-hash": "c989699d9", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-2-db-d668d649c-tnxkm", + "labels": { + "app": "app-db", + "instance": "node-2-db", + "intents.otterize.com/server": "app-node-2-db-cl-cluster-6999ba", + "pod-template-hash": "d668d649c", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-3-77c8fcf86f-r2l8x", + "labels": { + "app": "app", + "instance": "node-3", + "intents.otterize.com/server": "app-node-3-cl-cluster-04fa6d", + "network-partition-group": "network-group-1", + "pod-template-hash": "77c8fcf86f", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-3-db-69c7b7974c-9c2r6", + "labels": { + "app": "app-db", + "instance": "node-3-db", + "intents.otterize.com/server": "app-node-3-db-cl-cluster-31f364", + "pod-template-hash": "69c7b7974c", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-4-764b96b95c-gl6nn", + "labels": { + "app": "app", + "instance": "node-4", + "intents.otterize.com/server": "app-node-4-cl-cluster-816e93", + "network-partition-group": "network-group-1", + "pod-template-hash": "764b96b95c", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-4-db-58dcbd8c77-5hlbw", + "labels": { + "app": "app-db", + "instance": "node-4-db", + "intents.otterize.com/server": "app-node-4-db-cl-cluster-dd129f", + "pod-template-hash": "58dcbd8c77", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-5-6998f89557-925zr", + "labels": { + "app": "app", + "instance": "node-5", + "intents.otterize.com/server": "app-node-5-cl-cluster-a9916b", + "network-partition-group": "network-group-1", + "pod-template-hash": "6998f89557", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-5-db-77ff64c687-q4kph", + "labels": { + "app": "app-db", + "instance": "node-5-db", + "intents.otterize.com/server": "app-node-5-db-cl-cluster-a7c606", + "pod-template-hash": "77ff64c687", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-6-788f849997-862hp", + "labels": { + "app": "app", + "instance": "node-6", + "intents.otterize.com/server": "app-node-6-cl-cluster-6ab39a", + "network-partition-group": "network-group-1", + "pod-template-hash": "788f849997", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-6-db-58c5d55ccc-tfclz", + "labels": { + "app": "app-db", + "instance": "node-6-db", + "intents.otterize.com/server": "app-node-6-db-cl-cluster-cacf39", + "pod-template-hash": "58c5d55ccc", + "release": "app" + } + } + }, + { + "metadata": { + "name": "geth-55b64cdf59-rxmxt", + "labels": { + "app": "geth", + "intents.otterize.com/server": "geth-cl-cluster-331d92", + "network-partition-group": "network-group-3", + "pod-template-hash": "55b64cdf59", + "release": "app" + } + } + }, + { + "metadata": { + "name": "mockserver-7cb865999c-c8pcw", + "labels": { + "app": "mockserver", + "intents.otterize.com/server": "mockserver-cl-cluster-dedbac", + "network-partition-group": "network-group-4", + "pod-template-hash": "7cb865999c", + "release": "app" + } + } + } + ] +} diff --git a/havoc/testdata/deployments/deployment_crib_block_rewind.json b/havoc/testdata/deployments/deployment_crib_block_rewind.json new file mode 100755 index 000000000..a78d91ebf --- /dev/null +++ b/havoc/testdata/deployments/deployment_crib_block_rewind.json @@ -0,0 +1,219 @@ +{ + "items": [ + { + "metadata": { + "name": "app-node-1-bootstrap-5b47fb4dbc-msbzz", + "labels": { + "app": "app", + "instance": "node-1", + "intents.otterize.com/server": "app-node-1-bootstrap-cl-cluster-95bb1e", + "pod-template-hash": "5b47fb4dbc", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-1-db-6748d86b64-8hph5", + "labels": { + "app": "app-db", + "havoc-component-group": "db", + "havoc-network-group": "db", + "instance": "node-1-db", + "intents.otterize.com/server": "app-node-1-db-cl-cluster-162f92", + "pod-template-hash": "6748d86b64", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-2-596bb765d6-kph9d", + "labels": { + "app": "app", + "havoc-component-group": "node", + "havoc-network-group": "2", + "instance": "node-2", + "intents.otterize.com/server": "app-node-2-cl-cluster-23bb66", + "pod-template-hash": "596bb765d6", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-2-db-596cd6857d-6mqkd", + "labels": { + "app": "app-db", + "havoc-component-group": "db", + "havoc-network-group": "db", + "instance": "node-2-db", + "intents.otterize.com/server": "app-node-2-db-cl-cluster-6999ba", + "pod-template-hash": "596cd6857d", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-3-6f554cc8b6-g5vk9", + "labels": { + "app": "app", + "havoc-component-group": "node", + "havoc-network-group": "2", + "instance": "node-3", + "intents.otterize.com/server": "app-node-3-cl-cluster-04fa6d", + "pod-template-hash": "6f554cc8b6", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-3-db-bf4fcdd4-5w828", + "labels": { + "app": "app-db", + "havoc-component-group": "db", + "havoc-network-group": "db", + "instance": "node-3-db", + "intents.otterize.com/server": "app-node-3-db-cl-cluster-31f364", + "pod-template-hash": "bf4fcdd4", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-4-cf9977d9c-4gt28", + "labels": { + "app": "app", + "havoc-component-group": "node", + "havoc-network-group": "1", + "instance": "node-4", + "intents.otterize.com/server": "app-node-4-cl-cluster-816e93", + "pod-template-hash": "cf9977d9c", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-4-db-86b848b46b-crwxj", + "labels": { + "app": "app-db", + "havoc-component-group": "db", + "havoc-network-group": "db", + "instance": "node-4-db", + "intents.otterize.com/server": "app-node-4-db-cl-cluster-dd129f", + "pod-template-hash": "86b848b46b", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-5-d557ccf49-s2ffs", + "labels": { + "app": "app", + "havoc-component-group": "node", + "havoc-network-group": "1", + "instance": "node-5", + "intents.otterize.com/server": "app-node-5-cl-cluster-a9916b", + "pod-template-hash": "d557ccf49", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-5-db-56999b58d8-gg2ml", + "labels": { + "app": "app-db", + "havoc-component-group": "db", + "havoc-network-group": "db", + "instance": "node-5-db", + "intents.otterize.com/server": "app-node-5-db-cl-cluster-a7c606", + "pod-template-hash": "56999b58d8", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-6-5f964f4b9-wcbth", + "labels": { + "app": "app", + "havoc-component-group": "node", + "havoc-network-group": "1", + "instance": "node-6", + "intents.otterize.com/server": "app-node-6-cl-cluster-6ab39a", + "pod-template-hash": "5f964f4b9", + "release": "app" + } + } + }, + { + "metadata": { + "name": "app-node-6-db-757b5c49cd-5w796", + "labels": { + "app": "app-db", + "havoc-component-group": "db", + "havoc-network-group": "db", + "instance": "node-6-db", + "intents.otterize.com/server": "app-node-6-db-cl-cluster-cacf39", + "pod-template-hash": "757b5c49cd", + "release": "app" + } + } + }, + { + "metadata": { + "name": "geth-1337-7f7c9fb6c6-hzdhn", + "labels": { + "app": "geth", + "havoc-component-group": "blockchain", + "havoc-network-group": "blockchain", + "intents.otterize.com/server": "geth-cl-cluster-331d92", + "pod-template-hash": "7f7c9fb6c6", + "release": "app" + } + } + }, + { + "metadata": { + "name": "geth-2337-7f7c9fb6c6-hzdhn", + "labels": { + "app": "geth", + "havoc-component-group": "blockchain", + "havoc-network-group": "blockchain", + "intents.otterize.com/server": "geth-cl-cluster-331d92", + "pod-template-hash": "7f7c9fb6c6", + "release": "app" + } + } + }, + { + "metadata": { + "name": "mockserver-7cb865999c-qwdt9", + "labels": { + "app": "mockserver", + "intents.otterize.com/server": "mockserver-cl-cluster-dedbac", + "pod-template-hash": "7cb865999c", + "release": "app" + } + } + }, + { + "metadata": { + "name": "runner-64c589dd4b-qh4lj", + "labels": { + "app": "runner", + "instance": "runner-1", + "intents.otterize.com/server": "runner-cl-cluster-61468b", + "pod-template-hash": "64c589dd4b", + "release": "app" + } + } + } + ] +} diff --git a/havoc/testdata/deployments/deployment_single_group.json b/havoc/testdata/deployments/deployment_single_group.json new file mode 100755 index 000000000..9d59513e4 --- /dev/null +++ b/havoc/testdata/deployments/deployment_single_group.json @@ -0,0 +1,20 @@ +{ + "items": [ + { + "metadata": { + "name": "pod-1", + "labels": { + "havoc-component-group": "mygroup" + } + } + }, + { + "metadata": { + "name": "pod-2", + "labels": { + "havoc-component-group": "mygroup" + } + } + } + ] +} diff --git a/havoc/testdata/deployments/deployment_single_pod.json b/havoc/testdata/deployments/deployment_single_pod.json new file mode 100755 index 000000000..90df90fc8 --- /dev/null +++ b/havoc/testdata/deployments/deployment_single_pod.json @@ -0,0 +1,12 @@ +{ + "items": [ + { + "metadata": { + "name": "my-single-app", + "labels": { + "app": "app" + } + } + } + ] +} diff --git a/havoc/testdata/openapi_specs/petshop.yaml b/havoc/testdata/openapi_specs/petshop.yaml new file mode 100644 index 000000000..b01683dd6 --- /dev/null +++ b/havoc/testdata/openapi_specs/petshop.yaml @@ -0,0 +1,119 @@ +openapi: '3.0.0' +info: + version: 1.0.0 + title: Swagger Petstore + license: + name: MIT +servers: + - url: http://petstore.swagger.io/v1 +paths: + /pets: + get: + summary: List all pets + operationId: listPets + tags: + - pets + parameters: + - name: limit + in: query + description: How many items to return at one time (max 100) + required: false + schema: + type: integer + maximum: 100 + format: int32 + responses: + '200': + description: A paged array of pets + headers: + x-next: + description: A link to the next page of responses + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/Pets' + default: + description: unexpected error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + post: + summary: Create a pet + operationId: createPets + tags: + - pets + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Pet' + required: true + responses: + '201': + description: Null response + default: + description: unexpected error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /pets/{petId}: + get: + summary: Info for a specific pet + operationId: showPetById + tags: + - pets + parameters: + - name: petId + in: path + required: true + description: The id of the pet to retrieve + schema: + type: string + responses: + '200': + description: Expected response to a valid request + content: + application/json: + schema: + $ref: '#/components/schemas/Pet' + default: + description: unexpected error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' +components: + schemas: + Pet: + type: object + required: + - id + - name + properties: + id: + type: integer + format: int64 + name: + type: string + tag: + type: string + Pets: + type: array + maxItems: 100 + items: + $ref: '#/components/schemas/Pet' + Error: + type: object + required: + - code + - message + properties: + code: + type: integer + format: int32 + message: + type: string diff --git a/havoc/testdata/results/.gitkeep b/havoc/testdata/results/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-10.yaml b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-10.yaml new file mode 100755 index 000000000..f40a20f49 --- /dev/null +++ b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-10.yaml @@ -0,0 +1,9 @@ +kind: blockchain_rewind_head +name: blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-10 +metadata: + name: blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-10 +podName: geth-1337-7f7c9fb6c6-hzdhn +executorContainerName: geth-network +nodeInternalHTTPURL: geth-1337:8544 +namespace: cl-cluster +blocks: 10 diff --git a/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-20.yaml b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-20.yaml new file mode 100755 index 000000000..8b1b36c57 --- /dev/null +++ b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-20.yaml @@ -0,0 +1,9 @@ +kind: blockchain_rewind_head +name: blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-20 +metadata: + name: blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-20 +podName: geth-1337-7f7c9fb6c6-hzdhn +executorContainerName: geth-network +nodeInternalHTTPURL: geth-1337:8544 +namespace: cl-cluster +blocks: 20 diff --git a/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-30.yaml b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-30.yaml new file mode 100755 index 000000000..6b23d4632 --- /dev/null +++ b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-30.yaml @@ -0,0 +1,9 @@ +kind: blockchain_rewind_head +name: blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-30 +metadata: + name: blockchain_rewind_head-geth-1337-7f7c9fb6c6-hzdhn-30 +podName: geth-1337-7f7c9fb6c6-hzdhn +executorContainerName: geth-network +nodeInternalHTTPURL: geth-1337:8544 +namespace: cl-cluster +blocks: 30 diff --git a/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-10.yaml b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-10.yaml new file mode 100755 index 000000000..7fa36ae49 --- /dev/null +++ b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-10.yaml @@ -0,0 +1,9 @@ +kind: blockchain_rewind_head +name: blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-10 +metadata: + name: blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-10 +podName: geth-2337-7f7c9fb6c6-hzdhn +executorContainerName: geth-network +nodeInternalHTTPURL: geth-2337:8544 +namespace: cl-cluster +blocks: 10 diff --git a/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-20.yaml b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-20.yaml new file mode 100755 index 000000000..fa2046953 --- /dev/null +++ b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-20.yaml @@ -0,0 +1,9 @@ +kind: blockchain_rewind_head +name: blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-20 +metadata: + name: blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-20 +podName: geth-2337-7f7c9fb6c6-hzdhn +executorContainerName: geth-network +nodeInternalHTTPURL: geth-2337:8544 +namespace: cl-cluster +blocks: 20 diff --git a/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-30.yaml b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-30.yaml new file mode 100755 index 000000000..0da844330 --- /dev/null +++ b/havoc/testdata/snapshot/all/blockchain_rewind_head/blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-30.yaml @@ -0,0 +1,9 @@ +kind: blockchain_rewind_head +name: blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-30 +metadata: + name: blockchain_rewind_head-geth-2337-7f7c9fb6c6-hzdhn-30 +podName: geth-2337-7f7c9fb6c6-hzdhn +executorContainerName: geth-network +nodeInternalHTTPURL: geth-2337:8544 +namespace: cl-cluster +blocks: 30 diff --git a/havoc/testdata/snapshot/all/cpu/cpu-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml b/havoc/testdata/snapshot/all/cpu/cpu-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml new file mode 100755 index 000000000..54c68929c --- /dev/null +++ b/havoc/testdata/snapshot/all/cpu/cpu-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml @@ -0,0 +1,15 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: cpu-app-node-1-bootstrap-5b47fb4dbc-msbzz + namespace: cl-cluster +spec: + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: app-node-1-bootstrap-5b47fb4dbc-msbzz + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/cpu/cpu-mockserver-7cb865999c-qwdt9.yaml b/havoc/testdata/snapshot/all/cpu/cpu-mockserver-7cb865999c-qwdt9.yaml new file mode 100755 index 000000000..7bcd33890 --- /dev/null +++ b/havoc/testdata/snapshot/all/cpu/cpu-mockserver-7cb865999c-qwdt9.yaml @@ -0,0 +1,15 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: cpu-mockserver-7cb865999c-qwdt9 + namespace: cl-cluster +spec: + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: mockserver-7cb865999c-qwdt9 + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/cpu/cpu-runner-64c589dd4b-qh4lj.yaml b/havoc/testdata/snapshot/all/cpu/cpu-runner-64c589dd4b-qh4lj.yaml new file mode 100755 index 000000000..bd8bccfca --- /dev/null +++ b/havoc/testdata/snapshot/all/cpu/cpu-runner-64c589dd4b-qh4lj.yaml @@ -0,0 +1,15 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: cpu-runner-64c589dd4b-qh4lj + namespace: cl-cluster +spec: + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: runner-64c589dd4b-qh4lj + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/external/external-cl-cluster-0a137b375cc3881a70e186ce2172c8d1.yaml b/havoc/testdata/snapshot/all/external/external-cl-cluster-0a137b375cc3881a70e186ce2172c8d1.yaml new file mode 100755 index 000000000..c2d326f79 --- /dev/null +++ b/havoc/testdata/snapshot/all/external/external-cl-cluster-0a137b375cc3881a70e186ce2172c8d1.yaml @@ -0,0 +1,20 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: external-cl-cluster-0a137b375cc3881a70e186ce2172c8d1 + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + mode: all + action: partition + duration: 10s + direction: to + target: + selector: + namespaces: + - cl-cluster + mode: all + externalTargets: + - 'www.google.com' diff --git a/havoc/testdata/snapshot/all/failure/failure-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml b/havoc/testdata/snapshot/all/failure/failure-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml new file mode 100755 index 000000000..4277a7821 --- /dev/null +++ b/havoc/testdata/snapshot/all/failure/failure-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml @@ -0,0 +1,12 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: failure-app-node-1-bootstrap-5b47fb4dbc-msbzz + namespace: cl-cluster +spec: + action: pod-failure + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: app-node-1-bootstrap-5b47fb4dbc-msbzz diff --git a/havoc/testdata/snapshot/all/failure/failure-mockserver-7cb865999c-qwdt9.yaml b/havoc/testdata/snapshot/all/failure/failure-mockserver-7cb865999c-qwdt9.yaml new file mode 100755 index 000000000..193905cec --- /dev/null +++ b/havoc/testdata/snapshot/all/failure/failure-mockserver-7cb865999c-qwdt9.yaml @@ -0,0 +1,12 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: failure-mockserver-7cb865999c-qwdt9 + namespace: cl-cluster +spec: + action: pod-failure + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: mockserver-7cb865999c-qwdt9 diff --git a/havoc/testdata/snapshot/all/failure/failure-runner-64c589dd4b-qh4lj.yaml b/havoc/testdata/snapshot/all/failure/failure-runner-64c589dd4b-qh4lj.yaml new file mode 100755 index 000000000..1b14d7a63 --- /dev/null +++ b/havoc/testdata/snapshot/all/failure/failure-runner-64c589dd4b-qh4lj.yaml @@ -0,0 +1,12 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: failure-runner-64c589dd4b-qh4lj + namespace: cl-cluster +spec: + action: pod-failure + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: runner-64c589dd4b-qh4lj diff --git a/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-1-fixed.yaml b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-1-fixed.yaml new file mode 100755 index 000000000..6f19dace3 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-1-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-blockchain-1-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '1' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-2-fixed.yaml b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-2-fixed.yaml new file mode 100755 index 000000000..dda864472 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-2-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-blockchain-2-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '2' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-3-fixed.yaml b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-3-fixed.yaml new file mode 100755 index 000000000..672eddbb6 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-blockchain-3-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-blockchain-3-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '3' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-1-fixed.yaml b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-1-fixed.yaml new file mode 100755 index 000000000..8d829a849 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-1-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-node-1-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '1' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-2-fixed.yaml b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-2-fixed.yaml new file mode 100755 index 000000000..c5a513aa8 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-2-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-node-2-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '2' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-3-fixed.yaml b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-3-fixed.yaml new file mode 100755 index 000000000..ddfde5ea7 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-cpu/group-cpu-havoc-component-group-node-3-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-node-3-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '3' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-1-fixed.yaml b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-1-fixed.yaml new file mode 100755 index 000000000..a0bf4b7bc --- /dev/null +++ b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-1-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-blockchain-1-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '1' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' diff --git a/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-2-fixed.yaml b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-2-fixed.yaml new file mode 100755 index 000000000..069489995 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-2-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-blockchain-2-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '2' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' diff --git a/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-3-fixed.yaml b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-3-fixed.yaml new file mode 100755 index 000000000..d9d783550 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-blockchain-3-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-blockchain-3-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '3' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' diff --git a/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-1-fixed.yaml b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-1-fixed.yaml new file mode 100755 index 000000000..374d31385 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-1-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-node-1-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '1' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' diff --git a/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-2-fixed.yaml b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-2-fixed.yaml new file mode 100755 index 000000000..be3310deb --- /dev/null +++ b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-2-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-node-2-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '2' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' diff --git a/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-3-fixed.yaml b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-3-fixed.yaml new file mode 100755 index 000000000..e1242efcf --- /dev/null +++ b/havoc/testdata/snapshot/all/group-failure/group-failure-havoc-component-group-node-3-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-node-3-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '3' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' diff --git a/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-1-fixed.yaml b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-1-fixed.yaml new file mode 100755 index 000000000..3ec71dcdf --- /dev/null +++ b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-1-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-blockchain-1-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'blockchain' + mode: fixed + value: '1' + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'blockchain' + mode: fixed + value: '1' diff --git a/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-2-fixed.yaml b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-2-fixed.yaml new file mode 100755 index 000000000..c3bac3f3d --- /dev/null +++ b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-2-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-blockchain-2-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'blockchain' + mode: fixed + value: '2' + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'blockchain' + mode: fixed + value: '2' diff --git a/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-3-fixed.yaml b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-3-fixed.yaml new file mode 100755 index 000000000..667dee522 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-blockchain-3-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-blockchain-3-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'blockchain' + mode: fixed + value: '3' + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'blockchain' + mode: fixed + value: '3' diff --git a/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-1-fixed.yaml b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-1-fixed.yaml new file mode 100755 index 000000000..46ad46dc2 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-1-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-node-1-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'node' + mode: fixed + value: '1' + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'node' + mode: fixed + value: '1' diff --git a/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-2-fixed.yaml b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-2-fixed.yaml new file mode 100755 index 000000000..3fee88064 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-2-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-node-2-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'node' + mode: fixed + value: '2' + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'node' + mode: fixed + value: '2' diff --git a/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-3-fixed.yaml b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-3-fixed.yaml new file mode 100755 index 000000000..e6e2d3fed --- /dev/null +++ b/havoc/testdata/snapshot/all/group-latency/group-latency-havoc-component-group-node-3-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-node-3-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'node' + mode: fixed + value: '3' + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'node' + mode: fixed + value: '3' diff --git a/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-1-fixed.yaml b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-1-fixed.yaml new file mode 100755 index 000000000..7258e81cc --- /dev/null +++ b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-1-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-blockchain-1-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '1' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-2-fixed.yaml b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-2-fixed.yaml new file mode 100755 index 000000000..7224938f4 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-2-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-blockchain-2-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '2' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-3-fixed.yaml b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-3-fixed.yaml new file mode 100755 index 000000000..f1b20f768 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-blockchain-3-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-blockchain-3-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '3' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'blockchain' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-1-fixed.yaml b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-1-fixed.yaml new file mode 100755 index 000000000..f524039b5 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-1-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-node-1-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '1' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-2-fixed.yaml b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-2-fixed.yaml new file mode 100755 index 000000000..59a5e3ec2 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-2-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-node-2-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '2' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-3-fixed.yaml b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-3-fixed.yaml new file mode 100755 index 000000000..fc4cbd4b9 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-memory/group-memory-havoc-component-group-node-3-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-node-3-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '3' + duration: 10s + selector: + labelSelectors: + 'havoc-component-group': 'node' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-1-to-havoc-network-group-2-100-perc.yaml b/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-1-to-havoc-network-group-2-100-perc.yaml new file mode 100755 index 000000000..80e7f2d80 --- /dev/null +++ b/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-1-to-havoc-network-group-2-100-perc.yaml @@ -0,0 +1,24 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-partition-havoc-network-group-1-to-havoc-network-group-2-100-perc + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-network-group': '1' + action: partition + mode: fixed-percent + value: '100' + duration: 30s + direction: from + target: + mode: fixed-percent + value: '100' + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-network-group': '2' diff --git a/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-1-to-havoc-network-group-blockchain-100-perc.yaml b/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-1-to-havoc-network-group-blockchain-100-perc.yaml new file mode 100755 index 000000000..90aa573aa --- /dev/null +++ b/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-1-to-havoc-network-group-blockchain-100-perc.yaml @@ -0,0 +1,24 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-partition-havoc-network-group-1-to-havoc-network-group-blockchain-100-perc + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-network-group': '1' + action: partition + mode: fixed-percent + value: '100' + duration: 30s + direction: from + target: + mode: fixed-percent + value: '100' + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-network-group': 'blockchain' diff --git a/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-2-to-havoc-network-group-blockchain-100-perc.yaml b/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-2-to-havoc-network-group-blockchain-100-perc.yaml new file mode 100755 index 000000000..b5698d0ae --- /dev/null +++ b/havoc/testdata/snapshot/all/group-partition/group-partition-havoc-network-group-2-to-havoc-network-group-blockchain-100-perc.yaml @@ -0,0 +1,24 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-partition-havoc-network-group-2-to-havoc-network-group-blockchain-100-perc + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-network-group': '2' + action: partition + mode: fixed-percent + value: '100' + duration: 30s + direction: from + target: + mode: fixed-percent + value: '100' + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-network-group': 'blockchain' diff --git a/havoc/testdata/snapshot/all/latency/latency-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml b/havoc/testdata/snapshot/all/latency/latency-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml new file mode 100755 index 000000000..070ea19a0 --- /dev/null +++ b/havoc/testdata/snapshot/all/latency/latency-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml @@ -0,0 +1,24 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: latency-app-node-1-bootstrap-5b47fb4dbc-msbzz + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + fieldSelectors: + metadata.name: app-node-1-bootstrap-5b47fb4dbc-msbzz + mode: one + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + fieldSelectors: + metadata.name: app-node-1-bootstrap-5b47fb4dbc-msbzz + mode: one diff --git a/havoc/testdata/snapshot/all/latency/latency-mockserver-7cb865999c-qwdt9.yaml b/havoc/testdata/snapshot/all/latency/latency-mockserver-7cb865999c-qwdt9.yaml new file mode 100755 index 000000000..1e9b31c76 --- /dev/null +++ b/havoc/testdata/snapshot/all/latency/latency-mockserver-7cb865999c-qwdt9.yaml @@ -0,0 +1,24 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: latency-mockserver-7cb865999c-qwdt9 + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + fieldSelectors: + metadata.name: mockserver-7cb865999c-qwdt9 + mode: one + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + fieldSelectors: + metadata.name: mockserver-7cb865999c-qwdt9 + mode: one diff --git a/havoc/testdata/snapshot/all/latency/latency-runner-64c589dd4b-qh4lj.yaml b/havoc/testdata/snapshot/all/latency/latency-runner-64c589dd4b-qh4lj.yaml new file mode 100755 index 000000000..b5d5525b4 --- /dev/null +++ b/havoc/testdata/snapshot/all/latency/latency-runner-64c589dd4b-qh4lj.yaml @@ -0,0 +1,24 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: latency-runner-64c589dd4b-qh4lj + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + fieldSelectors: + metadata.name: runner-64c589dd4b-qh4lj + mode: one + action: delay + duration: 10s + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + fieldSelectors: + metadata.name: runner-64c589dd4b-qh4lj + mode: one diff --git a/havoc/testdata/snapshot/all/memory/memory-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml b/havoc/testdata/snapshot/all/memory/memory-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml new file mode 100755 index 000000000..fed593cdd --- /dev/null +++ b/havoc/testdata/snapshot/all/memory/memory-app-node-1-bootstrap-5b47fb4dbc-msbzz.yaml @@ -0,0 +1,15 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: memory-app-node-1-bootstrap-5b47fb4dbc-msbzz + namespace: cl-cluster +spec: + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: app-node-1-bootstrap-5b47fb4dbc-msbzz + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/all/memory/memory-mockserver-7cb865999c-qwdt9.yaml b/havoc/testdata/snapshot/all/memory/memory-mockserver-7cb865999c-qwdt9.yaml new file mode 100755 index 000000000..4c4c6fa26 --- /dev/null +++ b/havoc/testdata/snapshot/all/memory/memory-mockserver-7cb865999c-qwdt9.yaml @@ -0,0 +1,15 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: memory-mockserver-7cb865999c-qwdt9 + namespace: cl-cluster +spec: + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: mockserver-7cb865999c-qwdt9 + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/all/memory/memory-runner-64c589dd4b-qh4lj.yaml b/havoc/testdata/snapshot/all/memory/memory-runner-64c589dd4b-qh4lj.yaml new file mode 100755 index 000000000..af7940d7a --- /dev/null +++ b/havoc/testdata/snapshot/all/memory/memory-runner-64c589dd4b-qh4lj.yaml @@ -0,0 +1,15 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: memory-runner-64c589dd4b-qh4lj + namespace: cl-cluster +spec: + mode: one + duration: 10s + selector: + fieldSelectors: + metadata.name: runner-64c589dd4b-qh4lj + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-1-fixed.yaml b/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-1-fixed.yaml new file mode 100755 index 000000000..48239cd35 --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-1-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-mygroup-1-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '1' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-2-fixed.yaml b/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-2-fixed.yaml new file mode 100755 index 000000000..7f4a4e0d9 --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-2-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-mygroup-2-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '2' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-3-fixed.yaml b/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-3-fixed.yaml new file mode 100755 index 000000000..86754957d --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-cpu/group-cpu-havoc-component-group-mygroup-3-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-cpu-havoc-component-group-mygroup-3-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '3' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-1-fixed.yaml b/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-1-fixed.yaml new file mode 100755 index 000000000..5eca317b5 --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-1-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-mygroup-1-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '1' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' diff --git a/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-2-fixed.yaml b/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-2-fixed.yaml new file mode 100755 index 000000000..0e3a9d4f1 --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-2-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-mygroup-2-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '2' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' diff --git a/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-3-fixed.yaml b/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-3-fixed.yaml new file mode 100755 index 000000000..58a8dc3e1 --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-failure/group-failure-havoc-component-group-mygroup-3-fixed.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: group-failure-havoc-component-group-mygroup-3-fixed + namespace: cl-cluster +spec: + action: pod-failure + mode: fixed + value: '3' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' diff --git a/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-1-fixed.yaml b/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-1-fixed.yaml new file mode 100755 index 000000000..12acb9541 --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-1-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-mygroup-1-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'mygroup' + mode: fixed + value: '1' + action: delay + duration: 1m + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'mygroup' + mode: fixed + value: '1' diff --git a/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-2-fixed.yaml b/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-2-fixed.yaml new file mode 100755 index 000000000..fd59b1d53 --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-2-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-mygroup-2-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'mygroup' + mode: fixed + value: '2' + action: delay + duration: 1m + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'mygroup' + mode: fixed + value: '2' diff --git a/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-3-fixed.yaml b/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-3-fixed.yaml new file mode 100755 index 000000000..985c9280b --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-latency/group-latency-havoc-component-group-mygroup-3-fixed.yaml @@ -0,0 +1,26 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: group-latency-havoc-component-group-mygroup-3-fixed + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'mygroup' + mode: fixed + value: '3' + action: delay + duration: 1m + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + labelSelectors: + 'havoc-component-group': 'mygroup' + mode: fixed + value: '3' diff --git a/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-1-fixed.yaml b/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-1-fixed.yaml new file mode 100755 index 000000000..4950c703c --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-1-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-mygroup-1-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '1' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-2-fixed.yaml b/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-2-fixed.yaml new file mode 100755 index 000000000..f1ada38be --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-2-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-mygroup-2-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '2' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-3-fixed.yaml b/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-3-fixed.yaml new file mode 100755 index 000000000..5fe9602f5 --- /dev/null +++ b/havoc/testdata/snapshot/single_group/group-memory/group-memory-havoc-component-group-mygroup-3-fixed.yaml @@ -0,0 +1,16 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: group-memory-havoc-component-group-mygroup-3-fixed + namespace: cl-cluster +spec: + mode: fixed + value: '3' + duration: 1m + selector: + labelSelectors: + 'havoc-component-group': 'mygroup' + stressors: + memory: + workers: 1 + size: 512MB diff --git a/havoc/testdata/snapshot/single_pod/cpu/cpu-my-single-app.yaml b/havoc/testdata/snapshot/single_pod/cpu/cpu-my-single-app.yaml new file mode 100755 index 000000000..8acde5612 --- /dev/null +++ b/havoc/testdata/snapshot/single_pod/cpu/cpu-my-single-app.yaml @@ -0,0 +1,15 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: cpu-my-single-app + namespace: cl-cluster +spec: + mode: one + duration: 1m + selector: + fieldSelectors: + metadata.name: my-single-app + stressors: + cpu: + workers: 1 + load: 100 diff --git a/havoc/testdata/snapshot/single_pod/failure/failure-my-single-app.yaml b/havoc/testdata/snapshot/single_pod/failure/failure-my-single-app.yaml new file mode 100755 index 000000000..30cb81596 --- /dev/null +++ b/havoc/testdata/snapshot/single_pod/failure/failure-my-single-app.yaml @@ -0,0 +1,12 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: failure-my-single-app + namespace: cl-cluster +spec: + action: pod-failure + mode: one + duration: 1m + selector: + fieldSelectors: + metadata.name: my-single-app diff --git a/havoc/testdata/snapshot/single_pod/latency/latency-my-single-app.yaml b/havoc/testdata/snapshot/single_pod/latency/latency-my-single-app.yaml new file mode 100755 index 000000000..f197dbf62 --- /dev/null +++ b/havoc/testdata/snapshot/single_pod/latency/latency-my-single-app.yaml @@ -0,0 +1,24 @@ +kind: NetworkChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + name: latency-my-single-app + namespace: cl-cluster +spec: + selector: + namespaces: + - cl-cluster + fieldSelectors: + metadata.name: my-single-app + mode: one + action: delay + duration: 1m + delay: + latency: 300ms + direction: from + target: + selector: + namespaces: + - cl-cluster + fieldSelectors: + metadata.name: my-single-app + mode: one diff --git a/havoc/testdata/snapshot/single_pod/memory/memory-my-single-app.yaml b/havoc/testdata/snapshot/single_pod/memory/memory-my-single-app.yaml new file mode 100755 index 000000000..c842fe487 --- /dev/null +++ b/havoc/testdata/snapshot/single_pod/memory/memory-my-single-app.yaml @@ -0,0 +1,15 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: StressChaos +metadata: + name: memory-my-single-app + namespace: cl-cluster +spec: + mode: one + duration: 1m + selector: + fieldSelectors: + metadata.name: my-single-app + stressors: + memory: + workers: 1 + size: 512MB