diff --git a/.prettierignore b/.prettierignore index 5c08ba825..5c8b62609 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,3 +7,4 @@ k8s-test-runner/chart/**/*.yaml node_modules/ index.yaml wasp/** +havoc/** diff --git a/README.md b/README.md index 2616bedb7..4ab7f4af5 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ If you're looking to implement a new chain integration for the testing framework CTF contains a set of useful libraries: - [WASP](wasp/README.md) - Scalable protocol-agnostic load testing library for `Go` +- [Havoc](havoc/README.md) - Chaos testing library ## k8s package diff --git a/havoc/.gitignore b/havoc/.gitignore new file mode 100644 index 000000000..4a95cdd95 --- /dev/null +++ b/havoc/.gitignore @@ -0,0 +1,38 @@ +# IDE and environment +.idea/ +.vscode/ +.DS_STORE + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories +dist/ +vendor/ +node_modules/ +.yarn/ + +# Mercuy server configuration file +config.toml +config.*.toml + +# Other +tmp/ +*.log +*.swp +.air.toml +.DS_Store +output.txt + +# General env vars config +.envrc diff --git a/havoc/README.md b/havoc/README.md new file mode 100644 index 000000000..0a0aaff3c --- /dev/null +++ b/havoc/README.md @@ -0,0 +1,211 @@ +## Havoc + +The `havoc` package is a Go library designed to facilitate chaos testing within Kubernetes environments using Chaos Mesh. It offers a structured way to define, execute, and manage chaos experiments as code, directly integrated into Go applications or testing suites. This package simplifies the creation and control of Chaos Mesh experiments, including network chaos, pod failures, and stress testing on Kubernetes clusters. + +### Features + +- **Chaos Object Management:** Easily create, update, pause, resume, and delete chaos experiments using Go structures and methods. +- **Lifecycle Hooks:** Utilize chaos listeners to hook into lifecycle events of chaos experiments, such as creation, start, pause, resume, and finish. +- **Support for Various Chaos Experiments:** Create and manage different types of chaos experiments like NetworkChaos, IOChaos, StressChaos, PodChaos, and HTTPChaos. +- **Chaos Experiment Status Monitoring:** Monitor and react to the status of chaos experiments programmatically. + +### Installation + +To use `havoc` in your project, ensure you have a Go environment setup. Then, install the package using go get: + +``` +go get -u github.com/smartcontractkit/chainlink-testing-framework/havoc +``` + +Ensure your Kubernetes cluster is accessible and that you have Chaos Mesh installed and configured. + +### Monitoring and Observability in Chaos Experiments + +`havoc` enhances chaos experiment observability through structured logging and Grafana annotations, facilitated by implementing the ChaosListener interface. This approach allows for detailed monitoring, debugging, and visual representation of chaos experiments' impact. + +#### Structured Logging with ChaosLogger + +`ChaosLogger` leverages the zerolog library to provide structured, queryable logging of chaos events. It automatically logs key lifecycle events such as creation, start, pause, and termination of chaos experiments, including detailed contextual information. + +Instantiate `ChaosLogger` and register it as a listener to your chaos experiments: + +``` +logger := havoc.NewChaosLogger() +chaos.AddListener(logger) +``` + +### Default package logger + +`havoc/logger.go` contains default `Logger` instance for the package. + +#### Visual Monitoring with Grafana Annotations + +`SingleLineGrafanaAnnotator` is a `ChaosListener` that annotates Grafana dashboards with chaos experiment events. This visual representation helps correlate chaos events with their effects on system metrics and logs. + +Initialize `SingleLineGrafanaAnnotator` with your Grafana instance details and register it alongside `ChaosLogger`: + +``` +annotator := havoc.NewSingleLineGrafanaAnnotator( + "http://grafana-instance.com", + "grafana-access-token", + "dashboard-uid", +) +chaos.AddListener(annotator) +``` + +### Creating a Chaos Experiment + +To create a chaos experiment, define the chaos object options, initialize a chaos experiment with NewChaos, and then call Create to start the experiment. + +Here is an example of creating and starting a PodChaos experiment: + +``` +package main + +import ( + "context" + "github.com/smartcontractkit/chainlink-testing-framework/havoc" + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" +) + +func main() { + // Initialize dependencies + client, err := havoc.NewChaosMeshClient() + if err != nil { + panic(err) + } + logger := havoc.NewChaosLogger() + annotator := havoc.NewSingleLineGrafanaAnnotator( + "http://grafana-instance.com", + "grafana-access-token", + "dashboard-uid", + ) + + // Define chaos experiment + podChaos := &v1alpha1.PodChaos{ /* PodChaos spec */ } + chaos, err := havoc.NewChaos(havoc.ChaosOpts{ + Object: podChaos, + Description: "Pod failure example", + DelayCreate: 5 * time.Second, + Client: client, + }) + if err != nil { + panic(err) + } + + // Register listeners + chaos.AddListener(logger) + chaos.AddListener(annotator) + + // Start chaos experiment + chaos.Create(context.Background()) + + // Manage chaos lifecycle... +} +``` + +### Test Example + +``` +func TestChaosDON(t *testing.T) { + testDuration := time.Minute * 60 + + // Load test config + cfg := &config.MercuryQAEnvChaos{} + + // Define chaos experiments and their schedule + + k8sClient, err := havoc.NewChaosMeshClient() + require.NoError(t, err) + + // Test 3.2: Disable 2 nodes simultaneously + + podFailureChaos4, err := k8s_chaos.MercuryPodChaosSchedule(k8s_chaos.MercuryScheduledPodChaosOpts{ + Name: "schedule-don-ocr-node-failure-4", + Description: "Disable 2 nodes (clc-ocr-mercury-arb-testnet-qa-nodes-3 and clc-ocr-mercury-arb-testnet-qa-nodes-4)", + DelayCreate: time.Minute * 0, + Duration: time.Minute * 20, + Namespace: cfg.ChaosNodeNamespace, + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.ChaosNodeNamespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Key: "app.kubernetes.io/instance", + Operator: "In", + Values: []string{ + "clc-ocr-mercury-arb-testnet-qa-nodes-3", + "clc-ocr-mercury-arb-testnet-qa-nodes-4", + }, + }, + }, + }, + }, + }, + Client: k8sClient, + }) + require.NoError(t, err) + + // Test 3.3: Disable 3 nodes simultaneously + + podFailureChaos5, err := k8s_chaos.MercuryPodChaosSchedule(k8s_chaos.MercuryScheduledPodChaosOpts{ + Name: "schedule-don-ocr-node-failure-5", + Description: "Disable 3 nodes (clc-ocr-mercury-arb-testnet-qa-nodes-3, clc-ocr-mercury-arb-testnet-qa-nodes-4 and clc-ocr-mercury-arb-testnet-qa-nodes-5)", + DelayCreate: time.Minute * 40, + Duration: time.Minute * 20, + Namespace: cfg.ChaosNodeNamespace, + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.ChaosNodeNamespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Key: "app.kubernetes.io/instance", + Operator: "In", + Values: []string{ + "clc-ocr-mercury-arb-testnet-qa-nodes-3", + "clc-ocr-mercury-arb-testnet-qa-nodes-4", + "clc-ocr-mercury-arb-testnet-qa-nodes-5", + }, + }, + }, + }, + }, + }, + Client: k8sClient, + }) + require.NoError(t, err) + + chaosList := []havoc.ChaosEntity{ + podFailureChaos4, + podFailureChaos5, + } + + for _, chaos := range chaosList { + chaos.AddListener(havoc.NewChaosLogger()) + chaos.AddListener(havoc.NewSingleLineGrafanaAnnotator(cfg.GrafanaURL, cfg.GrafanaToken, cfg.GrafanaDashboardUID)) + + // Fail the test if the chaos object already exists + exists, err := havoc.ChaosObjectExists(chaos.GetObject(), k8sClient) + require.NoError(t, err) + require.False(t, exists, "chaos object already exists: %s. Delete it before starting the test", chaos.GetChaosName()) + + chaos.Create(context.Background()) + } + + t.Cleanup(func() { + for _, chaos := range chaosList { + // Delete chaos object if it still exists + chaos.Delete(context.Background()) + } + }) + + // Simulate user activity/load for the duration of the chaos experiments + runUserLoad(t, cfg, testDuration) +} +``` diff --git a/havoc/chaos.go b/havoc/chaos.go new file mode 100644 index 000000000..1b876684a --- /dev/null +++ b/havoc/chaos.go @@ -0,0 +1,622 @@ +package havoc + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/pkg/errors" + "github.com/rs/zerolog" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Chaos struct { + Object client.Object + Description string + DelayCreate time.Duration // Delay before creating the chaos object + Status ChaosStatus + Client client.Client + listeners []ChaosListener + cancelMonitor context.CancelFunc + startTime time.Time + endTime time.Time + logger *zerolog.Logger +} + +// ChaosStatus represents the status of a chaos experiment. +type ChaosStatus string + +// These constants define possible states of a chaos experiment. +const ( + StatusCreated ChaosStatus = "created" + StatusCreationFailed ChaosStatus = "creation_failed" + StatusRunning ChaosStatus = "running" + StatusPaused ChaosStatus = "paused" + StatusFinished ChaosStatus = "finished" + StatusDeleted ChaosStatus = "deleted" + StatusUnknown ChaosStatus = "unknown" // For any state that doesn't match the above +) + +type ChaosOpts struct { + Object client.Object + Description string + DelayCreate time.Duration + Client client.Client + Listeners []ChaosListener + Logger *zerolog.Logger +} + +func NewChaos(opts ChaosOpts) (*Chaos, error) { + if opts.Client == nil { + return nil, errors.New("client is required") + } + if opts.Object == nil { + return nil, errors.New("chaos object is required") + } + if opts.Logger == nil { + return nil, errors.New("logger is required") + } + + return &Chaos{ + Object: opts.Object, + Description: opts.Description, + DelayCreate: opts.DelayCreate, + Client: opts.Client, + listeners: opts.Listeners, + logger: opts.Logger, + }, nil +} + +// Create initiates a delayed creation of a chaos object, respecting context cancellation and deletion requests. +// It uses a timer based on `DelayCreate` and calls `create` method upon expiration unless preempted by deletion. +func (c *Chaos) Create(ctx context.Context) { + done := make(chan struct{}) + + // Create the timer with the delay to create the chaos object + timer := time.NewTimer(c.DelayCreate) + + go func() { + select { + case <-ctx.Done(): + // If the context is canceled, stop the timer and exit + if !timer.Stop() { + <-timer.C // If the timer already expired, drain the channel + } + close(done) // Signal that the operation was canceled + case <-timer.C: + // Timer expired, check if deletion was not requested + if c.Status != StatusDeleted { + c.createNow(ctx) + } + close(done) // Signal that the creation process is either done or skipped + } + }() +} + +func (c *Chaos) Update(ctx context.Context) error { + // Modify the resource + // For example, adding or updating an annotation + annotations := c.Object.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations["example.com/trigger-injection"] = "true" + c.Object.SetAnnotations(annotations) + + //nolint + if err := c.Client.Update(ctx, c.Object); err != nil { + return errors.Wrap(err, "failed to update chaos object") + } + + return nil +} + +// createNow is a private method that encapsulates the chaos object creation logic. +func (c *Chaos) createNow(ctx context.Context) { + if err := c.Client.Create(ctx, c.Object); err != nil { + c.notifyListeners(string(StatusCreationFailed), err) + return + } + c.notifyListeners(string(StatusCreated), nil) + + // Create a cancellable context for monitorStatus + monitorCtx, cancel := context.WithCancel(ctx) + c.cancelMonitor = cancel + go c.monitorStatus(monitorCtx) +} + +func (c *Chaos) Pause(ctx context.Context) error { + err := c.updateChaosObject(ctx) + if err != nil { + return errors.Wrap(err, "could not update the chaos object") + } + + annotations := c.Object.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[v1alpha1.PauseAnnotationKey] = strconv.FormatBool(true) + c.Object.SetAnnotations(annotations) + + err = c.Client.Update(ctx, c.Object) + if err != nil { + return errors.Wrap(err, "could not update the annotation to set the chaos experiment into pause state") + } + + c.notifyListeners("paused", nil) + return nil +} + +func (c *Chaos) Resume(ctx context.Context) error { + // Implement resume logic here + c.notifyListeners("resumed", nil) + return nil +} + +func (c *Chaos) Delete(ctx context.Context) error { + // Cancel the monitoring goroutine + if c.cancelMonitor != nil { + c.cancelMonitor() + } + + // If the chaos was running or paused, update the status and notify listeners + if c.Status == StatusPaused || c.Status == StatusRunning { + err := c.updateChaosObject(ctx) + if err != nil { + return errors.Wrap(err, "could not update the chaos object") + } + c.Status = StatusFinished + c.endTime = time.Now() + c.notifyListeners("finished", nil) + } + + if err := c.Client.Delete(ctx, c.Object); err != nil { + return errors.Wrap(err, "failed to delete chaos object") + } + + c.Status = StatusDeleted + + c.logger.Info().Str("name", c.GetChaosName()).Msg("Chaos deleted") + + return nil +} + +func (c *Chaos) GetObject() client.Object { + return c.Object +} + +func (c *Chaos) GetChaosName() string { + return c.Object.GetName() +} + +func (c *Chaos) GetChaosDescription() string { + return c.Description +} + +func (c *Chaos) GetChaosTypeStr() string { + switch c.Object.(type) { + case *v1alpha1.NetworkChaos: + return "NetworkChaos" + case *v1alpha1.IOChaos: + return "IOChaos" + case *v1alpha1.StressChaos: + return "StressChaos" + case *v1alpha1.PodChaos: + return "PodChaos" + case *v1alpha1.HTTPChaos: + return "HTTPChaos" + default: + return "Unknown" + } +} + +func (c *Chaos) GetChaosSpec() interface{} { + switch spec := c.Object.(type) { + case *v1alpha1.NetworkChaos: + return spec.Spec + case *v1alpha1.IOChaos: + return spec.Spec + case *v1alpha1.StressChaos: + return spec.Spec + case *v1alpha1.PodChaos: + return spec.Spec + case *v1alpha1.HTTPChaos: + return spec.Spec + default: + return nil + } +} + +func (c *Chaos) GetChaosDuration() (time.Duration, error) { + var durationStr *string + switch spec := c.Object.(type) { + case *v1alpha1.NetworkChaos: + durationStr = spec.Spec.Duration + case *v1alpha1.IOChaos: + durationStr = spec.Spec.Duration + case *v1alpha1.StressChaos: + durationStr = spec.Spec.Duration + case *v1alpha1.PodChaos: + durationStr = spec.Spec.Duration + case *v1alpha1.HTTPChaos: + durationStr = spec.Spec.Duration + } + + if durationStr == nil { + return time.Duration(0), fmt.Errorf("could not get duration for chaos object: %v", c.Object) + } + duration, err := time.ParseDuration(*durationStr) + if err != nil { + return time.Duration(0), fmt.Errorf("could not parse duration: %w", err) + } + return duration, nil +} + +func (c *Chaos) GetChaosEvents() (*corev1.EventList, error) { + listOpts := []client.ListOption{ + client.InNamespace(c.Object.GetNamespace()), + client.MatchingFields{"involvedObject.name": c.Object.GetName(), "involvedObject.kind": c.GetChaosKind()}, + } + events := &corev1.EventList{} + if err := c.Client.List(context.Background(), events, listOpts...); err != nil { + return nil, fmt.Errorf("could not list chaos events: %w", err) + } + + return events, nil +} + +func (c *Chaos) GetChaosKind() string { + switch c.Object.(type) { + case *v1alpha1.NetworkChaos: + return "NetworkChaos" + case *v1alpha1.IOChaos: + return "IOChaos" + case *v1alpha1.StressChaos: + return "StressChaos" + case *v1alpha1.PodChaos: + return "PodChaos" + case *v1alpha1.HTTPChaos: + return "HTTPChaos" + default: + panic(fmt.Sprintf("could not get chaos kind for object: %v", c.Object)) + } +} + +func (c *Chaos) GetChaosStatus() (*v1alpha1.ChaosStatus, error) { + switch obj := c.Object.(type) { + case *v1alpha1.NetworkChaos: + return obj.GetStatus(), nil + case *v1alpha1.IOChaos: + return obj.GetStatus(), nil + case *v1alpha1.StressChaos: + return obj.GetStatus(), nil + case *v1alpha1.PodChaos: + return obj.GetStatus(), nil + case *v1alpha1.HTTPChaos: + return obj.GetStatus(), nil + default: + return nil, fmt.Errorf("could not get chaos status for %s", c.GetChaosKind()) + } +} + +func (c *Chaos) GetExperimentStatus() (v1alpha1.ExperimentStatus, error) { + switch obj := c.Object.(type) { + case *v1alpha1.NetworkChaos: + return obj.Status.Experiment, nil + case *v1alpha1.IOChaos: + return obj.Status.Experiment, nil + case *v1alpha1.StressChaos: + return obj.Status.Experiment, nil + case *v1alpha1.PodChaos: + return obj.Status.Experiment, nil + case *v1alpha1.HTTPChaos: + return obj.Status.Experiment, nil + default: + return v1alpha1.ExperimentStatus{}, fmt.Errorf("could not experiment status for object: %v", c.Object) + } +} + +func ChaosObjectExists(object client.Object, c client.Client) (bool, error) { + switch obj := object.(type) { + case *v1alpha1.NetworkChaos, *v1alpha1.IOChaos, *v1alpha1.StressChaos, *v1alpha1.PodChaos, *v1alpha1.HTTPChaos, *v1alpha1.Schedule: + err := c.Get(context.Background(), client.ObjectKeyFromObject(obj), obj) + if err != nil { + if client.IgnoreNotFound(err) == nil { + // If the error is NotFound, the object does not exist. + return false, nil + } + // For any other errors, return the error. + return false, err + } + // If there's no error, the object exists. + return true, nil + default: + return false, fmt.Errorf("unsupported chaos object type: %T", obj) + } +} + +func (c *Chaos) updateChaosObject(ctx context.Context) error { + switch obj := c.Object.(type) { + case *v1alpha1.NetworkChaos: + var objOut = &v1alpha1.NetworkChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get network chaos object") + } + c.Object = objOut + case *v1alpha1.IOChaos: + var objOut = &v1alpha1.IOChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get IO chaos object") + } + c.Object = objOut + case *v1alpha1.StressChaos: + var objOut = &v1alpha1.StressChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get stress chaos object") + } + c.Object = objOut + case *v1alpha1.PodChaos: + var objOut = &v1alpha1.PodChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get pod chaos object") + } + c.Object = objOut + case *v1alpha1.HTTPChaos: + var objOut = &v1alpha1.HTTPChaos{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get HTTP chaos object") + } + c.Object = objOut + case *v1alpha1.Schedule: + var objOut = &v1alpha1.Schedule{} + err := c.Client.Get(ctx, client.ObjectKeyFromObject(obj), objOut) + if err != nil { + return errors.Wrap(err, "could not get schedule object") + } + c.Object = objOut + default: + return fmt.Errorf("unsupported chaos object type: %T", obj) + } + + return nil +} + +func isConditionTrue(status *v1alpha1.ChaosStatus, expectedCondition v1alpha1.ChaosCondition) bool { + if status == nil { + return false + } + + for _, condition := range status.Conditions { + if condition.Type == expectedCondition.Type { + return condition.Status == expectedCondition.Status + } + } + return false +} + +func (c *Chaos) AddListener(listener ChaosListener) { + c.listeners = append(c.listeners, listener) +} + +// GetStartTime returns the time when the chaos experiment started +func (c *Chaos) GetStartTime() time.Time { + return c.startTime +} + +// GetEndTime returns the time when the chaos experiment ended +func (c *Chaos) GetEndTime() time.Time { + return c.endTime +} + +// GetExpectedEndTime returns the time when the chaos experiment is expected to end +func (c *Chaos) GetExpectedEndTime() (time.Time, error) { + duration, err := c.GetChaosDuration() + if err != nil { + return time.Time{}, err + } + return c.startTime.Add(duration), nil +} + +type ChaosEventDetails struct { + Event string + Chaos *Chaos + Error error +} + +func (c *Chaos) notifyListeners(event string, err error) { + for _, listener := range c.listeners { + switch event { + case "created": + listener.OnChaosCreated(*c) + case string(StatusCreationFailed): + listener.OnChaosCreationFailed(*c, err) + case "started": + listener.OnChaosStarted(*c) + case "paused": + listener.OnChaosPaused(*c) + case "resumed": + listener.OnChaosStarted(*c) // Assuming "resumed" triggers "started" + case "finished": + listener.OnChaosEnded(*c) + case "unknown": + listener.OnChaosStatusUnknown(*c) + } + } +} + +func (c *Chaos) monitorStatus(ctx context.Context) { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + err := c.updateChaosObject(ctx) + if err != nil { + c.logger.Error().Err(err).Msg("failed to update chaos object") + continue + } + chaosStatus, err := c.GetChaosStatus() + if err != nil { + c.logger.Error().Err(err).Msg("failed to get chaos status") + continue + } + + var currentStatus ChaosStatus + + allRecovered := v1alpha1.ChaosCondition{ + Type: v1alpha1.ConditionAllRecovered, + Status: corev1.ConditionTrue, + } + allInjected := v1alpha1.ChaosCondition{ + Type: v1alpha1.ConditionAllInjected, + Status: corev1.ConditionTrue, + } + selected := v1alpha1.ChaosCondition{ + Type: v1alpha1.ConditionSelected, + Status: corev1.ConditionTrue, + } + paused := v1alpha1.ChaosCondition{ + Type: v1alpha1.ConditionPaused, + Status: corev1.ConditionTrue, + } + + if isConditionTrue(chaosStatus, selected) && isConditionTrue(chaosStatus, allInjected) { + currentStatus = StatusRunning + } else if isConditionTrue(chaosStatus, allRecovered) { + currentStatus = StatusFinished + } else if !isConditionTrue(chaosStatus, paused) && !isConditionTrue(chaosStatus, selected) { + currentStatus = StatusUnknown + } + + // If the status is unknown, always notify listeners + if currentStatus == StatusUnknown { + c.notifyListeners(string(StatusUnknown), nil) + continue + } + + // If the status has changed, update internal status and notify listeners + if c.Status != currentStatus { + c.Status = currentStatus + + switch c.Status { + case StatusCreated: + c.notifyListeners("created", nil) + case StatusRunning: + c.startTime = time.Now() + c.notifyListeners("started", nil) + case StatusPaused: + c.notifyListeners("paused", nil) + case StatusFinished: + c.endTime = time.Now() + c.notifyListeners("finished", nil) + + err := c.Delete(ctx) + if err != nil { + c.logger.Error().Err(err).Msg("failed to delete chaos object") + } + case StatusCreationFailed: + panic("not implemented") + case StatusDeleted: + panic("not implemented") + case StatusUnknown: + panic("not implemented") + } + } + } + } +} + +type NetworkChaosOpts struct { + Name string + Description string + DelayCreate time.Duration + Delay *v1alpha1.DelaySpec + Loss *v1alpha1.LossSpec + NodeCount int + Duration time.Duration + Selector v1alpha1.PodSelectorSpec + K8sClient client.Client +} + +func (o *NetworkChaosOpts) Validate() error { + if o.Delay != nil { + latency, err := time.ParseDuration(o.Delay.Latency) + if err != nil { + return fmt.Errorf("invalid latency: %v", err) + } + if latency > 500*time.Millisecond { + return fmt.Errorf("duration should be less than 500ms") + } + } + if o.Loss != nil { + lossInt, err := strconv.Atoi(o.Loss.Loss) // Convert the string to an integer + if err != nil { + return fmt.Errorf("invalid loss value: %s", err) + } + if lossInt > 100 { + return fmt.Errorf("loss should be less than 100") + } + } + if o.Loss == nil && o.Delay == nil { + return fmt.Errorf("either delay or loss should be specified") + } + return nil + +} + +type PodChaosOpts struct { + Name string + Description string + DelayCreate time.Duration + NodeCount int + Duration time.Duration + Spec v1alpha1.PodChaosSpec + K8sClient client.Client +} + +type StressChaosOpts struct { + Name string + Description string + DelayCreate time.Duration + NodeCount int + Stressors *v1alpha1.Stressors + Duration time.Duration + Selector v1alpha1.PodSelectorSpec + K8sClient client.Client +} + +// NewChaosMeshClient initializes and returns a new Kubernetes client configured for Chaos Mesh +func NewChaosMeshClient() (client.Client, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) + config, err := kubeConfig.ClientConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to load kubeconfig") + } + + // Ensure the Chaos Mesh types are added to the scheme + if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { + return nil, errors.Wrap(err, "could not add the Chaos Mesh scheme") + } + + // Create a new client for the Chaos Mesh API + chaosClient, err := client.New(config, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return nil, errors.Wrap(err, "failed to create a client for Chaos Mesh") + } + + return chaosClient, nil +} diff --git a/havoc/chaos_entity.go b/havoc/chaos_entity.go new file mode 100644 index 000000000..6c7e61517 --- /dev/null +++ b/havoc/chaos_entity.go @@ -0,0 +1,27 @@ +package havoc + +import ( + "context" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ChaosEntity is an interface that defines common behaviors for chaos management entities. +type ChaosEntity interface { + // Create initializes and submits the chaos object to Kubernetes. + Create(ctx context.Context) + // Delete removes the chaos object from Kubernetes. + Delete(ctx context.Context) error + // Registers a listener to receive updates about the chaos object's lifecycle. + AddListener(listener ChaosListener) + + GetObject() client.Object + GetChaosName() string + GetChaosDescription() string + GetChaosDuration() (time.Duration, error) + GetChaosSpec() interface{} + GetStartTime() time.Time + GetEndTime() time.Time + GetExpectedEndTime() (time.Time, error) +} diff --git a/havoc/chaos_helper.go b/havoc/chaos_helper.go new file mode 100644 index 000000000..831c78584 --- /dev/null +++ b/havoc/chaos_helper.go @@ -0,0 +1,44 @@ +package havoc + +import ( + "errors" + "time" +) + +// WaitForAllChaosRunning waits for all chaos experiments to be running +func WaitForAllChaosRunning(chaosObjects []*Chaos, timeoutDuration time.Duration) error { + timeout := time.NewTimer(timeoutDuration) + defer timeout.Stop() + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + runningStatus := make(map[*Chaos]bool) + for _, chaos := range chaosObjects { + runningStatus[chaos] = false + } + + for { + allRunning := true + + select { + case <-timeout.C: + return errors.New("timeout reached before all chaos experiments became running") + case <-ticker.C: + for chaos, isRunning := range runningStatus { + if !isRunning { // Only check if not already marked as running + if chaos.Status == StatusRunning { + runningStatus[chaos] = true + } else { + allRunning = false + } + } + } + + if allRunning { + return nil // All chaos objects are running, can exit + } + // Otherwise, continue the loop + } + } +} diff --git a/havoc/chaos_listener.go b/havoc/chaos_listener.go new file mode 100644 index 000000000..01476d702 --- /dev/null +++ b/havoc/chaos_listener.go @@ -0,0 +1,12 @@ +package havoc + +type ChaosListener interface { + OnChaosCreated(chaos Chaos) + OnChaosCreationFailed(chaos Chaos, reason error) + OnChaosStarted(chaos Chaos) + OnChaosPaused(chaos Chaos) + OnChaosEnded(chaos Chaos) // When the chaos is finished or deleted + OnChaosStatusUnknown(chaos Chaos) // When the chaos status is unknown + OnScheduleCreated(chaos Schedule) + OnScheduleDeleted(chaos Schedule) // When the chaos is finished or deleted +} diff --git a/havoc/console_logger.go b/havoc/console_logger.go new file mode 100644 index 000000000..cec59da44 --- /dev/null +++ b/havoc/console_logger.go @@ -0,0 +1,140 @@ +package havoc + +import ( + "time" + + "github.com/rs/zerolog" +) + +type ChaosLogger struct { + logger zerolog.Logger +} + +func NewChaosLogger(logger zerolog.Logger) *ChaosLogger { + return &ChaosLogger{logger: logger} +} + +func (l ChaosLogger) OnChaosCreated(chaos Chaos) { + l.commonChaosLog("info", chaos).Msg("Chaos created") +} + +func (l ChaosLogger) OnChaosCreationFailed(chaos Chaos, reason error) { + l.commonChaosLog("error", chaos). + Err(reason). + Msg("Failed to create chaos object") +} + +func (l ChaosLogger) OnChaosStarted(chaos Chaos) { + experiment, _ := chaos.GetExperimentStatus() + + l.commonChaosLog("info", chaos). + Interface("spec", chaos.GetChaosSpec()). + Interface("records", experiment.Records). + Msg("Chaos started") +} + +func (l ChaosLogger) OnChaosPaused(chaos Chaos) { + l.commonChaosLog("info", chaos). + Msg("Chaos paused") +} + +func (l ChaosLogger) OnChaosEnded(chaos Chaos) { + l.commonChaosLog("info", chaos). + Msg("Chaos ended") +} + +func (l ChaosLogger) OnChaosDeleted(chaos Chaos) { + l.commonChaosLog("info", chaos). + Msg("Chaos deleted") +} + +type SimplifiedEvent struct { + LastTimestamp string + Type string + Message string +} + +func (l ChaosLogger) OnChaosStatusUnknown(chaos Chaos) { + status, _ := chaos.GetExperimentStatus() + events, _ := chaos.GetChaosEvents() + + // Create a slice to hold the simplified events + simplifiedEvents := make([]SimplifiedEvent, 0, len(events.Items)) + + // Iterate over the events and extract the required information + for _, event := range events.Items { + simplifiedEvents = append(simplifiedEvents, SimplifiedEvent{ + LastTimestamp: event.LastTimestamp.Time.Format(time.RFC3339), + Type: event.Type, + Message: event.Message, + }) + } + + l.commonChaosLog("error", chaos). + Interface("status", status). + Interface("events", simplifiedEvents). + Msg("Chaos status unknown") +} + +func (l ChaosLogger) OnScheduleCreated(schedule Schedule) { + duration, _ := schedule.GetChaosDuration() + + l.logger.Info(). + Str("logger", "chaos"). + Str("name", schedule.GetObject().GetName()). + Str("namespace", schedule.GetObject().GetNamespace()). + Str("description", schedule.GetChaosDescription()). + Str("duration", duration.String()). + Time("startTime", schedule.GetStartTime()). + Time("endTime", schedule.GetEndTime()). + Interface("spec", schedule.GetChaosSpec()). + Msg("Chaos schedule created") +} + +func (l ChaosLogger) OnScheduleDeleted(schedule Schedule) { + duration, _ := schedule.GetChaosDuration() + + l.logger.Info(). + Str("logger", "chaos"). + Str("name", schedule.GetObject().GetName()). + Str("namespace", schedule.GetObject().GetNamespace()). + Str("description", schedule.GetChaosDescription()). + Str("duration", duration.String()). + Time("startTime", schedule.GetStartTime()). + Time("endTime", schedule.GetEndTime()). + Interface("spec", schedule.GetChaosSpec()). + Msg("Chaos schedule deleted") +} + +func (l ChaosLogger) commonChaosLog(logLevel string, chaos Chaos) *zerolog.Event { + // Create a base event based on the dynamic log level + var event *zerolog.Event + switch logLevel { + case "debug": + event = l.logger.Debug() + case "info": + event = l.logger.Info() + case "warn": + event = l.logger.Warn() + case "error": + event = l.logger.Error() + case "fatal": + event = l.logger.Fatal() + case "panic": + event = l.logger.Panic() + default: + // Default to info level if an unknown level is provided + event = l.logger.Info() + } + + duration, _ := chaos.GetChaosDuration() + + return event. + Str("logger", "chaos"). + Str("name", chaos.GetObject().GetName()). + Str("namespace", chaos.GetObject().GetNamespace()). + Str("description", chaos.GetChaosDescription()). + Str("duration", duration.String()). + Time("startTime", chaos.GetStartTime()). + Time("endTime", chaos.GetEndTime()) +} diff --git a/havoc/go.mod b/havoc/go.mod new file mode 100644 index 000000000..0b5cc7121 --- /dev/null +++ b/havoc/go.mod @@ -0,0 +1,80 @@ +module github.com/smartcontractkit/chainlink-testing-framework/havoc + +go 1.22.0 + +toolchain go1.22.6 + +exclude github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 + +require ( + github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a + github.com/pkg/errors v0.9.1 + github.com/rs/zerolog v1.33.0 + github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.1 + k8s.io/api v0.31.0 + k8s.io/client-go v0.31.0 + sigs.k8s.io/controller-runtime v0.16.2 +) + +require ( + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-resty/resty/v2 v2.11.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grafana/grafana-foundation-sdk/go v0.0.0-20240326122733-6f96a993222b // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.31.0 // indirect + k8s.io/apimachinery v0.31.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +replace sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.19.0 diff --git a/havoc/go.sum b/havoc/go.sum new file mode 100644 index 000000000..e3948ab67 --- /dev/null +++ b/havoc/go.sum @@ -0,0 +1,257 @@ +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= +github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a h1:6Pg3a6j/41QDzH/oYcMLwwKsf3x/HXcu9W/dBaf2Hzs= +github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a/go.mod h1:x11iCbZV6hzzSQWMq610B6Wl5Lg1dhwqcVfeiWQQnQQ= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/grafana-foundation-sdk/go v0.0.0-20240326122733-6f96a993222b h1:Msqs1nc2qWMxTriDCITKl58Td+7Md/RURmUmH7RXKns= +github.com/grafana/grafana-foundation-sdk/go v0.0.0-20240326122733-6f96a993222b/go.mod h1:WtWosval1KCZP9BGa42b8aVoJmVXSg0EvQXi9LDSVZQ= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.1 h1:1/r1wQZ4TOFpZ13w94r7amdF096Z96RuEnkOmrz1BGE= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.1/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= +k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= +k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= +sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/havoc/logger.go b/havoc/logger.go new file mode 100644 index 000000000..e35653bc4 --- /dev/null +++ b/havoc/logger.go @@ -0,0 +1,44 @@ +package havoc + +import ( + "os" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// Default logger +var Logger zerolog.Logger + +func init() { + // Default logger + Logger = CreateLogger(LoggerConfig{ + LogOutput: os.Getenv("CHAOS_LOG_OUTPUT"), + LogLevel: os.Getenv("CHAOS_LOG_LEVEL"), + LogType: "chaos", + }) +} + +type LoggerConfig struct { + LogOutput string // "json-console" for JSON output, empty or "console" for human-friendly console output + LogLevel string // Log level (e.g., "info", "debug", "error") + LogType string // Custom log type identifier +} + +// Create initializes a zerolog.Logger based on the specified configuration. +func CreateLogger(config LoggerConfig) zerolog.Logger { + // Parse the log level + lvl, err := zerolog.ParseLevel(config.LogLevel) + if err != nil { + panic(err) // Consider more graceful error handling based on your application's requirements + } + + switch config.LogOutput { + case "json-console": + // Configure for JSON console output + return zerolog.New(os.Stderr).Level(lvl).With().Timestamp().Str("type", config.LogType).Logger() + default: + // Configure for console (human-friendly) output + return log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "15:04:05"}).Level(lvl).With().Timestamp().Str("type", config.LogType).Logger() + } +} diff --git a/havoc/range_grafana_annotator.go b/havoc/range_grafana_annotator.go new file mode 100644 index 000000000..5b7e6a291 --- /dev/null +++ b/havoc/range_grafana_annotator.go @@ -0,0 +1,241 @@ +package havoc + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/rs/zerolog" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana" +) + +type RangeGrafanaAnnotator struct { + client *grafana.Client + dashboardUID string + chaosMap map[string]int64 // Maps Chaos ID to Grafana Annotation ID + logger zerolog.Logger +} + +func NewRangeGrafanaAnnotator(grafanaURL, grafanaToken, dashboardUID string, logger zerolog.Logger) *RangeGrafanaAnnotator { + return &RangeGrafanaAnnotator{ + client: grafana.NewGrafanaClient(grafanaURL, grafanaToken), + dashboardUID: dashboardUID, + chaosMap: make(map[string]int64), + logger: logger, + } +} + +func (l RangeGrafanaAnnotator) OnChaosCreated(chaos Chaos) { +} + +func (l RangeGrafanaAnnotator) OnChaosStarted(chaos Chaos) { + experiment, _ := chaos.GetExperimentStatus() + duration, _ := chaos.GetChaosDuration() + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Started

", chaos.GetChaosTypeStr())) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.GetName())) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.GetStartTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", duration.String())) + + spec := chaos.GetChaosSpec() + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + + if len(experiment.Records) > 0 { + sb.WriteString("
") + sb.WriteString("
Records:
") + sb.WriteString("") + } + + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.GetStartTime()), + Text: sb.String(), + } + res, _, err := l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } + + l.chaosMap[chaos.GetChaosName()] = res.ID +} + +func (l RangeGrafanaAnnotator) OnChaosPaused(chaos Chaos) { +} + +func (l RangeGrafanaAnnotator) OnChaosEnded(chaos Chaos) { + annotationID, exists := l.chaosMap[chaos.GetChaosName()] + if !exists { + l.logger.Error().Msgf("No Grafana annotation ID found for Chaos: %s", chaos.GetChaosName()) + return + } + + experiment, _ := chaos.GetExperimentStatus() + duration, _ := chaos.GetChaosDuration() + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s

", chaos.GetChaosTypeStr())) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.GetName())) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.GetStartTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
End Time: %s
", chaos.GetEndTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", duration.String())) + + spec := chaos.GetChaosSpec() + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + + if len(experiment.Records) > 0 { + sb.WriteString("
") + sb.WriteString("
Records:
") + sb.WriteString("") + } + + sb.WriteString("") + + // Delete the temporary start annotation + _, err = l.client.DeleteAnnotation(annotationID) + if err != nil { + l.logger.Error().Msgf("could not delete temporary start annotation: %s", err) + } + delete(l.chaosMap, chaos.GetChaosName()) + + // Create the final annotation (time range) + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.GetStartTime()), + TimeEnd: Ptr[time.Time](chaos.GetEndTime()), + Text: sb.String(), + } + res, _, err := l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } + l.chaosMap[chaos.GetChaosName()] = res.ID +} + +func (l RangeGrafanaAnnotator) OnChaosStatusUnknown(chaos Chaos) { +} + +func (l RangeGrafanaAnnotator) OnScheduleCreated(chaos Schedule) { + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Schedule Created

", chaos.Object.Spec.Type)) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.ObjectMeta.Name)) + sb.WriteString(fmt.Sprintf("
Schedule: %s
", chaos.Object.Spec.Schedule)) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.startTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", chaos.Duration.String())) + + spec := chaos.Object.Spec.ScheduleItem + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.startTime), + Text: sb.String(), + } + res, _, err := l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } + + l.chaosMap[chaos.Object.GetName()] = res.ID +} + +func (l RangeGrafanaAnnotator) OnScheduleDeleted(chaos Schedule) { + annotationID, exists := l.chaosMap[chaos.Object.GetName()] + if !exists { + l.logger.Error().Msgf("No Grafana annotation ID found for Chaos: %s", chaos.Object.GetName()) + return + } + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Schedule

", chaos.Object.Spec.Type)) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.ObjectMeta.Name)) + sb.WriteString(fmt.Sprintf("
Schedule: %s
", chaos.Object.Spec.Schedule)) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.startTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
End Time: %s
", chaos.endTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", chaos.Duration.String())) + + spec := chaos.Object.Spec.ScheduleItem + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + sb.WriteString("") + + // Delete the temporary start annotation + _, err = l.client.DeleteAnnotation(annotationID) + if err != nil { + l.logger.Error().Msgf("could not delete temporary start annotation: %s", err) + } + delete(l.chaosMap, chaos.Object.GetName()) + + // Create the final annotation (time range) + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.startTime), + TimeEnd: Ptr[time.Time](chaos.endTime), + Text: sb.String(), + } + res, _, err := l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } + l.chaosMap[chaos.Object.GetName()] = res.ID +} diff --git a/havoc/schedule.go b/havoc/schedule.go new file mode 100644 index 000000000..3bb08dc7a --- /dev/null +++ b/havoc/schedule.go @@ -0,0 +1,225 @@ +package havoc + +import ( + "context" + "time" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/pkg/errors" + "github.com/rs/zerolog" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ScheduleStatus string + +const ( + ScheduleStatusCreated ScheduleStatus = "created" + ScheduleStatusDeleted ScheduleStatus = "deleted" + ScheduleStatusUnknown ScheduleStatus = "unknown" // For any state that doesn't match the above +) + +type Schedule struct { + Object *v1alpha1.Schedule + Description string + DelayCreate time.Duration // Delay before creating the chaos object + Duration time.Duration // Duration for which the chaos object should exist + Status ChaosStatus + Client client.Client + listeners []ChaosListener + cancelMonitor context.CancelFunc + startTime time.Time + endTime time.Time + logger *zerolog.Logger +} + +type ScheduleOpts struct { + Object *v1alpha1.Schedule + Description string + DelayCreate time.Duration + Duration time.Duration + Client client.Client + Listeners []ChaosListener + Logger *zerolog.Logger +} + +func NewSchedule(opts ScheduleOpts) (*Schedule, error) { + if opts.Client == nil { + return nil, errors.New("client is required") + } + if opts.Object == nil { + return nil, errors.New("chaos object is required") + } + if opts.Logger == nil { + return nil, errors.New("logger is required") + } + + return &Schedule{ + Object: opts.Object, + Description: opts.Description, + DelayCreate: opts.DelayCreate, + Duration: opts.Duration, + Client: opts.Client, + listeners: opts.Listeners, + logger: opts.Logger, + }, nil +} + +// Create initiates a delayed creation of a chaos object, respecting context cancellation and deletion requests. +// It uses a timer based on `DelayCreate` and calls `create` method upon expiration unless preempted by deletion. +func (s *Schedule) Create(ctx context.Context) { + done := make(chan struct{}) + + // Create the timer with the delay to create the chaos object + timer := time.NewTimer(s.DelayCreate) + + go func() { + select { + case <-ctx.Done(): + // If the context is canceled, stop the timer and exit + if !timer.Stop() { + <-timer.C // If the timer already expired, drain the channel + } + close(done) // Signal that the operation was canceled + case <-timer.C: + // Timer expired, check if deletion was not requested + if s.Status != StatusDeleted { + s.createNow(ctx) + } + close(done) // Signal that the creation process is either done or skipped + } + }() +} + +func (s *Schedule) Delete(ctx context.Context) error { + if err := s.Client.Delete(ctx, s.Object); err != nil { + return errors.Wrap(err, "failed to delete chaos object") + } + + // Cancel the monitoring goroutine + if s.cancelMonitor != nil { + s.cancelMonitor() + } + + s.endTime = time.Now() + s.Status = StatusDeleted + s.notifyListeners(string(ScheduleStatusDeleted)) + + return nil +} + +func (s *Schedule) AddListener(listener ChaosListener) { + s.listeners = append(s.listeners, listener) +} + +func (s *Schedule) GetObject() client.Object { + return s.Object +} + +func (s *Schedule) GetChaosName() string { + return s.Object.GetName() +} + +func (s *Schedule) GetChaosDescription() string { + return s.Description +} + +func (s *Schedule) GetChaosSpec() interface{} { + return s.Object.Spec.ScheduleItem +} + +func (s *Schedule) GetChaosDuration() (time.Duration, error) { + return s.Duration, nil +} + +func (s *Schedule) GetStartTime() time.Time { + return s.startTime +} + +func (s *Schedule) GetEndTime() time.Time { + return s.endTime +} + +func (s *Schedule) GetExpectedEndTime() (time.Time, error) { + duration, err := s.GetChaosDuration() + if err != nil { + return time.Time{}, err + } + return s.startTime.Add(duration), nil +} + +func (s *Schedule) createNow(ctx context.Context) { + if err := s.Client.Create(ctx, s.Object); err != nil { + Logger.Error().Err(err).Interface("chaos", s).Msg("failed to create chaos object") + return + } + s.startTime = time.Now() + s.Status = StatusCreated + s.notifyListeners(string(ScheduleStatusCreated)) + + // Create a cancellable context for monitorStatus + monitorCtx, cancel := context.WithCancel(ctx) + s.cancelMonitor = cancel + go s.monitorStatus(monitorCtx) + + // Start a deletion timer to delete the chaos object after the specified duration + done := make(chan struct{}) + deleteTimer := time.NewTimer(s.Duration) + go func() { + select { + case <-ctx.Done(): + // Context was canceled, ensure chaos object is deleted + if !deleteTimer.Stop() { + <-deleteTimer.C // Drain the timer if it already fired + } + err := s.Delete(ctx) + if err != nil { + s.logger.Error().Err(err).Msg("failed to delete chaos object") + } + close(done) + case <-deleteTimer.C: + // Duration elapsed, delete the chaos object + err := s.Delete(ctx) + if err != nil { + s.logger.Error().Err(err).Msg("failed to delete chaos object") + } + close(done) + } + }() +} + +func (s *Schedule) notifyListeners(event string) { + for _, listener := range s.listeners { + switch event { + case string(ScheduleStatusCreated): + listener.OnScheduleCreated(*s) + case string(ScheduleStatusDeleted): + listener.OnScheduleDeleted(*s) + } + } +} + +func (s *Schedule) monitorStatus(ctx context.Context) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + // Context canceled, stop monitoring + return + case <-ticker.C: + // Fetch the latest state of the Schedule object + var schedule v1alpha1.Schedule + if err := s.Client.Get(ctx, client.ObjectKey{ + Namespace: s.Object.GetNamespace(), + Name: s.Object.GetName(), + }, &schedule); err != nil { + Logger.Error().Err(err).Msg("Failed to get Schedule object") + continue + } + + // Log or process the schedule's current status + // Loggerog.Info().Interface("status", schedule.Status).Msg("Current Schedule Status") + } + } +} diff --git a/havoc/single_line_grafana_annotator.go b/havoc/single_line_grafana_annotator.go new file mode 100644 index 000000000..24a6b17d4 --- /dev/null +++ b/havoc/single_line_grafana_annotator.go @@ -0,0 +1,206 @@ +package havoc + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/rs/zerolog" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana" +) + +type SingleLineGrafanaAnnotator struct { + client *grafana.Client + dashboardUID string + logger zerolog.Logger +} + +func NewSingleLineGrafanaAnnotator(grafanaURL, grafanaToken, dashboardUID string, logger zerolog.Logger) *SingleLineGrafanaAnnotator { + return &SingleLineGrafanaAnnotator{ + client: grafana.NewGrafanaClient(grafanaURL, grafanaToken), + dashboardUID: dashboardUID, + logger: logger, + } +} + +func (l SingleLineGrafanaAnnotator) OnChaosCreated(chaos Chaos) { +} + +func (l SingleLineGrafanaAnnotator) OnChaosCreationFailed(chaos Chaos, reason error) { +} + +func (l SingleLineGrafanaAnnotator) OnChaosStarted(chaos Chaos) { + experiment, _ := chaos.GetExperimentStatus() + duration, _ := chaos.GetChaosDuration() + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Started

", chaos.GetChaosTypeStr())) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.GetName())) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.GetStartTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", duration.String())) + + spec := chaos.GetChaosSpec() + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + + if len(experiment.Records) > 0 { + sb.WriteString("
") + sb.WriteString("
Records:
") + sb.WriteString("") + } + + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.GetStartTime()), + Text: sb.String(), + } + _, _, err = l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } +} + +func (l SingleLineGrafanaAnnotator) OnChaosPaused(chaos Chaos) { +} + +func (l SingleLineGrafanaAnnotator) OnChaosEnded(chaos Chaos) { + experiment, _ := chaos.GetExperimentStatus() + duration, _ := chaos.GetChaosDuration() + + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Ended

", chaos.GetChaosTypeStr())) + sb.WriteString(fmt.Sprintf("
Name: %s
", chaos.Object.GetName())) + if chaos.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", chaos.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", chaos.GetStartTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
End Time: %s
", chaos.GetEndTime().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", duration.String())) + + spec := chaos.GetChaosSpec() + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + + if len(experiment.Records) > 0 { + sb.WriteString("
") + sb.WriteString("
Records:
") + sb.WriteString("") + } + + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](chaos.GetEndTime()), + Text: sb.String(), + } + _, _, err = l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } +} + +func (l SingleLineGrafanaAnnotator) OnChaosStatusUnknown(chaos Chaos) { +} + +func (l SingleLineGrafanaAnnotator) OnScheduleCreated(s Schedule) { + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Schedule Created

", s.Object.Spec.Type)) + sb.WriteString(fmt.Sprintf("
Name: %s
", s.Object.ObjectMeta.Name)) + sb.WriteString(fmt.Sprintf("
Schedule: %s
", s.Object.Spec.Schedule)) + if s.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", s.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", s.startTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", s.Duration.String())) + + spec := s.Object.Spec.ScheduleItem + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Schedule Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](s.startTime), + Text: sb.String(), + } + _, _, err = l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } +} + +func (l SingleLineGrafanaAnnotator) OnScheduleDeleted(s Schedule) { + var sb strings.Builder + sb.WriteString("") + sb.WriteString(fmt.Sprintf("

%s Schedule Ended

", s.Object.Spec.Type)) + sb.WriteString(fmt.Sprintf("
Name: %s
", s.Object.ObjectMeta.Name)) + sb.WriteString(fmt.Sprintf("
Schedule: %s
", s.Object.Spec.Schedule)) + if s.Description != "" { + sb.WriteString(fmt.Sprintf("
Description: %s
", s.Description)) + } + sb.WriteString(fmt.Sprintf("
Start Time: %s
", s.startTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
End Time: %s
", s.endTime.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("
Duration: %s
", s.Duration.String())) + + spec := s.Object.Spec.ScheduleItem + specBytes, err := json.MarshalIndent(spec, "", " ") + if err == nil && len(specBytes) > 0 { + sb.WriteString("
") + sb.WriteString("
Schedule Spec:
") + sb.WriteString(string(specBytes)) + sb.WriteString("
") + } else { + l.logger.Warn().Msgf("could not get chaos spec: %s", err) + } + sb.WriteString("") + + a := grafana.PostAnnotation{ + DashboardUID: l.dashboardUID, + Time: Ptr[time.Time](s.endTime), + Text: sb.String(), + } + _, _, err = l.client.PostAnnotation(a) + if err != nil { + l.logger.Warn().Msgf("could not annotate on Grafana: %s", err) + } +} diff --git a/havoc/utils.go b/havoc/utils.go new file mode 100644 index 000000000..f9bc46340 --- /dev/null +++ b/havoc/utils.go @@ -0,0 +1,5 @@ +package havoc + +func Ptr[T any](value T) *T { + return &value +}