Fix race condition when applying a chaos experiment

smartcontractkit · Oct 23, 2023 · e48af06 · e48af06
1 parent cf7b4b1
commit e48af06
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 6 deletions.
diff --git a/env/client/chaos.go b/env/client/chaos.go
@@ -43,13 +43,13 @@ func (c *Chaos) Run(app cdk8s.App, id string, resource string) (string, error) {
 	config.JSIIGlobalMu.Unlock()
 	log.Trace().Str("Raw", manifest).Msg("Manifest")
 	c.ResourceByName[id] = resource
-	if err := c.Client.Apply(context.Background(), manifest, c.Namespace); err != nil {
+	if err := c.Client.Apply(context.Background(), manifest, c.Namespace, false); err != nil {
 		return id, err
 	}
 	if err := c.checkForPodsExistence(app); err != nil {
 		return id, err
 	}
-	err := c.waitForChaosStatus(id, v1alpha1.ConditionAllInjected, time.Minute)
+	err := c.waitForChaosStatus(id, v1alpha1.ConditionAllInjected, 5*time.Minute)
 	if err != nil {
 		return id, err
 	}

diff --git a/env/client/client.go b/env/client/client.go
@@ -437,7 +437,7 @@ func (m *K8sClient) WaitForDeploymentsAvailable(ctx context.Context, namespace s
 }
 
 // Apply applying a manifest to a currently connected k8s context
-func (m *K8sClient) Apply(ctx context.Context, manifest, namespace string) error {
+func (m *K8sClient) Apply(ctx context.Context, manifest, namespace string, waitForDeployment bool) error {
 	manifestFile := fmt.Sprintf(TempDebugManifest, uuid.NewString())
 	log.Info().Str("File", manifestFile).Msg("Applying manifest")
 	if err := os.WriteFile(manifestFile, []byte(manifest), os.ModePerm); err != nil {
@@ -448,7 +448,10 @@ func (m *K8sClient) Apply(ctx context.Context, manifest, namespace string) error
 	if err := ExecCmdWithContext(ctx, cmd); err != nil {
 		return err
 	}
-	return m.WaitForDeploymentsAvailable(ctx, namespace)
+	if waitForDeployment {
+		return m.WaitForDeploymentsAvailable(ctx, namespace)
+	}
+	return nil
 }
 
 // DeleteResource deletes resource

diff --git a/env/environment/environment.go b/env/environment/environment.go
@@ -309,7 +309,7 @@ func (m *Environment) initApp() error {
 	startTime := time.Now()
 	deadline, _ := ctx.Deadline()
 	for {
-		err = m.Client.Apply(ctx, m.CurrentManifest, m.Cfg.Namespace)
+		err = m.Client.Apply(ctx, m.CurrentManifest, m.Cfg.Namespace, true)
 		if err == nil || ctx.Err() != nil {
 			break
 		}
@@ -774,7 +774,7 @@ func (m *Environment) DeployCustomReadyConditions(customCheck *client.ReadyCheck
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), m.Cfg.ReadyCheckData.Timeout)
 	defer cancel()
-	err := m.Client.Apply(ctx, m.CurrentManifest, m.Cfg.Namespace)
+	err := m.Client.Apply(ctx, m.CurrentManifest, m.Cfg.Namespace, true)
 	if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 		return errors.New("timeout waiting for environment to be ready")
 	}