exit with error if any of the deployments failed

threefoldtech · Jun 6, 2024 · c97f096 · c97f096
1 parent 381d36b
commit c97f096
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 40 deletions.
diff --git a/.github/workflows/tfrobot_test.yml b/.github/workflows/tfrobot_test.yml
@@ -4,6 +4,7 @@ defaults:
   run:
     working-directory: tfrobot
 on:
+  push:
   schedule:
     - cron: 0 4 * * *
   workflow_dispatch:

diff --git a/tfrobot/cmd/deploy.go b/tfrobot/cmd/deploy.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/threefoldtech/tfgrid-sdk-go/tfrobot/internal/parser"
 	"github.com/threefoldtech/tfgrid-sdk-go/tfrobot/pkg/deployer"
@@ -93,8 +94,10 @@ var deployCmd = &cobra.Command{
 		}
 
 		ctx := context.Background()
-		if err = deployer.RunDeployer(ctx, cfg, tfPluginClient, outputPath, debug); err != nil {
-			return fmt.Errorf("failed to run the deployer with error: %w", err)
+		if errs := deployer.RunDeployer(ctx, cfg, tfPluginClient, outputPath, debug); errs != nil {
+			log.Error().Msg("deployments failed with errors: ")
+			fmt.Println(errs)
+			os.Exit(1)
 		}
 
 		return nil

diff --git a/tfrobot/cmd/load.go b/tfrobot/cmd/load.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/threefoldtech/tfgrid-sdk-go/tfrobot/internal/parser"
 	"github.com/threefoldtech/tfgrid-sdk-go/tfrobot/pkg/deployer"
@@ -80,9 +81,10 @@ var loadCmd = &cobra.Command{
 			return err
 		}
 
-		err = deployer.RunLoader(ctx, cfg, tfPluginClient, debug, outputPath)
-		if err != nil {
-			return fmt.Errorf("failed to load configured deployments with error: %w", err)
+		if err := deployer.RunLoader(ctx, cfg, tfPluginClient, debug, outputPath); err != nil {
+			log.Error().Msg("failed to load configured deployments")
+      fmt.Println(err)
+			os.Exit(1)
 		}
 
 		return nil

diff --git a/tfrobot/pkg/deployer/deployer.go b/tfrobot/pkg/deployer/deployer.go
@@ -25,9 +25,10 @@ const (
 	maxGoroutinesCount = 100
 )
 
-func RunDeployer(ctx context.Context, cfg Config, tfPluginClient deployer.TFPluginClient, output string, debug bool) error {
+func RunDeployer(ctx context.Context, cfg Config, tfPluginClient deployer.TFPluginClient, output string, debug bool) *multierror.Error {
 	passedGroups := map[string][]*workloads.Deployment{}
-	failedGroups := map[string]string{}
+	var failedGroupsErr *multierror.Error
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -74,8 +75,8 @@ func RunDeployer(ctx context.Context, cfg Config, tfPluginClient deployer.TFPlug
 
 			return nil
 		}); err != nil {
+			failedGroupsErr = multierror.Append(failedGroupsErr, fmt.Errorf("%s: %s", nodeGroup.Name, err.Error()))
 
-			failedGroups[nodeGroup.Name] = err.Error()
 			err := tfPluginClient.CancelByProjectName(fmt.Sprintf("vm/%s", nodeGroup.Name))
 			if err != nil {
 				log.Debug().Err(err).Send()
@@ -97,15 +98,20 @@ func RunDeployer(ctx context.Context, cfg Config, tfPluginClient deployer.TFPlug
 	endTime := time.Since(deploymentStart)
 
 	// load deployed deployments
-	outputBytes, err := loadAfterDeployment(ctx, tfPluginClient, passedGroups, failedGroups, cfg.MaxRetries, filepath.Ext(output) == ".json")
-	if err != nil {
-		return err
+	outputBytes, errs := loadAfterDeployment(ctx, tfPluginClient, passedGroups, cfg.MaxRetries, filepath.Ext(output) == ".json")
+	if errs != nil {
+		failedGroupsErr = multierror.Append(failedGroupsErr, errs.Errors...)
 	}
 
 	fmt.Println(string(outputBytes))
 	log.Info().Msgf("Deployment took %s", endTime)
 
-	return os.WriteFile(output, outputBytes, 0644)
+	err := os.WriteFile(output, outputBytes, 0644)
+	if err != nil {
+		log.Error().Err(err).Send()
+	}
+
+	return failedGroupsErr
 }
 
 func deployNodeGroup(
@@ -140,25 +146,25 @@ func loadAfterDeployment(
 	ctx context.Context,
 	tfPluginClient deployer.TFPluginClient,
 	deployedGroups map[string][]*workloads.Deployment,
-	failedGroups map[string]string,
 	retries uint64,
 	asJson bool,
-) ([]byte, error) {
+) ([]byte, *multierror.Error) {
 	var loadedgroups map[string][]vmOutput
+	var failedGroupsErr *multierror.Error
 
 	if len(deployedGroups) > 0 {
 		log.Info().Msg("Loading deployments")
 		groupsContracts := getDeploymentsContracts(deployedGroups)
 
-		var failed map[string]string
-		loadedgroups, failed = batchLoadNodeGroupsInfo(ctx, tfPluginClient, groupsContracts, retries)
+		loadedgroups, failedGroupsErr = batchLoadNodeGroupsInfo(ctx, tfPluginClient, groupsContracts, retries)
+	}
 
-		for nodeGroup, err := range failed {
-			failedGroups[nodeGroup] = err
-		}
+	output, err := parseDeploymentOutput(loadedgroups, asJson)
+	if err != nil {
+		log.Error().Err(err).Send()
 	}
 
-	return parseDeploymentOutput(loadedgroups, failedGroups, asJson)
+	return output, failedGroupsErr
 }
 
 func parseVMsGroup(vms []Vms, nodeGroup string, nodesIDs []int, sshKeys map[string]string) groupDeploymentsInfo {

diff --git a/tfrobot/pkg/deployer/load.go b/tfrobot/pkg/deployer/load.go
@@ -13,6 +13,7 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/rs/zerolog/log"
 	"github.com/sethvargo/go-retry"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/deployer"
@@ -28,26 +29,32 @@ func RunLoader(ctx context.Context, cfg Config, tfPluginClient deployer.TFPlugin
 
 	asJson := filepath.Ext(output) == ".json"
 
-	groupsContracts, failed := getGroupsContracts(ctx, tfPluginClient, cfg.NodeGroups)
-	passedGroups, failedGroups := batchLoadNodeGroupsInfo(ctx, tfPluginClient, groupsContracts, cfg.MaxRetries)
+	groupsContracts, failedGroupsErr := getGroupsContracts(tfPluginClient, cfg.NodeGroups)
+	passedGroups, errs := batchLoadNodeGroupsInfo(ctx, tfPluginClient, groupsContracts, cfg.MaxRetries)
 
 	// add projects failed to be loaded
-	for group, err := range failed {
-		failedGroups[group] = err
+	if errs != nil {
+		failedGroupsErr = multierror.Append(failedGroupsErr, errs.Errors...)
 	}
 
-	outputBytes, err := parseDeploymentOutput(passedGroups, failedGroups, asJson)
+	outputBytes, err := parseDeploymentOutput(passedGroups, asJson)
 	if err != nil {
-		return err
+		failedGroupsErr = multierror.Append(failedGroupsErr, err)
 	}
 
 	fmt.Println(string(outputBytes))
-	return os.WriteFile(output, outputBytes, 0644)
+
+	err = os.WriteFile(output, outputBytes, 0644)
+	if err != nil {
+		log.Info().Err(err).Send()
+	}
+
+	return failedGroupsErr
 }
 
-func getGroupsContracts(ctx context.Context, tfPluginClient deployer.TFPluginClient, nodeGroups []NodesGroup) (map[string]NodeContracts, map[string]string) {
+func getGroupsContracts(tfPluginClient deployer.TFPluginClient, nodeGroups []NodesGroup) (map[string]NodeContracts, *multierror.Error) {
 	loadedContracts := make(map[string]NodeContracts)
-	failedGroups := make(map[string]string)
+	var failedGroupsErr *multierror.Error
 
 	var lock sync.Mutex
 	var wg sync.WaitGroup
@@ -65,7 +72,7 @@ func getGroupsContracts(ctx context.Context, tfPluginClient deployer.TFPluginCli
 			defer lock.Unlock()
 
 			if err != nil {
-				failedGroups[nodeGroup] = err.Error()
+				failedGroupsErr = multierror.Append(failedGroupsErr, fmt.Errorf("%s: %s", nodeGroup, err.Error()))
 				return
 			}
 
@@ -74,18 +81,18 @@ func getGroupsContracts(ctx context.Context, tfPluginClient deployer.TFPluginCli
 	}
 
 	wg.Wait()
-	return loadedContracts, failedGroups
+	return loadedContracts, failedGroupsErr
 }
 
 func batchLoadNodeGroupsInfo(
 	ctx context.Context,
 	tfPluginClient deployer.TFPluginClient,
 	groupsContracts map[string]NodeContracts,
 	retries uint64,
-) (map[string][]vmOutput, map[string]string) {
+) (map[string][]vmOutput, *multierror.Error) {
 	trial := 1
-	failedGroups := map[string]string{}
 	nodeGroupsInfo := map[string][]vmOutput{}
+	var failedGroupsErr *multierror.Error
 
 	var lock sync.Mutex
 	var wg sync.WaitGroup
@@ -115,25 +122,23 @@ func batchLoadNodeGroupsInfo(
 				return nil
 			}); err != nil {
 				lock.Lock()
-				failedGroups[nodeGroup] = err.Error()
+				failedGroupsErr = multierror.Append(failedGroupsErr, fmt.Errorf("%s: %s", nodeGroup, err.Error()))
 				lock.Unlock()
 			}
 		}(nodeGroup, contracts)
 	}
 
 	wg.Wait()
-	return nodeGroupsInfo, failedGroups
+	return nodeGroupsInfo, failedGroupsErr
 }
 
-func parseDeploymentOutput(passedGroups map[string][]vmOutput, failedGroups map[string]string, asJson bool) ([]byte, error) {
+func parseDeploymentOutput(passedGroups map[string][]vmOutput, asJson bool) ([]byte, error) {
 	var err error
 	var outputBytes []byte
 	outData := struct {
-		OK    map[string][]vmOutput `json:"ok"`
-		Error map[string]string     `json:"error"`
+		OK map[string][]vmOutput `json:"ok"`
 	}{
-		OK:    passedGroups,
-		Error: failedGroups,
+		OK: passedGroups,
 	}
 
 	if asJson {