chore(kube): dump CRD if deployment fails after manifest apply

Signed-off-by: Artur Troian <[email protected]>

cluster/inventory.go

@@ -390,7 +390,7 @@ func (is *inventoryService) handleRequest(req inventoryRequest, state *inventory
 
 	{
 		jReservation, _ := json.Marshal(req.resources.GetResources())
-		is.log.Debug("reservation requested", "order", req.order, "resources", jReservation)
+		is.log.Debug("reservation requested", "order", req.order, fmt.Sprintf("resources=%s", jReservation))
 	}
 
 	if reservation.endpointQuantity != 0 {

cluster/kube/apply.go

@@ -12,6 +12,7 @@ import (
 	metricsutils "github.com/akash-network/node/util/metrics"
 
 	"github.com/akash-network/provider/cluster/kube/builder"
+	crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2"
 	crdapi "github.com/akash-network/provider/pkg/client/clientset/versioned"
 )
 
@@ -153,7 +154,7 @@ func applyService(ctx context.Context, kc kubernetes.Interface, b builder.Servic
 	return err
 }
 
-func applyManifest(ctx context.Context, kc crdapi.Interface, b builder.Manifest) error {
+func applyManifest(ctx context.Context, kc crdapi.Interface, b builder.Manifest) (*crd.Manifest, error) {
 	obj, err := kc.AkashV2beta2().Manifests(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{})
 
 	metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "akash-manifests-get", err, errors.IsNotFound)
@@ -173,5 +174,6 @@ func applyManifest(ctx context.Context, kc crdapi.Interface, b builder.Manifest)
 			metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "akash-manifests-create", err)
 		}
 	}
-	return err
+
+	return obj, err
 }

cluster/kube/client.go

@@ -2,6 +2,7 @@ package kube
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"strings"
@@ -219,12 +220,25 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) erro
 	}
 
 	cmanifest := builder.BuildManifest(c.log, settings, c.ns, cdeployment)
-	if err := applyManifest(ctx, c.ac, cmanifest); err != nil {
+	crdManifest, err := applyManifest(ctx, c.ac, cmanifest)
+	if err != nil {
 		c.log.Error("applying manifest", "err", err, "lease", lid)
 		return err
 	}
 
-	if err := cleanupStaleResources(ctx, c.kc, lid, group); err != nil {
+	// log actual stored manifest if deployment fails
+	defer func() {
+		if err != nil {
+			jData, err := json.Marshal(crdManifest)
+			if err == nil {
+				c.log.Error("while deploying", "err", err, fmt.Sprintf("manifest=%s", string(jData)))
+			} else {
+				c.log.Error("dump manifest crd while deploying", "err", err)
+			}
+		}
+	}()
+
+	if err = cleanupStaleResources(ctx, c.kc, lid, group); err != nil {
 		c.log.Error("cleaning stale resources", "err", err, "lease", lid)
 		return err
 	}
@@ -243,12 +257,12 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) erro
 		}
 
 		if persistent {
-			if err := applyStatefulSet(ctx, c.kc, builder.BuildStatefulSet(workload)); err != nil {
+			if err = applyStatefulSet(ctx, c.kc, builder.BuildStatefulSet(workload)); err != nil {
 				c.log.Error("applying statefulSet", "err", err, "lease", lid, "service", service.Name)
 				return err
 			}
 		} else {
-			if err := applyDeployment(ctx, c.kc, builder.NewDeployment(workload)); err != nil {
+			if err = applyDeployment(ctx, c.kc, builder.NewDeployment(workload)); err != nil {
 				c.log.Error("applying deployment", "err", err, "lease", lid, "service", service.Name)
 				return err
 			}
@@ -261,15 +275,15 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) erro
 
 		serviceBuilderLocal := builder.BuildService(workload, false)
 		if serviceBuilderLocal.Any() {
-			if err := applyService(ctx, c.kc, serviceBuilderLocal); err != nil {
+			if err = applyService(ctx, c.kc, serviceBuilderLocal); err != nil {
 				c.log.Error("applying local service", "err", err, "lease", lid, "service", service.Name)
 				return err
 			}
 		}
 
 		serviceBuilderGlobal := builder.BuildService(workload, true)
 		if serviceBuilderGlobal.Any() {
-			if err := applyService(ctx, c.kc, serviceBuilderGlobal); err != nil {
+			if err = applyService(ctx, c.kc, serviceBuilderGlobal); err != nil {
 				c.log.Error("applying global service", "err", err, "lease", lid, "service", service.Name)
 				return err
 			}