diff --git a/pkg/controller/postgressnapshot/postgressnapshot_controller.go b/pkg/controller/postgressnapshot/postgressnapshot_controller.go index c622f683a..d724e3b4e 100644 --- a/pkg/controller/postgressnapshot/postgressnapshot_controller.go +++ b/pkg/controller/postgressnapshot/postgressnapshot_controller.go @@ -3,8 +3,6 @@ package postgressnapshot import ( "context" "fmt" - "time" - "github.com/aws/aws-sdk-go/service/rds/rdsiface" "github.com/aws/aws-sdk-go/aws" @@ -113,7 +111,7 @@ func (r *ReconcilePostgresSnapshot) Reconcile(request reconcile.Request) (reconc // check status, if complete return if instance.Status.Phase == croType.PhaseComplete { r.logger.Infof("skipping creation of snapshot for %s as phase is complete", instance.Name) - return reconcile.Result{}, nil + return reconcile.Result{Requeue: true, RequeueAfter: resources.SuccessReconcileTime}, nil } // get postgres cr @@ -122,27 +120,27 @@ func (r *ReconcilePostgresSnapshot) Reconcile(request reconcile.Request) (reconc if err != nil { errMsg := fmt.Sprintf("failed to get postgres resource: %s", err.Error()) if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } - return reconcile.Result{}, errorUtil.New(errMsg) + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.New(errMsg) } // check postgres deployment strategy is aws if postgresCr.Status.Strategy != providers.AWSDeploymentStrategy { errMsg := fmt.Sprintf("the resource %s uses an unsupported provider strategy %s, only resources using the aws provider are valid", instance.Spec.ResourceName, postgresCr.Status.Strategy) if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } - return reconcile.Result{}, errorUtil.New(errMsg) + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.New(errMsg) } // get resource region stratCfg, err := r.ConfigManager.ReadStorageStrategy(ctx, providers.PostgresResourceType, postgresCr.Spec.Tier) if err != nil { if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(err.Error())); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } - return reconcile.Result{}, err + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, err } if stratCfg.Region == "" { stratCfg.Region = croAws.DefaultRegion @@ -153,9 +151,9 @@ func (r *ReconcilePostgresSnapshot) Reconcile(request reconcile.Request) (reconc if err != nil { errMsg := "failed to reconcile rds credentials" if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } - return reconcile.Result{}, errorUtil.Wrap(err, errMsg) + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.Wrap(err, errMsg) } // setup aws rds session @@ -167,12 +165,12 @@ func (r *ReconcilePostgresSnapshot) Reconcile(request reconcile.Request) (reconc // create the snapshot and return the phase phase, msg, err := r.createSnapshot(ctx, rdsSvc, instance, postgresCr) if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, phase, msg); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } if err != nil { - return reconcile.Result{}, err + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, err } - return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 60}, nil + return reconcile.Result{Requeue: true, RequeueAfter: resources.SuccessReconcileTime}, nil } func (r *ReconcilePostgresSnapshot) createSnapshot(ctx context.Context, rdsSvc rdsiface.RDSAPI, snapshot *integreatlyv1alpha1.PostgresSnapshot, postgres *integreatlyv1alpha1.Postgres) (croType.StatusPhase, croType.StatusMessage, error) { diff --git a/pkg/controller/postgressnapshot/postgressnapshot_controller_test.go b/pkg/controller/postgressnapshot/postgressnapshot_controller_test.go index 41d948715..1a6786fef 100644 --- a/pkg/controller/postgressnapshot/postgressnapshot_controller_test.go +++ b/pkg/controller/postgressnapshot/postgressnapshot_controller_test.go @@ -174,7 +174,7 @@ func TestReconcilePostgresSnapshot_createSnapshot(t *testing.T) { name: "test successful snapshot in progress", args: args{ ctx: ctx, - rdsSvc: &mockRdsClient{dbSnapshot: buildSnapshot(), dbSnapshots: buildSnapshots(snapshotName, "creatring")}, + rdsSvc: &mockRdsClient{dbSnapshot: buildSnapshot(), dbSnapshots: buildSnapshots(snapshotName, "creating")}, snapshot: buildPostgresSnapshot(), postgres: buildPostgres(), }, diff --git a/pkg/controller/redissnapshot/redissnapshot_controller.go b/pkg/controller/redissnapshot/redissnapshot_controller.go index 2fa024c01..c2e9885ba 100644 --- a/pkg/controller/redissnapshot/redissnapshot_controller.go +++ b/pkg/controller/redissnapshot/redissnapshot_controller.go @@ -3,13 +3,11 @@ package redissnapshot import ( "context" "fmt" - "github.com/aws/aws-sdk-go/service/elasticache/elasticacheiface" - "time" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/elasticache" + "github.com/aws/aws-sdk-go/service/elasticache/elasticacheiface" croType "github.com/integr8ly/cloud-resource-operator/pkg/apis/integreatly/v1alpha1/types" "github.com/integr8ly/cloud-resource-operator/pkg/providers" croAws "github.com/integr8ly/cloud-resource-operator/pkg/providers/aws" @@ -114,7 +112,7 @@ func (r *ReconcileRedisSnapshot) Reconcile(request reconcile.Request) (reconcile // check status, if complete return if instance.Status.Phase == croType.PhaseComplete { r.logger.Infof("skipping creation of snapshot for %s as phase is complete", instance.Name) - return reconcile.Result{}, nil + return reconcile.Result{Requeue: true, RequeueAfter: resources.SuccessReconcileTime}, nil } // get redis cr @@ -123,26 +121,27 @@ func (r *ReconcileRedisSnapshot) Reconcile(request reconcile.Request) (reconcile if err != nil { errMsg := fmt.Sprintf("failed to get redis cr : %s", err.Error()) if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.New(errMsg) } // check redis cr deployment type is aws if redisCr.Status.Strategy != providers.AWSDeploymentStrategy { errMsg := fmt.Sprintf("the resource %s uses an unsupported provider strategy %s, only resources using the aws provider are valid", instance.Spec.ResourceName, redisCr.Status.Strategy) if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } - return reconcile.Result{}, errorUtil.New(errMsg) + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.New(errMsg) } // get resource region stratCfg, err := r.ConfigManager.ReadStorageStrategy(ctx, providers.RedisResourceType, redisCr.Spec.Tier) if err != nil { if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(err.Error())); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } - return reconcile.Result{}, err + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, err } if stratCfg.Region == "" { stratCfg.Region = croAws.DefaultRegion @@ -153,9 +152,9 @@ func (r *ReconcileRedisSnapshot) Reconcile(request reconcile.Request) (reconcile if err != nil { errMsg := "failed to reconcile elasticache credentials" if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } - return reconcile.Result{}, errorUtil.Wrap(err, errMsg) + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.Wrap(err, errMsg) } // setup aws elasticache cluster sdk session @@ -167,13 +166,13 @@ func (r *ReconcileRedisSnapshot) Reconcile(request reconcile.Request) (reconcile // create snapshot of primary node phase, msg, err := r.createSnapshot(ctx, cacheSvc, instance, redisCr) if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, phase, msg); updateErr != nil { - return reconcile.Result{}, updateErr + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr } if err != nil { - return reconcile.Result{}, err + return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, err } - return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 60}, nil + return reconcile.Result{Requeue: true, RequeueAfter: resources.SuccessReconcileTime}, nil } func (r *ReconcileRedisSnapshot) createSnapshot(ctx context.Context, cacheSvc elasticacheiface.ElastiCacheAPI, snapshot *integreatlyv1alpha1.RedisSnapshot, redis *integreatlyv1alpha1.Redis) (croType.StatusPhase, croType.StatusMessage, error) { diff --git a/pkg/resources/config.go b/pkg/resources/config.go index cf7db61e9..53124dfbe 100644 --- a/pkg/resources/config.go +++ b/pkg/resources/config.go @@ -14,6 +14,8 @@ import ( const ( EnvForceReconcileTimeout = "ENV_FORCE_RECONCILE_TIMEOUT" DefaultTagKeyPrefix = "integreatly.org/" + ErrorReconcileTime = time.Second * 30 + SuccessReconcileTime = time.Second * 60 ) // returns envar for reconcile time else returns default time