Skip to content

Commit

Permalink
Update reconcile times
Browse files Browse the repository at this point in the history
  • Loading branch information
@ciaranRoche
ciaranRoche committed Jan 16, 2020
1 parent 9304f8d commit 7781424
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 29 deletions.
26 changes: 12 additions & 14 deletions pkg/controller/postgressnapshot/postgressnapshot_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ package postgressnapshot
import (
"context"
"fmt"
"time"

"github.com/aws/aws-sdk-go/service/rds/rdsiface"

"github.com/aws/aws-sdk-go/aws"
Expand Down Expand Up @@ -113,7 +111,7 @@ func (r *ReconcilePostgresSnapshot) Reconcile(request reconcile.Request) (reconc
// check status, if complete return
if instance.Status.Phase == croType.PhaseComplete {
r.logger.Infof("skipping creation of snapshot for %s as phase is complete", instance.Name)
return reconcile.Result{}, nil
return reconcile.Result{Requeue: true, RequeueAfter: resources.SuccessReconcileTime}, nil
}

// get postgres cr
Expand All @@ -122,27 +120,27 @@ func (r *ReconcilePostgresSnapshot) Reconcile(request reconcile.Request) (reconc
if err != nil {
errMsg := fmt.Sprintf("failed to get postgres resource: %s", err.Error())
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
return reconcile.Result{}, errorUtil.New(errMsg)
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.New(errMsg)
}

// check postgres deployment strategy is aws
if postgresCr.Status.Strategy != providers.AWSDeploymentStrategy {
errMsg := fmt.Sprintf("the resource %s uses an unsupported provider strategy %s, only resources using the aws provider are valid", instance.Spec.ResourceName, postgresCr.Status.Strategy)
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
return reconcile.Result{}, errorUtil.New(errMsg)
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.New(errMsg)
}

// get resource region
stratCfg, err := r.ConfigManager.ReadStorageStrategy(ctx, providers.PostgresResourceType, postgresCr.Spec.Tier)
if err != nil {
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(err.Error())); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
return reconcile.Result{}, err
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, err
}
if stratCfg.Region == "" {
stratCfg.Region = croAws.DefaultRegion
Expand All @@ -153,9 +151,9 @@ func (r *ReconcilePostgresSnapshot) Reconcile(request reconcile.Request) (reconc
if err != nil {
errMsg := "failed to reconcile rds credentials"
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
return reconcile.Result{}, errorUtil.Wrap(err, errMsg)
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.Wrap(err, errMsg)
}

// setup aws rds session
Expand All @@ -167,12 +165,12 @@ func (r *ReconcilePostgresSnapshot) Reconcile(request reconcile.Request) (reconc
// create the snapshot and return the phase
phase, msg, err := r.createSnapshot(ctx, rdsSvc, instance, postgresCr)
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, phase, msg); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
if err != nil {
return reconcile.Result{}, err
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, err
}
return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 60}, nil
return reconcile.Result{Requeue: true, RequeueAfter: resources.SuccessReconcileTime}, nil
}

func (r *ReconcilePostgresSnapshot) createSnapshot(ctx context.Context, rdsSvc rdsiface.RDSAPI, snapshot *integreatlyv1alpha1.PostgresSnapshot, postgres *integreatlyv1alpha1.Postgres) (croType.StatusPhase, croType.StatusMessage, error) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/postgressnapshot/postgressnapshot_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ func TestReconcilePostgresSnapshot_createSnapshot(t *testing.T) {
name: "test successful snapshot in progress",
args: args{
ctx: ctx,
rdsSvc: &mockRdsClient{dbSnapshot: buildSnapshot(), dbSnapshots: buildSnapshots(snapshotName, "creatring")},
rdsSvc: &mockRdsClient{dbSnapshot: buildSnapshot(), dbSnapshots: buildSnapshots(snapshotName, "creating")},
snapshot: buildPostgresSnapshot(),
postgres: buildPostgres(),
},
Expand Down
27 changes: 13 additions & 14 deletions pkg/controller/redissnapshot/redissnapshot_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,11 @@ package redissnapshot
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go/service/elasticache/elasticacheiface"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/elasticache"
"github.com/aws/aws-sdk-go/service/elasticache/elasticacheiface"
croType "github.com/integr8ly/cloud-resource-operator/pkg/apis/integreatly/v1alpha1/types"
"github.com/integr8ly/cloud-resource-operator/pkg/providers"
croAws "github.com/integr8ly/cloud-resource-operator/pkg/providers/aws"
Expand Down Expand Up @@ -114,7 +112,7 @@ func (r *ReconcileRedisSnapshot) Reconcile(request reconcile.Request) (reconcile
// check status, if complete return
if instance.Status.Phase == croType.PhaseComplete {
r.logger.Infof("skipping creation of snapshot for %s as phase is complete", instance.Name)
return reconcile.Result{}, nil
return reconcile.Result{Requeue: true, RequeueAfter: resources.SuccessReconcileTime}, nil
}

// get redis cr
Expand All @@ -123,26 +121,27 @@ func (r *ReconcileRedisSnapshot) Reconcile(request reconcile.Request) (reconcile
if err != nil {
errMsg := fmt.Sprintf("failed to get redis cr : %s", err.Error())
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.New(errMsg)
}

// check redis cr deployment type is aws
if redisCr.Status.Strategy != providers.AWSDeploymentStrategy {
errMsg := fmt.Sprintf("the resource %s uses an unsupported provider strategy %s, only resources using the aws provider are valid", instance.Spec.ResourceName, redisCr.Status.Strategy)
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
return reconcile.Result{}, errorUtil.New(errMsg)
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.New(errMsg)
}

// get resource region
stratCfg, err := r.ConfigManager.ReadStorageStrategy(ctx, providers.RedisResourceType, redisCr.Spec.Tier)
if err != nil {
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(err.Error())); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
return reconcile.Result{}, err
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, err
}
if stratCfg.Region == "" {
stratCfg.Region = croAws.DefaultRegion
Expand All @@ -153,9 +152,9 @@ func (r *ReconcileRedisSnapshot) Reconcile(request reconcile.Request) (reconcile
if err != nil {
errMsg := "failed to reconcile elasticache credentials"
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, croType.PhaseFailed, croType.StatusMessage(errMsg)); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
return reconcile.Result{}, errorUtil.Wrap(err, errMsg)
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, errorUtil.Wrap(err, errMsg)
}

// setup aws elasticache cluster sdk session
Expand All @@ -167,13 +166,13 @@ func (r *ReconcileRedisSnapshot) Reconcile(request reconcile.Request) (reconcile
// create snapshot of primary node
phase, msg, err := r.createSnapshot(ctx, cacheSvc, instance, redisCr)
if updateErr := resources.UpdateSnapshotPhase(ctx, r.client, instance, phase, msg); updateErr != nil {
return reconcile.Result{}, updateErr
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, updateErr
}
if err != nil {
return reconcile.Result{}, err
return reconcile.Result{Requeue: true, RequeueAfter: resources.ErrorReconcileTime}, err
}

return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 60}, nil
return reconcile.Result{Requeue: true, RequeueAfter: resources.SuccessReconcileTime}, nil
}

func (r *ReconcileRedisSnapshot) createSnapshot(ctx context.Context, cacheSvc elasticacheiface.ElastiCacheAPI, snapshot *integreatlyv1alpha1.RedisSnapshot, redis *integreatlyv1alpha1.Redis) (croType.StatusPhase, croType.StatusMessage, error) {
Expand Down
2 changes: 2 additions & 0 deletions pkg/resources/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
const (
EnvForceReconcileTimeout = "ENV_FORCE_RECONCILE_TIMEOUT"
DefaultTagKeyPrefix = "integreatly.org/"
ErrorReconcileTime = time.Second * 30
SuccessReconcileTime = time.Second * 60
)

// returns envar for reconcile time else returns default time
Expand Down

0 comments on commit 7781424

Please sign in to comment.