ebs br: make snapshot size calculation optional (#5202)

cmd/backup-manager/app/backup/manager.go

@@ -319,7 +319,7 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
 	defer func() {
 		// Calculate the backup size for ebs backup job even if it fails
 		if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize {
-			fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider)
+			fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider, backup.Spec.CalcSizeLevel)
 			if err != nil {
 				klog.Errorf("Failed to calc volume snapshot backup, err: %v", err)
 				return
@@ -332,6 +332,7 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
 				BackupSizeReadable:            &backupSizeReadable,
 				IncrementalBackupSize:         &incrementalBackupSize,
 				IncrementalBackupSizeReadable: &incrementalBackupSizeReadable,
+				TimeCompleted:                 &metav1.Time{Time: time.Now()},
 			}
 
 			if err := bm.StatusUpdater.Update(backup, nil, updateStatus); err != nil {

cmd/backup-manager/app/util/backup_size.go

@@ -45,16 +45,20 @@ import (
 // interface CalcVolSnapBackupSize called by backup and backup clean.
 
 const (
-	// This value can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned.
+	// DescribeSnapMaxReturnResult can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned.
 	DescribeSnapMaxReturnResult = 1000
-	// This value can be between 100 and 1,0000, and charge ~0.6$/1 million request
+	// ListSnapMaxReturnResult can be between 100 and 10,000, and charge ~0.6$/1 million request
 	ListSnapMaxReturnResult = 10000
-	// This value can be between 1 and 50 due to aws service quota
+	// EbsApiConcurrency can be between 1 and 50 due to aws service quota
 	EbsApiConcurrency = 40
+
+	CalculateFullSize    = "full"
+	CalculateIncremental = "incremental"
+	CalculateAll         = "all"
 )
 
 // CalcVolSnapBackupSize get snapshots from backup meta and then calc the backup size of snapshots.
-func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider) (fullBackupSize int64, incrementalBackupSize int64, err error) {
+func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
 	start := time.Now()
 	// retrieves all snapshots from backup meta file
 	volSnapshots, err := getSnapshotsFromBackupmeta(ctx, provider)
@@ -66,7 +70,7 @@ func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvide
 		return 0, 0, err
 	}
 
-	fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots)
+	fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots, level)
 
 	if err != nil {
 		return 0, 0, err
@@ -136,7 +140,7 @@ func getSnapshotsFromBackupmeta(ctx context.Context, provider v1alpha1.StoragePr
 }
 
 // calcBackupSize get a volume-snapshots backup size
-func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
+func calcBackupSize(ctx context.Context, volumes map[string]string, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
 	var apiReqCount, incrementalApiReqCount uint64
 
 	workerPool := util.NewWorkerPool(EbsApiConcurrency, "list snapshot size")
@@ -147,30 +151,35 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS
 		volumeId := vid
 		// sort snapshots by timestamp
 		workerPool.ApplyOnErrorGroup(eg, func() error {
-			snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId)
-			if err != nil {
-				return err
+			var snapSize uint64
+			if level == CalculateAll || level == CalculateFullSize {
+				snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId)
+				if err != nil {
+					return err
+				}
+				atomic.AddInt64(&fullBackupSize, int64(snapSize))
+				atomic.AddUint64(&apiReqCount, apiReq)
 			}
-			atomic.AddInt64(&fullBackupSize, int64(snapSize))
-			atomic.AddUint64(&apiReqCount, apiReq)
 
-			volSnapshots, err := getVolSnapshots(volumeId)
-			if err != nil {
-				return err
-			}
-			prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots)
-			if !existed {
-				// if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size
-				atomic.AddInt64(&incrementalBackupSize, int64(snapSize))
-				return nil
+			if level == CalculateAll || level == CalculateIncremental {
+				volSnapshots, err := getVolSnapshots(volumeId)
+				if err != nil {
+					return err
+				}
+				prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots)
+				if !existed {
+					// if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size
+					atomic.AddInt64(&incrementalBackupSize, int64(snapSize))
+					return nil
+				}
+				klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId)
+				incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId)
+				if err != nil {
+					return err
+				}
+				atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize))
+				atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq)
 			}
-			klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId)
-			incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId)
-			if err != nil {
-				return err
-			}
-			atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize))
-			atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq)
 			return nil
 		})
 	}
@@ -217,8 +226,10 @@ func calculateSnapshotSize(volumeId, snapshotId string) (uint64, uint64, error)
 		}
 		nextToken = resp.NextToken
 	}
+
 	klog.Infof("full snapshot size %s, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s",
 		humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
+
 	return snapshotSize, numApiReq, nil
 }
 
@@ -230,6 +241,7 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uin
 
 	klog.Infof("start to calculate incremental snapshot size for %s, base on prev snapshot %s, volume id %s",
 		snapshotId, preSnapshotId, volumeId)
+
 	ebsSession, err := util.NewEBSSession(util.CloudAPIConcurrency)
 	if err != nil {
 		klog.Errorf("new a ebs session failure.")
@@ -264,6 +276,7 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uin
 		}
 		nextToken = resp.NextToken
 	}
+
 	klog.Infof("incremental snapshot size %s, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s",
 		humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
 	return snapshotSize, numApiReq, nil

docs/api-references/docs.md

@@ -270,6 +270,18 @@ bool
 </tr>
 <tr>
 <td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>federalVolumeBackupPhase</code></br>
 <em>
 <a href="#federalvolumebackupphase">
@@ -4036,6 +4048,18 @@ bool
 </tr>
 <tr>
 <td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>federalVolumeBackupPhase</code></br>
 <em>
 <a href="#federalvolumebackupphase">

docs/api-references/federation-docs.md

@@ -695,6 +695,18 @@ string
 <p>PriorityClassName of Backup Job Pods</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="volumebackupmemberstatus">VolumeBackupMemberStatus</h3>

manifests/crd.yaml

@@ -503,6 +503,9 @@ spec:
                 required:
                 - cluster
                 type: object
+              calcSizeLevel:
+                default: all
+                type: string
               cleanOption:
                 properties:
                   backoffEnabled:
@@ -2126,6 +2129,9 @@ spec:
                     required:
                     - cluster
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanOption:
                     properties:
                       backoffEnabled:
@@ -3553,6 +3559,9 @@ spec:
                     required:
                     - cluster
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanOption:
                     properties:
                       backoffEnabled:

manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml

@@ -90,6 +90,9 @@ spec:
                       sendCredToTikv:
                         type: boolean
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanPolicy:
                     type: string
                   env:

manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml

@@ -98,6 +98,9 @@ spec:
                           sendCredToTikv:
                             type: boolean
                         type: object
+                      calcSizeLevel:
+                        default: all
+                        type: string
                       cleanPolicy:
                         type: string
                       env:

manifests/crd/v1/pingcap.com_backups.yaml

@@ -503,6 +503,9 @@ spec:
                 required:
                 - cluster
                 type: object
+              calcSizeLevel:
+                default: all
+                type: string
               cleanOption:
                 properties:
                   backoffEnabled:

manifests/crd/v1/pingcap.com_backupschedules.yaml

@@ -478,6 +478,9 @@ spec:
                     required:
                     - cluster
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanOption:
                     properties:
                       backoffEnabled:
@@ -1905,6 +1908,9 @@ spec:
                     required:
                     - cluster
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanOption:
                     properties:
                       backoffEnabled:

manifests/federation-crd.yaml

@@ -90,6 +90,9 @@ spec:
                       sendCredToTikv:
                         type: boolean
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanPolicy:
                     type: string
                   env:
@@ -1143,6 +1146,9 @@ spec:
                           sendCredToTikv:
                             type: boolean
                         type: object
+                      calcSizeLevel:
+                        default: all
+                        type: string
                       cleanPolicy:
                         type: string
                       env:

pkg/apis/federation/pingcap/v1alpha1/types.go

@@ -113,6 +113,10 @@ type VolumeBackupMemberSpec struct {
 	CleanPolicy pingcapv1alpha1.CleanPolicyType `json:"cleanPolicy,omitempty"`
 	// PriorityClassName of Backup Job Pods
 	PriorityClassName string `json:"priorityClassName,omitempty"`
+	// CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup
+	// +optional
+	// +kubebuilder:default="all"
+	CalcSizeLevel string `json:"calcSizeLevel,omitempty"`
 }
 
 // BRConfig contains config for BR

pkg/apis/pingcap/v1alpha1/types.go

@@ -1949,6 +1949,10 @@ type BackupSpec struct {
 	// LogStop indicates that will stop the log backup.
 	// +optional
 	LogStop bool `json:"logStop,omitempty"`
+	// CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup
+	// +optional
+	// +kubebuilder:default="all"
+	CalcSizeLevel string `json:"calcSizeLevel,omitempty"`
 	// FederalVolumeBackupPhase indicates which phase to execute in federal volume backup
 	// +optional
 	FederalVolumeBackupPhase FederalVolumeBackupPhase `json:"federalVolumeBackupPhase,omitempty"`

pkg/backup/snapshotter/snapshotter_test.go

@@ -429,8 +429,9 @@ func TestGenerateBackupMetadata(t *testing.T) {
 					Annotations: make(map[string]string),
 				},
 				Spec: v1alpha1.BackupSpec{
-					Type: v1alpha1.BackupTypeFull,
-					Mode: v1alpha1.BackupModeVolumeSnapshot,
+					Type:          v1alpha1.BackupTypeFull,
+					Mode:          v1alpha1.BackupModeVolumeSnapshot,
+					CalcSizeLevel: "all",
 				},
 			},
 			wantSSNil: false,