From 2f7fed23f5d239b1a47af731c45ccfb4fdcfa0d1 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Tue, 30 May 2023 13:59:57 +0800 Subject: [PATCH] *: support ebs backup schedule Signed-off-by: BornChanger --- docs/api-references/docs.md | 4 +- docs/api-references/federation-docs.md | 180 +++- manifests/crd.yaml | 6 +- ...ion.pingcap.com_volumebackupschedules.yaml | 953 +++++++++++++++++- ...ion.pingcap.com_volumebackupschedules.yaml | 951 ++++++++++++++++- .../crd/v1/pingcap.com_backupschedules.yaml | 6 +- .../v1beta1/pingcap.com_backupschedules.yaml | 6 +- manifests/crd_v1beta1.yaml | 6 +- manifests/federation-crd.yaml | 953 +++++++++++++++++- manifests/federation-crd_v1beta1.yaml | 951 ++++++++++++++++- .../pingcap/v1alpha1/openapi_generated.go | 41 + pkg/apis/federation/pingcap/v1alpha1/types.go | 31 +- .../pingcap/v1alpha1/volume_backup.go | 6 + .../v1alpha1/volume_backup_schedule.go | 25 + .../pingcap/v1alpha1/zz_generated.deepcopy.go | 23 +- .../pingcap/v1alpha1/openapi_generated.go | 2 +- pkg/apis/pingcap/v1alpha1/types.go | 3 +- .../backupschedule/backup_schedule_manager.go | 7 +- pkg/controller/controller_utils.go | 14 + .../fed_backup_schedule_status_updater.go | 113 +++ pkg/controller/fed_volume_backup_control.go | 11 +- .../fed_volume_backup_schedule_control.go | 34 +- ...fed_volume_backup_schedule_control_test.go | 120 +++ .../fed_volume_backup_schedule_controller.go | 2 +- ..._volume_backup_schedule_controller_test.go | 180 ++++ pkg/fedvolumebackup/backup/backup_manager.go | 34 +- .../backupschedule/backup_schedule_manager.go | 310 +++++- .../backup_schedule_manager_test.go | 516 ++++++++++ 28 files changed, 5423 insertions(+), 65 deletions(-) create mode 100644 pkg/apis/federation/pingcap/v1alpha1/volume_backup_schedule.go create mode 100644 pkg/controller/fed_backup_schedule_status_updater.go create mode 100644 pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control_test.go create mode 100644 pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller_test.go create mode 100644 pkg/fedvolumebackup/backupschedule/backup_schedule_manager_test.go diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 0cf873b5bf0..aa7ce061ee8 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -578,7 +578,6 @@ BackupSpec -(Optional)

BackupTemplate is the specification of the backup structure to get scheduled.

@@ -592,6 +591,7 @@ BackupSpec +(Optional)

LogBackupTemplate is the specification of the log backup structure to get scheduled.

@@ -3694,7 +3694,6 @@ BackupSpec -(Optional)

BackupTemplate is the specification of the backup structure to get scheduled.

@@ -3708,6 +3707,7 @@ BackupSpec +(Optional)

LogBackupTemplate is the specification of the log backup structure to get scheduled.

diff --git a/docs/api-references/federation-docs.md b/docs/api-references/federation-docs.md index 75ca98728ec..2f8c510677d 100644 --- a/docs/api-references/federation-docs.md +++ b/docs/api-references/federation-docs.md @@ -166,6 +166,66 @@ VolumeBackupScheduleSpec

+ + + + + + + + + + + + + + + + + + + +
+schedule
+ +string + +		 +

Schedule specifies the cron string used for backup scheduling.

+
+pause
+ +bool + +		 +

Pause means paused backupSchedule

+
+maxBackups
+ +int32 + +		 +

MaxBackups is to specify how many backups we want to keep +0 is magic number to indicate un-limited backups. +if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred +and MaxBackups is ignored.

+
+maxReservedTime
+ +string + +		 +

MaxReservedTime is to specify how long backups we want to keep.

+
+backupTemplate
+ + +VolumeBackupSpec + + +		 +

BackupTemplate is the specification of the volume backup structure to get scheduled.

+
@@ -713,6 +773,76 @@ string

VolumeBackupScheduleSpec describes the attributes that a user creates on a volume backup schedule.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+schedule
+ +string + +		 +

Schedule specifies the cron string used for backup scheduling.

+
+pause
+ +bool + +		 +

Pause means paused backupSchedule

+
+maxBackups
+ +int32 + +		 +

MaxBackups is to specify how many backups we want to keep +0 is magic number to indicate un-limited backups. +if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred +and MaxBackups is ignored.

+
+maxReservedTime
+ +string + +		 +

MaxReservedTime is to specify how long backups we want to keep.

+
+backupTemplate
+ + +VolumeBackupSpec + + +		 +

BackupTemplate is the specification of the volume backup structure to get scheduled.

+

VolumeBackupScheduleStatus

(Appears on: @@ -721,10 +851,58 @@ string

VolumeBackupScheduleStatus represents the current status of a volume backup schedule.

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+lastBackup
+ +string + +		 +

LastBackup represents the last backup.

+
+lastBackupTime
+ + +Kubernetes meta/v1.Time + + +		 +

LastBackupTime represents the last time the backup was successfully created.

+
+allBackupCleanTime
+ + +Kubernetes meta/v1.Time + + +		 +

AllBackupCleanTime represents the time when all backup entries are cleaned up

+

VolumeBackupSpec

(Appears on: -VolumeBackup) +VolumeBackup, +VolumeBackupScheduleSpec)

VolumeBackupSpec describes the attributes that a user creates on a volume backup.

diff --git a/manifests/crd.yaml b/manifests/crd.yaml index b82bbf49710..af635095bf0 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -1554,6 +1554,10 @@ spec: jsonPath: .spec.maxBackups name: MaxBackups type: integer + - description: How long backups we want to keep + jsonPath: .spec.maxReservedTime + name: MaxReservedTime + type: string - description: The last backup CR name jsonPath: .status.lastBackup name: LastBackup @@ -4232,7 +4236,7 @@ spec: storageSize: type: string required: - - logBackupTemplate + - backupTemplate - schedule type: object status: diff --git a/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml b/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml index 748bab61d90..386ccf98e5b 100644 --- a/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml +++ b/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml @@ -14,11 +14,37 @@ spec: listKind: VolumeBackupScheduleList plural: volumebackupschedules shortNames: - - vbks + - vbfs singular: volumebackupschedule scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: The cron format string used for backup scheduling + jsonPath: .spec.schedule + name: Schedule + type: string + - description: The max number of backups we want to keep + jsonPath: .spec.maxBackups + name: MaxBackups + type: integer + - description: How long backups we want to keep + jsonPath: .spec.maxReservedTime + name: MaxReservedTime + type: string + - description: The last backup CR name + jsonPath: .status.lastBackup + name: LastBackup + priority: 1 + type: string + - description: The last time the backup was successfully created + jsonPath: .status.lastBackupTime + name: LastBackupTime + priority: 1 + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: properties: @@ -29,8 +55,930 @@ spec: metadata: type: object spec: + properties: + backupTemplate: + properties: + clusters: + items: + properties: + k8sClusterName: + type: string + tcName: + type: string + tcNamespace: + type: string + type: object + type: array + template: + properties: + azblob: + properties: + accessTier: + type: string + container: + type: string + path: + type: string + prefix: + type: string + secretName: + type: string + type: object + br: + properties: + checkRequirements: + type: boolean + concurrency: + format: int32 + type: integer + options: + items: + type: string + type: array + sendCredToTikv: + type: boolean + type: object + cleanPolicy: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + gcs: + properties: + bucket: + type: string + bucketAcl: + type: string + location: + type: string + objectAcl: + type: string + path: + type: string + prefix: + type: string + projectId: + type: string + secretName: + type: string + storageClass: + type: string + required: + - projectId + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + local: + properties: + prefix: + type: string + volume: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + required: + - volume + - volumeMount + type: object + priorityClassName: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + s3: + properties: + acl: + type: string + bucket: + type: string + endpoint: + type: string + options: + items: + type: string + type: array + path: + type: string + prefix: + type: string + provider: + type: string + region: + type: string + secretName: + type: string + sse: + type: string + storageClass: + type: string + required: + - provider + type: object + serviceAccount: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + toolImage: + type: string + type: object + type: object + maxBackups: + format: int32 + type: integer + maxReservedTime: + type: string + pause: + type: boolean + schedule: + type: string + required: + - backupTemplate + - schedule type: object status: + properties: + allBackupCleanTime: + format: date-time + type: string + lastBackup: + type: string + lastBackupTime: + format: date-time + type: string type: object required: - metadata @@ -38,6 +986,7 @@ spec: type: object served: true storage: true + subresources: {} status: acceptedNames: kind: "" diff --git a/manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml b/manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml index 29d26b06e90..1809519859a 100644 --- a/manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml +++ b/manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml @@ -8,16 +8,43 @@ metadata: creationTimestamp: null name: volumebackupschedules.federation.pingcap.com spec: + additionalPrinterColumns: + - JSONPath: .spec.schedule + description: The cron format string used for backup scheduling + name: Schedule + type: string + - JSONPath: .spec.maxBackups + description: The max number of backups we want to keep + name: MaxBackups + type: integer + - JSONPath: .spec.maxReservedTime + description: How long backups we want to keep + name: MaxReservedTime + type: string + - JSONPath: .status.lastBackup + description: The last backup CR name + name: LastBackup + priority: 1 + type: string + - JSONPath: .status.lastBackupTime + description: The last time the backup was successfully created + name: LastBackupTime + priority: 1 + type: date + - JSONPath: .metadata.creationTimestamp + name: Age + type: date group: federation.pingcap.com names: kind: VolumeBackupSchedule listKind: VolumeBackupScheduleList plural: volumebackupschedules shortNames: - - vbks + - vbfs singular: volumebackupschedule preserveUnknownFields: false scope: Namespaced + subresources: {} validation: openAPIV3Schema: properties: @@ -28,8 +55,930 @@ spec: metadata: type: object spec: + properties: + backupTemplate: + properties: + clusters: + items: + properties: + k8sClusterName: + type: string + tcName: + type: string + tcNamespace: + type: string + type: object + type: array + template: + properties: + azblob: + properties: + accessTier: + type: string + container: + type: string + path: + type: string + prefix: + type: string + secretName: + type: string + type: object + br: + properties: + checkRequirements: + type: boolean + concurrency: + format: int32 + type: integer + options: + items: + type: string + type: array + sendCredToTikv: + type: boolean + type: object + cleanPolicy: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + gcs: + properties: + bucket: + type: string + bucketAcl: + type: string + location: + type: string + objectAcl: + type: string + path: + type: string + prefix: + type: string + projectId: + type: string + secretName: + type: string + storageClass: + type: string + required: + - projectId + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + local: + properties: + prefix: + type: string + volume: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + required: + - volume + - volumeMount + type: object + priorityClassName: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + s3: + properties: + acl: + type: string + bucket: + type: string + endpoint: + type: string + options: + items: + type: string + type: array + path: + type: string + prefix: + type: string + provider: + type: string + region: + type: string + secretName: + type: string + sse: + type: string + storageClass: + type: string + required: + - provider + type: object + serviceAccount: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + toolImage: + type: string + type: object + type: object + maxBackups: + format: int32 + type: integer + maxReservedTime: + type: string + pause: + type: boolean + schedule: + type: string + required: + - backupTemplate + - schedule type: object status: + properties: + allBackupCleanTime: + format: date-time + type: string + lastBackup: + type: string + lastBackupTime: + format: date-time + type: string type: object required: - metadata diff --git a/manifests/crd/v1/pingcap.com_backupschedules.yaml b/manifests/crd/v1/pingcap.com_backupschedules.yaml index 43175cd21e8..c6479d6916f 100644 --- a/manifests/crd/v1/pingcap.com_backupschedules.yaml +++ b/manifests/crd/v1/pingcap.com_backupschedules.yaml @@ -27,6 +27,10 @@ spec: jsonPath: .spec.maxBackups name: MaxBackups type: integer + - description: How long backups we want to keep + jsonPath: .spec.maxReservedTime + name: MaxReservedTime + type: string - description: The last backup CR name jsonPath: .status.lastBackup name: LastBackup @@ -2705,7 +2709,7 @@ spec: storageSize: type: string required: - - logBackupTemplate + - backupTemplate - schedule type: object status: diff --git a/manifests/crd/v1beta1/pingcap.com_backupschedules.yaml b/manifests/crd/v1beta1/pingcap.com_backupschedules.yaml index e865a48de32..c8ce37aef50 100644 --- a/manifests/crd/v1beta1/pingcap.com_backupschedules.yaml +++ b/manifests/crd/v1beta1/pingcap.com_backupschedules.yaml @@ -17,6 +17,10 @@ spec: description: The max number of backups we want to keep name: MaxBackups type: integer + - JSONPath: .spec.maxReservedTime + description: How long backups we want to keep + name: MaxReservedTime + type: string - JSONPath: .status.lastBackup description: The last backup CR name name: LastBackup @@ -2695,7 +2699,7 @@ spec: storageSize: type: string required: - - logBackupTemplate + - backupTemplate - schedule type: object status: diff --git a/manifests/crd_v1beta1.yaml b/manifests/crd_v1beta1.yaml index 9b909aca102..53742a0c660 100644 --- a/manifests/crd_v1beta1.yaml +++ b/manifests/crd_v1beta1.yaml @@ -1541,6 +1541,10 @@ spec: description: The max number of backups we want to keep name: MaxBackups type: integer + - JSONPath: .spec.maxReservedTime + description: How long backups we want to keep + name: MaxReservedTime + type: string - JSONPath: .status.lastBackup description: The last backup CR name name: LastBackup @@ -4219,7 +4223,7 @@ spec: storageSize: type: string required: - - logBackupTemplate + - backupTemplate - schedule type: object status: diff --git a/manifests/federation-crd.yaml b/manifests/federation-crd.yaml index ac30a565b30..7a3690d86f0 100644 --- a/manifests/federation-crd.yaml +++ b/manifests/federation-crd.yaml @@ -1042,11 +1042,37 @@ spec: listKind: VolumeBackupScheduleList plural: volumebackupschedules shortNames: - - vbks + - vbfs singular: volumebackupschedule scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: The cron format string used for backup scheduling + jsonPath: .spec.schedule + name: Schedule + type: string + - description: The max number of backups we want to keep + jsonPath: .spec.maxBackups + name: MaxBackups + type: integer + - description: How long backups we want to keep + jsonPath: .spec.maxReservedTime + name: MaxReservedTime + type: string + - description: The last backup CR name + jsonPath: .status.lastBackup + name: LastBackup + priority: 1 + type: string + - description: The last time the backup was successfully created + jsonPath: .status.lastBackupTime + name: LastBackupTime + priority: 1 + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: properties: @@ -1057,8 +1083,930 @@ spec: metadata: type: object spec: + properties: + backupTemplate: + properties: + clusters: + items: + properties: + k8sClusterName: + type: string + tcName: + type: string + tcNamespace: + type: string + type: object + type: array + template: + properties: + azblob: + properties: + accessTier: + type: string + container: + type: string + path: + type: string + prefix: + type: string + secretName: + type: string + type: object + br: + properties: + checkRequirements: + type: boolean + concurrency: + format: int32 + type: integer + options: + items: + type: string + type: array + sendCredToTikv: + type: boolean + type: object + cleanPolicy: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + gcs: + properties: + bucket: + type: string + bucketAcl: + type: string + location: + type: string + objectAcl: + type: string + path: + type: string + prefix: + type: string + projectId: + type: string + secretName: + type: string + storageClass: + type: string + required: + - projectId + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + local: + properties: + prefix: + type: string + volume: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + required: + - volume + - volumeMount + type: object + priorityClassName: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + s3: + properties: + acl: + type: string + bucket: + type: string + endpoint: + type: string + options: + items: + type: string + type: array + path: + type: string + prefix: + type: string + provider: + type: string + region: + type: string + secretName: + type: string + sse: + type: string + storageClass: + type: string + required: + - provider + type: object + serviceAccount: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + toolImage: + type: string + type: object + type: object + maxBackups: + format: int32 + type: integer + maxReservedTime: + type: string + pause: + type: boolean + schedule: + type: string + required: + - backupTemplate + - schedule type: object status: + properties: + allBackupCleanTime: + format: date-time + type: string + lastBackup: + type: string + lastBackupTime: + format: date-time + type: string type: object required: - metadata @@ -1066,6 +2014,7 @@ spec: type: object served: true storage: true + subresources: {} status: acceptedNames: kind: "" diff --git a/manifests/federation-crd_v1beta1.yaml b/manifests/federation-crd_v1beta1.yaml index 2f0438a4aa8..00dff0db474 100644 --- a/manifests/federation-crd_v1beta1.yaml +++ b/manifests/federation-crd_v1beta1.yaml @@ -1038,16 +1038,43 @@ metadata: creationTimestamp: null name: volumebackupschedules.federation.pingcap.com spec: + additionalPrinterColumns: + - JSONPath: .spec.schedule + description: The cron format string used for backup scheduling + name: Schedule + type: string + - JSONPath: .spec.maxBackups + description: The max number of backups we want to keep + name: MaxBackups + type: integer + - JSONPath: .spec.maxReservedTime + description: How long backups we want to keep + name: MaxReservedTime + type: string + - JSONPath: .status.lastBackup + description: The last backup CR name + name: LastBackup + priority: 1 + type: string + - JSONPath: .status.lastBackupTime + description: The last time the backup was successfully created + name: LastBackupTime + priority: 1 + type: date + - JSONPath: .metadata.creationTimestamp + name: Age + type: date group: federation.pingcap.com names: kind: VolumeBackupSchedule listKind: VolumeBackupScheduleList plural: volumebackupschedules shortNames: - - vbks + - vbfs singular: volumebackupschedule preserveUnknownFields: false scope: Namespaced + subresources: {} validation: openAPIV3Schema: properties: @@ -1058,8 +1085,930 @@ spec: metadata: type: object spec: + properties: + backupTemplate: + properties: + clusters: + items: + properties: + k8sClusterName: + type: string + tcName: + type: string + tcNamespace: + type: string + type: object + type: array + template: + properties: + azblob: + properties: + accessTier: + type: string + container: + type: string + path: + type: string + prefix: + type: string + secretName: + type: string + type: object + br: + properties: + checkRequirements: + type: boolean + concurrency: + format: int32 + type: integer + options: + items: + type: string + type: array + sendCredToTikv: + type: boolean + type: object + cleanPolicy: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + gcs: + properties: + bucket: + type: string + bucketAcl: + type: string + location: + type: string + objectAcl: + type: string + path: + type: string + prefix: + type: string + projectId: + type: string + secretName: + type: string + storageClass: + type: string + required: + - projectId + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + local: + properties: + prefix: + type: string + volume: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + required: + - volume + - volumeMount + type: object + priorityClassName: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + s3: + properties: + acl: + type: string + bucket: + type: string + endpoint: + type: string + options: + items: + type: string + type: array + path: + type: string + prefix: + type: string + provider: + type: string + region: + type: string + secretName: + type: string + sse: + type: string + storageClass: + type: string + required: + - provider + type: object + serviceAccount: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + toolImage: + type: string + type: object + type: object + maxBackups: + format: int32 + type: integer + maxReservedTime: + type: string + pause: + type: boolean + schedule: + type: string + required: + - backupTemplate + - schedule type: object status: + properties: + allBackupCleanTime: + format: date-time + type: string + lastBackup: + type: string + lastBackupTime: + format: date-time + type: string type: object required: - metadata diff --git a/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go index 16fd40b1ca5..51adaa8a29c 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go @@ -417,8 +417,49 @@ func schema_apis_federation_pingcap_v1alpha1_VolumeBackupScheduleSpec(ref common SchemaProps: spec.SchemaProps{ Description: "VolumeBackupScheduleSpec describes the attributes that a user creates on a volume backup schedule.", Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "schedule": { + SchemaProps: spec.SchemaProps{ + Description: "Schedule specifies the cron string used for backup scheduling.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "pause": { + SchemaProps: spec.SchemaProps{ + Description: "Pause means paused backupSchedule", + Type: []string{"boolean"}, + Format: "", + }, + }, + "maxBackups": { + SchemaProps: spec.SchemaProps{ + Description: "MaxBackups is to specify how many backups we want to keep 0 is magic number to indicate un-limited backups. if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred and MaxBackups is ignored.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "maxReservedTime": { + SchemaProps: spec.SchemaProps{ + Description: "MaxReservedTime is to specify how long backups we want to keep.", + Type: []string{"string"}, + Format: "", + }, + }, + "backupTemplate": { + SchemaProps: spec.SchemaProps{ + Description: "BackupTemplate is the specification of the volume backup structure to get scheduled.", + Default: map[string]interface{}{}, + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1.VolumeBackupSpec"), + }, + }, + }, + Required: []string{"schedule", "backupTemplate"}, }, }, + Dependencies: []string{ + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1.VolumeBackupSpec"}, } } diff --git a/pkg/apis/federation/pingcap/v1alpha1/types.go b/pkg/apis/federation/pingcap/v1alpha1/types.go index a2c5b4c4e28..3fa2cade293 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/types.go +++ b/pkg/apis/federation/pingcap/v1alpha1/types.go @@ -186,6 +186,8 @@ type VolumeBackupConditionType string const ( // VolumeBackupInvalid means the VolumeBackup is invalid VolumeBackupInvalid VolumeBackupConditionType = "Invalid" + // VolumeBackupPrepared means the VolumeBackup preparation is done + VolumeBackupPrepared VolumeBackupConditionType = "Prepared" // VolumeBackupRunning means the VolumeBackup is running VolumeBackupRunning VolumeBackupConditionType = "Running" // VolumeBackupComplete means all the backups in data plane are complete and the VolumeBackup is complete @@ -196,8 +198,6 @@ const ( VolumeBackupCleaned VolumeBackupConditionType = "Cleaned" // VolumeBackupCleanFailed means the VolumeBackup cleanup is failed VolumeBackupCleanFailed VolumeBackupConditionType = "CleanFailed" - // VolumeBackupPrepared means the VolumeBackup is prepared - VolumeBackupPrepared VolumeBackupConditionType = "Prepared" ) // +genclient @@ -206,8 +206,14 @@ const ( // VolumeBackupSchedule is the control script's spec // // +k8s:openapi-gen=true -// +kubebuilder:resource:shortName="vbks" +// +kubebuilder:resource:shortName="vbfs" // +genclient:noStatus +// +kubebuilder:printcolumn:name="Schedule",type=string,JSONPath=`.spec.schedule`,description="The cron format string used for backup scheduling" +// +kubebuilder:printcolumn:name="MaxBackups",type=integer,JSONPath=`.spec.maxBackups`,description="The max number of backups we want to keep" +// +kubebuilder:printcolumn:name="MaxReservedTime",type=string,JSONPath=`.spec.maxReservedTime`,description="How long backups we want to keep" +// +kubebuilder:printcolumn:name="LastBackup",type=string,JSONPath=`.status.lastBackup`,description="The last backup CR name",priority=1 +// +kubebuilder:printcolumn:name="LastBackupTime",type=date,JSONPath=`.status.lastBackupTime`,description="The last time the backup was successfully created",priority=1 +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type VolumeBackupSchedule struct { metav1.TypeMeta `json:",inline"` // +k8s:openapi-gen=false @@ -232,10 +238,29 @@ type VolumeBackupScheduleList struct { // VolumeBackupScheduleSpec describes the attributes that a user creates on a volume backup schedule. // +k8s:openapi-gen=true type VolumeBackupScheduleSpec struct { + // Schedule specifies the cron string used for backup scheduling. + Schedule string `json:"schedule"` + // Pause means paused backupSchedule + Pause bool `json:"pause,omitempty"` + // MaxBackups is to specify how many backups we want to keep + // 0 is magic number to indicate un-limited backups. + // if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred + // and MaxBackups is ignored. + MaxBackups *int32 `json:"maxBackups,omitempty"` + // MaxReservedTime is to specify how long backups we want to keep. + MaxReservedTime *string `json:"maxReservedTime,omitempty"` + // BackupTemplate is the specification of the volume backup structure to get scheduled. + BackupTemplate VolumeBackupSpec `json:"backupTemplate"` } // VolumeBackupScheduleStatus represents the current status of a volume backup schedule. type VolumeBackupScheduleStatus struct { + // LastBackup represents the last backup. + LastBackup string `json:"lastBackup,omitempty"` + // LastBackupTime represents the last time the backup was successfully created. + LastBackupTime *metav1.Time `json:"lastBackupTime,omitempty"` + // AllBackupCleanTime represents the time when all backup entries are cleaned up + AllBackupCleanTime *metav1.Time `json:"allBackupCleanTime,omitempty"` } // +genclient diff --git a/pkg/apis/federation/pingcap/v1alpha1/volume_backup.go b/pkg/apis/federation/pingcap/v1alpha1/volume_backup.go index a8115ddf7a9..dedfe72b40e 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/volume_backup.go +++ b/pkg/apis/federation/pingcap/v1alpha1/volume_backup.go @@ -88,6 +88,12 @@ func IsVolumeBackupRunning(volumeBackup *VolumeBackup) bool { return condition != nil && condition.Status == corev1.ConditionTrue } +// IsBackupPrepared returns true if VolumeBackup is running +func IsBackupPrepared(volumeBackup *VolumeBackup) bool { + _, condition := GetVolumeBackupCondition(&volumeBackup.Status, VolumeBackupPrepared) + return condition != nil && condition.Status == corev1.ConditionTrue +} + // IsVolumeBackupComplete returns true if VolumeBackup is complete func IsVolumeBackupComplete(volumeBackup *VolumeBackup) bool { _, condition := GetVolumeBackupCondition(&volumeBackup.Status, VolumeBackupComplete) diff --git a/pkg/apis/federation/pingcap/v1alpha1/volume_backup_schedule.go b/pkg/apis/federation/pingcap/v1alpha1/volume_backup_schedule.go new file mode 100644 index 00000000000..a76db525f94 --- /dev/null +++ b/pkg/apis/federation/pingcap/v1alpha1/volume_backup_schedule.go @@ -0,0 +1,25 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "fmt" + "time" + + constants "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" +) + +func (vbfs *VolumeBackupSchedule) GetBackupCRDName(timestamp time.Time) string { + return fmt.Sprintf("%s-%s", vbfs.GetName(), timestamp.UTC().Format(constants.BackupNameTimeFormat)) +} diff --git a/pkg/apis/federation/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/federation/pingcap/v1alpha1/zz_generated.deepcopy.go index 44f24e3898f..9d6e4a1afad 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/federation/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -216,8 +216,8 @@ func (in *VolumeBackupSchedule) DeepCopyInto(out *VolumeBackupSchedule) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) return } @@ -275,6 +275,17 @@ func (in *VolumeBackupScheduleList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeBackupScheduleSpec) DeepCopyInto(out *VolumeBackupScheduleSpec) { *out = *in + if in.MaxBackups != nil { + in, out := &in.MaxBackups, &out.MaxBackups + *out = new(int32) + **out = **in + } + if in.MaxReservedTime != nil { + in, out := &in.MaxReservedTime, &out.MaxReservedTime + *out = new(string) + **out = **in + } + in.BackupTemplate.DeepCopyInto(&out.BackupTemplate) return } @@ -291,6 +302,14 @@ func (in *VolumeBackupScheduleSpec) DeepCopy() *VolumeBackupScheduleSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeBackupScheduleStatus) DeepCopyInto(out *VolumeBackupScheduleStatus) { *out = *in + if in.LastBackupTime != nil { + in, out := &in.LastBackupTime, &out.LastBackupTime + *out = (*in).DeepCopy() + } + if in.AllBackupCleanTime != nil { + in, out := &in.AllBackupCleanTime, &out.AllBackupCleanTime + *out = (*in).DeepCopy() + } return } diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 0b2d2748322..99e813abd07 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -945,7 +945,7 @@ func schema_pkg_apis_pingcap_v1alpha1_BackupScheduleSpec(ref common.ReferenceCal }, }, }, - Required: []string{"schedule", "logBackupTemplate"}, + Required: []string{"schedule", "backupTemplate"}, }, }, Dependencies: []string{ diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 4e28b2d1230..788214f7d85 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -2197,6 +2197,7 @@ type BackupStatus struct { // +kubebuilder:resource:shortName="bks" // +kubebuilder:printcolumn:name="Schedule",type=string,JSONPath=`.spec.schedule`,description="The cron format string used for backup scheduling" // +kubebuilder:printcolumn:name="MaxBackups",type=integer,JSONPath=`.spec.maxBackups`,description="The max number of backups we want to keep" +// +kubebuilder:printcolumn:name="MaxReservedTime",type=string,JSONPath=`.spec.maxReservedTime`,description="How long backups we want to keep" // +kubebuilder:printcolumn:name="LastBackup",type=string,JSONPath=`.status.lastBackup`,description="The last backup CR name",priority=1 // +kubebuilder:printcolumn:name="LastBackupTime",type=date,JSONPath=`.status.lastBackupTime`,description="The last time the backup was successfully created",priority=1 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` @@ -2236,9 +2237,9 @@ type BackupScheduleSpec struct { // MaxReservedTime is to specify how long backups we want to keep. MaxReservedTime *string `json:"maxReservedTime,omitempty"` // BackupTemplate is the specification of the backup structure to get scheduled. - // +optional BackupTemplate BackupSpec `json:"backupTemplate"` // LogBackupTemplate is the specification of the log backup structure to get scheduled. + // +optional LogBackupTemplate *BackupSpec `json:"logBackupTemplate"` // The storageClassName of the persistent volume for Backup data storage if not storage class name set in BackupSpec. // Defaults to Kubernetes default storage class. diff --git a/pkg/backup/backupschedule/backup_schedule_manager.go b/pkg/backup/backupschedule/backup_schedule_manager.go index 33279f8f754..843467f582d 100644 --- a/pkg/backup/backupschedule/backup_schedule_manager.go +++ b/pkg/backup/backupschedule/backup_schedule_manager.go @@ -70,6 +70,7 @@ func (bm *backupScheduleManager) Sync(bs *v1alpha1.BackupSchedule) error { } // delete the last backup job for release the backup PVC + if err := bm.deleteLastBackupJob(bs); err != nil { return nil } @@ -373,7 +374,7 @@ func (bm *backupScheduleManager) backupGCByMaxReservedTime(bs *v1alpha1.BackupSc return } } else { - expiredBackups, err = caculateExpiredBackups(ascBackups, reservedTime) + expiredBackups, err = calculateExpiredBackups(ascBackups, reservedTime) if err != nil { klog.Errorf("caculate expired backups without log backup, err: %s", err) return @@ -404,7 +405,7 @@ func (bm *backupScheduleManager) backupGCByMaxReservedTime(bs *v1alpha1.BackupSc } } -// separateSnapshotBackupsAndLogBackup return snapot backups ordry by create time asc and log backup +// separateSnapshotBackupsAndLogBackup return snapshot backups order by create time asc and log backup func separateSnapshotBackupsAndLogBackup(backupsList []*v1alpha1.Backup) ([]*v1alpha1.Backup, *v1alpha1.Backup) { var ( ascBackupList = make([]*v1alpha1.Backup, 0) @@ -479,7 +480,7 @@ func calExpiredBackupsAndLogBackup(backupsList []*v1alpha1.Backup, logBackup *v1 return expiredBackups, truncateTSO, nil } -func caculateExpiredBackups(backupsList []*v1alpha1.Backup, reservedTime time.Duration) ([]*v1alpha1.Backup, error) { +func calculateExpiredBackups(backupsList []*v1alpha1.Backup, reservedTime time.Duration) ([]*v1alpha1.Backup, error) { expiredTS := config.TSToTSO(time.Now().Add(-1 * reservedTime).Unix()) i := 0 for ; i < len(backupsList); i++ { diff --git a/pkg/controller/controller_utils.go b/pkg/controller/controller_utils.go index b2c141dd359..92e8d04ae18 100644 --- a/pkg/controller/controller_utils.go +++ b/pkg/controller/controller_utils.go @@ -186,6 +186,20 @@ func GetBackupScheduleOwnerRef(bs *v1alpha1.BackupSchedule) metav1.OwnerReferenc } } +// GetFedVolumeBackupScheduleOwnerRef returns FedVolumeBackupSchedule's OwnerReference +func GetFedVolumeBackupScheduleOwnerRef(vbs *fedv1alpha1.VolumeBackupSchedule) metav1.OwnerReference { + controller := true + blockOwnerDeletion := true + return metav1.OwnerReference{ + APIVersion: backupScheduleControllerKind.GroupVersion().String(), + Kind: backupScheduleControllerKind.Kind, + Name: vbs.GetName(), + UID: vbs.GetUID(), + Controller: &controller, + BlockOwnerDeletion: &blockOwnerDeletion, + } +} + func GetTiDBMonitorOwnerRef(monitor *v1alpha1.TidbMonitor) metav1.OwnerReference { controller := true blockOwnerDeletion := true diff --git a/pkg/controller/fed_backup_schedule_status_updater.go b/pkg/controller/fed_backup_schedule_status_updater.go new file mode 100644 index 00000000000..88613eb5b12 --- /dev/null +++ b/pkg/controller/fed_backup_schedule_status_updater.go @@ -0,0 +1,113 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "fmt" + + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" + informers "github.com/pingcap/tidb-operator/pkg/client/federation/informers/externalversions/pingcap/v1alpha1" + listers "github.com/pingcap/tidb-operator/pkg/client/federation/listers/pingcap/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" +) + +// VolumeBackupScheduleStatusUpdaterInterface is an interface used to update the VolumeBackupScheduleStatus associated with a VolumeBackupSchedule. +// For any use other than testing, clients should create an instance using NewRealBackupScheduleStatusUpdater. +type VolumeBackupScheduleStatusUpdaterInterface interface { + // UpdateBackupScheduleStatus sets the backupSchedule's Status to status. Implementations are required to retry on conflicts, + // but fail on other errors. If the returned error is nil backup's Status has been successfully set to status. + UpdateBackupScheduleStatus(*v1alpha1.VolumeBackupSchedule, *v1alpha1.VolumeBackupScheduleStatus, *v1alpha1.VolumeBackupScheduleStatus) error +} + +// NewRealVolumeBackupScheduleStatusUpdater returns a VolumeBackupScheduleStatusUpdaterInterface that updates the Status of a VolumeBackupScheduleStatus, +// using the supplied client and bsLister. +func NewRealVolumeBackupScheduleStatusUpdater(deps *BrFedDependencies) VolumeBackupScheduleStatusUpdaterInterface { + return &realVolumeBackupScheduleStatusUpdater{ + deps: deps, + } +} + +type realVolumeBackupScheduleStatusUpdater struct { + deps *BrFedDependencies +} + +func (u *realVolumeBackupScheduleStatusUpdater) UpdateBackupScheduleStatus( + bs *v1alpha1.VolumeBackupSchedule, + newStatus *v1alpha1.VolumeBackupScheduleStatus, + oldStatus *v1alpha1.VolumeBackupScheduleStatus) error { + + ns := bs.GetNamespace() + bsName := bs.GetName() + // don't wait due to limited number of clients, but backoff after the default number of steps + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + _, updateErr := u.deps.Clientset.FederationV1alpha1().VolumeBackupSchedules(ns).Update(context.TODO(), bs, metav1.UpdateOptions{}) + if updateErr == nil { + klog.Infof("BackupSchedule: [%s/%s] updated successfully", ns, bsName) + return nil + } + if updated, err := u.deps.VolumeBackupScheduleLister.VolumeBackupSchedules(ns).Get(bsName); err == nil { + // make a copy so we don't mutate the shared cache + bs = updated.DeepCopy() + bs.Status = *newStatus + } else { + utilruntime.HandleError(fmt.Errorf("error getting updated backupSchedule %s/%s from lister: %v", ns, bsName, err)) + } + + return updateErr + }) + return err +} + +var _ VolumeBackupScheduleStatusUpdaterInterface = &realVolumeBackupScheduleStatusUpdater{} + +// FakeVolumeBackupScheduleStatusUpdater is a fake VolumeBackupScheduleStatusUpdaterInterface +type FakeVolumeBackupScheduleStatusUpdater struct { + BsLister listers.VolumeBackupScheduleLister + BsIndexer cache.Indexer + updateBsTracker RequestTracker +} + +// NewFakeVolumeBackupScheduleStatusUpdater returns a FakeVolumeBackupScheduleStatusUpdater +func NewFakeVolumeBackupScheduleStatusUpdater(bsInformer informers.VolumeBackupScheduleInformer) *FakeVolumeBackupScheduleStatusUpdater { + return &FakeVolumeBackupScheduleStatusUpdater{ + bsInformer.Lister(), + bsInformer.Informer().GetIndexer(), + RequestTracker{}, + } +} + +// SetUpdateBackupScheduleError sets the error attributes of updateBackupScheduleTracker +func (u *FakeVolumeBackupScheduleStatusUpdater) SetUpdateBackupScheduleError(err error, after int) { + u.updateBsTracker.err = err + u.updateBsTracker.after = after + u.updateBsTracker.SetError(err).SetAfter(after) +} + +// UpdateBackupScheduleStatus updates the BackupSchedule +func (u *FakeVolumeBackupScheduleStatusUpdater) UpdateBackupScheduleStatus(bs *v1alpha1.VolumeBackupSchedule, _ *v1alpha1.VolumeBackupScheduleStatus, _ *v1alpha1.VolumeBackupScheduleStatus) error { + defer u.updateBsTracker.Inc() + if u.updateBsTracker.ErrorReady() { + defer u.updateBsTracker.Reset() + return u.updateBsTracker.GetError() + } + + return u.BsIndexer.Update(bs) +} + +var _ VolumeBackupScheduleStatusUpdaterInterface = &FakeVolumeBackupScheduleStatusUpdater{} diff --git a/pkg/controller/fed_volume_backup_control.go b/pkg/controller/fed_volume_backup_control.go index 0de42e3bb4c..13ff5892b58 100644 --- a/pkg/controller/fed_volume_backup_control.go +++ b/pkg/controller/fed_volume_backup_control.go @@ -31,7 +31,7 @@ import ( listers "github.com/pingcap/tidb-operator/pkg/client/federation/listers/pingcap/v1alpha1" ) -// FedVolumeBackupControlInterface manages federaton VolumeBackups used in VolumeBackupSchedule +// FedVolumeBackupControlInterface manages federation VolumeBackups used in VolumeBackupSchedule type FedVolumeBackupControlInterface interface { CreateVolumeBackup(backup *v1alpha1.VolumeBackup) (*v1alpha1.VolumeBackup, error) DeleteVolumeBackup(backup *v1alpha1.VolumeBackup) error @@ -108,16 +108,18 @@ type FakeFedVolumeBackupControl struct { volumeBackupLister listers.VolumeBackupLister volumeBackupIndexer cache.Indexer createVolumeBackupTracker RequestTracker + updateVolumeBackupTracker RequestTracker deleteVolumeBackupTracker RequestTracker } -// NewFakeBackupControl returns a FakeBackupControl +// NewFakeFedVolumeBackupControl returns a FakeFedVolumeBackupControl func NewFakeFedVolumeBackupControl(volumeBackupInformer informers.VolumeBackupInformer) *FakeFedVolumeBackupControl { return &FakeFedVolumeBackupControl{ volumeBackupInformer.Lister(), volumeBackupInformer.Informer().GetIndexer(), RequestTracker{}, RequestTracker{}, + RequestTracker{}, } } @@ -126,6 +128,11 @@ func (fbc *FakeFedVolumeBackupControl) SetCreateVolumeBackupError(err error, aft fbc.createVolumeBackupTracker.SetError(err).SetAfter(after) } +// SetUpdateVolumeBackupError sets the error attributes of createVolumeBackupTracker +func (fbc *FakeFedVolumeBackupControl) SetUpdateVolumeBackupError(err error, after int) { + fbc.updateVolumeBackupTracker.SetError(err).SetAfter(after) +} + // SetDeleteVolumeBackupError sets the error attributes of deleteVolumeBackupTracker func (fbc *FakeFedVolumeBackupControl) SetDeleteVolumeBackupError(err error, after int) { fbc.deleteVolumeBackupTracker.SetError(err).SetAfter(after) diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control.go index ad568e2eaf8..8a7d965013c 100644 --- a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control.go +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control.go @@ -14,10 +14,11 @@ package fedvolumebackupschedule import ( + apiequality "k8s.io/apimachinery/pkg/api/equality" + errorutils "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/cache" "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" - "github.com/pingcap/tidb-operator/pkg/client/federation/clientset/versioned" informers "github.com/pingcap/tidb-operator/pkg/client/federation/informers/externalversions/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/fedvolumebackup" @@ -31,24 +32,41 @@ type ControlInterface interface { UpdateBackupSchedule(volumeBackupSchedule *v1alpha1.VolumeBackupSchedule) error } -// NewDefaultVolumeBackupScheduleControl returns a new instance of the default VolumeBackupSchedue ControlInterface implementation. +// NewDefaultVolumeBackupScheduleControl returns a new instance of the default VolumeBackupSchedule ControlInterface implementation. func NewDefaultVolumeBackupScheduleControl( - cli versioned.Interface, + statusUpdater controller.VolumeBackupScheduleStatusUpdaterInterface, backupScheduleManager fedvolumebackup.BackupScheduleManager) ControlInterface { return &defaultBackupScheduleControl{ - cli, + statusUpdater, backupScheduleManager, } } type defaultBackupScheduleControl struct { - cli versioned.Interface - bsManager fedvolumebackup.BackupScheduleManager + statusUpdater controller.VolumeBackupScheduleStatusUpdaterInterface + bsManager fedvolumebackup.BackupScheduleManager } // UpdateBackupSchedule executes the core logic loop for a VolumeBackupSchedule. -func (c *defaultBackupScheduleControl) UpdateBackupSchedule(volumeBackupSchedule *v1alpha1.VolumeBackupSchedule) error { - return c.bsManager.Sync(volumeBackupSchedule) +func (c *defaultBackupScheduleControl) UpdateBackupSchedule(vbs *v1alpha1.VolumeBackupSchedule) error { + var errs []error + oldStatus := vbs.Status.DeepCopy() + + if err := c.updateBackupSchedule(vbs); err != nil { + errs = append(errs, err) + } + if apiequality.Semantic.DeepEqual(&vbs.Status, oldStatus) { + return errorutils.NewAggregate(errs) + } + if err := c.statusUpdater.UpdateBackupScheduleStatus(vbs.DeepCopy(), &vbs.Status, oldStatus); err != nil { + errs = append(errs, err) + } + + return errorutils.NewAggregate(errs) +} + +func (c *defaultBackupScheduleControl) updateBackupSchedule(vbs *v1alpha1.VolumeBackupSchedule) error { + return c.bsManager.Sync(vbs) } var _ ControlInterface = &defaultBackupScheduleControl{} diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control_test.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control_test.go new file mode 100644 index 00000000000..b35040c90d8 --- /dev/null +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control_test.go @@ -0,0 +1,120 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. +package fedvolumebackupschedule + +import ( + "fmt" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/client/federation/clientset/versioned/fake" + informers "github.com/pingcap/tidb-operator/pkg/client/federation/informers/externalversions" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/fedvolumebackup/backupschedule" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBackupScheduleControlUpdateBackupSchedule(t *testing.T) { + g := NewGomegaWithT(t) + + type testcase struct { + name string + update func(bs *v1alpha1.VolumeBackupSchedule) + syncBsManagerErr bool + updateStatusErr bool + errExpectFn func(*GomegaWithT, error) + } + testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + + bs := newBackupSchedule() + if test.update != nil { + test.update(bs) + } + control, bsManager, bsStatusUpdater := newFakeBackupScheduleControl() + + if test.syncBsManagerErr { + bsManager.SetSyncError(fmt.Errorf("backup schedule sync error")) + } + + if test.updateStatusErr { + bsStatusUpdater.SetUpdateBackupScheduleError(fmt.Errorf("update backupSchedule status error"), 0) + } + + err := control.UpdateBackupSchedule(bs) + if test.errExpectFn != nil { + test.errExpectFn(g, err) + } + } + tests := []testcase{ + { + name: "backup schedule sync error", + update: nil, + syncBsManagerErr: true, + updateStatusErr: false, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "backup schedule sync error")).To(Equal(true)) + }, + }, + { + name: "backup schedule status is not updated", + update: nil, + syncBsManagerErr: false, + updateStatusErr: false, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).NotTo(HaveOccurred()) + }, + }, + { + name: "backup schedule status update failed", + update: func(bs *v1alpha1.VolumeBackupSchedule) { + bs.Status.LastBackupTime = &metav1.Time{Time: time.Now()} + }, + syncBsManagerErr: false, + updateStatusErr: true, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "update backupSchedule status error")).To(Equal(true)) + }, + }, + { + name: "normal", + update: func(bs *v1alpha1.VolumeBackupSchedule) { + bs.Status.LastBackupTime = &metav1.Time{Time: time.Now()} + }, + syncBsManagerErr: false, + updateStatusErr: false, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).NotTo(HaveOccurred()) + }, + }, + } + + for i := range tests { + testFn(&tests[i], t) + } +} + +func newFakeBackupScheduleControl() (ControlInterface, *backupschedule.FakeBackupScheduleManager, *controller.FakeVolumeBackupScheduleStatusUpdater) { + cli := fake.NewSimpleClientset() + bsInformer := informers.NewSharedInformerFactory(cli, 0).Federation().V1alpha1().VolumeBackupSchedules() + statusUpdater := controller.NewFakeVolumeBackupScheduleStatusUpdater(bsInformer) + bsManager := backupschedule.NewFakeBackupScheduleManager() + control := NewDefaultVolumeBackupScheduleControl(statusUpdater, bsManager) + + return control, bsManager, statusUpdater +} diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go index d3db48aa160..705500e0aa5 100644 --- a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go @@ -45,7 +45,7 @@ type Controller struct { func NewController(deps *controller.BrFedDependencies) *Controller { c := &Controller{ deps: deps, - control: NewDefaultVolumeBackupScheduleControl(deps.Clientset, backupschedule.NewBackupScheduleManager(deps)), + control: NewDefaultVolumeBackupScheduleControl(controller.NewRealVolumeBackupScheduleStatusUpdater(deps), backupschedule.NewBackupScheduleManager(deps)), queue: workqueue.NewNamedRateLimitingQueue( controller.NewControllerRateLimiter(1*time.Second, 100*time.Second), "volumeBackupSchedule", diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller_test.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller_test.go new file mode 100644 index 00000000000..02a118e5b0f --- /dev/null +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller_test.go @@ -0,0 +1,180 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package fedvolumebackupschedule + +import ( + "fmt" + "strings" + "testing" + + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/pointer" +) + +func TestBackupScheduleControllerEnqueueBackupSchedule(t *testing.T) { + g := NewGomegaWithT(t) + bks := newBackupSchedule() + bsc, _, _ := newFakeBackupScheduleController() + bsc.enqueueBackupSchedule(bks) + g.Expect(bsc.queue.Len()).To(Equal(1)) +} + +func TestBackupScheduleControllerEnqueueBackupScheduleFailed(t *testing.T) { + g := NewGomegaWithT(t) + bsc, _, _ := newFakeBackupScheduleController() + bsc.enqueueBackupSchedule(struct{}{}) + g.Expect(bsc.queue.Len()).To(Equal(0)) +} + +func TestBackupScheduleControllerSync(t *testing.T) { + g := NewGomegaWithT(t) + type testcase struct { + name string + addBsToIndexer bool + errWhenUpdateBackupSchedule bool + invalidKeyFn func(bs *v1alpha1.VolumeBackupSchedule) string + errExpectFn func(*GomegaWithT, error) + } + + testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + + bs := newBackupSchedule() + bsc, bsIndexer, bsControl := newFakeBackupScheduleController() + + if test.addBsToIndexer { + err := bsIndexer.Add(bs) + g.Expect(err).NotTo(HaveOccurred()) + } + + key, _ := cache.DeletionHandlingMetaNamespaceKeyFunc(bs) + if test.invalidKeyFn != nil { + key = test.invalidKeyFn(bs) + } + + if test.errWhenUpdateBackupSchedule { + bsControl.SetUpdateVolumeBackupError(fmt.Errorf("update backup schedule failed"), 0) + } + + err := bsc.sync(key) + + if test.errExpectFn != nil { + test.errExpectFn(g, err) + } + } + + tests := []testcase{ + { + name: "normal", + addBsToIndexer: true, + errWhenUpdateBackupSchedule: false, + invalidKeyFn: nil, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).NotTo(HaveOccurred()) + }, + }, + { + name: "invalid backup key", + addBsToIndexer: true, + errWhenUpdateBackupSchedule: false, + invalidKeyFn: func(bs *v1alpha1.VolumeBackupSchedule) string { + return fmt.Sprintf("test/demo/%s", bs.GetName()) + }, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).To(HaveOccurred()) + }, + }, + { + name: "can't found backup schedule", + addBsToIndexer: false, + errWhenUpdateBackupSchedule: false, + invalidKeyFn: nil, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).NotTo(HaveOccurred()) + }, + }, + { + name: "update backup schedule failed", + addBsToIndexer: true, + errWhenUpdateBackupSchedule: true, + invalidKeyFn: nil, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "update backup schedule failed")).To(Equal(true)) + }, + }, + } + + for i := range tests { + testFn(&tests[i], t) + } + +} + +func newFakeBackupScheduleController() (*Controller, cache.Indexer, *controller.FakeFedVolumeBackupControl) { + fakeDeps := controller.NewFakeDependencies() + bsc := NewController(fakeDeps) + bsInformer := fakeDeps.InformerFactory.Pingcap().V1alpha1().BackupSchedules() + backupScheduleControl := NewFakeBackupScheduleControl(bsInformer) + bsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: bsc.enqueueBackupSchedule, + UpdateFunc: func(old, cur interface{}) { + bsc.enqueueBackupSchedule(cur) + }, + DeleteFunc: bsc.enqueueBackupSchedule, + }) + bsc.control = backupScheduleControl + return bsc, bsInformer.Informer().GetIndexer(), backupScheduleControl +} + +func newBackupSchedule() *v1alpha1.VolumeBackupSchedule { + return &v1alpha1.VolumeBackupSchedule{ + TypeMeta: metav1.TypeMeta{ + Kind: "BackupScheduler", + APIVersion: "pingcap.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bks", + Namespace: corev1.NamespaceDefault, + UID: types.UID("test-bks"), + }, + Spec: v1alpha1.VolumeBackupScheduleSpec{ + Schedule: "1 */10 * * *", + MaxBackups: pointer.Int32Ptr(10), + BackupTemplate: v1alpha1.BackupSpec{ + From: &v1alpha1.TiDBAccessConfig{ + Host: "10.1.1.2", + Port: v1alpha1.DefaultTiDBServicePort, + User: v1alpha1.DefaultTidbUser, + SecretName: "demo1-tidb-secret", + }, + StorageProvider: v1alpha1.StorageProvider{ + S3: &v1alpha1.S3StorageProvider{ + Provider: v1alpha1.S3StorageProviderTypeCeph, + Endpoint: "http://10.0.0.1", + SecretName: "demo", + Bucket: "test1-demo1", + }, + }, + }, + }, + } +} diff --git a/pkg/fedvolumebackup/backup/backup_manager.go b/pkg/fedvolumebackup/backup/backup_manager.go index 0c183625717..2f825b280ca 100644 --- a/pkg/fedvolumebackup/backup/backup_manager.go +++ b/pkg/fedvolumebackup/backup/backup_manager.go @@ -118,20 +118,6 @@ func (bm *backupManager) runBackup(ctx context.Context, volumeBackup *v1alpha1.V bm.setVolumeBackupRunning(&volumeBackup.Status) } -<<<<<<< HEAD -======= - if bm.skipSync(volumeBackup) { - klog.Infof("skip VolumeBackup %s/%s", ns, name) - return nil - } - - ctx := context.Background() - backupMembers, err := bm.listAllBackupMembers(ctx, volumeBackup) - if err != nil { - return err - } - ->>>>>>> 7d84cf89e (br: update backup member to status) if len(backupMembers) == 0 { return false, bm.initializeVolumeBackup(ctx, volumeBackup) } @@ -184,6 +170,14 @@ func (bm *backupManager) setVolumeBackupRunning(volumeBackupStatus *v1alpha1.Vol }) } +func (bm *backupManager) setVolumeBackupPrepared(volumeBackupStatus *v1alpha1.VolumeBackupStatus) { + volumeBackupStatus.TimeStarted = metav1.Now() + v1alpha1.UpdateVolumeBackupCondition(volumeBackupStatus, &v1alpha1.VolumeBackupCondition{ + Type: v1alpha1.VolumeBackupPrepared, + Status: corev1.ConditionTrue, + }) +} + func (bm *backupManager) listAllBackupMembers(ctx context.Context, volumeBackup *v1alpha1.VolumeBackup) ([]*volumeBackupMember, error) { backupMembers := make([]*volumeBackupMember, 0, len(volumeBackup.Spec.Clusters)) for _, memberCluster := range volumeBackup.Spec.Clusters { @@ -349,14 +343,6 @@ func (bm *backupManager) setVolumeBackupCleaned(volumeBackupStatus *v1alpha1.Vol }) } -func (bm *backupManager) setVolumeBackupPrepared(volumeBackupStatus *v1alpha1.VolumeBackupStatus) { - volumeBackupStatus.TimeStarted = metav1.Now() - v1alpha1.UpdateVolumeBackupCondition(volumeBackupStatus, &v1alpha1.VolumeBackupCondition{ - Type: v1alpha1.VolumeBackupPrepared, - Status: corev1.ConditionTrue, - }) -} - func (bm *backupManager) setVolumeBackupSize(volumeBackupStatus *v1alpha1.VolumeBackupStatus, backupMembers []*volumeBackupMember) { var totalBackupSize int64 for _, backupMember := range backupMembers { @@ -417,11 +403,7 @@ func (bm *backupManager) buildBackupMember(volumeBackupName string, clusterMembe } func (bm *backupManager) skipSync(volumeBackup *v1alpha1.VolumeBackup) bool { -<<<<<<< HEAD - return volumeBackup.DeletionTimestamp == nil && (v1alpha1.IsVolumeBackupComplete(volumeBackup) || v1alpha1.IsVolumeBackupFailed(volumeBackup)) -======= return v1alpha1.IsVolumeBackupComplete(volumeBackup) || v1alpha1.IsVolumeBackupFailed(volumeBackup) ->>>>>>> 7d84cf89e (br: update backup member to status) } func (bm *backupManager) generateBackupMemberName(volumeBackupName, k8sClusterName string) string { diff --git a/pkg/fedvolumebackup/backupschedule/backup_schedule_manager.go b/pkg/fedvolumebackup/backupschedule/backup_schedule_manager.go index deeb552f971..be2d03f7e5e 100644 --- a/pkg/fedvolumebackup/backupschedule/backup_schedule_manager.go +++ b/pkg/fedvolumebackup/backupschedule/backup_schedule_manager.go @@ -14,8 +14,17 @@ package backupschedule import ( + "fmt" + "sort" "time" + perrors "github.com/pingcap/errors" + "github.com/pingcap/tidb-operator/pkg/apis/label" + "github.com/pingcap/tidb-operator/pkg/apis/util/config" + "github.com/pingcap/tidb-operator/pkg/util" + "github.com/robfig/cron" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" @@ -38,15 +47,306 @@ func NewBackupScheduleManager(deps *controller.BrFedDependencies) fedvolumebacku } } -func (bm *backupScheduleManager) Sync(volumeBackupSchedule *v1alpha1.VolumeBackupSchedule) error { - ns := volumeBackupSchedule.GetNamespace() - name := volumeBackupSchedule.GetName() - // TODO(federation): implement the main logic of backupSchedule - klog.Infof("sync VolumeBackupSchedule %s/%s", ns, name) +func (bm *backupScheduleManager) Sync(vbs *v1alpha1.VolumeBackupSchedule) error { + defer bm.backupGC(vbs) + if vbs.Spec.Pause { + return controller.IgnoreErrorf("backupSchedule %s/%s has been paused", vbs.GetNamespace(), vbs.GetName()) + } + + if err := bm.canPerformNextBackup(vbs); err != nil { + return err + } + + scheduledTime, err := getLastScheduledTime(vbs, bm.now) + if scheduledTime == nil { + return err + } + + // TODO: do we need to delete last backup job also? + backup, err := createBackup(bm.deps.FedVolumeBackupControl, vbs, *scheduledTime) + if err != nil { + return err + } + + vbs.Status.LastBackup = backup.GetName() + vbs.Status.LastBackupTime = &metav1.Time{Time: *scheduledTime} + vbs.Status.AllBackupCleanTime = nil return nil } +// getLastScheduledTime return the newest time need to be scheduled according last backup time. +// the return time is not before now and return nil if there's no such time. +func getLastScheduledTime(vbs *v1alpha1.VolumeBackupSchedule, nowFn nowFn) (*time.Time, error) { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + sched, err := cron.ParseStandard(vbs.Spec.Schedule) + if err != nil { + return nil, fmt.Errorf("parse backup schedule %s/%s cron format %s failed, err: %v", ns, bsName, vbs.Spec.Schedule, err) + } + + var earliestTime time.Time + if vbs.Status.LastBackupTime != nil { + earliestTime = vbs.Status.LastBackupTime.Time + } else if vbs.Status.AllBackupCleanTime != nil { + // Recovery from a long paused backup schedule may cause problem like "incorrect clock", + // so we introduce AllBackupCleanTime field to solve this problem. + earliestTime = vbs.Status.AllBackupCleanTime.Time + } else { + // If none found, then this is either a recently created backupSchedule, + // or the backupSchedule status info was somehow lost, + // or that we have started a backup, but have not update backupSchedule status yet + // (distributed systems can have arbitrary delays). + // In any case, use the creation time of the backupSchedule as last known start time. + earliestTime = vbs.ObjectMeta.CreationTimestamp.Time + } + + now := nowFn() + if earliestTime.After(now) { + // timestamp fallback, waiting for the next backup schedule period + klog.Errorf("backup schedule %s/%s timestamp fallback, lastBackupTime: %s, now: %s", + ns, bsName, earliestTime.Format(time.RFC3339), now.Format(time.RFC3339)) + return nil, nil + } + + var scheduledTimes []time.Time + for t := sched.Next(earliestTime); !t.After(now); t = sched.Next(t) { + scheduledTimes = append(scheduledTimes, t) + // If there is a bug somewhere, or incorrect clock + // on controller's server or apiservers (for setting creationTimestamp) + // then there could be so many missed start times (it could be off + // by decades or more), that it would eat up all the CPU and memory + // of this controller. In that case, we want to not try to list + // all the missed start times. + // + // I've somewhat arbitrarily picked 100, as more than 80, + // but less than "lots". + if len(scheduledTimes) > 100 { + // We can't get the last backup schedule time + if vbs.Status.LastBackupTime == nil && vbs.Status.AllBackupCleanTime != nil { + // Recovery backup schedule from pause status, should refresh AllBackupCleanTime to avoid unschedulable problem + vbs.Status.AllBackupCleanTime = &metav1.Time{Time: nowFn()} + return nil, controller.RequeueErrorf("recovery backup schedule %s/%s from pause status, refresh AllBackupCleanTime.", ns, bsName) + } + klog.Error("Too many missed start backup schedule time (> 100). Check the clock.") + return nil, nil + } + } + + if len(scheduledTimes) == 0 { + klog.V(4).Infof("unmet backup schedule %s/%s start time, waiting for the next backup schedule period", ns, bsName) + return nil, nil + } + scheduledTime := scheduledTimes[len(scheduledTimes)-1] + return &scheduledTime, nil +} + +func buildBackup(vbs *v1alpha1.VolumeBackupSchedule, timestamp time.Time) *v1alpha1.VolumeBackup { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + backupSpec := *vbs.Spec.BackupTemplate.DeepCopy() + + bsLabel := util.CombineStringMap(label.NewBackupSchedule().Instance(bsName).BackupSchedule(bsName), vbs.Labels) + backup := &v1alpha1.VolumeBackup{ + Spec: backupSpec, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: vbs.GetBackupCRDName(timestamp), + Labels: bsLabel, + Annotations: vbs.Annotations, + OwnerReferences: []metav1.OwnerReference{ + controller.GetFedVolumeBackupScheduleOwnerRef(vbs), + }, + }, + } + + return backup +} + +func createBackup(bkController controller.FedVolumeBackupControlInterface, vbs *v1alpha1.VolumeBackupSchedule, timestamp time.Time) (*v1alpha1.VolumeBackup, error) { + bk := buildBackup(vbs, timestamp) + return bkController.CreateVolumeBackup(bk) +} + +func (bm *backupScheduleManager) canPerformNextBackup(vbs *v1alpha1.VolumeBackupSchedule) error { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + backup, err := bm.deps.VolumeBackupLister.VolumeBackups(ns).Get(vbs.Status.LastBackup) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return fmt.Errorf("backup schedule %s/%s, get backup %s failed, err: %v", ns, bsName, vbs.Status.LastBackup, err) + } + + if v1alpha1.IsVolumeBackupComplete(backup) { + return nil + } + // If the last backup is in a failed state, but it is not scheduled yet, + // skip this sync round of the backup schedule and waiting the last backup. + return controller.RequeueErrorf("backup schedule %s/%s, the last backup %s is still running", ns, bsName, vbs.Status.LastBackup) +} + +func (bm *backupScheduleManager) backupGC(vbs *v1alpha1.VolumeBackupSchedule) { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + // if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred. + if vbs.Spec.MaxReservedTime != nil { + bm.backupGCByMaxReservedTime(vbs) + return + } + + if vbs.Spec.MaxBackups != nil && *vbs.Spec.MaxBackups > 0 { + bm.backupGCByMaxBackups(vbs) + return + } + klog.Warningf("backup schedule %s/%s does not set backup gc policy", ns, bsName) +} + +func (bm *backupScheduleManager) backupGCByMaxReservedTime(vbs *v1alpha1.VolumeBackupSchedule) { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + reservedTime, err := time.ParseDuration(*vbs.Spec.MaxReservedTime) + if err != nil { + klog.Errorf("backup schedule %s/%s, invalid MaxReservedTime %s", ns, bsName, *vbs.Spec.MaxReservedTime) + return + } + + backupsList, err := bm.getBackupList(vbs) + if err != nil { + klog.Errorf("backupGCByMaxReservedTime, err: %s", err) + return + } + + ascBackups := sortSnapshotBackups(backupsList) + if len(ascBackups) == 0 { + return + } + + var expiredBackups []*v1alpha1.VolumeBackup + + expiredBackups, err = calculateExpiredBackups(ascBackups, reservedTime) + if err != nil { + klog.Errorf("caculate expired backups without log backup, err: %s", err) + return + } + + for _, backup := range expiredBackups { + // delete the expired backup + if err = bm.deps.FedVolumeBackupControl.DeleteVolumeBackup(backup); err != nil { + klog.Errorf("backup schedule %s/%s gc backup %s failed, err %v", ns, bsName, backup.GetName(), err) + return + } + klog.Infof("backup schedule %s/%s gc backup %s success", ns, bsName, backup.GetName()) + } + + if len(expiredBackups) == len(backupsList) && len(expiredBackups) > 0 { + // All backups have been deleted, so the last backup information in the backupSchedule should be reset + bm.resetLastBackup(vbs) + } +} + +// sortSnapshotBackups return snapshot backups to be GCed order by create time asc +func sortSnapshotBackups(backupsList []*v1alpha1.VolumeBackup) []*v1alpha1.VolumeBackup { + var ascBackupList = make([]*v1alpha1.VolumeBackup, 0) + + for _, backup := range backupsList { + // the backup status CommitTs will be empty after created. without this, all newly created backups will be GC'ed + if v1alpha1.IsVolumeBackupRunning(backup) || v1alpha1.IsBackupPrepared(backup) { + continue + } + ascBackupList = append(ascBackupList, backup) + } + + sort.Slice(ascBackupList, func(i, j int) bool { + return ascBackupList[i].CreationTimestamp.Unix() < ascBackupList[j].CreationTimestamp.Unix() + }) + return ascBackupList +} + +func calculateExpiredBackups(backupsList []*v1alpha1.VolumeBackup, reservedTime time.Duration) ([]*v1alpha1.VolumeBackup, error) { + expiredTS := config.TSToTSO(time.Now().Add(-1 * reservedTime).Unix()) + i := 0 + for ; i < len(backupsList); i++ { + startTS, err := config.ParseTSString(backupsList[i].Status.CommitTs) + if err != nil { + return nil, perrors.Annotatef(err, "parse start tso: %s", backupsList[i].Status.CommitTs) + } + if startTS >= expiredTS { + break + } + } + return backupsList[:i], nil +} + +func (bm *backupScheduleManager) getBackupList(bs *v1alpha1.VolumeBackupSchedule) ([]*v1alpha1.VolumeBackup, error) { + ns := bs.GetNamespace() + bsName := bs.GetName() + + backupLabels := label.NewBackupSchedule().Instance(bsName).BackupSchedule(bsName) + selector, err := backupLabels.Selector() + if err != nil { + return nil, fmt.Errorf("generate backup schedule %s/%s label selector failed, err: %v", ns, bsName, err) + } + backupsList, err := bm.deps.VolumeBackupLister.VolumeBackups(ns).List(selector) + if err != nil { + return nil, fmt.Errorf("get backup schedule %s/%s backup list failed, selector: %s, err: %v", ns, bsName, selector, err) + } + + return backupsList, nil +} + +func (bm *backupScheduleManager) backupGCByMaxBackups(vbs *v1alpha1.VolumeBackupSchedule) { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + backupsList, err := bm.getBackupList(vbs) + if err != nil { + klog.Errorf("backupGCByMaxBackups failed, err: %s", err) + return + } + + sort.Sort(byCreateTimeDesc(backupsList)) + + var deleteCount int + for i, backup := range backupsList { + if i < int(*vbs.Spec.MaxBackups) { + continue + } + // delete the backup + if err := bm.deps.FedVolumeBackupControl.DeleteVolumeBackup(backup); err != nil { + klog.Errorf("backup schedule %s/%s gc backup %s failed, err %v", ns, bsName, backup.GetName(), err) + return + } + deleteCount += 1 + klog.Infof("backup schedule %s/%s gc backup %s success", ns, bsName, backup.GetName()) + } + + if deleteCount == len(backupsList) && deleteCount > 0 { + // All backups have been deleted, so the last backup information in the backupSchedule should be reset + bm.resetLastBackup(vbs) + } +} + +func (bm *backupScheduleManager) resetLastBackup(vbs *v1alpha1.VolumeBackupSchedule) { + vbs.Status.LastBackupTime = nil + vbs.Status.LastBackup = "" + vbs.Status.AllBackupCleanTime = &metav1.Time{Time: bm.now()} +} + +type byCreateTimeDesc []*v1alpha1.VolumeBackup + +func (b byCreateTimeDesc) Len() int { return len(b) } +func (b byCreateTimeDesc) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byCreateTimeDesc) Less(i, j int) bool { + return b[j].ObjectMeta.CreationTimestamp.Before(&b[i].ObjectMeta.CreationTimestamp) +} + var _ fedvolumebackup.BackupScheduleManager = &backupScheduleManager{} type FakeBackupScheduleManager struct { diff --git a/pkg/fedvolumebackup/backupschedule/backup_schedule_manager_test.go b/pkg/fedvolumebackup/backupschedule/backup_schedule_manager_test.go new file mode 100644 index 00000000000..a17926a2693 --- /dev/null +++ b/pkg/fedvolumebackup/backupschedule/backup_schedule_manager_test.go @@ -0,0 +1,516 @@ +package backupschedule + +import ( + "fmt" + "github.com/google/go-cmp/cmp" + "github.com/pingcap/tidb-operator/pkg/apis/label" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/backup/constants" + "github.com/pingcap/tidb-operator/pkg/controller" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/pointer" + "strconv" + "time" +) + +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package backupschedule + +import ( +"context" +"fmt" +"strconv" +"testing" +"time" + +"github.com/google/go-cmp/cmp" +. "github.com/onsi/gomega" +"github.com/pingcap/tidb-operator/pkg/apis/label" +"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" +"github.com/pingcap/tidb-operator/pkg/backup/constants" +"github.com/pingcap/tidb-operator/pkg/controller" +v1 "k8s.io/api/core/v1" +metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +"k8s.io/apimachinery/pkg/labels" +"k8s.io/utils/pointer" +) + +func TestManager(t *testing.T) { + g := NewGomegaWithT(t) + helper := newHelper(t) + defer helper.close() + deps := helper.deps + m := NewBackupScheduleManager(deps).(*backupScheduleManager) + var err error + bs := &v1alpha1.BackupSchedule{} + bs.Namespace = "ns" + bs.Name = "bsname" + + // test pause + bs.Spec.Pause = true + err = m.Sync(bs) + g.Expect(err).Should(BeAssignableToTypeOf(&controller.IgnoreError{})) + g.Expect(err.Error()).Should(MatchRegexp(".*has been paused.*")) + + // test canPerformNextBackup + // + // test not found last backup + bs.Spec.Pause = false + bs.Status.LastBackup = "backupname" + err = m.canPerformNextBackup(bs) + g.Expect(err).Should(BeNil()) + + // test backup complete + bk := &v1alpha1.Backup{} + bk.Namespace = bs.Namespace + bk.Name = bs.Status.LastBackup + bk.Status.Conditions = append(bk.Status.Conditions, v1alpha1.BackupCondition{ + Type: v1alpha1.BackupComplete, + Status: v1.ConditionTrue, + }) + helper.createBackup(bk) + err = m.canPerformNextBackup(bs) + g.Expect(err).Should(BeNil()) + helper.deleteBackup(bk) + + // test last backup failed state and not scheduled yet + bk.Status.Conditions = nil + bk.Status.Conditions = append(bk.Status.Conditions, v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: v1.ConditionTrue, + }) + helper.createBackup(bk) + err = m.canPerformNextBackup(bs) + g.Expect(err).Should(BeAssignableToTypeOf(&controller.RequeueError{})) + helper.deleteBackup(bk) + + t.Log("start test normal Sync") + bk.Status.Conditions = nil + bs.Spec.Schedule = "0 0 * * *" // Run at midnight every day + + now := time.Now() + m.now = func() time.Time { return now.AddDate(0, 0, -101) } + m.resetLastBackup(bs) + // 10 backup, one per day + for i := -9; i <= 0; i++ { + t.Log("loop id ", i) + m.now = func() time.Time { return now.AddDate(0, 0, i) } + err = m.Sync(bs) + g.Expect(err).Should(BeNil()) + bks := helper.checkBacklist(bs.Namespace, i+10, false) + // complete the backup created + for i := range bks.Items { + bk := bks.Items[i] + changed := v1alpha1.UpdateBackupCondition(&bk.Status, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupComplete, + Status: v1.ConditionTrue, + }) + if changed { + bk.CreationTimestamp = metav1.Time{Time: m.now()} + bk.Status.CommitTs = getTSOStr(m.now().Add(10 * time.Minute).Unix()) + t.Log("complete backup: ", bk.Name) + helper.updateBackup(&bk) + } + g.Expect(err).Should(BeNil()) + } + } + + t.Log("test setting MaxBackups") + m.now = time.Now + bs.Spec.MaxBackups = pointer.Int32Ptr(5) + err = m.Sync(bs) + g.Expect(err).Should(BeNil()) + helper.checkBacklist(bs.Namespace, 5, false) + + t.Log("test setting MaxReservedTime") + bs.Spec.MaxBackups = nil + bs.Spec.MaxReservedTime = pointer.StringPtr("71h") + err = m.Sync(bs) + g.Expect(err).Should(BeNil()) + helper.checkBacklist(bs.Namespace, 3, false) + + t.Log("test has log backup") + bs.Spec.LogBackupTemplate = &v1alpha1.BackupSpec{Mode: v1alpha1.BackupModeLog} + logBackup := buildLogBackup(bs, now.Add(-72*time.Hour)) + logBackup.Status.CommitTs = getTSOStr(now.Add(-72 * time.Hour).Unix()) + logBackup.Status.LogCheckpointTs = getTSOStr(now.Unix()) + helper.createBackup(logBackup) + bs.Status.LogBackup = &logBackup.Name + bs.Spec.MaxReservedTime = pointer.StringPtr("23h") + err = m.Sync(bs) + g.Expect(err).Should(BeNil()) + helper.checkBacklist(bs.Namespace, 2, true) +} + +func TestGetLastScheduledTime(t *testing.T) { + g := NewGomegaWithT(t) + + bs := &v1alpha1.BackupSchedule{ + Spec: v1alpha1.BackupScheduleSpec{}, + Status: v1alpha1.BackupScheduleStatus{ + LastBackupTime: &metav1.Time{}, + }, + } + var getTime *time.Time + var err error + + // test invalid format schedule + bs.Spec.Schedule = "#$#$#$@" + _, err = getLastScheduledTime(bs, time.Now) + g.Expect(err).ShouldNot(BeNil()) + + bs.Spec.Schedule = "0 0 * * *" // Run once a day at midnight + now := time.Now() + + // test last backup time after now + bs.Status.LastBackupTime.Time = now.AddDate(0, 0, 1) + getTime, err = getLastScheduledTime(bs, time.Now) + g.Expect(err).Should(BeNil()) + g.Expect(getTime).Should(BeNil()) + + // test scheduled + for i := 0; i < 10; i++ { + bs.Status.LastBackupTime.Time = now.AddDate(0, 0, -i-1) + getTime, err = getLastScheduledTime(bs, time.Now) + g.Expect(err).Should(BeNil()) + g.Expect(getTime).ShouldNot(BeNil()) + expectTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local) + g.Expect(*getTime).Should(Equal(expectTime)) + } + + // test too many miss + bs.Status.LastBackupTime.Time = now.AddDate(-1000, 0, 0) + getTime, err = getLastScheduledTime(bs, time.Now) + g.Expect(err).Should(BeNil()) + g.Expect(getTime).Should(BeNil()) +} + +func TestBuildBackup(t *testing.T) { + now := time.Now() + var get *v1alpha1.Backup + + // build BackupSchedule template + bs := &v1alpha1.BackupSchedule{ + Spec: v1alpha1.BackupScheduleSpec{}, + Status: v1alpha1.BackupScheduleStatus{ + LastBackupTime: &metav1.Time{}, + }, + } + bs.Namespace = "ns" + bs.Name = "bsname" + + // build Backup template + bk := &v1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: bs.Namespace, + Name: bs.GetBackupCRDName(now), + Labels: label.NewBackupSchedule().Instance(bs.Name).BackupSchedule(bs.Name).Labels(), + OwnerReferences: []metav1.OwnerReference{ + controller.GetBackupScheduleOwnerRef(bs), + }, + }, + Spec: v1alpha1.BackupSpec{ + StorageSize: constants.DefaultStorageSize, + }, + } + + // test BR == nil + get = buildBackup(bs, now) + if diff := cmp.Diff(bk, get); diff != "" { + t.Errorf("unexpected (-want, +got): %s", diff) + } + // should keep StorageSize from BackupSchedule + bs.Spec.StorageSize = "9527G" + bk.Spec.StorageSize = bs.Spec.StorageSize + get = buildBackup(bs, now) + if diff := cmp.Diff(bk, get); diff != "" { + t.Errorf("unexpected (-want, +got): %s", diff) + } + + // test BR != nil + bs.Spec.BackupTemplate.BR = &v1alpha1.BRConfig{} + bk.Spec.BR = bs.Spec.BackupTemplate.BR.DeepCopy() + bk.Spec.StorageSize = "" // no use for BR + get = buildBackup(bs, now) + if diff := cmp.Diff(bk, get); diff != "" { + t.Errorf("unexpected (-want, +got): %s", diff) + } +} + +func TestCaculateExpiredBackupsWithLogBackup(t *testing.T) { + g := NewGomegaWithT(t) + type testCase struct { + backups []*v1alpha1.Backup + logBackup *v1alpha1.Backup + reservedTime time.Duration + expectedDeleteBackupCount int + expectedTruncateTS uint64 + } + + var ( + now = time.Now() + last10Min = now.Add(-time.Minute * 10).Unix() + last1Day = now.Add(-time.Hour * 24 * 1).Unix() + last2Day = now.Add(-time.Hour * 24 * 2).Unix() + last3Day = now.Add(-time.Hour * 24 * 3).Unix() + last4Day = now.Add(-time.Hour * 24 * 4).Unix() + ) + + testCases := []*testCase{ + // no backup should be deleted and log backup just start, no commit ts/checkpoint ts + { + backups: []*v1alpha1.Backup{ + fakeBackup(&last10Min), + }, + logBackup: fakeLogBackup(nil, nil), + reservedTime: 24 * time.Hour, + expectedDeleteBackupCount: 0, + expectedTruncateTS: 0, + }, + // backup should be delete and log backup just start, no commit ts/checkpoint ts + { + backups: []*v1alpha1.Backup{ + fakeBackup(&last3Day), + fakeBackup(&last2Day), + fakeBackup(&last1Day), + fakeBackup(&last10Min), + }, + logBackup: fakeLogBackup(&last10Min, nil), + reservedTime: 24 * time.Hour, + expectedDeleteBackupCount: 1, + expectedTruncateTS: 0, + }, + // no backup should be deleted, has log backup + { + backups: []*v1alpha1.Backup{ + fakeBackup(&last3Day), + fakeBackup(&last1Day), + }, + logBackup: fakeLogBackup(&last4Day, &last10Min), + reservedTime: 24 * time.Hour, + expectedDeleteBackupCount: 0, + expectedTruncateTS: 0, + }, + // 1 backup should be deleted, no log backup should be truncated + { + backups: []*v1alpha1.Backup{ + fakeBackup(&last3Day), + fakeBackup(&last2Day), + fakeBackup(&last1Day), + }, + logBackup: fakeLogBackup(&last1Day, &last10Min), + reservedTime: 24 * time.Hour, + expectedDeleteBackupCount: 1, + expectedTruncateTS: 0, + }, + // 2 backup should be deleted, no log backup should be truncated + { + backups: []*v1alpha1.Backup{ + fakeBackup(&last4Day), + fakeBackup(&last3Day), + fakeBackup(&last2Day), + fakeBackup(&last1Day), + }, + logBackup: fakeLogBackup(&last1Day, &last10Min), + reservedTime: 24 * time.Hour, + expectedDeleteBackupCount: 2, + expectedTruncateTS: 0, + }, + // 2 backup should be deleted, has log backup should be truncated + { + backups: []*v1alpha1.Backup{ + fakeBackup(&last4Day), + fakeBackup(&last3Day), + fakeBackup(&last2Day), + fakeBackup(&last1Day), + }, + logBackup: fakeLogBackup(&last3Day, &last10Min), + reservedTime: 24 * time.Hour, + expectedDeleteBackupCount: 2, + expectedTruncateTS: getTSO(last2Day), + }, + } + + for _, tc := range testCases { + deletedBackups, truncateTS, err := calExpiredBackupsAndLogBackup(tc.backups, tc.logBackup, tc.reservedTime) + g.Expect(err).Should(BeNil()) + g.Expect(len(deletedBackups)).Should(Equal(tc.expectedDeleteBackupCount)) + g.Expect(truncateTS).Should(Equal(tc.expectedTruncateTS)) + } +} + +type helper struct { + t *testing.T + deps *controller.Dependencies + stop chan struct{} +} + +func newHelper(t *testing.T) *helper { + deps := controller.NewSimpleClientDependencies() + stop := make(chan struct{}) + deps.InformerFactory.Start(stop) + deps.KubeInformerFactory.Start(stop) + deps.InformerFactory.WaitForCacheSync(stop) + deps.KubeInformerFactory.WaitForCacheSync(stop) + + return &helper{ + t: t, + deps: deps, + stop: stop, + } +} + +func (h *helper) close() { + close(h.stop) +} + +// check for exists num Backup and return the exists backups "BackupList". +func (h *helper) checkBacklist(ns string, num int, checkLogBackupTruncate bool) (bks *v1alpha1.BackupList) { + t := h.t + deps := h.deps + g := NewGomegaWithT(t) + + check := func(backups []*v1alpha1.Backup) error { + snapshotBackups, logBackup := separateSnapshotBackupsAndLogBackup(backups) + // check snapshot backup num + if len(snapshotBackups) != num { + var names []string + for _, bk := range snapshotBackups { + names = append(names, bk.Name) + } + return fmt.Errorf("there %d backup, but should be %d, cur backups: %v", len(snapshotBackups), num, names) + } + if !checkLogBackupTruncate { + return nil + } + // check has log backup + if logBackup == nil { + return fmt.Errorf("there is no log backup, but should have") + } + // check truncateTSO, it should equal the earliest snapshot backup after gc + if len(snapshotBackups) == 0 { + return fmt.Errorf("there should have snapshot backup if need check log backup truncate tso") + } + if logBackup.Status.LogSuccessTruncateUntil != snapshotBackups[0].Spec.CommitTs { + return fmt.Errorf("log backup truncate tso should be %s, but cur is %s", snapshotBackups[0].Spec.CommitTs, logBackup.Status.LogSuccessTruncateUntil) + } + return nil + } + + t.Helper() + g.Eventually(func() error { + var err error + bks, err = deps.Clientset.PingcapV1alpha1().Backups(ns).List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).Should(BeNil()) + backups := convertToBackupPtrList(bks.Items) + return check(backups) + }, time.Second*30).Should(BeNil()) + + g.Eventually(func() error { + var err error + backups, err := deps.BackupLister.Backups(ns).List(labels.Everything()) + g.Expect(err).Should(BeNil()) + return check(backups) + }, time.Second*30).Should(BeNil()) + + return +} + +func convertToBackupPtrList(backups []v1alpha1.Backup) []*v1alpha1.Backup { + backupPtrs := make([]*v1alpha1.Backup, 0) + for i := 0; i < len(backups); i++ { + backupPtrs = append(backupPtrs, &backups[i]) + } + return backupPtrs +} + +func (h *helper) updateBackup(bk *v1alpha1.Backup) { + t := h.t + deps := h.deps + g := NewGomegaWithT(t) + _, err := deps.Clientset.PingcapV1alpha1().Backups(bk.Namespace).Update(context.TODO(), bk, metav1.UpdateOptions{}) + g.Expect(err).Should(BeNil()) + + g.Eventually(func() error { + get, err := deps.BackupLister.Backups(bk.Namespace).Get(bk.Name) + if err != nil { + return err + } + + diff := cmp.Diff(get, bk) + if diff == "" { + return nil + } + + return fmt.Errorf("not synced yet: %s", diff) + }, time.Second*10).Should(BeNil()) +} + +func (h *helper) createBackup(bk *v1alpha1.Backup) { + t := h.t + deps := h.deps + g := NewGomegaWithT(t) + _, err := deps.Clientset.PingcapV1alpha1().Backups(bk.Namespace).Create(context.TODO(), bk, metav1.CreateOptions{}) + g.Expect(err).Should(BeNil()) + g.Eventually(func() error { + _, err := deps.BackupLister.Backups(bk.Namespace).Get(bk.Name) + return err + }, time.Second*10).Should(BeNil()) +} + +func (h *helper) deleteBackup(bk *v1alpha1.Backup) { + t := h.t + deps := h.deps + g := NewGomegaWithT(t) + err := deps.Clientset.PingcapV1alpha1().Backups(bk.Namespace).Delete(context.TODO(), bk.Name, metav1.DeleteOptions{}) + g.Expect(err).Should(BeNil()) + g.Eventually(func() error { + _, err := deps.BackupLister.Backups(bk.Namespace).Get(bk.Name) + return err + }, time.Second*10).ShouldNot(BeNil()) +} + +func fakeBackup(ts *int64) *v1alpha1.Backup { + backup := &v1alpha1.Backup{} + if ts == nil { + return backup + } + backup.Status.CommitTs = getTSOStr(*ts) + return backup +} + +func fakeLogBackup(startTS, checkPointTS *int64) *v1alpha1.Backup { + logBackup := &v1alpha1.Backup{} + if startTS == nil { + return logBackup + } + logBackup.Status.CommitTs = getTSOStr(*startTS) + if checkPointTS == nil { + return logBackup + } + logBackup.Status.LogCheckpointTs = getTSOStr(*checkPointTS) + return logBackup +} + +func getTSOStr(ts int64) string { + tso := getTSO(ts) + return strconv.FormatUint(tso, 10) +} + +func getTSO(ts int64) uint64 { + return uint64((ts << 18) * 1000) +} +