BackupTemplate is the specification of the backup structure to get scheduled.
LogBackupTemplate is the specification of the log backup structure to get scheduled.
BackupTemplate is the specification of the backup structure to get scheduled.
LogBackupTemplate is the specification of the log backup structure to get scheduled.
+schedule
+
+string
+
+ |
+
+ Schedule specifies the cron string used for backup scheduling. + |
+
+pause
+
+bool
+
+ |
+
+ Pause means paused backupSchedule + |
+
+maxBackups
+
+int32
+
+ |
+
+ MaxBackups is to specify how many backups we want to keep +0 is magic number to indicate un-limited backups. +if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred +and MaxBackups is ignored. + |
+
+maxReservedTime
+
+string
+
+ |
+
+ MaxReservedTime is to specify how long backups we want to keep. + |
+
+backupTemplate
+
+
+VolumeBackupSpec
+
+
+ |
+
+ BackupTemplate is the specification of the volume backup structure to get scheduled. + |
+
(Members of StorageProvider are embedded into this type.)
StorageProvider configures where and how backups should be stored.
VolumeBackupScheduleSpec describes the attributes that a user creates on a volume backup schedule.
+| Field | +Description | +
|---|---|
+schedule
+
+string
+
+ |
+
+ Schedule specifies the cron string used for backup scheduling. + |
+
+pause
+
+bool
+
+ |
+
+ Pause means paused backupSchedule + |
+
+maxBackups
+
+int32
+
+ |
+
+ MaxBackups is to specify how many backups we want to keep +0 is magic number to indicate un-limited backups. +if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred +and MaxBackups is ignored. + |
+
+maxReservedTime
+
+string
+
+ |
+
+ MaxReservedTime is to specify how long backups we want to keep. + |
+
+backupTemplate
+
+
+VolumeBackupSpec
+
+
+ |
+
+ BackupTemplate is the specification of the volume backup structure to get scheduled. + |
+
(Appears on: @@ -746,10 +877,58 @@ string
VolumeBackupScheduleStatus represents the current status of a volume backup schedule.
+| Field | +Description | +
|---|---|
+lastBackup
+
+string
+
+ |
+
+ LastBackup represents the last backup. + |
+
+lastBackupTime
+
+
+Kubernetes meta/v1.Time
+
+
+ |
+
+ LastBackupTime represents the last time the backup was successfully created. + |
+
+allBackupCleanTime
+
+
+Kubernetes meta/v1.Time
+
+
+ |
+
+ AllBackupCleanTime represents the time when all backup entries are cleaned up + |
+
(Appears on: -VolumeBackup) +VolumeBackup, +VolumeBackupScheduleSpec)
VolumeBackupSpec describes the attributes that a user creates on a volume backup.
diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 591d9f992d..7d881192e5 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -1554,6 +1554,10 @@ spec: jsonPath: .spec.maxBackups name: MaxBackups type: integer + - description: How long backups we want to keep + jsonPath: .spec.maxReservedTime + name: MaxReservedTime + type: string - description: The last backup CR name jsonPath: .status.lastBackup name: LastBackup @@ -4232,7 +4236,7 @@ spec: storageSize: type: string required: - - logBackupTemplate + - backupTemplate - schedule type: object status: diff --git a/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml b/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml index 748bab61d9..e510ee5fbe 100644 --- a/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml +++ b/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml @@ -18,7 +18,33 @@ spec: singular: volumebackupschedule scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: The cron format string used for backup scheduling + jsonPath: .spec.schedule + name: Schedule + type: string + - description: The max number of backups we want to keep + jsonPath: .spec.maxBackups + name: MaxBackups + type: integer + - description: How long backups we want to keep + jsonPath: .spec.maxReservedTime + name: MaxReservedTime + type: string + - description: The last backup CR name + jsonPath: .status.lastBackup + name: LastBackup + priority: 1 + type: string + - description: The last time the backup was successfully created + jsonPath: .status.lastBackupTime + name: LastBackupTime + priority: 1 + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: properties: @@ -29,8 +55,930 @@ spec: metadata: type: object spec: + properties: + backupTemplate: + properties: + clusters: + items: + properties: + k8sClusterName: + type: string + tcName: + type: string + tcNamespace: + type: string + type: object + type: array + template: + properties: + azblob: + properties: + accessTier: + type: string + container: + type: string + path: + type: string + prefix: + type: string + secretName: + type: string + type: object + br: + properties: + checkRequirements: + type: boolean + concurrency: + format: int32 + type: integer + options: + items: + type: string + type: array + sendCredToTikv: + type: boolean + type: object + cleanPolicy: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + gcs: + properties: + bucket: + type: string + bucketAcl: + type: string + location: + type: string + objectAcl: + type: string + path: + type: string + prefix: + type: string + projectId: + type: string + secretName: + type: string + storageClass: + type: string + required: + - projectId + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + local: + properties: + prefix: + type: string + volume: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + required: + - volume + - volumeMount + type: object + priorityClassName: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + s3: + properties: + acl: + type: string + bucket: + type: string + endpoint: + type: string + options: + items: + type: string + type: array + path: + type: string + prefix: + type: string + provider: + type: string + region: + type: string + secretName: + type: string + sse: + type: string + storageClass: + type: string + required: + - provider + type: object + serviceAccount: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + toolImage: + type: string + type: object + type: object + maxBackups: + format: int32 + type: integer + maxReservedTime: + type: string + pause: + type: boolean + schedule: + type: string + required: + - backupTemplate + - schedule type: object status: + properties: + allBackupCleanTime: + format: date-time + type: string + lastBackup: + type: string + lastBackupTime: + format: date-time + type: string type: object required: - metadata @@ -38,6 +986,7 @@ spec: type: object served: true storage: true + subresources: {} status: acceptedNames: kind: "" diff --git a/manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml b/manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml index 29d26b06e9..05e64afc80 100644 --- a/manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml +++ b/manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml @@ -8,6 +8,32 @@ metadata: creationTimestamp: null name: volumebackupschedules.federation.pingcap.com spec: + additionalPrinterColumns: + - JSONPath: .spec.schedule + description: The cron format string used for backup scheduling + name: Schedule + type: string + - JSONPath: .spec.maxBackups + description: The max number of backups we want to keep + name: MaxBackups + type: integer + - JSONPath: .spec.maxReservedTime + description: How long backups we want to keep + name: MaxReservedTime + type: string + - JSONPath: .status.lastBackup + description: The last backup CR name + name: LastBackup + priority: 1 + type: string + - JSONPath: .status.lastBackupTime + description: The last time the backup was successfully created + name: LastBackupTime + priority: 1 + type: date + - JSONPath: .metadata.creationTimestamp + name: Age + type: date group: federation.pingcap.com names: kind: VolumeBackupSchedule @@ -18,6 +44,7 @@ spec: singular: volumebackupschedule preserveUnknownFields: false scope: Namespaced + subresources: {} validation: openAPIV3Schema: properties: @@ -28,8 +55,930 @@ spec: metadata: type: object spec: + properties: + backupTemplate: + properties: + clusters: + items: + properties: + k8sClusterName: + type: string + tcName: + type: string + tcNamespace: + type: string + type: object + type: array + template: + properties: + azblob: + properties: + accessTier: + type: string + container: + type: string + path: + type: string + prefix: + type: string + secretName: + type: string + type: object + br: + properties: + checkRequirements: + type: boolean + concurrency: + format: int32 + type: integer + options: + items: + type: string + type: array + sendCredToTikv: + type: boolean + type: object + cleanPolicy: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + gcs: + properties: + bucket: + type: string + bucketAcl: + type: string + location: + type: string + objectAcl: + type: string + path: + type: string + prefix: + type: string + projectId: + type: string + secretName: + type: string + storageClass: + type: string + required: + - projectId + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + local: + properties: + prefix: + type: string + volume: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + required: + - volume + - volumeMount + type: object + priorityClassName: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + s3: + properties: + acl: + type: string + bucket: + type: string + endpoint: + type: string + options: + items: + type: string + type: array + path: + type: string + prefix: + type: string + provider: + type: string + region: + type: string + secretName: + type: string + sse: + type: string + storageClass: + type: string + required: + - provider + type: object + serviceAccount: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + toolImage: + type: string + type: object + type: object + maxBackups: + format: int32 + type: integer + maxReservedTime: + type: string + pause: + type: boolean + schedule: + type: string + required: + - backupTemplate + - schedule type: object status: + properties: + allBackupCleanTime: + format: date-time + type: string + lastBackup: + type: string + lastBackupTime: + format: date-time + type: string type: object required: - metadata diff --git a/manifests/crd/v1/pingcap.com_backupschedules.yaml b/manifests/crd/v1/pingcap.com_backupschedules.yaml index 43175cd21e..c6479d6916 100644 --- a/manifests/crd/v1/pingcap.com_backupschedules.yaml +++ b/manifests/crd/v1/pingcap.com_backupschedules.yaml @@ -27,6 +27,10 @@ spec: jsonPath: .spec.maxBackups name: MaxBackups type: integer + - description: How long backups we want to keep + jsonPath: .spec.maxReservedTime + name: MaxReservedTime + type: string - description: The last backup CR name jsonPath: .status.lastBackup name: LastBackup @@ -2705,7 +2709,7 @@ spec: storageSize: type: string required: - - logBackupTemplate + - backupTemplate - schedule type: object status: diff --git a/manifests/crd/v1beta1/pingcap.com_backupschedules.yaml b/manifests/crd/v1beta1/pingcap.com_backupschedules.yaml index e865a48de3..c8ce37aef5 100644 --- a/manifests/crd/v1beta1/pingcap.com_backupschedules.yaml +++ b/manifests/crd/v1beta1/pingcap.com_backupschedules.yaml @@ -17,6 +17,10 @@ spec: description: The max number of backups we want to keep name: MaxBackups type: integer + - JSONPath: .spec.maxReservedTime + description: How long backups we want to keep + name: MaxReservedTime + type: string - JSONPath: .status.lastBackup description: The last backup CR name name: LastBackup @@ -2695,7 +2699,7 @@ spec: storageSize: type: string required: - - logBackupTemplate + - backupTemplate - schedule type: object status: diff --git a/manifests/crd_v1beta1.yaml b/manifests/crd_v1beta1.yaml index 56a5de17c1..21d2f0677c 100644 --- a/manifests/crd_v1beta1.yaml +++ b/manifests/crd_v1beta1.yaml @@ -1541,6 +1541,10 @@ spec: description: The max number of backups we want to keep name: MaxBackups type: integer + - JSONPath: .spec.maxReservedTime + description: How long backups we want to keep + name: MaxReservedTime + type: string - JSONPath: .status.lastBackup description: The last backup CR name name: LastBackup @@ -4219,7 +4223,7 @@ spec: storageSize: type: string required: - - logBackupTemplate + - backupTemplate - schedule type: object status: diff --git a/manifests/federation-crd.yaml b/manifests/federation-crd.yaml index f9d1d557f9..acb4e8c030 100644 --- a/manifests/federation-crd.yaml +++ b/manifests/federation-crd.yaml @@ -1046,7 +1046,33 @@ spec: singular: volumebackupschedule scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: The cron format string used for backup scheduling + jsonPath: .spec.schedule + name: Schedule + type: string + - description: The max number of backups we want to keep + jsonPath: .spec.maxBackups + name: MaxBackups + type: integer + - description: How long backups we want to keep + jsonPath: .spec.maxReservedTime + name: MaxReservedTime + type: string + - description: The last backup CR name + jsonPath: .status.lastBackup + name: LastBackup + priority: 1 + type: string + - description: The last time the backup was successfully created + jsonPath: .status.lastBackupTime + name: LastBackupTime + priority: 1 + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: properties: @@ -1057,8 +1083,930 @@ spec: metadata: type: object spec: + properties: + backupTemplate: + properties: + clusters: + items: + properties: + k8sClusterName: + type: string + tcName: + type: string + tcNamespace: + type: string + type: object + type: array + template: + properties: + azblob: + properties: + accessTier: + type: string + container: + type: string + path: + type: string + prefix: + type: string + secretName: + type: string + type: object + br: + properties: + checkRequirements: + type: boolean + concurrency: + format: int32 + type: integer + options: + items: + type: string + type: array + sendCredToTikv: + type: boolean + type: object + cleanPolicy: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + gcs: + properties: + bucket: + type: string + bucketAcl: + type: string + location: + type: string + objectAcl: + type: string + path: + type: string + prefix: + type: string + projectId: + type: string + secretName: + type: string + storageClass: + type: string + required: + - projectId + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + local: + properties: + prefix: + type: string + volume: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + required: + - volume + - volumeMount + type: object + priorityClassName: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + s3: + properties: + acl: + type: string + bucket: + type: string + endpoint: + type: string + options: + items: + type: string + type: array + path: + type: string + prefix: + type: string + provider: + type: string + region: + type: string + secretName: + type: string + sse: + type: string + storageClass: + type: string + required: + - provider + type: object + serviceAccount: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + toolImage: + type: string + type: object + type: object + maxBackups: + format: int32 + type: integer + maxReservedTime: + type: string + pause: + type: boolean + schedule: + type: string + required: + - backupTemplate + - schedule type: object status: + properties: + allBackupCleanTime: + format: date-time + type: string + lastBackup: + type: string + lastBackupTime: + format: date-time + type: string type: object required: - metadata @@ -1066,6 +2014,7 @@ spec: type: object served: true storage: true + subresources: {} status: acceptedNames: kind: "" diff --git a/manifests/federation-crd_v1beta1.yaml b/manifests/federation-crd_v1beta1.yaml index d5449d6473..5f51acbad9 100644 --- a/manifests/federation-crd_v1beta1.yaml +++ b/manifests/federation-crd_v1beta1.yaml @@ -1038,6 +1038,32 @@ metadata: creationTimestamp: null name: volumebackupschedules.federation.pingcap.com spec: + additionalPrinterColumns: + - JSONPath: .spec.schedule + description: The cron format string used for backup scheduling + name: Schedule + type: string + - JSONPath: .spec.maxBackups + description: The max number of backups we want to keep + name: MaxBackups + type: integer + - JSONPath: .spec.maxReservedTime + description: How long backups we want to keep + name: MaxReservedTime + type: string + - JSONPath: .status.lastBackup + description: The last backup CR name + name: LastBackup + priority: 1 + type: string + - JSONPath: .status.lastBackupTime + description: The last time the backup was successfully created + name: LastBackupTime + priority: 1 + type: date + - JSONPath: .metadata.creationTimestamp + name: Age + type: date group: federation.pingcap.com names: kind: VolumeBackupSchedule @@ -1048,6 +1074,7 @@ spec: singular: volumebackupschedule preserveUnknownFields: false scope: Namespaced + subresources: {} validation: openAPIV3Schema: properties: @@ -1058,8 +1085,930 @@ spec: metadata: type: object spec: + properties: + backupTemplate: + properties: + clusters: + items: + properties: + k8sClusterName: + type: string + tcName: + type: string + tcNamespace: + type: string + type: object + type: array + template: + properties: + azblob: + properties: + accessTier: + type: string + container: + type: string + path: + type: string + prefix: + type: string + secretName: + type: string + type: object + br: + properties: + checkRequirements: + type: boolean + concurrency: + format: int32 + type: integer + options: + items: + type: string + type: array + sendCredToTikv: + type: boolean + type: object + cleanPolicy: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + gcs: + properties: + bucket: + type: string + bucketAcl: + type: string + location: + type: string + objectAcl: + type: string + path: + type: string + prefix: + type: string + projectId: + type: string + secretName: + type: string + storageClass: + type: string + required: + - projectId + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + local: + properties: + prefix: + type: string + volume: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + required: + - volume + - volumeMount + type: object + priorityClassName: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + s3: + properties: + acl: + type: string + bucket: + type: string + endpoint: + type: string + options: + items: + type: string + type: array + path: + type: string + prefix: + type: string + provider: + type: string + region: + type: string + secretName: + type: string + sse: + type: string + storageClass: + type: string + required: + - provider + type: object + serviceAccount: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + toolImage: + type: string + type: object + type: object + maxBackups: + format: int32 + type: integer + maxReservedTime: + type: string + pause: + type: boolean + schedule: + type: string + required: + - backupTemplate + - schedule type: object status: + properties: + allBackupCleanTime: + format: date-time + type: string + lastBackup: + type: string + lastBackupTime: + format: date-time + type: string type: object required: - metadata diff --git a/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go index 51c99cfd16..5d06a89a25 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go @@ -419,8 +419,49 @@ func schema_apis_federation_pingcap_v1alpha1_VolumeBackupScheduleSpec(ref common SchemaProps: spec.SchemaProps{ Description: "VolumeBackupScheduleSpec describes the attributes that a user creates on a volume backup schedule.", Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "schedule": { + SchemaProps: spec.SchemaProps{ + Description: "Schedule specifies the cron string used for backup scheduling.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "pause": { + SchemaProps: spec.SchemaProps{ + Description: "Pause means paused backupSchedule", + Type: []string{"boolean"}, + Format: "", + }, + }, + "maxBackups": { + SchemaProps: spec.SchemaProps{ + Description: "MaxBackups is to specify how many backups we want to keep 0 is magic number to indicate un-limited backups. if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred and MaxBackups is ignored.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "maxReservedTime": { + SchemaProps: spec.SchemaProps{ + Description: "MaxReservedTime is to specify how long backups we want to keep.", + Type: []string{"string"}, + Format: "", + }, + }, + "backupTemplate": { + SchemaProps: spec.SchemaProps{ + Description: "BackupTemplate is the specification of the volume backup structure to get scheduled.", + Default: map[string]interface{}{}, + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1.VolumeBackupSpec"), + }, + }, + }, + Required: []string{"schedule", "backupTemplate"}, }, }, + Dependencies: []string{ + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1.VolumeBackupSpec"}, } } diff --git a/pkg/apis/federation/pingcap/v1alpha1/types.go b/pkg/apis/federation/pingcap/v1alpha1/types.go index e6b8500b3d..62ce1bc95c 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/types.go +++ b/pkg/apis/federation/pingcap/v1alpha1/types.go @@ -95,7 +95,8 @@ type VolumeBackupMemberSpec struct { // +optional Env []corev1.EnvVar `json:"env,omitempty"` // BRConfig is the configs for BR - BR *BRConfig `json:"br,omitempty"` + BR *BRConfig `json:"br,omitempty"` + // StorageProvider configures where and how backups should be stored. pingcapv1alpha1.StorageProvider `json:",inline"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` // ToolImage specifies the tool image used in `Backup`, which supports BR. @@ -203,35 +204,58 @@ const ( // // +k8s:openapi-gen=true // +kubebuilder:resource:shortName="vbks" -// +genclient:noStatus +// +kubebuilder:printcolumn:name="Schedule",type=string,JSONPath=`.spec.schedule`,description="The cron format string used for backup scheduling" +// +kubebuilder:printcolumn:name="MaxBackups",type=integer,JSONPath=`.spec.maxBackups`,description="The max number of backups we want to keep" +// +kubebuilder:printcolumn:name="MaxReservedTime",type=string,JSONPath=`.spec.maxReservedTime`,description="How long backups we want to keep" +// +kubebuilder:printcolumn:name="LastBackup",type=string,JSONPath=`.status.lastBackup`,description="The last backup CR name",priority=1 +// +kubebuilder:printcolumn:name="LastBackupTime",type=date,JSONPath=`.status.lastBackupTime`,description="The last time the backup was successfully created",priority=1 +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type VolumeBackupSchedule struct { metav1.TypeMeta `json:",inline"` // +k8s:openapi-gen=false metav1.ObjectMeta `json:"metadata"` Spec VolumeBackupScheduleSpec `json:"spec"` - // +k8s:openapi-gen=false Status VolumeBackupScheduleStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// VolumeBackupScheduleList is VolumeBackupSchedule list // +k8s:openapi-gen=true +// VolumeBackupScheduleList is VolumeBackupSchedule list type VolumeBackupScheduleList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata"` Items []VolumeBackupSchedule `json:"items"` } -// VolumeBackupScheduleSpec describes the attributes that a user creates on a volume backup schedule. // +k8s:openapi-gen=true +// VolumeBackupScheduleSpec describes the attributes that a user creates on a volume backup schedule. type VolumeBackupScheduleSpec struct { + // Schedule specifies the cron string used for backup scheduling. + Schedule string `json:"schedule"` + // Pause means paused backupSchedule + Pause bool `json:"pause,omitempty"` + // MaxBackups is to specify how many backups we want to keep + // 0 is magic number to indicate un-limited backups. + // if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred + // and MaxBackups is ignored. + MaxBackups *int32 `json:"maxBackups,omitempty"` + // MaxReservedTime is to specify how long backups we want to keep. + MaxReservedTime *string `json:"maxReservedTime,omitempty"` + // BackupTemplate is the specification of the volume backup structure to get scheduled. + BackupTemplate VolumeBackupSpec `json:"backupTemplate"` } // VolumeBackupScheduleStatus represents the current status of a volume backup schedule. type VolumeBackupScheduleStatus struct { + // LastBackup represents the last backup. + LastBackup string `json:"lastBackup,omitempty"` + // LastBackupTime represents the last time the backup was successfully created. + LastBackupTime *metav1.Time `json:"lastBackupTime,omitempty"` + // AllBackupCleanTime represents the time when all backup entries are cleaned up + AllBackupCleanTime *metav1.Time `json:"allBackupCleanTime,omitempty"` } // +genclient diff --git a/pkg/apis/federation/pingcap/v1alpha1/volume_backup_schedule.go b/pkg/apis/federation/pingcap/v1alpha1/volume_backup_schedule.go new file mode 100644 index 0000000000..dfee21349c --- /dev/null +++ b/pkg/apis/federation/pingcap/v1alpha1/volume_backup_schedule.go @@ -0,0 +1,25 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "fmt" + "time" + + constants "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" +) + +func (vbks *VolumeBackupSchedule) GetBackupCRDName(timestamp time.Time) string { + return fmt.Sprintf("%s-%s", vbks.GetName(), timestamp.UTC().Format(constants.BackupNameTimeFormat)) +} diff --git a/pkg/apis/federation/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/federation/pingcap/v1alpha1/zz_generated.deepcopy.go index 991207cc54..ed9d6b07cf 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/federation/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -216,8 +216,8 @@ func (in *VolumeBackupSchedule) DeepCopyInto(out *VolumeBackupSchedule) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) return } @@ -275,6 +275,17 @@ func (in *VolumeBackupScheduleList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeBackupScheduleSpec) DeepCopyInto(out *VolumeBackupScheduleSpec) { *out = *in + if in.MaxBackups != nil { + in, out := &in.MaxBackups, &out.MaxBackups + *out = new(int32) + **out = **in + } + if in.MaxReservedTime != nil { + in, out := &in.MaxReservedTime, &out.MaxReservedTime + *out = new(string) + **out = **in + } + in.BackupTemplate.DeepCopyInto(&out.BackupTemplate) return } @@ -291,6 +302,14 @@ func (in *VolumeBackupScheduleSpec) DeepCopy() *VolumeBackupScheduleSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeBackupScheduleStatus) DeepCopyInto(out *VolumeBackupScheduleStatus) { *out = *in + if in.LastBackupTime != nil { + in, out := &in.LastBackupTime, &out.LastBackupTime + *out = (*in).DeepCopy() + } + if in.AllBackupCleanTime != nil { + in, out := &in.AllBackupCleanTime, &out.AllBackupCleanTime + *out = (*in).DeepCopy() + } return } diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 5834ca8ceb..6aeedb5c1d 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -945,7 +945,7 @@ func schema_pkg_apis_pingcap_v1alpha1_BackupScheduleSpec(ref common.ReferenceCal }, }, }, - Required: []string{"schedule", "logBackupTemplate"}, + Required: []string{"schedule", "backupTemplate"}, }, }, Dependencies: []string{ diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 3a17ad77dc..152bd6a10d 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -2204,6 +2204,7 @@ type BackupStatus struct { // +kubebuilder:resource:shortName="bks" // +kubebuilder:printcolumn:name="Schedule",type=string,JSONPath=`.spec.schedule`,description="The cron format string used for backup scheduling" // +kubebuilder:printcolumn:name="MaxBackups",type=integer,JSONPath=`.spec.maxBackups`,description="The max number of backups we want to keep" +// +kubebuilder:printcolumn:name="MaxReservedTime",type=string,JSONPath=`.spec.maxReservedTime`,description="How long backups we want to keep" // +kubebuilder:printcolumn:name="LastBackup",type=string,JSONPath=`.status.lastBackup`,description="The last backup CR name",priority=1 // +kubebuilder:printcolumn:name="LastBackupTime",type=date,JSONPath=`.status.lastBackupTime`,description="The last time the backup was successfully created",priority=1 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` @@ -2243,9 +2244,9 @@ type BackupScheduleSpec struct { // MaxReservedTime is to specify how long backups we want to keep. MaxReservedTime *string `json:"maxReservedTime,omitempty"` // BackupTemplate is the specification of the backup structure to get scheduled. - // +optional BackupTemplate BackupSpec `json:"backupTemplate"` // LogBackupTemplate is the specification of the log backup structure to get scheduled. + // +optional LogBackupTemplate *BackupSpec `json:"logBackupTemplate"` // The storageClassName of the persistent volume for Backup data storage if not storage class name set in BackupSpec. // Defaults to Kubernetes default storage class. diff --git a/pkg/backup/backupschedule/backup_schedule_manager.go b/pkg/backup/backupschedule/backup_schedule_manager.go index 33279f8f75..843467f582 100644 --- a/pkg/backup/backupschedule/backup_schedule_manager.go +++ b/pkg/backup/backupschedule/backup_schedule_manager.go @@ -70,6 +70,7 @@ func (bm *backupScheduleManager) Sync(bs *v1alpha1.BackupSchedule) error { } // delete the last backup job for release the backup PVC + if err := bm.deleteLastBackupJob(bs); err != nil { return nil } @@ -373,7 +374,7 @@ func (bm *backupScheduleManager) backupGCByMaxReservedTime(bs *v1alpha1.BackupSc return } } else { - expiredBackups, err = caculateExpiredBackups(ascBackups, reservedTime) + expiredBackups, err = calculateExpiredBackups(ascBackups, reservedTime) if err != nil { klog.Errorf("caculate expired backups without log backup, err: %s", err) return @@ -404,7 +405,7 @@ func (bm *backupScheduleManager) backupGCByMaxReservedTime(bs *v1alpha1.BackupSc } } -// separateSnapshotBackupsAndLogBackup return snapot backups ordry by create time asc and log backup +// separateSnapshotBackupsAndLogBackup return snapshot backups order by create time asc and log backup func separateSnapshotBackupsAndLogBackup(backupsList []*v1alpha1.Backup) ([]*v1alpha1.Backup, *v1alpha1.Backup) { var ( ascBackupList = make([]*v1alpha1.Backup, 0) @@ -479,7 +480,7 @@ func calExpiredBackupsAndLogBackup(backupsList []*v1alpha1.Backup, logBackup *v1 return expiredBackups, truncateTSO, nil } -func caculateExpiredBackups(backupsList []*v1alpha1.Backup, reservedTime time.Duration) ([]*v1alpha1.Backup, error) { +func calculateExpiredBackups(backupsList []*v1alpha1.Backup, reservedTime time.Duration) ([]*v1alpha1.Backup, error) { expiredTS := config.TSToTSO(time.Now().Add(-1 * reservedTime).Unix()) i := 0 for ; i < len(backupsList); i++ { diff --git a/pkg/backup/backupschedule/backup_schedule_manager_test.go b/pkg/backup/backupschedule/backup_schedule_manager_test.go index 463cc83b1f..baa8d8c33e 100644 --- a/pkg/backup/backupschedule/backup_schedule_manager_test.go +++ b/pkg/backup/backupschedule/backup_schedule_manager_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 PingCAP, Inc. +// Copyright 2023 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -234,7 +234,7 @@ func TestBuildBackup(t *testing.T) { } } -func TestCaculateExpiredBackupsWithLogBackup(t *testing.T) { +func TestCalculateExpiredBackupsWithLogBackup(t *testing.T) { g := NewGomegaWithT(t) type testCase struct { backups []*v1alpha1.Backup diff --git a/pkg/backup/restore/restore_manager.go b/pkg/backup/restore/restore_manager.go index 1fedbabbff..e43f0a17c8 100644 --- a/pkg/backup/restore/restore_manager.go +++ b/pkg/backup/restore/restore_manager.go @@ -231,7 +231,7 @@ func (rm *restoreManager) syncRestoreJob(restore *v1alpha1.Restore) error { } // read cluster meta from external storage since k8s size limitation on annotation/configMap -// after volume retore job complete, br output a meta file for controller to reconfig the tikvs +// after volume restore job complete, br output a meta file for controller to reconfig the tikvs // since the meta file may big, so we use remote storage as bridge to pass it from restore manager to controller func (rm *restoreManager) readRestoreMetaFromExternalStorage(r *v1alpha1.Restore) (*snapshotter.CloudSnapBackup, string, error) { // since the restore meta is small (~5M), assume 1 minutes is enough diff --git a/pkg/client/federation/clientset/versioned/typed/pingcap/v1alpha1/fake/fake_volumebackupschedule.go b/pkg/client/federation/clientset/versioned/typed/pingcap/v1alpha1/fake/fake_volumebackupschedule.go index 7431f7f560..f02aa0f12b 100644 --- a/pkg/client/federation/clientset/versioned/typed/pingcap/v1alpha1/fake/fake_volumebackupschedule.go +++ b/pkg/client/federation/clientset/versioned/typed/pingcap/v1alpha1/fake/fake_volumebackupschedule.go @@ -99,6 +99,18 @@ func (c *FakeVolumeBackupSchedules) Update(ctx context.Context, volumeBackupSche return obj.(*v1alpha1.VolumeBackupSchedule), err } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeVolumeBackupSchedules) UpdateStatus(ctx context.Context, volumeBackupSchedule *v1alpha1.VolumeBackupSchedule, opts v1.UpdateOptions) (*v1alpha1.VolumeBackupSchedule, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(volumebackupschedulesResource, "status", c.ns, volumeBackupSchedule), &v1alpha1.VolumeBackupSchedule{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.VolumeBackupSchedule), err +} + // Delete takes name of the volumeBackupSchedule and deletes it. Returns an error if one occurs. func (c *FakeVolumeBackupSchedules) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. diff --git a/pkg/client/federation/clientset/versioned/typed/pingcap/v1alpha1/volumebackupschedule.go b/pkg/client/federation/clientset/versioned/typed/pingcap/v1alpha1/volumebackupschedule.go index 4c7d85845c..ca701242a5 100644 --- a/pkg/client/federation/clientset/versioned/typed/pingcap/v1alpha1/volumebackupschedule.go +++ b/pkg/client/federation/clientset/versioned/typed/pingcap/v1alpha1/volumebackupschedule.go @@ -37,6 +37,7 @@ type VolumeBackupSchedulesGetter interface { type VolumeBackupScheduleInterface interface { Create(ctx context.Context, volumeBackupSchedule *v1alpha1.VolumeBackupSchedule, opts v1.CreateOptions) (*v1alpha1.VolumeBackupSchedule, error) Update(ctx context.Context, volumeBackupSchedule *v1alpha1.VolumeBackupSchedule, opts v1.UpdateOptions) (*v1alpha1.VolumeBackupSchedule, error) + UpdateStatus(ctx context.Context, volumeBackupSchedule *v1alpha1.VolumeBackupSchedule, opts v1.UpdateOptions) (*v1alpha1.VolumeBackupSchedule, error) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.VolumeBackupSchedule, error) @@ -132,6 +133,22 @@ func (c *volumeBackupSchedules) Update(ctx context.Context, volumeBackupSchedule return } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *volumeBackupSchedules) UpdateStatus(ctx context.Context, volumeBackupSchedule *v1alpha1.VolumeBackupSchedule, opts v1.UpdateOptions) (result *v1alpha1.VolumeBackupSchedule, err error) { + result = &v1alpha1.VolumeBackupSchedule{} + err = c.client.Put(). + Namespace(c.ns). + Resource("volumebackupschedules"). + Name(volumeBackupSchedule.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeBackupSchedule). + Do(ctx). + Into(result) + return +} + // Delete takes name of the volumeBackupSchedule and deletes it. Returns an error if one occurs. func (c *volumeBackupSchedules) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { return c.client.Delete(). diff --git a/pkg/controller/backupschedule/backup_schedule_control_test.go b/pkg/controller/backupschedule/backup_schedule_control_test.go index 87921db45d..720c04faf2 100644 --- a/pkg/controller/backupschedule/backup_schedule_control_test.go +++ b/pkg/controller/backupschedule/backup_schedule_control_test.go @@ -81,26 +81,26 @@ func TestBackupScheduleControlUpdateBackupSchedule(t *testing.T) { }, }, { - name: "backup schedule status update failed", + name: "normal", update: func(bs *v1alpha1.BackupSchedule) { bs.Status.LastBackupTime = &metav1.Time{Time: time.Now()} }, syncBsManagerErr: false, - updateStatusErr: true, + updateStatusErr: false, errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(strings.Contains(err.Error(), "update backupSchedule status error")).To(Equal(true)) + g.Expect(err).NotTo(HaveOccurred()) }, }, { - name: "normal", + name: "backup schedule status update failed", update: func(bs *v1alpha1.BackupSchedule) { bs.Status.LastBackupTime = &metav1.Time{Time: time.Now()} }, syncBsManagerErr: false, - updateStatusErr: false, + updateStatusErr: true, errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).NotTo(HaveOccurred()) + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "update backupSchedule status error")).To(Equal(true)) }, }, } diff --git a/pkg/controller/br_fed_dependences.go b/pkg/controller/br_fed_dependences.go index 02bd0078b3..732c568bf5 100644 --- a/pkg/controller/br_fed_dependences.go +++ b/pkg/controller/br_fed_dependences.go @@ -101,6 +101,15 @@ func NewBrFedDependencies(cliCfg *BrFedCLIConfig, clientset versioned.Interface, return deps } +// NewSimpleFedClientDependencies returns a dependencies using NewSimpleClientset useful for testing. +func NewSimpleFedClientDependencies() *BrFedDependencies { + deps := NewFakeBrFedDependencies() + + // TODO make all controller use real controller with simple client. + deps.FedVolumeBackupControl = NewRealFedVolumeBackupControl(deps.Clientset, deps.Recorder) + return deps +} + func NewFakeBrFedDependencies() *BrFedDependencies { cli := fake.NewSimpleClientset() kubeCli := kubefake.NewSimpleClientset() diff --git a/pkg/controller/controller_utils.go b/pkg/controller/controller_utils.go index b2c141dd35..b708343732 100644 --- a/pkg/controller/controller_utils.go +++ b/pkg/controller/controller_utils.go @@ -186,6 +186,20 @@ func GetBackupScheduleOwnerRef(bs *v1alpha1.BackupSchedule) metav1.OwnerReferenc } } +// GetFedVolumeBackupScheduleOwnerRef returns FedVolumeBackupSchedule's OwnerReference +func GetFedVolumeBackupScheduleOwnerRef(vbks *fedv1alpha1.VolumeBackupSchedule) metav1.OwnerReference { + controller := true + blockOwnerDeletion := true + return metav1.OwnerReference{ + APIVersion: FedVolumeBackupScheduleControllerKind.GroupVersion().String(), + Kind: FedVolumeBackupScheduleControllerKind.Kind, + Name: vbks.GetName(), + UID: vbks.GetUID(), + Controller: &controller, + BlockOwnerDeletion: &blockOwnerDeletion, + } +} + func GetTiDBMonitorOwnerRef(monitor *v1alpha1.TidbMonitor) metav1.OwnerReference { controller := true blockOwnerDeletion := true diff --git a/pkg/controller/fed_backup_schedule_status_updater.go b/pkg/controller/fed_backup_schedule_status_updater.go new file mode 100644 index 0000000000..2277930a7a --- /dev/null +++ b/pkg/controller/fed_backup_schedule_status_updater.go @@ -0,0 +1,113 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "fmt" + + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" + informers "github.com/pingcap/tidb-operator/pkg/client/federation/informers/externalversions/pingcap/v1alpha1" + listers "github.com/pingcap/tidb-operator/pkg/client/federation/listers/pingcap/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" +) + +// VolumeBackupScheduleStatusUpdaterInterface is an interface used to update the VolumeBackupScheduleStatus associated with a VolumeBackupSchedule. +// For any use other than testing, clients should create an instance using NewRealBackupScheduleStatusUpdater. +type VolumeBackupScheduleStatusUpdaterInterface interface { + // UpdateBackupScheduleStatus sets the backupSchedule's Status to status. Implementations are required to retry on conflicts, + // but fail on other errors. If the returned error is nil backup's Status has been successfully set to status. + UpdateBackupScheduleStatus(*v1alpha1.VolumeBackupSchedule, *v1alpha1.VolumeBackupScheduleStatus, *v1alpha1.VolumeBackupScheduleStatus) error +} + +// NewRealVolumeBackupScheduleStatusUpdater returns a VolumeBackupScheduleStatusUpdaterInterface that updates the Status of a VolumeBackupScheduleStatus, +// using the supplied client and bsLister. +func NewRealVolumeBackupScheduleStatusUpdater(deps *BrFedDependencies) VolumeBackupScheduleStatusUpdaterInterface { + return &realVolumeBackupScheduleStatusUpdater{ + deps: deps, + } +} + +type realVolumeBackupScheduleStatusUpdater struct { + deps *BrFedDependencies +} + +func (u *realVolumeBackupScheduleStatusUpdater) UpdateBackupScheduleStatus( + bs *v1alpha1.VolumeBackupSchedule, + newStatus *v1alpha1.VolumeBackupScheduleStatus, + oldStatus *v1alpha1.VolumeBackupScheduleStatus) error { + + ns := bs.GetNamespace() + bsName := bs.GetName() + // don't wait due to limited number of clients, but backoff after the default number of steps + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + _, updateErr := u.deps.Clientset.FederationV1alpha1().VolumeBackupSchedules(ns).Update(context.TODO(), bs, metav1.UpdateOptions{}) + if updateErr == nil { + klog.Infof("VolumeBackupSchedule: [%s/%s] updated successfully", ns, bsName) + return nil + } + if updated, err := u.deps.VolumeBackupScheduleLister.VolumeBackupSchedules(ns).Get(bsName); err == nil { + // make a copy so we don't mutate the shared cache + bs = updated.DeepCopy() + bs.Status = *newStatus + } else { + utilruntime.HandleError(fmt.Errorf("error getting updated backupSchedule %s/%s from lister: %v", ns, bsName, err)) + } + + return updateErr + }) + return err +} + +var _ VolumeBackupScheduleStatusUpdaterInterface = &realVolumeBackupScheduleStatusUpdater{} + +// FakeVolumeBackupScheduleStatusUpdater is a fake VolumeBackupScheduleStatusUpdaterInterface +type FakeVolumeBackupScheduleStatusUpdater struct { + BsLister listers.VolumeBackupScheduleLister + BsIndexer cache.Indexer + updateBsTracker RequestTracker +} + +// NewFakeVolumeBackupScheduleStatusUpdater returns a FakeVolumeBackupScheduleStatusUpdater +func NewFakeVolumeBackupScheduleStatusUpdater(bsInformer informers.VolumeBackupScheduleInformer) *FakeVolumeBackupScheduleStatusUpdater { + return &FakeVolumeBackupScheduleStatusUpdater{ + bsInformer.Lister(), + bsInformer.Informer().GetIndexer(), + RequestTracker{}, + } +} + +// SetUpdateBackupScheduleError sets the error attributes of updateBackupScheduleTracker +func (u *FakeVolumeBackupScheduleStatusUpdater) SetUpdateBackupScheduleError(err error, after int) { + u.updateBsTracker.err = err + u.updateBsTracker.after = after + u.updateBsTracker.SetError(err).SetAfter(after) +} + +// UpdateBackupScheduleStatus updates the BackupSchedule +func (u *FakeVolumeBackupScheduleStatusUpdater) UpdateBackupScheduleStatus(bs *v1alpha1.VolumeBackupSchedule, _ *v1alpha1.VolumeBackupScheduleStatus, _ *v1alpha1.VolumeBackupScheduleStatus) error { + defer u.updateBsTracker.Inc() + if u.updateBsTracker.ErrorReady() { + defer u.updateBsTracker.Reset() + return u.updateBsTracker.GetError() + } + + return u.BsIndexer.Update(bs) +} + +var _ VolumeBackupScheduleStatusUpdaterInterface = &FakeVolumeBackupScheduleStatusUpdater{} diff --git a/pkg/controller/fed_volume_backup_control.go b/pkg/controller/fed_volume_backup_control.go index 0de42e3bb4..25d82a3982 100644 --- a/pkg/controller/fed_volume_backup_control.go +++ b/pkg/controller/fed_volume_backup_control.go @@ -31,7 +31,7 @@ import ( listers "github.com/pingcap/tidb-operator/pkg/client/federation/listers/pingcap/v1alpha1" ) -// FedVolumeBackupControlInterface manages federaton VolumeBackups used in VolumeBackupSchedule +// FedVolumeBackupControlInterface manages federation VolumeBackups used in VolumeBackupSchedule type FedVolumeBackupControlInterface interface { CreateVolumeBackup(backup *v1alpha1.VolumeBackup) (*v1alpha1.VolumeBackup, error) DeleteVolumeBackup(backup *v1alpha1.VolumeBackup) error @@ -111,7 +111,7 @@ type FakeFedVolumeBackupControl struct { deleteVolumeBackupTracker RequestTracker } -// NewFakeBackupControl returns a FakeBackupControl +// NewFakeFedVolumeBackupControl returns a FakeFedVolumeBackupControl func NewFakeFedVolumeBackupControl(volumeBackupInformer informers.VolumeBackupInformer) *FakeFedVolumeBackupControl { return &FakeFedVolumeBackupControl{ volumeBackupInformer.Lister(), diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control.go index ad568e2eaf..8a7d965013 100644 --- a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control.go +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control.go @@ -14,10 +14,11 @@ package fedvolumebackupschedule import ( + apiequality "k8s.io/apimachinery/pkg/api/equality" + errorutils "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/cache" "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" - "github.com/pingcap/tidb-operator/pkg/client/federation/clientset/versioned" informers "github.com/pingcap/tidb-operator/pkg/client/federation/informers/externalversions/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/fedvolumebackup" @@ -31,24 +32,41 @@ type ControlInterface interface { UpdateBackupSchedule(volumeBackupSchedule *v1alpha1.VolumeBackupSchedule) error } -// NewDefaultVolumeBackupScheduleControl returns a new instance of the default VolumeBackupSchedue ControlInterface implementation. +// NewDefaultVolumeBackupScheduleControl returns a new instance of the default VolumeBackupSchedule ControlInterface implementation. func NewDefaultVolumeBackupScheduleControl( - cli versioned.Interface, + statusUpdater controller.VolumeBackupScheduleStatusUpdaterInterface, backupScheduleManager fedvolumebackup.BackupScheduleManager) ControlInterface { return &defaultBackupScheduleControl{ - cli, + statusUpdater, backupScheduleManager, } } type defaultBackupScheduleControl struct { - cli versioned.Interface - bsManager fedvolumebackup.BackupScheduleManager + statusUpdater controller.VolumeBackupScheduleStatusUpdaterInterface + bsManager fedvolumebackup.BackupScheduleManager } // UpdateBackupSchedule executes the core logic loop for a VolumeBackupSchedule. -func (c *defaultBackupScheduleControl) UpdateBackupSchedule(volumeBackupSchedule *v1alpha1.VolumeBackupSchedule) error { - return c.bsManager.Sync(volumeBackupSchedule) +func (c *defaultBackupScheduleControl) UpdateBackupSchedule(vbs *v1alpha1.VolumeBackupSchedule) error { + var errs []error + oldStatus := vbs.Status.DeepCopy() + + if err := c.updateBackupSchedule(vbs); err != nil { + errs = append(errs, err) + } + if apiequality.Semantic.DeepEqual(&vbs.Status, oldStatus) { + return errorutils.NewAggregate(errs) + } + if err := c.statusUpdater.UpdateBackupScheduleStatus(vbs.DeepCopy(), &vbs.Status, oldStatus); err != nil { + errs = append(errs, err) + } + + return errorutils.NewAggregate(errs) +} + +func (c *defaultBackupScheduleControl) updateBackupSchedule(vbs *v1alpha1.VolumeBackupSchedule) error { + return c.bsManager.Sync(vbs) } var _ ControlInterface = &defaultBackupScheduleControl{} diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control_test.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control_test.go new file mode 100644 index 0000000000..eb58ae0ed7 --- /dev/null +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_control_test.go @@ -0,0 +1,121 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package fedvolumebackupschedule + +import ( + "fmt" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/client/federation/clientset/versioned/fake" + informers "github.com/pingcap/tidb-operator/pkg/client/federation/informers/externalversions" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/fedvolumebackup/backupschedule" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBackupScheduleControlUpdateBackupSchedule(t *testing.T) { + g := NewGomegaWithT(t) + + type testcase struct { + name string + update func(bs *v1alpha1.VolumeBackupSchedule) + syncBsManagerErr bool + updateStatusErr bool + errExpectFn func(*GomegaWithT, error) + } + testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + + bs := newBackupSchedule() + if test.update != nil { + test.update(bs) + } + control, bsManager, bsStatusUpdater := newFakeBackupScheduleControl() + + if test.syncBsManagerErr { + bsManager.SetSyncError(fmt.Errorf("backup schedule sync error")) + } + + if test.updateStatusErr { + bsStatusUpdater.SetUpdateBackupScheduleError(fmt.Errorf("update backupSchedule status error"), 0) + } + + err := control.UpdateBackupSchedule(bs) + if test.errExpectFn != nil { + test.errExpectFn(g, err) + } + } + tests := []testcase{ + { + name: "backup schedule sync error", + update: nil, + syncBsManagerErr: true, + updateStatusErr: false, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "backup schedule sync error")).To(Equal(true)) + }, + }, + { + name: "backup schedule status is not updated", + update: nil, + syncBsManagerErr: false, + updateStatusErr: false, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).NotTo(HaveOccurred()) + }, + }, + { + name: "normal", + update: func(bs *v1alpha1.VolumeBackupSchedule) { + bs.Status.LastBackupTime = &metav1.Time{Time: time.Now()} + }, + syncBsManagerErr: false, + updateStatusErr: false, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).NotTo(HaveOccurred()) + }, + }, + { + name: "backup schedule status update failed", + update: func(bs *v1alpha1.VolumeBackupSchedule) { + bs.Status.LastBackupTime = &metav1.Time{Time: time.Now()} + }, + syncBsManagerErr: false, + updateStatusErr: true, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "update backupSchedule status error")).To(Equal(true)) + }, + }, + } + + for i := range tests { + testFn(&tests[i], t) + } +} + +func newFakeBackupScheduleControl() (ControlInterface, *backupschedule.FakeBackupScheduleManager, *controller.FakeVolumeBackupScheduleStatusUpdater) { + cli := fake.NewSimpleClientset() + bsInformer := informers.NewSharedInformerFactory(cli, 0).Federation().V1alpha1().VolumeBackupSchedules() + statusUpdater := controller.NewFakeVolumeBackupScheduleStatusUpdater(bsInformer) + bsManager := backupschedule.NewFakeBackupScheduleManager() + control := NewDefaultVolumeBackupScheduleControl(statusUpdater, bsManager) + + return control, bsManager, statusUpdater +} diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go index d3db48aa16..705500e0aa 100644 --- a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go @@ -45,7 +45,7 @@ type Controller struct { func NewController(deps *controller.BrFedDependencies) *Controller { c := &Controller{ deps: deps, - control: NewDefaultVolumeBackupScheduleControl(deps.Clientset, backupschedule.NewBackupScheduleManager(deps)), + control: NewDefaultVolumeBackupScheduleControl(controller.NewRealVolumeBackupScheduleStatusUpdater(deps), backupschedule.NewBackupScheduleManager(deps)), queue: workqueue.NewNamedRateLimitingQueue( controller.NewControllerRateLimiter(1*time.Second, 100*time.Second), "volumeBackupSchedule", diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller_test.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller_test.go new file mode 100644 index 0000000000..346ed721c4 --- /dev/null +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller_test.go @@ -0,0 +1,148 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package fedvolumebackupschedule + +import ( + "fmt" + "testing" + + "k8s.io/utils/pointer" + + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + //"k8s.io/utils/pointer" +) + +func TestBackupScheduleControllerEnqueueBackupSchedule(t *testing.T) { + g := NewGomegaWithT(t) + bks := newBackupSchedule() + bsc, _, _ := newFakeBackupScheduleController() + bsc.enqueueBackupSchedule(bks) + g.Expect(bsc.queue.Len()).To(Equal(1)) +} + +func TestBackupScheduleControllerEnqueueBackupScheduleFailed(t *testing.T) { + g := NewGomegaWithT(t) + bsc, _, _ := newFakeBackupScheduleController() + bsc.enqueueBackupSchedule(struct{}{}) + g.Expect(bsc.queue.Len()).To(Equal(0)) +} + +func TestBackupScheduleControllerSync(t *testing.T) { + g := NewGomegaWithT(t) + type testcase struct { + name string + addBsToIndexer bool + invalidKeyFn func(bs *v1alpha1.VolumeBackupSchedule) string + errExpectFn func(*GomegaWithT, error) + } + + testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + + bs := newBackupSchedule() + bsc, bsIndexer, _ := newFakeBackupScheduleController() + + if test.addBsToIndexer { + err := bsIndexer.Add(bs) + g.Expect(err).NotTo(HaveOccurred()) + } + + key, _ := cache.DeletionHandlingMetaNamespaceKeyFunc(bs) + if test.invalidKeyFn != nil { + key = test.invalidKeyFn(bs) + } + + err := bsc.sync(key) + + if test.errExpectFn != nil { + test.errExpectFn(g, err) + } + } + + tests := []testcase{ + { + name: "normal", + addBsToIndexer: true, + invalidKeyFn: nil, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).NotTo(HaveOccurred()) + }, + }, + { + name: "invalid backup key", + addBsToIndexer: true, + invalidKeyFn: func(bs *v1alpha1.VolumeBackupSchedule) string { + return fmt.Sprintf("test/demo/%s", bs.GetName()) + }, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).To(HaveOccurred()) + }, + }, + { + name: "can't found backup schedule", + addBsToIndexer: false, + invalidKeyFn: nil, + errExpectFn: func(g *GomegaWithT, err error) { + g.Expect(err).NotTo(HaveOccurred()) + }, + }, + } + + for i := range tests { + testFn(&tests[i], t) + } + +} + +func newFakeBackupScheduleController() (*Controller, cache.Indexer, *controller.FakeFedVolumeBackupControl) { + fakeDeps := controller.NewFakeBrFedDependencies() + bsc := NewController(fakeDeps) + bsInformer := fakeDeps.InformerFactory.Federation().V1alpha1().VolumeBackups() + backupScheduleControl := controller.NewFakeFedVolumeBackupControl(bsInformer) + bsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: bsc.enqueueBackupSchedule, + UpdateFunc: func(old, cur interface{}) { + bsc.enqueueBackupSchedule(cur) + }, + DeleteFunc: bsc.enqueueBackupSchedule, + }) + //bsc.control = backupScheduleControl + return bsc, bsInformer.Informer().GetIndexer(), backupScheduleControl +} + +func newBackupSchedule() *v1alpha1.VolumeBackupSchedule { + return &v1alpha1.VolumeBackupSchedule{ + TypeMeta: metav1.TypeMeta{ + Kind: "BackupScheduler", + APIVersion: "pingcap.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bks", + Namespace: corev1.NamespaceDefault, + UID: types.UID("test-bks"), + }, + Spec: v1alpha1.VolumeBackupScheduleSpec{ + Schedule: "1 */10 * * *", + MaxBackups: pointer.Int32Ptr(10), + BackupTemplate: v1alpha1.VolumeBackupSpec{}, + }, + } +} diff --git a/pkg/fedvolumebackup/backupschedule/backup_schedule_manager.go b/pkg/fedvolumebackup/backupschedule/backup_schedule_manager.go index deeb552f97..618097a175 100644 --- a/pkg/fedvolumebackup/backupschedule/backup_schedule_manager.go +++ b/pkg/fedvolumebackup/backupschedule/backup_schedule_manager.go @@ -14,11 +14,22 @@ package backupschedule import ( + "fmt" + "path" + "sort" "time" + perrors "github.com/pingcap/errors" + "github.com/pingcap/tidb-operator/pkg/apis/label" + "github.com/pingcap/tidb-operator/pkg/apis/util/config" + "github.com/pingcap/tidb-operator/pkg/util" + "github.com/robfig/cron" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" + pingcapv1alpha1 "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/fedvolumebackup" ) @@ -38,15 +49,337 @@ func NewBackupScheduleManager(deps *controller.BrFedDependencies) fedvolumebacku } } -func (bm *backupScheduleManager) Sync(volumeBackupSchedule *v1alpha1.VolumeBackupSchedule) error { - ns := volumeBackupSchedule.GetNamespace() - name := volumeBackupSchedule.GetName() - // TODO(federation): implement the main logic of backupSchedule - klog.Infof("sync VolumeBackupSchedule %s/%s", ns, name) +func (bm *backupScheduleManager) Sync(vbs *v1alpha1.VolumeBackupSchedule) error { + defer bm.backupGC(vbs) + if vbs.Spec.Pause { + return controller.IgnoreErrorf("backupSchedule %s/%s has been paused", vbs.GetNamespace(), vbs.GetName()) + } + + if err := bm.canPerformNextBackup(vbs); err != nil { + return err + } + + scheduledTime, err := getLastScheduledTime(vbs, bm.now) + if scheduledTime == nil { + return err + } + + backup, err := createBackup(bm.deps.FedVolumeBackupControl, vbs, *scheduledTime) + if err != nil { + return err + } + + vbs.Status.LastBackup = backup.GetName() + vbs.Status.LastBackupTime = &metav1.Time{Time: *scheduledTime} + vbs.Status.AllBackupCleanTime = nil return nil } +// getLastScheduledTime return the newest time need to be scheduled according last backup time. +// the return time is not before now and return nil if there's no such time. +func getLastScheduledTime(vbs *v1alpha1.VolumeBackupSchedule, nowFn nowFn) (*time.Time, error) { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + sched, err := cron.ParseStandard(vbs.Spec.Schedule) + if err != nil { + return nil, fmt.Errorf("parse backup schedule %s/%s cron format %s failed, err: %v", ns, bsName, vbs.Spec.Schedule, err) + } + + var earliestTime time.Time + if vbs.Status.LastBackupTime != nil { + earliestTime = vbs.Status.LastBackupTime.Time + } else if vbs.Status.AllBackupCleanTime != nil { + // Recovery from a long paused backup schedule may cause problem like "incorrect clock", + // so we introduce AllBackupCleanTime field to solve this problem. + earliestTime = vbs.Status.AllBackupCleanTime.Time + } else { + // If none found, then this is either a recently created backupSchedule, + // or the backupSchedule status info was somehow lost, + // or that we have started a backup, but have not update backupSchedule status yet + // (distributed systems can have arbitrary delays). + // In any case, use the creation time of the backupSchedule as last known start time. + earliestTime = vbs.ObjectMeta.CreationTimestamp.Time + } + + now := nowFn() + if earliestTime.After(now) { + // timestamp fallback, waiting for the next backup schedule period + klog.Errorf("backup schedule %s/%s timestamp fallback, lastBackupTime: %s, now: %s", + ns, bsName, earliestTime.Format(time.RFC3339), now.Format(time.RFC3339)) + return nil, nil + } + + var scheduledTimes []time.Time + for t := sched.Next(earliestTime); !t.After(now); t = sched.Next(t) { + scheduledTimes = append(scheduledTimes, t) + // If there is a bug somewhere, or incorrect clock + // on controller's server or apiservers (for setting creationTimestamp) + // then there could be so many missed start times (it could be off + // by decades or more), that it would eat up all the CPU and memory + // of this controller. In that case, we want to not try to list + // all the missed start times. + // + // I've somewhat arbitrarily picked 100, as more than 80, + // but less than "lots". + if len(scheduledTimes) > 100 { + // We can't get the last backup schedule time + if vbs.Status.LastBackupTime == nil && vbs.Status.AllBackupCleanTime != nil { + // Recovery backup schedule from pause status, should refresh AllBackupCleanTime to avoid unschedulable problem + vbs.Status.AllBackupCleanTime = &metav1.Time{Time: nowFn()} + return nil, controller.RequeueErrorf("recovery backup schedule %s/%s from pause status, refresh AllBackupCleanTime.", ns, bsName) + } + klog.Error("Too many missed start backup schedule time (> 100). Check the clock.") + return nil, nil + } + } + + if len(scheduledTimes) == 0 { + klog.V(4).Infof("unmet backup schedule %s/%s start time, waiting for the next backup schedule period", ns, bsName) + return nil, nil + } + scheduledTime := scheduledTimes[len(scheduledTimes)-1] + return &scheduledTime, nil +} + +func buildBackup(vbs *v1alpha1.VolumeBackupSchedule, timestamp time.Time) *v1alpha1.VolumeBackup { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + backupSpec := *vbs.Spec.BackupTemplate.DeepCopy() + + bsLabel := util.CombineStringMap(label.NewBackupSchedule().Instance(bsName).BackupSchedule(bsName), vbs.Labels) + + if backupSpec.Template.BR == nil { + klog.Errorf("Information on BR missing in template") + return nil + } + + if backupSpec.Template.S3 != nil { + backupSpec.Template.S3.Prefix = path.Join(backupSpec.Template.S3.Prefix, "-"+timestamp.UTC().Format(pingcapv1alpha1.BackupNameTimeFormat)) + } else { + klog.Errorf("Information on S3 missing in template") + return nil + } + + if vbs.Spec.BackupTemplate.Template.ImagePullSecrets != nil { + backupSpec.Template.ImagePullSecrets = vbs.Spec.BackupTemplate.Template.ImagePullSecrets + } + backup := &v1alpha1.VolumeBackup{ + Spec: backupSpec, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: vbs.GetBackupCRDName(timestamp), + Labels: bsLabel, + Annotations: vbs.Annotations, + OwnerReferences: []metav1.OwnerReference{ + controller.GetFedVolumeBackupScheduleOwnerRef(vbs), + }, + }, + } + + klog.V(4).Infof("created backup is [%v], at time %v", backup, timestamp) + + return backup +} + +func createBackup(bkController controller.FedVolumeBackupControlInterface, vbs *v1alpha1.VolumeBackupSchedule, timestamp time.Time) (*v1alpha1.VolumeBackup, error) { + bk := buildBackup(vbs, timestamp) + if bk == nil { + return nil, controller.IgnoreErrorf("Invalid backup template for volume backup schedule [%s], BR or S3 information missing", vbs.GetName()) + } + + return bkController.CreateVolumeBackup(bk) +} + +func (bm *backupScheduleManager) canPerformNextBackup(vbs *v1alpha1.VolumeBackupSchedule) error { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + backup, err := bm.deps.VolumeBackupLister.VolumeBackups(ns).Get(vbs.Status.LastBackup) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return fmt.Errorf("backup schedule %s/%s, get backup %s failed, err: %v", ns, bsName, vbs.Status.LastBackup, err) + } + + if v1alpha1.IsVolumeBackupComplete(backup) || v1alpha1.IsVolumeBackupFailed(backup) { + return nil + } + // skip this sync round of the backup schedule and waiting the last backup. + return controller.RequeueErrorf("backup schedule %s/%s, the last backup %s is still running", ns, bsName, vbs.Status.LastBackup) +} + +func (bm *backupScheduleManager) backupGC(vbs *v1alpha1.VolumeBackupSchedule) { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + // if MaxBackups and MaxReservedTime are set at the same time, MaxReservedTime is preferred. + if vbs.Spec.MaxReservedTime != nil { + bm.backupGCByMaxReservedTime(vbs) + return + } + + if vbs.Spec.MaxBackups != nil && *vbs.Spec.MaxBackups > 0 { + bm.backupGCByMaxBackups(vbs) + return + } + klog.Warningf("backup schedule %s/%s does not set backup gc policy", ns, bsName) +} + +func (bm *backupScheduleManager) backupGCByMaxReservedTime(vbs *v1alpha1.VolumeBackupSchedule) { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + reservedTime, err := time.ParseDuration(*vbs.Spec.MaxReservedTime) + if err != nil { + klog.Errorf("backup schedule %s/%s, invalid MaxReservedTime %s", ns, bsName, *vbs.Spec.MaxReservedTime) + return + } + + backupsList, err := bm.getBackupList(vbs) + if err != nil { + klog.Errorf("backupGCByMaxReservedTime, err: %s", err) + return + } + + ascBackups := sortSnapshotBackups(backupsList) + if len(ascBackups) == 0 { + return + } + + var expiredBackups []*v1alpha1.VolumeBackup + + expiredBackups, err = calculateExpiredBackups(ascBackups, reservedTime) + if err != nil { + klog.Errorf("calculate expired backups without log backup, err: %s", err) + return + } + + for _, backup := range expiredBackups { + // delete the expired backup + if err = bm.deps.FedVolumeBackupControl.DeleteVolumeBackup(backup); err != nil { + klog.Errorf("backup schedule %s/%s gc backup %s failed, err %v", ns, bsName, backup.GetName(), err) + return + } + klog.Infof("backup schedule %s/%s gc backup %s success", ns, bsName, backup.GetName()) + } + + if len(expiredBackups) == len(backupsList) && len(expiredBackups) > 0 { + // All backups have been deleted, so the last backup information in the backupSchedule should be reset + bm.resetLastBackup(vbs) + } +} + +// sortSnapshotBackups return snapshot backups to be GCed order by create time asc +func sortSnapshotBackups(backupsList []*v1alpha1.VolumeBackup) []*v1alpha1.VolumeBackup { + var ascBackupList = make([]*v1alpha1.VolumeBackup, 0) + + for _, backup := range backupsList { + // Only try to GC Completed or Failed VolumeBackup + if !(v1alpha1.IsVolumeBackupFailed(backup) || v1alpha1.IsVolumeBackupComplete(backup)) { + continue + } + ascBackupList = append(ascBackupList, backup) + } + + sort.Slice(ascBackupList, func(i, j int) bool { + return ascBackupList[i].CreationTimestamp.Unix() < ascBackupList[j].CreationTimestamp.Unix() + }) + return ascBackupList +} + +// sortAllSnapshotBackups return all snapshot backups order by create time asc +// it's for test only now +func sortAllSnapshotBackups(backupsList []*v1alpha1.VolumeBackup) []*v1alpha1.VolumeBackup { + var ascBackupList = make([]*v1alpha1.VolumeBackup, 0) + ascBackupList = append(ascBackupList, backupsList...) + + sort.Slice(ascBackupList, func(i, j int) bool { + return ascBackupList[i].CreationTimestamp.Unix() < ascBackupList[j].CreationTimestamp.Unix() + }) + return ascBackupList +} + +func calculateExpiredBackups(backupsList []*v1alpha1.VolumeBackup, reservedTime time.Duration) ([]*v1alpha1.VolumeBackup, error) { + expiredTS := config.TSToTSO(time.Now().Add(-1 * reservedTime).Unix()) + i := 0 + for ; i < len(backupsList); i++ { + startTS, err := config.ParseTSString(backupsList[i].Status.CommitTs) + if err != nil { + return nil, perrors.Annotatef(err, "parse start tso: %s", backupsList[i].Status.CommitTs) + } + if startTS >= expiredTS { + break + } + } + return backupsList[:i], nil +} + +func (bm *backupScheduleManager) getBackupList(bs *v1alpha1.VolumeBackupSchedule) ([]*v1alpha1.VolumeBackup, error) { + ns := bs.GetNamespace() + bsName := bs.GetName() + + backupLabels := label.NewBackupSchedule().Instance(bsName).BackupSchedule(bsName) + selector, err := backupLabels.Selector() + if err != nil { + return nil, fmt.Errorf("generate backup schedule %s/%s label selector failed, err: %v", ns, bsName, err) + } + backupsList, err := bm.deps.VolumeBackupLister.VolumeBackups(ns).List(selector) + if err != nil { + return nil, fmt.Errorf("get backup schedule %s/%s backup list failed, selector: %s, err: %v", ns, bsName, selector, err) + } + + return backupsList, nil +} + +func (bm *backupScheduleManager) backupGCByMaxBackups(vbs *v1alpha1.VolumeBackupSchedule) { + ns := vbs.GetNamespace() + bsName := vbs.GetName() + + backupsList, err := bm.getBackupList(vbs) + if err != nil { + klog.Errorf("backupGCByMaxBackups failed, err: %s", err) + return + } + + sort.Sort(byCreateTimeDesc(backupsList)) + var deleteCount int + for i, backup := range backupsList { + if i < int(*vbs.Spec.MaxBackups) { + continue + } + // delete the backup + if err := bm.deps.FedVolumeBackupControl.DeleteVolumeBackup(backup); err != nil { + klog.Errorf("backup schedule %s/%s gc backup %s failed, err %v", ns, bsName, backup.GetName(), err) + return + } + deleteCount += 1 + klog.Infof("backup schedule %s/%s gc backup %s success", ns, bsName, backup.GetName()) + } + + if deleteCount == len(backupsList) && deleteCount > 0 { + // All backups have been deleted, so the last backup information in the backupSchedule should be reset + bm.resetLastBackup(vbs) + } +} + +func (bm *backupScheduleManager) resetLastBackup(vbs *v1alpha1.VolumeBackupSchedule) { + vbs.Status.LastBackupTime = nil + vbs.Status.LastBackup = "" + vbs.Status.AllBackupCleanTime = &metav1.Time{Time: bm.now()} +} + +type byCreateTimeDesc []*v1alpha1.VolumeBackup + +func (b byCreateTimeDesc) Len() int { return len(b) } +func (b byCreateTimeDesc) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byCreateTimeDesc) Less(i, j int) bool { + return b[j].ObjectMeta.CreationTimestamp.Before(&b[i].ObjectMeta.CreationTimestamp) +} + var _ fedvolumebackup.BackupScheduleManager = &backupScheduleManager{} type FakeBackupScheduleManager struct { @@ -61,8 +394,15 @@ func (m *FakeBackupScheduleManager) SetSyncError(err error) { m.err = err } -func (m *FakeBackupScheduleManager) Sync(_ *v1alpha1.VolumeBackupSchedule) error { - return m.err +func (m *FakeBackupScheduleManager) Sync(vbs *v1alpha1.VolumeBackupSchedule) error { + if m.err != nil { + return m.err + } + if vbs.Status.LastBackupTime != nil { + // simulate status update + vbs.Status.LastBackupTime = &metav1.Time{Time: vbs.Status.LastBackupTime.Add(1 * time.Hour)} + } + return nil } var _ fedvolumebackup.BackupScheduleManager = &FakeBackupScheduleManager{} diff --git a/pkg/fedvolumebackup/backupschedule/backup_schedule_manager_test.go b/pkg/fedvolumebackup/backupschedule/backup_schedule_manager_test.go new file mode 100644 index 0000000000..6ab06eb1b5 --- /dev/null +++ b/pkg/fedvolumebackup/backupschedule/backup_schedule_manager_test.go @@ -0,0 +1,402 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package backupschedule + +import ( + "context" + "fmt" + "strconv" + "testing" + "time" + + pingcapv1alpha1 "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + + "github.com/google/go-cmp/cmp" + . "github.com/onsi/gomega" + "github.com/pingcap/tidb-operator/pkg/apis/federation/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/apis/label" + "github.com/pingcap/tidb-operator/pkg/controller" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/pointer" +) + +func TestManager(t *testing.T) { + g := NewGomegaWithT(t) + helper := newHelper(t) + defer helper.close() + deps := helper.deps + m := NewBackupScheduleManager(deps).(*backupScheduleManager) + var err error + bs := &v1alpha1.VolumeBackupSchedule{} + bs.Namespace = "ns" + bs.Name = "bsname" + bs.Spec.BackupTemplate.Template.BR = &v1alpha1.BRConfig{} + bs.Spec.BackupTemplate.Template.S3 = &pingcapv1alpha1.S3StorageProvider{} + + // test pause + bs.Spec.Pause = true + err = m.Sync(bs) + g.Expect(err).Should(BeAssignableToTypeOf(&controller.IgnoreError{})) + g.Expect(err.Error()).Should(MatchRegexp(".*has been paused.*")) + + // test canPerformNextBackup + // + // test not found last backup + bs.Spec.Pause = false + bs.Status.LastBackup = "backupname" + err = m.canPerformNextBackup(bs) + g.Expect(err).Should(BeNil()) + + // test backup complete + bk := &v1alpha1.VolumeBackup{} + bk.Namespace = bs.Namespace + bk.Name = bs.Status.LastBackup + bk.Status.Conditions = append(bk.Status.Conditions, v1alpha1.VolumeBackupCondition{ + Type: v1alpha1.VolumeBackupComplete, + Status: v1.ConditionTrue, + }) + helper.createBackup(bk) + err = m.canPerformNextBackup(bs) + g.Expect(err).Should(BeNil()) + helper.deleteBackup(bk) + + // test last backup failed state + bk.Status.Conditions = nil + bk.Status.Conditions = append(bk.Status.Conditions, v1alpha1.VolumeBackupCondition{ + Type: v1alpha1.VolumeBackupFailed, + Status: v1.ConditionTrue, + }) + helper.createBackup(bk) + err = m.canPerformNextBackup(bs) + g.Expect(err).Should(BeNil()) + helper.deleteBackup(bk) + + t.Log("start test normal Sync") + bk.Status.Conditions = nil + bs.Spec.Schedule = "0 0 * * *" // Run at midnight every day + + now := time.Now() + m.now = func() time.Time { return now.AddDate(0, 0, -101) } + m.resetLastBackup(bs) + // 10 backup, one per day + for i := -9; i <= 0; i++ { + t.Log("loop id ", i) + m.now = func() time.Time { return now.AddDate(0, 0, i) } + err = m.Sync(bs) + g.Expect(err).Should(BeNil()) + bks := helper.checkBacklist(bs.Namespace, i+10) + // complete the backup created + for i := range bks.Items { + bk := bks.Items[i].DeepCopy() + changed := !v1alpha1.IsVolumeBackupComplete(bk) + v1alpha1.UpdateVolumeBackupCondition(&bk.Status, &v1alpha1.VolumeBackupCondition{ + Type: v1alpha1.VolumeBackupComplete, + Status: v1.ConditionTrue, + }) + + if changed { + bk.CreationTimestamp = metav1.Time{Time: m.now()} + bk.Status.CommitTs = getTSOStr(m.now().Add(10 * time.Minute).Unix()) + t.Log("complete backup: ", bk.Name) + helper.updateBackup(bk) + } + + g.Expect(err).Should(BeNil()) + } + } + + t.Log("test setting MaxBackups") + m.now = time.Now + bs.Spec.MaxBackups = pointer.Int32Ptr(5) + err = m.Sync(bs) + g.Expect(err).Should(BeNil()) + helper.checkBacklist(bs.Namespace, 5) + + t.Log("test setting MaxReservedTime") + bs.Spec.MaxBackups = nil + bs.Spec.MaxReservedTime = pointer.StringPtr("71h") + err = m.Sync(bs) + g.Expect(err).Should(BeNil()) + helper.checkBacklist(bs.Namespace, 3) +} + +func TestGetLastScheduledTime(t *testing.T) { + g := NewGomegaWithT(t) + + bs := &v1alpha1.VolumeBackupSchedule{ + Spec: v1alpha1.VolumeBackupScheduleSpec{}, + Status: v1alpha1.VolumeBackupScheduleStatus{ + LastBackupTime: &metav1.Time{}, + }, + } + var getTime *time.Time + var err error + + // test invalid format schedule + bs.Spec.Schedule = "#$#$#$@" + _, err = getLastScheduledTime(bs, time.Now) + g.Expect(err).ShouldNot(BeNil()) + + bs.Spec.Schedule = "0 0 * * *" // Run once a day at midnight + now := time.Now() + + // test last backup time after now + bs.Status.LastBackupTime.Time = now.AddDate(0, 0, 1) + getTime, err = getLastScheduledTime(bs, time.Now) + g.Expect(err).Should(BeNil()) + g.Expect(getTime).Should(BeNil()) + + // test scheduled + for i := 0; i < 10; i++ { + bs.Status.LastBackupTime.Time = now.AddDate(0, 0, -i-1) + getTime, err = getLastScheduledTime(bs, time.Now) + g.Expect(err).Should(BeNil()) + g.Expect(getTime).ShouldNot(BeNil()) + expectTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local) + g.Expect(*getTime).Should(Equal(expectTime)) + } + + // test too many miss + bs.Status.LastBackupTime.Time = now.AddDate(-1000, 0, 0) + getTime, err = getLastScheduledTime(bs, time.Now) + g.Expect(err).Should(BeNil()) + g.Expect(getTime).Should(BeNil()) +} + +func TestBuildBackup(t *testing.T) { + now := time.Now() + var get *v1alpha1.VolumeBackup + + // build BackupSchedule template + bs := &v1alpha1.VolumeBackupSchedule{ + Spec: v1alpha1.VolumeBackupScheduleSpec{}, + Status: v1alpha1.VolumeBackupScheduleStatus{ + LastBackupTime: &metav1.Time{}, + }, + } + bs.Namespace = "ns" + bs.Name = "bsname" + + // build Backup template + bk := &v1alpha1.VolumeBackup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: bs.Namespace, + Name: bs.GetBackupCRDName(now), + Labels: label.NewBackupSchedule().Instance(bs.Name).BackupSchedule(bs.Name).Labels(), + OwnerReferences: []metav1.OwnerReference{ + controller.GetFedVolumeBackupScheduleOwnerRef(bs), + }, + }, + Spec: v1alpha1.VolumeBackupSpec{}, + } + + // test BR != nil + bs.Spec.BackupTemplate.Template.BR = &v1alpha1.BRConfig{} + bs.Spec.BackupTemplate.Template.S3 = &pingcapv1alpha1.S3StorageProvider{} + + bk.Spec.Template.BR = bs.Spec.BackupTemplate.Template.BR.DeepCopy() + bk.Spec.Template.S3 = bs.Spec.BackupTemplate.Template.S3.DeepCopy() + get = buildBackup(bs, now) + // have to reset the dynamic prefix + bk.Spec.Template.S3.Prefix = get.Spec.Template.S3.Prefix + if diff := cmp.Diff(bk, get); diff != "" { + t.Errorf("unexpected (-want, +got): %s", diff) + } +} + +func TestCalculateExpiredBackups(t *testing.T) { + g := NewGomegaWithT(t) + type testCase struct { + backups []*v1alpha1.VolumeBackup + reservedTime time.Duration + expectedDeleteBackupCount int + } + + var ( + now = time.Now() + last10Min = now.Add(-time.Minute * 10).Unix() + last1Day = now.Add(-time.Hour * 24 * 1).Unix() + last2Day = now.Add(-time.Hour * 24 * 2).Unix() + last3Day = now.Add(-time.Hour * 24 * 3).Unix() + ) + + testCases := []*testCase{ + // no backup should be deleted + { + backups: []*v1alpha1.VolumeBackup{ + fakeBackup(&last10Min), + }, + reservedTime: 24 * time.Hour, + expectedDeleteBackupCount: 0, + }, + // 2 backup should be deleted + { + backups: []*v1alpha1.VolumeBackup{ + fakeBackup(&last3Day), + fakeBackup(&last2Day), + fakeBackup(&last1Day), + fakeBackup(&last10Min), + }, + reservedTime: 24 * time.Hour, + expectedDeleteBackupCount: 2, + }, + } + + for _, tc := range testCases { + deletedBackups, err := calculateExpiredBackups(tc.backups, tc.reservedTime) + g.Expect(err).Should(BeNil()) + g.Expect(len(deletedBackups)).Should(Equal(tc.expectedDeleteBackupCount)) + } +} + +type helper struct { + t *testing.T + deps *controller.BrFedDependencies + stop chan struct{} +} + +func newHelper(t *testing.T) *helper { + deps := controller.NewSimpleFedClientDependencies() + stop := make(chan struct{}) + deps.InformerFactory.Start(stop) + deps.KubeInformerFactory.Start(stop) + deps.InformerFactory.WaitForCacheSync(stop) + deps.KubeInformerFactory.WaitForCacheSync(stop) + + return &helper{ + t: t, + deps: deps, + stop: stop, + } +} + +func (h *helper) close() { + close(h.stop) +} + +// check for exists num Backup and return the exists backups "BackupList". +func (h *helper) checkBacklist(ns string, num int) (bks *v1alpha1.VolumeBackupList) { + t := h.t + deps := h.deps + g := NewGomegaWithT(t) + + check := func(backups []*v1alpha1.VolumeBackup) error { + snapshotBackups := sortAllSnapshotBackups(backups) + // check snapshot backup num + if len(snapshotBackups) != num { + var names []string + for _, bk := range snapshotBackups { + names = append(names, bk.Name) + } + return fmt.Errorf("there %d backup, but should be %d, cur backups: %v", len(snapshotBackups), num, names) + } + // check truncateTSO, it should equal the earliest snapshot backup after gc + if len(snapshotBackups) == 0 { + return fmt.Errorf("there should have snapshot backup if need check log backup truncate tso") + } + return nil + } + + t.Helper() + g.Eventually(func() error { + var err error + bks, err = deps.Clientset.FederationV1alpha1().VolumeBackups(ns).List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).Should(BeNil()) + backups := convertToBackupPtrList(bks.Items) + return check(backups) + }, time.Second*30).Should(BeNil()) + + g.Eventually(func() error { + var err error + backups, err := deps.VolumeBackupLister.VolumeBackups(ns).List(labels.Everything()) + g.Expect(err).Should(BeNil()) + return check(backups) + }, time.Second*30).Should(BeNil()) + + return +} + +func convertToBackupPtrList(backups []v1alpha1.VolumeBackup) []*v1alpha1.VolumeBackup { + backupPtrs := make([]*v1alpha1.VolumeBackup, 0) + for i := 0; i < len(backups); i++ { + backupPtrs = append(backupPtrs, &backups[i]) + } + return backupPtrs +} + +func (h *helper) createBackup(bk *v1alpha1.VolumeBackup) { + t := h.t + deps := h.deps + g := NewGomegaWithT(t) + _, err := deps.Clientset.FederationV1alpha1().VolumeBackups(bk.Namespace).Create(context.TODO(), bk, metav1.CreateOptions{}) + g.Expect(err).Should(BeNil()) + g.Eventually(func() error { + _, err := deps.VolumeBackupLister.VolumeBackups(bk.Namespace).Get(bk.Name) + return err + }, time.Second*10).Should(BeNil()) +} + +func (h *helper) deleteBackup(bk *v1alpha1.VolumeBackup) { + t := h.t + deps := h.deps + g := NewGomegaWithT(t) + err := deps.Clientset.FederationV1alpha1().VolumeBackups(bk.Namespace).Delete(context.TODO(), bk.Name, metav1.DeleteOptions{}) + g.Expect(err).Should(BeNil()) + g.Eventually(func() error { + _, err := deps.VolumeBackupLister.VolumeBackups(bk.Namespace).Get(bk.Name) + return err + }, time.Second*10).ShouldNot(BeNil()) +} + +func (h *helper) updateBackup(bk *v1alpha1.VolumeBackup) { + t := h.t + deps := h.deps + g := NewGomegaWithT(t) + _, err := deps.Clientset.FederationV1alpha1().VolumeBackups(bk.Namespace).Update(context.TODO(), bk, metav1.UpdateOptions{}) + g.Expect(err).Should(BeNil()) + + g.Eventually(func() error { + get, err := deps.VolumeBackupLister.VolumeBackups(bk.Namespace).Get(bk.Name) + if err != nil { + return err + } + + diff := cmp.Diff(get, bk) + if diff == "" { + return nil + } + + return fmt.Errorf("not synced yet: %s", diff) + }, time.Second*10).Should(BeNil()) +} + +func fakeBackup(ts *int64) *v1alpha1.VolumeBackup { + backup := &v1alpha1.VolumeBackup{} + if ts == nil { + return backup + } + backup.Status.CommitTs = getTSOStr(*ts) + return backup +} + +func getTSOStr(ts int64) string { + tso := getTSO(ts) + return strconv.FormatUint(tso, 10) +} + +func getTSO(ts int64) uint64 { + return uint64((ts << 18) * 1000) +}