br: add warmup process to volume snapshot restore (#5229) (#5253)

docs/api-references/docs.md

@@ -1485,6 +1485,32 @@ For BR image, if it does not contain tag, Pod will use image ‘ToolImage:${
 </tr>
 <tr>
 <td>
+<code>warmup</code></br>
+<em>
+<a href="#restorewarmupmode">
+RestoreWarmupMode
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Warmup represents whether to initialize TiKV volumes after volume snapshot restore</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>warmupImage</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>WarmupImage represents using what image to initialize TiKV volumes</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>podSecurityContext</code></br>
 <em>
 <a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#podsecuritycontext-v1-core">
@@ -13959,6 +13985,32 @@ For BR image, if it does not contain tag, Pod will use image &lsquo;ToolImage:${
 </tr>
 <tr>
 <td>
+<code>warmup</code></br>
+<em>
+<a href="#restorewarmupmode">
+RestoreWarmupMode
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Warmup represents whether to initialize TiKV volumes after volume snapshot restore</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>warmupImage</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>WarmupImage represents using what image to initialize TiKV volumes</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>podSecurityContext</code></br>
 <em>
 <a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#podsecuritycontext-v1-core">
@@ -14088,6 +14140,14 @@ RestoreConditionType
 </tr>
 </tbody>
 </table>
+<h3 id="restorewarmupmode">RestoreWarmupMode</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#restorespec">RestoreSpec</a>)
+</p>
+<p>
+<p>RestoreWarmupMode represents when to initialize TiKV volumes</p>
+</p>
 <h3 id="s3storageprovider">S3StorageProvider</h3>
 <p>
 (<em>Appears on:</em>

docs/api-references/federation-docs.md

@@ -1459,6 +1459,30 @@ string
 <p>PriorityClassName of Restore Job Pods</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>warmup</code></br>
+<em>
+github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.RestoreWarmupMode
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Warmup represents whether to initialize TiKV volumes after volume snapshot restore</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>warmupImage</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>WarmupImage represents using what image to initialize TiKV volumes</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="volumerestorememberstatus">VolumeRestoreMemberStatus</h3>

manifests/crd.yaml

@@ -14339,6 +14339,10 @@ spec:
                 type: boolean
               volumeAZ:
                 type: string
+              warmup:
+                type: string
+              warmupImage:
+                type: string
             type: object
           status:
             properties:

manifests/crd/federation/v1/federation.pingcap.com_volumerestores.yaml

@@ -945,6 +945,10 @@ spec:
                     type: array
                   toolImage:
                     type: string
+                  warmup:
+                    type: string
+                  warmupImage:
+                    type: string
                 type: object
             type: object
           status:

manifests/crd/v1/pingcap.com_restores.yaml

@@ -2087,6 +2087,10 @@ spec:
                 type: boolean
               volumeAZ:
                 type: string
+              warmup:
+                type: string
+              warmupImage:
+                type: string
             type: object
           status:
             properties:

manifests/federation-crd.yaml

@@ -2981,6 +2981,10 @@ spec:
                     type: array
                   toolImage:
                     type: string
+                  warmup:
+                    type: string
+                  warmupImage:
+                    type: string
                 type: object
             type: object
           status:

pkg/apis/federation/pingcap/v1alpha1/types.go

@@ -361,6 +361,12 @@ type VolumeRestoreMemberSpec struct {
 	ServiceAccount string `json:"serviceAccount,omitempty"`
 	// PriorityClassName of Restore Job Pods
 	PriorityClassName string `json:"priorityClassName,omitempty"`
+	// Warmup represents whether to initialize TiKV volumes after volume snapshot restore
+	// +optional
+	Warmup pingcapv1alpha1.RestoreWarmupMode `json:"warmup,omitempty"`
+	// WarmupImage represents using what image to initialize TiKV volumes
+	// +optional
+	WarmupImage string `json:"warmupImage,omitempty"`
 }
 
 type VolumeRestoreMemberBackupInfo struct {
@@ -441,6 +447,11 @@ const (
 	VolumeRestoreRunning VolumeRestoreConditionType = "Running"
 	// VolumeRestoreVolumeComplete means all the restore members are volume complete
 	VolumeRestoreVolumeComplete VolumeRestoreConditionType = "VolumeComplete"
+	// VolumeRestoreWarmUpStarted means the Restore has successfully started warmup pods to
+	// initialize volumes restored from snapshots
+	VolumeRestoreWarmUpStarted VolumeRestoreConditionType = "WarmUpStarted"
+	// VolumeRestoreWarmUpComplete means the Restore has successfully warmed up all TiKV volumes
+	VolumeRestoreWarmUpComplete VolumeRestoreConditionType = "WarmUpComplete"
 	// VolumeRestoreTiKVComplete means all the restore members are tikv complete
 	VolumeRestoreTiKVComplete VolumeRestoreConditionType = "TikvComplete"
 	// VolumeRestoreDataComplete means all the restore members are data complete
@@ -460,6 +471,8 @@ const (
 	VolumeRestoreStepRestoreVolume VolumeRestoreStepType = "RestoreVolume"
 	// VolumeRestoreStepStartTiKV is start tikv step
 	VolumeRestoreStepStartTiKV VolumeRestoreStepType = "StartTiKV"
+	// VolumeRestoreStepWarmUp is warm up tikv volumes step
+	VolumeRestoreStepWarmUp VolumeRestoreStepType = "WarmUp"
 	// VolumeRestoreStepRestoreData is restore data step
 	VolumeRestoreStepRestoreData VolumeRestoreStepType = "RestoreData"
 	// VolumeRestoreStepRestartTiKV is restart tikv step

pkg/apis/federation/pingcap/v1alpha1/volume_restore.go

@@ -31,6 +31,16 @@ func IsVolumeRestoreVolumeComplete(volumeRestore *VolumeRestore) bool {
 	return condition != nil && condition.Status == corev1.ConditionTrue
 }
 
+func IsVolumeRestoreWarmUpStarted(volumeRestore *VolumeRestore) bool {
+	_, condition := GetVolumeRestoreCondition(&volumeRestore.Status, VolumeRestoreWarmUpStarted)
+	return condition != nil && condition.Status == corev1.ConditionTrue
+}
+
+func IsVolumeRestoreWarmUpComplete(volumeRestore *VolumeRestore) bool {
+	_, condition := GetVolumeRestoreCondition(&volumeRestore.Status, VolumeRestoreWarmUpComplete)
+	return condition != nil && condition.Status == corev1.ConditionTrue
+}
+
 func IsVolumeRestoreTiKVComplete(volumeRestore *VolumeRestore) bool {
 	_, condition := GetVolumeRestoreCondition(&volumeRestore.Status, VolumeRestoreTiKVComplete)
 	return condition != nil && condition.Status == corev1.ConditionTrue

pkg/apis/label/label.go

@@ -58,6 +58,8 @@ const (
 
 	// RestoreLabelKey is restore key
 	RestoreLabelKey string = "tidb.pingcap.com/restore"
+	// RestoreWarmUpLabelKey defines which pod the restore warms up
+	RestoreWarmUpLabelKey string = "tidb.pingcap.com/warm-up-pod"
 
 	// BackupProtectionFinalizer is the name of finalizer on backups or federation backups
 	BackupProtectionFinalizer string = "tidb.pingcap.com/backup-protection"
@@ -168,6 +170,8 @@ const (
 	CleanJobLabelVal string = "clean"
 	// RestoreJobLabelVal is restore job label value
 	RestoreJobLabelVal string = "restore"
+	// RestoreWarmUpJobLabelVal is restore warmup job label value
+	RestoreWarmUpJobLabelVal string = "warmup"
 	// BackupJobLabelVal is backup job label value
 	BackupJobLabelVal string = "backup"
 	// BackupScheduleJobLabelVal is backup schedule job label value
@@ -352,6 +356,10 @@ func (l Label) RestoreJob() Label {
 	return l.Component(RestoreJobLabelVal)
 }
 
+func (l Label) RestoreWarmUpJob() Label {
+	return l.Component(RestoreWarmUpJobLabelVal)
+}
+
 // Backup assigns specific value to backup key in label
 func (l Label) Backup(val string) Label {
 	l[BackupLabelKey] = val

pkg/apis/pingcap/v1alpha1/restore.go

@@ -130,6 +130,18 @@ func IsRestoreVolumeComplete(restore *Restore) bool {
 	return condition != nil && condition.Status == corev1.ConditionTrue
 }
 
+// IsRestoreWarmUpStarted returns true if all the warmup jobs has successfully started
+func IsRestoreWarmUpStarted(restore *Restore) bool {
+	_, condition := GetRestoreCondition(&restore.Status, RestoreWarmUpStarted)
+	return condition != nil && condition.Status == corev1.ConditionTrue
+}
+
+// IsRestoreWarmUpComplete returns true if all the warmup jobs has successfully finished
+func IsRestoreWarmUpComplete(restore *Restore) bool {
+	_, condition := GetRestoreCondition(&restore.Status, RestoreWarmUpComplete)
+	return condition != nil && condition.Status == corev1.ConditionTrue
+}
+
 // IsRestoreTiKVComplete returns true if all TiKVs run successfully during volume restore
 func IsRestoreTiKVComplete(restore *Restore) bool {
 	_, condition := GetRestoreCondition(&restore.Status, RestoreTiKVComplete)

pkg/apis/pingcap/v1alpha1/types.go

@@ -2340,6 +2340,11 @@ const (
 	// RestoreVolumeComplete means the Restore has successfully executed part-1 and the
 	// backup volumes have been rebuilded from the corresponding snapshot
 	RestoreVolumeComplete RestoreConditionType = "VolumeComplete"
+	// RestoreWarmUpStarted means the Restore has successfully started warm up pods to
+	// initialize volumes restored from snapshots
+	RestoreWarmUpStarted RestoreConditionType = "WarmUpStarted"
+	// RestoreWarmUpComplete means the Restore has successfully warmed up all TiKV volumes
+	RestoreWarmUpComplete RestoreConditionType = "WarmUpComplete"
 	// RestoreDataComplete means the Restore has successfully executed part-2 and the
 	// data in restore volumes has been deal with consistency based on min_resolved_ts
 	RestoreDataComplete RestoreConditionType = "DataComplete"
@@ -2444,6 +2449,12 @@ type RestoreSpec struct {
 	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
 	// TableFilter means Table filter expression for 'db.table' matching. BR supports this from v4.0.3.
 	TableFilter []string `json:"tableFilter,omitempty"`
+	// Warmup represents whether to initialize TiKV volumes after volume snapshot restore
+	// +optional
+	Warmup RestoreWarmupMode `json:"warmup,omitempty"`
+	// WarmupImage represents using what image to initialize TiKV volumes
+	// +optional
+	WarmupImage string `json:"warmupImage,omitempty"`
 
 	// PodSecurityContext of the component
 	// +optional
@@ -2465,6 +2476,16 @@ const (
 	FederalVolumeRestoreFinish FederalVolumeRestorePhase = "restore-finish"
 )
 
+// RestoreWarmupMode represents when to initialize TiKV volumes
+type RestoreWarmupMode string
+
+const (
+	// RestoreWarmupModeSync means initialize TiKV volumes before TiKV starts
+	RestoreWarmupModeSync RestoreWarmupMode = "sync"
+	// RestoreWarmupModeASync means initialize TiKV volumes after restore complete
+	RestoreWarmupModeASync RestoreWarmupMode = "async"
+)
+
 // RestoreStatus represents the current status of a tidb cluster restore.
 type RestoreStatus struct {
 	// TimeStarted is the time at which the restore was started.