From 36053770348984996031913ab9804956f89d7e88 Mon Sep 17 00:00:00 2001 From: huangfumingyue <41893849+huangfumingyue@users.noreply.github.com> Date: Fri, 28 May 2021 15:03:56 +0900 Subject: [PATCH] Get the correct increment backup (#176) * get the correct increment backup Before, we actually got the difference backup, not the incremental backup. TO solve this probem ,revert #125. However, if a physical backup file is deleted(rm command, etc.) other than the pg_rman delete command of pg_rman, It cannot find the correct status of a full backup and incremental backup.(please refer to #154 to details) WE will solve this problem, at the next major upgarde. * Revert "get the correct increment backup" * get the correct increment backup Before, we actually got the difference backup, not the incremental backup. TO solve this probem ,revert #125. However, if a physical backup file is deleted(rm command, etc.) other than the pg_rman delete command of pg_rman, It cannot find the correct status of a full backup and incremental backup.(please refer to #154 to details) WE will solve this problem, at the next major upgarde. * add the regression test of increment backup * add the regression test of increment backup * add the comment add the comment about the increment backup's problem * Add a comment Add a comment * add the regression test of increment backup * add the regression test of increment backup add the regression test of increment backup * add the comment * add the regression test * delete file becaust upload the wrong place,delete it * add the regression test * Reflected the comment * Reflected the comment * Reflected the comment * Reflected the comment * Reflected the comment * Reflected the comment * Reflected the comment * Regression test command was modified * Regression test command was modified * Regression test command was modified --- backup.c | 8 ++++++-- catalog.c | 7 +++---- expected/backup.out | 16 ++++++++++++---- pg_rman.h | 2 +- sql/backup.sh | 25 ++++++++++++++++++++----- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/backup.c b/backup.c index 9b6db171..9e3bd0c0 100644 --- a/backup.c +++ b/backup.c @@ -103,7 +103,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt) pgBackup *prev_backup; /* find last completed database backup */ - prev_backup = catalog_get_last_full_backup(backup_list); + prev_backup = catalog_get_last_data_backup(backup_list); if (prev_backup == NULL) { if (current.full_backup_on_error) @@ -185,6 +185,10 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt) /* * To take incremental backup, the file list of the latest validated * full database backup is needed. + * TODO: fix for issue #154 + * When a backup list is deleted with rm command or pg_rman's delete command with '--force' option, + * pg_rman can't detect there is a missing piece of backup. + * We need the way tracing the backup chains or something else... */ if (current.backup_mode < BACKUP_MODE_FULL) { @@ -192,7 +196,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt) uint32 xlogid, xrecoff; /* find last completed database backup */ - prev_backup = catalog_get_last_full_backup(backup_list); + prev_backup = catalog_get_last_data_backup(backup_list); if (prev_backup == NULL || prev_backup->tli != current.tli) { if (current.full_backup_on_error) diff --git a/catalog.c b/catalog.c index 2648b094..80749c6e 100644 --- a/catalog.c +++ b/catalog.c @@ -266,7 +266,7 @@ catalog_get_backup_list(const pgBackupRange *range) * Find the last completed database full valid backup from the backup list. */ pgBackup * -catalog_get_last_full_backup(parray *backup_list) +catalog_get_last_data_backup(parray *backup_list) { int i; pgBackup *backup = NULL; @@ -276,9 +276,8 @@ catalog_get_last_full_backup(parray *backup_list) { backup = (pgBackup *) parray_get(backup_list, i); - /* Return the first full valid backup. */ - if (backup->backup_mode == BACKUP_MODE_FULL && - backup->status == BACKUP_STATUS_OK) + /* we need completed database backup */ + if (backup -> status == BACKUP_STATUS_OK && HAVE_DATABASE(backup)) return backup; } diff --git a/expected/backup.out b/expected/backup.out index 1734572a..67c9692d 100644 --- a/expected/backup.out +++ b/expected/backup.out @@ -16,15 +16,23 @@ 0 1 ###### BACKUP COMMAND TEST-0005 ###### +###### Make sure that pg_rman does not take a differential backup, but a incremental backup ###### +0 +CREATE TABLE +INSERT 0 1000000 +0 +0 +INCR 16kB OK +###### BACKUP COMMAND TEST-0006 ###### ###### full backup with compression ###### 0 1 1 -###### BACKUP COMMAND TEST-0006 ###### +###### BACKUP COMMAND TEST-0007 ###### ###### full backup with smooth checkpoint ###### 0 1 -###### BACKUP COMMAND TEST-0007 ###### +###### BACKUP COMMAND TEST-0008 ###### ###### switch backup mode from incremental to full ###### incremental backup without validated full backup INFO: copying database files @@ -43,7 +51,7 @@ INFO: Please execute 'pg_rman validate' to verify the files are correctly copied 0 1 1 -###### BACKUP COMMAND TEST-0008 ###### +###### BACKUP COMMAND TEST-0009 ###### ###### switch backup mode from archive to full ###### archive backup without validated full backup ERROR: cannot take an incremental backup @@ -61,7 +69,7 @@ INFO: Please execute 'pg_rman validate' to verify the files are correctly copied 0 1 1 -###### BACKUP COMMAND TEST-0009 ###### +###### BACKUP COMMAND TEST-0010 ###### ###### failure in backup with different system identifier database ###### ERROR: could not start backup DETAIL: system identifier of target database is different from the one of initially configured database diff --git a/pg_rman.h b/pg_rman.h index 1394f634..63e1e46e 100644 --- a/pg_rman.h +++ b/pg_rman.h @@ -285,7 +285,7 @@ extern void pgBackupValidate(pgBackup *backup, bool size_only, bool for_get_time /* in catalog.c */ extern pgBackup *catalog_get_backup(time_t timestamp); extern parray *catalog_get_backup_list(const pgBackupRange *range); -extern pgBackup *catalog_get_last_full_backup(parray *backup_list); +extern pgBackup *catalog_get_last_data_backup(parray *backup_list); extern pgBackup *catalog_get_last_arclog_backup(parray *backup_list); extern pgBackup *catalog_get_last_srvlog_backup(parray *backup_list); diff --git a/sql/backup.sh b/sql/backup.sh index 0eeb62d0..85430ce7 100644 --- a/sql/backup.sh +++ b/sql/backup.sh @@ -118,7 +118,6 @@ pg_rman validate -B ${BACKUP_PATH} --quiet pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0001.log 2>&1 grep -c OK ${TEST_BASE}/TEST-0001.log - echo '###### BACKUP COMMAND TEST-0002 ######' echo '###### incremental backup mode ######' pg_rman backup -B ${BACKUP_PATH} -b incremental -p ${TEST_PGPORT} -d postgres --quiet;echo $? @@ -141,7 +140,23 @@ pg_rman validate -B ${BACKUP_PATH} --quiet pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0004.log 2>&1 grep -c OK ${TEST_BASE}/TEST-0004.log +# If a differential backup was taken, the size of third backup is same as the second one and the size is bigger (XXMB). +# 16kB is the base backup size if nothing was changed in the database cluster. echo '###### BACKUP COMMAND TEST-0005 ######' +echo '###### Make sure that pg_rman does not take a differential backup, but a incremental backup ######' +init_catalog +pg_rman backup -B ${BACKUP_PATH} -b full -p ${TEST_PGPORT} -d postgres --quiet;echo $? +pg_rman validate -B ${BACKUP_PATH} --quiet +psql -p ${TEST_PGPORT} -d postgres -c 'create table test (c1 int);' +psql -p ${TEST_PGPORT} -d postgres -c 'insert into test values(generate_series(1,1000000));' +pg_rman backup -B ${BACKUP_PATH} -b incremental -p ${TEST_PGPORT} -d postgres --quiet;echo $? +pg_rman validate -B ${BACKUP_PATH} --quiet +pg_rman backup -B ${BACKUP_PATH} -b incremental -p ${TEST_PGPORT} -d postgres --quiet;echo $? +pg_rman validate -B ${BACKUP_PATH} --quiet +pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0012.log 2>&1 +cat ${TEST_BASE}/TEST-0012.log | head -n 4 | tail -n 1 | awk '{print $5, $6, $13}' + +echo '###### BACKUP COMMAND TEST-0006 ######' echo '###### full backup with compression ######' init_catalog pg_rman backup -B ${BACKUP_PATH} -b full -s -Z -p ${TEST_PGPORT} -d postgres --quiet;echo $? @@ -150,7 +165,7 @@ pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0005.log 2>&1 grep -c OK ${TEST_BASE}/TEST-0005.log grep OK ${TEST_BASE}/TEST-0005.log | grep -c true -echo '###### BACKUP COMMAND TEST-0006 ######' +echo '###### BACKUP COMMAND TEST-0007 ######' echo '###### full backup with smooth checkpoint ######' init_catalog pg_rman backup -B ${BACKUP_PATH} -b full -s -C -p ${TEST_PGPORT} -d postgres --quiet;echo $? @@ -158,7 +173,7 @@ pg_rman validate -B ${BACKUP_PATH} --quiet pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0006.log 2>&1 grep -c OK ${TEST_BASE}/TEST-0006.log -echo '###### BACKUP COMMAND TEST-0007 ######' +echo '###### BACKUP COMMAND TEST-0008 ######' echo '###### switch backup mode from incremental to full ######' init_catalog echo 'incremental backup without validated full backup' @@ -171,7 +186,7 @@ pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0010.log 2>&1 grep OK ${TEST_BASE}/TEST-0010.log | grep FULL | wc -l grep ERROR ${TEST_BASE}/TEST-0010.log | grep INCR | wc -l -echo '###### BACKUP COMMAND TEST-0008 ######' +echo '###### BACKUP COMMAND TEST-0009 ######' echo '###### switch backup mode from archive to full ######' init_catalog echo 'archive backup without validated full backup' @@ -184,7 +199,7 @@ pg_rman show detail -B ${BACKUP_PATH} > ${TEST_BASE}/TEST-0011.log 2>&1 grep OK ${TEST_BASE}/TEST-0011.log | grep FULL | wc -l grep ERROR ${TEST_BASE}/TEST-0011.log | grep ARCH | wc -l -echo '###### BACKUP COMMAND TEST-0009 ######' +echo '###### BACKUP COMMAND TEST-0010 ######' echo '###### failure in backup with different system identifier database ######' init_catalog pg_ctl stop -m immediate > /dev/null 2>&1