diff --git a/e2e-tests/functions b/e2e-tests/functions index e32ac221d7..0c16637126 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -759,3 +759,17 @@ get_container_image() { echo "${IMAGE_BASE}:${operatorVersion}-ppg${pgVersion}-${component}" } + +get_postgresql_logs() { + local pgVersion=$1 + + for pod in $(kubectl get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + local phase=$(kubectl -n ${NAMESPACE} get ${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "find /pgdata/pg${pgVersion}/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done +} diff --git a/e2e-tests/tests/major-upgrade/01-create-cluster.yaml b/e2e-tests/tests/major-upgrade/01-create-cluster.yaml index 1380fae4aa..729fbcc51f 100644 --- a/e2e-tests/tests/major-upgrade/01-create-cluster.yaml +++ b/e2e-tests/tests/major-upgrade/01-create-cluster.yaml @@ -7,7 +7,7 @@ commands: set -o xtrace source ../../functions - + get_cr \ | yq eval ' .spec.postgresVersion = 12 | diff --git a/e2e-tests/tests/major-upgrade/10-assert.yaml b/e2e-tests/tests/major-upgrade/10-assert.yaml index 66d99bc294..ae8dcb18f8 100644 --- a/e2e-tests/tests/major-upgrade/10-assert.yaml +++ b/e2e-tests/tests/major-upgrade/10-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 360 +commands: +- script: |- + kubectl -n ${NAMESPACE} get pg,pod,job + sleep 5 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -20,3 +24,19 @@ status: ready: 3 size: 3 state: ready +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: major-upgrade + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/11-assert.yaml b/e2e-tests/tests/major-upgrade/11-assert.yaml index 525d522912..47ccba03b5 100644 --- a/e2e-tests/tests/major-upgrade/11-assert.yaml +++ b/e2e-tests/tests/major-upgrade/11-assert.yaml @@ -1,31 +1,21 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 660 +commands: +- script: |- + kubectl get postgrescluster major-upgrade \ + -n ${NAMESPACE} \ + -o yaml \ + | yq eval '.status.pgbackrest.repos' - --- -kind: Job -apiVersion: batch/v1 +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-12-to-13 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true + name: major-upgrade status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-12-to-13 -spec: - pgCluster: major-upgrade - repoName: repo1 - options: - - --type=full -status: - state: Succeeded + pgbackrest: + repos: + - bound: true + name: repo2 + replicaCreateBackupComplete: true + stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/11-change-repo.yaml b/e2e-tests/tests/major-upgrade/11-change-repo.yaml new file mode 100644 index 0000000000..22094b19f4 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/11-change-repo.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl patch \ + -n $NAMESPACE \ + perconapgcluster major-upgrade \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]' diff --git a/e2e-tests/tests/major-upgrade/12-assert.yaml b/e2e-tests/tests/major-upgrade/12-assert.yaml index cb8a90c13f..d1790eabf5 100644 --- a/e2e-tests/tests/major-upgrade/12-assert.yaml +++ b/e2e-tests/tests/major-upgrade/12-assert.yaml @@ -1,30 +1,31 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 +timeout: 660 --- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore +kind: Job +apiVersion: batch/v1 metadata: - name: restore-after-12-to-13 -spec: - pgCluster: major-upgrade - repoName: repo1 + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-12-to-13 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo2 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true status: - state: Succeeded + succeeded: 1 --- apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster +kind: PerconaPGBackup metadata: - name: major-upgrade + name: backup-after-12-to-13 +spec: + pgCluster: major-upgrade + repoName: repo2 + options: + - --type=full status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready + state: Succeeded diff --git a/e2e-tests/tests/major-upgrade/11-run-backup.yaml b/e2e-tests/tests/major-upgrade/12-run-backup.yaml similarity index 89% rename from e2e-tests/tests/major-upgrade/11-run-backup.yaml rename to e2e-tests/tests/major-upgrade/12-run-backup.yaml index 0cfac94a95..601008d4ca 100644 --- a/e2e-tests/tests/major-upgrade/11-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/12-run-backup.yaml @@ -4,6 +4,6 @@ metadata: name: backup-after-12-to-13 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo2 options: - --type=full diff --git a/e2e-tests/tests/major-upgrade/13-assert.yaml b/e2e-tests/tests/major-upgrade/13-assert.yaml index 7c65443fbe..2d05bb7d88 100644 --- a/e2e-tests/tests/major-upgrade/13-assert.yaml +++ b/e2e-tests/tests/major-upgrade/13-assert.yaml @@ -1,10 +1,52 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg13/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 30 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 05-read-from-primary -data: - data: ' 100500' + name: restore-after-12-to-13 +spec: + pgCluster: major-upgrade + repoName: repo2 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/12-run-restore.yaml b/e2e-tests/tests/major-upgrade/13-run-restore.yaml similarity index 87% rename from e2e-tests/tests/major-upgrade/12-run-restore.yaml rename to e2e-tests/tests/major-upgrade/13-run-restore.yaml index 6350f5d6f1..bba96fe985 100644 --- a/e2e-tests/tests/major-upgrade/12-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/13-run-restore.yaml @@ -4,4 +4,4 @@ metadata: name: restore-after-12-to-13 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo2 diff --git a/e2e-tests/tests/major-upgrade/14-assert.yaml b/e2e-tests/tests/major-upgrade/14-assert.yaml new file mode 100644 index 0000000000..7c65443fbe --- /dev/null +++ b/e2e-tests/tests/major-upgrade/14-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 05-read-from-primary +data: + data: ' 100500' diff --git a/e2e-tests/tests/major-upgrade/13-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/14-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/13-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/14-read-from-primary.yaml diff --git a/e2e-tests/tests/major-upgrade/20-assert.yaml b/e2e-tests/tests/major-upgrade/20-assert.yaml index d60a84442f..c506a745f5 100644 --- a/e2e-tests/tests/major-upgrade/20-assert.yaml +++ b/e2e-tests/tests/major-upgrade/20-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 360 +commands: +- script: |- + kubectl -n ${NAMESPACE} get pg,pod,job + sleep 5 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -20,3 +24,19 @@ status: ready: 3 size: 3 state: ready +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: major-upgrade + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo2 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/21-assert.yaml b/e2e-tests/tests/major-upgrade/21-assert.yaml index 788d3de044..301240b0f5 100644 --- a/e2e-tests/tests/major-upgrade/21-assert.yaml +++ b/e2e-tests/tests/major-upgrade/21-assert.yaml @@ -1,31 +1,21 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 660 +commands: +- script: |- + kubectl get postgrescluster major-upgrade \ + -n ${NAMESPACE} \ + -o yaml \ + | yq eval '.status.pgbackrest.repos' - --- -kind: Job -apiVersion: batch/v1 +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-13-to-14 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true + name: major-upgrade status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-13-to-14 -spec: - pgCluster: major-upgrade - repoName: repo1 - options: - - --type=full -status: - state: Succeeded + pgbackrest: + repos: + - bound: true + name: repo3 + replicaCreateBackupComplete: true + stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/21-change-repo.yaml b/e2e-tests/tests/major-upgrade/21-change-repo.yaml new file mode 100644 index 0000000000..57f9edc8a4 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/21-change-repo.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl patch \ + -n $NAMESPACE \ + perconapgcluster major-upgrade \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo3"}]' diff --git a/e2e-tests/tests/major-upgrade/22-assert.yaml b/e2e-tests/tests/major-upgrade/22-assert.yaml index 2b8c164f6e..7a2f6c1c0f 100644 --- a/e2e-tests/tests/major-upgrade/22-assert.yaml +++ b/e2e-tests/tests/major-upgrade/22-assert.yaml @@ -1,30 +1,31 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 +timeout: 660 --- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore +kind: Job +apiVersion: batch/v1 metadata: - name: restore-after-13-to-14 -spec: - pgCluster: major-upgrade - repoName: repo1 + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-13-to-14 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo3 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true status: - state: Succeeded + succeeded: 1 --- apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster +kind: PerconaPGBackup metadata: - name: major-upgrade + name: backup-after-13-to-14 +spec: + pgCluster: major-upgrade + repoName: repo3 + options: + - --type=full status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready + state: Succeeded diff --git a/e2e-tests/tests/major-upgrade/21-run-backup.yaml b/e2e-tests/tests/major-upgrade/22-run-backup.yaml similarity index 89% rename from e2e-tests/tests/major-upgrade/21-run-backup.yaml rename to e2e-tests/tests/major-upgrade/22-run-backup.yaml index 7f60f47386..a7a1050c53 100644 --- a/e2e-tests/tests/major-upgrade/21-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/22-run-backup.yaml @@ -4,6 +4,6 @@ metadata: name: backup-after-13-to-14 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo3 options: - --type=full diff --git a/e2e-tests/tests/major-upgrade/23-assert.yaml b/e2e-tests/tests/major-upgrade/23-assert.yaml index c86dbb55fc..75b542e689 100644 --- a/e2e-tests/tests/major-upgrade/23-assert.yaml +++ b/e2e-tests/tests/major-upgrade/23-assert.yaml @@ -1,10 +1,52 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 30 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 07-read-from-primary -data: - data: ' 100500' + name: restore-after-13-to-14 +spec: + pgCluster: major-upgrade + repoName: repo3 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/22-run-restore.yaml b/e2e-tests/tests/major-upgrade/23-run-restore.yaml similarity index 87% rename from e2e-tests/tests/major-upgrade/22-run-restore.yaml rename to e2e-tests/tests/major-upgrade/23-run-restore.yaml index 7ce7fed535..079ebf7b77 100644 --- a/e2e-tests/tests/major-upgrade/22-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/23-run-restore.yaml @@ -4,4 +4,4 @@ metadata: name: restore-after-13-to-14 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo3 diff --git a/e2e-tests/tests/major-upgrade/24-assert.yaml b/e2e-tests/tests/major-upgrade/24-assert.yaml new file mode 100644 index 0000000000..c86dbb55fc --- /dev/null +++ b/e2e-tests/tests/major-upgrade/24-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-from-primary +data: + data: ' 100500' diff --git a/e2e-tests/tests/major-upgrade/23-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/24-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/23-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/24-read-from-primary.yaml diff --git a/e2e-tests/tests/major-upgrade/30-assert.yaml b/e2e-tests/tests/major-upgrade/30-assert.yaml index 1d058344ee..b871b4faf8 100644 --- a/e2e-tests/tests/major-upgrade/30-assert.yaml +++ b/e2e-tests/tests/major-upgrade/30-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 360 +commands: +- script: |- + kubectl -n ${NAMESPACE} get pg,pod,job + sleep 5 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -20,3 +24,19 @@ status: ready: 3 size: 3 state: ready +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: major-upgrade + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo3 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/31-assert.yaml b/e2e-tests/tests/major-upgrade/31-assert.yaml index 14ca1327c2..6af985e4b7 100644 --- a/e2e-tests/tests/major-upgrade/31-assert.yaml +++ b/e2e-tests/tests/major-upgrade/31-assert.yaml @@ -1,31 +1,21 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 660 +commands: +- script: |- + kubectl get postgrescluster major-upgrade \ + -n ${NAMESPACE} \ + -o yaml \ + | yq eval '.status.pgbackrest.repos' - --- -kind: Job -apiVersion: batch/v1 +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-14-to-15 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true + name: major-upgrade status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-14-to-15 -spec: - pgCluster: major-upgrade - repoName: repo1 - options: - - --type=full -status: - state: Succeeded + pgbackrest: + repos: + - bound: true + name: repo4 + replicaCreateBackupComplete: true + stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/31-change-repo.yaml b/e2e-tests/tests/major-upgrade/31-change-repo.yaml new file mode 100644 index 0000000000..33711377c4 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/31-change-repo.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl patch \ + -n $NAMESPACE \ + perconapgcluster major-upgrade \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo4"}]' diff --git a/e2e-tests/tests/major-upgrade/32-assert.yaml b/e2e-tests/tests/major-upgrade/32-assert.yaml index 0246ec0e79..75daf8364c 100644 --- a/e2e-tests/tests/major-upgrade/32-assert.yaml +++ b/e2e-tests/tests/major-upgrade/32-assert.yaml @@ -1,30 +1,31 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 +timeout: 660 --- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore +kind: Job +apiVersion: batch/v1 metadata: - name: restore-after-14-to-15 -spec: - pgCluster: major-upgrade - repoName: repo1 + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-14-to-15 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo4 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true status: - state: Succeeded + succeeded: 1 --- apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster +kind: PerconaPGBackup metadata: - name: major-upgrade + name: backup-after-14-to-15 +spec: + pgCluster: major-upgrade + repoName: repo4 + options: + - --type=full status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready + state: Succeeded diff --git a/e2e-tests/tests/major-upgrade/31-run-backup.yaml b/e2e-tests/tests/major-upgrade/32-run-backup.yaml similarity index 89% rename from e2e-tests/tests/major-upgrade/31-run-backup.yaml rename to e2e-tests/tests/major-upgrade/32-run-backup.yaml index e787fedfca..792209c20c 100644 --- a/e2e-tests/tests/major-upgrade/31-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/32-run-backup.yaml @@ -4,6 +4,6 @@ metadata: name: backup-after-14-to-15 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo4 options: - --type=full diff --git a/e2e-tests/tests/major-upgrade/33-assert.yaml b/e2e-tests/tests/major-upgrade/33-assert.yaml index 7321c61abe..1d3a8a9aae 100644 --- a/e2e-tests/tests/major-upgrade/33-assert.yaml +++ b/e2e-tests/tests/major-upgrade/33-assert.yaml @@ -1,10 +1,52 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 30 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 09-read-from-primary -data: - data: ' 100500' + name: restore-after-14-to-15 +spec: + pgCluster: major-upgrade + repoName: repo4 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/32-run-restore.yaml b/e2e-tests/tests/major-upgrade/33-run-restore.yaml similarity index 87% rename from e2e-tests/tests/major-upgrade/32-run-restore.yaml rename to e2e-tests/tests/major-upgrade/33-run-restore.yaml index 24845cbb29..8e7c7c6dab 100644 --- a/e2e-tests/tests/major-upgrade/32-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/33-run-restore.yaml @@ -4,5 +4,5 @@ metadata: name: restore-after-14-to-15 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo4 diff --git a/e2e-tests/tests/major-upgrade/34-assert.yaml b/e2e-tests/tests/major-upgrade/34-assert.yaml new file mode 100644 index 0000000000..7321c61abe --- /dev/null +++ b/e2e-tests/tests/major-upgrade/34-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 09-read-from-primary +data: + data: ' 100500' diff --git a/e2e-tests/tests/major-upgrade/33-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/34-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/33-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/34-read-from-primary.yaml diff --git a/e2e-tests/tests/major-upgrade/40-assert.yaml b/e2e-tests/tests/major-upgrade/40-assert.yaml index 58f19f4e69..de3a338c83 100644 --- a/e2e-tests/tests/major-upgrade/40-assert.yaml +++ b/e2e-tests/tests/major-upgrade/40-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 360 +commands: +- script: |- + kubectl -n ${NAMESPACE} get pg,pod,job + sleep 5 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -20,3 +24,19 @@ status: ready: 3 size: 3 state: ready +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: major-upgrade + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo4 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/41-assert.yaml b/e2e-tests/tests/major-upgrade/41-assert.yaml index 2b003b69a9..1b95780c8f 100644 --- a/e2e-tests/tests/major-upgrade/41-assert.yaml +++ b/e2e-tests/tests/major-upgrade/41-assert.yaml @@ -1,31 +1,21 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 660 +commands: +- script: |- + kubectl get postgrescluster major-upgrade \ + -n ${NAMESPACE} \ + -o yaml \ + | yq eval '.status.pgbackrest.repos' - --- -kind: Job -apiVersion: batch/v1 +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-15-to-16 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true + name: major-upgrade status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-15-to-16 -spec: - pgCluster: major-upgrade - repoName: repo1 - options: - - --type=full -status: - state: Succeeded + pgbackrest: + repos: + - bound: true + name: repo1 + replicaCreateBackupComplete: true + stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/41-change-repo.yaml b/e2e-tests/tests/major-upgrade/41-change-repo.yaml new file mode 100644 index 0000000000..d3d277599f --- /dev/null +++ b/e2e-tests/tests/major-upgrade/41-change-repo.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl patch \ + -n $NAMESPACE \ + perconapgcluster major-upgrade \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo1"}]' diff --git a/e2e-tests/tests/major-upgrade/42-assert.yaml b/e2e-tests/tests/major-upgrade/42-assert.yaml index ff07785690..2b003b69a9 100644 --- a/e2e-tests/tests/major-upgrade/42-assert.yaml +++ b/e2e-tests/tests/major-upgrade/42-assert.yaml @@ -1,30 +1,31 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 +timeout: 660 +--- +kind: Job +apiVersion: batch/v1 +metadata: + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-15-to-16 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 --- apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore +kind: PerconaPGBackup metadata: - name: restore-after-15-to-16 + name: backup-after-15-to-16 spec: pgCluster: major-upgrade repoName: repo1 + options: + - --type=full status: state: Succeeded ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: major-upgrade -status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready diff --git a/e2e-tests/tests/major-upgrade/41-run-backup.yaml b/e2e-tests/tests/major-upgrade/42-run-backup.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/41-run-backup.yaml rename to e2e-tests/tests/major-upgrade/42-run-backup.yaml diff --git a/e2e-tests/tests/major-upgrade/43-assert.yaml b/e2e-tests/tests/major-upgrade/43-assert.yaml index 95d57c293f..3114c23118 100644 --- a/e2e-tests/tests/major-upgrade/43-assert.yaml +++ b/e2e-tests/tests/major-upgrade/43-assert.yaml @@ -1,10 +1,52 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 30 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 11-read-from-primary -data: - data: ' 100500' + name: restore-after-15-to-16 +spec: + pgCluster: major-upgrade + repoName: repo1 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/42-run-restore.yaml b/e2e-tests/tests/major-upgrade/43-run-restore.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/42-run-restore.yaml rename to e2e-tests/tests/major-upgrade/43-run-restore.yaml diff --git a/e2e-tests/tests/major-upgrade/44-assert.yaml b/e2e-tests/tests/major-upgrade/44-assert.yaml new file mode 100644 index 0000000000..95d57c293f --- /dev/null +++ b/e2e-tests/tests/major-upgrade/44-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 11-read-from-primary +data: + data: ' 100500' diff --git a/e2e-tests/tests/major-upgrade/43-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/44-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/43-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/44-read-from-primary.yaml