pgupgrade: Use remote migration from SCL images (PROJQUAY-5631)

- Switch to rhel8 images for more efficient implementation

controllers/quay/quayregistry_controller.go

@@ -27,6 +27,7 @@ import (
 	prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/tidwall/sjson"
 	"gopkg.in/yaml.v2"
+	appsv1 "k8s.io/api/apps/v1"
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2"
 	batchv1 "k8s.io/api/batch/v1"
@@ -242,6 +243,61 @@ func (r *QuayRegistryReconciler) checkPostgresUpgradeStatus(
 
 		if job.Status.Succeeded == 1 {
 			log.Info(fmt.Sprintf("%s upgrade complete", jobName))
+			var oldPostgresDeploymentName string
+			if jobName == clairPostgresUpgradeJobName {
+				oldPostgresDeploymentName = fmt.Sprintf("%s-%s", quay.GetName(), "clair-postgres-old")
+			} else {
+				oldPostgresDeploymentName = fmt.Sprintf("%s-%s", quay.GetName(), "quay-database-old")
+			}
+			oldPostgresDeployment := &appsv1.Deployment{}
+			if err := r.Client.Get(
+				ctx,
+				types.NamespacedName{
+					Name:      oldPostgresDeploymentName,
+					Namespace: quay.GetNamespace(),
+				},
+				oldPostgresDeployment,
+			); err != nil {
+				r.Log.Info(fmt.Sprintf("%s deployment not found, skipping", oldPostgresDeploymentName))
+				continue
+			}
+
+			// Remove owner reference
+			obj, err := v1.RemoveOwnerReference(quay, oldPostgresDeployment)
+			if err != nil {
+				log.Error(err, "could not remove owner reference from old postgres deployment")
+			}
+
+			// Delete old postgres deployment
+			if err := r.Client.Delete(
+				ctx,
+				obj,
+			); err != nil {
+				r.Log.Error(err, fmt.Sprintf("%s deployment could not be deleted", oldPostgresDeploymentName))
+			}
+
+			// Remove owner reference from old pvc so user can delete when ready
+			var oldPostgresPVCName string
+			if jobName == clairPostgresUpgradeJobName {
+				oldPostgresPVCName = fmt.Sprintf("%s-%s", quay.GetName(), "clair-postgres")
+			} else {
+				oldPostgresPVCName = fmt.Sprintf("%s-%s", quay.GetName(), "quay-database")
+			}
+			oldPostgresPVC := &corev1.PersistentVolumeClaim{}
+			if err := r.Client.Get(
+				ctx,
+				types.NamespacedName{
+					Name:      oldPostgresPVCName,
+					Namespace: quay.GetNamespace(),
+				},
+				oldPostgresPVC,
+			); err != nil {
+				r.Log.Info(fmt.Sprintf("%s pvc not found, skipping", oldPostgresDeploymentName))
+				continue
+			}
+			if _, err := v1.RemoveOwnerReference(quay, oldPostgresPVC); err != nil {
+				log.Error(err, "could not remove owner reference from old postgres pvc")
+			}
 			continue
 		}
 

hack/build.sh

@@ -71,7 +71,7 @@ digest docker.io/redis:7.0 REDIS_DIGEST
 # shellcheck disable=SC2034
 POSTGRES_DIGEST='quay.io/sclorg/postgresql-13-c9s@sha256:efe7ca31ff169cc8d5f458cc0da4e844b6646a7c1fe76ac4d61a79dcc749f5d1'
 # shellcheck disable=SC2034
-POSTGRES_UPGRADE_DIGEST='centos/postgresql-12-centos7@sha256:be8803d45d64870f8dfd018f3110af62e2e1558d64191faea461005e1bd03243'
+POSTGRES_OLD_DIGEST='centos/postgresql-10-centos7@sha256:de1560cb35e5ec643e7b3a772ebaac8e3a7a2a8e8271d9e91ff023539b4dfb33'
 
 # need exporting so that yq can see them
 export OPERATOR_DIGEST
@@ -98,7 +98,7 @@ yq eval -i '
 	.spec.install.spec.deployments[0].spec.template.spec.containers[0].env[] |= select(.name == "RELATED_IMAGE_COMPONENT_BUILDER") .value = strenv(BUILDER_DIGEST) |
 	.spec.install.spec.deployments[0].spec.template.spec.containers[0].env[] |= select(.name == "RELATED_IMAGE_COMPONENT_BUILDER_QEMU") .value = strenv(BUILDER_QEMU_DIGEST) |
 	.spec.install.spec.deployments[0].spec.template.spec.containers[0].env[] |= select(.name == "RELATED_IMAGE_COMPONENT_POSTGRES") .value = strenv(POSTGRES_DIGEST) |
-	.spec.install.spec.deployments[0].spec.template.spec.containers[0].env[] |= select(.name == "RELATED_IMAGE_COMPONENT_POSTGRES_UPGRADE") .value = strenv(POSTGRES_UPGRADE_DIGEST) |
+	.spec.install.spec.deployments[0].spec.template.spec.containers[0].env[] |= select(.name == "RELATED_IMAGE_COMPONENT_POSTGRES_PREVIOUS") .value = strenv(POSTGRES_OLD_DIGEST) |
 	.spec.install.spec.deployments[0].spec.template.spec.containers[0].env[] |= select(.name == "RELATED_IMAGE_COMPONENT_REDIS") .value = strenv(REDIS_DIGEST)
 	' "${CSV_PATH}"
 

kustomize/components/clairpgupgrade/clair-pg-old.deployment.yaml

@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: clair-postgres-old
+  labels:
+    quay-component: clair-postgres
+  annotations:
+    quay-component: clair-postgres
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      quay-component: clair-postgres
+  template:
+    metadata:
+      labels:
+        quay-component: clair-postgres
+    spec:
+      terminationGracePeriodSeconds: 180
+      serviceAccountName: clair-postgres
+      volumes:
+        - name: clair-postgres-conf-sample
+          configMap:
+            name: clair-postgres-conf-sample
+        - name: postgres-data
+          persistentVolumeClaim:
+            claimName: clair-postgres
+      containers:
+        - name: postgres
+          image: centos/postgresql-10-centos7@sha256:de1560cb35e5ec643e7b3a772ebaac8e3a7a2a8e8271d9e91ff023539b4dfb33
+          imagePullPolicy: "IfNotPresent"
+          ports:
+            - containerPort: 5432
+              protocol: TCP
+          env:
+            - name: POSTGRESQL_USER
+              value: postgres
+            - name: POSTGRESQL_DATABASE
+              value: postgres
+            - name: POSTGRESQL_PASSWORD
+              value: postgres
+            - name: POSTGRESQL_ADMIN_PASSWORD
+              value: postgres
+            - name: POSTGRESQL_SHARED_BUFFERS
+              value: 256MB
+            - name: POSTGRESQL_MAX_CONNECTIONS
+              value: "2000"
+          volumeMounts:
+            - name: clair-postgres-conf-sample
+              mountPath: /usr/share/pgsql/postgresql.conf.sample
+              subPath: postgresql.conf.sample
+            - name: postgres-data
+              mountPath: /var/lib/pgsql/data
+          resources:
+            requests:
+              cpu: 500m
+              memory: 2Gi

kustomize/components/pgupgrade/pg-upgrade.persistentvolumeclaim.yaml → kustomize/components/clairpgupgrade/clair-pg-old.persistentvolumeclaim.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: postgres-migration
+  name: clair-postgres
 spec:
   accessModes:
     - ReadWriteOnce

kustomize/components/clairpgupgrade/clair-pg-upgrade.job.yaml

@@ -9,90 +9,49 @@ metadata:
 spec:
   template:
     spec:
-      restartPolicy: Never
+      restartPolicy: OnFailure
       terminationGracePeriodSeconds: 180
       serviceAccountName: clair-postgres
       volumes:
         - name: clair-postgres-conf-sample
           configMap:
             name: clair-postgres-conf-sample
-        - name: postgres-data
+        - name: clair-postgres-13
           persistentVolumeClaim:
-            claimName: clair-postgres
-        - name: migration-data
-          persistentVolumeClaim:
-            claimName: clair-postgres-migration
-      initContainers:
-        - name: postgres-old
-          image: centos/postgresql-12-centos7@sha256:be8803d45d64870f8dfd018f3110af62e2e1558d64191faea461005e1bd03243
-          imagePullPolicy: IfNotPresent
-          ports:
-            - containerPort: 5432
-              protocol: TCP
-          env:
-            - name: POSTGRESQL_UPGRADE
-              value: copy
-            - name: POSTGRESQL_USER
-              value: postgres
-            - name: POSTGRESQL_DATABASE
-              value: postgres
-            - name: POSTGRESQL_PASSWORD
-              value: postgres
-            - name: POSTGRESQL_ADMIN_PASSWORD
-              value: postgres
-            - name: POSTGRESQL_MAX_CONNECTIONS
-              value: "1000"
-          volumeMounts:
-            - name: clair-postgres-conf-sample
-              mountPath: /usr/share/pgsql/postgresql.conf.sample
-              subPath: postgresql.conf.sample
-            - name: postgres-data
-              mountPath: /var/lib/pgsql/data
-            - name: migration-data
-              mountPath: /var/lib/pgsql/backup
-          resources:
-            requests:
-              cpu: 500m
-              memory: 2Gi
-          command:
-            - "/bin/bash"
-            - "-c"
-            - "cp -r /var/lib/pgsql/data /var/lib/pgsql/backup/clair && run-postgresql --help"
-
+            claimName: clair-postgres-13
       containers:
-        - name: postgres-new
+        - name: clair-postgres-13
           image: centos/postgresql-13-centos7@sha256:71b24684d64da46f960682cc4216222a7e4ed8b1a31dd5a865b3e71afdea20d2
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 5432
               protocol: TCP
           env:
-            - name: POSTGRESQL_UPGRADE
-              value: copy
-            - name: POSTGRESQL_USER
-              value: postgres
-            - name: POSTGRESQL_DATABASE
-              value: postgres
-            - name: POSTGRESQL_PASSWORD
-              value: postgres
-            - name: POSTGRESQL_ADMIN_PASSWORD
-              value: postgres
+            - name: POSTGRESQL_MIGRATION_REMOTE_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: clair-config-secret
+                  key: clair-db-host
+            - name: POSTGRESQL_MIGRATION_ADMIN_PASSWORD
+              value: postgres
+            - name: POSTGRESQL_SHARED_BUFFERS
+              value: 256MB
             - name: POSTGRESQL_MAX_CONNECTIONS
-              value: "1000"
+              value: "2000"
           volumeMounts:
             - name: clair-postgres-conf-sample
               mountPath: /usr/share/pgsql/postgresql.conf.sample
               subPath: postgresql.conf.sample
-            - name: postgres-data
+            - name: clair-postgres-13
               mountPath: /var/lib/pgsql/data
-            - name: migration-data
-              mountPath: /var/lib/pgsql/backup
           resources:
             requests:
               cpu: 500m
               memory: 2Gi
           command:
-            - "run-postgresql"
+          - "/bin/sh"
+          - "-c"
           args:
-            - "--version"
-  backoffLimit: 20
+          - >
+            run-postgresql --version || (echo "postgres migration command failed, cleaning up..." && rm -rf /var/lib/pgsql/data/* && exit 1)
+  backoffLimit: 50

kustomize/components/clairpgupgrade/kustomization.yaml

@@ -1,5 +1,10 @@
+# Overlay variant for upgrading to current Project Quay release.
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
 resources:
   - ./clair-pg-upgrade.job.yaml
-  - ./clair-pg-upgrade.persistentvolumeclaim.yaml
+  - ./clair-pg-old.persistentvolumeclaim.yaml
+  - ./clair-pg-old.deployment.yaml
+patchesStrategicMerge:
+  - ./clair.deployment.patch.yaml
+  - ./clair-pg.deployment.patch.yaml

kustomize/components/clairpostgres/postgres.deployment.yaml

@@ -26,9 +26,9 @@ spec:
             name: clair-postgres-conf-sample
         - name: postgres-data
           persistentVolumeClaim:
-            claimName: clair-postgres
+            claimName: clair-postgres-13
       containers:
-        - name: postgres
+        - name: clair-postgres
           image: centos/postgresql-13-centos7@sha256:71b24684d64da46f960682cc4216222a7e4ed8b1a31dd5a865b3e71afdea20d2
           imagePullPolicy: "IfNotPresent"
           ports:
@@ -43,8 +43,10 @@ spec:
               value: postgres
             - name: POSTGRESQL_ADMIN_PASSWORD
               value: postgres
+            - name: POSTGRESQL_SHARED_BUFFERS
+              value: 256MB
             - name: POSTGRESQL_MAX_CONNECTIONS
-              value: "1000"
+              value: "2000"
           volumeMounts:
             - name: clair-postgres-conf-sample
               mountPath: /usr/share/pgsql/postgresql.conf.sample

kustomize/components/clairpostgres/postgres.persistentvolumeclaim.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: clair-postgres
+  name: clair-postgres-13
   labels:
     quay-component: clair-postgres
   annotations:

kustomize/components/pgupgrade/kustomization.yaml

@@ -1,5 +1,10 @@
+# Overlay variant for upgrading to current Project Quay release.
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
 resources:
-  - ./pg-upgrade.job.yaml
-  - ./pg-upgrade.persistentvolumeclaim.yaml
+  - ./quay-pg-upgrade.job.yaml
+  - ./quay-pg-old.persistentvolumeclaim.yaml
+  - ./quay-pg-old.deployment.yaml
+patchesStrategicMerge:
+  - ./quay.deployment.patch.yaml
+  - ./quay-pg.deployment.patch.yaml