spectrum fusion updates (#2004)

* create utility image Signed-off-by: Ben Luzarraga <[email protected]> * update test resoruces Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * up memory Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * update permissions Signed-off-by: Ben Luzarraga <[email protected]> * replace xargs Signed-off-by: Ben Luzarraga <[email protected]> * update image for cpfs-util Signed-off-by: Ben Luzarraga <[email protected]> * upload test recipes Signed-off-by: Ben Luzarraga <[email protected]> * update multins recipe Signed-off-by: Ben Luzarraga <[email protected]> * enable log collection Signed-off-by: Ben Luzarraga <[email protected]> * add cleanup function Signed-off-by: Ben Luzarraga <[email protected]> * increase timeout for zen service Signed-off-by: Ben Luzarraga <[email protected]> * increase timeout Signed-off-by: Ben Luzarraga <[email protected]> * update resources for docs Signed-off-by: Ben Luzarraga <[email protected]> * update recipe and scripts Signed-off-by: Ben Luzarraga <[email protected]> * update images, add wait for cluster cr, update operand restore Signed-off-by: Ben Luzarraga <[email protected]> * replace namespaces, other test data Signed-off-by: Ben Luzarraga <[email protected]> --------- Signed-off-by: Ben Luzarraga <[email protected]>
IBM · May 31, 2024 · 0a3587e · 0a3587e
1 parent b74b524
commit 0a3587e
Show file tree

Hide file tree

Showing 21 changed files with 2,096 additions and 6 deletions.
diff --git a/velero/schedule/common-service-db/cs-db-backup-deployment.yaml b/velero/schedule/common-service-db/cs-db-backup-deployment.yaml
@@ -3,6 +3,8 @@ apiVersion: apps/v1
 metadata:
   name: cs-db-backup
   namespace: <cs-db namespace>
+  labels:
+    foundationservices.cloudpak.ibm.com: cs-db-data
 spec:
   selector:
     matchLabels:
@@ -28,7 +30,7 @@ spec:
           - sh
           - -c
           - sleep infinity
-          image: icr.io/cpopen/cpfs/cpfs-utils:4.3.0 #4.1.0 if using CS 4.1, 4.2.0 if using CS 4.2
+          image: icr.io/cpopen/cpfs/cpfs-utils:4.6.0 #4.1.0 if using CS 4.1, 4.2.0 if using CS 4.2
           imagePullPolicy: IfNotPresent
           name: cs-db-br
           resources:
@@ -47,6 +49,8 @@ spec:
             name: cs-db-backup-mount
           - name: scripts
             mountPath: "/cs-db"
+          - mountPath: /cs-db/cs-db-backup/logs
+            name: logs
         dnsPolicy: ClusterFirst
         schedulerName: default-scheduler
         securityContext:
@@ -61,4 +65,6 @@ spec:
         - name: scripts
           configMap:
             name: cs-db-br-configmap
-            defaultMode: 0777
+            defaultMode: 0777
+        - emptyDir: {}
+          name: logs
diff --git a/velero/schedule/common-service-db/cs-db-br-script-cm.yaml b/velero/schedule/common-service-db/cs-db-br-script-cm.yaml
@@ -24,15 +24,20 @@ data:
 
     MODE=$1
     CSDB_NAMESPACE=$2
+    CLUSTER_CR=common-service-db
 
     BACKUP_DIR=/cs-db/cs-db-backup
 
     function main {
         if [[ $MODE == "backup" ]]; then
+            save_log "logs" "backup_log"
+            trap cleanup_log EXIT
             info "Mode set to backup, beginning backup process."
             backup
             success "Backup completed successfully."
         elif [[ $MODE == "restore" ]]; then
+            save_log "logs" "restore_log"
+            trap cleanup_log EXIT
             info "Mode is set to restore, beginning restore process."
             restore
             success "Restore completed successfully."
@@ -63,6 +68,7 @@ data:
     
     function restore {
 
+        wait_for_cluster_cr
         CNPG_PRIMARY_POD=`oc get cluster.postgresql.k8s.enterprisedb.io common-service-db -o jsonpath="{.status.currentPrimary}" -n $CSDB_NAMESPACE` 
         oc exec $CNPG_PRIMARY_POD -n $CSDB_NAMESPACE -- mkdir -p /run/cs-db_backup
         oc cp $BACKUP_DIR/database/cs-db_cloudpak_backup.dump $CSDB_NAMESPACE/$CNPG_PRIMARY_POD:/run/cs-db_backup/cs-db_cloudpak_backup.dump
@@ -114,6 +120,60 @@ data:
         fi
     }
 
+    function wait_for_cluster_cr {
+        info "Waiting for EDB Cluster CR $CLUSTER_CR to complete in namespace $CSDB_NAMESPACE."
+        cluster_cr_exists=$(oc get clusters.postgresql.k8s.enterprisedb.io $CLUSTER_CR -n $CSDB_NAMESPACE --no-headers || echo fail)
+        if [[ $cluster_cr_exists != "fail" ]]; then
+            completed=$(oc get clusters.postgresql.k8s.enterprisedb.io $CLUSTER_CR -n $CSDB_NAMESPACE -o=jsonpath='{.status.phase}')
+            retry_count=40
+            while [[ $completed != "Cluster in healthy state" ]] && [[ $retry_count > 0 ]]
+            do
+                info "Wait for cluster $CLUSTER_CR to complete. Try again in 15s."
+                sleep 15
+                completed=$(oc get clusters.postgresql.k8s.enterprisedb.io $CLUSTER_CR -n $CSDB_NAMESPACE -o=jsonpath='{.status.phase}')
+                retry_count=$retry_count-1
+            done
+
+            if [[ $retry_count == 0 ]] && [[ $completed != "1/1" ]]; then
+                error "Timed out waiting for cluster $CLUSTER_CR."
+            else
+                info "EDB cluster $CLUSTER_CR ready."
+            fi
+        else
+            error "EDB cluster $CLUSTER_CR not present."
+        fi
+    }
+
+    function save_log(){
+        local LOG_DIR="$BACKUP_DIR/$1"
+        LOG_FILE="$LOG_DIR/$2_$(date +'%Y%m%d%H%M%S').log"
+
+        if [[ ! -d $LOG_DIR ]]; then
+            mkdir -p "$LOG_DIR"
+        fi
+
+        # Create a named pipe
+        PIPE=$(mktemp -u)
+        mkfifo "$PIPE"
+
+        # Tee the output to both the log file and the terminal
+        tee "$LOG_FILE" < "$PIPE" &
+
+        # Redirect stdout and stderr to the named pipe
+        exec > "$PIPE" 2>&1
+
+        # Remove the named pipe
+        rm "$PIPE"
+    }
+
+    function cleanup_log() {
+        # Check if the log file already exists
+        if [[ -e $LOG_FILE ]]; then
+            # Remove ANSI escape sequences from log file
+            sed -E 's/\x1B\[[0-9;]+[A-Za-z]//g' "$LOG_FILE" > "$LOG_FILE.tmp" && mv "$LOG_FILE.tmp" "$LOG_FILE"
+        fi
+    }
+
     function msg() {
         printf '%b\n' "$1"
     }

diff --git a/velero/schedule/keycloak/keycloak-backup-deployment.yaml b/velero/schedule/keycloak/keycloak-backup-deployment.yaml
@@ -28,7 +28,7 @@ spec:
           - sh
           - -c
           - sleep infinity
-          image: icr.io/cpopen/cpfs/cpfs-utils:4.3.0 #4.1.0 if using CS 4.1, 4.2.0 if using CS 4.2
+          image: icr.io/cpopen/cpfs/cpfs-utils:4.6.0 #4.1.0 if using CS 4.1, 4.2.0 if using CS 4.2
           imagePullPolicy: IfNotPresent
           name: keycloak-br
           resources:

diff --git a/velero/schedule/license_service_reporter/lsr-backup-deployment.yaml b/velero/schedule/license_service_reporter/lsr-backup-deployment.yaml
@@ -29,7 +29,7 @@ spec:
           - sh
           - -c
           - sleep infinity
-          image: icr.io/cpopen/cpfs/cpfs-utils:4.3.0 #4.1.0 if using CS 4.1, 4.2.0 if using CS 4.2
+          image: icr.io/cpopen/cpfs/cpfs-utils:4.6.0 #4.1.0 if using CS 4.1, 4.2.0 if using CS 4.2
           imagePullPolicy: IfNotPresent
           name: lsr-backup-job
           resources:

diff --git a/velero/schedule/zen5-backup-deployment.yaml b/velero/schedule/zen5-backup-deployment.yaml
@@ -29,7 +29,7 @@ spec:
           - sh
           - -c
           - sleep infinity
-          image: icr.io/cpopen/cpfs/cpfs-utils:4.3.0 #4.1.0 if using CS 4.1, 4.2.0 if using CS 4.2
+          image: icr.io/cpopen/cpfs/cpfs-utils:4.6.0 #4.1.0 if using CS 4.1, 4.2.0 if using CS 4.2
           imagePullPolicy: IfNotPresent
           name: zen5-backup-job
           resources:
@@ -50,6 +50,8 @@ spec:
             mountPath: "/zen5"
           - name: ext
             mountPath: /zen5/extensions
+          - name: logs
+            mountPath: /zen5/zen-backup/logs
         dnsPolicy: ClusterFirst
         schedulerName: default-scheduler
         securityContext:
@@ -66,4 +68,6 @@ spec:
             name: zen5-br-configmap
             defaultMode: 0777
         - emptyDir: {}
-          name: ext
+          name: ext
+        - emptyDir: {}
+          name: logs
diff --git a/velero/schedule/zen5-br-scripts-cm.yaml b/velero/schedule/zen5-br-scripts-cm.yaml
@@ -30,6 +30,8 @@ data:
 
     #Setup backup location
     function main {
+        save_log "logs" "backup_log"
+        trap cleanup_log EXIT
         title "Beginning zen 5 backup process in namespace $ZEN_NAMESPACE."
         info "Creating necessary directories"
         mkdir -p $BACKUP_DIR/workspace
@@ -89,6 +91,35 @@ data:
         success "Backup completed."
     }
 
+    function save_log(){
+        local LOG_DIR="$BACKUP_DIR/$1"
+        LOG_FILE="$LOG_DIR/$2_$(date +'%Y%m%d%H%M%S').log"
+
+        if [[ ! -d $LOG_DIR ]]; then
+            mkdir -p "$LOG_DIR"
+        fi
+
+        # Create a named pipe
+        PIPE=$(mktemp -u)
+        mkfifo "$PIPE"
+
+        # Tee the output to both the log file and the terminal
+        tee "$LOG_FILE" < "$PIPE" &
+
+        # Redirect stdout and stderr to the named pipe
+        exec > "$PIPE" 2>&1
+
+        # Remove the named pipe
+        rm "$PIPE"
+    }
+
+    function cleanup_log() {
+        # Check if the log file already exists
+        if [[ -e $LOG_FILE ]]; then
+            # Remove ANSI escape sequences from log file
+            sed -E 's/\x1B\[[0-9;]+[A-Za-z]//g' "$LOG_FILE" > "$LOG_FILE.tmp" && mv "$LOG_FILE.tmp" "$LOG_FILE"
+        fi
+    }
 
     function msg() {
         printf '%b\n' "$1"
@@ -138,7 +169,11 @@ data:
     BACKUP_DIR=/zen5/zen-backup
 
     function main {
+        save_log "logs" "restore_log"
+        trap cleanup_log EXIT
         title "Beginning restore process for zenservice $ZENSERVICE_NAME in namespace $ZEN_NAMESPACE."
+        info "Wait for zenservice $ZENSERVICE_NAME to be ready..."
+        wait_for_zenservice
         info "Enabling Zen operator maintenance mode and scale down deployments"
         #[2.2.2] Enable Zen operator maintenance mode and scale down deployments
         oc patch zenservice ${ZENSERVICE_NAME} --namespace ${ZEN_NAMESPACE} --type=merge --patch '{"spec": {"ignoreForMaintenance": true}}'
@@ -286,6 +321,60 @@ data:
         success "Restore complete for zenservice $ZENSERVICE_NAME in namespace $ZEN_NAMESPACE."
     }
 
+    function wait_for_zenservice {
+        info "Waiting for zenservice $ZENSERVICE_NAME to complete in namespace $ZEN_NAMESPACE."
+        zenservice_exists=$(oc get zenservice $ZENSERVICE_NAME -n $ZEN_NAMESPACE --no-headers || echo fail)
+        if [[ $zenservice_exists != "fail" ]]; then
+            completed=$(oc get zenservice --no-headers $ZENSERVICE_NAME -n $ZEN_NAMESPACE | awk '{print $2}')
+            retry_count=60
+            while [[ $completed != "Completed" ]] && [[ $retry_count > 0 ]]
+            do
+                info "Wait for zenservice $ZENSERVICE_NAME to complete. Try again in 60s."
+                sleep 60
+                completed=$(oc get zenservice --no-headers $ZENSERVICE_NAME -n $ZEN_NAMESPACE | awk '{print $2}')
+                retry_count=$retry_count-1
+            done
+
+            if [[ $retry_count == 0 ]] && [[ $completed != "1/1" ]]; then
+                error "Timed out waiting for zenservice $ZENSERVICE_NAME."
+            else
+                info "Zenservice $ZENSERVICE_NAME ready."
+            fi
+        else
+            error "Zenservice $ZENSERVICE_NAME not present."
+        fi
+    }
+
+    function save_log(){
+        local LOG_DIR="$BACKUP_DIR/$1"
+        LOG_FILE="$LOG_DIR/$2_$(date +'%Y%m%d%H%M%S').log"
+
+        if [[ ! -d $LOG_DIR ]]; then
+            mkdir -p "$LOG_DIR"
+        fi
+
+        # Create a named pipe
+        PIPE=$(mktemp -u)
+        mkfifo "$PIPE"
+
+        # Tee the output to both the log file and the terminal
+        tee "$LOG_FILE" < "$PIPE" &
+
+        # Redirect stdout and stderr to the named pipe
+        exec > "$PIPE" 2>&1
+
+        # Remove the named pipe
+        rm "$PIPE"
+    }
+
+    function cleanup_log() {
+        # Check if the log file already exists
+        if [[ -e $LOG_FILE ]]; then
+            # Remove ANSI escape sequences from log file
+            sed -E 's/\x1B\[[0-9;]+[A-Za-z]//g' "$LOG_FILE" > "$LOG_FILE.tmp" && mv "$LOG_FILE.tmp" "$LOG_FILE"
+        fi
+    }
+
     function msg() {
         printf '%b\n' "$1"
     }

diff --git a/velero/spectrum-fusion/application.yaml b/velero/spectrum-fusion/application.yaml
@@ -0,0 +1,17 @@
+apiVersion: application.isf.ibm.com/v1alpha1
+kind: Application
+metadata:
+  name: cs-application
+  namespace: ibm-spectrum-fusion-ns
+spec:
+  enableDR: false
+  includedNamespaces:
+    - <operator namespace>
+    - <service namespace>
+    - <tenant namespace 1>
+    - <tenant namespace 2>
+    - <cert manager namespace>
+    - <licensing namespace>
+    - openshift-marketplace    
+    - openshift-config
+    - kube-public