Feature/us 521643- Resume capability Support for ZDT upgrade (pegasys…

…tems#628) * custom resume support changes from Pega-Abhinav * US-5121643-2: resume capability support for ZDT * US-5121643-2: resume capability support for ZDT * US-5121643-2: resume capability support for ZDT * US-5121643-2: resume capability support for ZDT * US-5121643-2: resume capability support for ZDT * US-521643 : added comments --------- Co-authored-by: kistc <[email protected]> Co-authored-by: MadhuriArugula <[email protected]>
maracle6 · Oct 5, 2023 · b31d316 · b31d316
1 parent 8298729
commit b31d316
Show file tree

Hide file tree

Showing 15 changed files with 243 additions and 56 deletions.
diff --git a/charts/pega/charts/installer/config/setupDatabase.properties.tmpl b/charts/pega/charts/installer/config/setupDatabase.properties.tmpl
@@ -88,4 +88,7 @@ upgrade.enable.admin=true
 
 custom.jvm.args=-Xmx4g {{ .Env.CUSTOM_JVM_ARGS }}
 
+# Enable the automatic resume parameter to support resuming rules_upgrade from point of failure.
+automatic.resume={{ .Env.AUTOMATIC_RESUME_ENABLED }}
+
 {{ .Env.ADVANCED_SETTINGS }}
diff --git a/charts/pega/charts/installer/templates/_helpers.tpl b/charts/pega/charts/installer/templates/_helpers.tpl
@@ -23,6 +23,7 @@
 {{- define "pegaInstallEnvironmentConfig" -}}pega-install-environment-config{{- end -}}
 {{- define "pegaUpgradeEnvironmentConfig" -}}pega-upgrade-environment-config{{- end -}}
 {{- define "pegaDistributionKitVolume" -}}pega-distribution-kit-volume{{- end -}}
+{{- define "pegaInstallerMountVolume" -}}pega-installer-mount-volume{{- end -}}
 {{- define "k8sWaitForWaitTime" -}}
   {{- if (.Values.global.utilityImages.k8s_wait_for) -}}
     {{- if (.Values.global.utilityImages.k8s_wait_for.waitTimeSeconds) -}}

diff --git a/charts/pega/charts/installer/templates/_pega-installer-job.tpl b/charts/pega/charts/installer/templates/_pega-installer-job.tpl
@@ -36,20 +36,25 @@ spec:
       annotations:
 {{- if .root.Values.podAnnotations}}
 {{ toYaml .root.Values.podAnnotations | indent 8 }}
-{{- end }}     
+{{- end }}
     spec:
       shareProcessNamespace: {{ .root.Values.shareProcessNamespace }}
 {{- if .root.Values.serviceAccountName }}
       serviceAccountName: {{ .root.Values.serviceAccountName }}
-{{- end }}   
+{{- end }}
       volumes:
+{{- if .root.Values.installerMountVolumeClaimName }}
+      - name: {{ template "pegaInstallerMountVolume" }}
+        persistentVolumeClaim:
+          claimName: {{ .root.Values.installerMountVolumeClaimName }}
+{{- end }}
 {{- if and .root.Values.distributionKitVolumeClaimName (not .root.Values.distributionKitURL) }}
       - name: {{ template "pegaDistributionKitVolume" }}
         persistentVolumeClaim:
           claimName: {{ .root.Values.distributionKitVolumeClaimName }}
 {{- end }}
 {{- if .root.Values.custom }}{{- if .root.Values.custom.volumes }}
-{{ toYaml .root.Values.custom.volumes | indent 6 }}          
+{{ toYaml .root.Values.custom.volumes | indent 6 }}
 {{- end }}{{- end }}
       - name: {{ template "pegaInstallerCredentialsVolume" }}
         projected:
@@ -107,14 +112,18 @@ spec:
             cpu: "{{ .root.Values.resources.limits.cpu }}"
             memory: "{{ .root.Values.resources.limits.memory }}"
         volumeMounts:
+{{- if .root.Values.installerMountVolumeClaimName }}
+        - name: {{ template "pegaInstallerMountVolume" }}
+          mountPath: "/opt/pega/mount/installer"
+{{- end }}
         # The given mountpath is mapped to volume with the specified name.  The config map files are mounted here.
         - name: {{ template "pegaVolumeInstall" }}
           mountPath: "/opt/pega/config"
         - name: {{ template "pegaInstallerCredentialsVolume" }}
           mountPath: "/opt/pega/secrets"
-{{- if and .root.Values.distributionKitVolumeClaimName (not .root.Values.distributionKitURL) }}          
+{{- if and .root.Values.distributionKitVolumeClaimName (not .root.Values.distributionKitURL) }}
         - name: {{ template "pegaDistributionKitVolume" }}
-          mountPath: "/opt/pega/mount/kit"                           
+          mountPath: "/opt/pega/mount/kit"
 {{- end }}
 {{- if .root.Values.custom }}
 {{- if .root.Values.custom.volumeMounts }}
@@ -159,9 +168,9 @@ spec:
 {{- end }}
 {{- if .root.Values.sidecarContainers }}
 {{ toYaml .root.Values.sidecarContainers | indent 6 }}
-{{- end }}                
+{{- end }}
       restartPolicy: Never
       imagePullSecrets:
 {{- include "imagePullSecrets" .root | indent 6 }}
 ---
-{{- end -}}
+{{- end -}}
diff --git a/charts/pega/charts/installer/templates/pega-installer-action-validate.yaml b/charts/pega/charts/installer/templates/pega-installer-action-validate.yaml
@@ -5,7 +5,7 @@
 {{- end }}
 {{- end }}
 {{- if (eq (include "performUpgradeAndDeployment" .) "true") }}
-{{- if not (or (eq .Values.upgrade.upgradeType "zero-downtime") (eq .Values.upgrade.upgradeType "out-of-place")) }}
+{{- if not (or (eq .Values.upgrade.upgradeType "zero-downtime") (eq .Values.upgrade.upgradeType "out-of-place") (eq .Values.upgrade.upgradeType "custom") ) }}
 {{- fail "Upgrade Type value is not correct for upgrade-deploy action. Supported value is 'zero-downtime' and @deprecated 'out-of-place' "  }}
 {{- end }}
 {{- end }}
diff --git a/charts/pega/charts/installer/templates/pega-installer-job.yaml b/charts/pega/charts/installer/templates/pega-installer-job.yaml
@@ -4,15 +4,17 @@
 {{ if (and  (eq (include "performOnlyUpgrade" .) "true") (eq .Values.upgrade.upgradeType "in-place"))  }}
 {{ template "pega.installer" dict "root" $ "name" (include "pegaDBInPlaceUpgrade" .) "action" "upgrade" }}
 {{ end }}
-{{ if (eq (include "performUpgradeAndDeployment" .) "true") }}
+# allowing ZDT to trigger only incase of action upgrade-deploy and upgradeType as zero-downtime
+{{ if ( and (eq (include "performUpgradeAndDeployment" .) "true") (eq .Values.upgrade.upgradeType "zero-downtime")) }}
 {{ template "pega.installer" dict "root" $ "name" (include "pegaPreDBUpgrade" .) "action" "pre-upgrade" }}
 {{ template "pega.installer" dict "root" $ "name" (include "pegaDBZDTUpgrade" .) "action" "upgrade" "initContainers" (list "waitForPreDBUpgrade") }}
 {{ template "pega.installer" dict "root" $ "name" (include "pegaPostDBUpgrade" .) "action" "post-upgrade" "initContainers" (list "waitForPegaDBZDTUpgrade" "waitForRollingUpdates") }}
 {{ end }}
 {{ if ( and (eq (include "performOnlyUpgrade" .) "true") (eq .Values.upgrade.upgradeType "out-of-place")) }}
 {{ template "pega.installer" dict "root" $ "name" (include "pegaDBOOPUpgrade" .) "action" "upgrade" }}
 {{ end }}
-{{ if ( and  (eq (include "performOnlyUpgrade" .) "true") (eq .Values.upgrade.upgradeType "custom")) }}
+# enabled custom upgrade in case of upgrade-deploy , this allows ZDT to recover from failure and can switch over to custom and perfrom rest of steps in upgrade process
+{{ if ( and  ( or (eq (include "performUpgradeAndDeployment" .) "true") (eq (include "performOnlyUpgrade" .) "true") ) (eq .Values.upgrade.upgradeType "custom")) }}
 {{ template "pega.installer" dict "root" $ "name" (include "pegaDBCustomUpgrade" .) "action" "upgrade" }}
 {{ end }}
 {{ if (and  (eq (include "performOnlyUpgrade" .) "true") (eq .Values.upgrade.upgradeType "out-of-place-rules")) }}

diff --git a/charts/pega/charts/installer/templates/pega-upgrade-environment-config.yaml b/charts/pega/charts/installer/templates/pega-upgrade-environment-config.yaml
@@ -77,6 +77,8 @@ data:
   RUN_RULESET_CLEANUP: {{ .Values.upgrade.runRulesetCleanup | quote }}
   # Rebuild Database Rules Indexes after Rules Load to improve Database Access Performance
   REBUILD_INDEXES:  {{ .Values.upgrade.rebuildIndexes | quote }}
+  # Automatic resume parameter to support resuming rules_upgrade from point of failure
+  AUTOMATIC_RESUME_ENABLED: {{ .Values.upgrade.automaticResumeEnabled | quote }}
 {{- if .Values.distributionKitURL }}
   # Distribution kit URL
   DISTRIBUTION_KIT_URL: {{ .Values.distributionKitURL }}

diff --git a/charts/pega/charts/installer/values.yaml b/charts/pega/charts/installer/values.yaml
@@ -20,6 +20,10 @@ assembler: ""
 bypassTruncateUpdatescache: "false"
 # Distribution kit URL
 distributionKitURL: ""
+# A manually managed Persistent Volume Claim for mounting upgrade artifacts. You must create this PVC manually before you bind the volume.
+# The installer job persists upgrade artifacts to this Persistent Volume to support automatically resuming rules_upgrade from point of failure when you use custom upgradeType.
+# To use this function, set the automaticResumeEnabled parameter to true.
+installerMountVolumeClaimName: ""
 # A manually managed Persistent Volume Claim for mounting distribution kit. Given PVC must be created manually before volume will be bound.
 # This as an alternative to distributionKitURL. If both are specified then distributionKitURL will take precedence.
 # Only distriubtion kit zip file is expected inside the volume.
@@ -76,6 +80,8 @@ upgrade:
   pegaRESTServerURL: ""
   pegaRESTUsername: ""
   pegaRESTPassword: ""
+  # Specify automaticResumeEnabled to support resuming rules_upgrade from point of failure. You can use this functionality only when you use "custom" upgradeType.
+  automaticResumeEnabled: "false"
 
 # Memory and CPU settings for installer
 resources:

diff --git a/charts/pega/templates/pega-tier-deployment.yaml b/charts/pega/templates/pega-tier-deployment.yaml
@@ -23,7 +23,7 @@
 {{ end }}
 
 {{ if (eq (include "performInstallAndDeployment" $) "true") }}
-{{ $containerWaitList = append $containerWaitList "waitForPegaDBInstall" }} 
+{{ $containerWaitList = append $containerWaitList "waitForPegaDBInstall" }}
 {{ if not $.Values.pegasearch.externalSearchService }}
 {{ $containerWaitList = append $containerWaitList "waitForPegaSearch" }}
 {{ end }}
@@ -32,7 +32,8 @@
 {{ end }}
 {{ end }}
 
-{{ if (eq (include "performUpgradeAndDeployment" $) "true") }}
+# Allowing ZDT wait containers to trigger only in case of upgradeType as zero-downtime, so that custom upgrade moves forward in case of failure in ZDT
+{{ if ( and  (eq (include "performUpgradeAndDeployment" $) "true") (eq $.Values.installer.upgrade.upgradeType "zero-downtime")) }}
 {{ $containerWaitList = append $containerWaitList "waitForPegaDBZDTUpgrade" }}
 {{ end }}
 

diff --git a/docs/resuming-rules-upgrade-failures.md b/docs/resuming-rules-upgrade-failures.md
@@ -0,0 +1,20 @@
+# Resuming failed upgrades from point of failures
+
+## You can only resume rules_upgrade failures from point of failure when you set the upgradeType parameter to “custom” when you upgrade.
+
+## Client-required steps
+
+### Steps to enable resume functionality
+- Specify `action.execute: upgrade` to upgrade your application using the software version contained in your Pega-provided "installer" image.
+- Specify `installer.upgrade.upgradeType: custom`
+- Specify `installer.upgrade.upgradeSteps: rules_upgrade` to run rules_upgrade
+- Provide a Persistent Volume Claim name in the `installer.installerMountVolumeClaimName` parameter. This is a client-managed PVC for mounting upgrade artifacts.
+- You must create the PVC manually in the same namespace where you deploy Pega Platform before the volume will be bound.
+- Set `installer.upgrade.automaticResumeEnabled` to `true` to enable this functionality
+- Run the upgrade process by using the `helm upgrade release --namespace mypega` command. For more information, see  [Upgrading your Pega Platform deployment using the command line](https://github.com/pegasystems/pega-helm-charts/blob/master/docs/upgrading-pega-deployment-zero-downtime.md#upgrading-your-pega-platform-deployment-using-the-command-line).
+
+### Steps to resume failed rules_upgrade
+- To debug upgrade failure, follow the instructions in the README section  - [Debugging failed upgrades using helm commands](https://github.com/pegasystems/pega-helm-charts/blob/master/README.md#debugging-failed-upgrades-using-helm-commands)
+- Identify the root cause and make necessary changes to fix the issue before resuming the upgrade
+- Resume the upgrade process by using the `helm upgrade release --namespace mypega` command. For more information, see - [Upgrading your Pega Platform deployment using the command line](https://github.com/pegasystems/pega-helm-charts/blob/master/docs/upgrading-pega-deployment-zero-downtime.md#upgrading-your-pega-platform-deployment-using-the-command-line).
+- The upgrade skips steps that were completed successfully in the previous run.