From b31d316c489f10e2e0669cfad6c7c02492765e4a Mon Sep 17 00:00:00 2001 From: chandraprakash kistaiahgari Date: Thu, 5 Oct 2023 18:10:40 +0530 Subject: [PATCH] Feature/us 521643- Resume capability Support for ZDT upgrade (#628) * custom resume support changes from Pega-Abhinav * US-5121643-2: resume capability support for ZDT * US-5121643-2: resume capability support for ZDT * US-5121643-2: resume capability support for ZDT * US-5121643-2: resume capability support for ZDT * US-5121643-2: resume capability support for ZDT * US-521643 : added comments --------- Co-authored-by: kistc Co-authored-by: MadhuriArugula --- .../config/setupDatabase.properties.tmpl | 3 + .../charts/installer/templates/_helpers.tpl | 1 + .../templates/_pega-installer-job.tpl | 23 +++-- .../pega-installer-action-validate.yaml | 2 +- .../templates/pega-installer-job.yaml | 6 +- .../pega-upgrade-environment-config.yaml | 2 + charts/pega/charts/installer/values.yaml | 6 ++ .../pega/templates/pega-tier-deployment.yaml | 5 +- docs/resuming-rules-upgrade-failures.md | 20 +++++ ...upgrading-pega-deployment-zero-downtime.md | 31 ++++++- .../data/expectedSetupdatabase.properties | 3 + .../values_with_automatic_resume_enabled.yaml | 31 +++++++ ...pega-custom-upgrade-resume-enabled_test.go | 85 +++++++++++++++++++ .../src/test/pega/pega-custom-upgrade_test.go | 2 +- .../pega-upgrade-environment-config_test.go | 79 ++++++++--------- 15 files changed, 243 insertions(+), 56 deletions(-) create mode 100644 docs/resuming-rules-upgrade-failures.md create mode 100644 terratest/src/test/pega/data/values_with_automatic_resume_enabled.yaml create mode 100644 terratest/src/test/pega/pega-custom-upgrade-resume-enabled_test.go diff --git a/charts/pega/charts/installer/config/setupDatabase.properties.tmpl b/charts/pega/charts/installer/config/setupDatabase.properties.tmpl index 7c5a0a44b..593d03930 100644 --- a/charts/pega/charts/installer/config/setupDatabase.properties.tmpl +++ b/charts/pega/charts/installer/config/setupDatabase.properties.tmpl @@ -88,4 +88,7 @@ upgrade.enable.admin=true custom.jvm.args=-Xmx4g {{ .Env.CUSTOM_JVM_ARGS }} +# Enable the automatic resume parameter to support resuming rules_upgrade from point of failure. +automatic.resume={{ .Env.AUTOMATIC_RESUME_ENABLED }} + {{ .Env.ADVANCED_SETTINGS }} \ No newline at end of file diff --git a/charts/pega/charts/installer/templates/_helpers.tpl b/charts/pega/charts/installer/templates/_helpers.tpl index 3150effb8..ca7a6926b 100644 --- a/charts/pega/charts/installer/templates/_helpers.tpl +++ b/charts/pega/charts/installer/templates/_helpers.tpl @@ -23,6 +23,7 @@ {{- define "pegaInstallEnvironmentConfig" -}}pega-install-environment-config{{- end -}} {{- define "pegaUpgradeEnvironmentConfig" -}}pega-upgrade-environment-config{{- end -}} {{- define "pegaDistributionKitVolume" -}}pega-distribution-kit-volume{{- end -}} +{{- define "pegaInstallerMountVolume" -}}pega-installer-mount-volume{{- end -}} {{- define "k8sWaitForWaitTime" -}} {{- if (.Values.global.utilityImages.k8s_wait_for) -}} {{- if (.Values.global.utilityImages.k8s_wait_for.waitTimeSeconds) -}} diff --git a/charts/pega/charts/installer/templates/_pega-installer-job.tpl b/charts/pega/charts/installer/templates/_pega-installer-job.tpl index d2f5f5a6c..5e15023f7 100644 --- a/charts/pega/charts/installer/templates/_pega-installer-job.tpl +++ b/charts/pega/charts/installer/templates/_pega-installer-job.tpl @@ -36,20 +36,25 @@ spec: annotations: {{- if .root.Values.podAnnotations}} {{ toYaml .root.Values.podAnnotations | indent 8 }} -{{- end }} +{{- end }} spec: shareProcessNamespace: {{ .root.Values.shareProcessNamespace }} {{- if .root.Values.serviceAccountName }} serviceAccountName: {{ .root.Values.serviceAccountName }} -{{- end }} +{{- end }} volumes: +{{- if .root.Values.installerMountVolumeClaimName }} + - name: {{ template "pegaInstallerMountVolume" }} + persistentVolumeClaim: + claimName: {{ .root.Values.installerMountVolumeClaimName }} +{{- end }} {{- if and .root.Values.distributionKitVolumeClaimName (not .root.Values.distributionKitURL) }} - name: {{ template "pegaDistributionKitVolume" }} persistentVolumeClaim: claimName: {{ .root.Values.distributionKitVolumeClaimName }} {{- end }} {{- if .root.Values.custom }}{{- if .root.Values.custom.volumes }} -{{ toYaml .root.Values.custom.volumes | indent 6 }} +{{ toYaml .root.Values.custom.volumes | indent 6 }} {{- end }}{{- end }} - name: {{ template "pegaInstallerCredentialsVolume" }} projected: @@ -107,14 +112,18 @@ spec: cpu: "{{ .root.Values.resources.limits.cpu }}" memory: "{{ .root.Values.resources.limits.memory }}" volumeMounts: +{{- if .root.Values.installerMountVolumeClaimName }} + - name: {{ template "pegaInstallerMountVolume" }} + mountPath: "/opt/pega/mount/installer" +{{- end }} # The given mountpath is mapped to volume with the specified name. The config map files are mounted here. - name: {{ template "pegaVolumeInstall" }} mountPath: "/opt/pega/config" - name: {{ template "pegaInstallerCredentialsVolume" }} mountPath: "/opt/pega/secrets" -{{- if and .root.Values.distributionKitVolumeClaimName (not .root.Values.distributionKitURL) }} +{{- if and .root.Values.distributionKitVolumeClaimName (not .root.Values.distributionKitURL) }} - name: {{ template "pegaDistributionKitVolume" }} - mountPath: "/opt/pega/mount/kit" + mountPath: "/opt/pega/mount/kit" {{- end }} {{- if .root.Values.custom }} {{- if .root.Values.custom.volumeMounts }} @@ -159,9 +168,9 @@ spec: {{- end }} {{- if .root.Values.sidecarContainers }} {{ toYaml .root.Values.sidecarContainers | indent 6 }} -{{- end }} +{{- end }} restartPolicy: Never imagePullSecrets: {{- include "imagePullSecrets" .root | indent 6 }} --- -{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/pega/charts/installer/templates/pega-installer-action-validate.yaml b/charts/pega/charts/installer/templates/pega-installer-action-validate.yaml index 0728178c4..a8a40d4a1 100644 --- a/charts/pega/charts/installer/templates/pega-installer-action-validate.yaml +++ b/charts/pega/charts/installer/templates/pega-installer-action-validate.yaml @@ -5,7 +5,7 @@ {{- end }} {{- end }} {{- if (eq (include "performUpgradeAndDeployment" .) "true") }} -{{- if not (or (eq .Values.upgrade.upgradeType "zero-downtime") (eq .Values.upgrade.upgradeType "out-of-place")) }} +{{- if not (or (eq .Values.upgrade.upgradeType "zero-downtime") (eq .Values.upgrade.upgradeType "out-of-place") (eq .Values.upgrade.upgradeType "custom") ) }} {{- fail "Upgrade Type value is not correct for upgrade-deploy action. Supported value is 'zero-downtime' and @deprecated 'out-of-place' " }} {{- end }} {{- end }} diff --git a/charts/pega/charts/installer/templates/pega-installer-job.yaml b/charts/pega/charts/installer/templates/pega-installer-job.yaml index 92498a26c..849f7b15b 100644 --- a/charts/pega/charts/installer/templates/pega-installer-job.yaml +++ b/charts/pega/charts/installer/templates/pega-installer-job.yaml @@ -4,7 +4,8 @@ {{ if (and (eq (include "performOnlyUpgrade" .) "true") (eq .Values.upgrade.upgradeType "in-place")) }} {{ template "pega.installer" dict "root" $ "name" (include "pegaDBInPlaceUpgrade" .) "action" "upgrade" }} {{ end }} -{{ if (eq (include "performUpgradeAndDeployment" .) "true") }} +# allowing ZDT to trigger only incase of action upgrade-deploy and upgradeType as zero-downtime +{{ if ( and (eq (include "performUpgradeAndDeployment" .) "true") (eq .Values.upgrade.upgradeType "zero-downtime")) }} {{ template "pega.installer" dict "root" $ "name" (include "pegaPreDBUpgrade" .) "action" "pre-upgrade" }} {{ template "pega.installer" dict "root" $ "name" (include "pegaDBZDTUpgrade" .) "action" "upgrade" "initContainers" (list "waitForPreDBUpgrade") }} {{ template "pega.installer" dict "root" $ "name" (include "pegaPostDBUpgrade" .) "action" "post-upgrade" "initContainers" (list "waitForPegaDBZDTUpgrade" "waitForRollingUpdates") }} @@ -12,7 +13,8 @@ {{ if ( and (eq (include "performOnlyUpgrade" .) "true") (eq .Values.upgrade.upgradeType "out-of-place")) }} {{ template "pega.installer" dict "root" $ "name" (include "pegaDBOOPUpgrade" .) "action" "upgrade" }} {{ end }} -{{ if ( and (eq (include "performOnlyUpgrade" .) "true") (eq .Values.upgrade.upgradeType "custom")) }} +# enabled custom upgrade in case of upgrade-deploy , this allows ZDT to recover from failure and can switch over to custom and perfrom rest of steps in upgrade process +{{ if ( and ( or (eq (include "performUpgradeAndDeployment" .) "true") (eq (include "performOnlyUpgrade" .) "true") ) (eq .Values.upgrade.upgradeType "custom")) }} {{ template "pega.installer" dict "root" $ "name" (include "pegaDBCustomUpgrade" .) "action" "upgrade" }} {{ end }} {{ if (and (eq (include "performOnlyUpgrade" .) "true") (eq .Values.upgrade.upgradeType "out-of-place-rules")) }} diff --git a/charts/pega/charts/installer/templates/pega-upgrade-environment-config.yaml b/charts/pega/charts/installer/templates/pega-upgrade-environment-config.yaml index 6005961f6..9dfc80e43 100644 --- a/charts/pega/charts/installer/templates/pega-upgrade-environment-config.yaml +++ b/charts/pega/charts/installer/templates/pega-upgrade-environment-config.yaml @@ -77,6 +77,8 @@ data: RUN_RULESET_CLEANUP: {{ .Values.upgrade.runRulesetCleanup | quote }} # Rebuild Database Rules Indexes after Rules Load to improve Database Access Performance REBUILD_INDEXES: {{ .Values.upgrade.rebuildIndexes | quote }} + # Automatic resume parameter to support resuming rules_upgrade from point of failure + AUTOMATIC_RESUME_ENABLED: {{ .Values.upgrade.automaticResumeEnabled | quote }} {{- if .Values.distributionKitURL }} # Distribution kit URL DISTRIBUTION_KIT_URL: {{ .Values.distributionKitURL }} diff --git a/charts/pega/charts/installer/values.yaml b/charts/pega/charts/installer/values.yaml index f2f88e6a2..fb7ad3d1a 100644 --- a/charts/pega/charts/installer/values.yaml +++ b/charts/pega/charts/installer/values.yaml @@ -20,6 +20,10 @@ assembler: "" bypassTruncateUpdatescache: "false" # Distribution kit URL distributionKitURL: "" +# A manually managed Persistent Volume Claim for mounting upgrade artifacts. You must create this PVC manually before you bind the volume. +# The installer job persists upgrade artifacts to this Persistent Volume to support automatically resuming rules_upgrade from point of failure when you use custom upgradeType. +# To use this function, set the automaticResumeEnabled parameter to true. +installerMountVolumeClaimName: "" # A manually managed Persistent Volume Claim for mounting distribution kit. Given PVC must be created manually before volume will be bound. # This as an alternative to distributionKitURL. If both are specified then distributionKitURL will take precedence. # Only distriubtion kit zip file is expected inside the volume. @@ -76,6 +80,8 @@ upgrade: pegaRESTServerURL: "" pegaRESTUsername: "" pegaRESTPassword: "" + # Specify automaticResumeEnabled to support resuming rules_upgrade from point of failure. You can use this functionality only when you use "custom" upgradeType. + automaticResumeEnabled: "false" # Memory and CPU settings for installer resources: diff --git a/charts/pega/templates/pega-tier-deployment.yaml b/charts/pega/templates/pega-tier-deployment.yaml index 6c3e3afe1..7f67fa5ab 100644 --- a/charts/pega/templates/pega-tier-deployment.yaml +++ b/charts/pega/templates/pega-tier-deployment.yaml @@ -23,7 +23,7 @@ {{ end }} {{ if (eq (include "performInstallAndDeployment" $) "true") }} -{{ $containerWaitList = append $containerWaitList "waitForPegaDBInstall" }} +{{ $containerWaitList = append $containerWaitList "waitForPegaDBInstall" }} {{ if not $.Values.pegasearch.externalSearchService }} {{ $containerWaitList = append $containerWaitList "waitForPegaSearch" }} {{ end }} @@ -32,7 +32,8 @@ {{ end }} {{ end }} -{{ if (eq (include "performUpgradeAndDeployment" $) "true") }} +# Allowing ZDT wait containers to trigger only in case of upgradeType as zero-downtime, so that custom upgrade moves forward in case of failure in ZDT +{{ if ( and (eq (include "performUpgradeAndDeployment" $) "true") (eq $.Values.installer.upgrade.upgradeType "zero-downtime")) }} {{ $containerWaitList = append $containerWaitList "waitForPegaDBZDTUpgrade" }} {{ end }} diff --git a/docs/resuming-rules-upgrade-failures.md b/docs/resuming-rules-upgrade-failures.md new file mode 100644 index 000000000..c45d1cae9 --- /dev/null +++ b/docs/resuming-rules-upgrade-failures.md @@ -0,0 +1,20 @@ +# Resuming failed upgrades from point of failures + +## You can only resume rules_upgrade failures from point of failure when you set the upgradeType parameter to “custom” when you upgrade. + +## Client-required steps + +### Steps to enable resume functionality +- Specify `action.execute: upgrade` to upgrade your application using the software version contained in your Pega-provided "installer" image. +- Specify `installer.upgrade.upgradeType: custom` +- Specify `installer.upgrade.upgradeSteps: rules_upgrade` to run rules_upgrade +- Provide a Persistent Volume Claim name in the `installer.installerMountVolumeClaimName` parameter. This is a client-managed PVC for mounting upgrade artifacts. +- You must create the PVC manually in the same namespace where you deploy Pega Platform before the volume will be bound. +- Set `installer.upgrade.automaticResumeEnabled` to `true` to enable this functionality +- Run the upgrade process by using the `helm upgrade release --namespace mypega` command. For more information, see [Upgrading your Pega Platform deployment using the command line](https://github.com/pegasystems/pega-helm-charts/blob/master/docs/upgrading-pega-deployment-zero-downtime.md#upgrading-your-pega-platform-deployment-using-the-command-line). + +### Steps to resume failed rules_upgrade +- To debug upgrade failure, follow the instructions in the README section - [Debugging failed upgrades using helm commands](https://github.com/pegasystems/pega-helm-charts/blob/master/README.md#debugging-failed-upgrades-using-helm-commands) +- Identify the root cause and make necessary changes to fix the issue before resuming the upgrade +- Resume the upgrade process by using the `helm upgrade release --namespace mypega` command. For more information, see - [Upgrading your Pega Platform deployment using the command line](https://github.com/pegasystems/pega-helm-charts/blob/master/docs/upgrading-pega-deployment-zero-downtime.md#upgrading-your-pega-platform-deployment-using-the-command-line). +- The upgrade skips steps that were completed successfully in the previous run. \ No newline at end of file diff --git a/docs/upgrading-pega-deployment-zero-downtime.md b/docs/upgrading-pega-deployment-zero-downtime.md index 8e7b82149..a4d9326af 100644 --- a/docs/upgrading-pega-deployment-zero-downtime.md +++ b/docs/upgrading-pega-deployment-zero-downtime.md @@ -112,8 +112,10 @@ To complete an upgrade with zero downtime, configure the following settings in - In the installer section of the Helm chart, update the following: + - Specify `installer.installerMountVolumeClaimName` persistent Volume Claim name. This is a client-managed PVC for mounting upgrade artifacts. - Specify `installer.upgradeType: "Zero-downtime"` to use the zero-downtime upgrade process. - Specify `installer.targetRulesSchema: ""` and `installer.targetDataSchema: ""` for the new target and data schema name that the process creates in your existing database for the upgrade process. + - Specify `installer.upgrade.automaticResumeEnabled` to support resuming from point of failure You can leave the existing customized parameters as is; the upgrade process will use the remaining existing settings in your deployment. @@ -126,8 +128,8 @@ You can leave the existing customized parameters as is; the upgrade process will | Chart parameter name | Purpose | Your setting | |-------------------------|-------------------------------------------|--------------| | actions.execute: | To upgrade using the zero-downtime upgrade process, specify an “upgrade-deploy” deployment type. | execute: "upgrade-deploy" | - | jdbc.rulesSchema: "YOUR_RULES_SCHEMA" | Specify the name of the existing rules schema from which the upgrade process migrates the existing rules structure to your new rules schema. | rulesSchema: "YOUR_RULES_SCHEMA" | - | jdbc.dataSchema: "YOUR_DATA_SCHEMA" | Specify the name of the existing data schema to which the upgrade process migrates the existing data structure from the existing data schema | dataSchema: "YOUR_DATA_SCHEMA" | + | jdbc.rulesSchema: "YOUR_RULES_SCHEMA" | Specify the name of the existing rules schema from which the upgrade process migrates the existing rules structure to your new rules schema. | rulesSchema: "YOUR_RULES_SCHEMA" | + | jdbc.dataSchema: "YOUR_DATA_SCHEMA" | Specify the name of the existing data schema to which the upgrade process migrates the existing data structure from the existing data schema | dataSchema: "YOUR_DATA_SCHEMA" | | Optional: jdbc.customerDataSchema: "YOUR_DATA_SCHEMA" | Optionally specify the name of a data schema separate from case data that is for your business and customer data. The `customerDataSchema` parameter defaults to the value of `dataSchema` if you leave this set to default value. | customerDataSchema : "YOUR_DATA_SCHEMA" | | docker.registry.url: username: and password: | If using a new registry since you installed Pega Platform, update the host name of a registry to an object that contains the “username” and “password” values for that registry. For more information, search for “index.docker.io/v1” in [Engine API v1.24](https://docs.docker.com/engine/api/v1.24/). You can skip this section if the registry is the same as your initial installation. |
  • url: “”
  • username: "\"
  • password: "\< DockerHub account password\>"
| | docker.pega.image: | Update the tagging details, including the version and date of your latest Pega-provided `platform/pega` Docker image that you downloaded and pushed to your Docker registry. This image should match the version of the installer image with which you complete the upgrade. | Image: "\/my-pega:\" | @@ -135,11 +137,13 @@ You can leave the existing customized parameters as is; the upgrade process will | pegasearch.image: | Update the tagging details, including the version and date of your latest Pega-provided `platform/pega` Docker image that you downloaded and pushed to your Docker registry. | Image: "\/my-pega-search:\" | installer.image: | Update the tagging details, including the version and date of your latest Pega-provided `platform/installer` Docker image that you downloaded and pushed to your Docker registry. | Image: "\/my-pega-installer:\" | | installer.adminPassword: | Specify an initial administrator@pega.com password for your installation. This will need to be changed at first login. The adminPassword value cannot start with "@".| adminPassword: "\" | + | installer.installerMountVolumeClaimName: | A manually managed Persistent Volume Claim for mounting upgrade artifacts. You must create this PVC manually before you bind the volume.The installer job persists upgrade artifacts to this Persistent Volume to support automatically resuming rules_upgrade from point of failure when you use custom upgradeType.To use this function, set the installer.upgrade.automaticResumeEnabled parameter to true. | installerMountVolumeClaimName: "" | | installer.upgrade.upgradeType | Specify an zero-downtime upgrade to upgrade using the zero-downtime upgrade process. | upgradeType: "zero-downtime" | | installer.upgrade.targetRulesSchema | For upgrades from 8.4.2 and later, specify a new rules schema name within the quotes that the process uses to create the schema in your existing database to support the upgrade process.| targetRulesSchema: "" | | installer.upgrade.targetDataSchema | For upgrades from 8.4.2 and later, specify the new target data schema name within the quotes that the process uses to create the schema in your existing database to support the upgrade process. You must delete this schema from your database after you complete your upgrade. For upgrades starting at earlier versions, you can leave this value empty, as is (do not leave it blank). | targetDataSchema: "" | - | installer.upgrade.pegaRESTUsername | For zero-downtime upgrades, specify username to access Pega REST server within the quotes that the process uses to perform pre-upgrade/post-upgrade actions required for the upgrade orchestration. | pegaRESTUsername: "" | - | installer.upgrade.pegaRESTPassword | For zero-downtime upgrades, specify user's password to access Pega REST server within the quotes that the process uses to perform pre-upgrade/post-upgrade actions required for the upgrade orchestration. | pegaRESTPassword: "" | + | installer.upgrade.pegaRESTUsername | For zero-downtime upgrades, specify username to access Pega REST server within the quotes that the process uses to perform pre-upgrade/post-upgrade actions required for the upgrade orchestration. | pegaRESTUsername: "" | + | installer.upgrade.pegaRESTPassword | For zero-downtime upgrades, specify user's password to access Pega REST server within the quotes that the process uses to perform pre-upgrade/post-upgrade actions required for the upgrade orchestration. | pegaRESTPassword: "" | + | installer.upgrade.automaticResumeEnabled | For zero-downtime upgrades, Specify automaticResumeEnabled to support resuming rules_upgrade from point of failure. You can use this functionality only when you use "custom" upgradeType. | automaticResumeEnabled: "" | 2. Save the file. @@ -180,3 +184,22 @@ In this document, you specify that the Helm chart always “deploys” by using A successful deployment does not show errors across the various workloads. The `mypega--demo` Namespace **Overview** view shows charts of the percentage of complete tiers and resources configurations. A successful deployment has 100% complete **Workloads**. It takes a little over an hour for the upgrade process to upgrade the applicable rules and then perform a rolling reboot of your nodes. + + +### Steps to resume failed rules_upgrade +- To debug upgrade failure, follow the instructions in the README section - [Debugging failed upgrades using helm commands](https://github.com/pegasystems/pega-helm-charts/blob/master/README.md#debugging-failed-upgrades-using-helm-commands) +- Identify the root cause and make necessary changes to fix the issue before resuming the upgrade +- Specify `installer.upgrade.upgradeType: custom` and `installer.upgrade.upgradeSteps: rules_upgrade` to run rules_upgrade +- Resume the upgrade process by using the `helm upgrade release --namespace mypega` command. For more information, see - [Upgrading your Pega Platform deployment using the command line](https://github.com/pegasystems/pega-helm-charts/blob/master/docs/upgrading-pega-deployment-zero-downtime.md#upgrading-your-pega-platform-deployment-using-the-command-line). +- The upgrade skips steps that were completed successfully in the previous run. +- Specify `installer.upgrade.upgradeType: custom` and `installer.upgrade.upgradeSteps: data_upgrade` to run data_upgrade +- Resume the upgrade process by using the `helm upgrade release --namespace mypega` command. For more information, see - [Upgrading your Pega Platform deployment using the command line](https://github.com/pegasystems/pega-helm-charts/blob/master/docs/upgrading-pega-deployment-zero-downtime.md#upgrading-your-pega-platform-deployment-using-the-command-line). +- Specify + - `action.execute: deploy` this action performs rolling restart + - `jdbc.rulesSchema: ""` update `jdbc.rulesSchema` with `installer.targetRulesSchema` which you have used in upgrade process +- Resume the upgrade process by using the `helm upgrade release --namespace mypega` command. For more information, see - [Upgrading your Pega Platform deployment using the command line](https://github.com/pegasystems/pega-helm-charts/blob/master/docs/upgrading-pega-deployment-zero-downtime.md#upgrading-your-pega-platform-deployment-using-the-command-line). +- Specify + - `action.execute: upgrade-deploy` + - `installer.upgrade.upgradeType: custom` + - `installer.upgrade.upgradeSteps: disable_cluster_upgrade` to run disable_cluster_upgrade +- Resume the upgrade process by using the `helm upgrade release --namespace mypega` command. For more information, see - [Upgrading your Pega Platform deployment using the command line](https://github.com/pegasystems/pega-helm-charts/blob/master/docs/upgrading-pega-deployment-zero-downtime.md#upgrading-your-pega-platform-deployment-using-the-command-line). \ No newline at end of file diff --git a/terratest/src/test/pega/data/expectedSetupdatabase.properties b/terratest/src/test/pega/data/expectedSetupdatabase.properties index 7c5a0a44b..593d03930 100644 --- a/terratest/src/test/pega/data/expectedSetupdatabase.properties +++ b/terratest/src/test/pega/data/expectedSetupdatabase.properties @@ -88,4 +88,7 @@ upgrade.enable.admin=true custom.jvm.args=-Xmx4g {{ .Env.CUSTOM_JVM_ARGS }} +# Enable the automatic resume parameter to support resuming rules_upgrade from point of failure. +automatic.resume={{ .Env.AUTOMATIC_RESUME_ENABLED }} + {{ .Env.ADVANCED_SETTINGS }} \ No newline at end of file diff --git a/terratest/src/test/pega/data/values_with_automatic_resume_enabled.yaml b/terratest/src/test/pega/data/values_with_automatic_resume_enabled.yaml new file mode 100644 index 000000000..5dd699517 --- /dev/null +++ b/terratest/src/test/pega/data/values_with_automatic_resume_enabled.yaml @@ -0,0 +1,31 @@ +--- +global: + # Enter your Kubernetes provider. + provider: "YOUR_KUBERNETES_PROVIDER" + deployment: + # Specify the prefix for all Pega nodes (for example, you can replace "pega" with "app1-dev"). + name: "pega" + # Deploy Pega nodes + actions: + execute: "upgrade" +installer: + image: "YOUR_INSTALLER_IMAGE:TAG" + # Set the initial administrator@pega.com password for your installation. Change the password at first login. + # The adminPassword value cannot start with "@". + adminPassword: "ADMIN_PASSWORD" + # Upgrade specific properties + installerMountVolumeClaimName: "test-claim" + upgrade: + # Type of upgrade + # Valid upgradeType values are “in-place”, “zero-downtime”, “custom”, “out-of-place-rules”, “out-of-place-data”. + upgradeType: "custom" + # Specify a name for a target rules schema that the upgrade process creates for patches and upgrades. + targetRulesSchema: "" + # Specify a name for a target data schema that the upgrade process creates for patches and upgrades. + # For PostgreSQL databases that you are upgrading from Pega Platform version 8.4.0 and later + # And for Oracle databases that you are upgrading from Pega Infinity version 8.4.3 and later. + targetDataSchema: "" + # Specify the username and password to access the pre-upgrade Pega Platform to perform pre- and post- actions during zero-downtime upgrades. + pegaRESTUsername: "" + pegaRESTPassword: "" + automaticResumeEnabled: "true" diff --git a/terratest/src/test/pega/pega-custom-upgrade-resume-enabled_test.go b/terratest/src/test/pega/pega-custom-upgrade-resume-enabled_test.go new file mode 100644 index 000000000..25e8ddb89 --- /dev/null +++ b/terratest/src/test/pega/pega-custom-upgrade-resume-enabled_test.go @@ -0,0 +1,85 @@ +package pega + +import ( + "github.com/gruntwork-io/terratest/modules/helm" + "github.com/stretchr/testify/require" + k8sbatch "k8s.io/api/batch/v1" + k8score "k8s.io/api/core/v1" + "path/filepath" + "strings" + "testing" +) + +func TestPegaCustomUpgradeJobResumeEnabled(t *testing.T) { + + var supportedVendors = []string{"eks"} + var supportedOperations = []string{"upgrade", "upgrade-deploy"} + var deploymentNames = []string{"pega"} + var upgradeType = []string{"custom"} + var upgradeSteps = []string{"rules_upgrade"} + + helmChartPath, err := filepath.Abs(PegaHelmChartPath) + require.NoError(t, err) + + for _, vendor := range supportedVendors { + for _, operation := range supportedOperations { + for _, depName := range deploymentNames { + for _, upType := range upgradeType { + for _, upSteps := range upgradeSteps { + var options = &helm.Options{ + ValuesFiles: []string{"data/values_with_automatic_resume_enabled.yaml"}, + SetValues: map[string]string{ + "global.deployment.name": depName, + "global.provider": vendor, + "global.actions.execute": operation, + "installer.upgrade.upgradeType": upType, + "installer.upgrade.upgradeSteps": upSteps, + }, + } + yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-installer-job.yaml"}) + yamlSplit := strings.Split(yamlContent, "---") + assertUpgradeJobJobResumeEnabled(t, yamlSplit[1], pegaDbJob{"pega-db-custom-upgrade", []string{}, "pega-upgrade-environment-config", "pega-installer"}, options) + + } + } + } + } + } +} + +func assertUpgradeJobJobResumeEnabled(t *testing.T, jobYaml string, expectedJob pegaDbJob, options *helm.Options) { + var jobObj k8sbatch.Job + UnmarshalK8SYaml(t, jobYaml, &jobObj) + + jobSpec := jobObj.Spec.Template.Spec + jobContainers := jobObj.Spec.Template.Spec.Containers + + var containerPort int32 = 8080 + + require.Equal(t, jobSpec.Volumes[0].Name, "pega-installer-mount-volume") + require.Equal(t, jobSpec.Volumes[0].VolumeSource.PersistentVolumeClaim.ClaimName, "test-claim") + + if jobContainers[0].Name == "pega-db-upgrade-rules-migration" || jobContainers[0].Name == "pega-db-upgrade-rules-upgrade" || jobContainers[0].Name == "pega-db-upgrade-data-upgrade" { + require.Equal(t, jobContainers[0].Name, "pega-installer") + } + + require.Equal(t, "YOUR_INSTALLER_IMAGE:TAG", jobContainers[0].Image) + require.Equal(t, jobContainers[0].Ports[0].ContainerPort, containerPort) + require.Equal(t, jobContainers[0].VolumeMounts[0].Name, "pega-installer-mount-volume") + require.Equal(t, jobContainers[0].VolumeMounts[0].MountPath, "/opt/pega/mount/installer") + require.Equal(t, jobContainers[0].EnvFrom[0].ConfigMapRef.LocalObjectReference.Name, expectedJob.configMapName) + + require.Equal(t, jobSpec.ImagePullSecrets[0].Name, getObjName(options, "-registry-secret")) + + require.Equal(t, jobSpec.RestartPolicy, k8score.RestartPolicy("Never")) + + actualInitContainers := jobSpec.InitContainers + count := len(actualInitContainers) + actualInitContainerNames := make([]string, count) + for i := 0; i < count; i++ { + actualInitContainerNames[i] = actualInitContainers[i].Name + } + + require.Equal(t, expectedJob.initContainers, actualInitContainerNames) + VerifyInitContainerData(t, actualInitContainers, options) +} diff --git a/terratest/src/test/pega/pega-custom-upgrade_test.go b/terratest/src/test/pega/pega-custom-upgrade_test.go index f0dfd17b5..ded3765df 100644 --- a/terratest/src/test/pega/pega-custom-upgrade_test.go +++ b/terratest/src/test/pega/pega-custom-upgrade_test.go @@ -13,7 +13,7 @@ import ( func TestPegaUpgradeJob(t *testing.T) { var supportedVendors = []string{"k8s", "openshift", "eks", "gke", "aks", "pks"} - var supportedOperations = []string{"upgrade"} + var supportedOperations = []string{"upgrade", "upgrade-deploy"} var deploymentNames = []string{"pega", "myapp-dev"} var upgradeType = []string{"custom"} var upgradeSteps = []string{"rules_migration", "rules_upgrade", "data_upgrade"} diff --git a/terratest/src/test/pega/pega-upgrade-environment-config_test.go b/terratest/src/test/pega/pega-upgrade-environment-config_test.go index efb45ab15..eb8ceef01 100644 --- a/terratest/src/test/pega/pega-upgrade-environment-config_test.go +++ b/terratest/src/test/pega/pega-upgrade-environment-config_test.go @@ -37,8 +37,8 @@ func TestPegaUpgradeEnvironmentConfig(t *testing.T) { } type restURLExpectation struct { - valuesFile string - expectedURL string + valuesFile string + expectedURL string } func TestPegaUpgradeEnvironmentConfig_DetermineRestUrl(t *testing.T) { @@ -46,49 +46,49 @@ func TestPegaUpgradeEnvironmentConfig_DetermineRestUrl(t *testing.T) { var supportedOperations = []string{"upgrade-deploy"} var expectedValues = map[string]string{"PEGA_REST_SERVER_URL": "http://pega-web:80/prweb/PRRestService", "PEGA_REST_USERNAME": "username", "PEGA_REST_PASSWORD": "password"} - var expectations = []restURLExpectation{ - { - valuesFile: "/data/values_zdt_upgrade_http_non_default_port81.yaml", - expectedURL: "http://pega-web:81/prweb/PRRestService", - }, - { - valuesFile: "/data/values_zdt_upgrade_https_default_port.yaml", - expectedURL: "https://pega-web:443/prweb/PRRestService", - }, - { - valuesFile: "/data/values_zdt_upgrade_https_non_default_port.yaml", - expectedURL: "https://pega-web:5443/prweb/PRRestService", - }, - { - valuesFile: "/data/values_zdt_upgrade_http_default_port_renamed_deployment_renamed_tier.yaml", - expectedURL: "http://xxx-webcustom:80/somethingotherthanprweb/PRRestService", - }, - } + var expectations = []restURLExpectation{ + { + valuesFile: "/data/values_zdt_upgrade_http_non_default_port81.yaml", + expectedURL: "http://pega-web:81/prweb/PRRestService", + }, + { + valuesFile: "/data/values_zdt_upgrade_https_default_port.yaml", + expectedURL: "https://pega-web:443/prweb/PRRestService", + }, + { + valuesFile: "/data/values_zdt_upgrade_https_non_default_port.yaml", + expectedURL: "https://pega-web:5443/prweb/PRRestService", + }, + { + valuesFile: "/data/values_zdt_upgrade_http_default_port_renamed_deployment_renamed_tier.yaml", + expectedURL: "http://xxx-webcustom:80/somethingotherthanprweb/PRRestService", + }, + } helmChartPath, err := filepath.Abs(PegaHelmChartPath) require.NoError(t, err) - testsPath, err := filepath.Abs(PegaHelmChartTestsPath) + testsPath, err := filepath.Abs(PegaHelmChartTestsPath) require.NoError(t, err) - for _, expectation := range expectations { - expectedValues["PEGA_REST_SERVER_URL"] = expectation.expectedURL - for _, vendor := range supportedVendors { - for _, operation := range supportedOperations { - var options = &helm.Options{ - SetValues: map[string]string{ - "global.provider": vendor, - "global.actions.execute": operation, - "installer.upgrade.upgradeType": "zero-downtime", - "installer.upgrade.pegaRESTUsername": expectedValues["PEGA_REST_USERNAME"], - "installer.upgrade.pegaRESTPassword": expectedValues["PEGA_REST_PASSWORD"], - }, - } - - yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-upgrade-environment-config.yaml"}, "--values", testsPath+expectation.valuesFile) - assertUpgradeEnvironmentConfig(t, yamlContent, options, expectedValues) - } - } + for _, expectation := range expectations { + expectedValues["PEGA_REST_SERVER_URL"] = expectation.expectedURL + for _, vendor := range supportedVendors { + for _, operation := range supportedOperations { + var options = &helm.Options{ + SetValues: map[string]string{ + "global.provider": vendor, + "global.actions.execute": operation, + "installer.upgrade.upgradeType": "zero-downtime", + "installer.upgrade.pegaRESTUsername": expectedValues["PEGA_REST_USERNAME"], + "installer.upgrade.pegaRESTPassword": expectedValues["PEGA_REST_PASSWORD"], + }, + } + + yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-upgrade-environment-config.yaml"}, "--values", testsPath+expectation.valuesFile) + assertUpgradeEnvironmentConfig(t, yamlContent, options, expectedValues) + } + } } } @@ -124,4 +124,5 @@ func assertUpgradeEnvironmentConfig(t *testing.T, configYaml string, options *he require.Equal(t, upgradeEnvConfigData["PEGA_REST_PASSWORD"], expectedValues["PEGA_REST_PASSWORD"]) require.Equal(t, upgradeEnvConfigData["DISTRIBUTION_KIT_URL"], "") require.Equal(t, upgradeEnvConfigData["ENABLE_CUSTOM_ARTIFACTORY_SSL_VERIFICATION"], "true") + require.Equal(t, upgradeEnvConfigData["AUTOMATIC_RESUME_ENABLED"], "false") }