From 96def55be406e78dd8db17b4d7352645347595b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Rod=C3=A1k?= Date: Thu, 24 Oct 2024 14:01:58 +0200 Subject: [PATCH] Configure HealthCheck with `podman update` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New flags in a `podman update` can change the configuration of HealthCheck when the container is started, without having to restart or recreate the container. This can help determine why a given container suddenly started failing HealthCheck without interfering with the services it provides. For example, reconfigure HealthCheck to keep logs longer than the usual last X results, store logs to other destinations, etc. Fixes: https://issues.redhat.com/browse/RHEL-60561 Signed-off-by: Jan Rodák --- cmd/podman/common/create.go | 252 ++++---- cmd/podman/containers/update.go | 12 +- docs/source/markdown/options/health-cmd.md | 2 +- .../markdown/options/health-interval.md | 2 +- .../options/health-log-destination.md | 2 +- .../markdown/options/health-max-log-count.md | 2 +- .../markdown/options/health-max-log-size.md | 2 +- .../markdown/options/health-on-failure.md | 2 +- .../source/markdown/options/health-retries.md | 2 +- .../markdown/options/health-start-period.md | 2 +- .../markdown/options/health-startup-cmd.md | 2 +- .../options/health-startup-interval.md | 2 +- .../options/health-startup-retries.md | 2 +- .../options/health-startup-success.md | 2 +- .../options/health-startup-timeout.md | 2 +- .../source/markdown/options/health-timeout.md | 2 +- .../source/markdown/options/no-healthcheck.md | 2 +- docs/source/markdown/podman-update.1.md.in | 40 +- libpod/container_api.go | 7 +- libpod/container_internal.go | 327 +++++++++++ libpod/healthcheck.go | 59 +- pkg/api/handlers/compat/containers.go | 5 +- pkg/api/handlers/libpod/containers.go | 11 +- pkg/api/handlers/swagger/models.go | 19 +- pkg/api/handlers/types.go | 4 +- pkg/api/server/register_containers.go | 2 +- pkg/bindings/containers/update.go | 9 +- pkg/domain/entities/containers.go | 2 + pkg/domain/entities/types/containers.go | 23 +- pkg/domain/infra/abi/containers.go | 2 +- pkg/specgenutil/specgen.go | 54 +- test/system/280-update.bats | 548 ++++++++++++++++++ 32 files changed, 1226 insertions(+), 180 deletions(-) diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index 54a2e781f5..c9531479f1 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -168,78 +168,6 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, ) _ = cmd.RegisterFlagCompletionFunc(groupAddFlagName, completion.AutocompleteNone) - healthCmdFlagName := "health-cmd" - createFlags.StringVar( - &cf.HealthCmd, - healthCmdFlagName, "", - "set a healthcheck command for the container ('none' disables the existing healthcheck)", - ) - _ = cmd.RegisterFlagCompletionFunc(healthCmdFlagName, completion.AutocompleteNone) - - healthIntervalFlagName := "health-interval" - createFlags.StringVar( - &cf.HealthInterval, - healthIntervalFlagName, define.DefaultHealthCheckInterval, - "set an interval for the healthcheck (a value of disable results in no automatic timer setup)", - ) - _ = cmd.RegisterFlagCompletionFunc(healthIntervalFlagName, completion.AutocompleteNone) - - healthLogDestinationFlagName := "health-log-destination" - createFlags.StringVar( - &cf.HealthLogDestination, - healthLogDestinationFlagName, define.DefaultHealthCheckLocalDestination, - "set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file)", - ) - _ = cmd.RegisterFlagCompletionFunc(healthLogDestinationFlagName, completion.AutocompleteNone) - - healthMaxLogCountFlagName := "health-max-log-count" - createFlags.UintVar( - &cf.HealthMaxLogCount, - healthMaxLogCountFlagName, define.DefaultHealthMaxLogCount, - "set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file)", - ) - _ = cmd.RegisterFlagCompletionFunc(healthMaxLogCountFlagName, completion.AutocompleteNone) - - healthMaxLogSizeFlagName := "health-max-log-size" - createFlags.UintVar( - &cf.HealthMaxLogSize, - healthMaxLogSizeFlagName, define.DefaultHealthMaxLogSize, - "set maximum length in characters of stored HealthCheck log. ('0' value means an infinite log length)", - ) - _ = cmd.RegisterFlagCompletionFunc(healthMaxLogSizeFlagName, completion.AutocompleteNone) - - healthRetriesFlagName := "health-retries" - createFlags.UintVar( - &cf.HealthRetries, - healthRetriesFlagName, define.DefaultHealthCheckRetries, - "the number of retries allowed before a healthcheck is considered to be unhealthy", - ) - _ = cmd.RegisterFlagCompletionFunc(healthRetriesFlagName, completion.AutocompleteNone) - - healthStartPeriodFlagName := "health-start-period" - createFlags.StringVar( - &cf.HealthStartPeriod, - healthStartPeriodFlagName, define.DefaultHealthCheckStartPeriod, - "the initialization time needed for a container to bootstrap", - ) - _ = cmd.RegisterFlagCompletionFunc(healthStartPeriodFlagName, completion.AutocompleteNone) - - healthTimeoutFlagName := "health-timeout" - createFlags.StringVar( - &cf.HealthTimeout, - healthTimeoutFlagName, define.DefaultHealthCheckTimeout, - "the maximum time allowed to complete the healthcheck before an interval is considered failed", - ) - _ = cmd.RegisterFlagCompletionFunc(healthTimeoutFlagName, completion.AutocompleteNone) - - healthOnFailureFlagName := "health-on-failure" - createFlags.StringVar( - &cf.HealthOnFailure, - healthOnFailureFlagName, "none", - "action to take once the container turns unhealthy", - ) - _ = cmd.RegisterFlagCompletionFunc(healthOnFailureFlagName, AutocompleteHealthOnFailure) - createFlags.BoolVar( &cf.HTTPProxy, "http-proxy", podmanConfig.ContainersConfDefaultsRO.Containers.HTTPProxy, @@ -311,11 +239,6 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, ) _ = cmd.RegisterFlagCompletionFunc(logOptFlagName, AutocompleteLogOpt) - createFlags.BoolVar( - &cf.NoHealthCheck, - "no-healthcheck", false, - "Disable healthchecks on container", - ) createFlags.BoolVar( &cf.OOMKillDisable, "oom-kill-disable", false, @@ -452,46 +375,6 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, ) _ = cmd.RegisterFlagCompletionFunc(secretFlagName, AutocompleteSecrets) - startupHCCmdFlagName := "health-startup-cmd" - createFlags.StringVar( - &cf.StartupHCCmd, - startupHCCmdFlagName, "", - "Set a startup healthcheck command for the container", - ) - _ = cmd.RegisterFlagCompletionFunc(startupHCCmdFlagName, completion.AutocompleteNone) - - startupHCIntervalFlagName := "health-startup-interval" - createFlags.StringVar( - &cf.StartupHCInterval, - startupHCIntervalFlagName, define.DefaultHealthCheckInterval, - "Set an interval for the startup healthcheck", - ) - _ = cmd.RegisterFlagCompletionFunc(startupHCIntervalFlagName, completion.AutocompleteNone) - - startupHCRetriesFlagName := "health-startup-retries" - createFlags.UintVar( - &cf.StartupHCRetries, - startupHCRetriesFlagName, 0, - "Set the maximum number of retries before the startup healthcheck will restart the container", - ) - _ = cmd.RegisterFlagCompletionFunc(startupHCRetriesFlagName, completion.AutocompleteNone) - - startupHCSuccessesFlagName := "health-startup-success" - createFlags.UintVar( - &cf.StartupHCSuccesses, - startupHCSuccessesFlagName, 0, - "Set the number of consecutive successes before the startup healthcheck is marked as successful and the normal healthcheck begins (0 indicates any success will start the regular healthcheck)", - ) - _ = cmd.RegisterFlagCompletionFunc(startupHCSuccessesFlagName, completion.AutocompleteNone) - - startupHCTimeoutFlagName := "health-startup-timeout" - createFlags.StringVar( - &cf.StartupHCTimeout, - startupHCTimeoutFlagName, define.DefaultHealthCheckTimeout, - "Set the maximum amount of time that the startup healthcheck may take before it is considered failed", - ) - _ = cmd.RegisterFlagCompletionFunc(startupHCTimeoutFlagName, completion.AutocompleteNone) - stopSignalFlagName := "stop-signal" createFlags.StringVar( &cf.StopSignal, @@ -665,6 +548,141 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, `If a container with the same name exists, replace it`, ) } + if mode == entities.CreateMode || mode == entities.UpdateMode { + // TODO: Focus on disable + createFlags.BoolVar( + &cf.NoHealthCheck, + "no-healthcheck", false, + "Disable healthchecks on container", + ) + + healthCmdFlagName := "health-cmd" + createFlags.StringVar( + &cf.HealthCmd, + healthCmdFlagName, "", + "set a healthcheck command for the container ('none' disables the existing healthcheck)", + ) + _ = cmd.RegisterFlagCompletionFunc(healthCmdFlagName, completion.AutocompleteNone) + + info := "" + if mode == entities.UpdateMode { + info = "Changing this setting resets timer." + } + healthIntervalFlagName := "health-interval" + createFlags.StringVar( + &cf.HealthInterval, + healthIntervalFlagName, define.DefaultHealthCheckInterval, + "set an interval for the healthcheck. (a value of disable results in no automatic timer setup) "+info, + ) + _ = cmd.RegisterFlagCompletionFunc(healthIntervalFlagName, completion.AutocompleteNone) + + warning := "" + if mode == entities.UpdateMode { + warning = "Warning: Changing this setting may cause the loss of previous logs!" + } + healthLogDestinationFlagName := "health-log-destination" + createFlags.StringVar( + &cf.HealthLogDestination, + healthLogDestinationFlagName, define.DefaultHealthCheckLocalDestination, + "set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file) "+warning, + ) + _ = cmd.RegisterFlagCompletionFunc(healthLogDestinationFlagName, completion.AutocompleteNone) + + healthMaxLogCountFlagName := "health-max-log-count" + createFlags.UintVar( + &cf.HealthMaxLogCount, + healthMaxLogCountFlagName, define.DefaultHealthMaxLogCount, + "set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file)", + ) + _ = cmd.RegisterFlagCompletionFunc(healthMaxLogCountFlagName, completion.AutocompleteNone) + + healthMaxLogSizeFlagName := "health-max-log-size" + createFlags.UintVar( + &cf.HealthMaxLogSize, + healthMaxLogSizeFlagName, define.DefaultHealthMaxLogSize, + "set maximum length in characters of stored HealthCheck log. ('0' value means an infinite log length)", + ) + _ = cmd.RegisterFlagCompletionFunc(healthMaxLogSizeFlagName, completion.AutocompleteNone) + + healthRetriesFlagName := "health-retries" + createFlags.UintVar( + &cf.HealthRetries, + healthRetriesFlagName, define.DefaultHealthCheckRetries, + "the number of retries allowed before a healthcheck is considered to be unhealthy", + ) + _ = cmd.RegisterFlagCompletionFunc(healthRetriesFlagName, completion.AutocompleteNone) + + healthStartPeriodFlagName := "health-start-period" + createFlags.StringVar( + &cf.HealthStartPeriod, + healthStartPeriodFlagName, define.DefaultHealthCheckStartPeriod, + "the initialization time needed for a container to bootstrap", + ) + _ = cmd.RegisterFlagCompletionFunc(healthStartPeriodFlagName, completion.AutocompleteNone) + + healthTimeoutFlagName := "health-timeout" + createFlags.StringVar( + &cf.HealthTimeout, + healthTimeoutFlagName, define.DefaultHealthCheckTimeout, + "the maximum time allowed to complete the healthcheck before an interval is considered failed", + ) + _ = cmd.RegisterFlagCompletionFunc(healthTimeoutFlagName, completion.AutocompleteNone) + + healthOnFailureFlagName := "health-on-failure" + createFlags.StringVar( + &cf.HealthOnFailure, + healthOnFailureFlagName, "none", + "action to take once the container turns unhealthy", + ) + _ = cmd.RegisterFlagCompletionFunc(healthOnFailureFlagName, AutocompleteHealthOnFailure) + + // Startup HealthCheck + + startupHCCmdFlagName := "health-startup-cmd" + createFlags.StringVar( + &cf.StartupHCCmd, + startupHCCmdFlagName, "", + "Set a startup healthcheck command for the container", + ) + _ = cmd.RegisterFlagCompletionFunc(startupHCCmdFlagName, completion.AutocompleteNone) + + info = "" + if mode == entities.UpdateMode { + info = "Changing this setting resets the timer, depending on the state of the container." + } + startupHCIntervalFlagName := "health-startup-interval" + createFlags.StringVar( + &cf.StartupHCInterval, + startupHCIntervalFlagName, define.DefaultHealthCheckInterval, + "Set an interval for the startup healthcheck. "+info, + ) + _ = cmd.RegisterFlagCompletionFunc(startupHCIntervalFlagName, completion.AutocompleteNone) + + startupHCRetriesFlagName := "health-startup-retries" + createFlags.UintVar( + &cf.StartupHCRetries, + startupHCRetriesFlagName, 0, + "Set the maximum number of retries before the startup healthcheck will restart the container", + ) + _ = cmd.RegisterFlagCompletionFunc(startupHCRetriesFlagName, completion.AutocompleteNone) + + startupHCSuccessesFlagName := "health-startup-success" + createFlags.UintVar( + &cf.StartupHCSuccesses, + startupHCSuccessesFlagName, 0, + "Set the number of consecutive successes before the startup healthcheck is marked as successful and the normal healthcheck begins (0 indicates any success will start the regular healthcheck)", + ) + _ = cmd.RegisterFlagCompletionFunc(startupHCSuccessesFlagName, completion.AutocompleteNone) + + startupHCTimeoutFlagName := "health-startup-timeout" + createFlags.StringVar( + &cf.StartupHCTimeout, + startupHCTimeoutFlagName, define.DefaultHealthCheckTimeout, + "Set the maximum amount of time that the startup healthcheck may take before it is considered failed", + ) + _ = cmd.RegisterFlagCompletionFunc(startupHCTimeoutFlagName, completion.AutocompleteNone) + } + // Restart is allowed for created, updated, and infra ctr if mode == entities.InfraMode || mode == entities.CreateMode || mode == entities.UpdateMode { restartFlagName := "restart" diff --git a/cmd/podman/containers/update.go b/cmd/podman/containers/update.go index 9e8e28070a..74cb1f132a 100644 --- a/cmd/podman/containers/update.go +++ b/cmd/podman/containers/update.go @@ -17,7 +17,7 @@ import ( ) var ( - updateDescription = `Updates the cgroup configuration of a given container` + updateDescription = `Updates the configuration of an already existing container, allowing different resource limits to be set, and HealthCheck configuration. The currently supported options are a subset of the podman create/run.` updateCommand = &cobra.Command{ Use: "update [options] CONTAINER", @@ -89,9 +89,15 @@ func update(cmd *cobra.Command, args []string) error { return err } + healthCheckConfig := specgenutil.GetChangedHealthCheckConfiguration(cmd) + if err != nil { + return err + } + opts := &entities.ContainerUpdateOptions{ - NameOrID: strings.TrimPrefix(args[0], "/"), - Specgen: s, + NameOrID: strings.TrimPrefix(args[0], "/"), + Specgen: s, + ChangedHealthCheckConfiguration: &healthCheckConfig, } rep, err := registry.ContainerEngine().ContainerUpdate(context.Background(), opts) if err != nil { diff --git a/docs/source/markdown/options/health-cmd.md b/docs/source/markdown/options/health-cmd.md index a135a2c435..4e09430849 100644 --- a/docs/source/markdown/options/health-cmd.md +++ b/docs/source/markdown/options/health-cmd.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-cmd**=*"command"* | *'["command", "arg1", ...]'* diff --git a/docs/source/markdown/options/health-interval.md b/docs/source/markdown/options/health-interval.md index 9aa86dcd76..0b25517cfa 100644 --- a/docs/source/markdown/options/health-interval.md +++ b/docs/source/markdown/options/health-interval.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-interval**=*interval* diff --git a/docs/source/markdown/options/health-log-destination.md b/docs/source/markdown/options/health-log-destination.md index 16b99ecc4c..91e91e269c 100644 --- a/docs/source/markdown/options/health-log-destination.md +++ b/docs/source/markdown/options/health-log-destination.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-log-destination**=*directory_path* diff --git a/docs/source/markdown/options/health-max-log-count.md b/docs/source/markdown/options/health-max-log-count.md index 96a7d60861..137d470830 100644 --- a/docs/source/markdown/options/health-max-log-count.md +++ b/docs/source/markdown/options/health-max-log-count.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-max-log-count**=*number of stored logs* diff --git a/docs/source/markdown/options/health-max-log-size.md b/docs/source/markdown/options/health-max-log-size.md index 96cc399e4a..1c3169c7f4 100644 --- a/docs/source/markdown/options/health-max-log-size.md +++ b/docs/source/markdown/options/health-max-log-size.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-max-log-size**=*size of stored logs* diff --git a/docs/source/markdown/options/health-on-failure.md b/docs/source/markdown/options/health-on-failure.md index 4075556a2d..6c539e7332 100644 --- a/docs/source/markdown/options/health-on-failure.md +++ b/docs/source/markdown/options/health-on-failure.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-on-failure**=*action* diff --git a/docs/source/markdown/options/health-retries.md b/docs/source/markdown/options/health-retries.md index 224bd0d552..13d68afb95 100644 --- a/docs/source/markdown/options/health-retries.md +++ b/docs/source/markdown/options/health-retries.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-retries**=*retries* diff --git a/docs/source/markdown/options/health-start-period.md b/docs/source/markdown/options/health-start-period.md index 5b1fde4bd5..4391338e46 100644 --- a/docs/source/markdown/options/health-start-period.md +++ b/docs/source/markdown/options/health-start-period.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-start-period**=*period* diff --git a/docs/source/markdown/options/health-startup-cmd.md b/docs/source/markdown/options/health-startup-cmd.md index b3792a584f..67b2db32ad 100644 --- a/docs/source/markdown/options/health-startup-cmd.md +++ b/docs/source/markdown/options/health-startup-cmd.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-startup-cmd**=*"command"* | *'["command", "arg1", ...]'* diff --git a/docs/source/markdown/options/health-startup-interval.md b/docs/source/markdown/options/health-startup-interval.md index dbba969a55..052d703a78 100644 --- a/docs/source/markdown/options/health-startup-interval.md +++ b/docs/source/markdown/options/health-startup-interval.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-startup-interval**=*interval* diff --git a/docs/source/markdown/options/health-startup-retries.md b/docs/source/markdown/options/health-startup-retries.md index db213dcf97..2a0f9fdf90 100644 --- a/docs/source/markdown/options/health-startup-retries.md +++ b/docs/source/markdown/options/health-startup-retries.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-startup-retries**=*retries* diff --git a/docs/source/markdown/options/health-startup-success.md b/docs/source/markdown/options/health-startup-success.md index c8c85e1bfb..e1f911b215 100644 --- a/docs/source/markdown/options/health-startup-success.md +++ b/docs/source/markdown/options/health-startup-success.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-startup-success**=*retries* diff --git a/docs/source/markdown/options/health-startup-timeout.md b/docs/source/markdown/options/health-startup-timeout.md index f6b8c75e07..deffa14025 100644 --- a/docs/source/markdown/options/health-startup-timeout.md +++ b/docs/source/markdown/options/health-startup-timeout.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-startup-timeout**=*timeout* diff --git a/docs/source/markdown/options/health-timeout.md b/docs/source/markdown/options/health-timeout.md index 1324628008..0540a48db3 100644 --- a/docs/source/markdown/options/health-timeout.md +++ b/docs/source/markdown/options/health-timeout.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--health-timeout**=*timeout* diff --git a/docs/source/markdown/options/no-healthcheck.md b/docs/source/markdown/options/no-healthcheck.md index 14704db8aa..9fab16a563 100644 --- a/docs/source/markdown/options/no-healthcheck.md +++ b/docs/source/markdown/options/no-healthcheck.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman create, run +####> podman create, run, update ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--no-healthcheck** diff --git a/docs/source/markdown/podman-update.1.md.in b/docs/source/markdown/podman-update.1.md.in index 9cce804aa9..e6f346a987 100644 --- a/docs/source/markdown/podman-update.1.md.in +++ b/docs/source/markdown/podman-update.1.md.in @@ -10,8 +10,8 @@ podman\-update - Update the configuration of a given container ## DESCRIPTION -Updates the configuration of an already existing container, allowing different resource limits to be set. -The currently supported options are a subset of the podman create/run resource limit options. +Updates the configuration of an already existing container, allowing different resource limits to be set, and HealthCheck configuration. +The currently supported options are a subset of the podman create/run. ## OPTIONS @@ -43,6 +43,40 @@ The currently supported options are a subset of the podman create/run resource l @@option device-write-iops +@@option health-cmd + +@@option health-interval + +Changing this setting resets timer. + +@@option health-log-destination + +Warning: Changing this setting may cause the loss of previous logs. + +@@option health-max-log-count + +@@option health-max-log-size + +@@option health-on-failure + +@@option health-retries + +@@option health-start-period + +@@option health-startup-cmd + +@@option health-startup-interval + +Changing this setting resets the timer, depending on the state of the container. + +@@option health-startup-retries + +@@option health-startup-success + +@@option health-startup-timeout + +@@option health-timeout + @@option memory @@option memory-reservation @@ -51,6 +85,8 @@ The currently supported options are a subset of the podman create/run resource l @@option memory-swappiness +@@option no-healthcheck + @@option pids-limit @@option restart diff --git a/libpod/container_api.go b/libpod/container_api.go index cae6817ec3..0d4ee4b116 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -14,6 +14,7 @@ import ( "github.com/containers/common/pkg/resize" "github.com/containers/podman/v5/libpod/define" "github.com/containers/podman/v5/libpod/events" + "github.com/containers/podman/v5/pkg/domain/entities/types" "github.com/containers/storage/pkg/archive" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -119,7 +120,7 @@ func (c *Container) Start(ctx context.Context, recursive bool) (finalErr error) // Either resource limits or restart policy can be updated. // Either resources or restartPolicy must not be nil. // If restartRetries is not nil, restartPolicy must be set and must be "on-failure". -func (c *Container) Update(resources *spec.LinuxResources, restartPolicy *string, restartRetries *uint) error { +func (c *Container) Update(resources *spec.LinuxResources, restartPolicy *string, restartRetries *uint, changedHealthCheckConfig *types.UpdateHealthCheckConfig) error { if !c.batched { c.lock.Lock() defer c.lock.Unlock() @@ -133,6 +134,10 @@ func (c *Container) Update(resources *spec.LinuxResources, restartPolicy *string return fmt.Errorf("container %s is being removed, cannot update: %w", c.ID(), define.ErrCtrStateInvalid) } + if err := c.updateHealthCheckConfiguration(changedHealthCheckConfig); err != nil { + return err + } + return c.update(resources, restartPolicy, restartRetries) } diff --git a/libpod/container_internal.go b/libpod/container_internal.go index c7efd18e4b..8bd5bf6710 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -30,13 +30,16 @@ import ( "github.com/containers/common/pkg/hooks/exec" "github.com/containers/common/pkg/timezone" cutil "github.com/containers/common/pkg/util" + "github.com/containers/image/v5/manifest" "github.com/containers/podman/v5/libpod/define" "github.com/containers/podman/v5/libpod/events" "github.com/containers/podman/v5/libpod/shutdown" "github.com/containers/podman/v5/pkg/ctime" + "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/pkg/lookup" "github.com/containers/podman/v5/pkg/rootless" "github.com/containers/podman/v5/pkg/selinux" + "github.com/containers/podman/v5/pkg/specgenutil" "github.com/containers/podman/v5/pkg/systemd/notifyproxy" "github.com/containers/podman/v5/pkg/util" "github.com/containers/storage" @@ -2738,3 +2741,327 @@ func (c *Container) update(resources *spec.LinuxResources, restartPolicy *string return nil } + +func (c *Container) getCopyOfHealthCheckAndStartupHelathCheck() (*manifest.Schema2HealthConfig, *define.StartupHealthCheck) { + var healthCheck manifest.Schema2HealthConfig + if c.config.HealthCheckConfig != nil { + healthCheck = manifest.Schema2HealthConfig{ + Test: slices.Clone(c.config.HealthCheckConfig.Test), + Interval: c.config.HealthCheckConfig.Interval, + Retries: c.config.HealthCheckConfig.Retries, + StartInterval: c.config.HealthCheckConfig.StartInterval, + StartPeriod: c.config.HealthCheckConfig.StartPeriod, + Timeout: c.config.HealthCheckConfig.Timeout, + } + } + + var startupHealthCheck define.StartupHealthCheck + if c.config.StartupHealthCheckConfig != nil { + startupHealthCheck = define.StartupHealthCheck{ + Successes: c.config.StartupHealthCheckConfig.Successes, + } + startupHealthCheck.Test = slices.Clone(c.config.StartupHealthCheckConfig.Test) + startupHealthCheck.Interval = c.config.StartupHealthCheckConfig.Interval + startupHealthCheck.Retries = c.config.StartupHealthCheckConfig.Retries + startupHealthCheck.StartInterval = c.config.StartupHealthCheckConfig.StartInterval + startupHealthCheck.StartPeriod = c.config.StartupHealthCheckConfig.StartPeriod + startupHealthCheck.Timeout = c.config.StartupHealthCheckConfig.Timeout + } + return &healthCheck, &startupHealthCheck +} + +func (c *Container) changeHealthCheckConfiguration(changedHealthCheckConfig *entities.UpdateHealthCheckConfig) (bool, error) { + containsHealthCheckCmd := changedHealthCheckConfig.HealthCmd != "" + containsFlags := (changedHealthCheckConfig.HealthInterval != "" || + changedHealthCheckConfig.HealthRetries != "" || + changedHealthCheckConfig.HealthTimeout != "" || + changedHealthCheckConfig.HealthStartPeriod != "") + if c.config.HealthCheckConfig == nil && !containsHealthCheckCmd && containsFlags { + return false, errors.New("healthcheck command is not set") + } + + cmd := "" + interval := define.DefaultHealthCheckInterval + retries := int(define.DefaultHealthCheckRetries) + timeout := define.DefaultHealthCheckTimeout + startPeriod := define.DefaultHealthCheckStartPeriod + + if c.config.HealthCheckConfig != nil { + cmd = strings.Join(c.config.HealthCheckConfig.Test, " ") + interval = c.config.HealthCheckConfig.Interval.String() + retries = c.config.HealthCheckConfig.Retries + timeout = c.config.HealthCheckConfig.Timeout.String() + startPeriod = c.config.HealthCheckConfig.StartPeriod.String() + } + + changed := false + changedTimer := false + noHealthCheck := changedHealthCheckConfig.NoHealthCheck == "true" + + if changedHealthCheckConfig.HealthCmd != "" { + cmd = changedHealthCheckConfig.HealthCmd + changed = true + } + if changedHealthCheckConfig.HealthInterval != "" { + interval = changedHealthCheckConfig.HealthInterval + changed = true + changedTimer = true + } + if changedHealthCheckConfig.HealthRetries != "" { + val, err := strconv.Atoi(changedHealthCheckConfig.HealthRetries) + if err != nil { + return false, err + } + retries = val + changed = true + } + if changedHealthCheckConfig.HealthTimeout != "" { + timeout = changedHealthCheckConfig.HealthTimeout + changed = true + } + if changedHealthCheckConfig.HealthStartPeriod != "" { + startPeriod = changedHealthCheckConfig.HealthStartPeriod + changed = true + } + + switch { + case noHealthCheck && changed: + return false, errors.New("cannot specify both --no-healthcheck and other HealthCheck flags") + case noHealthCheck: + c.config.HealthCheckConfig = &manifest.Schema2HealthConfig{ + Test: []string{"NONE"}, + } + + case changed: + healthCheckConfig_, err := specgenutil.MakeHealthCheckFromCli( + cmd, + interval, + uint(retries), + timeout, + startPeriod, + false, + ) + if err != nil { + return false, err + } + c.config.HealthCheckConfig = healthCheckConfig_ + } + return changedTimer, nil +} + +func (c *Container) changeStartupHealthCheckConfiguration(changedHealthCheckConfig *entities.UpdateHealthCheckConfig) (bool, error) { + containsStartupHealthCheckCmd := changedHealthCheckConfig.HealthStartupCmd != "" + containsFlags := (changedHealthCheckConfig.HealthStartupInterval != "" || + changedHealthCheckConfig.HealthStartupRetries != "" || + changedHealthCheckConfig.HealthStartupTimeout != "" || + changedHealthCheckConfig.HealthStartupSuccess != "") + if c.config.StartupHealthCheckConfig == nil && !containsStartupHealthCheckCmd && containsFlags { + return false, errors.New("startup healthcheck command is not set") + } + + cmd := "" + interval := define.DefaultHealthCheckInterval + retries := 0 + timeout := define.DefaultHealthCheckTimeout + successes := 0 + + if c.config.StartupHealthCheckConfig != nil { + cmd = strings.Join(c.config.StartupHealthCheckConfig.Test, " ") + interval = c.config.StartupHealthCheckConfig.Interval.String() + retries = c.config.StartupHealthCheckConfig.Retries + timeout = c.config.StartupHealthCheckConfig.Timeout.String() + successes = c.config.StartupHealthCheckConfig.Successes + } + + changed := false + changedTimer := false + noHealthCheck := changedHealthCheckConfig.NoHealthCheck == "true" + + if changedHealthCheckConfig.HealthStartupCmd != "" { + cmd = changedHealthCheckConfig.HealthStartupCmd + changed = true + } + + if changedHealthCheckConfig.HealthStartupInterval != "" { + interval = changedHealthCheckConfig.HealthStartupInterval + changed = true + changedTimer = true + } + + if changedHealthCheckConfig.HealthStartupRetries != "" { + val, err := strconv.Atoi(changedHealthCheckConfig.HealthStartupRetries) + if err != nil { + return false, err + } + retries = val + changed = true + } + + if changedHealthCheckConfig.HealthStartupTimeout != "" { + timeout = changedHealthCheckConfig.HealthStartupTimeout + changed = true + } + + if changedHealthCheckConfig.HealthStartupSuccess != "" { + val, err := strconv.Atoi(changedHealthCheckConfig.HealthStartupSuccess) + if err != nil { + return false, err + } + successes = val + changed = true + } + + switch { + case noHealthCheck && changed: + return false, errors.New("cannot specify both --no-healthcheck and other HealthCheck flags") + case noHealthCheck: + c.config.StartupHealthCheckConfig = nil + case changed: + tmpHcConfig, err := specgenutil.MakeHealthCheckFromCli( + cmd, + interval, + uint(retries), + timeout, + "1s", + true, + ) + if err != nil { + return false, err + } + c.config.StartupHealthCheckConfig = new(define.StartupHealthCheck) + c.config.StartupHealthCheckConfig.Test = tmpHcConfig.Test + c.config.StartupHealthCheckConfig.Interval = tmpHcConfig.Interval + c.config.StartupHealthCheckConfig.Timeout = tmpHcConfig.Timeout + c.config.StartupHealthCheckConfig.Retries = tmpHcConfig.Retries + c.config.StartupHealthCheckConfig.Successes = successes + } + return changedTimer, nil +} + +func (c *Container) changeGlobalHealthCheckConfiguration(changedHealthCheckConfig *entities.UpdateHealthCheckConfig) error { + if changedHealthCheckConfig.HealthLogDestination != "" { + c.valid = false + err := WithHealthCheckLogDestination(changedHealthCheckConfig.HealthLogDestination)(c) + if err != nil { + return err + } + c.valid = true + } + + if changedHealthCheckConfig.HealthMaxLogSize != "" { + val, err := strconv.ParseUint(changedHealthCheckConfig.HealthMaxLogSize, 10, 64) + if err != nil { + return err + } + c.config.HealthMaxLogSize = uint(val) + } + + if changedHealthCheckConfig.HealthMaxLogCount != "" { + val, err := strconv.ParseUint(changedHealthCheckConfig.HealthMaxLogCount, 10, 64) + if err != nil { + return err + } + c.config.HealthMaxLogCount = uint(val) + } + + if changedHealthCheckConfig.HealthOnFailure != "" { + val, err := define.ParseHealthCheckOnFailureAction(changedHealthCheckConfig.HealthOnFailure) + if err != nil { + return err + } + c.config.HealthCheckOnFailureAction = val + } + return nil +} + +func (c *Container) resetHealthCheckTimers(noHealthCheck bool, changedItervalTimer bool, changedStartupItervalTimer bool) error { + if !c.ensureState(define.ContainerStateCreated, define.ContainerStateRunning) { + return nil + } + + if noHealthCheck { + if err := c.removeTransientFiles(context.Background(), + c.config.StartupHealthCheckConfig != nil && !c.state.StartupHCPassed, + c.state.HCUnitName); err != nil { + return err + } + return nil + } + + if c.config.StartupHealthCheckConfig != nil && !c.state.StartupHCPassed && changedStartupItervalTimer { + c.state.StartupHCPassed = false + c.state.StartupHCSuccessCount = 0 + c.state.StartupHCFailureCount = 0 + if err := c.save(); err != nil { + return err + } + c.recreateHealthCheckTimer(context.Background(), true, true) + } + + if c.config.HealthCheckConfig != nil && c.state.StartupHCPassed && changedItervalTimer { + c.recreateHealthCheckTimer(context.Background(), false, false) + return nil + } + + if c.config.HealthCheckConfig != nil && c.config.StartupHealthCheckConfig == nil && changedItervalTimer { + if err := c.createTimer(c.config.HealthCheckConfig.Interval.String(), false); err != nil { + logrus.Errorf("Error recreating container %s healthcheck: %v", c.ID(), err) + return nil + } + if err := c.startTimer(false); err != nil { + logrus.Errorf("Error restarting container %s healthcheck timer: %v", c.ID(), err) + return nil + } + return nil + } + return nil +} + +func (c *Container) updateHealthCheckConfiguration(changedHealthCheckConfig *entities.UpdateHealthCheckConfig) error { + if changedHealthCheckConfig == nil { + logrus.Debug("New HealthCheck configuration is empty.") + return nil + } + + oldHealthCheck, oldStartupHealthCheck := c.getCopyOfHealthCheckAndStartupHelathCheck() + oldHealthCheckOnFailureAction := c.config.HealthCheckOnFailureAction + oldHealthLogDestination := c.config.HealthLogDestination + oldHealthMaxLogCount := c.config.HealthMaxLogCount + oldHealthMaxLogSize := c.config.HealthMaxLogSize + + err := c.changeGlobalHealthCheckConfiguration(changedHealthCheckConfig) + if err != nil { + return err + } + + changedItervalTimer, err := c.changeHealthCheckConfiguration(changedHealthCheckConfig) + if err != nil { + return err + } + + changedStartupItervalTimer, err := c.changeStartupHealthCheckConfiguration(changedHealthCheckConfig) + if err != nil { + return err + } + noHealthCheck := changedHealthCheckConfig.NoHealthCheck == "true" + + if err := c.runtime.state.SafeRewriteContainerConfig(c, "", "", c.config); err != nil { + // Assume DB write failed, revert to old resources block + c.config.HealthCheckConfig = oldHealthCheck + c.config.StartupHealthCheckConfig = oldStartupHealthCheck + c.config.HealthCheckOnFailureAction = oldHealthCheckOnFailureAction + c.config.HealthLogDestination = oldHealthLogDestination + c.config.HealthMaxLogCount = oldHealthMaxLogCount + c.config.HealthMaxLogSize = oldHealthMaxLogSize + return err + } + + err = c.resetHealthCheckTimers(noHealthCheck, changedItervalTimer, changedStartupItervalTimer) + if err != nil { + return err + } + + logrus.Debugf("HealthCheck updated for container %s", c.ID()) + c.newContainerEvent(events.Update) + return nil +} diff --git a/libpod/healthcheck.go b/libpod/healthcheck.go index ada4e004d3..093f11bd65 100644 --- a/libpod/healthcheck.go +++ b/libpod/healthcheck.go @@ -258,33 +258,42 @@ func (c *Container) incrementStartupHCSuccessCounter(ctx context.Context) { } if recreateTimer { - logrus.Infof("Startup healthcheck for container %s passed, recreating timer", c.ID()) + c.recreateHealthCheckTimer(ctx, false, true) + } +} - oldUnit := c.state.HCUnitName - // Create the new, standard healthcheck timer first. - if err := c.createTimer(c.HealthCheckConfig().Interval.String(), false); err != nil { - logrus.Errorf("Error recreating container %s healthcheck: %v", c.ID(), err) - return - } - if err := c.startTimer(false); err != nil { - logrus.Errorf("Error restarting container %s healthcheck timer: %v", c.ID(), err) - } +func (c *Container) recreateHealthCheckTimer(ctx context.Context, isStartup bool, isStartupRemoved bool) { + logrus.Infof("Startup healthcheck for container %s passed, recreating timer", c.ID()) - // This kills the process the healthcheck is running. - // Which happens to be us. - // So this has to be last - after this, systemd serves us a - // SIGTERM and we exit. - // Special case, via SIGTERM we exit(1) which means systemd logs a failure in the unit. - // We do not want this as the unit will be leaked on failure states unless "reset-failed" - // is called. Fundamentally this is expected so switch it to exit 0. - // NOTE: This is only safe while being called from "podman healthcheck run" which we know - // is the case here as we should not alter the exit code of another process that just - // happened to call this. - shutdown.SetExitCode(0) - if err := c.removeTransientFiles(ctx, true, oldUnit); err != nil { - logrus.Errorf("Error removing container %s healthcheck: %v", c.ID(), err) - return - } + oldUnit := c.state.HCUnitName + // Create the new, standard healthcheck timer first. + interval := c.HealthCheckConfig().Interval.String() + if isStartup { + interval = c.config.StartupHealthCheckConfig.StartInterval.String() + } + + if err := c.createTimer(interval, isStartup); err != nil { + logrus.Errorf("Error recreating container %s (isStartup: %t) healthcheck: %v", c.ID(), isStartup, err) + return + } + if err := c.startTimer(isStartup); err != nil { + logrus.Errorf("Error restarting container %s (isStartup: %t) healthcheck timer: %v", c.ID(), isStartup, err) + } + + // This kills the process the healthcheck is running. + // Which happens to be us. + // So this has to be last - after this, systemd serves us a + // SIGTERM and we exit. + // Special case, via SIGTERM we exit(1) which means systemd logs a failure in the unit. + // We do not want this as the unit will be leaked on failure states unless "reset-failed" + // is called. Fundamentally this is expected so switch it to exit 0. + // NOTE: This is only safe while being called from "podman healthcheck run" which we know + // is the case here as we should not alter the exit code of another process that just + // happened to call this. + shutdown.SetExitCode(0) + if err := c.removeTransientFiles(ctx, isStartupRemoved, oldUnit); err != nil { + logrus.Errorf("Error removing container %s healthcheck: %v", c.ID(), err) + return } } diff --git a/pkg/api/handlers/compat/containers.go b/pkg/api/handlers/compat/containers.go index 706d2d97ba..777b4d9d6e 100644 --- a/pkg/api/handlers/compat/containers.go +++ b/pkg/api/handlers/compat/containers.go @@ -19,6 +19,7 @@ import ( "github.com/containers/podman/v5/pkg/api/handlers/utils" api "github.com/containers/podman/v5/pkg/api/types" "github.com/containers/podman/v5/pkg/domain/entities" + entitiesTypes "github.com/containers/podman/v5/pkg/domain/entities/types" "github.com/containers/podman/v5/pkg/domain/filters" "github.com/containers/podman/v5/pkg/domain/infra/abi" "github.com/containers/podman/v5/pkg/ps" @@ -785,8 +786,8 @@ func UpdateContainer(w http.ResponseWriter, r *http.Request) { localRetries := uint(options.RestartPolicy.MaximumRetryCount) restartRetries = &localRetries } - - if err := ctr.Update(resources, restartPolicy, restartRetries); err != nil { + // Update HealthCheck config for Docker is not available + if err := ctr.Update(resources, restartPolicy, restartRetries, &entitiesTypes.UpdateHealthCheckConfig{}); err != nil { utils.Error(w, http.StatusInternalServerError, fmt.Errorf("updating container: %w", err)) return } diff --git a/pkg/api/handlers/libpod/containers.go b/pkg/api/handlers/libpod/containers.go index 726c30b2e7..be096dd68d 100644 --- a/pkg/api/handlers/libpod/containers.go +++ b/pkg/api/handlers/libpod/containers.go @@ -17,6 +17,7 @@ import ( "github.com/containers/podman/v5/pkg/api/handlers/utils" api "github.com/containers/podman/v5/pkg/api/types" "github.com/containers/podman/v5/pkg/domain/entities" + "github.com/containers/podman/v5/pkg/domain/entities/types" "github.com/containers/podman/v5/pkg/domain/infra/abi" "github.com/containers/podman/v5/pkg/util" "github.com/gorilla/schema" @@ -443,12 +444,16 @@ func UpdateContainer(w http.ResponseWriter, r *http.Request) { return } - options := &handlers.UpdateEntities{Resources: &specs.LinuxResources{}} - if err := json.NewDecoder(r.Body).Decode(&options.Resources); err != nil { + options := &handlers.UpdateEntities{ + Resources: &specs.LinuxResources{}, + ChangedHealthCheckConfiguration: &types.UpdateHealthCheckConfig{}, + } + if err := json.NewDecoder(r.Body).Decode(&options); err != nil { utils.Error(w, http.StatusInternalServerError, fmt.Errorf("decode(): %w", err)) return } - err = ctr.Update(options.Resources, restartPolicy, restartRetries) + + err = ctr.Update(options.Resources, restartPolicy, restartRetries, options.ChangedHealthCheckConfiguration) if err != nil { utils.InternalServerError(w, err) return diff --git a/pkg/api/handlers/swagger/models.go b/pkg/api/handlers/swagger/models.go index 56fd5d8b56..fb8471a94c 100644 --- a/pkg/api/handlers/swagger/models.go +++ b/pkg/api/handlers/swagger/models.go @@ -54,4 +54,21 @@ type networkUpdateRequestLibpod entities.NetworkUpdateOptions // Container update // swagger:model -type containerUpdateRequest container.UpdateConfig +type containerUpdateRequest struct { + container.UpdateConfig + HealthLogDestination string + HealthMaxLogSize string + HealthMaxLogCount string + HealthOnFailure string + HoHealthCheck string + HealthCmd string + HealthInterval string + HealthRetries string + HealthTimeout string + HealthStartPeriod string + HealthStartupCmd string + HealthStartupInterval string + HealthStartupRetries string + HealthStartupTimeout string + HealthStartupSuccess string +} diff --git a/pkg/api/handlers/types.go b/pkg/api/handlers/types.go index 5a13530b90..a2291cfca3 100644 --- a/pkg/api/handlers/types.go +++ b/pkg/api/handlers/types.go @@ -2,6 +2,7 @@ package handlers import ( "github.com/containers/podman/v5/pkg/domain/entities" + "github.com/containers/podman/v5/pkg/domain/entities/types" docker "github.com/docker/docker/api/types" dockerBackend "github.com/docker/docker/api/types/backend" dockerContainer "github.com/docker/docker/api/types/container" @@ -73,7 +74,8 @@ type LibpodContainersRmReport struct { // UpdateEntities used to wrap the oci resource spec in a swagger model // swagger:model type UpdateEntities struct { - Resources *specs.LinuxResources + Resources *specs.LinuxResources `json:"resources,omitempty"` + ChangedHealthCheckConfiguration *types.UpdateHealthCheckConfig `json:"changed_healthcheck_configuration,omitempty"` } type Info struct { diff --git a/pkg/api/server/register_containers.go b/pkg/api/server/register_containers.go index cc1cc5e74d..7e68ef3414 100644 --- a/pkg/api/server/register_containers.go +++ b/pkg/api/server/register_containers.go @@ -1815,4 +1815,4 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error { // $ref: "#/responses/internalError" r.HandleFunc(VersionedPath("/libpod/containers/{name}/update"), s.APIHandler(libpod.UpdateContainer)).Methods(http.MethodPost) return nil -} +} // TODO Update docs? diff --git a/pkg/bindings/containers/update.go b/pkg/bindings/containers/update.go index 37cf74426e..a1ad2474f3 100644 --- a/pkg/bindings/containers/update.go +++ b/pkg/bindings/containers/update.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/containers/podman/v5/pkg/api/handlers" "github.com/containers/podman/v5/pkg/bindings" "github.com/containers/podman/v5/pkg/domain/entities/types" jsoniter "github.com/json-iterator/go" @@ -26,11 +27,15 @@ func Update(ctx context.Context, options *types.ContainerUpdateOptions) (string, } } - resources, err := jsoniter.MarshalToString(options.Specgen.ResourceLimits) + updateEntities := &handlers.UpdateEntities{ + Resources: options.Specgen.ResourceLimits, + ChangedHealthCheckConfiguration: options.ChangedHealthCheckConfiguration, + } + requestData, err := jsoniter.MarshalToString(updateEntities) if err != nil { return "", err } - stringReader := strings.NewReader(resources) + stringReader := strings.NewReader(requestData) response, err := conn.DoRequest(ctx, stringReader, http.MethodPost, "/containers/%s/update", params, nil, options.NameOrID) if err != nil { return "", err diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index 105b091549..8f64287573 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -495,3 +495,5 @@ type ContainerCloneOptions struct { // ContainerUpdateOptions containers options for updating an existing containers cgroup configuration type ContainerUpdateOptions = types.ContainerUpdateOptions + +type UpdateHealthCheckConfig = types.UpdateHealthCheckConfig diff --git a/pkg/domain/entities/types/containers.go b/pkg/domain/entities/types/containers.go index f9d922e229..b9859ef937 100644 --- a/pkg/domain/entities/types/containers.go +++ b/pkg/domain/entities/types/containers.go @@ -36,6 +36,25 @@ type ContainerStatsReport struct { } type ContainerUpdateOptions struct { - NameOrID string - Specgen *specgen.SpecGenerator + NameOrID string + Specgen *specgen.SpecGenerator + ChangedHealthCheckConfiguration *UpdateHealthCheckConfig +} + +type UpdateHealthCheckConfig struct { + HealthLogDestination string `json:"health_log_destination,omitempty"` + HealthMaxLogSize string `json:"health_max_log_size,omitempty"` + HealthMaxLogCount string `json:"health_max_log_count,omitempty"` + HealthOnFailure string `json:"health_jn_failure,omitempty"` + NoHealthCheck string `json:"no_healthcheck,omitempty"` + HealthCmd string `json:"health_cmd,omitempty"` + HealthInterval string `json:"health_interval,omitempty"` + HealthRetries string `json:"health_retries,omitempty"` + HealthTimeout string `json:"health_timeout,omitempty"` + HealthStartPeriod string `json:"health_start_period,omitempty"` + HealthStartupCmd string `json:"health_startup_cmd,omitempty"` + HealthStartupInterval string `json:"health_startup_interval,omitempty"` + HealthStartupRetries string `json:"health_startup_retries,omitempty"` + HealthStartupTimeout string `json:"health_startup_timeout,omitempty"` + HealthStartupSuccess string `json:"health_startup_success,omitempty"` } diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 0a0dbe3357..e1d722a417 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -1812,7 +1812,7 @@ func (ic *ContainerEngine) ContainerUpdate(ctx context.Context, updateOptions *e restartPolicy = &updateOptions.Specgen.RestartPolicy } - if err = containers[0].Update(updateOptions.Specgen.ResourceLimits, restartPolicy, updateOptions.Specgen.RestartRetries); err != nil { + if err = containers[0].Update(updateOptions.Specgen.ResourceLimits, restartPolicy, updateOptions.Specgen.RestartRetries, updateOptions.ChangedHealthCheckConfiguration); err != nil { return "", err } return containers[0].ID(), nil diff --git a/pkg/specgenutil/specgen.go b/pkg/specgenutil/specgen.go index 0bc9b419a7..3f26ca36af 100644 --- a/pkg/specgenutil/specgen.go +++ b/pkg/specgenutil/specgen.go @@ -22,6 +22,9 @@ import ( "github.com/docker/go-units" "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/selinux/go-selinux" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" ) const ( @@ -354,7 +357,7 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions if c.NoHealthCheck { return errors.New("cannot specify both --no-healthcheck and --health-cmd") } - s.HealthConfig, err = makeHealthCheckFromCli(c.HealthCmd, c.HealthInterval, c.HealthRetries, c.HealthTimeout, c.HealthStartPeriod, false) + s.HealthConfig, err = MakeHealthCheckFromCli(c.HealthCmd, c.HealthInterval, c.HealthRetries, c.HealthTimeout, c.HealthStartPeriod, false) if err != nil { return err } @@ -383,7 +386,7 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions // The hardcoded "1s" will be discarded, as the startup // healthcheck does not have a period. So just hardcode // something that parses correctly. - tmpHcConfig, err := makeHealthCheckFromCli(c.StartupHCCmd, c.StartupHCInterval, c.StartupHCRetries, c.StartupHCTimeout, "1s", true) + tmpHcConfig, err := MakeHealthCheckFromCli(c.StartupHCCmd, c.StartupHCInterval, c.StartupHCRetries, c.StartupHCTimeout, "1s", true) if err != nil { return err } @@ -948,7 +951,7 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions return nil } -func makeHealthCheckFromCli(inCmd, interval string, retries uint, timeout, startPeriod string, isStartup bool) (*manifest.Schema2HealthConfig, error) { +func MakeHealthCheckFromCli(inCmd, interval string, retries uint, timeout, startPeriod string, isStartup bool) (*manifest.Schema2HealthConfig, error) { cmdArr := []string{} isArr := true err := json.Unmarshal([]byte(inCmd), &cmdArr) // array unmarshalling @@ -1017,7 +1020,6 @@ func makeHealthCheckFromCli(inCmd, interval string, retries uint, timeout, start return nil, errors.New("healthcheck-start-period must be 0 seconds or greater") } hc.StartPeriod = startPeriodDuration - return &hc, nil } @@ -1297,3 +1299,47 @@ func GetResources(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions) } return s.ResourceLimits, nil } + +func GetChangedHealthCheckConfiguration(cmd *cobra.Command) entities.UpdateHealthCheckConfig { + updateHealthCheckConfig := entities.UpdateHealthCheckConfig{} + cmd.Flags().Visit(func(f *pflag.Flag) { + name := strings.ToLower(f.Name) + if strings.Contains(name, "health") { + switch name { + case "health-log-destination": + updateHealthCheckConfig.HealthLogDestination = f.Value.String() + case "health-max-log-size": + updateHealthCheckConfig.HealthMaxLogSize = f.Value.String() + case "health-max-log-count": + updateHealthCheckConfig.HealthMaxLogCount = f.Value.String() + case "health-on-failure": + updateHealthCheckConfig.HealthOnFailure = f.Value.String() + case "no-healthcheck": + updateHealthCheckConfig.NoHealthCheck = f.Value.String() + case "health-cmd": + updateHealthCheckConfig.HealthCmd = f.Value.String() + case "health-interval": + updateHealthCheckConfig.HealthInterval = f.Value.String() + case "health-retries": + updateHealthCheckConfig.HealthRetries = f.Value.String() + case "health-timeout": + updateHealthCheckConfig.HealthTimeout = f.Value.String() + case "health-start-period": + updateHealthCheckConfig.HealthStartPeriod = f.Value.String() + case "health-startup-cmd": + updateHealthCheckConfig.HealthStartupCmd = f.Value.String() + case "health-startup-interval": + updateHealthCheckConfig.HealthStartupInterval = f.Value.String() + case "health-startup-retries": + updateHealthCheckConfig.HealthStartupRetries = f.Value.String() + case "health-startup-timeout": + updateHealthCheckConfig.HealthStartupTimeout = f.Value.String() + case "health-startup-success": + updateHealthCheckConfig.HealthStartupSuccess = f.Value.String() + default: + logrus.Debug("Unexpected HealthCheck flag.") + } + } + }) + return updateHealthCheckConfig +} diff --git a/test/system/280-update.bats b/test/system/280-update.bats index 6013aa17e7..8020e4c4d4 100644 --- a/test/system/280-update.bats +++ b/test/system/280-update.bats @@ -161,4 +161,552 @@ device-write-iops = /dev/zero:4000 | - | - run_podman rm -f -t0 testctr } +function _create_container_check_change_of_HealthCheck_configuration_with_podman_update(){ + local ctrname="$1" + local msg="$2" + local format="$3" + local flag="$4" + local value="$5" + local expect="$6" + local expect_msg="$7" + local expect_exit_code="$8" + local is_startup="$9" + local no_hc="${10}" + + if [[ $is_startup = "yes" ]]; then + run_podman run -d --name $ctrname \ + --health-cmd "echo $msg" \ + --health-startup-cmd "echo $msg" \ + $IMAGE /home/podman/pause + cid="$output" + else + if [[ $no_hc = "yes" ]]; then + run_podman run -d --name $ctrname \ + $IMAGE /home/podman/pause + cid="$output" + else + run_podman run -d --name $ctrname \ + --health-cmd "echo $msg" \ + $IMAGE /home/podman/pause + cid="$output" + fi + fi + + + if [ -n "${value}" ]; then + run_podman $expect_exit_code update $ctrname $flag "$value" + else + run_podman $expect_exit_code update $ctrname $flag + fi + + if [[ $expect_exit_code = 0 ]]; then + run_podman inspect $ctrname --format $format + assert "$output" == "$expect" "$expect_msg" + fi + + output=$cid +} + +# HealthCheck configuration + +@test "podman update - --health-cmd" { + local msg="healthmsg-$(random_string)" + local new_msg="healthmsg-new-msg" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.Test}}" \ + "--health-cmd" "echo $new_msg" \ + "[CMD-SHELL echo $new_msg]" \ + ".Config.Healthcheck.Test" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-interval" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.Interval}}"\ + "--health-interval" "10s" \ + "10s" \ + ".Config.Healthcheck.Interval" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-log-destination" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + local TMP_DIR_HEALTHCHECK="$PODMAN_TMPDIR/healthcheck" + mkdir $TMP_DIR_HEALTHCHECK + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.HealthLogDestination}}" \ + "--health-log-destination" "$TMP_DIR_HEALTHCHECK" \ + "$TMP_DIR_HEALTHCHECK" \ + ".Config.HealthLogDestination" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-max-log-count" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.HealthMaxLogCount}}" \ + "--health-max-log-count" "20" \ + "20" \ + ".Config.HealthMaxLogCount" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-max-log-size" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.HealthMaxLogSize}}" \ + "--health-max-log-size" "10" \ + "10" \ + ".Config.HealthMaxLogSize" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-on-failure" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.HealthcheckOnFailureAction}}" \ + "--health-on-failure" "restart" \ + "restart" \ + ".Config.Healthcheck.HealthcheckOnFailureAction" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-retries" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.Retries}}" \ + "--health-retries" "5" \ + "5" \ + ".Config.Healthcheck.Retries" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-timeout" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.Timeout}}" \ + "--health-timeout" "10s" \ + "10s" \ + ".Config.Healthcheck.Timeout" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --no-healthcheck" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.Test}}" \ + "--no-healthcheck" "" \ + "[NONE]" \ + "HealthCheck command is disabled" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + + +@test "podman update - --health-start-period" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.StartPeriod}}" \ + "--health-start-period" "10s" \ + "10s" \ + ".Config.Healthcheck.StartPeriod" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - no hc command --health-interval" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.Interval}}"\ + "--health-interval" "10s" \ + "10s" \ + ".Config.Healthcheck.Interval" \ + 125 "no" "yes" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - no hc command --health-retries" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.Retries}}" \ + "--health-retries" "5" \ + "5" \ + ".Config.Healthcheck.Retries" \ + 125 "no" "yes" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - no hc command --health-timeout" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.Timeout}}" \ + "--health-timeout" "10s" \ + "10s" \ + ".Config.Healthcheck.Timeout" \ + 125 "no" "yes" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - no hc command --health-start-period" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.Healthcheck.StartPeriod}}" \ + "--health-start-period" "10s" \ + "10s" \ + ".Config.Healthcheck.StartPeriod" \ + 125 "no" "yes" + + run_podman rm -t 0 -f $ctrname +} + +# Startup HealthCheck Configuration + +@test "podman update - --health-startup-cmd" { + local msg="healthmsg-$(random_string)" + local new_msg="healthmsg-new-msg" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Test}}" \ + "--health-startup-cmd" "echo $new_msg" \ + "[CMD-SHELL echo $new_msg]" \ + ".Config.StartupHealthCheck.Test" \ + 0 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-startup-interval" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Interval}}" \ + "--health-startup-interval" "10s" \ + "10s" \ + ".Config.StartupHealthCheck.Interval" \ + 0 "yes" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-startup-retries" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Retries}}" \ + "--health-startup-retries" "5" \ + "5" \ + ".Config.StartupHealthCheck.Retries" \ + 0 "yes" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-startup-success" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Successes}}" \ + "--health-startup-success " "10" \ + "10" \ + ".Config.StartupHealthCheck.Successes" \ + 0 "yes" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - --health-startup-timeout" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Timeout}}" \ + "--health-startup-timeout" "10s" \ + "10s" \ + ".Config.StartupHealthCheck.Timeout" \ + 0 "yes" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - no startup hc command --health-startup-interval" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Interval}}" \ + "--health-startup-interval" "10s" \ + "10s" \ + ".Config.StartupHealthCheck.Interval" \ + 125 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - no startup hc command --health-startup-retries" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Retries}}" \ + "--health-startup-retries" "5" \ + "5" \ + ".Config.StartupHealthCheck.Retries" \ + 125 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - no startup hc command --health-startup-success" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Successes}}" \ + "--health-startup-success" "10" \ + "10" \ + ".Config.StartupHealthCheck.Successes" \ + 125 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - no startup hc command --health-startup-timeout" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + + _create_container_check_change_of_HealthCheck_configuration_with_podman_update \ + $ctrname \ + $msg \ + "{{.Config.StartupHealthCheck.Timeout}}" \ + "--health-startup-timeout" "10s" \ + "10s" \ + ".Config.StartupHealthCheck.Timeout" \ + 125 "no" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - check behavior - change cmd and destination healthcheck" { + local TMP_DIR_HEALTHCHECK="$PODMAN_TMPDIR/healthcheck" + mkdir $TMP_DIR_HEALTHCHECK + local ctrname="c-h-$(safename)" + local msg="healthmsg-$(random_string)" + + run_podman run -d --name $ctrname \ + --health-cmd "echo $msg" \ + $IMAGE /home/podman/pause + cid="$output" + + run_podman healthcheck run $ctrname + is "$output" "" "output from 'podman healthcheck run'" + + run_podman update $ctrname --health-cmd "echo healthmsg-new" --health-log-destination $TMP_DIR_HEALTHCHECK + + run_podman healthcheck run $ctrname + is "$output" "" "output from 'podman healthcheck run'" + + healthcheck_log_path="${TMP_DIR_HEALTHCHECK}/${cid}-healthcheck.log" + # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd. + # And since `run_podman healthcheck run` is also run manually, it will result in two runs. + count=$(grep -co "healthmsg-new" $healthcheck_log_path) + assert "$count" -ge 1 "Number of matching health log messages" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - check behavior - change healthcheck interval" { + local ctrname="c-h-$(safename)" + local msg="healthmsg-$(random_string)" + + run_podman run -d --name $ctrname \ + --health-cmd "echo $msg" \ + --health-interval 2s \ + $IMAGE top + cid="$output" + + sleep 2s + + run_podman update $ctrname --health-interval 1s + + sleep 5s + + run_podman inspect $ctrname --format "{{.State.Health.Log}}" + count=$(grep -co "$msg" <<< "$output") + assert "$count" -ge 3 "Number of matching health log messages" + + run_podman rm -t 0 -f $ctrname +} + + +@test "podman update - check behavior - set healthcheck interval" { + local ctrname="c-h-$(safename)" + local msg="healthmsg-$(random_string)" + + run_podman run -d --name $ctrname \ + --health-cmd "echo $msg" \ + --health-interval disable \ + $IMAGE top + cid="$output" + + run_podman update $ctrname --health-interval 1s + + sleep 5s + + run_podman inspect $ctrname --format "{{.State.Health.Log}}" + count=$(grep -co "$msg" <<< "$output") + assert "$count" -ge 3 "Number of matching health log messages" + + run_podman rm -t 0 -f $ctrname +} + +@test "podman update - check behavior - change healthcheck startup interval" { + local ctrname="c-h-$(safename)" + local msg="healthmsg-$(random_string)" + + run_podman run -d --name $ctrname \ + --health-cmd "echo normal$msg" \ + --health-startup-cmd "echo startup$msg" \ + --health-startup-interval 30s \ + --health-startup-success 3 \ + $IMAGE top + cid="$output" + + sleep 1s + + run_podman update $ctrname --health-startup-interval 1s + + sleep 5s + + run_podman inspect $ctrname --format "{{.State.Health.Log}}" + count=$(grep -co "startup$msg" <<< "$output") + assert "$count" -ge 3 "Number of matching startup health log messages" + + count=$(grep -co "normal$msg" <<< "$output") + assert "$count" -ge 1 "Number of matching health log messages" + + run_podman rm -t 0 -f $ctrname +} + + +@test "podman update - check behavior - set healthcheck startup interval" { + local ctrname="c-h-$(safename)" + local msg="healthmsg-$(random_string)" + + run_podman run -d --name $ctrname \ + --health-cmd "echo normal$msg" \ + --health-startup-cmd "echo startup$msg" \ + --health-startup-interval disable \ + $IMAGE top + cid="$output" + + sleep 1s + + run_podman update $ctrname --health-startup-interval 1s + + sleep 5s + + run_podman inspect $ctrname --format "{{.State.Health.Log}}" + count=$(grep -co "$msg" <<< "$output") + assert "$count" -ge 1 "Number of matching health log messages" + + run_podman rm -t 0 -f $ctrname +} # vim: filetype=sh