From 838570bfb64ec6a4155d0e3cf33b79d904159675 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Fri, 27 Sep 2024 13:44:10 +0100 Subject: [PATCH] Wait for the old CRD Manager to stop before starting a new one --- CHANGELOG.md | 3 +++ .../component/prometheus/operator/common/component.go | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4660a4c1f3..6eb2972f08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ Main (unreleased) - Update yet-another-cloudwatch-exporter from v0.60.0 vo v0.61.0: (@morremeyer) - Fixes a bug where cloudwatch S3 metrics are reported as `0` +- `prometheus.operator.*` components: Fixed a bug which would sometimes cause a + "failed to create service discovery refresh metrics" error after a config reload. (@ptodev) + ### Other changes - Small fix in UI stylesheet to fit more content into visible table area. (@defanator) diff --git a/internal/component/prometheus/operator/common/component.go b/internal/component/prometheus/operator/common/component.go index 666ec42aeb..e54f512374 100644 --- a/internal/component/prometheus/operator/common/component.go +++ b/internal/component/prometheus/operator/common/component.go @@ -74,6 +74,8 @@ func (c *Component) Run(ctx context.Context) error { c.reportHealth(nil) errChan := make(chan error, 1) + runWg := sync.WaitGroup{} + defer runWg.Wait() for { select { case <-ctx.Done(): @@ -87,15 +89,23 @@ func (c *Component) Run(ctx context.Context) error { c.mut.Lock() manager := newCrdManager(c.opts, c.cluster, c.opts.Logger, c.config, c.kind, c.ls) c.manager = manager + + // Wait for the old manager to stop. + // If we start the new manager before stopping the old one, + // the new manager might not be able to register its debug metrics due to a duplicate registration error. if cancel != nil { cancel() } + runWg.Wait() + innerCtx, cancel = context.WithCancel(ctx) + runWg.Add(1) go func() { if err := manager.Run(innerCtx); err != nil { level.Error(c.opts.Logger).Log("msg", "error running crd manager", "err", err) errChan <- err } + runWg.Done() }() c.mut.Unlock() }