From 2884fad46d2688a1573243044bc922262c32ebec Mon Sep 17 00:00:00 2001 From: Eron Wright Date: Wed, 16 Oct 2024 19:02:21 -0700 Subject: [PATCH] Clean up "object has already been modified" warnings #700 (#717) ### Proposed changes This PR optimizes the watches to reduce the likelihood of a race between the cache and the reconciliation loop. The general idea is to use predicates to filter on the 'edge' events that would allow the reconciler to make forward progress. For example, the workspace reconciler applies a StatefulSet, then waits for it to be ready. Only when the StatefulSet is indeed ready should reconciliation be queued. Before this PR, the watch on StatefulSet was triggering upon any revision change (way too eager). Overall, a number of optimizations were implemented: - workspace controller should wait for the statefulset to be ready - workspace controller need not watch the Service - stack controller should wait for flux source to be ready - stack controller should not trigger reconciliation upon adding its finalizer (which increments the generation). - stack controller should wait for the update to complete - update controller should wait for workspace readiness - update controller should not enqueue the completed updates of a given workspace Also, some debug statements were introduced to reason about the revision changes over time, since the "conflicts" are detected based on revision. In particular, we emit a debug statement when reconciliation is triggered by a given watch, to understand 'why'. ### Related issues (optional) Closes #700 Supercedes https://github.com/pulumi/pulumi-kubernetes-operator/pull/679 ### Example Here's the result of applying [operator/examples/random-yaml/stack.yaml](https://github.com/pulumi/pulumi-kubernetes-operator/blob/78836991b2691034b2c029df4040ea2016240cd8/operator/examples/random-yaml/stack.yaml). #### Before ```log controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.032Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "841b4d5b-029c-4457-a2b7-8c1821475adf", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:181 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.033Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "841b4d5b-029c-4457-a2b7-8c1821475adf", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.047Z ERROR unable to save object status {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "6a99e260-ca2d-4376-b95f-ab490acbe32a", "error": "Operation cannot be fulfilled on stacks.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/pulumi.(*StackReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/pulumi/stack_controller.go:445 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/pulumi.(*StackReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/pulumi/stack_controller.go:741 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.047Z ERROR Reconciler error {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "6a99e260-ca2d-4376-b95f-ab490acbe32a", "error": "Operation cannot be fulfilled on stacks.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.091Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d397d7d1-c74b-43fe-b4a0-80c3d39ee3ea", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:188 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.091Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d397d7d1-c74b-43fe-b4a0-80c3d39ee3ea", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.111Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "b94710c8-8165-4937-a99c-a4e065f63788", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:188 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.111Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "b94710c8-8165-4937-a99c-a4e065f63788", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:32.082Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "7cb6660a-508c-4392-80cd-749fe72def9f", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:252 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:32.082Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "7cb6660a-508c-4392-80cd-749fe72def9f", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:36.114Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "480ad23a-2d02-40a6-b4a4-198fb9df01c4", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:316 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:36.114Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "480ad23a-2d02-40a6-b4a4-198fb9df01c4", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 ``` #### After ```log controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.212Z INFO Reconciling Stack {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "585499fc-fb33-4505-8594-6a9fd8c810b2", "revision": "893372"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z INFO KubeAPIWarningLogger metadata.finalizers: "finalizer.stack.pulumi.com": prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z INFO installing watcher for newly seen source kind {"GroupVersionKind": "source.toolkit.fluxcd.io/v1, Kind=GitRepository"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z INFO Starting EventSource {"controller": "stack-controller", "source": "kind source: *unstructured.Unstructured"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.238Z INFO Reconciling Workspace {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d1595cfb-0fe6-46d5-9e9c-2227f334eb7f", "revision": "893374"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.238Z INFO Applying StatefulSet {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d1595cfb-0fe6-46d5-9e9c-2227f334eb7f", "revision": "893374", "hash": "608b7e170ccd95754972cde1178ab426", "source": {"Generation":1,"ForceRequest":"","Git":null,"Flux":{"Url":"http://source-controller.flux-system.svc.cluster.local./gitrepository/default/pulumi-examples/1432c2ce26299516223c0ad228e92294eba4b2c1.tar.gz","Digest":"sha256:44d554df090dcdeb9bae908cd38155c8c93db05f64dc72982027e8e1294af0d3","Dir":"random-yaml/"}}} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.609Z INFO Reconciling Workspace {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.609Z INFO Applying StatefulSet {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "hash": "608b7e170ccd95754972cde1178ab426", "source": {"Generation":1,"ForceRequest":"","Git":null,"Flux":{"Url":"http://source-controller.flux-system.svc.cluster.local./gitrepository/default/pulumi-examples/1432c2ce26299516223c0ad228e92294eba4b2c1.tar.gz","Digest":"sha256:44d554df090dcdeb9bae908cd38155c8c93db05f64dc72982027e8e1294af0d3","Dir":"random-yaml/"}}} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.719Z INFO Connecting to workspace pod {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.724Z INFO Connected to workspace pod {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.724Z INFO Running pulumi install {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:51.509Z INFO Creating Pulumi stack(s) {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893413"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.250Z INFO workspace pod initialized {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893422"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.250Z INFO Ready {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893422"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.255Z INFO Reconciling Stack {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "910a2faa-5b49-4590-a23a-d9ef8cab0023", "revision": "893375"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.271Z INFO Reconciling Update {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.271Z INFO Updating the status {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.275Z INFO Connecting {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.278Z INFO Selecting the stack {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "stackName": "dev"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:57.333Z INFO Applying the update {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "type": "up"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:57.334Z INFO Executing update operation {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "request": "message:\"Stack Update (up)\" target_dependents:false refresh:true"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.421Z INFO Update complete {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "result": "stdout:\"Updating (dev)\\n\\nView Live: https://app.pulumi.com/eron-pulumi-corp/random/dev/updates/4536\\n\\n+ pulumi:pulumi:Stack: (create)\\n [urn=urn:pulumi:dev::random::pulumi:pulumi:Stack::random-dev]\\n + random:index/randomPassword:RandomPassword: (create)\\n [urn=urn:pulumi:dev::random::random:index/randomPassword:RandomPassword::randomPassword]\\n length : 16\\n overrideSpecial: \\\"_%@\\\"\\n special : true\\n --outputs:--\\n bcryptHash : [secret]\\n id : \\\"none\\\"\\n lower : true\\n minLower : 0\\n minNumeric : 0\\n minSpecial : 0\\n minUpper : 0\\n number : true\\n numeric : true\\n result : [secret]\\n upper : true\\n --outputs:--\\n password: [secret]\\nResources:\\n + 2 created\\n\\nDuration: 3s\\n\" summary:{start_time:{seconds:1729044119} end_time:{seconds:1729044122} result:\"succeeded\" message:\"\\\"Stack Update (up)\\\"\"} permalink:\"https://app.pulumi.com/eron-pulumi-corp/random/dev/updates/4536\" outputs:{key:\"password\" value:{value:\"\\\"fAMcmXI2DfW7KCey\\\"\" secret:true}}"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.438Z INFO Reconciling Stack {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "a100c4f3-c5b9-4894-8644-fabc1f71d008", "revision": "893437"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.546Z INFO Commit hash unchanged. Will wait for Source update or resync. {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "a100c4f3-c5b9-4894-8644-fabc1f71d008", "revision": "893437"} ``` --- operator/Makefile | 2 +- .../controller/auto/update_controller.go | 71 ++++++++- operator/internal/controller/auto/utils.go | 31 ++++ .../controller/auto/workspace_controller.go | 48 +++++- operator/internal/controller/pulumi/flux.go | 70 +++++---- .../controller/pulumi/stack_controller.go | 137 ++++++++++++++---- .../pulumi/stack_controller_test.go | 2 +- 7 files changed, 294 insertions(+), 67 deletions(-) diff --git a/operator/Makefile b/operator/Makefile index 2a090b7b..df161463 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -192,7 +192,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller manager to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply --server-side=true -oyaml -f - + $(KUSTOMIZE) build config/default | $(KUBECTL) apply --server-side=true -f - .PHONY: undeploy undeploy: ## Undeploy controller manager from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. diff --git a/operator/internal/controller/auto/update_controller.go b/operator/internal/controller/auto/update_controller.go index e2d58a9b..e07242f0 100644 --- a/operator/internal/controller/auto/update_controller.go +++ b/operator/internal/controller/auto/update_controller.go @@ -32,6 +32,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -41,6 +42,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -77,13 +79,15 @@ type UpdateReconciler struct { // Reconcile manages the Update CRD and initiates Pulumi operations. func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { l := log.FromContext(ctx) - l.Info("Reconciling Update") obj := &autov1alpha1.Update{} err := r.Get(ctx, req.NamespacedName, obj) if err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + l = l.WithValues("revision", obj.ResourceVersion) + ctx = log.IntoContext(ctx, l) + l.Info("Reconciling Update") rs := newReconcileSession(r.Client, obj) @@ -116,12 +120,32 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, fmt.Errorf("failed to update the status: %w", err) } - // TODO check the w status before proceeding + // Get the workspace and check that it is ready w := &autov1alpha1.Workspace{} err = r.Client.Get(ctx, client.ObjectKey{Namespace: obj.Namespace, Name: obj.Spec.WorkspaceName}, w) if err != nil { + if apierrors.IsNotFound(err) { + l.Info("Workspace not found", "workspace", obj.Spec.WorkspaceName) + rs.progressing.Status = metav1.ConditionFalse + rs.progressing.Reason = "WorkspaceNotFound" + rs.failed.Status = metav1.ConditionFalse + rs.failed.Reason = UpdateConditionReasonProgressing + rs.complete.Status = metav1.ConditionFalse + rs.complete.Reason = UpdateConditionReasonProgressing + return ctrl.Result{}, rs.updateStatus(ctx, obj) + } return ctrl.Result{}, fmt.Errorf("failed to get workspace: %w", err) } + if !isWorkspaceReady(w) { + l.Info("Workspace not ready", "workspace", w.Name) + rs.progressing.Status = metav1.ConditionFalse + rs.progressing.Reason = "WorkspaceNotReady" + rs.failed.Status = metav1.ConditionFalse + rs.failed.Reason = UpdateConditionReasonProgressing + rs.complete.Status = metav1.ConditionFalse + rs.complete.Reason = UpdateConditionReasonProgressing + return ctrl.Result{}, rs.updateStatus(ctx, obj) + } // Connect to the workspace's GRPC server addr := fmt.Sprintf("%s:%d", fqdnForService(w), WorkspaceGrpcPort) @@ -168,6 +192,36 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } } +func isWorkspaceReady(ws *autov1alpha1.Workspace) bool { + if ws == nil || ws.Generation != ws.Status.ObservedGeneration { + return false + } + return meta.IsStatusConditionTrue(ws.Status.Conditions, autov1alpha1.WorkspaceReady) +} + +type workspaceReadyPredicate struct{} + +var _ predicate.Predicate = &workspaceReadyPredicate{} + +func (workspaceReadyPredicate) Create(e event.CreateEvent) bool { + return isWorkspaceReady(e.Object.(*autov1alpha1.Workspace)) +} + +func (workspaceReadyPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (workspaceReadyPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !isWorkspaceReady(e.ObjectOld.(*autov1alpha1.Workspace)) && isWorkspaceReady(e.ObjectNew.(*autov1alpha1.Workspace)) +} + +func (workspaceReadyPredicate) Generic(_ event.GenericEvent) bool { + return false +} + type reconcileSession struct { progressing *metav1.Condition complete *metav1.Condition @@ -375,13 +429,18 @@ func (r *UpdateReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&autov1alpha1.Update{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Watches(&autov1alpha1.Workspace{}, handler.EnqueueRequestsFromMapFunc(r.mapWorkspaceToUpdate), - builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})). + builder.WithPredicates(&workspaceReadyPredicate{})). Complete(r) } func indexUpdateByWorkspace(obj client.Object) []string { - w := obj.(*autov1alpha1.Update) - return []string{w.Spec.WorkspaceName} + u := obj.(*autov1alpha1.Update) + complete := meta.IsStatusConditionTrue(u.Status.Conditions, UpdateConditionTypeComplete) + if complete { + // don't index the completed updates, to avoid unnecessary reconciles when their workspace is updated + return []string{} + } + return []string{u.Spec.WorkspaceName} } func (r *UpdateReconciler) mapWorkspaceToUpdate(ctx context.Context, obj client.Object) []reconcile.Request { @@ -494,7 +553,7 @@ func (s streamReader[T]) Result() (result, error) { continue // No result yet. } - s.l.Info("Result received", "result", res) + s.l.Info("Update complete", "result", res) s.obj.Status.StartTime = metav1.NewTime(res.GetSummary().StartTime.AsTime()) s.obj.Status.EndTime = metav1.NewTime(res.GetSummary().EndTime.AsTime()) diff --git a/operator/internal/controller/auto/utils.go b/operator/internal/controller/auto/utils.go index db029182..4aa06f53 100644 --- a/operator/internal/controller/auto/utils.go +++ b/operator/internal/controller/auto/utils.go @@ -12,6 +12,9 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/types/known/structpb" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" ) func connect(ctx context.Context, addr string) (*grpc.ClientConn, error) { @@ -84,3 +87,31 @@ func marshalConfigValue(item autov1alpha1.ConfigItem) *agentpb.ConfigValue { } return v } + +var l = log.Log.WithName("predicate").WithName("debug") + +type DebugPredicate struct { + Controller string +} + +var _ predicate.Predicate = &DebugPredicate{} + +func (p *DebugPredicate) Create(e event.CreateEvent) bool { + l.V(1).Info("Create", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion()) + return true +} + +func (p *DebugPredicate) Delete(e event.DeleteEvent) bool { + l.V(1).Info("Delete", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion()) + return true +} + +func (p *DebugPredicate) Update(e event.UpdateEvent) bool { + l.V(1).Info("Update", "controller", p.Controller, "type", fmt.Sprintf("%T", e.ObjectOld), "name", e.ObjectOld.GetName(), "old-revision", e.ObjectOld.GetResourceVersion(), "new-revision", e.ObjectNew.GetResourceVersion()) + return true +} + +func (p *DebugPredicate) Generic(e event.GenericEvent) bool { + l.V(1).Info("Generic", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion()) + return true +} diff --git a/operator/internal/controller/auto/workspace_controller.go b/operator/internal/controller/auto/workspace_controller.go index b557cbe5..af9a63be 100644 --- a/operator/internal/controller/auto/workspace_controller.go +++ b/operator/internal/controller/auto/workspace_controller.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -78,13 +79,14 @@ type WorkspaceReconciler struct { func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { l := log.FromContext(ctx) - l.Info("Reconciling Workspace") w := &autov1alpha1.Workspace{} err := r.Get(ctx, req.NamespacedName, w) if err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + l = l.WithValues("revision", w.ResourceVersion) + l.Info("Reconciling Workspace") // apply defaults to the workspace spec // future: use a mutating webhook to apply defaults @@ -108,7 +110,11 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( err := r.Status().Update(ctx, w) if err != nil { l.Error(err, "updating status") + } else { + l = log.FromContext(ctx).WithValues("revision", w.ResourceVersion) + l.V(1).Info("Status updated") } + return err } @@ -319,15 +325,47 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // SetupWithManager sets up the controller with the Manager. func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). + Named("workspace-controller"). For(&autov1alpha1.Workspace{}, - builder.WithPredicates(predicate.GenerationChangedPredicate{})). - Owns(&corev1.Service{}, - builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})). + builder.WithPredicates(predicate.GenerationChangedPredicate{}, &DebugPredicate{Controller: "workspace-controller"})). Owns(&appsv1.StatefulSet{}, - builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})). + builder.WithPredicates(&statefulSetReadyPredicate{}, &DebugPredicate{Controller: "workspace-controller"})). Complete(r) } +type statefulSetReadyPredicate struct{} + +var _ predicate.Predicate = &statefulSetReadyPredicate{} + +func isStatefulSetReady(ss *appsv1.StatefulSet) bool { + if ss.Status.ObservedGeneration != ss.Generation || ss.Status.UpdateRevision != ss.Status.CurrentRevision { + return false + } + if ss.Status.AvailableReplicas < 1 { + return false + } + return true +} + +func (statefulSetReadyPredicate) Create(e event.CreateEvent) bool { + return isStatefulSetReady(e.Object.(*appsv1.StatefulSet)) +} + +func (statefulSetReadyPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (statefulSetReadyPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !isStatefulSetReady(e.ObjectOld.(*appsv1.StatefulSet)) && isStatefulSetReady(e.ObjectNew.(*appsv1.StatefulSet)) +} + +func (statefulSetReadyPredicate) Generic(_ event.GenericEvent) bool { + return false +} + const ( FieldManager = "pulumi-kubernetes-operator" WorkspacePulumiContainerName = "pulumi" diff --git a/operator/internal/controller/pulumi/flux.go b/operator/internal/controller/pulumi/flux.go index 6ccfc5c2..61ec9bf5 100644 --- a/operator/internal/controller/pulumi/flux.go +++ b/operator/internal/controller/pulumi/flux.go @@ -11,6 +11,8 @@ import ( "github.com/pulumi/pulumi-kubernetes-operator/v2/operator/api/pulumi/shared" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" ) func (sess *stackReconcilerSession) SetupWorkspaceFromFluxSource(ctx context.Context, source unstructured.Unstructured, fluxSource *shared.FluxSource) (string, error) { @@ -70,37 +72,28 @@ func checksumOrDigest(source unstructured.Unstructured) (string, error) { return checksum, nil } -// checkFluxSourceReady looks for the conventional "Ready" condition to see if the supplied object -// can be considered _not_ ready. It returns an error if it can determine that the object is not -// ready, and nil if it cannot determine so. -func checkFluxSourceReady(obj unstructured.Unstructured) error { +func checkFluxSourceReady(obj *unstructured.Unstructured) bool { + observedGeneration, ok, err := unstructured.NestedInt64(obj.Object, "status", "observedGeneration") + if !ok || err != nil || observedGeneration != obj.GetGeneration() { + return false + } conditions, ok, err := unstructured.NestedSlice(obj.Object, "status", "conditions") - if ok && err == nil { - // didn't find a []Condition, so there's nothing to indicate that it's not ready there - for _, c0 := range conditions { - var c map[string]interface{} - if c, ok = c0.(map[string]interface{}); !ok { - // condition isn't the right shape, try the next one - continue - } - if t, ok, err := unstructured.NestedString(c, "type"); ok && err == nil && t == "Ready" { - if v, ok, err := unstructured.NestedString(c, "status"); ok && err == nil && v == "True" { - // found the Ready condition and it is actually ready; proceed to next check - break - } - // found the Ready condition and it's something other than ready - return fmt.Errorf("source Ready condition does not have status True %#v", c) + if !ok || err != nil { + return false + } + for _, c0 := range conditions { + var c map[string]interface{} + if c, ok = c0.(map[string]interface{}); !ok { + // condition isn't the right shape, try the next one + continue + } + if t, ok, err := unstructured.NestedString(c, "type"); ok && err == nil && t == "Ready" { + if v, ok, err := unstructured.NestedString(c, "status"); ok && err == nil && v == "True" { + return true } } - // Ready=true, or no ready condition to tell us either way } - - _, ok, err = unstructured.NestedMap(obj.Object, "status", "artifact") - if !ok || err != nil { - return fmt.Errorf(".status.artifact does not have an Artifact object") - } - - return nil + return false } func getSourceGVK(src shared.FluxSourceReference) (schema.GroupVersionKind, error) { @@ -111,3 +104,26 @@ func getSourceGVK(src shared.FluxSourceReference) (schema.GroupVersionKind, erro func fluxSourceKey(gvk schema.GroupVersionKind, name string) string { return fmt.Sprintf("%s:%s", gvk, name) } + +type fluxSourceReadyPredicate struct{} + +var _ predicate.Predicate = &fluxSourceReadyPredicate{} + +func (fluxSourceReadyPredicate) Create(e event.CreateEvent) bool { + return checkFluxSourceReady(e.Object.(*unstructured.Unstructured)) +} + +func (fluxSourceReadyPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (fluxSourceReadyPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !checkFluxSourceReady(e.ObjectOld.(*unstructured.Unstructured)) && checkFluxSourceReady(e.ObjectNew.(*unstructured.Unstructured)) +} + +func (fluxSourceReadyPredicate) Generic(_ event.GenericEvent) bool { + return false +} diff --git a/operator/internal/controller/pulumi/stack_controller.go b/operator/internal/controller/pulumi/stack_controller.go index 5fdf922c..cf4b8131 100644 --- a/operator/internal/controller/pulumi/stack_controller.go +++ b/operator/internal/controller/pulumi/stack_controller.go @@ -58,7 +58,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" ctrlhandler "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -101,7 +101,9 @@ func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error { // Filter for update events where an object's metadata.generation is changed (no spec change!), // or the "force reconcile" annotation is used (and not marked as handled). predicates := []predicate.Predicate{ - predicate.Or(predicate.GenerationChangedPredicate{}, ReconcileRequestedPredicate{}), + predicate.Or( + predicate.And(predicate.GenerationChangedPredicate{}, predicate.Not(&finalizerAddedPredicate{})), + ReconcileRequestedPredicate{}), } // Track metrics about stacks. @@ -194,13 +196,16 @@ func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error { enqueueStacksForSourceFunc(programRefIndexFieldName, func(obj client.Object) string { return obj.GetName() - }))) + })), + builder.WithPredicates(&auto.DebugPredicate{Controller: "stack-controller"})) // Watch the stack's workspace and update objects - blder = blder.Watches(&autov1alpha1.Workspace{}, ctrlhandler.EnqueueRequestForOwner( - mgr.GetScheme(), mgr.GetRESTMapper(), &pulumiv1.Stack{})) - blder = blder.Watches(&autov1alpha1.Update{}, ctrlhandler.EnqueueRequestForOwner( - mgr.GetScheme(), mgr.GetRESTMapper(), &pulumiv1.Stack{})) + blder = blder.Watches(&autov1alpha1.Workspace{}, + ctrlhandler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &pulumiv1.Stack{}), + builder.WithPredicates(&workspaceReadyPredicate{}, &auto.DebugPredicate{Controller: "stack-controller"})) + blder = blder.Watches(&autov1alpha1.Update{}, + ctrlhandler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &pulumiv1.Stack{}), + builder.WithPredicates(&updateCompletePredicate{}, &auto.DebugPredicate{Controller: "stack-controller"})) c, err := blder.WithOptions(opts).Build(r) if err != nil { @@ -254,9 +259,7 @@ func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error { } watchedMu.Unlock() if !ok { - // Using PartialObjectMetadata means we don't need the actual types registered in the - // schema. - var sourceKind metav1.PartialObjectMetadata + var sourceKind unstructured.Unstructured sourceKind.SetGroupVersionKind(gvk) mgr.GetLogger().Info("installing watcher for newly seen source kind", "GroupVersionKind", gvk) err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &sourceKind, @@ -264,7 +267,7 @@ func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error { enqueueStacksForSourceFunc(fluxSourceIndexFieldName, func(obj client.Object) string { gvk := obj.GetObjectKind().GroupVersionKind() return fluxSourceKey(gvk, obj.GetName()) - })))) + })), &fluxSourceReadyPredicate{}, &auto.DebugPredicate{Controller: "stack-controller"})) if err != nil { watchedMu.Lock() delete(watched, gvk) @@ -344,6 +347,66 @@ func (p ReconcileRequestedPredicate) Update(e event.UpdateEvent) bool { return false // either removed, or present in neither object } +func isWorkspaceReady(ws *autov1alpha1.Workspace) bool { + if ws == nil || ws.Generation != ws.Status.ObservedGeneration { + return false + } + return meta.IsStatusConditionTrue(ws.Status.Conditions, autov1alpha1.WorkspaceReady) +} + +type workspaceReadyPredicate struct{} + +var _ predicate.Predicate = &workspaceReadyPredicate{} + +func (workspaceReadyPredicate) Create(e event.CreateEvent) bool { + return isWorkspaceReady(e.Object.(*autov1alpha1.Workspace)) +} + +func (workspaceReadyPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (workspaceReadyPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !isWorkspaceReady(e.ObjectOld.(*autov1alpha1.Workspace)) && isWorkspaceReady(e.ObjectNew.(*autov1alpha1.Workspace)) +} + +func (workspaceReadyPredicate) Generic(_ event.GenericEvent) bool { + return false +} + +func isUpdateComplete(update *autov1alpha1.Update) bool { + if update == nil || update.Generation != update.Status.ObservedGeneration { + return false + } + return meta.IsStatusConditionTrue(update.Status.Conditions, autov1alpha1.UpdateConditionTypeComplete) +} + +type updateCompletePredicate struct{} + +var _ predicate.Predicate = &updateCompletePredicate{} + +func (updateCompletePredicate) Create(e event.CreateEvent) bool { + return isUpdateComplete(e.Object.(*autov1alpha1.Update)) +} + +func (updateCompletePredicate) Delete(e event.DeleteEvent) bool { + return false +} + +func (updateCompletePredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !isUpdateComplete(e.ObjectOld.(*autov1alpha1.Update)) && isUpdateComplete(e.ObjectNew.(*autov1alpha1.Update)) +} + +func (updateCompletePredicate) Generic(e event.GenericEvent) bool { + return false +} + // StackReconciler reconciles a Stack object type StackReconciler struct { // This client, initialized using mgr.Client() above, is a split client @@ -395,8 +458,7 @@ var errProgramNotFound = fmt.Errorf("unable to retrieve program for stack") // Reconcile reads that state of the cluster for a Stack object and makes changes based on the state read // and what is in the Stack.Spec func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) (res ctrl.Result, reterr error) { - log := log.FromContext(ctx) - log.Info("Reconciling Stack") + log := ctrllog.FromContext(ctx) // Fetch the Stack instance instance := &pulumiv1.Stack{} @@ -407,6 +469,8 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( // Return and don't requeue return reconcile.Result{}, client.IgnoreNotFound(err) } + log = log.WithValues("revision", instance.ResourceVersion) + log.Info("Reconciling Stack") // Update the observed generation and "reconcile request" of the object. instance.Status.ObservedGeneration = instance.GetGeneration() @@ -445,6 +509,8 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( log.Error(err, "unable to save object status") return err } + log = ctrllog.FromContext(ctx).WithValues("revision", instance.ResourceVersion) + log.V(1).Info("Status updated") return nil } @@ -463,7 +529,8 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( return reconcile.Result{}, fmt.Errorf("get current update: %w", err) } - completed := meta.IsStatusConditionTrue(sess.update.Status.Conditions, autov1alpha1.UpdateConditionTypeComplete) + completed := sess.update.Generation == sess.update.Status.ObservedGeneration && + meta.IsStatusConditionTrue(sess.update.Status.Conditions, autov1alpha1.UpdateConditionTypeComplete) if !completed { // wait for the update to complete instance.Status.MarkReconcilingCondition(pulumiv1.ReconcilingProcessingReason, pulumiv1.ReconcilingProcessingUpdateMessage) @@ -619,9 +686,9 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( return reconcile.Result{}, err } - if err := checkFluxSourceReady(sourceObject); err != nil { + if !checkFluxSourceReady(&sourceObject) { // Wait until the source is ready, at which time the watch mechanism will requeue it. - instance.Status.MarkStalledCondition(pulumiv1.StalledSourceUnavailableReason, err.Error()) + instance.Status.MarkStalledCondition(pulumiv1.StalledSourceUnavailableReason, "Flux source not ready") return reconcile.Result{}, saveStatus() } @@ -735,7 +802,7 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( return reconcile.Result{}, fmt.Errorf("unable to create workspace: %w", err) } - if !sess.isWorkspaceReady() { + if !isWorkspaceReady(sess.ws) { // watch the workspace for status updates log.V(1).Info("waiting for workspace to be ready") return reconcile.Result{}, saveStatus() @@ -1190,16 +1257,6 @@ func (sess *stackReconcilerSession) CreateWorkspace(ctx context.Context) error { return nil } -func (sess *stackReconcilerSession) isWorkspaceReady() bool { - if sess.ws == nil { - return false - } - if sess.ws.Generation != sess.ws.Status.ObservedGeneration { - return false - } - return meta.IsStatusConditionTrue(sess.ws.Status.Conditions, autov1alpha1.WorkspaceReady) -} - // setupWorkspace sets all the extra configuration specified by the Stack object, after you have // constructed a workspace from a source. func (sess *stackReconcilerSession) setupWorkspace(ctx context.Context) error { @@ -1396,3 +1453,29 @@ func patchObject[T any, V any](base T, patch V) (*T, error) { return &result, nil } + +// finalizerAddedPredicate detects when a finalizer is added to an object. +// It is used to suppress reconciliation when the stack controller adds its finalizer, which causes +// a generation change that would otherwise trigger reconciliation. +type finalizerAddedPredicate struct{} + +var _ predicate.Predicate = &finalizerAddedPredicate{} + +func (p *finalizerAddedPredicate) Create(_ event.CreateEvent) bool { + return false +} + +func (p *finalizerAddedPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (p *finalizerAddedPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !controllerutil.ContainsFinalizer(e.ObjectOld, pulumiFinalizer) && controllerutil.ContainsFinalizer(e.ObjectNew, pulumiFinalizer) +} + +func (p *finalizerAddedPredicate) Generic(_ event.GenericEvent) bool { + return false +} diff --git a/operator/internal/controller/pulumi/stack_controller_test.go b/operator/internal/controller/pulumi/stack_controller_test.go index 11f341e5..38370b00 100644 --- a/operator/internal/controller/pulumi/stack_controller_test.go +++ b/operator/internal/controller/pulumi/stack_controller_test.go @@ -1155,7 +1155,7 @@ var _ = Describe("Stack Controller", func() { {Type: "Ready", Status: metav1.ConditionFalse, Reason: "Unknown", LastTransitionTime: metav1.Now()}, } }) - beStalled(pulumiv1.StalledSourceUnavailableReason, ContainSubstring("source Ready condition does not have status True")) + beStalled(pulumiv1.StalledSourceUnavailableReason, ContainSubstring("Flux source not ready")) }) When("the flux source is ready", func() {