Skip to content

Commit

Permalink
Clean up "object has already been modified" warnings #700 (#717)
Browse files Browse the repository at this point in the history 
<!--Thanks for your contribution. See [CONTRIBUTING](CONTRIBUTING.md)
    for Pulumi's contribution guidelines.

    Help us merge your changes more quickly by adding more details such
    as labels, milestones, and reviewers.-->

### Proposed changes

<!--Give us a brief description of what you've done and what it solves.
-->

This PR optimizes the watches to reduce the likelihood of a race between
the cache and the reconciliation loop.

The general idea is to use predicates to filter on the 'edge' events
that would allow the reconciler to make forward progress. For example,
the workspace reconciler applies a StatefulSet, then waits for it to be
ready. Only when the StatefulSet is indeed ready should reconciliation
be queued. Before this PR, the watch on StatefulSet was triggering upon
any revision change (way too eager).

Overall, a number of optimizations were implemented:
- workspace controller should wait for the statefulset to be ready
- workspace controller need not watch the Service
- stack controller should wait for flux source to be ready
- stack controller should not trigger reconciliation upon adding its
finalizer (which increments the generation).
- stack controller should wait for the update to complete
- update controller should wait for workspace readiness
- update controller should not enqueue the completed updates of a given
workspace

Also, some debug statements were introduced to reason about the revision
changes over time, since the "conflicts" are detected based on revision.
In particular, we emit a debug statement when reconciliation is
triggered by a given watch, to understand 'why'.

### Related issues (optional)

<!--Refer to related PRs or issues: #1234, or 'Fixes #1234' or 'Closes
#1234'.
Or link to full URLs to issues or pull requests in other GitHub
repositories. -->

Closes #700 
Supercedes #679

### Example

Here's the result of applying
[operator/examples/random-yaml/stack.yaml](https://github.com/pulumi/pulumi-kubernetes-operator/blob/78836991b2691034b2c029df4040ea2016240cd8/operator/examples/random-yaml/stack.yaml).

#### Before
```log
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.032Z    ERROR   updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "841b4d5b-029c-4457-a2b7-8c1821475adf", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:110
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:181
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.033Z    ERROR   Reconciler error        {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "841b4d5b-029c-4457-a2b7-8c1821475adf", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:324
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.047Z    ERROR   unable to save object status    {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "6a99e260-ca2d-4376-b95f-ab490acbe32a", "error": "Operation cannot be fulfilled on stacks.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/pulumi.(*StackReconciler).Reconcile.func1
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/pulumi/stack_controller.go:445
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/pulumi.(*StackReconciler).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/pulumi/stack_controller.go:741
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.047Z    ERROR   Reconciler error        {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "6a99e260-ca2d-4376-b95f-ab490acbe32a", "error": "Operation cannot be fulfilled on stacks.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:324
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.091Z    ERROR   updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d397d7d1-c74b-43fe-b4a0-80c3d39ee3ea", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:110
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:188
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.091Z    ERROR   Reconciler error        {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d397d7d1-c74b-43fe-b4a0-80c3d39ee3ea", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:324
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.111Z    ERROR   updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "b94710c8-8165-4937-a99c-a4e065f63788", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:110
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:188
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.111Z    ERROR   Reconciler error        {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "b94710c8-8165-4937-a99c-a4e065f63788", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:324
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:32.082Z    ERROR   updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "7cb6660a-508c-4392-80cd-749fe72def9f", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:110
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:252
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:32.082Z    ERROR   Reconciler error        {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "7cb6660a-508c-4392-80cd-749fe72def9f", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:324
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:36.114Z    ERROR   updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "480ad23a-2d02-40a6-b4a4-198fb9df01c4", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:110
controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/operator/internal/controller/auto/workspace_controller.go:316
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:36.114Z    ERROR   Reconciler error        {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "480ad23a-2d02-40a6-b4a4-198fb9df01c4", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"}
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:324
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:261
controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
controller-manager-6f849dcb6b-4wscw manager     /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:222
```

#### After
```log

controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.212Z    INFO    Reconciling Stack       {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "585499fc-fb33-4505-8594-6a9fd8c810b2", "revision": "893372"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z    INFO    KubeAPIWarningLogger    metadata.finalizers: "finalizer.stack.pulumi.com": prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z    INFO    installing watcher for newly seen source kind   {"GroupVersionKind": "source.toolkit.fluxcd.io/v1, Kind=GitRepository"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z    INFO    Starting EventSource    {"controller": "stack-controller", "source": "kind source: *unstructured.Unstructured"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.238Z    INFO    Reconciling Workspace   {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d1595cfb-0fe6-46d5-9e9c-2227f334eb7f", "revision": "893374"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.238Z    INFO    Applying StatefulSet    {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d1595cfb-0fe6-46d5-9e9c-2227f334eb7f", "revision": "893374", "hash": "608b7e170ccd95754972cde1178ab426", "source": {"Generation":1,"ForceRequest":"","Git":null,"Flux":{"Url":"http://source-controller.flux-system.svc.cluster.local./gitrepository/default/pulumi-examples/1432c2ce26299516223c0ad228e92294eba4b2c1.tar.gz","Digest":"sha256:44d554df090dcdeb9bae908cd38155c8c93db05f64dc72982027e8e1294af0d3","Dir":"random-yaml/"}}}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.609Z    INFO    Reconciling Workspace   {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.609Z    INFO    Applying StatefulSet    {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "hash": "608b7e170ccd95754972cde1178ab426", "source": {"Generation":1,"ForceRequest":"","Git":null,"Flux":{"Url":"http://source-controller.flux-system.svc.cluster.local./gitrepository/default/pulumi-examples/1432c2ce26299516223c0ad228e92294eba4b2c1.tar.gz","Digest":"sha256:44d554df090dcdeb9bae908cd38155c8c93db05f64dc72982027e8e1294af0d3","Dir":"random-yaml/"}}}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.719Z    INFO    Connecting to workspace pod     {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.724Z    INFO    Connected to workspace pod      {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.724Z    INFO    Running pulumi install  {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:51.509Z    INFO    Creating Pulumi stack(s)        {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893413"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.250Z    INFO    workspace pod initialized       {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893422"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.250Z    INFO    Ready   {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893422"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.255Z    INFO    Reconciling Stack       {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "910a2faa-5b49-4590-a23a-d9ef8cab0023", "revision": "893375"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.271Z    INFO    Reconciling Update      {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.271Z    INFO    Updating the status     {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.275Z    INFO    Connecting      {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.278Z    INFO    Selecting the stack     {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "stackName": "dev"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:57.333Z    INFO    Applying the update     {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "type": "up"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:57.334Z    INFO    Executing update operation      {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "request": "message:\"Stack Update (up)\"  target_dependents:false  refresh:true"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.421Z    INFO    Update complete {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "result": "stdout:\"Updating (dev)\\n\\nView Live: https://app.pulumi.com/eron-pulumi-corp/random/dev/updates/4536\\n\\n+ pulumi:pulumi:Stack: (create)\\n    [urn=urn:pulumi:dev::random::pulumi:pulumi:Stack::random-dev]\\n    + random:index/randomPassword:RandomPassword: (create)\\n        [urn=urn:pulumi:dev::random::random:index/randomPassword:RandomPassword::randomPassword]\\n        length         : 16\\n        overrideSpecial: \\\"_%@\\\"\\n        special        : true\\n        --outputs:--\\n        bcryptHash     : [secret]\\n        id             : \\\"none\\\"\\n        lower          : true\\n        minLower       : 0\\n        minNumeric     : 0\\n        minSpecial     : 0\\n        minUpper       : 0\\n        number         : true\\n        numeric        : true\\n        result         : [secret]\\n        upper          : true\\n    --outputs:--\\n    password: [secret]\\nResources:\\n    + 2 created\\n\\nDuration: 3s\\n\"  summary:{start_time:{seconds:1729044119}  end_time:{seconds:1729044122}  result:\"succeeded\"  message:\"\\\"Stack Update (up)\\\"\"}  permalink:\"https://app.pulumi.com/eron-pulumi-corp/random/dev/updates/4536\"  outputs:{key:\"password\"  value:{value:\"\\\"fAMcmXI2DfW7KCey\\\"\"  secret:true}}"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.438Z    INFO    Reconciling Stack       {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "a100c4f3-c5b9-4894-8644-fabc1f71d008", "revision": "893437"}
controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.546Z    INFO    Commit hash unchanged. Will wait for Source update or resync.   {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "a100c4f3-c5b9-4894-8644-fabc1f71d008", "revision": "893437"}
```
  • Loading branch information
@EronWright
EronWright authored Oct 17, 2024
1 parent 7883699 commit 2884fad
Show file tree
Hide file tree
Showing 7 changed files with 294 additions and 67 deletions.
2 changes: 1 addition & 1 deletion operator/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
.PHONY: deploy
deploy: manifests kustomize ## Deploy controller manager to the K8s cluster specified in ~/.kube/config.
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
$(KUSTOMIZE) build config/default | $(KUBECTL) apply --server-side=true -oyaml -f -
$(KUSTOMIZE) build config/default | $(KUBECTL) apply --server-side=true -f -

.PHONY: undeploy
undeploy: ## Undeploy controller manager from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
Expand Down
71 changes: 65 additions & 6 deletions operator/internal/controller/auto/update_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -41,6 +42,7 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
Expand Down Expand Up @@ -77,13 +79,15 @@ type UpdateReconciler struct {
// Reconcile manages the Update CRD and initiates Pulumi operations.
func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
l := log.FromContext(ctx)
l.Info("Reconciling Update")

obj := &autov1alpha1.Update{}
err := r.Get(ctx, req.NamespacedName, obj)
if err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
l = l.WithValues("revision", obj.ResourceVersion)
ctx = log.IntoContext(ctx, l)
l.Info("Reconciling Update")

rs := newReconcileSession(r.Client, obj)

Expand Down Expand Up @@ -116,12 +120,32 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
return ctrl.Result{}, fmt.Errorf("failed to update the status: %w", err)
}

// TODO check the w status before proceeding
// Get the workspace and check that it is ready
w := &autov1alpha1.Workspace{}
err = r.Client.Get(ctx, client.ObjectKey{Namespace: obj.Namespace, Name: obj.Spec.WorkspaceName}, w)
if err != nil {
if apierrors.IsNotFound(err) {
l.Info("Workspace not found", "workspace", obj.Spec.WorkspaceName)
rs.progressing.Status = metav1.ConditionFalse
rs.progressing.Reason = "WorkspaceNotFound"
rs.failed.Status = metav1.ConditionFalse
rs.failed.Reason = UpdateConditionReasonProgressing
rs.complete.Status = metav1.ConditionFalse
rs.complete.Reason = UpdateConditionReasonProgressing
return ctrl.Result{}, rs.updateStatus(ctx, obj)
}
return ctrl.Result{}, fmt.Errorf("failed to get workspace: %w", err)
}
if !isWorkspaceReady(w) {
l.Info("Workspace not ready", "workspace", w.Name)
rs.progressing.Status = metav1.ConditionFalse
rs.progressing.Reason = "WorkspaceNotReady"
rs.failed.Status = metav1.ConditionFalse
rs.failed.Reason = UpdateConditionReasonProgressing
rs.complete.Status = metav1.ConditionFalse
rs.complete.Reason = UpdateConditionReasonProgressing
return ctrl.Result{}, rs.updateStatus(ctx, obj)
}

// Connect to the workspace's GRPC server
addr := fmt.Sprintf("%s:%d", fqdnForService(w), WorkspaceGrpcPort)
Expand Down Expand Up @@ -168,6 +192,36 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
}
}

func isWorkspaceReady(ws *autov1alpha1.Workspace) bool {
if ws == nil || ws.Generation != ws.Status.ObservedGeneration {
return false
}
return meta.IsStatusConditionTrue(ws.Status.Conditions, autov1alpha1.WorkspaceReady)
}

type workspaceReadyPredicate struct{}

var _ predicate.Predicate = &workspaceReadyPredicate{}

func (workspaceReadyPredicate) Create(e event.CreateEvent) bool {
return isWorkspaceReady(e.Object.(*autov1alpha1.Workspace))
}

func (workspaceReadyPredicate) Delete(_ event.DeleteEvent) bool {
return false
}

func (workspaceReadyPredicate) Update(e event.UpdateEvent) bool {
if e.ObjectOld == nil || e.ObjectNew == nil {
return false
}
return !isWorkspaceReady(e.ObjectOld.(*autov1alpha1.Workspace)) && isWorkspaceReady(e.ObjectNew.(*autov1alpha1.Workspace))
}

func (workspaceReadyPredicate) Generic(_ event.GenericEvent) bool {
return false
}

type reconcileSession struct {
progressing *metav1.Condition
complete *metav1.Condition
Expand Down Expand Up @@ -375,13 +429,18 @@ func (r *UpdateReconciler) SetupWithManager(mgr ctrl.Manager) error {
For(&autov1alpha1.Update{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Watches(&autov1alpha1.Workspace{},
handler.EnqueueRequestsFromMapFunc(r.mapWorkspaceToUpdate),
builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})).
builder.WithPredicates(&workspaceReadyPredicate{})).
Complete(r)
}

func indexUpdateByWorkspace(obj client.Object) []string {
w := obj.(*autov1alpha1.Update)
return []string{w.Spec.WorkspaceName}
u := obj.(*autov1alpha1.Update)
complete := meta.IsStatusConditionTrue(u.Status.Conditions, UpdateConditionTypeComplete)
if complete {
// don't index the completed updates, to avoid unnecessary reconciles when their workspace is updated
return []string{}
}
return []string{u.Spec.WorkspaceName}
}

func (r *UpdateReconciler) mapWorkspaceToUpdate(ctx context.Context, obj client.Object) []reconcile.Request {
Expand Down Expand Up @@ -494,7 +553,7 @@ func (s streamReader[T]) Result() (result, error) {
continue // No result yet.
}

s.l.Info("Result received", "result", res)
s.l.Info("Update complete", "result", res)

s.obj.Status.StartTime = metav1.NewTime(res.GetSummary().StartTime.AsTime())
s.obj.Status.EndTime = metav1.NewTime(res.GetSummary().EndTime.AsTime())
Expand Down
31 changes: 31 additions & 0 deletions operator/internal/controller/auto/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ import (
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/protobuf/types/known/structpb"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
)

func connect(ctx context.Context, addr string) (*grpc.ClientConn, error) {
Expand Down Expand Up @@ -84,3 +87,31 @@ func marshalConfigValue(item autov1alpha1.ConfigItem) *agentpb.ConfigValue {
}
return v
}

var l = log.Log.WithName("predicate").WithName("debug")

type DebugPredicate struct {
Controller string
}

var _ predicate.Predicate = &DebugPredicate{}

func (p *DebugPredicate) Create(e event.CreateEvent) bool {
l.V(1).Info("Create", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion())
return true
}

func (p *DebugPredicate) Delete(e event.DeleteEvent) bool {
l.V(1).Info("Delete", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion())
return true
}

func (p *DebugPredicate) Update(e event.UpdateEvent) bool {
l.V(1).Info("Update", "controller", p.Controller, "type", fmt.Sprintf("%T", e.ObjectOld), "name", e.ObjectOld.GetName(), "old-revision", e.ObjectOld.GetResourceVersion(), "new-revision", e.ObjectNew.GetResourceVersion())
return true
}

func (p *DebugPredicate) Generic(e event.GenericEvent) bool {
l.V(1).Info("Generic", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion())
return true
}
48 changes: 43 additions & 5 deletions operator/internal/controller/auto/workspace_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
)
Expand Down Expand Up @@ -78,13 +79,14 @@ type WorkspaceReconciler struct {

func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
l := log.FromContext(ctx)
l.Info("Reconciling Workspace")

w := &autov1alpha1.Workspace{}
err := r.Get(ctx, req.NamespacedName, w)
if err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
l = l.WithValues("revision", w.ResourceVersion)
l.Info("Reconciling Workspace")

// apply defaults to the workspace spec
// future: use a mutating webhook to apply defaults
Expand All @@ -108,7 +110,11 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
err := r.Status().Update(ctx, w)
if err != nil {
l.Error(err, "updating status")
} else {
l = log.FromContext(ctx).WithValues("revision", w.ResourceVersion)
l.V(1).Info("Status updated")
}

return err
}

Expand Down Expand Up @@ -319,15 +325,47 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
// SetupWithManager sets up the controller with the Manager.
func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
Named("workspace-controller").
For(&autov1alpha1.Workspace{},
builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Owns(&corev1.Service{},
builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})).
builder.WithPredicates(predicate.GenerationChangedPredicate{}, &DebugPredicate{Controller: "workspace-controller"})).
Owns(&appsv1.StatefulSet{},
builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})).
builder.WithPredicates(&statefulSetReadyPredicate{}, &DebugPredicate{Controller: "workspace-controller"})).
Complete(r)
}

type statefulSetReadyPredicate struct{}

var _ predicate.Predicate = &statefulSetReadyPredicate{}

func isStatefulSetReady(ss *appsv1.StatefulSet) bool {
if ss.Status.ObservedGeneration != ss.Generation || ss.Status.UpdateRevision != ss.Status.CurrentRevision {
return false
}
if ss.Status.AvailableReplicas < 1 {
return false
}
return true
}

func (statefulSetReadyPredicate) Create(e event.CreateEvent) bool {
return isStatefulSetReady(e.Object.(*appsv1.StatefulSet))
}

func (statefulSetReadyPredicate) Delete(_ event.DeleteEvent) bool {
return false
}

func (statefulSetReadyPredicate) Update(e event.UpdateEvent) bool {
if e.ObjectOld == nil || e.ObjectNew == nil {
return false
}
return !isStatefulSetReady(e.ObjectOld.(*appsv1.StatefulSet)) && isStatefulSetReady(e.ObjectNew.(*appsv1.StatefulSet))
}

func (statefulSetReadyPredicate) Generic(_ event.GenericEvent) bool {
return false
}

const (
FieldManager = "pulumi-kubernetes-operator"
WorkspacePulumiContainerName = "pulumi"
Expand Down
70 changes: 43 additions & 27 deletions operator/internal/controller/pulumi/flux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"github.com/pulumi/pulumi-kubernetes-operator/v2/operator/api/pulumi/shared"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"
)

func (sess *stackReconcilerSession) SetupWorkspaceFromFluxSource(ctx context.Context, source unstructured.Unstructured, fluxSource *shared.FluxSource) (string, error) {
Expand Down Expand Up @@ -70,37 +72,28 @@ func checksumOrDigest(source unstructured.Unstructured) (string, error) {
return checksum, nil
}

// checkFluxSourceReady looks for the conventional "Ready" condition to see if the supplied object
// can be considered _not_ ready. It returns an error if it can determine that the object is not
// ready, and nil if it cannot determine so.
func checkFluxSourceReady(obj unstructured.Unstructured) error {
func checkFluxSourceReady(obj *unstructured.Unstructured) bool {
observedGeneration, ok, err := unstructured.NestedInt64(obj.Object, "status", "observedGeneration")
if !ok || err != nil || observedGeneration != obj.GetGeneration() {
return false
}
conditions, ok, err := unstructured.NestedSlice(obj.Object, "status", "conditions")
if ok && err == nil {
// didn't find a []Condition, so there's nothing to indicate that it's not ready there
for _, c0 := range conditions {
var c map[string]interface{}
if c, ok = c0.(map[string]interface{}); !ok {
// condition isn't the right shape, try the next one
continue
}
if t, ok, err := unstructured.NestedString(c, "type"); ok && err == nil && t == "Ready" {
if v, ok, err := unstructured.NestedString(c, "status"); ok && err == nil && v == "True" {
// found the Ready condition and it is actually ready; proceed to next check
break
}
// found the Ready condition and it's something other than ready
return fmt.Errorf("source Ready condition does not have status True %#v", c)
if !ok || err != nil {
return false
}
for _, c0 := range conditions {
var c map[string]interface{}
if c, ok = c0.(map[string]interface{}); !ok {
// condition isn't the right shape, try the next one
continue
}
if t, ok, err := unstructured.NestedString(c, "type"); ok && err == nil && t == "Ready" {
if v, ok, err := unstructured.NestedString(c, "status"); ok && err == nil && v == "True" {
return true
}
}
// Ready=true, or no ready condition to tell us either way
}

_, ok, err = unstructured.NestedMap(obj.Object, "status", "artifact")
if !ok || err != nil {
return fmt.Errorf(".status.artifact does not have an Artifact object")
}

return nil
return false
}

func getSourceGVK(src shared.FluxSourceReference) (schema.GroupVersionKind, error) {
Expand All @@ -111,3 +104,26 @@ func getSourceGVK(src shared.FluxSourceReference) (schema.GroupVersionKind, erro
func fluxSourceKey(gvk schema.GroupVersionKind, name string) string {
return fmt.Sprintf("%s:%s", gvk, name)
}

type fluxSourceReadyPredicate struct{}

var _ predicate.Predicate = &fluxSourceReadyPredicate{}

func (fluxSourceReadyPredicate) Create(e event.CreateEvent) bool {
return checkFluxSourceReady(e.Object.(*unstructured.Unstructured))
}

func (fluxSourceReadyPredicate) Delete(_ event.DeleteEvent) bool {
return false
}

func (fluxSourceReadyPredicate) Update(e event.UpdateEvent) bool {
if e.ObjectOld == nil || e.ObjectNew == nil {
return false
}
return !checkFluxSourceReady(e.ObjectOld.(*unstructured.Unstructured)) && checkFluxSourceReady(e.ObjectNew.(*unstructured.Unstructured))
}

func (fluxSourceReadyPredicate) Generic(_ event.GenericEvent) bool {
return false
}
Loading

0 comments on commit 2884fad

Please sign in to comment.