From 78836991b2691034b2c029df4040ea2016240cd8 Mon Sep 17 00:00:00 2001 From: Eron Wright Date: Fri, 11 Oct 2024 17:10:23 -0700 Subject: [PATCH 1/2] Implement an authorization layer for operator-to-workspace communication (#712) ### Overview This PR implements an authentication and authorization layer for the agent's RPC endpoint. Authentication is performed by authenticating a bearer token via the TokenReview API. The operator uses its built-in service account token. Authorization is performed via the SubjectAccessReview API, which checks for following RBAC permission: ```yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role rules: - apiGroups: - auto.pulumi.com resources: - workspaces/rpc verbs: - use ``` The workspace pod's service account must be granted the `system:auth-delegator` role using a `ClusterRoleBinding`. For. convenience, the installer creates a service account named `pulumi` into the `default` namespace, with an associated binding. The operator itself is granted the necessary permission to access the RPC endpoint. ### Proposed changes - [x] agent grpc interceptor - [x] agent command args (`--auth-mode=kube`, `--kube-workspace-name=random-yaml`) - [x] operator client credentials - [x] operator RBAC permissions - [ ] ~cluster role binding for workspace service account (to ClusterRole named `system:auth-delegator`)~ - [x] install a "default/pulumi" service account with RBAC - [x] Provide a Flux sample network policy - [x] Update the e2e test manifests to have requisite account, rbac, and network policy. ### Future Enhancement This implementation uses the operator's default service account token, but to further improve security it should use an audience-scoped token, where the audience is the agent service address as opposed to the API server. Such tokens may be created by the operator with a call to TokenRequest, and checked with TokenReview by adding the expected audience to the context (`authenticator.WithAudience`). ### Related issues (optional) Closes #609 #### Examples Some example requests: ``` random-yaml-workspace-0 pulumi 2024-10-09T21:09:43.905Z INFO cmd.serve.grpc finished unary call with code OK {"grpc.start_time": "2024-10-09T21:09:43Z", "grpc.request.deadline": "2024-10-09T21:59:43Z", "system": "grpc", "span.kind": "server", "grpc.service": "agent.AutomationService", "grpc.method": "WhoAmI", "user.id": "81be050c-9ad4-4708-9a52-413064700747", "user.name": "system:serviceaccount:default:dev", "peer.address": "127.0.0.1:56394", "auth.mode": "kubernetes", "grpc.code": "OK", "grpc.time_ms": 441.086} random-yaml-workspace-0 pulumi 2024-10-09T21:09:52.934Z INFO cmd.serve.grpc finished unary call with code Unauthenticated {"grpc.start_time": "2024-10-09T21:09:52Z", "grpc.request.deadline": "2024-10-09T21:59:52Z", "system": "grpc", "span.kind": "server", "grpc.service": "agent.AutomationService", "grpc.method": "WhoAmI", "peer.address": "127.0.0.1:57380", "auth.mode": "kubernetes", "error": "rpc error: code = Unauthenticated desc = Request unauthenticated with Bearer", "grpc.code": "Unauthenticated", "grpc.time_ms": 0.095} ``` --- .vscode/launch.json | 19 +- agent/cmd/root.go | 23 +- agent/cmd/serve.go | 78 ++++-- agent/hack/token.yaml | 27 ++ agent/pkg/client/client.go | 75 ++++++ agent/pkg/server/auth.go | 231 ++++++++++++++++ agent/pkg/server/auth_test.go | 248 ++++++++++++++++++ agent/pkg/server/grpc.go | 16 +- agent/pkg/server/grpc_test.go | 2 +- .../templates/clusterrole.yaml | 6 + .../helm/pulumi-operator/templates/role.yaml | 6 + deploy/yaml/install.yaml | 84 +++++- go.mod | 8 +- go.sum | 17 +- operator/Makefile | 2 +- operator/config/flux/network_policy.yaml | 26 ++ operator/config/quickstart/kustomization.yaml | 4 + operator/config/quickstart/rbac.yaml | 14 + .../config/quickstart/service_account.yaml | 8 + operator/config/rbac/role.yaml | 6 + .../testdata/git-auth-nonroot/manifests.yaml | 20 ++ .../random-yaml-nonroot/manifests.yaml | 46 +++- operator/e2e/testdata/targets/manifests.yaml | 20 ++ operator/examples/random-yaml/stack.yaml | 21 ++ .../controller/auto/update_controller.go | 1 + operator/internal/controller/auto/utils.go | 18 +- .../controller/auto/workspace_controller.go | 27 +- 27 files changed, 1018 insertions(+), 35 deletions(-) create mode 100644 agent/hack/token.yaml create mode 100644 agent/pkg/client/client.go create mode 100644 agent/pkg/server/auth.go create mode 100644 agent/pkg/server/auth_test.go create mode 100644 operator/config/flux/network_policy.yaml create mode 100644 operator/config/quickstart/kustomization.yaml create mode 100644 operator/config/quickstart/rbac.yaml create mode 100644 operator/config/quickstart/service_account.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index 50127707..5a9d4d62 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -45,9 +45,26 @@ "-v=false", "--workspace=${input:workdir}", "-s=dev" + ] + }, + { + "name": "Agent (kubernetes)", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "agent", + "args": [ + "serve", + "-v=true", + "--workspace=${input:workdir}", + "-s=dev", + "--auth-mode=kube", + "--kube-workspace-namespace=default", + "--kube-workspace-name=random-yaml" ], "env": { - "AWS_REGION": "us-west-1", + "POD_NAMESPACE": "default", + "POD_SA_NAME": "fake" } } ], diff --git a/agent/cmd/root.go b/agent/cmd/root.go index 5bc3a741..fe9d30cb 100644 --- a/agent/cmd/root.go +++ b/agent/cmd/root.go @@ -18,11 +18,18 @@ package cmd import ( "os" + "flag" + "github.com/spf13/cobra" "go.uber.org/zap" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/config" ) -var verbose bool +var ( + verbose bool + kubeContext string +) // a command-specific logger var log *zap.SugaredLogger @@ -51,6 +58,7 @@ to use to perform stack operations.`, // initialize a command-specific logger log = zap.L().Named("cmd").Named(cmd.Name()).Sugar() + cmd.SilenceErrors = true return nil }, PersistentPostRun: func(cmd *cobra.Command, args []string) { @@ -64,10 +72,23 @@ to use to perform stack operations.`, func Execute() { err := rootCmd.Execute() if err != nil { + if log != nil { + log.Error(err.Error()) + } os.Exit(1) } } func init() { rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose logging") + + // register the Kubernetes flags (e.g. for serve command when using Kubernetes RBAC for authorization) + fs := flag.NewFlagSet("kubernetes", flag.ExitOnError) + config.RegisterFlags(fs) + rootCmd.PersistentFlags().AddGoFlagSet(fs) + rootCmd.PersistentFlags().StringVar(&kubeContext, "context", "", "Kubernetes context override") +} + +func GetKubeConfig() (*rest.Config, error) { + return config.GetConfigWithContext(kubeContext) } diff --git a/agent/cmd/serve.go b/agent/cmd/serve.go index 8a1a13fd..6499bd5d 100644 --- a/agent/cmd/serve.go +++ b/agent/cmd/serve.go @@ -25,6 +25,7 @@ import ( "runtime/debug" "syscall" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/server" "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/version" "github.com/pulumi/pulumi/sdk/v3/go/auto" @@ -32,6 +33,12 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapio" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" +) + +const ( + AuthModeNone = "none" + AuthModeKubernetes = "kube" ) var ( @@ -40,6 +47,10 @@ var ( _stack string _host string _port int + + _authMode string + _workspaceNamespace string + _workspaceName string ) // serveCmd represents the serve command @@ -48,7 +59,22 @@ var serveCmd = &cobra.Command{ Short: "Serve the agent RPC service", Long: `Start the agent gRPC server. `, - Run: func(cmd *cobra.Command, args []string) { + PreRunE: func(cmd *cobra.Command, args []string) error { + if _authMode != AuthModeNone && _authMode != AuthModeKubernetes { + return fmt.Errorf("unsupported auth mode: %s", _authMode) + } + if _authMode == AuthModeKubernetes { + if _workspaceNamespace == "" { + return fmt.Errorf("--kube-workspace-namespace is required when auth mode is kubernetes") + } + if _workspaceName == "" { + return fmt.Errorf("--kube-workspace-name is required when auth mode is kubernetes") + } + } + cmd.SilenceUsage = true + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() log.Infow("Pulumi Kubernetes Agent", "version", version.Version) @@ -63,23 +89,42 @@ var serveCmd = &cobra.Command{ } } + // Prepare the authorizer function + var authFunc grpc_auth.AuthFunc + switch _authMode { + case AuthModeKubernetes: + kubeConfig, err := GetKubeConfig() + if err != nil { + return fmt.Errorf("unable to load the kubeconfig: %w", err) + } + + authFunc, err = server.NewKubeAuth(log.Desugar(), kubeConfig, server.KubeAuthOptions{ + WorkspaceName: types.NamespacedName{ + Namespace: _workspaceNamespace, + Name: _workspaceName, + }, + }) + if err != nil { + return fmt.Errorf("unable to initialize the Kubernetes authorizer: %w", err) + } + log.Infow("activated the Kubernetes authorization mode", + zap.String("workspace.namespace", _workspaceNamespace), zap.String("workspace.name", _workspaceName)) + } + // open the workspace using auto api workspaceOpts := []auto.LocalWorkspaceOption{} workDir, err := filepath.EvalSymlinks(_workDir) // resolve the true location of the workspace if err != nil { - log.Fatalw("unable to resolve the workspace directory", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to resolve the workspace directory: %w", err) } workspaceOpts = append(workspaceOpts, auto.WorkDir(workDir)) workspace, err := auto.NewLocalWorkspace(ctx, workspaceOpts...) if err != nil { - log.Fatalw("unable to open the workspace", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to open the workspace: %w", err) } proj, err := workspace.ProjectSettings(ctx) if err != nil { - log.Fatalw("unable to get the project settings", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to get the project settings: %w", err) } log.Infow("opened a local workspace", "workspace", workDir, "project", proj.Name, "runtime", proj.Runtime.Name()) @@ -96,8 +141,7 @@ var serveCmd = &cobra.Command{ } log.Infow("installing project dependencies") if err := workspace.Install(ctx, opts); err != nil { - log.Fatalw("installation failed", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to install project dependencies: %w", err) } log.Infow("installation completed") } else { @@ -110,30 +154,28 @@ var serveCmd = &cobra.Command{ StackName: _stack, }) if err != nil { - log.Fatalw("unable to make an automation server", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to make an automation server: %w", err) } address := fmt.Sprintf("%s:%d", _host, _port) log.Infow("starting the RPC server", "address", address) - s := server.NewGRPC(autoServer, log) + s := server.NewGRPC(log, autoServer, authFunc) // Start the grpc server lis, err := net.Listen("tcp", address) if err != nil { - log.Errorw("fatal: unable to start the RPC server", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to listen on %s: %w", address, err) } log.Infow("server listening", "address", lis.Addr(), "workspace", workDir) ctx, cancel := context.WithCancel(ctx) setupSignalHandler(cancel) if err := s.Serve(ctx, lis); err != nil { - log.Errorw("fatal: server failure", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unexpected serve error: %w", err) } log.Infow("server stopped") + return nil }, } @@ -166,4 +208,8 @@ func init() { serveCmd.Flags().StringVar(&_host, "host", "0.0.0.0", "Server bind address (default: 0.0.0.0)") serveCmd.Flags().IntVar(&_port, "port", 50051, "Server port (default: 50051)") + + serveCmd.Flags().StringVar(&_authMode, "auth-mode", AuthModeNone, "Authorization mode (none, kube)") + serveCmd.Flags().StringVar(&_workspaceNamespace, "kube-workspace-namespace", os.Getenv("WORKSPACE_NAMESPACE"), "The Workspace object namespace (for kubernetes auth mode)") + serveCmd.Flags().StringVar(&_workspaceName, "kube-workspace-name", os.Getenv("WORKSPACE_NAME"), "The Workspace object name (for kubernetes auth mode)") } diff --git a/agent/hack/token.yaml b/agent/hack/token.yaml new file mode 100644 index 00000000..aa48390c --- /dev/null +++ b/agent/hack/token.yaml @@ -0,0 +1,27 @@ +# Apply this manifest file to create a token with which to authenticate to the agent. +# To get the token, run the following command: kubectl describe secret/dev-token +# To test: kubectl auth can-i use workspaces/random-yaml --subresource rpc --as system:serviceaccount:default:dev +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dev +--- +apiVersion: v1 +kind: Secret +metadata: + name: dev-token + annotations: + kubernetes.io/service-account.name: dev +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: dev:cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: dev \ No newline at end of file diff --git a/agent/pkg/client/client.go b/agent/pkg/client/client.go new file mode 100644 index 00000000..57800359 --- /dev/null +++ b/agent/pkg/client/client.go @@ -0,0 +1,75 @@ +/* +Copyright © 2024 Pulumi Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + + "golang.org/x/oauth2" + "google.golang.org/grpc/credentials" + "k8s.io/client-go/transport" +) + +// NewTokenCredentials adds the provided bearer token to a request. +// If tokenFile is non-empty, it is periodically read, +// and the last successfully read content is used as the bearer token. +// If tokenFile is non-empty and bearer is empty, the tokenFile is read +// immediately to populate the initial bearer token. +func NewTokenCredentials(bearer string, tokenFile string) (*TokenCredentials, error) { + if len(tokenFile) == 0 { + return &TokenCredentials{bearer, nil}, nil + } + source := transport.NewCachedFileTokenSource(tokenFile) + if len(bearer) == 0 { + token, err := source.Token() + if err != nil { + return nil, err + } + bearer = token.AccessToken + } + return &TokenCredentials{bearer, source}, nil +} + +type TokenCredentials struct { + bearer string + source oauth2.TokenSource +} + +// GetRequestMetadata gets the current request metadata, refreshing tokens +// if required. This should be called by the transport layer on each +// request, and the data should be populated in headers or other +// context. If a status code is returned, it will be used as the status for +// the RPC (restricted to an allowable set of codes as defined by gRFC +// A54). uri is the URI of the entry point for the request. When supported +// by the underlying implementation, ctx can be used for timeout and +// cancellation. Additionally, RequestInfo data will be available via ctx +// to this call. +func (k *TokenCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + token := k.bearer + if k.source != nil { + if refreshedToken, err := k.source.Token(); err == nil { + token = refreshedToken.AccessToken + } + } + return map[string]string{"authorization": "Bearer " + token}, nil +} + +func (k *TokenCredentials) RequireTransportSecurity() bool { + return false +} + +var _ credentials.PerRPCCredentials = &TokenCredentials{} diff --git a/agent/pkg/server/auth.go b/agent/pkg/server/auth.go new file mode 100644 index 00000000..e147d83c --- /dev/null +++ b/agent/pkg/server/auth.go @@ -0,0 +1,231 @@ +/* +Copyright © 2024 Pulumi Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "fmt" + "os" + "time" + + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" + grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/token/cache" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" + authenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1" + authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1" + "k8s.io/client-go/rest" +) + +const ( + ServiceAccountPermissionsErrorMessage = ` +The stack's service account needs permission to access the Kubernetes API. +Please add a ClusterRoleBinding for the ClusterRole 'system:auth-delegator'. +For example: + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: %s:%s:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + namespace: %s + name: %s + +` +) + +func formattedServiceAccountPermissionsErrorMessage() string { + saName := types.NamespacedName{ + Namespace: os.Getenv("POD_NAMESPACE"), + Name: os.Getenv("POD_SA_NAME"), + } + return fmt.Sprintf(ServiceAccountPermissionsErrorMessage, saName.Namespace, saName.Name, saName.Namespace, saName.Name) +} + +type KubeAuthOptions struct { + WorkspaceName types.NamespacedName +} + +// NewKubeAuth provides a grpc_auth.AuthFunc for authentication and authorization. +// Requests will be authenticated (via TokenReviews) and authorized (via SubjectAccessReviews) with the +// kube-apiserver. +// For the authentication and authorization the agent needs a role +// with the following rules: +// * apiGroups: authentication.k8s.io, resources: tokenreviews, verbs: create +// * apiGroups: authorization.k8s.io, resources: subjectaccessreviews, verbs: create +// +// To make RPC requests e.g. as the Operator the client needs a role +// with the following rule: +// * apiGroups: auto.pulumi.com, resources: workspaces/rpc, verbs: use +func NewKubeAuth(rootLogger *zap.Logger, config *rest.Config, opts KubeAuthOptions) (grpc_auth.AuthFunc, error) { + authenticationV1Client, err := authenticationv1.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create authentication client: %w", err) + } + authorizationV1Client, err := authorizationv1.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create authorization client: %w", err) + } + + tokenAuth, err := webhooktoken.NewFromInterface( + authenticationV1Client, + nil, + wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + 10*time.Second, /* requestTimeout */ + webhooktoken.AuthenticatorMetrics{ + RecordRequestTotal: RecordRequestTotal, + RecordRequestLatency: RecordRequestLatency, + }) + if err != nil { + return nil, fmt.Errorf("failed to create webhook authenticator: %w", err) + } + delegatingAuthenticator := cache.New(tokenAuth, false, 1*time.Hour /* successTTL */, 1*time.Minute /* failureTTL */) + + authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{ + SubjectAccessReviewClient: authorizationV1Client, + AllowCacheTTL: 5 * time.Minute, + DenyCacheTTL: 5 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthorizer, err := authorizerConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authorizer: %w", err) + } + + a := &kubeAuth{ + log: rootLogger.Named("grpc").Sugar(), + authn: delegatingAuthenticator, + authz: delegatingAuthorizer, + workspaceName: opts.WorkspaceName, + } + return a.Authenticate, nil +} + +type kubeAuth struct { + log *zap.SugaredLogger + authn authenticator.Token + authz authorizer.Authorizer + workspaceName types.NamespacedName +} + +// Authenticate implements grpc_auth.AuthFunc to perform authentication and authorization. +// Authentication is done via TokenReview and authorization via SubjectAccessReview. +// +// The passed in `Context` will contain the gRPC metadata.MD object (for header-based authentication) and +// the peer.Peer information that can contain transport-based credentials (e.g. `credentials.AuthInfo`). +// +// The returned context will be propagated to handlers, allowing user changes to `Context`. However, +// please make sure that the `Context` returned is a child `Context` of the one passed in. +// +// If error is returned, its `grpc.Code()` will be returned to the user as well as the verbatim message. +// Please make sure you use `codes.Unauthenticated` (lacking auth) and `codes.PermissionDenied` +// (authed, but lacking perms) appropriately. +func (a *kubeAuth) Authenticate(ctx context.Context) (context.Context, error) { + tags := grpc_ctxtags.Extract(ctx) + tags.Set("auth.mode", "kubernetes") + + token, err := grpc_auth.AuthFromMD(ctx, "Bearer") + if err != nil { + return nil, err + } + + // Authenticate the user. + res, ok, err := a.authn.AuthenticateToken(ctx, token) + if err != nil { + if apierrors.IsForbidden(err) { + a.log.Errorw(formattedServiceAccountPermissionsErrorMessage()) + return nil, status.Error(codes.Unauthenticated, "TokenReview API is unavailable") + } + a.log.Warn("authentication failed with an error", zap.Error(err)) + return nil, status.Error(codes.Unauthenticated, err.Error()) + } + if !ok { + a.log.Warn("authentication failed (unauthenticated)") + return nil, status.Error(codes.Unauthenticated, "unauthenticated") + } + a.log.Debugw("authenticated user", zap.String("name", res.User.GetName()), zap.String("uid", res.User.GetUID())) + tags.Set("user.id", res.User.GetUID()) + tags.Set("user.name", res.User.GetName()) + + // Authorize the user to use the workspace RPC endpoint. + + attributes := authorizer.AttributesRecord{ + User: res.User, + Namespace: a.workspaceName.Namespace, + Name: a.workspaceName.Name, + ResourceRequest: true, + APIGroup: "auto.pulumi.com", + APIVersion: "v1alpha1", + Resource: "workspaces", + Subresource: "rpc", + Verb: "use", + } + authorized, reason, err := a.authz.Authorize(ctx, attributes) + if err != nil { + if apierrors.IsForbidden(err) { + a.log.Errorw(formattedServiceAccountPermissionsErrorMessage()) + return nil, status.Error(codes.Unauthenticated, "SubjectAccessReview API is unavailable") + } + a.log.Errorw("authorization failed with an error", zap.Error(err), "user", attributes.User.GetName()) + return nil, err + } + + if authorized != authorizer.DecisionAllow { + if reason == "" { + reason = "forbidden" + } + a.log.Warnw("access denied", zap.Error(err), zap.String("user", attributes.User.GetName()), zap.String("reason", reason)) + return nil, status.Error(codes.PermissionDenied, reason) + } + a.log.Debugw("authorization allowed", zap.String("reason", reason)) + + return context.WithValue(ctx, "k8s.user", res.User), nil +} + +// RecordRequestTotal increments the total number of requests for the delegated authentication. +func RecordRequestTotal(ctx context.Context, code string) { +} + +// RecordRequestLatency measures request latency in seconds for the delegated authentication. Broken down by status code. +func RecordRequestLatency(ctx context.Context, code string, latency float64) { +} diff --git a/agent/pkg/server/auth_test.go b/agent/pkg/server/auth_test.go new file mode 100644 index 00000000..ef93030e --- /dev/null +++ b/agent/pkg/server/auth_test.go @@ -0,0 +1,248 @@ +/* +Copyright © 2024 Pulumi Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "errors" + "testing" + + grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" + "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" + "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + "go.uber.org/zap" + grpc_codes "google.golang.org/grpc/codes" + grpc_status "google.golang.org/grpc/status" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/utils/ptr" +) + +func TestKubernetes(t *testing.T) { + t.Parallel() + log := zap.L().Named("TestKubernetes").Sugar() + + unavailable := func() authenticator.TokenFunc { + return func(ctx context.Context, token string) (*authenticator.Response, bool, error) { + return nil, false, apierrors.NewForbidden(schema.GroupResource{}, "", errors.New("testing")) + } + } + + authenticate := func(knownToken string, knownUser user.DefaultInfo) authenticator.TokenFunc { + return func(ctx context.Context, token string) (*authenticator.Response, bool, error) { + switch token { + case "": + return nil, false, errors.New("Request unauthenticated with Bearer") + case "b4d": + return nil, false, errors.New("invalid bearer token") + case knownToken: + return &authenticator.Response{ + User: &knownUser, + }, true, nil + default: + return nil, false, nil + } + } + } + + authorize := func(knownUser user.DefaultInfo, knownWorkspace types.NamespacedName) authorizer.AuthorizerFunc { + return func(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { + if !(a.GetUser() != nil && a.GetUser().GetName() == knownUser.Name) { + return authorizer.DecisionDeny, "", nil + } + if !(a.GetVerb() == "use" && a.GetResource() == "workspaces" && a.GetSubresource() == "rpc") { + return authorizer.DecisionDeny, "", nil + } + if !(a.GetNamespace() == knownWorkspace.Namespace && a.GetName() == knownWorkspace.Name) { + return authorizer.DecisionDeny, "", nil + } + return authorizer.DecisionAllow, "", nil + } + } + + testWorkspace := types.NamespacedName{ + Namespace: "default", + Name: "test", + } + otherWorkspace := types.NamespacedName{ + Namespace: "default", + Name: "other", + } + testUser := user.DefaultInfo{ + Name: "system:serviceaccount:default:test", + UID: "81be050c-9ad4-4708-9a52-413064700747", + } + otherUser := user.DefaultInfo{ + Name: "system:serviceaccount:default:other", + UID: "71be050c-9ad4-4708-9a52-413064700747", + } + + tests := []struct { + name string + resourceName types.NamespacedName + authHeaderValue *string + authn authenticator.TokenFunc + authz authorizer.AuthorizerFunc + wantStatus *grpc_status.Status + wantTags gstruct.Keys + }{ + { + name: "Unavailable (AuthN)", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer g00d"), + authn: unavailable(), + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "TokenReview API is unavailable"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "NoToken", + resourceName: testWorkspace, + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "Request unauthenticated with Bearer"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "NotBearerToken", + resourceName: testWorkspace, + authHeaderValue: ptr.To("basic dXNlcm5hbWU6cGFzc3dvcmQ="), + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "Request unauthenticated with Bearer"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "InvalidToken", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer b4d"), + authn: authenticate("g00d", testUser), + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "invalid bearer token"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "AuthenticationFailure", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer 0ther"), + authn: authenticate("g00d", testUser), + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "unauthenticated"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "Denied_User", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer g00d"), + authn: authenticate("g00d", testUser), + authz: authorize(otherUser, testWorkspace), + wantStatus: grpc_status.New(grpc_codes.PermissionDenied, "forbidden"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + "user.id": gomega.Equal(testUser.UID), + "user.name": gomega.Equal(testUser.Name), + }, + }, + { + name: "Denied_ResourceName", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer g00d"), + authn: authenticate("g00d", testUser), + authz: authorize(testUser, otherWorkspace), + wantStatus: grpc_status.New(grpc_codes.PermissionDenied, "forbidden"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + "user.id": gomega.Equal(testUser.UID), + "user.name": gomega.Equal(testUser.Name), + }, + }, + { + name: "Allowed", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer g00d"), + authn: authenticate("g00d", testUser), + authz: authorize(testUser, testWorkspace), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + "user.id": gomega.Equal(testUser.UID), + "user.name": gomega.Equal(testUser.Name), + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + g := gomega.NewWithT(t) + log := log.Named(t.Name()) + + if tt.authn == nil { + tt.authn = func(ctx context.Context, token string) (*authenticator.Response, bool, error) { + t.Error("unexpected call to AuthenticateToken") + return nil, false, nil + } + } + if tt.authz == nil { + tt.authz = func(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { + t.Error("unexpected call to Authorize") + return authorizer.DecisionNoOpinion, "", nil + } + } + + kubeAuth := &kubeAuth{ + log: log, + authn: tt.authn, + authz: tt.authz, + workspaceName: tt.resourceName, + } + + // prepare the context + md := metautils.NiceMD{} + if tt.authHeaderValue != nil { + md.Add("authorization", *tt.authHeaderValue) + } + tags := grpc_ctxtags.NewTags() + ctx := md.ToIncoming(context.Background()) + ctx = grpc_ctxtags.SetInContext(ctx, tags) + + // execute the auth function + ctx, err := kubeAuth.Authenticate(ctx) + + // validate the tags, some of which are set even if the function fails + if tt.wantTags != nil { + g.Expect(tags.Values()).To(gstruct.MatchAllKeys(tt.wantTags)) + } + + // validate the resultant status + status := grpc_status.Convert(err) + if tt.wantStatus != nil { + g.Expect(status).To(gomega.HaveValue(gomega.Equal(*tt.wantStatus)), "an unexpected status") + } + if err == nil { + g.Expect(ctx.Value("k8s.user")).ToNot(gomega.BeNil()) + } + }) + } +} diff --git a/agent/pkg/server/grpc.go b/agent/pkg/server/grpc.go index 1bdce0d5..ad08445d 100644 --- a/agent/pkg/server/grpc.go +++ b/agent/pkg/server/grpc.go @@ -19,6 +19,7 @@ import ( "context" "net" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" pb "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/proto" @@ -33,8 +34,8 @@ type GRPC struct { log *zap.SugaredLogger } -// NewGRPC constructs a new gRPC server with logging and graceful shutdown. -func NewGRPC(server *Server, rootLogger *zap.SugaredLogger) *GRPC { +// NewGRPC constructs a new gRPC server with logging and authentication support. +func NewGRPC(rootLogger *zap.SugaredLogger, server *Server, authF grpc_auth.AuthFunc) *GRPC { log := rootLogger.Named("grpc") // Configure the grpc server. // Apply zap logging and use filters to reduce log verbosity as needed. @@ -43,17 +44,26 @@ func NewGRPC(server *Server, rootLogger *zap.SugaredLogger) *GRPC { return true }), } - grpc_zap.ReplaceGrpcLoggerV2WithVerbosity(log.Desugar(), int(log.Level())) + // Apply a default authentication function. + if authF == nil { + authF = func(ctx context.Context) (context.Context, error) { + return ctx, nil + } + } + + // Create the gRPC server. s := grpc.NewServer( grpc.ChainUnaryInterceptor( grpc_ctxtags.UnaryServerInterceptor(grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor)), grpc_zap.UnaryServerInterceptor(log.Desugar(), serverOpts...), + grpc_auth.UnaryServerInterceptor(authF), ), grpc.ChainStreamInterceptor( grpc_ctxtags.StreamServerInterceptor(grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor)), grpc_zap.StreamServerInterceptor(log.Desugar(), serverOpts...), + grpc_auth.StreamServerInterceptor(authF), ), ) pb.RegisterAutomationServiceServer(s, server) diff --git a/agent/pkg/server/grpc_test.go b/agent/pkg/server/grpc_test.go index 5a8fb076..263719b1 100644 --- a/agent/pkg/server/grpc_test.go +++ b/agent/pkg/server/grpc_test.go @@ -45,7 +45,7 @@ func TestGracefulShutdown(t *testing.T) { tc := newTC(ctx, t, tcOptions{ProjectDir: "./testdata/hang", Stacks: []string{"test"}}) log := zap.L().Named("test").Sugar() lis := bufconn.Listen(1024) - s := server.NewGRPC(tc.server, log) + s := server.NewGRPC(log, tc.server, nil) go func() { // This should exit cleanly if we shut down gracefully. if err := s.Serve(ctx, lis); err != nil { diff --git a/deploy/helm/pulumi-operator/templates/clusterrole.yaml b/deploy/helm/pulumi-operator/templates/clusterrole.yaml index 46b46840..5e09cdb6 100644 --- a/deploy/helm/pulumi-operator/templates/clusterrole.yaml +++ b/deploy/helm/pulumi-operator/templates/clusterrole.yaml @@ -82,6 +82,12 @@ rules: - workspaces/finalizers verbs: - update +- apiGroups: + - auto.pulumi.com + resources: + - workspaces/rpc + verbs: + - use - apiGroups: - auto.pulumi.com resources: diff --git a/deploy/helm/pulumi-operator/templates/role.yaml b/deploy/helm/pulumi-operator/templates/role.yaml index 828f9efc..354ad28e 100644 --- a/deploy/helm/pulumi-operator/templates/role.yaml +++ b/deploy/helm/pulumi-operator/templates/role.yaml @@ -82,6 +82,12 @@ rules: - workspaces/finalizers verbs: - update +- apiGroups: + - auto.pulumi.com + resources: + - workspaces/rpc + verbs: + - use - apiGroups: - auto.pulumi.com resources: diff --git a/deploy/yaml/install.yaml b/deploy/yaml/install.yaml index 4d38710a..50604a50 100644 --- a/deploy/yaml/install.yaml +++ b/deploy/yaml/install.yaml @@ -28045,6 +28045,12 @@ spec: --- apiVersion: v1 kind: ServiceAccount +metadata: + name: pulumi + namespace: default +--- +apiVersion: v1 +kind: ServiceAccount metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -28095,6 +28101,34 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: pulumi-kubernetes-operator-controller-manager rules: @@ -28171,6 +28205,12 @@ rules: - workspaces/finalizers verbs: - update +- apiGroups: + - auto.pulumi.com + resources: + - workspaces/rpc + verbs: + - use - apiGroups: - auto.pulumi.com resources: @@ -28342,6 +28382,32 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: default:pulumi:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: pulumi + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -28358,6 +28424,22 @@ subjects: --- apiVersion: v1 kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: pulumi-kubernetes-operator +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service metadata: labels: control-plane: controller-manager @@ -28401,9 +28483,9 @@ spec: containers: - args: - /manager + - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 - - --metrics-bind-address=:8383 - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local:80 - --zap-log-level=error - --zap-time-encoding=iso8601 diff --git a/go.mod b/go.mod index 464ceb56..64e2f788 100644 --- a/go.mod +++ b/go.mod @@ -22,11 +22,13 @@ require ( github.com/stretchr/testify v1.9.0 go.uber.org/mock v0.4.0 go.uber.org/zap v1.27.0 + golang.org/x/oauth2 v0.20.0 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 k8s.io/api v0.30.1 k8s.io/apiextensions-apiserver v0.30.1 k8s.io/apimachinery v0.30.1 + k8s.io/apiserver v0.30.1 k8s.io/client-go v0.30.1 k8s.io/kubernetes v1.30.3 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 @@ -41,7 +43,7 @@ require ( github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -131,7 +133,7 @@ require ( github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect @@ -153,7 +155,6 @@ require ( golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/oauth2 v0.20.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.21.0 // indirect @@ -168,7 +169,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.30.1 // indirect k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect diff --git a/go.sum b/go.sum index 92f4bd33..9bd1e244 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -17,8 +21,8 @@ github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7l github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -302,10 +306,12 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -314,6 +320,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqajr6t1lOv8GyGE2U= diff --git a/operator/Makefile b/operator/Makefile index c9ad6416..2a090b7b 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -173,7 +173,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml + $(KUSTOMIZE) build config/quickstart > dist/install.yaml ##@ Deployment diff --git a/operator/config/flux/network_policy.yaml b/operator/config/flux/network_policy.yaml new file mode 100644 index 00000000..2c5d5143 --- /dev/null +++ b/operator/config/flux/network_policy.yaml @@ -0,0 +1,26 @@ +# A network policy to allow Pulumi workspaces in the `default` namespace to +# fetch Flux artifacts from the source-controller in the `flux-system` namespace. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-pulumi-fetch-flux-artifacts + namespace: flux-system +spec: + podSelector: + matchLabels: + app: source-controller + ingress: + - ports: + - protocol: TCP + port: http + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default + - podSelector: + matchLabels: + app.kubernetes.io/managed-by: pulumi-kubernetes-operator + app.kubernetes.io/name: pulumi + app.kubernetes.io/component: workspace + policyTypes: + - Ingress \ No newline at end of file diff --git a/operator/config/quickstart/kustomization.yaml b/operator/config/quickstart/kustomization.yaml new file mode 100644 index 00000000..671c9773 --- /dev/null +++ b/operator/config/quickstart/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - ../default + - service_account.yaml + - rbac.yaml diff --git a/operator/config/quickstart/rbac.yaml b/operator/config/quickstart/rbac.yaml new file mode 100644 index 00000000..7d5816d1 --- /dev/null +++ b/operator/config/quickstart/rbac.yaml @@ -0,0 +1,14 @@ +# Grant `system:auth-delegator` to the `default/pulumi` service account, +# to enable Kubernetes RBAC for the Pulumi workspace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:pulumi:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator # permissions: TokenReview, SubjectAccessReview +subjects: +- kind: ServiceAccount + namespace: default + name: pulumi diff --git a/operator/config/quickstart/service_account.yaml b/operator/config/quickstart/service_account.yaml new file mode 100644 index 00000000..da2ba593 --- /dev/null +++ b/operator/config/quickstart/service_account.yaml @@ -0,0 +1,8 @@ +# A service account named `default/pulumi` for the Pulumi workspace (execution environment). +# If your Pulumi program uses the Kubernetes resource provider, this service account will be used to +# authenticate with the Kubernetes cluster. +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: default + name: pulumi diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index cfe38a91..604f10a5 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -77,6 +77,12 @@ rules: - workspaces/finalizers verbs: - update +- apiGroups: + - auto.pulumi.com + resources: + - workspaces/rpc + verbs: + - use - apiGroups: - auto.pulumi.com resources: diff --git a/operator/e2e/testdata/git-auth-nonroot/manifests.yaml b/operator/e2e/testdata/git-auth-nonroot/manifests.yaml index 2457753e..96776469 100644 --- a/operator/e2e/testdata/git-auth-nonroot/manifests.yaml +++ b/operator/e2e/testdata/git-auth-nonroot/manifests.yaml @@ -5,6 +5,25 @@ metadata: name: git-auth-nonroot --- apiVersion: v1 +kind: ServiceAccount +metadata: + name: git-auth-nonroot + namespace: git-auth-nonroot +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: git-auth-nonroot:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: git-auth-nonroot + namespace: git-auth-nonroot +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: state @@ -54,6 +73,7 @@ spec: value: "test" workspaceTemplate: spec: + serviceAccountName: git-auth-nonroot podTemplate: spec: containers: diff --git a/operator/e2e/testdata/random-yaml-nonroot/manifests.yaml b/operator/e2e/testdata/random-yaml-nonroot/manifests.yaml index fd9fa73b..4eb65650 100644 --- a/operator/e2e/testdata/random-yaml-nonroot/manifests.yaml +++ b/operator/e2e/testdata/random-yaml-nonroot/manifests.yaml @@ -1,10 +1,54 @@ --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-random-yaml-nonroot-fetch + namespace: flux-system +spec: + podSelector: + matchLabels: + app: source-controller + ingress: + - ports: + - protocol: TCP + port: http + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: random-yaml-nonroot + - podSelector: + matchLabels: + app.kubernetes.io/managed-by: pulumi-kubernetes-operator + app.kubernetes.io/name: pulumi + app.kubernetes.io/component: workspace + policyTypes: + - Ingress +--- apiVersion: v1 kind: Namespace metadata: name: random-yaml-nonroot --- apiVersion: v1 +kind: ServiceAccount +metadata: + name: random-yaml-nonroot + namespace: random-yaml-nonroot +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: random-yaml-nonroot:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: random-yaml-nonroot + namespace: random-yaml-nonroot +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: state @@ -59,7 +103,7 @@ spec: value: "test" workspaceTemplate: spec: - image: pulumi/pulumi:3.134.1-nonroot + serviceAccountName: random-yaml-nonroot podTemplate: spec: containers: diff --git a/operator/e2e/testdata/targets/manifests.yaml b/operator/e2e/testdata/targets/manifests.yaml index 4d03328d..f4a8bf9b 100644 --- a/operator/e2e/testdata/targets/manifests.yaml +++ b/operator/e2e/testdata/targets/manifests.yaml @@ -5,6 +5,25 @@ metadata: name: targets --- apiVersion: v1 +kind: ServiceAccount +metadata: + name: targets + namespace: targets +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: targets:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: targets + namespace: targets +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: state @@ -61,6 +80,7 @@ spec: value: "test" workspaceTemplate: spec: + serviceAccountName: targets podTemplate: spec: containers: diff --git a/operator/examples/random-yaml/stack.yaml b/operator/examples/random-yaml/stack.yaml index b4a15399..b3e1d2ad 100644 --- a/operator/examples/random-yaml/stack.yaml +++ b/operator/examples/random-yaml/stack.yaml @@ -1,7 +1,27 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: random-yaml + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: random-yaml:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: random-yaml + namespace: default +--- apiVersion: pulumi.com/v1 kind: Stack metadata: name: random-yaml + namespace: default spec: fluxSource: sourceRef: @@ -24,5 +44,6 @@ spec: key: accessToken workspaceTemplate: spec: + serviceAccountName: random-yaml image: pulumi/pulumi:3.134.1-nonroot diff --git a/operator/internal/controller/auto/update_controller.go b/operator/internal/controller/auto/update_controller.go index 798191ba..e2d58a9b 100644 --- a/operator/internal/controller/auto/update_controller.go +++ b/operator/internal/controller/auto/update_controller.go @@ -71,6 +71,7 @@ type UpdateReconciler struct { //+kubebuilder:rbac:groups=auto.pulumi.com,resources=updates/status,verbs=get;update;patch //+kubebuilder:rbac:groups=auto.pulumi.com,resources=updates/finalizers,verbs=update //+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces,verbs=get;list;watch +//+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces/rpc,verbs=use //+kubebuilder:rbac:groups="",resources=secrets,verbs=create // Reconcile manages the Update CRD and initiates Pulumi operations. diff --git a/operator/internal/controller/auto/utils.go b/operator/internal/controller/auto/utils.go index 2d5fa356..db029182 100644 --- a/operator/internal/controller/auto/utils.go +++ b/operator/internal/controller/auto/utils.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + agentclient "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/client" agentpb "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/proto" autov1alpha1 "github.com/pulumi/pulumi-kubernetes-operator/v2/operator/api/auto/v1alpha1" "google.golang.org/grpc" @@ -17,7 +18,22 @@ func connect(ctx context.Context, addr string) (*grpc.ClientConn, error) { if os.Getenv("WORKSPACE_LOCALHOST") != "" { addr = os.Getenv("WORKSPACE_LOCALHOST") } - conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + + token := os.Getenv("WORKSPACE_TOKEN") + tokenFile := os.Getenv("WORKSPACE_TOKEN_FILE") + if token == "" && tokenFile == "" { + // use in-cluster configuration using the operator's service account token + tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + creds, err := agentclient.NewTokenCredentials(token, tokenFile) + if err != nil { + return nil, fmt.Errorf("unable to use token credentials: %w", err) + } + + conn, err := grpc.NewClient( + addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithPerRPCCredentials(creds)) if err != nil { return nil, fmt.Errorf("unable to connect to workspace: %w", err) } diff --git a/operator/internal/controller/auto/workspace_controller.go b/operator/internal/controller/auto/workspace_controller.go index f83de2eb..b557cbe5 100644 --- a/operator/internal/controller/auto/workspace_controller.go +++ b/operator/internal/controller/auto/workspace_controller.go @@ -71,6 +71,7 @@ type WorkspaceReconciler struct { //+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces/status,verbs=get;update;patch //+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces/finalizers,verbs=update +//+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces/rpc,verbs=use //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete @@ -217,6 +218,7 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( defer func() { _ = conn.Close() }() + l.Info("Connected to workspace pod", "addr", addr) w.Status.Address = addr wc := agentpb.NewAutomationServiceClient(conn) @@ -368,9 +370,26 @@ func newStatefulSet(ctx context.Context, w *autov1alpha1.Workspace, source *sour "--workspace", "/share/workspace", "--skip-install", } - env := w.Spec.Env + // provide some pod information to the agent for informational purposes + env = append(env, corev1.EnvVar{ + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }) + env = append(env, corev1.EnvVar{ + Name: "POD_SA_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.serviceAccountName", + }, + }, + }) + // limit the memory usage to the reserved amount // https://github.com/pulumi/pulumi-kubernetes-operator/issues/698 env = append(env, corev1.EnvVar{ @@ -383,6 +402,12 @@ func newStatefulSet(ctx context.Context, w *autov1alpha1.Workspace, source *sour }, }) + // enable workspace endpoint protection + command = append(command, + "--auth-mode", "kube", + "--kube-workspace-namespace", w.Namespace, + "--kube-workspace-name", w.Name) + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", From 2884fad46d2688a1573243044bc922262c32ebec Mon Sep 17 00:00:00 2001 From: Eron Wright Date: Wed, 16 Oct 2024 19:02:21 -0700 Subject: [PATCH 2/2] Clean up "object has already been modified" warnings #700 (#717) ### Proposed changes This PR optimizes the watches to reduce the likelihood of a race between the cache and the reconciliation loop. The general idea is to use predicates to filter on the 'edge' events that would allow the reconciler to make forward progress. For example, the workspace reconciler applies a StatefulSet, then waits for it to be ready. Only when the StatefulSet is indeed ready should reconciliation be queued. Before this PR, the watch on StatefulSet was triggering upon any revision change (way too eager). Overall, a number of optimizations were implemented: - workspace controller should wait for the statefulset to be ready - workspace controller need not watch the Service - stack controller should wait for flux source to be ready - stack controller should not trigger reconciliation upon adding its finalizer (which increments the generation). - stack controller should wait for the update to complete - update controller should wait for workspace readiness - update controller should not enqueue the completed updates of a given workspace Also, some debug statements were introduced to reason about the revision changes over time, since the "conflicts" are detected based on revision. In particular, we emit a debug statement when reconciliation is triggered by a given watch, to understand 'why'. ### Related issues (optional) Closes #700 Supercedes https://github.com/pulumi/pulumi-kubernetes-operator/pull/679 ### Example Here's the result of applying [operator/examples/random-yaml/stack.yaml](https://github.com/pulumi/pulumi-kubernetes-operator/blob/78836991b2691034b2c029df4040ea2016240cd8/operator/examples/random-yaml/stack.yaml). #### Before ```log controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.032Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "841b4d5b-029c-4457-a2b7-8c1821475adf", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:181 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.033Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "841b4d5b-029c-4457-a2b7-8c1821475adf", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.047Z ERROR unable to save object status {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "6a99e260-ca2d-4376-b95f-ab490acbe32a", "error": "Operation cannot be fulfilled on stacks.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/pulumi.(*StackReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/pulumi/stack_controller.go:445 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/pulumi.(*StackReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/pulumi/stack_controller.go:741 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.047Z ERROR Reconciler error {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "6a99e260-ca2d-4376-b95f-ab490acbe32a", "error": "Operation cannot be fulfilled on stacks.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.091Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d397d7d1-c74b-43fe-b4a0-80c3d39ee3ea", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:188 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.091Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d397d7d1-c74b-43fe-b4a0-80c3d39ee3ea", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.111Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "b94710c8-8165-4937-a99c-a4e065f63788", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:188 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:26.111Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "b94710c8-8165-4937-a99c-a4e065f63788", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:32.082Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "7cb6660a-508c-4392-80cd-749fe72def9f", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:252 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:32.082Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "7cb6660a-508c-4392-80cd-749fe72def9f", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:36.114Z ERROR updating status {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "480ad23a-2d02-40a6-b4a4-198fb9df01c4", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile.func1 controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:110 controller-manager-6f849dcb6b-4wscw manager github.com/pulumi/pulumi-kubernetes-operator/v2/operator/internal/controller/auto.(*WorkspaceReconciler).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/operator/internal/controller/auto/workspace_controller.go:316 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 controller-manager-6f849dcb6b-4wscw manager 2024-10-16T01:47:36.114Z ERROR Reconciler error {"controller": "workspace", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "480ad23a-2d02-40a6-b4a4-198fb9df01c4", "error": "Operation cannot be fulfilled on workspaces.auto.pulumi.com \"random-yaml\": the object has been modified; please apply your changes to the latest version and try again"} controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 controller-manager-6f849dcb6b-4wscw manager sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 controller-manager-6f849dcb6b-4wscw manager /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 ``` #### After ```log controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.212Z INFO Reconciling Stack {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "585499fc-fb33-4505-8594-6a9fd8c810b2", "revision": "893372"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z INFO KubeAPIWarningLogger metadata.finalizers: "finalizer.stack.pulumi.com": prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z INFO installing watcher for newly seen source kind {"GroupVersionKind": "source.toolkit.fluxcd.io/v1, Kind=GitRepository"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.225Z INFO Starting EventSource {"controller": "stack-controller", "source": "kind source: *unstructured.Unstructured"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.238Z INFO Reconciling Workspace {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d1595cfb-0fe6-46d5-9e9c-2227f334eb7f", "revision": "893374"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:45.238Z INFO Applying StatefulSet {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "d1595cfb-0fe6-46d5-9e9c-2227f334eb7f", "revision": "893374", "hash": "608b7e170ccd95754972cde1178ab426", "source": {"Generation":1,"ForceRequest":"","Git":null,"Flux":{"Url":"http://source-controller.flux-system.svc.cluster.local./gitrepository/default/pulumi-examples/1432c2ce26299516223c0ad228e92294eba4b2c1.tar.gz","Digest":"sha256:44d554df090dcdeb9bae908cd38155c8c93db05f64dc72982027e8e1294af0d3","Dir":"random-yaml/"}}} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.609Z INFO Reconciling Workspace {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.609Z INFO Applying StatefulSet {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "hash": "608b7e170ccd95754972cde1178ab426", "source": {"Generation":1,"ForceRequest":"","Git":null,"Flux":{"Url":"http://source-controller.flux-system.svc.cluster.local./gitrepository/default/pulumi-examples/1432c2ce26299516223c0ad228e92294eba4b2c1.tar.gz","Digest":"sha256:44d554df090dcdeb9bae908cd38155c8c93db05f64dc72982027e8e1294af0d3","Dir":"random-yaml/"}}} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.719Z INFO Connecting to workspace pod {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.724Z INFO Connected to workspace pod {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:48.724Z INFO Running pulumi install {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893380"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:51.509Z INFO Creating Pulumi stack(s) {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893413"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.250Z INFO workspace pod initialized {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893422"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.250Z INFO Ready {"controller": "workspace-controller", "controllerGroup": "auto.pulumi.com", "controllerKind": "Workspace", "Workspace": {"name":"random-yaml","namespace":"default"}, "namespace": "default", "name": "random-yaml", "reconcileID": "5e7b9b87-fab5-4da0-b025-65dadefc097e", "revision": "893422"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.255Z INFO Reconciling Stack {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "910a2faa-5b49-4590-a23a-d9ef8cab0023", "revision": "893375"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.271Z INFO Reconciling Update {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.271Z INFO Updating the status {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.275Z INFO Connecting {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "addr": "random-yaml-workspace.default.svc.cluster.local:50051"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:55.278Z INFO Selecting the stack {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "stackName": "dev"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:57.333Z INFO Applying the update {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "type": "up"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:01:57.334Z INFO Executing update operation {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "request": "message:\"Stack Update (up)\" target_dependents:false refresh:true"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.421Z INFO Update complete {"controller": "update", "controllerGroup": "auto.pulumi.com", "controllerKind": "Update", "Update": {"name":"random-yaml-b7kng2l5","namespace":"default"}, "namespace": "default", "name": "random-yaml-b7kng2l5", "reconcileID": "5b9ced73-989f-46fa-a485-746686519832", "result": "stdout:\"Updating (dev)\\n\\nView Live: https://app.pulumi.com/eron-pulumi-corp/random/dev/updates/4536\\n\\n+ pulumi:pulumi:Stack: (create)\\n [urn=urn:pulumi:dev::random::pulumi:pulumi:Stack::random-dev]\\n + random:index/randomPassword:RandomPassword: (create)\\n [urn=urn:pulumi:dev::random::random:index/randomPassword:RandomPassword::randomPassword]\\n length : 16\\n overrideSpecial: \\\"_%@\\\"\\n special : true\\n --outputs:--\\n bcryptHash : [secret]\\n id : \\\"none\\\"\\n lower : true\\n minLower : 0\\n minNumeric : 0\\n minSpecial : 0\\n minUpper : 0\\n number : true\\n numeric : true\\n result : [secret]\\n upper : true\\n --outputs:--\\n password: [secret]\\nResources:\\n + 2 created\\n\\nDuration: 3s\\n\" summary:{start_time:{seconds:1729044119} end_time:{seconds:1729044122} result:\"succeeded\" message:\"\\\"Stack Update (up)\\\"\"} permalink:\"https://app.pulumi.com/eron-pulumi-corp/random/dev/updates/4536\" outputs:{key:\"password\" value:{value:\"\\\"fAMcmXI2DfW7KCey\\\"\" secret:true}}"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.438Z INFO Reconciling Stack {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "a100c4f3-c5b9-4894-8644-fabc1f71d008", "revision": "893437"} controller-manager-778bdbdfbb-cdrpb manager 2024-10-16T02:02:05.546Z INFO Commit hash unchanged. Will wait for Source update or resync. {"controller": "stack-controller", "namespace": "default", "name": "random-yaml", "reconcileID": "a100c4f3-c5b9-4894-8644-fabc1f71d008", "revision": "893437"} ``` --- operator/Makefile | 2 +- .../controller/auto/update_controller.go | 71 ++++++++- operator/internal/controller/auto/utils.go | 31 ++++ .../controller/auto/workspace_controller.go | 48 +++++- operator/internal/controller/pulumi/flux.go | 70 +++++---- .../controller/pulumi/stack_controller.go | 137 ++++++++++++++---- .../pulumi/stack_controller_test.go | 2 +- 7 files changed, 294 insertions(+), 67 deletions(-) diff --git a/operator/Makefile b/operator/Makefile index 2a090b7b..df161463 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -192,7 +192,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller manager to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply --server-side=true -oyaml -f - + $(KUSTOMIZE) build config/default | $(KUBECTL) apply --server-side=true -f - .PHONY: undeploy undeploy: ## Undeploy controller manager from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. diff --git a/operator/internal/controller/auto/update_controller.go b/operator/internal/controller/auto/update_controller.go index e2d58a9b..e07242f0 100644 --- a/operator/internal/controller/auto/update_controller.go +++ b/operator/internal/controller/auto/update_controller.go @@ -32,6 +32,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -41,6 +42,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -77,13 +79,15 @@ type UpdateReconciler struct { // Reconcile manages the Update CRD and initiates Pulumi operations. func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { l := log.FromContext(ctx) - l.Info("Reconciling Update") obj := &autov1alpha1.Update{} err := r.Get(ctx, req.NamespacedName, obj) if err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + l = l.WithValues("revision", obj.ResourceVersion) + ctx = log.IntoContext(ctx, l) + l.Info("Reconciling Update") rs := newReconcileSession(r.Client, obj) @@ -116,12 +120,32 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, fmt.Errorf("failed to update the status: %w", err) } - // TODO check the w status before proceeding + // Get the workspace and check that it is ready w := &autov1alpha1.Workspace{} err = r.Client.Get(ctx, client.ObjectKey{Namespace: obj.Namespace, Name: obj.Spec.WorkspaceName}, w) if err != nil { + if apierrors.IsNotFound(err) { + l.Info("Workspace not found", "workspace", obj.Spec.WorkspaceName) + rs.progressing.Status = metav1.ConditionFalse + rs.progressing.Reason = "WorkspaceNotFound" + rs.failed.Status = metav1.ConditionFalse + rs.failed.Reason = UpdateConditionReasonProgressing + rs.complete.Status = metav1.ConditionFalse + rs.complete.Reason = UpdateConditionReasonProgressing + return ctrl.Result{}, rs.updateStatus(ctx, obj) + } return ctrl.Result{}, fmt.Errorf("failed to get workspace: %w", err) } + if !isWorkspaceReady(w) { + l.Info("Workspace not ready", "workspace", w.Name) + rs.progressing.Status = metav1.ConditionFalse + rs.progressing.Reason = "WorkspaceNotReady" + rs.failed.Status = metav1.ConditionFalse + rs.failed.Reason = UpdateConditionReasonProgressing + rs.complete.Status = metav1.ConditionFalse + rs.complete.Reason = UpdateConditionReasonProgressing + return ctrl.Result{}, rs.updateStatus(ctx, obj) + } // Connect to the workspace's GRPC server addr := fmt.Sprintf("%s:%d", fqdnForService(w), WorkspaceGrpcPort) @@ -168,6 +192,36 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } } +func isWorkspaceReady(ws *autov1alpha1.Workspace) bool { + if ws == nil || ws.Generation != ws.Status.ObservedGeneration { + return false + } + return meta.IsStatusConditionTrue(ws.Status.Conditions, autov1alpha1.WorkspaceReady) +} + +type workspaceReadyPredicate struct{} + +var _ predicate.Predicate = &workspaceReadyPredicate{} + +func (workspaceReadyPredicate) Create(e event.CreateEvent) bool { + return isWorkspaceReady(e.Object.(*autov1alpha1.Workspace)) +} + +func (workspaceReadyPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (workspaceReadyPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !isWorkspaceReady(e.ObjectOld.(*autov1alpha1.Workspace)) && isWorkspaceReady(e.ObjectNew.(*autov1alpha1.Workspace)) +} + +func (workspaceReadyPredicate) Generic(_ event.GenericEvent) bool { + return false +} + type reconcileSession struct { progressing *metav1.Condition complete *metav1.Condition @@ -375,13 +429,18 @@ func (r *UpdateReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&autov1alpha1.Update{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Watches(&autov1alpha1.Workspace{}, handler.EnqueueRequestsFromMapFunc(r.mapWorkspaceToUpdate), - builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})). + builder.WithPredicates(&workspaceReadyPredicate{})). Complete(r) } func indexUpdateByWorkspace(obj client.Object) []string { - w := obj.(*autov1alpha1.Update) - return []string{w.Spec.WorkspaceName} + u := obj.(*autov1alpha1.Update) + complete := meta.IsStatusConditionTrue(u.Status.Conditions, UpdateConditionTypeComplete) + if complete { + // don't index the completed updates, to avoid unnecessary reconciles when their workspace is updated + return []string{} + } + return []string{u.Spec.WorkspaceName} } func (r *UpdateReconciler) mapWorkspaceToUpdate(ctx context.Context, obj client.Object) []reconcile.Request { @@ -494,7 +553,7 @@ func (s streamReader[T]) Result() (result, error) { continue // No result yet. } - s.l.Info("Result received", "result", res) + s.l.Info("Update complete", "result", res) s.obj.Status.StartTime = metav1.NewTime(res.GetSummary().StartTime.AsTime()) s.obj.Status.EndTime = metav1.NewTime(res.GetSummary().EndTime.AsTime()) diff --git a/operator/internal/controller/auto/utils.go b/operator/internal/controller/auto/utils.go index db029182..4aa06f53 100644 --- a/operator/internal/controller/auto/utils.go +++ b/operator/internal/controller/auto/utils.go @@ -12,6 +12,9 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/types/known/structpb" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" ) func connect(ctx context.Context, addr string) (*grpc.ClientConn, error) { @@ -84,3 +87,31 @@ func marshalConfigValue(item autov1alpha1.ConfigItem) *agentpb.ConfigValue { } return v } + +var l = log.Log.WithName("predicate").WithName("debug") + +type DebugPredicate struct { + Controller string +} + +var _ predicate.Predicate = &DebugPredicate{} + +func (p *DebugPredicate) Create(e event.CreateEvent) bool { + l.V(1).Info("Create", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion()) + return true +} + +func (p *DebugPredicate) Delete(e event.DeleteEvent) bool { + l.V(1).Info("Delete", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion()) + return true +} + +func (p *DebugPredicate) Update(e event.UpdateEvent) bool { + l.V(1).Info("Update", "controller", p.Controller, "type", fmt.Sprintf("%T", e.ObjectOld), "name", e.ObjectOld.GetName(), "old-revision", e.ObjectOld.GetResourceVersion(), "new-revision", e.ObjectNew.GetResourceVersion()) + return true +} + +func (p *DebugPredicate) Generic(e event.GenericEvent) bool { + l.V(1).Info("Generic", "controller", p.Controller, "type", fmt.Sprintf("%T", e.Object), "name", e.Object.GetName(), "revision", e.Object.GetResourceVersion()) + return true +} diff --git a/operator/internal/controller/auto/workspace_controller.go b/operator/internal/controller/auto/workspace_controller.go index b557cbe5..af9a63be 100644 --- a/operator/internal/controller/auto/workspace_controller.go +++ b/operator/internal/controller/auto/workspace_controller.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -78,13 +79,14 @@ type WorkspaceReconciler struct { func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { l := log.FromContext(ctx) - l.Info("Reconciling Workspace") w := &autov1alpha1.Workspace{} err := r.Get(ctx, req.NamespacedName, w) if err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + l = l.WithValues("revision", w.ResourceVersion) + l.Info("Reconciling Workspace") // apply defaults to the workspace spec // future: use a mutating webhook to apply defaults @@ -108,7 +110,11 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( err := r.Status().Update(ctx, w) if err != nil { l.Error(err, "updating status") + } else { + l = log.FromContext(ctx).WithValues("revision", w.ResourceVersion) + l.V(1).Info("Status updated") } + return err } @@ -319,15 +325,47 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // SetupWithManager sets up the controller with the Manager. func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). + Named("workspace-controller"). For(&autov1alpha1.Workspace{}, - builder.WithPredicates(predicate.GenerationChangedPredicate{})). - Owns(&corev1.Service{}, - builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})). + builder.WithPredicates(predicate.GenerationChangedPredicate{}, &DebugPredicate{Controller: "workspace-controller"})). Owns(&appsv1.StatefulSet{}, - builder.WithPredicates(&predicate.ResourceVersionChangedPredicate{})). + builder.WithPredicates(&statefulSetReadyPredicate{}, &DebugPredicate{Controller: "workspace-controller"})). Complete(r) } +type statefulSetReadyPredicate struct{} + +var _ predicate.Predicate = &statefulSetReadyPredicate{} + +func isStatefulSetReady(ss *appsv1.StatefulSet) bool { + if ss.Status.ObservedGeneration != ss.Generation || ss.Status.UpdateRevision != ss.Status.CurrentRevision { + return false + } + if ss.Status.AvailableReplicas < 1 { + return false + } + return true +} + +func (statefulSetReadyPredicate) Create(e event.CreateEvent) bool { + return isStatefulSetReady(e.Object.(*appsv1.StatefulSet)) +} + +func (statefulSetReadyPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (statefulSetReadyPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !isStatefulSetReady(e.ObjectOld.(*appsv1.StatefulSet)) && isStatefulSetReady(e.ObjectNew.(*appsv1.StatefulSet)) +} + +func (statefulSetReadyPredicate) Generic(_ event.GenericEvent) bool { + return false +} + const ( FieldManager = "pulumi-kubernetes-operator" WorkspacePulumiContainerName = "pulumi" diff --git a/operator/internal/controller/pulumi/flux.go b/operator/internal/controller/pulumi/flux.go index 6ccfc5c2..61ec9bf5 100644 --- a/operator/internal/controller/pulumi/flux.go +++ b/operator/internal/controller/pulumi/flux.go @@ -11,6 +11,8 @@ import ( "github.com/pulumi/pulumi-kubernetes-operator/v2/operator/api/pulumi/shared" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" ) func (sess *stackReconcilerSession) SetupWorkspaceFromFluxSource(ctx context.Context, source unstructured.Unstructured, fluxSource *shared.FluxSource) (string, error) { @@ -70,37 +72,28 @@ func checksumOrDigest(source unstructured.Unstructured) (string, error) { return checksum, nil } -// checkFluxSourceReady looks for the conventional "Ready" condition to see if the supplied object -// can be considered _not_ ready. It returns an error if it can determine that the object is not -// ready, and nil if it cannot determine so. -func checkFluxSourceReady(obj unstructured.Unstructured) error { +func checkFluxSourceReady(obj *unstructured.Unstructured) bool { + observedGeneration, ok, err := unstructured.NestedInt64(obj.Object, "status", "observedGeneration") + if !ok || err != nil || observedGeneration != obj.GetGeneration() { + return false + } conditions, ok, err := unstructured.NestedSlice(obj.Object, "status", "conditions") - if ok && err == nil { - // didn't find a []Condition, so there's nothing to indicate that it's not ready there - for _, c0 := range conditions { - var c map[string]interface{} - if c, ok = c0.(map[string]interface{}); !ok { - // condition isn't the right shape, try the next one - continue - } - if t, ok, err := unstructured.NestedString(c, "type"); ok && err == nil && t == "Ready" { - if v, ok, err := unstructured.NestedString(c, "status"); ok && err == nil && v == "True" { - // found the Ready condition and it is actually ready; proceed to next check - break - } - // found the Ready condition and it's something other than ready - return fmt.Errorf("source Ready condition does not have status True %#v", c) + if !ok || err != nil { + return false + } + for _, c0 := range conditions { + var c map[string]interface{} + if c, ok = c0.(map[string]interface{}); !ok { + // condition isn't the right shape, try the next one + continue + } + if t, ok, err := unstructured.NestedString(c, "type"); ok && err == nil && t == "Ready" { + if v, ok, err := unstructured.NestedString(c, "status"); ok && err == nil && v == "True" { + return true } } - // Ready=true, or no ready condition to tell us either way } - - _, ok, err = unstructured.NestedMap(obj.Object, "status", "artifact") - if !ok || err != nil { - return fmt.Errorf(".status.artifact does not have an Artifact object") - } - - return nil + return false } func getSourceGVK(src shared.FluxSourceReference) (schema.GroupVersionKind, error) { @@ -111,3 +104,26 @@ func getSourceGVK(src shared.FluxSourceReference) (schema.GroupVersionKind, erro func fluxSourceKey(gvk schema.GroupVersionKind, name string) string { return fmt.Sprintf("%s:%s", gvk, name) } + +type fluxSourceReadyPredicate struct{} + +var _ predicate.Predicate = &fluxSourceReadyPredicate{} + +func (fluxSourceReadyPredicate) Create(e event.CreateEvent) bool { + return checkFluxSourceReady(e.Object.(*unstructured.Unstructured)) +} + +func (fluxSourceReadyPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (fluxSourceReadyPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !checkFluxSourceReady(e.ObjectOld.(*unstructured.Unstructured)) && checkFluxSourceReady(e.ObjectNew.(*unstructured.Unstructured)) +} + +func (fluxSourceReadyPredicate) Generic(_ event.GenericEvent) bool { + return false +} diff --git a/operator/internal/controller/pulumi/stack_controller.go b/operator/internal/controller/pulumi/stack_controller.go index 5fdf922c..cf4b8131 100644 --- a/operator/internal/controller/pulumi/stack_controller.go +++ b/operator/internal/controller/pulumi/stack_controller.go @@ -58,7 +58,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" ctrlhandler "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -101,7 +101,9 @@ func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error { // Filter for update events where an object's metadata.generation is changed (no spec change!), // or the "force reconcile" annotation is used (and not marked as handled). predicates := []predicate.Predicate{ - predicate.Or(predicate.GenerationChangedPredicate{}, ReconcileRequestedPredicate{}), + predicate.Or( + predicate.And(predicate.GenerationChangedPredicate{}, predicate.Not(&finalizerAddedPredicate{})), + ReconcileRequestedPredicate{}), } // Track metrics about stacks. @@ -194,13 +196,16 @@ func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error { enqueueStacksForSourceFunc(programRefIndexFieldName, func(obj client.Object) string { return obj.GetName() - }))) + })), + builder.WithPredicates(&auto.DebugPredicate{Controller: "stack-controller"})) // Watch the stack's workspace and update objects - blder = blder.Watches(&autov1alpha1.Workspace{}, ctrlhandler.EnqueueRequestForOwner( - mgr.GetScheme(), mgr.GetRESTMapper(), &pulumiv1.Stack{})) - blder = blder.Watches(&autov1alpha1.Update{}, ctrlhandler.EnqueueRequestForOwner( - mgr.GetScheme(), mgr.GetRESTMapper(), &pulumiv1.Stack{})) + blder = blder.Watches(&autov1alpha1.Workspace{}, + ctrlhandler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &pulumiv1.Stack{}), + builder.WithPredicates(&workspaceReadyPredicate{}, &auto.DebugPredicate{Controller: "stack-controller"})) + blder = blder.Watches(&autov1alpha1.Update{}, + ctrlhandler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &pulumiv1.Stack{}), + builder.WithPredicates(&updateCompletePredicate{}, &auto.DebugPredicate{Controller: "stack-controller"})) c, err := blder.WithOptions(opts).Build(r) if err != nil { @@ -254,9 +259,7 @@ func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error { } watchedMu.Unlock() if !ok { - // Using PartialObjectMetadata means we don't need the actual types registered in the - // schema. - var sourceKind metav1.PartialObjectMetadata + var sourceKind unstructured.Unstructured sourceKind.SetGroupVersionKind(gvk) mgr.GetLogger().Info("installing watcher for newly seen source kind", "GroupVersionKind", gvk) err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &sourceKind, @@ -264,7 +267,7 @@ func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error { enqueueStacksForSourceFunc(fluxSourceIndexFieldName, func(obj client.Object) string { gvk := obj.GetObjectKind().GroupVersionKind() return fluxSourceKey(gvk, obj.GetName()) - })))) + })), &fluxSourceReadyPredicate{}, &auto.DebugPredicate{Controller: "stack-controller"})) if err != nil { watchedMu.Lock() delete(watched, gvk) @@ -344,6 +347,66 @@ func (p ReconcileRequestedPredicate) Update(e event.UpdateEvent) bool { return false // either removed, or present in neither object } +func isWorkspaceReady(ws *autov1alpha1.Workspace) bool { + if ws == nil || ws.Generation != ws.Status.ObservedGeneration { + return false + } + return meta.IsStatusConditionTrue(ws.Status.Conditions, autov1alpha1.WorkspaceReady) +} + +type workspaceReadyPredicate struct{} + +var _ predicate.Predicate = &workspaceReadyPredicate{} + +func (workspaceReadyPredicate) Create(e event.CreateEvent) bool { + return isWorkspaceReady(e.Object.(*autov1alpha1.Workspace)) +} + +func (workspaceReadyPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (workspaceReadyPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !isWorkspaceReady(e.ObjectOld.(*autov1alpha1.Workspace)) && isWorkspaceReady(e.ObjectNew.(*autov1alpha1.Workspace)) +} + +func (workspaceReadyPredicate) Generic(_ event.GenericEvent) bool { + return false +} + +func isUpdateComplete(update *autov1alpha1.Update) bool { + if update == nil || update.Generation != update.Status.ObservedGeneration { + return false + } + return meta.IsStatusConditionTrue(update.Status.Conditions, autov1alpha1.UpdateConditionTypeComplete) +} + +type updateCompletePredicate struct{} + +var _ predicate.Predicate = &updateCompletePredicate{} + +func (updateCompletePredicate) Create(e event.CreateEvent) bool { + return isUpdateComplete(e.Object.(*autov1alpha1.Update)) +} + +func (updateCompletePredicate) Delete(e event.DeleteEvent) bool { + return false +} + +func (updateCompletePredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !isUpdateComplete(e.ObjectOld.(*autov1alpha1.Update)) && isUpdateComplete(e.ObjectNew.(*autov1alpha1.Update)) +} + +func (updateCompletePredicate) Generic(e event.GenericEvent) bool { + return false +} + // StackReconciler reconciles a Stack object type StackReconciler struct { // This client, initialized using mgr.Client() above, is a split client @@ -395,8 +458,7 @@ var errProgramNotFound = fmt.Errorf("unable to retrieve program for stack") // Reconcile reads that state of the cluster for a Stack object and makes changes based on the state read // and what is in the Stack.Spec func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) (res ctrl.Result, reterr error) { - log := log.FromContext(ctx) - log.Info("Reconciling Stack") + log := ctrllog.FromContext(ctx) // Fetch the Stack instance instance := &pulumiv1.Stack{} @@ -407,6 +469,8 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( // Return and don't requeue return reconcile.Result{}, client.IgnoreNotFound(err) } + log = log.WithValues("revision", instance.ResourceVersion) + log.Info("Reconciling Stack") // Update the observed generation and "reconcile request" of the object. instance.Status.ObservedGeneration = instance.GetGeneration() @@ -445,6 +509,8 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( log.Error(err, "unable to save object status") return err } + log = ctrllog.FromContext(ctx).WithValues("revision", instance.ResourceVersion) + log.V(1).Info("Status updated") return nil } @@ -463,7 +529,8 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( return reconcile.Result{}, fmt.Errorf("get current update: %w", err) } - completed := meta.IsStatusConditionTrue(sess.update.Status.Conditions, autov1alpha1.UpdateConditionTypeComplete) + completed := sess.update.Generation == sess.update.Status.ObservedGeneration && + meta.IsStatusConditionTrue(sess.update.Status.Conditions, autov1alpha1.UpdateConditionTypeComplete) if !completed { // wait for the update to complete instance.Status.MarkReconcilingCondition(pulumiv1.ReconcilingProcessingReason, pulumiv1.ReconcilingProcessingUpdateMessage) @@ -619,9 +686,9 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( return reconcile.Result{}, err } - if err := checkFluxSourceReady(sourceObject); err != nil { + if !checkFluxSourceReady(&sourceObject) { // Wait until the source is ready, at which time the watch mechanism will requeue it. - instance.Status.MarkStalledCondition(pulumiv1.StalledSourceUnavailableReason, err.Error()) + instance.Status.MarkStalledCondition(pulumiv1.StalledSourceUnavailableReason, "Flux source not ready") return reconcile.Result{}, saveStatus() } @@ -735,7 +802,7 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( return reconcile.Result{}, fmt.Errorf("unable to create workspace: %w", err) } - if !sess.isWorkspaceReady() { + if !isWorkspaceReady(sess.ws) { // watch the workspace for status updates log.V(1).Info("waiting for workspace to be ready") return reconcile.Result{}, saveStatus() @@ -1190,16 +1257,6 @@ func (sess *stackReconcilerSession) CreateWorkspace(ctx context.Context) error { return nil } -func (sess *stackReconcilerSession) isWorkspaceReady() bool { - if sess.ws == nil { - return false - } - if sess.ws.Generation != sess.ws.Status.ObservedGeneration { - return false - } - return meta.IsStatusConditionTrue(sess.ws.Status.Conditions, autov1alpha1.WorkspaceReady) -} - // setupWorkspace sets all the extra configuration specified by the Stack object, after you have // constructed a workspace from a source. func (sess *stackReconcilerSession) setupWorkspace(ctx context.Context) error { @@ -1396,3 +1453,29 @@ func patchObject[T any, V any](base T, patch V) (*T, error) { return &result, nil } + +// finalizerAddedPredicate detects when a finalizer is added to an object. +// It is used to suppress reconciliation when the stack controller adds its finalizer, which causes +// a generation change that would otherwise trigger reconciliation. +type finalizerAddedPredicate struct{} + +var _ predicate.Predicate = &finalizerAddedPredicate{} + +func (p *finalizerAddedPredicate) Create(_ event.CreateEvent) bool { + return false +} + +func (p *finalizerAddedPredicate) Delete(_ event.DeleteEvent) bool { + return false +} + +func (p *finalizerAddedPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + return !controllerutil.ContainsFinalizer(e.ObjectOld, pulumiFinalizer) && controllerutil.ContainsFinalizer(e.ObjectNew, pulumiFinalizer) +} + +func (p *finalizerAddedPredicate) Generic(_ event.GenericEvent) bool { + return false +} diff --git a/operator/internal/controller/pulumi/stack_controller_test.go b/operator/internal/controller/pulumi/stack_controller_test.go index 11f341e5..38370b00 100644 --- a/operator/internal/controller/pulumi/stack_controller_test.go +++ b/operator/internal/controller/pulumi/stack_controller_test.go @@ -1155,7 +1155,7 @@ var _ = Describe("Stack Controller", func() { {Type: "Ready", Status: metav1.ConditionFalse, Reason: "Unknown", LastTransitionTime: metav1.Now()}, } }) - beStalled(pulumiv1.StalledSourceUnavailableReason, ContainSubstring("source Ready condition does not have status True")) + beStalled(pulumiv1.StalledSourceUnavailableReason, ContainSubstring("Flux source not ready")) }) When("the flux source is ready", func() {