From 78836991b2691034b2c029df4040ea2016240cd8 Mon Sep 17 00:00:00 2001 From: Eron Wright Date: Fri, 11 Oct 2024 17:10:23 -0700 Subject: [PATCH] Implement an authorization layer for operator-to-workspace communication (#712) ### Overview This PR implements an authentication and authorization layer for the agent's RPC endpoint. Authentication is performed by authenticating a bearer token via the TokenReview API. The operator uses its built-in service account token. Authorization is performed via the SubjectAccessReview API, which checks for following RBAC permission: ```yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role rules: - apiGroups: - auto.pulumi.com resources: - workspaces/rpc verbs: - use ``` The workspace pod's service account must be granted the `system:auth-delegator` role using a `ClusterRoleBinding`. For. convenience, the installer creates a service account named `pulumi` into the `default` namespace, with an associated binding. The operator itself is granted the necessary permission to access the RPC endpoint. ### Proposed changes - [x] agent grpc interceptor - [x] agent command args (`--auth-mode=kube`, `--kube-workspace-name=random-yaml`) - [x] operator client credentials - [x] operator RBAC permissions - [ ] ~cluster role binding for workspace service account (to ClusterRole named `system:auth-delegator`)~ - [x] install a "default/pulumi" service account with RBAC - [x] Provide a Flux sample network policy - [x] Update the e2e test manifests to have requisite account, rbac, and network policy. ### Future Enhancement This implementation uses the operator's default service account token, but to further improve security it should use an audience-scoped token, where the audience is the agent service address as opposed to the API server. Such tokens may be created by the operator with a call to TokenRequest, and checked with TokenReview by adding the expected audience to the context (`authenticator.WithAudience`). ### Related issues (optional) Closes #609 #### Examples Some example requests: ``` random-yaml-workspace-0 pulumi 2024-10-09T21:09:43.905Z INFO cmd.serve.grpc finished unary call with code OK {"grpc.start_time": "2024-10-09T21:09:43Z", "grpc.request.deadline": "2024-10-09T21:59:43Z", "system": "grpc", "span.kind": "server", "grpc.service": "agent.AutomationService", "grpc.method": "WhoAmI", "user.id": "81be050c-9ad4-4708-9a52-413064700747", "user.name": "system:serviceaccount:default:dev", "peer.address": "127.0.0.1:56394", "auth.mode": "kubernetes", "grpc.code": "OK", "grpc.time_ms": 441.086} random-yaml-workspace-0 pulumi 2024-10-09T21:09:52.934Z INFO cmd.serve.grpc finished unary call with code Unauthenticated {"grpc.start_time": "2024-10-09T21:09:52Z", "grpc.request.deadline": "2024-10-09T21:59:52Z", "system": "grpc", "span.kind": "server", "grpc.service": "agent.AutomationService", "grpc.method": "WhoAmI", "peer.address": "127.0.0.1:57380", "auth.mode": "kubernetes", "error": "rpc error: code = Unauthenticated desc = Request unauthenticated with Bearer", "grpc.code": "Unauthenticated", "grpc.time_ms": 0.095} ``` --- .vscode/launch.json | 19 +- agent/cmd/root.go | 23 +- agent/cmd/serve.go | 78 ++++-- agent/hack/token.yaml | 27 ++ agent/pkg/client/client.go | 75 ++++++ agent/pkg/server/auth.go | 231 ++++++++++++++++ agent/pkg/server/auth_test.go | 248 ++++++++++++++++++ agent/pkg/server/grpc.go | 16 +- agent/pkg/server/grpc_test.go | 2 +- .../templates/clusterrole.yaml | 6 + .../helm/pulumi-operator/templates/role.yaml | 6 + deploy/yaml/install.yaml | 84 +++++- go.mod | 8 +- go.sum | 17 +- operator/Makefile | 2 +- operator/config/flux/network_policy.yaml | 26 ++ operator/config/quickstart/kustomization.yaml | 4 + operator/config/quickstart/rbac.yaml | 14 + .../config/quickstart/service_account.yaml | 8 + operator/config/rbac/role.yaml | 6 + .../testdata/git-auth-nonroot/manifests.yaml | 20 ++ .../random-yaml-nonroot/manifests.yaml | 46 +++- operator/e2e/testdata/targets/manifests.yaml | 20 ++ operator/examples/random-yaml/stack.yaml | 21 ++ .../controller/auto/update_controller.go | 1 + operator/internal/controller/auto/utils.go | 18 +- .../controller/auto/workspace_controller.go | 27 +- 27 files changed, 1018 insertions(+), 35 deletions(-) create mode 100644 agent/hack/token.yaml create mode 100644 agent/pkg/client/client.go create mode 100644 agent/pkg/server/auth.go create mode 100644 agent/pkg/server/auth_test.go create mode 100644 operator/config/flux/network_policy.yaml create mode 100644 operator/config/quickstart/kustomization.yaml create mode 100644 operator/config/quickstart/rbac.yaml create mode 100644 operator/config/quickstart/service_account.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index 50127707..5a9d4d62 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -45,9 +45,26 @@ "-v=false", "--workspace=${input:workdir}", "-s=dev" + ] + }, + { + "name": "Agent (kubernetes)", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "agent", + "args": [ + "serve", + "-v=true", + "--workspace=${input:workdir}", + "-s=dev", + "--auth-mode=kube", + "--kube-workspace-namespace=default", + "--kube-workspace-name=random-yaml" ], "env": { - "AWS_REGION": "us-west-1", + "POD_NAMESPACE": "default", + "POD_SA_NAME": "fake" } } ], diff --git a/agent/cmd/root.go b/agent/cmd/root.go index 5bc3a741..fe9d30cb 100644 --- a/agent/cmd/root.go +++ b/agent/cmd/root.go @@ -18,11 +18,18 @@ package cmd import ( "os" + "flag" + "github.com/spf13/cobra" "go.uber.org/zap" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/config" ) -var verbose bool +var ( + verbose bool + kubeContext string +) // a command-specific logger var log *zap.SugaredLogger @@ -51,6 +58,7 @@ to use to perform stack operations.`, // initialize a command-specific logger log = zap.L().Named("cmd").Named(cmd.Name()).Sugar() + cmd.SilenceErrors = true return nil }, PersistentPostRun: func(cmd *cobra.Command, args []string) { @@ -64,10 +72,23 @@ to use to perform stack operations.`, func Execute() { err := rootCmd.Execute() if err != nil { + if log != nil { + log.Error(err.Error()) + } os.Exit(1) } } func init() { rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose logging") + + // register the Kubernetes flags (e.g. for serve command when using Kubernetes RBAC for authorization) + fs := flag.NewFlagSet("kubernetes", flag.ExitOnError) + config.RegisterFlags(fs) + rootCmd.PersistentFlags().AddGoFlagSet(fs) + rootCmd.PersistentFlags().StringVar(&kubeContext, "context", "", "Kubernetes context override") +} + +func GetKubeConfig() (*rest.Config, error) { + return config.GetConfigWithContext(kubeContext) } diff --git a/agent/cmd/serve.go b/agent/cmd/serve.go index 8a1a13fd..6499bd5d 100644 --- a/agent/cmd/serve.go +++ b/agent/cmd/serve.go @@ -25,6 +25,7 @@ import ( "runtime/debug" "syscall" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/server" "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/version" "github.com/pulumi/pulumi/sdk/v3/go/auto" @@ -32,6 +33,12 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapio" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" +) + +const ( + AuthModeNone = "none" + AuthModeKubernetes = "kube" ) var ( @@ -40,6 +47,10 @@ var ( _stack string _host string _port int + + _authMode string + _workspaceNamespace string + _workspaceName string ) // serveCmd represents the serve command @@ -48,7 +59,22 @@ var serveCmd = &cobra.Command{ Short: "Serve the agent RPC service", Long: `Start the agent gRPC server. `, - Run: func(cmd *cobra.Command, args []string) { + PreRunE: func(cmd *cobra.Command, args []string) error { + if _authMode != AuthModeNone && _authMode != AuthModeKubernetes { + return fmt.Errorf("unsupported auth mode: %s", _authMode) + } + if _authMode == AuthModeKubernetes { + if _workspaceNamespace == "" { + return fmt.Errorf("--kube-workspace-namespace is required when auth mode is kubernetes") + } + if _workspaceName == "" { + return fmt.Errorf("--kube-workspace-name is required when auth mode is kubernetes") + } + } + cmd.SilenceUsage = true + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() log.Infow("Pulumi Kubernetes Agent", "version", version.Version) @@ -63,23 +89,42 @@ var serveCmd = &cobra.Command{ } } + // Prepare the authorizer function + var authFunc grpc_auth.AuthFunc + switch _authMode { + case AuthModeKubernetes: + kubeConfig, err := GetKubeConfig() + if err != nil { + return fmt.Errorf("unable to load the kubeconfig: %w", err) + } + + authFunc, err = server.NewKubeAuth(log.Desugar(), kubeConfig, server.KubeAuthOptions{ + WorkspaceName: types.NamespacedName{ + Namespace: _workspaceNamespace, + Name: _workspaceName, + }, + }) + if err != nil { + return fmt.Errorf("unable to initialize the Kubernetes authorizer: %w", err) + } + log.Infow("activated the Kubernetes authorization mode", + zap.String("workspace.namespace", _workspaceNamespace), zap.String("workspace.name", _workspaceName)) + } + // open the workspace using auto api workspaceOpts := []auto.LocalWorkspaceOption{} workDir, err := filepath.EvalSymlinks(_workDir) // resolve the true location of the workspace if err != nil { - log.Fatalw("unable to resolve the workspace directory", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to resolve the workspace directory: %w", err) } workspaceOpts = append(workspaceOpts, auto.WorkDir(workDir)) workspace, err := auto.NewLocalWorkspace(ctx, workspaceOpts...) if err != nil { - log.Fatalw("unable to open the workspace", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to open the workspace: %w", err) } proj, err := workspace.ProjectSettings(ctx) if err != nil { - log.Fatalw("unable to get the project settings", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to get the project settings: %w", err) } log.Infow("opened a local workspace", "workspace", workDir, "project", proj.Name, "runtime", proj.Runtime.Name()) @@ -96,8 +141,7 @@ var serveCmd = &cobra.Command{ } log.Infow("installing project dependencies") if err := workspace.Install(ctx, opts); err != nil { - log.Fatalw("installation failed", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to install project dependencies: %w", err) } log.Infow("installation completed") } else { @@ -110,30 +154,28 @@ var serveCmd = &cobra.Command{ StackName: _stack, }) if err != nil { - log.Fatalw("unable to make an automation server", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to make an automation server: %w", err) } address := fmt.Sprintf("%s:%d", _host, _port) log.Infow("starting the RPC server", "address", address) - s := server.NewGRPC(autoServer, log) + s := server.NewGRPC(log, autoServer, authFunc) // Start the grpc server lis, err := net.Listen("tcp", address) if err != nil { - log.Errorw("fatal: unable to start the RPC server", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unable to listen on %s: %w", address, err) } log.Infow("server listening", "address", lis.Addr(), "workspace", workDir) ctx, cancel := context.WithCancel(ctx) setupSignalHandler(cancel) if err := s.Serve(ctx, lis); err != nil { - log.Errorw("fatal: server failure", zap.Error(err)) - os.Exit(1) + return fmt.Errorf("unexpected serve error: %w", err) } log.Infow("server stopped") + return nil }, } @@ -166,4 +208,8 @@ func init() { serveCmd.Flags().StringVar(&_host, "host", "0.0.0.0", "Server bind address (default: 0.0.0.0)") serveCmd.Flags().IntVar(&_port, "port", 50051, "Server port (default: 50051)") + + serveCmd.Flags().StringVar(&_authMode, "auth-mode", AuthModeNone, "Authorization mode (none, kube)") + serveCmd.Flags().StringVar(&_workspaceNamespace, "kube-workspace-namespace", os.Getenv("WORKSPACE_NAMESPACE"), "The Workspace object namespace (for kubernetes auth mode)") + serveCmd.Flags().StringVar(&_workspaceName, "kube-workspace-name", os.Getenv("WORKSPACE_NAME"), "The Workspace object name (for kubernetes auth mode)") } diff --git a/agent/hack/token.yaml b/agent/hack/token.yaml new file mode 100644 index 00000000..aa48390c --- /dev/null +++ b/agent/hack/token.yaml @@ -0,0 +1,27 @@ +# Apply this manifest file to create a token with which to authenticate to the agent. +# To get the token, run the following command: kubectl describe secret/dev-token +# To test: kubectl auth can-i use workspaces/random-yaml --subresource rpc --as system:serviceaccount:default:dev +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dev +--- +apiVersion: v1 +kind: Secret +metadata: + name: dev-token + annotations: + kubernetes.io/service-account.name: dev +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: dev:cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: dev \ No newline at end of file diff --git a/agent/pkg/client/client.go b/agent/pkg/client/client.go new file mode 100644 index 00000000..57800359 --- /dev/null +++ b/agent/pkg/client/client.go @@ -0,0 +1,75 @@ +/* +Copyright © 2024 Pulumi Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + + "golang.org/x/oauth2" + "google.golang.org/grpc/credentials" + "k8s.io/client-go/transport" +) + +// NewTokenCredentials adds the provided bearer token to a request. +// If tokenFile is non-empty, it is periodically read, +// and the last successfully read content is used as the bearer token. +// If tokenFile is non-empty and bearer is empty, the tokenFile is read +// immediately to populate the initial bearer token. +func NewTokenCredentials(bearer string, tokenFile string) (*TokenCredentials, error) { + if len(tokenFile) == 0 { + return &TokenCredentials{bearer, nil}, nil + } + source := transport.NewCachedFileTokenSource(tokenFile) + if len(bearer) == 0 { + token, err := source.Token() + if err != nil { + return nil, err + } + bearer = token.AccessToken + } + return &TokenCredentials{bearer, source}, nil +} + +type TokenCredentials struct { + bearer string + source oauth2.TokenSource +} + +// GetRequestMetadata gets the current request metadata, refreshing tokens +// if required. This should be called by the transport layer on each +// request, and the data should be populated in headers or other +// context. If a status code is returned, it will be used as the status for +// the RPC (restricted to an allowable set of codes as defined by gRFC +// A54). uri is the URI of the entry point for the request. When supported +// by the underlying implementation, ctx can be used for timeout and +// cancellation. Additionally, RequestInfo data will be available via ctx +// to this call. +func (k *TokenCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + token := k.bearer + if k.source != nil { + if refreshedToken, err := k.source.Token(); err == nil { + token = refreshedToken.AccessToken + } + } + return map[string]string{"authorization": "Bearer " + token}, nil +} + +func (k *TokenCredentials) RequireTransportSecurity() bool { + return false +} + +var _ credentials.PerRPCCredentials = &TokenCredentials{} diff --git a/agent/pkg/server/auth.go b/agent/pkg/server/auth.go new file mode 100644 index 00000000..e147d83c --- /dev/null +++ b/agent/pkg/server/auth.go @@ -0,0 +1,231 @@ +/* +Copyright © 2024 Pulumi Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "fmt" + "os" + "time" + + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" + grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/token/cache" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" + authenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1" + authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1" + "k8s.io/client-go/rest" +) + +const ( + ServiceAccountPermissionsErrorMessage = ` +The stack's service account needs permission to access the Kubernetes API. +Please add a ClusterRoleBinding for the ClusterRole 'system:auth-delegator'. +For example: + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: %s:%s:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + namespace: %s + name: %s + +` +) + +func formattedServiceAccountPermissionsErrorMessage() string { + saName := types.NamespacedName{ + Namespace: os.Getenv("POD_NAMESPACE"), + Name: os.Getenv("POD_SA_NAME"), + } + return fmt.Sprintf(ServiceAccountPermissionsErrorMessage, saName.Namespace, saName.Name, saName.Namespace, saName.Name) +} + +type KubeAuthOptions struct { + WorkspaceName types.NamespacedName +} + +// NewKubeAuth provides a grpc_auth.AuthFunc for authentication and authorization. +// Requests will be authenticated (via TokenReviews) and authorized (via SubjectAccessReviews) with the +// kube-apiserver. +// For the authentication and authorization the agent needs a role +// with the following rules: +// * apiGroups: authentication.k8s.io, resources: tokenreviews, verbs: create +// * apiGroups: authorization.k8s.io, resources: subjectaccessreviews, verbs: create +// +// To make RPC requests e.g. as the Operator the client needs a role +// with the following rule: +// * apiGroups: auto.pulumi.com, resources: workspaces/rpc, verbs: use +func NewKubeAuth(rootLogger *zap.Logger, config *rest.Config, opts KubeAuthOptions) (grpc_auth.AuthFunc, error) { + authenticationV1Client, err := authenticationv1.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create authentication client: %w", err) + } + authorizationV1Client, err := authorizationv1.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create authorization client: %w", err) + } + + tokenAuth, err := webhooktoken.NewFromInterface( + authenticationV1Client, + nil, + wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + 10*time.Second, /* requestTimeout */ + webhooktoken.AuthenticatorMetrics{ + RecordRequestTotal: RecordRequestTotal, + RecordRequestLatency: RecordRequestLatency, + }) + if err != nil { + return nil, fmt.Errorf("failed to create webhook authenticator: %w", err) + } + delegatingAuthenticator := cache.New(tokenAuth, false, 1*time.Hour /* successTTL */, 1*time.Minute /* failureTTL */) + + authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{ + SubjectAccessReviewClient: authorizationV1Client, + AllowCacheTTL: 5 * time.Minute, + DenyCacheTTL: 5 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthorizer, err := authorizerConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authorizer: %w", err) + } + + a := &kubeAuth{ + log: rootLogger.Named("grpc").Sugar(), + authn: delegatingAuthenticator, + authz: delegatingAuthorizer, + workspaceName: opts.WorkspaceName, + } + return a.Authenticate, nil +} + +type kubeAuth struct { + log *zap.SugaredLogger + authn authenticator.Token + authz authorizer.Authorizer + workspaceName types.NamespacedName +} + +// Authenticate implements grpc_auth.AuthFunc to perform authentication and authorization. +// Authentication is done via TokenReview and authorization via SubjectAccessReview. +// +// The passed in `Context` will contain the gRPC metadata.MD object (for header-based authentication) and +// the peer.Peer information that can contain transport-based credentials (e.g. `credentials.AuthInfo`). +// +// The returned context will be propagated to handlers, allowing user changes to `Context`. However, +// please make sure that the `Context` returned is a child `Context` of the one passed in. +// +// If error is returned, its `grpc.Code()` will be returned to the user as well as the verbatim message. +// Please make sure you use `codes.Unauthenticated` (lacking auth) and `codes.PermissionDenied` +// (authed, but lacking perms) appropriately. +func (a *kubeAuth) Authenticate(ctx context.Context) (context.Context, error) { + tags := grpc_ctxtags.Extract(ctx) + tags.Set("auth.mode", "kubernetes") + + token, err := grpc_auth.AuthFromMD(ctx, "Bearer") + if err != nil { + return nil, err + } + + // Authenticate the user. + res, ok, err := a.authn.AuthenticateToken(ctx, token) + if err != nil { + if apierrors.IsForbidden(err) { + a.log.Errorw(formattedServiceAccountPermissionsErrorMessage()) + return nil, status.Error(codes.Unauthenticated, "TokenReview API is unavailable") + } + a.log.Warn("authentication failed with an error", zap.Error(err)) + return nil, status.Error(codes.Unauthenticated, err.Error()) + } + if !ok { + a.log.Warn("authentication failed (unauthenticated)") + return nil, status.Error(codes.Unauthenticated, "unauthenticated") + } + a.log.Debugw("authenticated user", zap.String("name", res.User.GetName()), zap.String("uid", res.User.GetUID())) + tags.Set("user.id", res.User.GetUID()) + tags.Set("user.name", res.User.GetName()) + + // Authorize the user to use the workspace RPC endpoint. + + attributes := authorizer.AttributesRecord{ + User: res.User, + Namespace: a.workspaceName.Namespace, + Name: a.workspaceName.Name, + ResourceRequest: true, + APIGroup: "auto.pulumi.com", + APIVersion: "v1alpha1", + Resource: "workspaces", + Subresource: "rpc", + Verb: "use", + } + authorized, reason, err := a.authz.Authorize(ctx, attributes) + if err != nil { + if apierrors.IsForbidden(err) { + a.log.Errorw(formattedServiceAccountPermissionsErrorMessage()) + return nil, status.Error(codes.Unauthenticated, "SubjectAccessReview API is unavailable") + } + a.log.Errorw("authorization failed with an error", zap.Error(err), "user", attributes.User.GetName()) + return nil, err + } + + if authorized != authorizer.DecisionAllow { + if reason == "" { + reason = "forbidden" + } + a.log.Warnw("access denied", zap.Error(err), zap.String("user", attributes.User.GetName()), zap.String("reason", reason)) + return nil, status.Error(codes.PermissionDenied, reason) + } + a.log.Debugw("authorization allowed", zap.String("reason", reason)) + + return context.WithValue(ctx, "k8s.user", res.User), nil +} + +// RecordRequestTotal increments the total number of requests for the delegated authentication. +func RecordRequestTotal(ctx context.Context, code string) { +} + +// RecordRequestLatency measures request latency in seconds for the delegated authentication. Broken down by status code. +func RecordRequestLatency(ctx context.Context, code string, latency float64) { +} diff --git a/agent/pkg/server/auth_test.go b/agent/pkg/server/auth_test.go new file mode 100644 index 00000000..ef93030e --- /dev/null +++ b/agent/pkg/server/auth_test.go @@ -0,0 +1,248 @@ +/* +Copyright © 2024 Pulumi Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "errors" + "testing" + + grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" + "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" + "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + "go.uber.org/zap" + grpc_codes "google.golang.org/grpc/codes" + grpc_status "google.golang.org/grpc/status" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/utils/ptr" +) + +func TestKubernetes(t *testing.T) { + t.Parallel() + log := zap.L().Named("TestKubernetes").Sugar() + + unavailable := func() authenticator.TokenFunc { + return func(ctx context.Context, token string) (*authenticator.Response, bool, error) { + return nil, false, apierrors.NewForbidden(schema.GroupResource{}, "", errors.New("testing")) + } + } + + authenticate := func(knownToken string, knownUser user.DefaultInfo) authenticator.TokenFunc { + return func(ctx context.Context, token string) (*authenticator.Response, bool, error) { + switch token { + case "": + return nil, false, errors.New("Request unauthenticated with Bearer") + case "b4d": + return nil, false, errors.New("invalid bearer token") + case knownToken: + return &authenticator.Response{ + User: &knownUser, + }, true, nil + default: + return nil, false, nil + } + } + } + + authorize := func(knownUser user.DefaultInfo, knownWorkspace types.NamespacedName) authorizer.AuthorizerFunc { + return func(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { + if !(a.GetUser() != nil && a.GetUser().GetName() == knownUser.Name) { + return authorizer.DecisionDeny, "", nil + } + if !(a.GetVerb() == "use" && a.GetResource() == "workspaces" && a.GetSubresource() == "rpc") { + return authorizer.DecisionDeny, "", nil + } + if !(a.GetNamespace() == knownWorkspace.Namespace && a.GetName() == knownWorkspace.Name) { + return authorizer.DecisionDeny, "", nil + } + return authorizer.DecisionAllow, "", nil + } + } + + testWorkspace := types.NamespacedName{ + Namespace: "default", + Name: "test", + } + otherWorkspace := types.NamespacedName{ + Namespace: "default", + Name: "other", + } + testUser := user.DefaultInfo{ + Name: "system:serviceaccount:default:test", + UID: "81be050c-9ad4-4708-9a52-413064700747", + } + otherUser := user.DefaultInfo{ + Name: "system:serviceaccount:default:other", + UID: "71be050c-9ad4-4708-9a52-413064700747", + } + + tests := []struct { + name string + resourceName types.NamespacedName + authHeaderValue *string + authn authenticator.TokenFunc + authz authorizer.AuthorizerFunc + wantStatus *grpc_status.Status + wantTags gstruct.Keys + }{ + { + name: "Unavailable (AuthN)", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer g00d"), + authn: unavailable(), + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "TokenReview API is unavailable"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "NoToken", + resourceName: testWorkspace, + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "Request unauthenticated with Bearer"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "NotBearerToken", + resourceName: testWorkspace, + authHeaderValue: ptr.To("basic dXNlcm5hbWU6cGFzc3dvcmQ="), + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "Request unauthenticated with Bearer"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "InvalidToken", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer b4d"), + authn: authenticate("g00d", testUser), + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "invalid bearer token"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "AuthenticationFailure", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer 0ther"), + authn: authenticate("g00d", testUser), + wantStatus: grpc_status.New(grpc_codes.Unauthenticated, "unauthenticated"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + }, + }, + { + name: "Denied_User", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer g00d"), + authn: authenticate("g00d", testUser), + authz: authorize(otherUser, testWorkspace), + wantStatus: grpc_status.New(grpc_codes.PermissionDenied, "forbidden"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + "user.id": gomega.Equal(testUser.UID), + "user.name": gomega.Equal(testUser.Name), + }, + }, + { + name: "Denied_ResourceName", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer g00d"), + authn: authenticate("g00d", testUser), + authz: authorize(testUser, otherWorkspace), + wantStatus: grpc_status.New(grpc_codes.PermissionDenied, "forbidden"), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + "user.id": gomega.Equal(testUser.UID), + "user.name": gomega.Equal(testUser.Name), + }, + }, + { + name: "Allowed", + resourceName: testWorkspace, + authHeaderValue: ptr.To("Bearer g00d"), + authn: authenticate("g00d", testUser), + authz: authorize(testUser, testWorkspace), + wantTags: gstruct.Keys{ + "auth.mode": gomega.Equal("kubernetes"), + "user.id": gomega.Equal(testUser.UID), + "user.name": gomega.Equal(testUser.Name), + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + g := gomega.NewWithT(t) + log := log.Named(t.Name()) + + if tt.authn == nil { + tt.authn = func(ctx context.Context, token string) (*authenticator.Response, bool, error) { + t.Error("unexpected call to AuthenticateToken") + return nil, false, nil + } + } + if tt.authz == nil { + tt.authz = func(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { + t.Error("unexpected call to Authorize") + return authorizer.DecisionNoOpinion, "", nil + } + } + + kubeAuth := &kubeAuth{ + log: log, + authn: tt.authn, + authz: tt.authz, + workspaceName: tt.resourceName, + } + + // prepare the context + md := metautils.NiceMD{} + if tt.authHeaderValue != nil { + md.Add("authorization", *tt.authHeaderValue) + } + tags := grpc_ctxtags.NewTags() + ctx := md.ToIncoming(context.Background()) + ctx = grpc_ctxtags.SetInContext(ctx, tags) + + // execute the auth function + ctx, err := kubeAuth.Authenticate(ctx) + + // validate the tags, some of which are set even if the function fails + if tt.wantTags != nil { + g.Expect(tags.Values()).To(gstruct.MatchAllKeys(tt.wantTags)) + } + + // validate the resultant status + status := grpc_status.Convert(err) + if tt.wantStatus != nil { + g.Expect(status).To(gomega.HaveValue(gomega.Equal(*tt.wantStatus)), "an unexpected status") + } + if err == nil { + g.Expect(ctx.Value("k8s.user")).ToNot(gomega.BeNil()) + } + }) + } +} diff --git a/agent/pkg/server/grpc.go b/agent/pkg/server/grpc.go index 1bdce0d5..ad08445d 100644 --- a/agent/pkg/server/grpc.go +++ b/agent/pkg/server/grpc.go @@ -19,6 +19,7 @@ import ( "context" "net" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" pb "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/proto" @@ -33,8 +34,8 @@ type GRPC struct { log *zap.SugaredLogger } -// NewGRPC constructs a new gRPC server with logging and graceful shutdown. -func NewGRPC(server *Server, rootLogger *zap.SugaredLogger) *GRPC { +// NewGRPC constructs a new gRPC server with logging and authentication support. +func NewGRPC(rootLogger *zap.SugaredLogger, server *Server, authF grpc_auth.AuthFunc) *GRPC { log := rootLogger.Named("grpc") // Configure the grpc server. // Apply zap logging and use filters to reduce log verbosity as needed. @@ -43,17 +44,26 @@ func NewGRPC(server *Server, rootLogger *zap.SugaredLogger) *GRPC { return true }), } - grpc_zap.ReplaceGrpcLoggerV2WithVerbosity(log.Desugar(), int(log.Level())) + // Apply a default authentication function. + if authF == nil { + authF = func(ctx context.Context) (context.Context, error) { + return ctx, nil + } + } + + // Create the gRPC server. s := grpc.NewServer( grpc.ChainUnaryInterceptor( grpc_ctxtags.UnaryServerInterceptor(grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor)), grpc_zap.UnaryServerInterceptor(log.Desugar(), serverOpts...), + grpc_auth.UnaryServerInterceptor(authF), ), grpc.ChainStreamInterceptor( grpc_ctxtags.StreamServerInterceptor(grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor)), grpc_zap.StreamServerInterceptor(log.Desugar(), serverOpts...), + grpc_auth.StreamServerInterceptor(authF), ), ) pb.RegisterAutomationServiceServer(s, server) diff --git a/agent/pkg/server/grpc_test.go b/agent/pkg/server/grpc_test.go index 5a8fb076..263719b1 100644 --- a/agent/pkg/server/grpc_test.go +++ b/agent/pkg/server/grpc_test.go @@ -45,7 +45,7 @@ func TestGracefulShutdown(t *testing.T) { tc := newTC(ctx, t, tcOptions{ProjectDir: "./testdata/hang", Stacks: []string{"test"}}) log := zap.L().Named("test").Sugar() lis := bufconn.Listen(1024) - s := server.NewGRPC(tc.server, log) + s := server.NewGRPC(log, tc.server, nil) go func() { // This should exit cleanly if we shut down gracefully. if err := s.Serve(ctx, lis); err != nil { diff --git a/deploy/helm/pulumi-operator/templates/clusterrole.yaml b/deploy/helm/pulumi-operator/templates/clusterrole.yaml index 46b46840..5e09cdb6 100644 --- a/deploy/helm/pulumi-operator/templates/clusterrole.yaml +++ b/deploy/helm/pulumi-operator/templates/clusterrole.yaml @@ -82,6 +82,12 @@ rules: - workspaces/finalizers verbs: - update +- apiGroups: + - auto.pulumi.com + resources: + - workspaces/rpc + verbs: + - use - apiGroups: - auto.pulumi.com resources: diff --git a/deploy/helm/pulumi-operator/templates/role.yaml b/deploy/helm/pulumi-operator/templates/role.yaml index 828f9efc..354ad28e 100644 --- a/deploy/helm/pulumi-operator/templates/role.yaml +++ b/deploy/helm/pulumi-operator/templates/role.yaml @@ -82,6 +82,12 @@ rules: - workspaces/finalizers verbs: - update +- apiGroups: + - auto.pulumi.com + resources: + - workspaces/rpc + verbs: + - use - apiGroups: - auto.pulumi.com resources: diff --git a/deploy/yaml/install.yaml b/deploy/yaml/install.yaml index 4d38710a..50604a50 100644 --- a/deploy/yaml/install.yaml +++ b/deploy/yaml/install.yaml @@ -28045,6 +28045,12 @@ spec: --- apiVersion: v1 kind: ServiceAccount +metadata: + name: pulumi + namespace: default +--- +apiVersion: v1 +kind: ServiceAccount metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -28095,6 +28101,34 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: pulumi-kubernetes-operator-controller-manager rules: @@ -28171,6 +28205,12 @@ rules: - workspaces/finalizers verbs: - update +- apiGroups: + - auto.pulumi.com + resources: + - workspaces/rpc + verbs: + - use - apiGroups: - auto.pulumi.com resources: @@ -28342,6 +28382,32 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: default:pulumi:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: pulumi + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -28358,6 +28424,22 @@ subjects: --- apiVersion: v1 kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: pulumi-kubernetes-operator +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service metadata: labels: control-plane: controller-manager @@ -28401,9 +28483,9 @@ spec: containers: - args: - /manager + - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 - - --metrics-bind-address=:8383 - --program-fs-adv-addr=pulumi-kubernetes-operator.$(POD_NAMESPACE).svc.cluster.local:80 - --zap-log-level=error - --zap-time-encoding=iso8601 diff --git a/go.mod b/go.mod index 464ceb56..64e2f788 100644 --- a/go.mod +++ b/go.mod @@ -22,11 +22,13 @@ require ( github.com/stretchr/testify v1.9.0 go.uber.org/mock v0.4.0 go.uber.org/zap v1.27.0 + golang.org/x/oauth2 v0.20.0 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 k8s.io/api v0.30.1 k8s.io/apiextensions-apiserver v0.30.1 k8s.io/apimachinery v0.30.1 + k8s.io/apiserver v0.30.1 k8s.io/client-go v0.30.1 k8s.io/kubernetes v1.30.3 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 @@ -41,7 +43,7 @@ require ( github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -131,7 +133,7 @@ require ( github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect @@ -153,7 +155,6 @@ require ( golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/oauth2 v0.20.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.21.0 // indirect @@ -168,7 +169,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.30.1 // indirect k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect diff --git a/go.sum b/go.sum index 92f4bd33..9bd1e244 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -17,8 +21,8 @@ github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7l github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -302,10 +306,12 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -314,6 +320,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqajr6t1lOv8GyGE2U= diff --git a/operator/Makefile b/operator/Makefile index c9ad6416..2a090b7b 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -173,7 +173,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml + $(KUSTOMIZE) build config/quickstart > dist/install.yaml ##@ Deployment diff --git a/operator/config/flux/network_policy.yaml b/operator/config/flux/network_policy.yaml new file mode 100644 index 00000000..2c5d5143 --- /dev/null +++ b/operator/config/flux/network_policy.yaml @@ -0,0 +1,26 @@ +# A network policy to allow Pulumi workspaces in the `default` namespace to +# fetch Flux artifacts from the source-controller in the `flux-system` namespace. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-pulumi-fetch-flux-artifacts + namespace: flux-system +spec: + podSelector: + matchLabels: + app: source-controller + ingress: + - ports: + - protocol: TCP + port: http + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default + - podSelector: + matchLabels: + app.kubernetes.io/managed-by: pulumi-kubernetes-operator + app.kubernetes.io/name: pulumi + app.kubernetes.io/component: workspace + policyTypes: + - Ingress \ No newline at end of file diff --git a/operator/config/quickstart/kustomization.yaml b/operator/config/quickstart/kustomization.yaml new file mode 100644 index 00000000..671c9773 --- /dev/null +++ b/operator/config/quickstart/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - ../default + - service_account.yaml + - rbac.yaml diff --git a/operator/config/quickstart/rbac.yaml b/operator/config/quickstart/rbac.yaml new file mode 100644 index 00000000..7d5816d1 --- /dev/null +++ b/operator/config/quickstart/rbac.yaml @@ -0,0 +1,14 @@ +# Grant `system:auth-delegator` to the `default/pulumi` service account, +# to enable Kubernetes RBAC for the Pulumi workspace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:pulumi:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator # permissions: TokenReview, SubjectAccessReview +subjects: +- kind: ServiceAccount + namespace: default + name: pulumi diff --git a/operator/config/quickstart/service_account.yaml b/operator/config/quickstart/service_account.yaml new file mode 100644 index 00000000..da2ba593 --- /dev/null +++ b/operator/config/quickstart/service_account.yaml @@ -0,0 +1,8 @@ +# A service account named `default/pulumi` for the Pulumi workspace (execution environment). +# If your Pulumi program uses the Kubernetes resource provider, this service account will be used to +# authenticate with the Kubernetes cluster. +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: default + name: pulumi diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index cfe38a91..604f10a5 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -77,6 +77,12 @@ rules: - workspaces/finalizers verbs: - update +- apiGroups: + - auto.pulumi.com + resources: + - workspaces/rpc + verbs: + - use - apiGroups: - auto.pulumi.com resources: diff --git a/operator/e2e/testdata/git-auth-nonroot/manifests.yaml b/operator/e2e/testdata/git-auth-nonroot/manifests.yaml index 2457753e..96776469 100644 --- a/operator/e2e/testdata/git-auth-nonroot/manifests.yaml +++ b/operator/e2e/testdata/git-auth-nonroot/manifests.yaml @@ -5,6 +5,25 @@ metadata: name: git-auth-nonroot --- apiVersion: v1 +kind: ServiceAccount +metadata: + name: git-auth-nonroot + namespace: git-auth-nonroot +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: git-auth-nonroot:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: git-auth-nonroot + namespace: git-auth-nonroot +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: state @@ -54,6 +73,7 @@ spec: value: "test" workspaceTemplate: spec: + serviceAccountName: git-auth-nonroot podTemplate: spec: containers: diff --git a/operator/e2e/testdata/random-yaml-nonroot/manifests.yaml b/operator/e2e/testdata/random-yaml-nonroot/manifests.yaml index fd9fa73b..4eb65650 100644 --- a/operator/e2e/testdata/random-yaml-nonroot/manifests.yaml +++ b/operator/e2e/testdata/random-yaml-nonroot/manifests.yaml @@ -1,10 +1,54 @@ --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-random-yaml-nonroot-fetch + namespace: flux-system +spec: + podSelector: + matchLabels: + app: source-controller + ingress: + - ports: + - protocol: TCP + port: http + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: random-yaml-nonroot + - podSelector: + matchLabels: + app.kubernetes.io/managed-by: pulumi-kubernetes-operator + app.kubernetes.io/name: pulumi + app.kubernetes.io/component: workspace + policyTypes: + - Ingress +--- apiVersion: v1 kind: Namespace metadata: name: random-yaml-nonroot --- apiVersion: v1 +kind: ServiceAccount +metadata: + name: random-yaml-nonroot + namespace: random-yaml-nonroot +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: random-yaml-nonroot:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: random-yaml-nonroot + namespace: random-yaml-nonroot +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: state @@ -59,7 +103,7 @@ spec: value: "test" workspaceTemplate: spec: - image: pulumi/pulumi:3.134.1-nonroot + serviceAccountName: random-yaml-nonroot podTemplate: spec: containers: diff --git a/operator/e2e/testdata/targets/manifests.yaml b/operator/e2e/testdata/targets/manifests.yaml index 4d03328d..f4a8bf9b 100644 --- a/operator/e2e/testdata/targets/manifests.yaml +++ b/operator/e2e/testdata/targets/manifests.yaml @@ -5,6 +5,25 @@ metadata: name: targets --- apiVersion: v1 +kind: ServiceAccount +metadata: + name: targets + namespace: targets +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: targets:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: targets + namespace: targets +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: state @@ -61,6 +80,7 @@ spec: value: "test" workspaceTemplate: spec: + serviceAccountName: targets podTemplate: spec: containers: diff --git a/operator/examples/random-yaml/stack.yaml b/operator/examples/random-yaml/stack.yaml index b4a15399..b3e1d2ad 100644 --- a/operator/examples/random-yaml/stack.yaml +++ b/operator/examples/random-yaml/stack.yaml @@ -1,7 +1,27 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: random-yaml + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: random-yaml:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: random-yaml + namespace: default +--- apiVersion: pulumi.com/v1 kind: Stack metadata: name: random-yaml + namespace: default spec: fluxSource: sourceRef: @@ -24,5 +44,6 @@ spec: key: accessToken workspaceTemplate: spec: + serviceAccountName: random-yaml image: pulumi/pulumi:3.134.1-nonroot diff --git a/operator/internal/controller/auto/update_controller.go b/operator/internal/controller/auto/update_controller.go index 798191ba..e2d58a9b 100644 --- a/operator/internal/controller/auto/update_controller.go +++ b/operator/internal/controller/auto/update_controller.go @@ -71,6 +71,7 @@ type UpdateReconciler struct { //+kubebuilder:rbac:groups=auto.pulumi.com,resources=updates/status,verbs=get;update;patch //+kubebuilder:rbac:groups=auto.pulumi.com,resources=updates/finalizers,verbs=update //+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces,verbs=get;list;watch +//+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces/rpc,verbs=use //+kubebuilder:rbac:groups="",resources=secrets,verbs=create // Reconcile manages the Update CRD and initiates Pulumi operations. diff --git a/operator/internal/controller/auto/utils.go b/operator/internal/controller/auto/utils.go index 2d5fa356..db029182 100644 --- a/operator/internal/controller/auto/utils.go +++ b/operator/internal/controller/auto/utils.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + agentclient "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/client" agentpb "github.com/pulumi/pulumi-kubernetes-operator/v2/agent/pkg/proto" autov1alpha1 "github.com/pulumi/pulumi-kubernetes-operator/v2/operator/api/auto/v1alpha1" "google.golang.org/grpc" @@ -17,7 +18,22 @@ func connect(ctx context.Context, addr string) (*grpc.ClientConn, error) { if os.Getenv("WORKSPACE_LOCALHOST") != "" { addr = os.Getenv("WORKSPACE_LOCALHOST") } - conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + + token := os.Getenv("WORKSPACE_TOKEN") + tokenFile := os.Getenv("WORKSPACE_TOKEN_FILE") + if token == "" && tokenFile == "" { + // use in-cluster configuration using the operator's service account token + tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + creds, err := agentclient.NewTokenCredentials(token, tokenFile) + if err != nil { + return nil, fmt.Errorf("unable to use token credentials: %w", err) + } + + conn, err := grpc.NewClient( + addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithPerRPCCredentials(creds)) if err != nil { return nil, fmt.Errorf("unable to connect to workspace: %w", err) } diff --git a/operator/internal/controller/auto/workspace_controller.go b/operator/internal/controller/auto/workspace_controller.go index f83de2eb..b557cbe5 100644 --- a/operator/internal/controller/auto/workspace_controller.go +++ b/operator/internal/controller/auto/workspace_controller.go @@ -71,6 +71,7 @@ type WorkspaceReconciler struct { //+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces/status,verbs=get;update;patch //+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces/finalizers,verbs=update +//+kubebuilder:rbac:groups=auto.pulumi.com,resources=workspaces/rpc,verbs=use //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete @@ -217,6 +218,7 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( defer func() { _ = conn.Close() }() + l.Info("Connected to workspace pod", "addr", addr) w.Status.Address = addr wc := agentpb.NewAutomationServiceClient(conn) @@ -368,9 +370,26 @@ func newStatefulSet(ctx context.Context, w *autov1alpha1.Workspace, source *sour "--workspace", "/share/workspace", "--skip-install", } - env := w.Spec.Env + // provide some pod information to the agent for informational purposes + env = append(env, corev1.EnvVar{ + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }) + env = append(env, corev1.EnvVar{ + Name: "POD_SA_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.serviceAccountName", + }, + }, + }) + // limit the memory usage to the reserved amount // https://github.com/pulumi/pulumi-kubernetes-operator/issues/698 env = append(env, corev1.EnvVar{ @@ -383,6 +402,12 @@ func newStatefulSet(ctx context.Context, w *autov1alpha1.Workspace, source *sour }, }) + // enable workspace endpoint protection + command = append(command, + "--auth-mode", "kube", + "--kube-workspace-namespace", w.Namespace, + "--kube-workspace-name", w.Name) + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet",