From 6ca31d38ef8404face3e7913c64bbaf2e0aaf2e5 Mon Sep 17 00:00:00 2001 From: Giulio Micheloni Date: Tue, 27 Aug 2024 12:24:45 +0200 Subject: [PATCH] buildx: support server-side proxy Part of: NSL-3935 --- internal/cli/cmd/cluster/buildx.go | 13 +- .../cli/cmd/cluster/server_side_buildx.go | 190 ++++++++++++++++++ internal/fnapi/tenants.go | 25 +++ internal/providers/nscloud/api/rpc.go | 32 +++ 4 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 internal/cli/cmd/cluster/server_side_buildx.go diff --git a/internal/cli/cmd/cluster/buildx.go b/internal/cli/cmd/cluster/buildx.go index 8529fefc3..26e65056c 100644 --- a/internal/cli/cmd/cluster/buildx.go +++ b/internal/cli/cmd/cluster/buildx.go @@ -41,7 +41,7 @@ import ( const ( metadataFile = "metadata.json" - defaultBuilder = "nsc-remote" + defaultBuilder = "namespace" proxyDir = "proxy" buildkitProxyPath = "buildkit/" + proxyDir ) @@ -72,6 +72,8 @@ func newSetupBuildxCmd() *cobra.Command { _ = cmd.Flags().MarkHidden("buildkit_sock_path") defaultLoad := cmd.Flags().Bool("default_load", false, "If true, load images to the Docker Engine image store if no other output is specified.") _ = cmd.Flags().MarkHidden("default_load") + useServerSideProxy := cmd.Flags().Bool("use_server_side_proxy", false, "If set, buildx is setup to use transparent server-side proxy powered by Namespace") + _ = cmd.Flags().MarkHidden("use_server_side_proxy") cmd.RunE = fncobra.RunE(func(ctx context.Context, args []string) error { if *debugDir != "" && !*background { @@ -99,8 +101,6 @@ func newSetupBuildxCmd() *cobra.Command { return err } - eg := executor.New(ctx, "proxies") - available, err := determineAvailable(ctx) if err != nil { return err @@ -115,6 +115,12 @@ func newSetupBuildxCmd() *cobra.Command { return err } + // NSL-3935 use remote-side buildx proxy + // This will be soon the default + if *useServerSideProxy { + return setupServerSideBuildxProxy(ctx, state, *name, *use, *defaultLoad, dockerCli, available) + } + fmt.Fprintf(console.Debug(ctx), "Using state path %q\n", state) if proxyAlreadyExists(state) { @@ -169,6 +175,7 @@ func newSetupBuildxCmd() *cobra.Command { md.Instances = append(md.Instances, instanceMD) } + eg := executor.New(ctx, "proxies") var instances []BuildCluster for i, p := range md.Instances { // Always create one, in case it's needed below. This instance has a zero-ish cost if we never call NewConn. diff --git a/internal/cli/cmd/cluster/server_side_buildx.go b/internal/cli/cmd/cluster/server_side_buildx.go new file mode 100644 index 000000000..6c1f5e2ce --- /dev/null +++ b/internal/cli/cmd/cluster/server_side_buildx.go @@ -0,0 +1,190 @@ +// Copyright 2022 Namespace Labs Inc; All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package cluster + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "fmt" + "maps" + "os" + "path" + + builderv1beta "buf.build/gen/go/namespace/cloud/protocolbuffers/go/proto/namespace/cloud/builder/v1beta" + "github.com/docker/buildx/store" + "github.com/docker/buildx/util/dockerutil" + "github.com/docker/cli/cli/command" + "github.com/pkg/errors" + "namespacelabs.dev/foundation/internal/fnapi" + "namespacelabs.dev/foundation/internal/providers/nscloud/api" +) + +func setupServerSideBuildxProxy(ctx context.Context, stateDir, builderName string, use, defaultLoad bool, dockerCli *command.DockerCli, platforms []api.BuildPlatform) error { + // Generate private and public keys + privKeyPem, pubKeyPem, err := genPrivAndPublicKeysPEM() + if err != nil { + return err + } + + // Ask IAM server to exchange our tenant token with a certificate using this public key + cliCert, err := fnapi.ExchangeTenantTokenForClientCert(ctx, string(pubKeyPem)) + if err != nil { + return err + } + + serverBuilderConfigs := []*builderv1beta.GetBuilderConfigurationResponse{} + for _, plat := range platforms { + // Download the builder config for this platform + resp, err := api.GetBuilderConfiguration(ctx, plat) + if err != nil { + return err + } + + serverBuilderConfigs = append(serverBuilderConfigs, resp) + } + + // Write key files in ns state directory + state, err := ensureStateDir(stateDir, buildkitProxyPath) + if err != nil { + return err + } + + privKeyPath := path.Join(state, "private_key.pem") + if err := writeFileToPath(privKeyPem, privKeyPath); err != nil { + return err + } + + clientCertPath := path.Join(state, "client_cert.pem") + if err := writeFileToPath([]byte(cliCert.ClientCertificatePem), clientCertPath); err != nil { + return err + } + + builderConfigs := []builderConfig{} + for _, bc := range serverBuilderConfigs { + serverCAPath := path.Join(state, fmt.Sprintf("server_%s_cert.pem", bc.GetShape().GetMachineArch())) + if err := writeFileToPath([]byte(bc.GetServerCaPem()), serverCAPath); err != nil { + return err + } + + builderConfigs = append(builderConfigs, builderConfig{ + serverConfig: bc, + serverCAPath: serverCAPath, + }) + } + + // Create buildx builders + if err := wireRemoteBuildxProxy(dockerCli, builderName, use, defaultLoad, builderConfigs, privKeyPath, clientCertPath); err != nil { + return err + } + + return nil +} + +type builderConfig struct { + serverConfig *builderv1beta.GetBuilderConfigurationResponse + serverCAPath string +} + +func genPrivAndPublicKeysPEM() ([]byte, []byte, error) { + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, err + } + + publicKey := &privateKey.PublicKey + + publicKeyBytes, err := x509.MarshalPKIXPublicKey(publicKey) + if err != nil { + return nil, nil, err + } + + publicKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PUBLIC KEY", + Bytes: publicKeyBytes, + }) + + privateKeyBytes, err := x509.MarshalECPrivateKey(privateKey) + if err != nil { + return nil, nil, err + } + + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "EC PRIVATE KEY", + Bytes: privateKeyBytes, + }) + + return privateKeyPEM, publicKeyPEM, nil +} + +func writeFileToPath(content []byte, path string) error { + return os.WriteFile(path, content, 0600) +} + +func wireRemoteBuildxProxy(dockerCli *command.DockerCli, name string, use, defaultLoad bool, builderConfigs []builderConfig, privKeyPath, clientCertPath string) error { + return withStore(dockerCli, func(txn *store.Txn) error { + ng, err := txn.NodeGroupByName(name) + if err != nil { + if !os.IsNotExist(errors.Cause(err)) { + return err + } + } + + const driver = "remote" + + if ng == nil { + ng = &store.NodeGroup{ + Name: name, + Driver: driver, + } + } + + driverOpts := map[string]string{ + "cert": clientCertPath, + "key": privKeyPath, + } + + if defaultLoad { + // Supported starting with v0.14.0 + driverOpts["default-load"] = "true" + } + + for _, bc := range builderConfigs { + var platforms []string + if bc.serverConfig.GetShape().GetMachineArch() == "arm64" { + platforms = []string{"linux/arm64"} + } else { + platforms = []string{"linux/amd64"} + } + + doCopy := maps.Clone(driverOpts) + doCopy["cacert"] = bc.serverCAPath + + if err := ng.Update(bc.serverConfig.GetShape().GetMachineArch(), "tcp://"+bc.serverConfig.GetBuildkitEndpoint()+":443", platforms, true, true, nil, "", doCopy); err != nil { + return err + } + } + + if use { + ep, err := dockerutil.GetCurrentEndpoint(dockerCli) + if err != nil { + return err + } + + if err := txn.SetCurrent(ep, name, false, false); err != nil { + return err + } + } + + if err := txn.Save(ng); err != nil { + return err + } + + return nil + }) +} diff --git a/internal/fnapi/tenants.go b/internal/fnapi/tenants.go index 9a9a2f305..ca1589369 100644 --- a/internal/fnapi/tenants.go +++ b/internal/fnapi/tenants.go @@ -41,6 +41,14 @@ type ExchangeOIDCTokenRequest struct { OidcToken string `json:"oidc_token,omitempty"` } +type ExchangeTenantTokenForClientCertRequest struct { + PublicKeyPem string `json:"public_key_pem,omitempty"` +} + +type ExchangeTenantTokenForClientCertResponse struct { + ClientCertificatePem string `json:"client_certificate_pem,omitempty"` +} + type ExchangeTokenResponse struct { TenantToken string `json:"tenant_token,omitempty"` Tenant *Tenant `json:"tenant,omitempty"` @@ -220,6 +228,23 @@ func TrustAWSCognitoJWT(ctx context.Context, tenantID, identityPool, identityPro }.Do(ctx, req, ResolveIAMEndpoint, nil) } +func ExchangeTenantTokenForClientCert(ctx context.Context, publicKey string) (ExchangeTenantTokenForClientCertResponse, error) { + var res ExchangeTenantTokenForClientCertResponse + req := ExchangeTenantTokenForClientCertRequest{ + PublicKeyPem: publicKey, + } + + if err := (Call[ExchangeTenantTokenForClientCertRequest]{ + Method: "nsl.tenants.TenantsService/ExchangeTenantTokenForClientCert", + IssueBearerToken: IssueBearerToken, + Retryable: true, + }).Do(ctx, req, ResolveIAMEndpoint, DecodeJSONResponse(&res)); err != nil { + return res, err + } + + return res, nil +} + type GetTenantResponse struct { Tenant *Tenant `json:"tenant,omitempty"` } diff --git a/internal/providers/nscloud/api/rpc.go b/internal/providers/nscloud/api/rpc.go index 9fde1456a..66fe78520 100644 --- a/internal/providers/nscloud/api/rpc.go +++ b/internal/providers/nscloud/api/rpc.go @@ -359,6 +359,38 @@ func CreateAndWaitCluster(ctx context.Context, api API, opts CreateClusterOpts) return WaitClusterReady(ctx, api, cluster.ClusterId, opts.WaitClusterOpts) } +func GetBuilderConfiguration(ctx context.Context, platform BuildPlatform) (*builderv1beta.GetBuilderConfigurationResponse, error) { + token, err := fnapi.IssueBearerToken(ctx) + if err != nil { + return nil, err + } + + return tasks.Return(ctx, tasks.Action("nsc.get-builder-configuration").HumanReadablef(fmt.Sprintf("Fetching Builder config for platform: %s", platform)), + func(ctx context.Context) (*builderv1beta.GetBuilderConfigurationResponse, error) { + tid := ids.NewRandomBase32ID(4) + + cli, conn, err := public.NewBuilderServiceClient(ctx, tid, token) + if err != nil { + return nil, err + } + + defer conn.Close() + + t := time.Now() + fmt.Fprintf(console.Debug(ctx), "[%s] RPC: calling EnsureBuildInstance {platform: %v}\n", tid, platform) + response, err := cli.GetBuilderConfiguration(ctx, &builderv1beta.GetBuilderConfigurationRequest{ + Platform: string(platform), + }) + if err != nil { + return nil, fnerrors.New("failed while creating %v build cluster: %w", platform, err) + } + + fmt.Fprintf(console.Debug(ctx), "[%s] RPC: got buildkit_endpoint=%s server_ca_pem=%s shape=%s (took %v)\n", + tid, response.GetBuildkitEndpoint(), response.GetServerCaPem(), response.GetShape(), time.Since(t)) + return response, nil + }) +} + func EnsureBuildCluster(ctx context.Context, platform BuildPlatform) (*builderv1beta.EnsureBuildInstanceResponse, error) { token, err := fnapi.IssueBearerToken(ctx) if err != nil {