Merge pull request #2129 from BarDweller/mountpoints

Add ephemeral lifecycle image, enabling podman support

acceptance/acceptance_test.go

@@ -836,6 +836,8 @@ func testAcceptance(
 
 						when("builder is untrusted", func() {
 							it("uses the 5 phases, and runs the extender (build)", func() {
+								origLifecycle := lifecycle.Image()
+
 								output := pack.RunSuccessfully(
 									"build", repoName,
 									"-p", filepath.Join("testdata", "mock_app"),
@@ -846,7 +848,7 @@ func testAcceptance(
 								assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulImageBuild(repoName)
 
 								assertOutput := assertions.NewLifecycleOutputAssertionManager(t, output)
-								assertOutput.IncludesLifecycleImageTag(lifecycle.Image())
+								assertOutput.IncludesTagOrEphemeralLifecycle(origLifecycle)
 								assertOutput.IncludesSeparatePhasesWithBuildExtension()
 
 								t.Log("inspecting image")
@@ -886,6 +888,8 @@ func testAcceptance(
 								})
 
 								it("uses the 5 phases, and runs the extender (run)", func() {
+									origLifecycle := lifecycle.Image()
+
 									output := pack.RunSuccessfully(
 										"build", repoName,
 										"-p", filepath.Join("testdata", "mock_app"),
@@ -897,7 +901,8 @@ func testAcceptance(
 									assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulImageBuild(repoName)
 
 									assertOutput := assertions.NewLifecycleOutputAssertionManager(t, output)
-									assertOutput.IncludesLifecycleImageTag(lifecycle.Image())
+
+									assertOutput.IncludesTagOrEphemeralLifecycle(origLifecycle)
 									assertOutput.IncludesSeparatePhasesWithRunExtension()
 
 									t.Log("inspecting image")
@@ -977,6 +982,8 @@ func testAcceptance(
 
 					when("daemon", func() {
 						it("uses the 5 phases", func() {
+							origLifecycle := lifecycle.Image()
+
 							output := pack.RunSuccessfully(
 								"build", repoName,
 								"-p", filepath.Join("testdata", "mock_app"),
@@ -986,13 +993,15 @@ func testAcceptance(
 							assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulImageBuild(repoName)
 
 							assertOutput := assertions.NewLifecycleOutputAssertionManager(t, output)
-							assertOutput.IncludesLifecycleImageTag(lifecycle.Image())
+							assertOutput.IncludesTagOrEphemeralLifecycle(origLifecycle)
 							assertOutput.IncludesSeparatePhases()
 						})
 					})
 
 					when("--publish", func() {
 						it("uses the 5 phases", func() {
+							origLifecycle := lifecycle.Image()
+
 							buildArgs := []string{
 								repoName,
 								"-p", filepath.Join("testdata", "mock_app"),
@@ -1008,7 +1017,7 @@ func testAcceptance(
 							assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulImageBuild(repoName)
 
 							assertOutput := assertions.NewLifecycleOutputAssertionManager(t, output)
-							assertOutput.IncludesLifecycleImageTag(lifecycle.Image())
+							assertOutput.IncludesTagOrEphemeralLifecycle(origLifecycle)
 							assertOutput.IncludesSeparatePhases()
 						})
 					})

acceptance/assertions/lifecycle_output.go

@@ -6,6 +6,7 @@ package assertions
 import (
 	"fmt"
 	"regexp"
+	"strings"
 	"testing"
 
 	h "github.com/buildpacks/pack/testhelpers"
@@ -85,8 +86,12 @@ func (l LifecycleOutputAssertionManager) IncludesSeparatePhasesWithRunExtension(
 	l.assert.ContainsAll(l.output, "[detector]", "[analyzer]", "[extender (run)]", "[exporter]")
 }
 
-func (l LifecycleOutputAssertionManager) IncludesLifecycleImageTag(tag string) {
+func (l LifecycleOutputAssertionManager) IncludesTagOrEphemeralLifecycle(tag string) {
 	l.testObject.Helper()
 
-	l.assert.Contains(l.output, tag)
+	if !strings.Contains(l.output, tag) {
+		if !strings.Contains(l.output, "pack.local/lifecyle") {
+			l.testObject.Fatalf("Unable to locate reference to lifecycle image within output")
+		}
+	}
 }

pkg/client/build.go

@@ -13,6 +13,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -32,6 +33,7 @@ import (
 	"github.com/buildpacks/pack/internal/build"
 	"github.com/buildpacks/pack/internal/builder"
 	internalConfig "github.com/buildpacks/pack/internal/config"
+	"github.com/buildpacks/pack/internal/layer"
 	pname "github.com/buildpacks/pack/internal/name"
 	"github.com/buildpacks/pack/internal/paths"
 	"github.com/buildpacks/pack/internal/stack"
@@ -425,6 +427,29 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error {
 				return fmt.Errorf("fetching lifecycle image: %w", err)
 			}
 
+			// if lifecyle container os isn't windows, use ephemeral lifecycle to add /workspace with correct ownership
+			imageOS, err := lifecycleImage.OS()
+			if err != nil {
+				return errors.Wrap(err, "getting lifecycle image OS")
+			}
+			if imageOS != "windows" {
+				// obtain uid/gid from builder to use when extending lifecycle image
+				uid, gid, err := userAndGroupIDs(rawBuilderImage)
+				if err != nil {
+					return fmt.Errorf("obtaining build uid/gid from builder image: %w", err)
+				}
+
+				c.logger.Debugf("Creating ephemeral lifecycle from %s with uid %d and gid %d. With workspace dir %s", lifecycleImage.Name(), uid, gid, opts.Workspace)
+				// extend lifecycle image with mountpoints, and use it instead of current lifecycle image
+				lifecycleImage, err = c.createEphemeralLifecycle(lifecycleImage, opts.Workspace, uid, gid)
+				if err != nil {
+					return err
+				}
+				c.logger.Debugf("Selecting ephemeral lifecycle image %s for build", lifecycleImage.Name())
+				// cleanup the extended lifecycle image when done
+				defer c.docker.ImageRemove(context.Background(), lifecycleImage.Name(), types.ImageRemoveOptions{Force: true})
+			}
+
 			lifecycleOptsLifecycleImage = lifecycleImage.Name()
 			labels, err := lifecycleImage.Labels()
 			if err != nil {
@@ -639,7 +664,17 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error {
 			if len(manifestContents) < 1 {
 				return "", errors.New("missing manifest entries")
 			}
-			return manifestContents[0].Layers[len(manifestContents[0].Layers)-1], nil // we can assume the lifecycle layer is the last in the tar
+			// we can assume the lifecycle layer is the last in the tar, except if the lifecycle has been extended as an ephemeral lifecycle
+			layerOffset := 1
+			if strings.Contains(lifecycleOpts.LifecycleImage, "pack.local/lifecycle") {
+				layerOffset = 2
+			}
+
+			if (len(manifestContents[0].Layers) - layerOffset) < 0 {
+				return "", errors.New("Lifecycle image did not contain expected layer count")
+			}
+
+			return manifestContents[0].Layers[len(manifestContents[0].Layers)-layerOffset], nil
 		}()
 		if err != nil {
 			return "", err
@@ -1320,6 +1355,109 @@ func (c *Client) processExtensions(ctx context.Context, builderImage imgutil.Ima
 	return fetchedExs, orderExtensions, nil
 }
 
+func userAndGroupIDs(img imgutil.Image) (int, int, error) {
+	sUID, err := img.Env(builder.EnvUID)
+	if err != nil {
+		return 0, 0, errors.Wrap(err, "reading builder env variables")
+	} else if sUID == "" {
+		return 0, 0, fmt.Errorf("image %s missing required env var %s", style.Symbol(img.Name()), style.Symbol(builder.EnvUID))
+	}
+
+	sGID, err := img.Env(builder.EnvGID)
+	if err != nil {
+		return 0, 0, errors.Wrap(err, "reading builder env variables")
+	} else if sGID == "" {
+		return 0, 0, fmt.Errorf("image %s missing required env var %s", style.Symbol(img.Name()), style.Symbol(builder.EnvGID))
+	}
+
+	var uid, gid int
+	uid, err = strconv.Atoi(sUID)
+	if err != nil {
+		return 0, 0, fmt.Errorf("failed to parse %s, value %s should be an integer", style.Symbol(builder.EnvUID), style.Symbol(sUID))
+	}
+
+	gid, err = strconv.Atoi(sGID)
+	if err != nil {
+		return 0, 0, fmt.Errorf("failed to parse %s, value %s should be an integer", style.Symbol(builder.EnvGID), style.Symbol(sGID))
+	}
+
+	return uid, gid, nil
+}
+
+func workspacePathForOS(os, workspace string) string {
+	if workspace == "" {
+		workspace = "workspace"
+	}
+	if os == "windows" {
+		// note we don't use ephemeral lifecycle when os is windows..
+		return "c:\\" + workspace
+	}
+	return "/" + workspace
+}
+
+func (c *Client) addUserMountpoints(lifecycleImage imgutil.Image, dest string, workspace string, uid int, gid int) (string, error) {
+	// today only workspace needs to be added, easy to add future dirs if required.
+
+	imageOS, err := lifecycleImage.OS()
+	if err != nil {
+		return "", errors.Wrap(err, "getting image OS")
+	}
+	layerWriterFactory, err := layer.NewWriterFactory(imageOS)
+	if err != nil {
+		return "", err
+	}
+
+	workspace = workspacePathForOS(imageOS, workspace)
+
+	fh, err := os.Create(filepath.Join(dest, "dirs.tar"))
+	if err != nil {
+		return "", err
+	}
+	defer fh.Close()
+
+	lw := layerWriterFactory.NewWriter(fh)
+	defer lw.Close()
+
+	for _, path := range []string{workspace} {
+		if err := lw.WriteHeader(&tar.Header{
+			Typeflag: tar.TypeDir,
+			Name:     path,
+			Mode:     0755,
+			ModTime:  archive.NormalizedDateTime,
+			Uid:      uid,
+			Gid:      gid,
+		}); err != nil {
+			return "", errors.Wrapf(err, "creating %s mountpoint dir in layer", style.Symbol(path))
+		}
+	}
+
+	return fh.Name(), nil
+}
+
+func (c *Client) createEphemeralLifecycle(lifecycleImage imgutil.Image, workspace string, uid int, gid int) (imgutil.Image, error) {
+	lifecycleImage.Rename(fmt.Sprintf("pack.local/lifecycle/%x:latest", randString(10)))
+
+	tmpDir, err := os.MkdirTemp("", "create-lifecycle-scratch")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmpDir)
+	dirsTar, err := c.addUserMountpoints(lifecycleImage, tmpDir, workspace, uid, gid)
+	if err != nil {
+		return nil, err
+	}
+	if err := lifecycleImage.AddLayer(dirsTar); err != nil {
+		return nil, errors.Wrap(err, "adding mountpoint dirs layer")
+	}
+
+	err = lifecycleImage.Save()
+	if err != nil {
+		return nil, err
+	}
+
+	return lifecycleImage, nil
+}
+
 func (c *Client) createEphemeralBuilder(
 	rawBuilderImage imgutil.Image,
 	env map[string]string,

pkg/client/build_test.go

@@ -2098,16 +2098,18 @@ api = "0.2"
 				when("builder is untrusted", func() {
 					when("lifecycle image is available", func() {
 						it("uses the 5 phases with the lifecycle image", func() {
+							origLifecyleName := fakeLifecycleImage.Name()
+
 							h.AssertNil(t, subject.Build(context.TODO(), BuildOptions{
 								Image:        "some/app",
 								Builder:      defaultBuilderName,
 								Publish:      true,
 								TrustBuilder: func(string) bool { return false },
 							}))
 							h.AssertEq(t, fakeLifecycle.Opts.UseCreator, false)
-							h.AssertEq(t, fakeLifecycle.Opts.LifecycleImage, fakeLifecycleImage.Name())
-
-							args := fakeImageFetcher.FetchCalls[fakeLifecycleImage.Name()]
+							h.AssertContains(t, fakeLifecycle.Opts.LifecycleImage, "pack.local/lifecycle")
+							args := fakeImageFetcher.FetchCalls[origLifecyleName]
+							h.AssertNotNil(t, args)
 							h.AssertEq(t, args.Daemon, true)
 							h.AssertEq(t, args.PullPolicy, image.PullAlways)
 							h.AssertEq(t, args.Platform, "linux/amd64")
@@ -2196,16 +2198,17 @@ api = "0.2"
 				when("builder is untrusted", func() {
 					when("lifecycle image is available", func() {
 						it("uses the 5 phases with the lifecycle image", func() {
+							origLifecyleName := fakeLifecycleImage.Name()
 							h.AssertNil(t, subject.Build(context.TODO(), BuildOptions{
 								Image:        "some/app",
 								Builder:      defaultBuilderName,
 								Publish:      false,
 								TrustBuilder: func(string) bool { return false },
 							}))
 							h.AssertEq(t, fakeLifecycle.Opts.UseCreator, false)
-							h.AssertEq(t, fakeLifecycle.Opts.LifecycleImage, fakeLifecycleImage.Name())
-
-							args := fakeImageFetcher.FetchCalls[fakeLifecycleImage.Name()]
+							h.AssertContains(t, fakeLifecycle.Opts.LifecycleImage, "pack.local/lifecycle")
+							args := fakeImageFetcher.FetchCalls[origLifecyleName]
+							h.AssertNotNil(t, args)
 							h.AssertEq(t, args.Daemon, true)
 							h.AssertEq(t, args.PullPolicy, image.PullAlways)
 							h.AssertEq(t, args.Platform, "linux/amd64")