Merge pull request #1026 from buildpacks/add-benchmarks

Add Benchmark tests for Build
Signed-off-by: David Freilich <[email protected]>

.gitignore

@@ -1,6 +1,8 @@
 .pack/
 /pack
 out/
+benchmarks.test
+*.out
 
 # Jetbrains Goland
 .idea/
@@ -10,4 +12,5 @@ artifacts/
 .DS_Store
 
 # Travis unencrypted file
-.travis/key.pem
+.travis/key.pem
+

Makefile

@@ -114,7 +114,7 @@ acceptance-all:
 
 clean:
 	@echo "> Cleaning workspace..."
-	@$(RMRF) .$/out || (exit 0)
+	@$(RMRF) .$/out benchmarks.test || (exit 0)
 
 verify: verify-format lint
 
@@ -137,9 +137,18 @@ prepare-for-pr: tidy verify test
 	echo "-----------------\n"  &&\
 	exit 0)
 
+benchmark: out
+	@echo "> Running Benchmarks"
+	$(GOCMD) test -run=^$  -bench=. -benchtime=1s -benchmem -memprofile=./out/bench_mem.out -cpuprofile=./out/bench_cpu.out -tags=benchmarks ./benchmarks/ -v
+# NOTE: You can analyze the results, using go tool pprof. For instance, you can start a server to see a graph of the cpu usage by running
+# go tool pprof -http=":8082" out/bench_cpu.out. Alternatively, you can run go tool pprof, and in the ensuing cli, run
+# commands like top10 or web to dig down into the cpu and memory usage
+# For more, see https://blog.golang.org/pprof
+
 # NOTE: Windows doesn't support `-p`
 out:
-	@mkdir out
-	mkdir out$/tests
+	@mkdir out || (exit 0)
+	mkdir out$/tests || (exit 0)
+
 
-.PHONY: clean build format imports lint test unit acceptance prepare-for-pr verify verify-format
+.PHONY: clean build format imports lint test unit acceptance prepare-for-pr verify verify-format benchmark

benchmarks/build_test.go

@@ -0,0 +1,85 @@
+// +build benchmarks
+
+package benchmarks
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	dockerCli "github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/buildpacks/pack"
+	"github.com/buildpacks/pack/internal/commands"
+	cfg "github.com/buildpacks/pack/internal/config"
+	ilogging "github.com/buildpacks/pack/internal/logging"
+	h "github.com/buildpacks/pack/testhelpers"
+)
+
+var (
+	baseImg     = "some-org/" + h.RandString(10)
+	trustedImg  = baseImg + "-trusted-"
+	builder     = "cnbs/sample-builder:bionic"
+	mockAppPath = filepath.Join("..", "acceptance", "testdata", "mock_app")
+)
+
+func BenchmarkBuild(b *testing.B) {
+	dockerClient, err := dockerCli.NewClientWithOpts(dockerCli.FromEnv, dockerCli.WithVersion("1.38"))
+	if err != nil {
+		b.Error(errors.Wrap(err, "creating docker client"))
+	}
+
+	if err = h.PullImageWithAuth(dockerClient, builder, ""); err != nil {
+		b.Error(errors.Wrapf(err, "pulling builder %s", builder))
+	}
+
+	cmd := createCmd(b, dockerClient)
+
+	b.Run("with Untrusted Builder", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			// perform the operation we're analyzing
+			cmd.SetArgs([]string{fmt.Sprintf("%s%d", baseImg, i), "-p", mockAppPath, "-B", builder})
+			if err = cmd.Execute(); err != nil {
+				b.Error(errors.Wrapf(err, "running build #%d", i))
+			}
+		}
+	})
+
+	b.Run("with Trusted Builder", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			// perform the operation we're analyzing
+			cmd.SetArgs([]string{fmt.Sprintf("%s%d", trustedImg, i), "-p", mockAppPath, "-B", builder, "--trust-builder"})
+			if err = cmd.Execute(); err != nil {
+				b.Error(errors.Wrapf(err, "running build #%d", i))
+			}
+		}
+	})
+
+	// Cleanup
+	for i := 0; i < b.N; i++ {
+		if err = h.DockerRmi(dockerClient, fmt.Sprintf("%s%d", baseImg, i)); err != nil {
+			b.Error(errors.Wrapf(err, "deleting image #%d", i))
+		}
+
+		if err = h.DockerRmi(dockerClient, fmt.Sprintf("%s%d", trustedImg, i)); err != nil {
+			b.Error(errors.Wrapf(err, "deleting image #%d", i))
+		}
+	}
+
+	if err = h.DockerRmi(dockerClient, builder); err != nil {
+		b.Error(errors.Wrapf(err, "deleting builder %s", builder))
+	}
+}
+
+func createCmd(b *testing.B, docker *dockerCli.Client) *cobra.Command {
+	outBuf := bytes.Buffer{}
+	logger := ilogging.NewLogWithWriters(&outBuf, &outBuf)
+	packClient, err := pack.NewClient(pack.WithLogger(logger), pack.WithDockerClient(docker), pack.WithExperimental(true))
+	if err != nil {
+		b.Error(errors.Wrap(err, "creating packClient"))
+	}
+	return commands.Build(logger, cfg.Config{}, packClient)
+}