From 6349eac2ddd92784560d388c11075a61b44b669c Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Thu, 10 Oct 2024 18:09:03 +0000 Subject: [PATCH] chore: add comparison benchmarks Signed-off-by: Arjun Raja Yogidas --- Makefile | 2 +- benchmark/all/all_test.go | 4 +- benchmark/benchmark.go | 53 ++++++++++++ benchmark/container/container_test.go | 33 +++++++- benchmark/suite.go | 111 ++++++++++++++++++++++---- 5 files changed, 181 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index c5adf413f..5eeca3527 100644 --- a/Makefile +++ b/Makefile @@ -284,7 +284,7 @@ test-benchmark-vm: .PHONY: test-benchmark-container test-benchmark-container: - cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)" + cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem -benchtime=1x -timeout 2h --installed="$(INSTALLED)" .PHONY: gen-code # Since different projects may have different versions of tool binaries, diff --git a/benchmark/all/all_test.go b/benchmark/all/all_test.go index 1ba257d8d..ea548b9c1 100644 --- a/benchmark/all/all_test.go +++ b/benchmark/all/all_test.go @@ -31,11 +31,11 @@ func BenchmarkAll(b *testing.B) { } b.Run("BenchmarkContainerRun", func(b *testing.B) { - suite.BenchmarkContainerRun(b) + suite.BenchmarkContainerRun(b, "finch") }) b.Run("BenchmarkImageBuild", func(b *testing.B) { - suite.BenchmarkImageBuild(b) + suite.BenchmarkImageBuild(b, "finch") }) err = suite.StopVM() diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go index 941333b04..4f79b42cb 100644 --- a/benchmark/benchmark.go +++ b/benchmark/benchmark.go @@ -57,6 +57,59 @@ func GetSubject() (string, error) { return subject, nil } +// GetDocker gets the path of docker installation +func GetDocker() (string, error) { + subject := filepath.Join("/usr/bin/", "docker") + return subject, nil +} + +// func CleanUpFunc() error { +// cmd0 := exec.Command("sudo", "systemctl", "stop", "docker") +// if err := cmd0.Run(); err != nil { +// return fmt.Errorf("docker stop failed: %v", err) +// } +// cmd1 := exec.Command("sudo", "rm", "-rf", "/var/lib/docker") +// if err := cmd1.Run(); err != nil { +// return fmt.Errorf("docker cleanup failed: %v", err) +// } +// cmd2 := exec.Command("sudo", "rm", "-rf", "/var/run/docker.pid") +// if err := cmd2.Run(); err != nil { +// return fmt.Errorf("failed to stop docker process: %v", err) +// } +// cmd3 := exec.Command("sudo", "systemctl", "restart", "docker") +// if err := cmd3.Run(); err != nil { +// return fmt.Errorf("docker failed to restart: %v", err) +// } + +// cmd4 := exec.Command("sudo", "rm", "-rf", "/var/lib/finch/buildkit/cache.db") +// if err := cmd4.Run(); err != nil { +// return fmt.Errorf("finch buildkit cleanup failed: %v", err) +// } + +// cmd5 := exec.Command("sudo", "docker", "builder", "prune", "-a", "-f") +// if err := cmd5.Run(); err != nil { +// return fmt.Errorf("docker builder prune failed: %v", err) +// } + +// cmd6 := exec.Command("sudo", "docker", "image", "prune", "-a", "-f") +// if err := cmd6.Run(); err != nil { +// return fmt.Errorf("docker image prune failed: %v", err) +// } + +// cmd7 := exec.Command("sudo", "rm", "-rf", "/var/lib/containerd") +// if err := cmd7.Run(); err != nil { +// return fmt.Errorf("containerd cleanup failed: %v", err) +// } + +// cmd8 := exec.Command("sudo", "systemctl", "restart", "containerd") +// if err := cmd8.Run(); err != nil { +// return fmt.Errorf("docker failed to restart: %v", err) +// } + +// return nil + +// } + // Wrapper reports the benchmarking metrics of targetFunc to testing.B. func Wrapper(b *testing.B, targetFunc func(), cleanupFunc func()) { metricsSum := Metrics{} diff --git a/benchmark/container/container_test.go b/benchmark/container/container_test.go index cc84d7432..3970a1be5 100644 --- a/benchmark/container/container_test.go +++ b/benchmark/container/container_test.go @@ -17,11 +17,36 @@ func BenchmarkContainer(b *testing.B) { b.Fatal(err) } - b.Run("BenchmarkContainerRun", func(b *testing.B) { - suite.BenchmarkContainerRun(b) + b.Run("BenchmarkContainerRun-docker", func(b *testing.B) { + suite.BenchmarkContainerRun(b, "docker") }) - b.Run("BenchmarkImageBuild", func(b *testing.B) { - suite.BenchmarkImageBuild(b) + b.Run("BenchmarkContainerRun-finch", func(b *testing.B) { + suite.BenchmarkContainerRun(b, "finch") }) + + b.Run("BenchmarkContainerPull-docker", func(b *testing.B) { + suite.BenchmarkContainerPull(b, "docker") + }) + + b.Run("BenchmarkContainerPull-finch", func(b *testing.B) { + suite.BenchmarkContainerPull(b, "finch") + }) + + b.Run("BenchmarkImageBuild-docker", func(b *testing.B) { + suite.BenchmarkImageBuild(b, "docker") + }) + + b.Run("BenchmarkImageBuild-finch", func(b *testing.B) { + suite.BenchmarkImageBuild(b, "finch") + }) + + b.Run("BenchmarkImageDelete-docker", func(b *testing.B) { + suite.BenchmarkImageDelete(b, "docker") + }) + + b.Run("BenchmarkImageDelete-finch", func(b *testing.B) { + suite.BenchmarkImageDelete(b, "finch") + }) + } diff --git a/benchmark/suite.go b/benchmark/suite.go index ec9b0c7da..6138e92cb 100644 --- a/benchmark/suite.go +++ b/benchmark/suite.go @@ -16,14 +16,17 @@ import ( const ( virtualMachineRootCmd = "vm" - alpineImage = "public.ecr.aws/docker/library/alpine:latest" + alpineImage = "public.ecr.aws/y0o4y9o3/anaconda-pkg-build:latest" testImageName = "test:tag" testContainerName = "ctr-test" + ligthImage = "public.ecr.aws/docker/library/amazonlinux:latest" ) +// public.ecr.aws/soci-workshop-examples/mongo:latest public.ecr.aws/soci-workshop-examples/redis:latest public.ecr.aws/docker/library/alpine:latest // Suite is a struct that groups benchmark functions and shared state. type Suite struct { subject string + docker string } // Setup initializes the Suite by getting the subject. @@ -32,7 +35,13 @@ func (suite *Suite) Setup() error { if err != nil { return err } + + docker, err := GetDocker() + if err != nil { + return err + } suite.subject = subject + suite.docker = docker return nil } @@ -80,18 +89,49 @@ func (suite *Suite) BenchmarkVMStart(b *testing.B) { } // BenchmarkContainerRun measures the metrics to run a container. -func (suite *Suite) BenchmarkContainerRun(b *testing.B) { - assert.NoError(b, exec.Command(suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only - Wrapper(b, func() { - assert.NoError(b, exec.Command(suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only - }, func() { - assert.NoError(b, exec.Command(suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only - }) - assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only +func (suite *Suite) BenchmarkContainerRun(b *testing.B, binaryName string) { + // assert.NoError(b, CleanUpFunc()) + if binaryName == "finch" { + assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only + }) + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only + } else { + // assert.NoError(b, CleanUpFunc()) + assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only + }) + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only + } + +} + +func (suite *Suite) BenchmarkContainerPull(b *testing.B, binaryName string) { + if binaryName == "finch" { + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage, "--namespace=finch").Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only + }) + } else { + assert.NoError(b, exec.Command("sudo", suite.docker, "images", "prune", "-a").Run()) + assert.NoError(b, exec.Command("sudo", suite.docker, "volume", "prune", "-a").Run()) + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only + }) + } } // BenchmarkImageBuild measures the metrics to build an image. -func (suite *Suite) BenchmarkImageBuild(b *testing.B) { +func (suite *Suite) BenchmarkImageBuild(b *testing.B, binaryName string) { homeDir, err := os.UserHomeDir() assert.NoError(b, err) tempDir, err := os.MkdirTemp(homeDir, "finch-test") @@ -103,9 +143,50 @@ func (suite *Suite) BenchmarkImageBuild(b *testing.B) { assert.NoError(b, err) buildContext := filepath.Dir(dockerFilePath) defer os.RemoveAll(buildContext) //nolint:errcheck // testing only - Wrapper(b, func() { - assert.NoError(b, exec.Command(suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only - }, func() { - assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only - }) + // assert.NoError(b, CleanUpFunc()) + if binaryName == "finch" { + assert.NoError(b, exec.Command("sudo", suite.subject, "builder", "prune").Run()) + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext, "--namespace=finch").Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }) + } else { + // assert.NoError(b, CleanUpFunc()) + assert.NoError(b, exec.Command("sudo", suite.docker, "builder", "prune").Run()) + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext, "--no-cache").Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }) + } +} + +func (suite *Suite) BenchmarkImageDelete(b *testing.B, binaryImage string) { + homeDir, err := os.UserHomeDir() + assert.NoError(b, err) + tempDir, err := os.MkdirTemp(homeDir, "finch-test") + assert.NoError(b, err) + dockerFilePath := filepath.Join(tempDir, "Dockerfile") + err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s + CMD ["echo", "finch-test-dummy-output"] + `, alpineImage)), 0o600) + assert.NoError(b, err) + buildContext := filepath.Dir(dockerFilePath) + defer os.RemoveAll(buildContext) //nolint:errcheck // testing only + if binaryImage == "finch" { + assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--help").Run()) + }) + } else { + assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--help").Run()) + }) + } }