Add debug flag to include delve

.github/workflows/e2e.yaml

@@ -83,3 +83,6 @@ jobs:
           - "-X main.version=${{ github.sha }}"
         EOF
         docker run $(go run ./ build ./test/ --platform=${PLATFORM}) --wait=false 2>&1 | grep "${{ github.sha }}"
+
+        # Check that --debug adds dlv to the image, and that dlv is runnable.
+        docker run --entrypoint="dlv" $(go run ./ build ./test/ --platform=${PLATFORM} --debug) version | grep "Delve Debugger"

docs/features/debugging.md

@@ -0,0 +1,29 @@
+# Debugging
+
+Sometimes it's challenging to track down the cause of unexpected behavior in an app. Because `ko` makes it simple to make tweaks to your app and immediately rebuild your image, it's possible to iteratively explore various aspects of your app, such as by adding log lines that print variable values.
+
+But to help you solve the problem _as fast as possible_, `ko` supports debugging your Go app with [delve](https://github.com/go-delve/delve).
+
+To use this feature, just add the `--debug` flag to your `ko build` command. This adjusts how the image is built:
+
+- It installs `delve` in the image (in addition to your own app).
+- It sets the image's `ENTRYPOINT` to a `delve exec ...` command that runs the Go app in debug-mode, listening on port `40000` for a debugger client.
+- It ensures your compiled Go app includes debug symbols needed to enable debugging.
+
+**Note:** This feature is geared toward development workflows. It **should not** be used in production.
+
+### How it works
+
+Build the image using the debug feature.
+
+```plaintext
+ko build . --debug
+```
+
+Run the container, ensuring that the debug port (`40000`) is exposed to allow clients to connect to it.
+
+```plaintext
+docker run -p 40000:40000 <img>
+```
+
+This sets up your app to be waiting to run the command you've specified. All that's needed now is to connect your debugger client to the running container!

docs/reference/ko_apply.md

@@ -47,6 +47,7 @@ ko apply -f FILENAME [flags]
 ```
       --bare                     Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags).
   -B, --base-import-paths        Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags).
+      --debug                    Include Delve debugger into image and wrap around ko-app. This debugger will listen to port 40000.
       --disable-optimizations    Disable optimizations when building Go code. Useful when you want to interactively debug the created container.
   -f, --filename strings         Filename, directory, or URL to files to use to create the resource
   -h, --help                     help for apply

docs/reference/ko_build.md

@@ -44,6 +44,7 @@ ko build IMPORTPATH... [flags]
 ```
       --bare                     Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags).
   -B, --base-import-paths        Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags).
+      --debug                    Include Delve debugger into image and wrap around ko-app. This debugger will listen to port 40000.
       --disable-optimizations    Disable optimizations when building Go code. Useful when you want to interactively debug the created container.
   -h, --help                     help for build
       --image-label strings      Which labels (key=value) to add to the image.

docs/reference/ko_create.md

@@ -47,6 +47,7 @@ ko create -f FILENAME [flags]
 ```
       --bare                     Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags).
   -B, --base-import-paths        Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags).
+      --debug                    Include Delve debugger into image and wrap around ko-app. This debugger will listen to port 40000.
       --disable-optimizations    Disable optimizations when building Go code. Useful when you want to interactively debug the created container.
   -f, --filename strings         Filename, directory, or URL to files to use to create the resource
   -h, --help                     help for create

docs/reference/ko_resolve.md

@@ -40,6 +40,7 @@ ko resolve -f FILENAME [flags]
 ```
       --bare                     Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags).
   -B, --base-import-paths        Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags).
+      --debug                    Include Delve debugger into image and wrap around ko-app. This debugger will listen to port 40000.
       --disable-optimizations    Disable optimizations when building Go code. Useful when you want to interactively debug the created container.
   -f, --filename strings         Filename, directory, or URL to files to use to create the resource
   -h, --help                     help for resolve

docs/reference/ko_run.md

@@ -32,6 +32,7 @@ ko run IMPORTPATH [flags]
 ```
       --bare                     Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags).
   -B, --base-import-paths        Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags).
+      --debug                    Include Delve debugger into image and wrap around ko-app. This debugger will listen to port 40000.
       --disable-optimizations    Disable optimizations when building Go code. Useful when you want to interactively debug the created container.
   -h, --help                     help for run
       --image-label strings      Which labels (key=value) to add to the image.

mkdocs.yml

@@ -24,6 +24,7 @@ nav:
     - features/k8s.md
     - features/static-assets.md
     - features/build-cache.md
+    - features/debugging.md
   - Advanced:
     - advanced/go-packages.md
     - advanced/limitations.md
@@ -65,4 +66,4 @@ markdown_extensions:
       pygments_lang_class: true
   - pymdownx.inlinehilite
   - pymdownx.snippets
-  - pymdownx.superfences
+  - pymdownx.superfences

pkg/build/gobuild.go

@@ -101,6 +101,7 @@ type gobuild struct {
 	platformMatcher      *platformMatcher
 	dir                  string
 	labels               map[string]string
+	debug                bool
 	semaphore            *semaphore.Weighted
 
 	cache *layerCache
@@ -127,6 +128,7 @@ type gobuildOpener struct {
 	labels               map[string]string
 	dir                  string
 	jobs                 int
+	debug                bool
 }
 
 func (gbo *gobuildOpener) Open() (Interface, error) {
@@ -156,6 +158,7 @@ func (gbo *gobuildOpener) Open() (Interface, error) {
 		defaultLdflags:       gbo.defaultLdflags,
 		labels:               gbo.labels,
 		dir:                  gbo.dir,
+		debug:                gbo.debug,
 		platformMatcher:      matcher,
 		cache: &layerCache{
 			buildToDiff: map[string]buildIDToDiffID{},
@@ -274,6 +277,77 @@ func getGoBinary() string {
 	return defaultGoBin
 }
 
+func doesPlatformSupportDebugging(platform v1.Platform) bool {
+	// Here's the list of supported platforms by Delve:
+	//
+	// https://github.com/go-delve/delve/blob/master/Documentation/faq.md#unsupportedplatforms
+	//
+	// For the time being, we'll support only linux/amd64 and linux/arm64.
+
+	return platform.OS == "linux" && (platform.Architecture == "amd64" || platform.Architecture == "arm64")
+}
+
+func getDelve(ctx context.Context, platform v1.Platform) (string, error) {
+	const delveCloneURL = "https://github.com/go-delve/delve.git"
+
+	if platform.OS == "" || platform.Architecture == "" {
+		return "", fmt.Errorf("platform os (%q) or arch (%q) is empty",
+			platform.OS,
+			platform.Architecture,
+		)
+	}
+
+	env, err := buildEnv(platform, os.Environ(), nil)
+	if err != nil {
+		return "", fmt.Errorf("could not create env for Delve build: %w", err)
+	}
+
+	tmpInstallDir, err := os.MkdirTemp("", "delve")
+	if err != nil {
+		return "", fmt.Errorf("could not create tmp dir for Delve installation: %w", err)
+	}
+	cloneDir := filepath.Join(tmpInstallDir, "delve")
+	err = os.MkdirAll(cloneDir, 0755)
+	if err != nil {
+		return "", fmt.Errorf("making dir for delve clone: %w", err)
+	}
+	err = git.Clone(ctx, cloneDir, delveCloneURL)
+	if err != nil {
+		return "", fmt.Errorf("cloning delve repo: %w", err)
+	}
+	osArchDir := fmt.Sprintf("%s_%s", platform.OS, platform.Architecture)
+	delveBinaryPath := filepath.Join(tmpInstallDir, "bin", osArchDir, "dlv")
+
+	// install delve to tmp directory
+	args := []string{
+		"build",
+		"-o",
+		delveBinaryPath,
+		"./cmd/dlv",
+	}
+
+	gobin := getGoBinary()
+	cmd := exec.CommandContext(ctx, gobin, args...)
+	cmd.Env = env
+	cmd.Dir = cloneDir
+
+	var output bytes.Buffer
+	cmd.Stderr = &output
+	cmd.Stdout = &output
+
+	log.Printf("Building Delve for %s", platform)
+	if err := cmd.Run(); err != nil {
+		os.RemoveAll(tmpInstallDir)
+		return "", fmt.Errorf("go build Delve: %w: %s", err, output.String())
+	}
+
+	if _, err := os.Stat(delveBinaryPath); err != nil {
+		return "", fmt.Errorf("could not find Delve binary at %q: %w", delveBinaryPath, err)
+	}
+
+	return delveBinaryPath, nil
+}
+
 func build(ctx context.Context, buildCtx buildContext) (string, error) {
 	// Create the set of build arguments from the config flags/ldflags with any
 	// template parameters applied.
@@ -887,6 +961,10 @@ func (g *gobuild) configForImportPath(ip string) Config {
 	return config
 }
 
+func (g gobuild) useDebugging(platform v1.Platform) bool {
+	return g.debug && doesPlatformSupportDebugging(platform)
+}
+
 func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, platform *v1.Platform) (oci.SignedImage, error) {
 	if err := g.semaphore.Acquire(ctx, 1); err != nil {
 		return nil, err
@@ -914,12 +992,22 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
 		return nil, err
 	}
 	if platform == nil {
+		if cf.OS == "" {
+			cf.OS = "linux"
+		}
+		if cf.Architecture == "" {
+			cf.Architecture = "amd64"
+		}
+
 		platform = &v1.Platform{
 			OS:           cf.OS,
 			Architecture: cf.Architecture,
 			OSVersion:    cf.OSVersion,
 		}
 	}
+	if g.debug && !doesPlatformSupportDebugging(*platform) {
+		log.Printf("image for platform %q will be built without debugging enabled because debugging is not supported for that platform", *platform)
+	}
 
 	if !g.platformMatcher.matches(platform) {
 		return nil, fmt.Errorf("base image platform %q does not match desired platforms %v", platform, g.platformMatcher.platforms)
@@ -1030,6 +1118,45 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
 		},
 	})
 
+	delvePath := "" // path for delve in image
+	if g.useDebugging(*platform) {
+		// get delve locally
+		delveBinary, err := getDelve(ctx, *platform)
+		if err != nil {
+			return nil, fmt.Errorf("building Delve: %w", err)
+		}
+		defer os.RemoveAll(filepath.Dir(delveBinary))
+
+		delvePath = path.Join("/ko-app", filepath.Base(delveBinary))
+
+		// add layer with delve binary
+		delveLayer, err := g.cache.get(ctx, delveBinary, func() (v1.Layer, error) {
+			return buildLayer(delvePath, delveBinary, platform, layerMediaType, &lo)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("cache.get(%q): %w", delveBinary, err)
+		}
+
+		layers = append(layers, mutate.Addendum{
+			Layer:     delveLayer,
+			MediaType: layerMediaType,
+			History: v1.History{
+				Author:    "ko",
+				Created:   g.creationTime,
+				CreatedBy: "ko build " + ref.String(),
+				Comment:   "Delve debugger, at " + delvePath,
+			},
+		})
+	}
+	delveArgs := []string{
+		"exec",
+		"--listen=:40000",
+		"--headless",
+		"--log",
+		"--accept-multiclient",
+		"--api-version=2",
+	}
+
 	// Augment the base image with our application layer.
 	withApp, err := mutate.Append(base, layers...)
 	if err != nil {
@@ -1047,10 +1174,22 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
 	cfg.Config.Entrypoint = []string{appPath}
 	cfg.Config.Cmd = nil
 	if platform.OS == "windows" {
-		cfg.Config.Entrypoint = []string{`C:\ko-app\` + appFileName}
+		appPath := `C:\ko-app\` + appFileName
+		if g.debug {
+			cfg.Config.Entrypoint = append([]string{"C:\\" + delvePath}, delveArgs...)
+			cfg.Config.Entrypoint = append(cfg.Config.Entrypoint, appPath)
+		} else {
+			cfg.Config.Entrypoint = []string{appPath}
+		}
+
 		updatePath(cfg, `C:\ko-app`)
 		cfg.Config.Env = append(cfg.Config.Env, `KO_DATA_PATH=C:\var\run\ko`)
 	} else {
+		if g.useDebugging(*platform) {
+			cfg.Config.Entrypoint = append([]string{delvePath}, delveArgs...)
+			cfg.Config.Entrypoint = append(cfg.Config.Entrypoint, appPath)
+		}
+
 		updatePath(cfg, appDir)
 		cfg.Config.Env = append(cfg.Config.Env, "KO_DATA_PATH="+kodataRoot)
 	}

pkg/build/gobuild_test.go

@@ -1453,3 +1453,59 @@ func TestGoBuildConsistentMediaTypes(t *testing.T) {
 		})
 	}
 }
+
+func TestDebugger(t *testing.T) {
+	base, err := random.Image(1024, 3)
+	if err != nil {
+		t.Fatalf("random.Image() = %v", err)
+	}
+	importpath := "github.com/google/ko"
+
+	ng, err := NewGo(
+		context.Background(),
+		"",
+		WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, base, nil }),
+		WithPlatforms("linux/amd64"),
+		WithDebugger(),
+	)
+	if err != nil {
+		t.Fatalf("NewGo() = %v", err)
+	}
+
+	result, err := ng.Build(context.Background(), StrictScheme+importpath)
+	if err != nil {
+		t.Fatalf("Build() = %v", err)
+	}
+
+	img, ok := result.(v1.Image)
+	if !ok {
+		t.Fatalf("Build() not an Image: %T", result)
+	}
+
+	// Check that the entrypoint of the image is not overwritten
+	cfg, err := img.ConfigFile()
+	if err != nil {
+		t.Errorf("ConfigFile() = %v", err)
+	}
+	gotEntrypoint := cfg.Config.Entrypoint
+	wantEntrypoint := []string{
+		"/ko-app/dlv",
+		"exec",
+		"--listen=:40000",
+		"--headless",
+		"--log",
+		"--accept-multiclient",
+		"--api-version=2",
+		"/ko-app/ko",
+	}
+
+	if got, want := len(gotEntrypoint), len(wantEntrypoint); got != want {
+		t.Fatalf("len(entrypoint) = %v, want %v", got, want)
+	}
+
+	for i := range wantEntrypoint {
+		if got, want := gotEntrypoint[i], wantEntrypoint[i]; got != want {
+			t.Errorf("entrypoint[%d] = %v, want %v", i, got, want)
+		}
+	}
+}

pkg/build/options.go

@@ -202,3 +202,10 @@ func WithSBOMDir(dir string) Option {
 		return nil
 	}
 }
+
+func WithDebugger() Option {
+	return func(gbo *gobuildOpener) error {
+		gbo.debug = true
+		return nil
+	}
+}

pkg/commands/options/build.go

@@ -65,6 +65,7 @@ type BuildOptions struct {
 	SBOMDir              string
 	Platforms            []string
 	Labels               []string
+	Debug                bool
 	// UserAgent enables overriding the default value of the `User-Agent` HTTP
 	// request header used when retrieving the base image.
 	UserAgent string
@@ -94,6 +95,8 @@ func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) {
 		"Which platform to use when pulling a multi-platform base. Format: all | <os>[/<arch>[/<variant>]][,platform]*")
 	cmd.Flags().StringSliceVar(&bo.Labels, "image-label", []string{},
 		"Which labels (key=value) to add to the image.")
+	cmd.Flags().BoolVar(&bo.Debug, "debug", bo.Debug,
+		"Include Delve debugger into image and wrap around ko-app. This debugger will listen to port 40000.")
 	bo.Trimpath = true
 }
 

pkg/commands/resolver.go

@@ -101,6 +101,10 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) {
 	if bo.DisableOptimizations {
 		opts = append(opts, build.WithDisabledOptimizations())
 	}
+	if bo.Debug {
+		opts = append(opts, build.WithDebugger())
+		opts = append(opts, build.WithDisabledOptimizations()) // also needed for Delve
+	}
 	switch bo.SBOM {
 	case "none":
 		opts = append(opts, build.WithDisabledSBOM())