Merge branch 'main' into issue-11900

.github/workflows/stale.yml

@@ -0,0 +1,23 @@
+name: 'Close stale issues'
+on:
+  schedule:
+    - cron: '0 0 * * 0,3' # at midnight UTC every Sunday and Wednesday
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          stale-issue-message: >
+            This issue has been automatically marked as stale because it has not had
+            recent activity. It will be closed if no further activity occurs. Thank you
+            for your contributions.
+          days-before-issue-stale: 150 # marks stale after 5 months
+          days-before-issue-close: 30 # closes 1 month after being marked with no action
+          stale-issue-label: "stale"
+          exempt-issue-labels: "kind/feature,kind/enhancement"
+

cmd/compose/compose.go

@@ -29,8 +29,10 @@ import (
 	"syscall"
 
 	"github.com/compose-spec/compose-go/v2/cli"
+	"github.com/compose-spec/compose-go/v2/dotenv"
 	"github.com/compose-spec/compose-go/v2/loader"
 	"github.com/compose-spec/compose-go/v2/types"
+	composegoutils "github.com/compose-spec/compose-go/v2/utils"
 	"github.com/docker/buildx/util/logutil"
 	dockercli "github.com/docker/cli/cli"
 	"github.com/docker/cli/cli-plugins/manager"
@@ -342,11 +344,14 @@ func (o *ProjectOptions) ToProject(ctx context.Context, dockerCli command.Cli, s
 		project.Services[name] = s
 	}
 
+	project, err = project.WithSelectedServices(services)
+	if err != nil {
+		return nil, tracing.Metrics{}, err
+	}
+
 	if !o.All {
 		project = project.WithoutUnnecessaryResources()
 	}
-
-	project, err = project.WithSelectedServices(services)
 	return project, metrics, err
 }
 
@@ -447,6 +452,11 @@ func RootCommand(dockerCli command.Cli, backend Backend) *cobra.Command { //noli
 			if verbose {
 				logrus.SetLevel(logrus.TraceLevel)
 			}
+
+			err := setEnvWithDotEnv(opts)
+			if err != nil {
+				return err
+			}
 			if noAnsi {
 				if ansi != "auto" {
 					return errors.New(`cannot specify DEPRECATED "--no-ansi" and "--ansi". Please use only "--ansi"`)
@@ -538,7 +548,7 @@ func RootCommand(dockerCli command.Cli, backend Backend) *cobra.Command { //noli
 			}
 
 			// dry run detection
-			ctx, err := backend.DryRunMode(ctx, dryRun)
+			ctx, err = backend.DryRunMode(ctx, dryRun)
 			if err != nil {
 				return err
 			}
@@ -638,6 +648,30 @@ func RootCommand(dockerCli command.Cli, backend Backend) *cobra.Command { //noli
 	return c
 }
 
+func setEnvWithDotEnv(opts ProjectOptions) error {
+	options, err := cli.NewProjectOptions(opts.ConfigPaths,
+		cli.WithWorkingDirectory(opts.ProjectDir),
+		cli.WithOsEnv,
+		cli.WithEnvFiles(opts.EnvFiles...),
+		cli.WithDotEnv,
+	)
+	if err != nil {
+		return nil
+	}
+	envFromFile, err := dotenv.GetEnvFromFile(composegoutils.GetAsEqualsMap(os.Environ()), options.EnvFiles)
+	if err != nil {
+		return nil
+	}
+	for k, v := range envFromFile {
+		if _, ok := os.LookupEnv(k); !ok {
+			if err = os.Setenv(k, v); err != nil {
+				return nil
+			}
+		}
+	}
+	return err
+}
+
 var printerModes = []string{
 	ui.ModeAuto,
 	ui.ModeTTY,

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/buger/goterm v1.0.4
-	github.com/compose-spec/compose-go/v2 v2.1.3
+	github.com/compose-spec/compose-go/v2 v2.1.4-0.20240708103555-327323ea44d4
 	github.com/containerd/containerd v1.7.18
 	github.com/davecgh/go-spew v1.1.1
 	github.com/distribution/reference v0.6.0

go.sum

@@ -86,8 +86,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
-github.com/compose-spec/compose-go/v2 v2.1.3 h1:bD67uqLuL/XgkAK6ir3xZvNLFPxPScEi1KW7R5esrLE=
-github.com/compose-spec/compose-go/v2 v2.1.3/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc=
+github.com/compose-spec/compose-go/v2 v2.1.4-0.20240708103555-327323ea44d4 h1:jpoA30Hs8qGuT2Rv/mk9pga4y/Glw/5O+jyOCv2la3I=
+github.com/compose-spec/compose-go/v2 v2.1.4-0.20240708103555-327323ea44d4/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc=
 github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
 github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
 github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro=

pkg/compose/containers.go

@@ -118,17 +118,16 @@ func isRunning() containerPredicate {
 	}
 }
 
-func isNotService(services ...string) containerPredicate {
-	return func(c moby.Container) bool {
-		service := c.Labels[api.ServiceLabel]
-		return !utils.StringContains(services, service)
-	}
-}
-
 // isOrphaned is a predicate to select containers without a matching service definition in compose project
 func isOrphaned(project *types.Project) containerPredicate {
 	services := append(project.ServiceNames(), project.DisabledServiceNames()...)
 	return func(c moby.Container) bool {
+		// One-off container
+		v, ok := c.Labels[api.OneoffLabel]
+		if ok && v == "True" {
+			return c.State == ContainerExited || c.State == ContainerDead
+		}
+		// Service that is not defined in the compose model
 		service := c.Labels[api.ServiceLabel]
 		return !utils.StringContains(services, service)
 	}

pkg/compose/create.go

@@ -101,8 +101,7 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt
 		return err
 	}
 
-	allServiceNames := append(project.ServiceNames(), project.DisabledServiceNames()...)
-	orphans := observedState.filter(isNotService(allServiceNames...))
+	orphans := observedState.filter(isOrphaned(project))
 	if len(orphans) > 0 && !options.IgnoreOrphans {
 		if options.RemoveOrphans {
 			err := s.removeContainers(ctx, orphans, nil, false)
@@ -1313,11 +1312,10 @@ func (s *composeService) resolveExternalNetwork(ctx context.Context, n *types.Ne
 	if len(networks) == 0 {
 		// in this instance, n.Name is really an ID
 		sn, err := s.apiClient().NetworkInspect(ctx, n.Name, network.InspectOptions{})
-		if err != nil {
+		if err != nil && !errdefs.IsNotFound(err) {
 			return err
 		}
 		networks = append(networks, sn)
-
 	}
 
 	// NetworkList API doesn't return the exact name match, so we can retrieve more than one network with a request

pkg/compose/down.go

@@ -327,6 +327,10 @@ func (s *composeService) stopAndRemoveContainer(ctx context.Context, container m
 	w := progress.ContextWriter(ctx)
 	eventName := getContainerProgressName(container)
 	err := s.stopContainer(ctx, w, container, timeout)
+	if errdefs.IsNotFound(err) {
+		w.Event(progress.RemovedEvent(eventName))
+		return nil
+	}
 	if err != nil {
 		return err
 	}

pkg/compose/kill_test.go

@@ -112,6 +112,7 @@ func testContainer(service string, id string, oneOff bool) moby.Container {
 		ID:     id,
 		Names:  []string{name},
 		Labels: containerLabels(service, oneOff),
+		State:  ContainerExited,
 	}
 }
 

pkg/e2e/compose_environment_test.go

@@ -222,3 +222,24 @@ func TestCommentsInEnvFile(t *testing.T) {
 		c.RunDockerComposeCmd(t, "--project-name", "env-file-comments", "down", "--rmi", "all")
 	})
 }
+
+func TestUnsetEnv(t *testing.T) {
+	c := NewParallelCLI(t)
+	t.Cleanup(func() {
+		c.RunDockerComposeCmd(t, "--project-name", "empty-variable", "down", "--rmi", "all")
+	})
+
+	t.Run("override env variable", func(t *testing.T) {
+		c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/empty-variable/compose.yaml", "build")
+
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/empty-variable/compose.yaml",
+			"run", "-e", "EMPTY=hello", "--rm", "empty-variable")
+		res.Assert(t, icmd.Expected{Out: `=hello=`})
+	})
+
+	t.Run("unset env variable", func(t *testing.T) {
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/empty-variable/compose.yaml",
+			"run", "--rm", "empty-variable")
+		res.Assert(t, icmd.Expected{Out: `==`})
+	})
+}

pkg/e2e/compose_run_test.go

@@ -37,7 +37,7 @@ func TestLocalComposeRun(t *testing.T) {
 			"Hello one more time")
 		lines = Lines(res.Stdout())
 		assert.Equal(t, lines[len(lines)-1], "Hello one more time", res.Stdout())
-		assert.Assert(t, !strings.Contains(res.Combined(), "orphan"))
+		assert.Assert(t, strings.Contains(res.Combined(), "orphan"))
 	})
 
 	t.Run("check run container exited", func(t *testing.T) {

pkg/e2e/compose_test.go

@@ -388,3 +388,20 @@ func TestNestedDotEnv(t *testing.T) {
 	})
 
 }
+
+func TestUnnecesaryResources(t *testing.T) {
+	const projectName = "compose-e2e-unnecessary-resources"
+	c := NewParallelCLI(t)
+	t.Cleanup(func() {
+		c.RunDockerComposeCmd(t, "-p", projectName, "down", "-t=0")
+	})
+
+	res := c.RunDockerComposeCmdNoCheck(t, "-f", "./fixtures/external/compose.yaml", "-p", projectName, "up", "-d")
+	res.Assert(t, icmd.Expected{
+		ExitCode: 1,
+		Err:      "network foo_bar declared as external, but could not be found",
+	})
+
+	c.RunDockerComposeCmd(t, "-f", "./fixtures/external/compose.yaml", "-p", projectName, "up", "-d", "test")
+	// Should not fail as missing external network is not used
+}

pkg/e2e/fixtures/environment/empty-variable/Dockerfile

@@ -0,0 +1,17 @@
+#   Copyright 2020 Docker Compose CLI authors
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+FROM alpine
+ENV  EMPTY=not_empty
+CMD ["sh", "-c", "echo \"=$EMPTY=\""]

pkg/e2e/fixtures/environment/empty-variable/compose.yaml

@@ -0,0 +1,7 @@
+services:
+  empty-variable:
+    build:
+      context: .
+    image: empty-variable
+    environment:
+      - EMPTY # expect to propagate value from user's env OR unset in container

pkg/e2e/fixtures/external/compose.yaml

@@ -0,0 +1,14 @@
+services:
+  test:
+    image: nginx:alpine
+
+  other:
+    image: nginx:alpine
+    networks:
+      test_network:
+        ipv4_address: 8.8.8.8
+
+networks:
+  test_network:
+    external: true
+    name: foo_bar

pkg/e2e/fixtures/orphans/.env

@@ -0,0 +1 @@
+COMPOSE_REMOVE_ORPHANS=true

pkg/e2e/fixtures/orphans/compose.yaml

@@ -0,0 +1,7 @@
+services:
+  orphan:
+    profiles: [run]
+    image: alpine
+    command: echo hello
+  test:
+    image: nginx:alpine

pkg/e2e/orphans_test.go

@@ -0,0 +1,39 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package e2e
+
+import (
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+func TestRemoveOrphans(t *testing.T) {
+	c := NewCLI(t)
+
+	const projectName = "compose-e2e-orphans"
+
+	c.RunDockerComposeCmd(t, "-f", "./fixtures/orphans/compose.yaml", "-p", projectName, "run", "orphan")
+	res := c.RunDockerComposeCmd(t, "-p", projectName, "ps", "--all")
+	assert.Check(t, strings.Contains(res.Combined(), "compose-e2e-orphans-orphan-run-"))
+
+	c.RunDockerComposeCmd(t, "-f", "./fixtures/orphans/compose.yaml", "-p", projectName, "up", "-d")
+
+	res = c.RunDockerComposeCmd(t, "-p", projectName, "ps", "--all")
+	assert.Check(t, !strings.Contains(res.Combined(), "compose-e2e-orphans-orphan-run-"))
+}