From 9c03797f9dc653ba09d13a7427df9f0c8023fa76 Mon Sep 17 00:00:00 2001 From: Joana Hrotko Date: Tue, 6 Aug 2024 13:49:55 +0100 Subject: [PATCH] initial sync files that modified after image creation Signed-off-by: Joana Hrotko --- go.mod | 2 +- go.sum | 4 +- pkg/compose/watch.go | 101 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 18be6a4f71f..7f6d0bac842 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v1.0.4 - github.com/compose-spec/compose-go/v2 v2.1.5 + github.com/compose-spec/compose-go/v2 v2.1.6 github.com/containerd/containerd v1.7.20 github.com/containerd/platforms v0.2.1 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 159258f74bc..25468f0c4c3 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= -github.com/compose-spec/compose-go/v2 v2.1.5 h1:6YoC9ik3NXdSYtgRn51EMZ2DxzGPyGjZ8M0B7mXTXeQ= -github.com/compose-spec/compose-go/v2 v2.1.5/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= +github.com/compose-spec/compose-go/v2 v2.1.6 h1:d0Cs0DffmOwmSzs0YPHwKCskknGq2jfGg4uGowlEpps= +github.com/compose-spec/compose-go/v2 v2.1.6/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= diff --git a/pkg/compose/watch.go b/pkg/compose/watch.go index 5133f1a3e30..37ac3457e3f 100644 --- a/pkg/compose/watch.go +++ b/pkg/compose/watch.go @@ -163,7 +163,10 @@ func (s *composeService) watch(ctx context.Context, syncChannel chan bool, proje success, err := trigger.Extensions.Get("x-initialSync", &initialSync) if err == nil && success && initialSync && (trigger.Action == types.WatchActionSync || trigger.Action == types.WatchActionSyncRestart) { // Need to check initial files are in container that are meant to be synched from watch action - s.initialSync(ctx, service, trigger, ignore, syncer) + err := s.initialSync(ctx, project, service, trigger, ignore, syncer) + if err != nil { + return err + } } } paths = append(paths, trigger.Path) @@ -583,12 +586,20 @@ func (s *composeService) pruneDanglingImagesOnRebuild(ctx context.Context, proje } } -func (s *composeService) initialSync(ctx context.Context, service types.ServiceConfig, trigger types.Trigger, ignore watch.PathMatcher, syncer sync.Syncer) error { - dockerFileIgnore, _ := watch.NewDockerPatternMatcher("/", []string{"Dockerfile", "*compose*.y*ml"}) - triggerIgnore, _ := watch.NewDockerPatternMatcher("/", trigger.Ignore) +// Walks develop.watch.path and checks which files should be copied inside the container +// ignores develop.watch.ignore, Dockerfile, compose files, bind mounted paths and .git +func (s *composeService) initialSync(ctx context.Context, project *types.Project, service types.ServiceConfig, trigger types.Trigger, ignore watch.PathMatcher, syncer sync.Syncer) error { + dockerFileIgnore, err := watch.NewDockerPatternMatcher("/", []string{"Dockerfile", "*compose*.y*ml"}) + if err != nil { + return err + } + triggerIgnore, err := watch.NewDockerPatternMatcher("/", trigger.Ignore) + if err != nil { + return err + } ignoreInitialSync := watch.NewCompositeMatcher(ignore, dockerFileIgnore, triggerIgnore) - pathsToCopy, err := initialSyncFiles(service, trigger, ignoreInitialSync) + pathsToCopy, err := s.initialSyncFiles(ctx, project, service, trigger, ignoreInitialSync) if err != nil { return err } @@ -596,16 +607,22 @@ func (s *composeService) initialSync(ctx context.Context, service types.ServiceC return syncer.Sync(ctx, service, pathsToCopy) } -func initialSyncFiles(service types.ServiceConfig, trigger types.Trigger, ignore watch.PathMatcher) ([]sync.PathMapping, error) { +// Syncs files from develop.watch.path if thy have been modified after the image has been created +// +//nolint:gocyclo +func (s *composeService) initialSyncFiles(ctx context.Context, project *types.Project, service types.ServiceConfig, trigger types.Trigger, ignore watch.PathMatcher) ([]sync.PathMapping, error) { fi, err := os.Stat(trigger.Path) if err != nil { return nil, err } - + timeImageCreated, err := s.imageCreatedTime(ctx, project, service.Name) + if err != nil { + return nil, err + } var pathsToCopy []sync.PathMapping switch mode := fi.Mode(); { case mode.IsDir(): - // do directory stuff + // process directory err = filepath.WalkDir(trigger.Path, func(path string, d fs.DirEntry, err error) error { if err != nil { // handle possible path err, just in case... @@ -615,34 +632,76 @@ func initialSyncFiles(service types.ServiceConfig, trigger types.Trigger, ignore // walk starts at the root directory return nil } - rel, _ := filepath.Rel(trigger.Path, path) - if shouldIgnoreOrSkip(filepath.Base(path), ignore) || checkIfPathAlreadyBindMounted(path, service.Volumes) { + if shouldIgnore(filepath.Base(path), ignore) || checkIfPathAlreadyBindMounted(path, service.Volumes) { // By definition sync ignores bind mounted paths if d.IsDir() { - return fs.SkipDir // ignore or skip folder + // skip folder + return fs.SkipDir } - return nil // ignore or skip file + return nil // skip file + } + info, err := d.Info() + if err != nil { + return err + } + if !d.IsDir() { + if info.ModTime().Before(timeImageCreated) { + // skip file if it was modified before image creation + return nil + } + rel, err := filepath.Rel(trigger.Path, path) + if err != nil { + return err + } + // only copy files (and not full directories) + pathsToCopy = append(pathsToCopy, sync.PathMapping{ + HostPath: path, + ContainerPath: filepath.Join(trigger.Target, rel), + }) } - pathsToCopy = append(pathsToCopy, sync.PathMapping{ - HostPath: path, - ContainerPath: filepath.Join(trigger.Target, rel), - }) return nil }) case mode.IsRegular(): - // do file stuff - if !shouldIgnoreOrSkip(filepath.Base(trigger.Path), ignore) && !checkIfPathAlreadyBindMounted(trigger.Path, service.Volumes) { + // process file + if fi.ModTime().After(timeImageCreated) && !shouldIgnore(filepath.Base(trigger.Path), ignore) && !checkIfPathAlreadyBindMounted(trigger.Path, service.Volumes) { pathsToCopy = append(pathsToCopy, sync.PathMapping{ HostPath: trigger.Path, ContainerPath: trigger.Target, }) } } - return pathsToCopy, nil + return pathsToCopy, err } -func shouldIgnoreOrSkip(rel string, ignore watch.PathMatcher) bool { - shouldIgnore, _ := ignore.Matches(rel) +func shouldIgnore(name string, ignore watch.PathMatcher) bool { + shouldIgnore, _ := ignore.Matches(name) // ignore files that match any ignore pattern return shouldIgnore } + +// gets the image creation time for a service +func (s *composeService) imageCreatedTime(ctx context.Context, project *types.Project, serviceName string) (time.Time, error) { + containers, err := s.apiClient().ContainerList(ctx, container.ListOptions{ + All: true, + Filters: filters.NewArgs( + filters.Arg("label", fmt.Sprintf("%s=%s", api.ProjectLabel, project.Name)), + filters.Arg("label", fmt.Sprintf("%s=%s", api.ServiceLabel, serviceName))), + }) + if err != nil { + return time.Now(), err + } + if len(containers) == 0 { + return time.Now(), fmt.Errorf("Could not get created time for service's image") + } + + img, _, err := s.apiClient().ImageInspectWithRaw(ctx, containers[0].ImageID) + if err != nil { + return time.Now(), err + } + // Need to get oldest one? + timeCreated, err := time.Parse(time.RFC3339Nano, img.Created) + if err != nil { + return time.Now(), err + } + return timeCreated, nil +}