From f68ee7254238f78923198102d39ff4fa4e9c8528 Mon Sep 17 00:00:00 2001 From: Christophe de Vienne Date: Tue, 7 May 2024 19:09:07 +0200 Subject: [PATCH] git: add a cache based on bundles Signed-off-by: Christophe de Vienne --- pkg/vendir/directory/directory.go | 2 +- pkg/vendir/fetch/git/git.go | 10 ++++-- pkg/vendir/fetch/git/git_test.go | 4 +-- pkg/vendir/fetch/git/sync.go | 53 +++++++++++++++++++++++++++++-- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/pkg/vendir/directory/directory.go b/pkg/vendir/directory/directory.go index be2f89cf..632fa565 100644 --- a/pkg/vendir/directory/directory.go +++ b/pkg/vendir/directory/directory.go @@ -112,7 +112,7 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { d.ui.PrintLinef("Fetching: %s + %s (git from %s)", d.opts.Path, contents.Path, gitSync.Desc()) - lock, err := gitSync.Sync(stagingDstPath, stagingDir.TempArea()) + lock, err := gitSync.Sync(stagingDstPath, stagingDir.TempArea(), syncOpts.Cache) if err != nil { return lockConfig, fmt.Errorf("Syncing directory '%s' with git contents: %s", contents.Path, err) } diff --git a/pkg/vendir/fetch/git/git.go b/pkg/vendir/fetch/git/git.go index 644a9ed3..07883a7f 100644 --- a/pkg/vendir/fetch/git/git.go +++ b/pkg/vendir/fetch/git/git.go @@ -47,12 +47,12 @@ type GitInfo struct { CommitTitle string } -func (t *Git) Retrieve(dstPath string, tempArea ctlfetch.TempArea) (GitInfo, error) { +func (t *Git) Retrieve(dstPath string, tempArea ctlfetch.TempArea, bundle string) (GitInfo, error) { if len(t.opts.URL) == 0 { return GitInfo{}, fmt.Errorf("Expected non-empty URL") } - err := t.fetch(dstPath, tempArea) + err := t.fetch(dstPath, tempArea, bundle) if err != nil { return GitInfo{}, err } @@ -81,7 +81,7 @@ func (t *Git) Retrieve(dstPath string, tempArea ctlfetch.TempArea) (GitInfo, err return info, nil } -func (t *Git) fetch(dstPath string, tempArea ctlfetch.TempArea) error { +func (t *Git) fetch(dstPath string, tempArea ctlfetch.TempArea, bundle string) error { authOpts, err := t.getAuthOpts() if err != nil { return err @@ -168,6 +168,10 @@ func (t *Git) fetch(dstPath string, tempArea ctlfetch.TempArea) error { argss = append(argss, []string{"config", "remote.origin.tagOpt", "--tags"}) + if bundle != "" { + argss = append(argss, []string{"bundle", "unbundle", bundle}) + } + { fetchArgs := []string{"fetch", "origin"} if strings.HasPrefix(t.opts.Ref, "origin/") { diff --git a/pkg/vendir/fetch/git/git_test.go b/pkg/vendir/fetch/git/git_test.go index c6a4de08..3ccdd286 100644 --- a/pkg/vendir/fetch/git/git_test.go +++ b/pkg/vendir/fetch/git/git_test.go @@ -32,7 +32,7 @@ func TestGit_Retrieve(t *testing.T) { SecretRef: &config.DirectoryContentsLocalRef{Name: "some-secret"}, ForceHTTPBasicAuth: true, }, os.Stdout, secretFetcher, runner) - _, err := gitRetriever.Retrieve("", &tmpFolder{t}) + _, err := gitRetriever.Retrieve("", &tmpFolder{t}, "") require.NoError(t, err) isPresent := false // Check that the header was added with the correct values @@ -62,7 +62,7 @@ func TestGit_Retrieve(t *testing.T) { SecretRef: &config.DirectoryContentsLocalRef{Name: "some-secret"}, ForceHTTPBasicAuth: true, }, os.Stdout, secretFetcher, runner) - _, err := gitRetriever.Retrieve("", &tmpFolder{t}) + _, err := gitRetriever.Retrieve("", &tmpFolder{t}, "") require.ErrorContains(t, err, "Username/password authentication is only supported for https remotes") }) } diff --git a/pkg/vendir/fetch/git/sync.go b/pkg/vendir/fetch/git/sync.go index ec08611d..c5df3936 100644 --- a/pkg/vendir/fetch/git/sync.go +++ b/pkg/vendir/fetch/git/sync.go @@ -4,15 +4,20 @@ package git import ( + "crypto/sha256" "fmt" "io" "os" + "path/filepath" "strings" ctlconf "carvel.dev/vendir/pkg/vendir/config" ctlfetch "carvel.dev/vendir/pkg/vendir/fetch" + ctlcache "carvel.dev/vendir/pkg/vendir/fetch/cache" ) +const gitCacheType = "git-bundle" + type Sync struct { opts ctlconf.DirectoryContentsGit log io.Writer @@ -36,7 +41,7 @@ func (d Sync) Desc() string { return fmt.Sprintf("%s@%s", d.opts.URL, ref) } -func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea) (ctlconf.LockDirectoryContentsGit, error) { +func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea, cache ctlcache.Cache) (ctlconf.LockDirectoryContentsGit, error) { gitLockConf := ctlconf.LockDirectoryContentsGit{} incomingTmpPath, err := tempArea.NewTempDir("git") @@ -46,9 +51,24 @@ func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea) (ctlconf.LockDire defer os.RemoveAll(incomingTmpPath) + sum := sha256.New() + if _, err := sum.Write([]byte(d.opts.URL)); err != nil { + if err != nil { + return gitLockConf, err + } + } + + cacheID := string(sum.Sum(nil)) + git := NewGit(d.opts, d.log, d.refFetcher) - info, err := git.Retrieve(incomingTmpPath, tempArea) + var bundle string + if cacheEntry, hasCache := cache.Has(gitCacheType, cacheID); hasCache { + bundle = filepath.Join(cacheEntry, "bundle") + } + fmt.Println("We got cache?", cacheID, bundle) + + info, err := git.Retrieve(incomingTmpPath, tempArea, bundle) if err != nil { return gitLockConf, fmt.Errorf("Fetching git repository: %s", err) } @@ -57,6 +77,35 @@ func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea) (ctlconf.LockDire gitLockConf.Tags = info.Tags gitLockConf.CommitTitle = d.singleLineCommitTitle(info.CommitTitle) + if os.Getenv("VENDIR_CACHE_DIR") != "" { + // attempt to save a bundle to the cache + bundleDir, err := tempArea.NewTempDir("bundleCache") + if err != nil { + return gitLockConf, err + } + defer os.RemoveAll(bundleDir) + bundle := filepath.Join(bundleDir, "bundle") + // get all refs + + out, _, err := git.cmdRunner.Run([]string{"for-each-ref", "--format=%(refname)"}, nil, incomingTmpPath) + if err != nil { + return gitLockConf, err + } + var refs []string + for _, ref := range strings.Split(string(out), "\n") { + ref = strings.TrimSpace(ref) + if ref != "" { + refs = append(refs, ref) + } + } + if _, _, err := git.cmdRunner.Run(append([]string{"bundle", "create", bundle}, refs...), nil, incomingTmpPath); err != nil { + return gitLockConf, err + } + if err := cache.Save(gitCacheType, cacheID, bundleDir); err != nil { + return gitLockConf, err + } + } + err = os.RemoveAll(dstPath) if err != nil { return gitLockConf, fmt.Errorf("Deleting dir %s: %s", dstPath, err)