git: add a cache based on bundles

Signed-off-by: Christophe de Vienne <[email protected]>

pkg/vendir/directory/directory.go

@@ -112,7 +112,7 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 
 			d.ui.PrintLinef("Fetching: %s + %s (git from %s)", d.opts.Path, contents.Path, gitSync.Desc())
 
-			lock, err := gitSync.Sync(stagingDstPath, stagingDir.TempArea())
+			lock, err := gitSync.Sync(stagingDstPath, stagingDir.TempArea(), syncOpts.Cache)
 			if err != nil {
 				return lockConfig, fmt.Errorf("Syncing directory '%s' with git contents: %s", contents.Path, err)
 			}

pkg/vendir/fetch/git/git.go

@@ -47,12 +47,12 @@ type GitInfo struct {
 	CommitTitle string
 }
 
-func (t *Git) Retrieve(dstPath string, tempArea ctlfetch.TempArea) (GitInfo, error) {
+func (t *Git) Retrieve(dstPath string, tempArea ctlfetch.TempArea, bundle string) (GitInfo, error) {
 	if len(t.opts.URL) == 0 {
 		return GitInfo{}, fmt.Errorf("Expected non-empty URL")
 	}
 
-	err := t.fetch(dstPath, tempArea)
+	err := t.fetch(dstPath, tempArea, bundle)
 	if err != nil {
 		return GitInfo{}, err
 	}
@@ -81,7 +81,7 @@ func (t *Git) Retrieve(dstPath string, tempArea ctlfetch.TempArea) (GitInfo, err
 	return info, nil
 }
 
-func (t *Git) fetch(dstPath string, tempArea ctlfetch.TempArea) error {
+func (t *Git) fetch(dstPath string, tempArea ctlfetch.TempArea, bundle string) error {
 	authOpts, err := t.getAuthOpts()
 	if err != nil {
 		return err
@@ -168,6 +168,10 @@ func (t *Git) fetch(dstPath string, tempArea ctlfetch.TempArea) error {
 
 	argss = append(argss, []string{"config", "remote.origin.tagOpt", "--tags"})
 
+	if bundle != "" {
+		argss = append(argss, []string{"bundle", "unbundle", bundle})
+	}
+
 	{
 		fetchArgs := []string{"fetch", "origin"}
 		if strings.HasPrefix(t.opts.Ref, "origin/") {

pkg/vendir/fetch/git/git_test.go

@@ -32,7 +32,7 @@ func TestGit_Retrieve(t *testing.T) {
 			SecretRef:          &config.DirectoryContentsLocalRef{Name: "some-secret"},
 			ForceHTTPBasicAuth: true,
 		}, os.Stdout, secretFetcher, runner)
-		_, err := gitRetriever.Retrieve("", &tmpFolder{t})
+		_, err := gitRetriever.Retrieve("", &tmpFolder{t}, "")
 		require.NoError(t, err)
 		isPresent := false
 		// Check that the header was added with the correct values
@@ -62,7 +62,7 @@ func TestGit_Retrieve(t *testing.T) {
 			SecretRef:          &config.DirectoryContentsLocalRef{Name: "some-secret"},
 			ForceHTTPBasicAuth: true,
 		}, os.Stdout, secretFetcher, runner)
-		_, err := gitRetriever.Retrieve("", &tmpFolder{t})
+		_, err := gitRetriever.Retrieve("", &tmpFolder{t}, "")
 		require.ErrorContains(t, err, "Username/password authentication is only supported for https remotes")
 	})
 }

pkg/vendir/fetch/git/sync.go

@@ -4,15 +4,20 @@
 package git
 
 import (
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
 	ctlconf "carvel.dev/vendir/pkg/vendir/config"
 	ctlfetch "carvel.dev/vendir/pkg/vendir/fetch"
+	ctlcache "carvel.dev/vendir/pkg/vendir/fetch/cache"
 )
 
+const gitCacheType = "git-bundle"
+
 type Sync struct {
 	opts       ctlconf.DirectoryContentsGit
 	log        io.Writer
@@ -36,7 +41,7 @@ func (d Sync) Desc() string {
 	return fmt.Sprintf("%s@%s", d.opts.URL, ref)
 }
 
-func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea) (ctlconf.LockDirectoryContentsGit, error) {
+func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea, cache ctlcache.Cache) (ctlconf.LockDirectoryContentsGit, error) {
 	gitLockConf := ctlconf.LockDirectoryContentsGit{}
 
 	incomingTmpPath, err := tempArea.NewTempDir("git")
@@ -46,9 +51,24 @@ func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea) (ctlconf.LockDire
 
 	defer os.RemoveAll(incomingTmpPath)
 
+	sum := sha256.New()
+	if _, err := sum.Write([]byte(d.opts.URL)); err != nil {
+		if err != nil {
+			return gitLockConf, err
+		}
+	}
+
+	cacheID := string(sum.Sum(nil))
+
 	git := NewGit(d.opts, d.log, d.refFetcher)
 
-	info, err := git.Retrieve(incomingTmpPath, tempArea)
+	var bundle string
+	if cacheEntry, hasCache := cache.Has(gitCacheType, cacheID); hasCache {
+		bundle = filepath.Join(cacheEntry, "bundle")
+	}
+	fmt.Println("We got cache?", cacheID, bundle)
+
+	info, err := git.Retrieve(incomingTmpPath, tempArea, bundle)
 	if err != nil {
 		return gitLockConf, fmt.Errorf("Fetching git repository: %s", err)
 	}
@@ -57,6 +77,35 @@ func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea) (ctlconf.LockDire
 	gitLockConf.Tags = info.Tags
 	gitLockConf.CommitTitle = d.singleLineCommitTitle(info.CommitTitle)
 
+	if os.Getenv("VENDIR_CACHE_DIR") != "" {
+		// attempt to save a bundle to the cache
+		bundleDir, err := tempArea.NewTempDir("bundleCache")
+		if err != nil {
+			return gitLockConf, err
+		}
+		defer os.RemoveAll(bundleDir)
+		bundle := filepath.Join(bundleDir, "bundle")
+		// get all refs
+
+		out, _, err := git.cmdRunner.Run([]string{"for-each-ref", "--format=%(refname)"}, nil, incomingTmpPath)
+		if err != nil {
+			return gitLockConf, err
+		}
+		var refs []string
+		for _, ref := range strings.Split(string(out), "\n") {
+			ref = strings.TrimSpace(ref)
+			if ref != "" {
+				refs = append(refs, ref)
+			}
+		}
+		if _, _, err := git.cmdRunner.Run(append([]string{"bundle", "create", bundle}, refs...), nil, incomingTmpPath); err != nil {
+			return gitLockConf, err
+		}
+		if err := cache.Save(gitCacheType, cacheID, bundleDir); err != nil {
+			return gitLockConf, err
+		}
+	}
+
 	err = os.RemoveAll(dstPath)
 	if err != nil {
 		return gitLockConf, fmt.Errorf("Deleting dir %s: %s", dstPath, err)