Add prune feature to keep the cache from growing forever

caches/fileSystem.go

@@ -2,16 +2,29 @@ package caches
 
 import (
 	"encoding/hex"
+	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/user"
 	"path"
+	"path/filepath"
+	"time"
 )
 
 type FileSystemCache struct {
 	root string
 }
 
+type Entry struct {
+	Content  string    `json:"content,omitempty"`
+	LastUsed time.Time `json:"last_used"`
+}
+
+type Entries map[string]Entry
+
+const ENTRIES_FILE = "entries.json"
+
 // GetFileSystemCachePath gets the path to the directory to use for storing the
 // cache. It defaults to ~/.ctcache/cache and can be overridden by setting
 // CLANG_TIDY_CACHE_DIR environment variable.
@@ -29,7 +42,42 @@ func NewFsCache() *FileSystemCache {
 	}
 }
 
-func (c *FileSystemCache) FindEntry(digest []byte) ([]byte, error) {
+// Read the cache entries from JSON. For errors, we log and return an empty
+// `Entries` map so that execution can continue.
+func readJson(filepath string) Entries {
+	if _, err := os.Stat(filepath); os.IsNotExist(err) {
+		return Entries{} // file doesn't exist yet, equivalent to empty file
+	}
+
+	jsonData, err := ioutil.ReadFile(filepath)
+	if err != nil {
+		fmt.Printf("Error reading cache JSON: %v\n", err)
+		return Entries{}
+	}
+
+	entries := Entries{}
+	err = json.Unmarshal(jsonData, &entries)
+	if err != nil {
+		fmt.Printf("Error decoding cache JSON: %v\n", err)
+	}
+	return entries
+}
+
+// Check if we have a cache hit in JSON
+func checkJsonEntry(c *FileSystemCache, digest []byte) []byte {
+	entries := readJson(path.Join(c.root, ENTRIES_FILE))
+	entry, exists := entries[hex.EncodeToString(digest)]
+	if !exists {
+		return nil
+	}
+
+	result := []byte(entry.Content)
+	c.SaveEntry(digest, result) // to update the last used time
+	return result
+}
+
+// Check if we have a cache hit in the filesystem
+func checkFsEntry(c *FileSystemCache, digest []byte) ([]byte, error) {
 	_, entryPath := defineEntryPath(c.root, digest)
 	_, err := os.Stat(entryPath)
 
@@ -50,6 +98,15 @@ func (c *FileSystemCache) FindEntry(digest []byte) ([]byte, error) {
 	return ioutil.ReadAll(source)
 }
 
+// `Prune()` consolidates entries into the JSON file so we want to check that first.
+// A hit in the filesystem is a fallback and it means that `Prune()` has not run yet.
+func (c *FileSystemCache) FindEntry(digest []byte) ([]byte, error) {
+	if content := checkJsonEntry(c, digest); content != nil {
+		return content, nil
+	}
+	return checkFsEntry(c, digest)
+}
+
 func (c *FileSystemCache) SaveEntry(digest []byte, content []byte) error {
 	entryRoot, entryPath := defineEntryPath(c.root, digest)
 
@@ -77,3 +134,87 @@ func defineEntryPath(root string, digest []byte) (string, string) {
 	entryPath := path.Join(entryRoot, encodedDigest[4:])
 	return entryRoot, entryPath
 }
+
+// Remove cache entries that have not been used in the last `numWeeks` and
+// consolidate the remainder in a single JSON file. The consolidation helps
+// speed up later pruning since we only need to look up the single file.
+func Prune(numWeeks int) error {
+	root := GetFileSystemCachePath()
+	err := os.MkdirAll(root, 0755)
+	if err != nil {
+		return err
+	}
+
+	// Populate `Entries` from the many files in the filesystem
+	entries := readJson(path.Join(root, ENTRIES_FILE))
+	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() || info.Name() == ENTRIES_FILE {
+			return nil
+		}
+		content, err := ioutil.ReadFile(path)
+		if err != nil {
+			fmt.Println("Error reading file:", err)
+			return nil
+		}
+
+		// The digest is split over 2 parent dir name and the file name, e.g. `ab/cd/efg...`
+		parent1 := filepath.Base(filepath.Dir(filepath.Dir(path)))
+		parent2 := filepath.Base(filepath.Dir(path))
+		digest := parent1 + parent2 + info.Name()
+		entries[digest] = Entry{Content: string(content), LastUsed: info.ModTime()}
+
+		// We no longer need the file since the content is going into JSON.
+		err = os.Remove(path)
+		if err != nil {
+			fmt.Println("Error deleting file:", err)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Remove all the directories as well now that they are empty
+	paths, err := os.ReadDir(root)
+	if err != nil {
+		return err
+	}
+	for _, pathInfo := range paths {
+		if !pathInfo.IsDir() {
+			continue
+		}
+
+		if err := os.RemoveAll(filepath.Join(root, pathInfo.Name())); err != nil {
+			fmt.Println("Error deleting path:", err)
+			return err
+		}
+	}
+
+	// Keep only the most recent entries
+	now := time.Now()
+	duration := time.Duration(numWeeks*7*24) * time.Hour
+	prunedEntries := Entries{}
+	for key, value := range entries {
+		if now.Sub(value.LastUsed) <= duration {
+			prunedEntries[key] = value
+		}
+	}
+
+	fmt.Println("Found", len(entries), "cache entries in", root)
+	diff := len(entries) - len(prunedEntries)
+	if diff == 0 {
+		fmt.Println("No outdated entries")
+	} else {
+		fmt.Println("Removed", diff, "outdated cache entries")
+	}
+
+	// Write to JSON
+	jsonData, err := json.MarshalIndent(prunedEntries, "", "  ")
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path.Join(root, ENTRIES_FILE), jsonData, 0644)
+}

main.go

@@ -3,18 +3,20 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/ejfitzgerald/clang-tidy-cache/caches"
-	"github.com/ejfitzgerald/clang-tidy-cache/clang"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"os/user"
 	"path"
 	"path/filepath"
+	"strconv"
+
+	"github.com/ejfitzgerald/clang-tidy-cache/caches"
+	"github.com/ejfitzgerald/clang-tidy-cache/clang"
 )
 
-const VERSION = "0.6.0"
+const VERSION = "0.7.0"
 
 type Configuration struct {
 	ClangTidyPath string                   `json:"clang_tidy_path"`
@@ -224,6 +226,20 @@ func main() {
 		os.Exit(0)
 	}
 
+	if len(args) >= 1 && args[0] == "prune" {
+		numWeeks, err := strconv.Atoi(args[1])
+		if err != nil {
+			fmt.Printf("Failed to prune the cache: %v\n", err)
+			os.Exit(1)
+		}
+		err = caches.Prune(numWeeks)
+		if err != nil {
+			fmt.Printf("Failed to prune the cache: %v\n", err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	}
+
 	cfg, err := loadConfiguration()
 	if err != nil {
 		fmt.Printf("Failed to load configuration: %v\n", err)