Interfaces for Extendable GC and LRU implementation

gc/gc.go

@@ -0,0 +1,44 @@
+package gc
+
+import "github.com/filecoin-project/dagstore/shard"
+
+// GarbageCollectionStrategy manages the algorithm for deciding the order in which
+// reclaimable transients should be garbage collected when the dagstore runs an automated GC.
+// It is the dagstore's responsibility to inform the `GarbageCollectionStrategy` about which transients are reclaimable and
+// which are not using the public interface of the `GarbageCollectionStrategy`.
+//
+// Note: The `GarbageCollectionStrategy` is only responsible for deciding the order in which reclaimable transients
+// should be GC'd. The actual Garbage Collection is done by the dagstore which "owns" the transients.
+//
+// Implementations of `GarbageCollectionStrategy` are not meant to be thread safe and the dagstore should only
+// invoke the `GarbageCollectionStrategy` methods from the event loop.
+type GarbageCollectionStrategy interface {
+	// NotifyAccessed notifies the strategy when the shard with the given key is accessed for a read operation.
+	NotifyAccessed(shard.Key)
+	// NotifyReclaimable notifies the strategy that the shard with the given key is reclaimable.
+	NotifyReclaimable(shard.Key)
+	// NotifyNotReclaimable notifies the strategy that the shard with the given key is not reclaimable.
+	NotifyNotReclaimable(key shard.Key)
+	// NotifyRemoved notifies the strategy that the shard with the given key has been removed by the dagstore.
+	NotifyRemoved(shard.Key)
+	// NotifyReclaimed notifies the strategy that the shards with the given key have been reclaimed by the dagstore.
+	// The dagstore will ideally call `Reclaimable` -> get an ordered list of shards that can be GC'd and then call `NotifyReclaimed`
+	// on the `GarbageCollectionStrategy` for all shards that were actually GC'd/reclaimed.
+	NotifyReclaimed([]shard.Key)
+	// Reclaimable is called by the dagstore when it wants an ordered list of shards whose transients can
+	// be reclaimed by GC. The shards are ordered in descending order of their eligibility for GC.
+	Reclaimable() []shard.Key
+}
+
+var _ GarbageCollectionStrategy = (*NoOpStrategy)(nil)
+
+type NoOpStrategy struct{}
+
+func (n *NoOpStrategy) NotifyAccessed(shard.Key)           {}
+func (n *NoOpStrategy) NotifyReclaimable(shard.Key)        {}
+func (n *NoOpStrategy) NotifyNotReclaimable(key shard.Key) {}
+func (n *NoOpStrategy) NotifyRemoved(shard.Key)            {}
+func (n *NoOpStrategy) NotifyReclaimed([]shard.Key)        {}
+func (n *NoOpStrategy) Reclaimable() []shard.Key {
+	return nil
+}

gc/lru.go

@@ -0,0 +1,83 @@
+package gc
+
+import (
+	"sort"
+	"time"
+
+	"github.com/filecoin-project/dagstore/shard"
+)
+
+var _ GarbageCollectionStrategy = (*LRUGarbageCollector)(nil)
+
+type shardMetadata struct {
+	key            shard.Key
+	reclaimable    bool
+	lastAccessedAt time.Time
+}
+
+// LRUGarbageCollector implements a `Least Recently Used` strategy for
+// determining the order in which the reclaimable shards it is tracking should be GC'd.
+type LRUGarbageCollector struct {
+	shards map[shard.Key]*shardMetadata
+}
+
+func NewLRUGarbageCollector() *LRUGarbageCollector {
+	return &LRUGarbageCollector{
+		shards: make(map[shard.Key]*shardMetadata),
+	}
+}
+
+func (l *LRUGarbageCollector) NotifyReclaimable(key shard.Key) {
+	l.updateF(key, func(sm *shardMetadata) { sm.reclaimable = true })
+}
+
+func (l *LRUGarbageCollector) NotifyNotReclaimable(key shard.Key) {
+	l.updateF(key, func(sm *shardMetadata) { sm.reclaimable = false })
+}
+
+func (l *LRUGarbageCollector) NotifyAccessed(key shard.Key) {
+	l.updateF(key, func(sm *shardMetadata) { sm.lastAccessedAt = time.Now() })
+}
+
+func (l *LRUGarbageCollector) updateF(key shard.Key, update func(sm *shardMetadata)) {
+	sm, ok := l.shards[key]
+	if !ok {
+		sm = &shardMetadata{
+			key: key,
+		}
+	}
+	update(sm)
+	l.shards[key] = sm
+}
+
+func (l *LRUGarbageCollector) NotifyRemoved(key shard.Key) {
+	delete(l.shards, key)
+}
+
+func (l *LRUGarbageCollector) Reclaimable() []shard.Key {
+	var reclaim []shardMetadata
+	for _, s := range l.shards {
+		sm := *s
+		if s.reclaimable {
+			reclaim = append(reclaim, sm)
+		}
+	}
+
+	// Sort in LRU order
+	sort.Slice(reclaim, func(i, j int) bool {
+		return reclaim[i].lastAccessedAt.Before(reclaim[j].lastAccessedAt)
+	})
+
+	keys := make([]shard.Key, 0, len(reclaim))
+	for _, s := range reclaim {
+		keys = append(keys, s.key)
+	}
+
+	return keys
+}
+
+func (l *LRUGarbageCollector) NotifyReclaimed(keys []shard.Key) {
+	for _, k := range keys {
+		delete(l.shards, k)
+	}
+}

gc/lru_test.go

@@ -0,0 +1,47 @@
+package gc
+
+import (
+	"testing"
+
+	"github.com/filecoin-project/dagstore/shard"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLRUGarbageCollector(t *testing.T) {
+	lru := NewLRUGarbageCollector()
+
+	sk1 := shard.KeyFromString("key")
+	sk2 := shard.KeyFromString("key2")
+	sk3 := shard.KeyFromString("key3")
+
+	lru.NotifyReclaimable(sk1)
+	require.Equal(t, []shard.Key{sk1}, lru.Reclaimable())
+
+	lru.NotifyReclaimable(sk2)
+	lru.NotifyAccessed(sk1)
+	require.Equal(t, []shard.Key{sk2, sk1}, lru.Reclaimable())
+
+	lru.NotifyAccessed(sk2)
+	require.Equal(t, []shard.Key{sk1, sk2}, lru.Reclaimable())
+
+	lru.NotifyNotReclaimable(sk1)
+	require.Equal(t, []shard.Key{sk2}, lru.Reclaimable())
+
+	lru.NotifyReclaimable(sk3)
+	require.Equal(t, []shard.Key{sk3, sk2}, lru.Reclaimable())
+
+	lru.NotifyReclaimable(sk1)
+	require.Equal(t, sk2, lru.Reclaimable()[2])
+
+	lru.NotifyAccessed(sk3)
+	require.Equal(t, []shard.Key{sk1, sk2, sk3}, lru.Reclaimable())
+
+	lru.NotifyRemoved(sk2)
+	require.Equal(t, []shard.Key{sk1, sk3}, lru.Reclaimable())
+
+	lru.NotifyReclaimed([]shard.Key{sk1})
+	require.Equal(t, []shard.Key{sk3}, lru.Reclaimable())
+
+	lru.NotifyReclaimed([]shard.Key{sk1, sk3})
+	require.Equal(t, []shard.Key{}, lru.Reclaimable())
+}