Skip to content

Commit

Permalink
[scudo] Update secondary cache time-based release logic.
Browse files Browse the repository at this point in the history
Secondary cache entries are now released to the OS from least recent
to most recent entries. This helps to avoid unnecessary scans of the
cache since entries ready to be released (specifically, entries
that are considered old relative to the configurable release interval)
will always be at the tail of the list of committed entries by the LRU
ordering. For this same reason, the OldestTime variable is no longer
needed to indicate when releases are necessary so it has been removed.
  • Loading branch information
JoshuaMBa committed Sep 9, 2024
1 parent 9e9971b commit 647eb80
Showing 1 changed file with 30 additions and 21 deletions.
51 changes: 30 additions & 21 deletions compiler-rt/lib/scudo/standalone/secondary.h
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,10 @@ class MapAllocatorCache {
}
CachedBlock PrevEntry = Quarantine[QuarantinePos];
Quarantine[QuarantinePos] = Entry;
if (OldestTime == 0)
OldestTime = Entry.Time;
Entry = PrevEntry;
// Set entry time once more to reflect time
// that quarantined memory was placed in the cache
Entry.Time = Time;
}

// All excess entries are evicted from the cache
Expand All @@ -331,9 +332,6 @@ class MapAllocatorCache {
}

insert(Entry);

if (OldestTime == 0)
OldestTime = Entry.Time;
} while (0);

for (MemMapT &EvictMemMap : EvictionMemMaps)
Expand Down Expand Up @@ -532,6 +530,9 @@ class MapAllocatorCache {
Entries[LRUHead].Prev = static_cast<u16>(FreeIndex);
}

if (LastUnreleasedEntry == CachedBlock::InvalidEntry)
LastUnreleasedEntry = static_cast<u16>(FreeIndex);

Entries[FreeIndex] = Entry;
Entries[FreeIndex].Next = LRUHead;
Entries[FreeIndex].Prev = CachedBlock::InvalidEntry;
Expand All @@ -549,6 +550,9 @@ class MapAllocatorCache {

Entries[I].invalidate();

if (I == LastUnreleasedEntry)
LastUnreleasedEntry = Entries[LastUnreleasedEntry].Prev;

if (I == LRUHead)
LRUHead = Entries[I].Next;
else
Expand Down Expand Up @@ -590,35 +594,37 @@ class MapAllocatorCache {
}
}

void releaseIfOlderThan(CachedBlock &Entry, u64 Time) REQUIRES(Mutex) {
if (!Entry.isValid() || !Entry.Time)
return;
if (Entry.Time > Time) {
if (OldestTime == 0 || Entry.Time < OldestTime)
OldestTime = Entry.Time;
return;
}
inline void release(CachedBlock &Entry) {
DCHECK(Entry.Time != 0);
Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, Entry.CommitSize);
Entry.Time = 0;
}

void releaseOlderThan(u64 Time) EXCLUDES(Mutex) {
ScopedLock L(Mutex);
if (!EntriesCount || OldestTime == 0 || OldestTime > Time)
if (!EntriesCount)
return;
OldestTime = 0;
for (uptr I = 0; I < Config::getQuarantineSize(); I++)
releaseIfOlderThan(Quarantine[I], Time);
for (uptr I = 0; I < Config::getEntriesArraySize(); I++)
releaseIfOlderThan(Entries[I], Time);
}

for (uptr I = 0; I < Config::getQuarantineSize(); I++) {
CachedBlock &ReleaseEntry = Quarantine[I];
if (!ReleaseEntry.isValid() || ReleaseEntry.Time > Time ||
!ReleaseEntry.Time)
continue;
release(ReleaseEntry);
}

// Release oldest entries first by releasing from decommit base
while (LastUnreleasedEntry != CachedBlock::InvalidEntry &&
Entries[LastUnreleasedEntry].Time <= Time) {
release(Entries[LastUnreleasedEntry]);
LastUnreleasedEntry = Entries[LastUnreleasedEntry].Prev;
}
}
HybridMutex Mutex;
u32 EntriesCount GUARDED_BY(Mutex) = 0;
u32 QuarantinePos GUARDED_BY(Mutex) = 0;
atomic_u32 MaxEntriesCount = {};
atomic_uptr MaxEntrySize = {};
u64 OldestTime GUARDED_BY(Mutex) = 0;
atomic_s32 ReleaseToOsIntervalMs = {};
u32 CallsToRetrieve GUARDED_BY(Mutex) = 0;
u32 SuccessfulRetrieves GUARDED_BY(Mutex) = 0;
Expand All @@ -633,6 +639,9 @@ class MapAllocatorCache {
u16 LRUTail GUARDED_BY(Mutex) = 0;
// The AvailableHead is the top of the stack of available entries
u16 AvailableHead GUARDED_BY(Mutex) = 0;
// The LastUnreleasedEntry is the least recently used entry that has not
// been released
u16 LastUnreleasedEntry GUARDED_BY(Mutex) = 0;
};

template <typename Config> class MapAllocator {
Expand Down

0 comments on commit 647eb80

Please sign in to comment.