From 4cb51af05e611ffb25cbc541e3fba1003d0c47d1 Mon Sep 17 00:00:00 2001 From: Peter Hofer Date: Tue, 7 May 2024 13:51:30 +0200 Subject: [PATCH] Improvements and minor fixes. --- .../core/genscavenge/AlignedHeapChunk.java | 19 +- .../genscavenge/CompactingOldGeneration.java | 285 ++++++++++-------- .../genscavenge/CopyingOldGeneration.java | 22 +- .../oracle/svm/core/genscavenge/GCImpl.java | 4 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 50 +-- .../svm/core/genscavenge/HeapVerifier.java | 5 + .../core/genscavenge/ObjectHeaderImpl.java | 6 +- .../svm/core/genscavenge/OldGeneration.java | 23 +- .../ReferenceObjectProcessing.java | 3 +- .../svm/core/genscavenge/SerialGCOptions.java | 26 +- .../oracle/svm/core/genscavenge/Space.java | 43 ++- .../oracle/svm/core/genscavenge/Timers.java | 63 ++-- .../svm/core/genscavenge/YoungGeneration.java | 16 + .../genscavenge/compacting/MarkStack.java | 27 +- .../compacting/ObjectMoveInfo.java | 24 +- .../compacting/ObjectRefFixupVisitor.java | 4 +- .../compacting/PlanningVisitor.java | 28 +- .../RuntimeCodeCacheFixupWalker.java | 9 +- .../compacting/SweepingVisitor.java | 19 +- .../remset/AlignedChunkRememberedSet.java | 2 +- .../core/genscavenge/remset/BrickTable.java | 2 +- .../core/genscavenge/remset/CardTable.java | 10 +- 22 files changed, 386 insertions(+), 304 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/{ => compacting}/RuntimeCodeCacheFixupWalker.java (87%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index ceb08d41804b..f8d34e8e5dce 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -52,17 +52,14 @@ * Most allocation within a AlignedHeapChunk is via fast-path allocation snippets, but a slow-path * allocation method is available. *

- * Objects in a AlignedHeapChunk have to be promoted by copying from their current HeapChunk to a - * destination HeapChunk. - *

- * An AlignedHeapChunk is laid out: + * An AlignedHeapChunk is laid out as follows: * * 

- * +===============+-------+--------+----------------------+
- * | AlignedHeader | Card  | First  | Object ...           |
- * | Fields        | Table | Object |                      |
- * |               |       | Table  |                      |
- * +===============+-------+--------+----------------------+
+ * +===============+-------+--------+-----------------+-----------------+
+ * | AlignedHeader | Card  | First  | Initial Object  | Object ...      |
+ * | Fields        | Table | Object | Move Info (only |                 |
+ * |               |       | Table  | Compacting GC)  |                 |
+ * +===============+-------+--------+-----------------+-----------------+
  *
* * The size of both the CardTable and the FirstObjectTable depends on the used {@link RememberedSet} @@ -107,6 +104,10 @@ public static Pointer getObjectsEnd(AlignedHeader that) { return HeapChunk.getEndPointer(that); } + public static boolean isEmpty(AlignedHeader that) { + return HeapChunk.getTopOffset(that).equal(getObjectsStartOffset()); + } + /** Allocate uninitialized memory within this AlignedHeapChunk. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java index 7b9344d96965..80b43f4011bb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java @@ -26,8 +26,6 @@ import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress; -import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY; -import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -38,7 +36,6 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.code.RuntimeCodeInfoMemory; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; @@ -48,23 +45,65 @@ import com.oracle.svm.core.genscavenge.compacting.ObjectMoveInfo; import com.oracle.svm.core.genscavenge.compacting.ObjectRefFixupVisitor; import com.oracle.svm.core.genscavenge.compacting.PlanningVisitor; +import com.oracle.svm.core.genscavenge.compacting.RuntimeCodeCacheFixupWalker; import com.oracle.svm.core.genscavenge.compacting.SweepingVisitor; +import com.oracle.svm.core.genscavenge.remset.BrickTable; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.graal.RuntimeCompilation; import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.heap.ObjectVisitor; -import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.VMThreadLocalSupport; -import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.word.Word; /** - * The compacting old generation has only one {@link Space} for existing and promoted objects and - * uses a mark-compact algorithm for garbage collection. + * Core of the mark-compact implementation for the old generation, which collects using (almost) + * only memory that is already in use, while {@link CopyingOldGeneration} has a worst-case memory + * usage of 2x the heap size during collections. This implementation has a single {@link Space}. + * + * Complete collections are carried out in the following stages: + * + * + * + * While updating references using lookups in the brick table and structures seems expensive, it + * frequently needs only few accesses. It would be possible to introduce a field in each object that + * stores its new location during collections, but that would add significant memory overhead even + * outside of GC. In contrast, using entirely separate side tables would require extra memory only + * during GC and enable collecting with fewer passes over the heap, but requires allocating the + * tables precisely at a time when memory might be scarce. + * + * Some parts of the implementation are scattered over the GC code and can be found by following the + * usages of {@link SerialGCOptions#useCompactingOldGen()}. */ final class CompactingOldGeneration extends OldGeneration { @@ -86,9 +125,9 @@ final class CompactingOldGeneration extends OldGeneration { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void beginPromotion(YoungGeneration youngGen, boolean incrementalGc) { + void beginPromotion(boolean incrementalGc) { if (!incrementalGc) { - absorb(youngGen); + absorb(HeapImpl.getHeapImpl().getYoungGeneration()); } toGreyObjectsWalker.setScanStart(space); } @@ -117,10 +156,13 @@ boolean scanGreyObjects(boolean incrementalGc) { } toGreyObjectsWalker.walkGreyObjects(); } else { + if (markStack.isEmpty()) { + return false; + } GreyToBlackObjectVisitor visitor = GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); - while (!markStack.isEmpty()) { + do { visitor.visitObjectInline(markStack.pop()); - } + } while (!markStack.isEmpty()); } return true; } @@ -176,13 +218,11 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected boolean promotePinnedObject(Object obj, HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace) { if (!GCImpl.getGCImpl().isCompleteCollection()) { - if (originalSpace != space) { - assert originalSpace.isFromSpace(); - if (isAligned) { - space.promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk, originalSpace); - } else { - space.promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk, originalSpace); - } + assert originalSpace != space && originalSpace.isFromSpace(); + if (isAligned) { + space.promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk, originalSpace); + } else { + space.promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk, originalSpace); } return true; } @@ -201,37 +241,32 @@ protected boolean promotePinnedObject(Object obj, HeapChunk.Header originalCh @Override void sweepAndCompact(Timers timers, ChunkReleaser chunkReleaser) { - long startTicks = JfrGCEvents.startGCPhasePause(); - /* - * Update or null reference objects now because planning below overwrites referent objects - * that do not survive or have been copied (e.g. adding for an identity hashcode field). + * Update or clear reference object referent fields now because planning below overwrites + * referent objects that do not survive or have been copied (e.g. adding for an identity + * hashcode field). */ ReferenceObjectProcessing.updateForwardedRefs(); - Timer tenuredPlanningTimer = timers.tenuredPlanning.open(); + Timer oldPlanningTimer = timers.oldPlanning.open(); try { - try { - planCompaction(); - } finally { - JfrGCEvents.emitGCPhasePauseEvent(GCImpl.getGCImpl().getCollectionEpoch(), "Tenured Planning", startTicks); - } + planCompaction(); } finally { - tenuredPlanningTimer.close(); + oldPlanningTimer.close(); } - Timer tenuredFixingTimer = timers.tenuredFixing.open(); + Timer oldFixupTimer = timers.oldFixup.open(); try { fixupReferencesBeforeCompaction(chunkReleaser, timers); } finally { - tenuredFixingTimer.close(); + oldFixupTimer.close(); } - Timer tenuredCompactingTimer = timers.tenuredCompacting.open(); + Timer oldCompactionTimer = timers.oldCompaction.open(); try { compact(timers); } finally { - tenuredCompactingTimer.close(); + oldCompactionTimer.close(); } } @@ -242,80 +277,86 @@ private void planCompaction() { @Uninterruptible(reason = "Avoid unnecessary safepoint checks in GC for performance.") private void fixupReferencesBeforeCompaction(ChunkReleaser chunkReleaser, Timers timers) { - timers.tenuredFixingAlignedChunks.open(); - AlignedHeapChunk.AlignedHeader aChunk = space.getFirstAlignedHeapChunk(); - while (aChunk.isNonNull()) { - ObjectMoveInfo.walkObjects(aChunk, fixupVisitor); - aChunk = HeapChunk.getNext(aChunk); + Timer oldFixupAlignedChunksTimer = timers.oldFixupAlignedChunks.open(); + try { + AlignedHeapChunk.AlignedHeader aChunk = space.getFirstAlignedHeapChunk(); + while (aChunk.isNonNull()) { + ObjectMoveInfo.walkObjects(aChunk, fixupVisitor); + aChunk = HeapChunk.getNext(aChunk); + } + } finally { + oldFixupAlignedChunksTimer.close(); } - timers.tenuredFixingAlignedChunks.close(); - timers.tenuredFixingImageHeap.open(); - for (ImageHeapInfo info = HeapImpl.getFirstImageHeapInfo(); info != null; info = info.next) { - GCImpl.walkImageHeapRoots(info, fixupVisitor); - } - if (AuxiliaryImageHeap.isPresent()) { - ImageHeapInfo auxImageHeapInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo(); - if (auxImageHeapInfo != null) { - GCImpl.walkImageHeapRoots(auxImageHeapInfo, fixupVisitor); + Timer oldFixupImageHeapTimer = timers.oldFixupImageHeap.open(); + try { + for (ImageHeapInfo info = HeapImpl.getFirstImageHeapInfo(); info != null; info = info.next) { + GCImpl.walkImageHeapRoots(info, fixupVisitor); } - } - timers.tenuredFixingImageHeap.close(); - - timers.tenuredFixingThreadLocal.open(); - if (SubstrateOptions.MultiThreaded.getValue()) { - Timer walkThreadLocalsTimer = timers.walkThreadLocals.open(); - try { - for (IsolateThread isolateThread = VMThreads.firstThread(); isolateThread.isNonNull(); isolateThread = VMThreads.nextThread(isolateThread)) { - VMThreadLocalSupport.singleton().walk(isolateThread, refFixupVisitor); + if (AuxiliaryImageHeap.isPresent()) { + ImageHeapInfo auxImageHeapInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo(); + if (auxImageHeapInfo != null) { + GCImpl.walkImageHeapRoots(auxImageHeapInfo, fixupVisitor); } - } finally { - walkThreadLocalsTimer.close(); } + } finally { + oldFixupImageHeapTimer.close(); + } + + Timer oldFixupThreadLocalsTimer = timers.oldFixupThreadLocals.open(); + try { + for (IsolateThread isolateThread = VMThreads.firstThread(); isolateThread.isNonNull(); isolateThread = VMThreads.nextThread(isolateThread)) { + VMThreadLocalSupport.singleton().walk(isolateThread, refFixupVisitor); + } + } finally { + oldFixupThreadLocalsTimer.close(); } - timers.tenuredFixingThreadLocal.close(); - timers.tenuredFixingStack.open(); + Timer oldFixupStackTimer = timers.oldFixupStack.open(); try { fixupStackReferences(); } finally { - timers.tenuredFixingStack.close(); + oldFixupStackTimer.close(); } /* - * Check unaligned objects. Fix its contained references if the object is marked. Add the - * chunk to the releaser's list in case the object is not marked and thus won't survive. + * Check each unaligned object and fix its references if the object is marked. Add the chunk + * to the releaser's list in case the object is not marked and therefore won't survive. */ - timers.tenuredFixingUnalignedChunks.open(); + Timer oldFixupUnalignedChunksTimer = timers.oldFixupUnalignedChunks.open(); try { - UnalignedHeapChunk.UnalignedHeader uChunk = space.getFirstUnalignedHeapChunk(); - while (uChunk.isNonNull()) { - UnalignedHeapChunk.UnalignedHeader next = HeapChunk.getNext(uChunk); - Pointer objPointer = UnalignedHeapChunk.getObjectStart(uChunk); - Object obj = objPointer.toObject(); - if (ObjectHeaderImpl.isMarked(obj)) { - ObjectHeaderImpl.unsetMarkedAndKeepRememberedSetBit(obj); - RememberedSet.get().clearRememberedSet(uChunk); - - UnalignedHeapChunk.walkObjectsInline(uChunk, fixupVisitor); - - UnsignedWord objSize = LayoutEncoding.getSizeFromObjectInlineInGC(obj); - assert UnalignedHeapChunk.getObjectStart(uChunk).add(objSize).equal(HeapChunk.getTopPointer(uChunk)); - } else { - space.extractUnalignedHeapChunk(uChunk); - chunkReleaser.add(uChunk); - } - uChunk = next; + fixupUnalignedChunkReferences(chunkReleaser); + } finally { + oldFixupUnalignedChunksTimer.close(); + } + + Timer oldFixupRuntimeCodeCacheTimer = timers.oldFixupRuntimeCodeCache.open(); + try { + if (RuntimeCompilation.isEnabled()) { + RuntimeCodeInfoMemory.singleton().walkRuntimeMethodsDuringGC(runtimeCodeCacheFixupWalker); } } finally { - timers.tenuredFixingUnalignedChunks.close(); + oldFixupRuntimeCodeCacheTimer.close(); } + } - timers.tenuredFixingRuntimeCodeCache.open(); - if (RuntimeCompilation.isEnabled()) { - RuntimeCodeInfoMemory.singleton().walkRuntimeMethodsDuringGC(runtimeCodeCacheFixupWalker); + @Uninterruptible(reason = "Avoid unnecessary safepoint checks in GC for performance.") + private void fixupUnalignedChunkReferences(ChunkReleaser chunkReleaser) { + UnalignedHeapChunk.UnalignedHeader uChunk = space.getFirstUnalignedHeapChunk(); + while (uChunk.isNonNull()) { + UnalignedHeapChunk.UnalignedHeader next = HeapChunk.getNext(uChunk); + Pointer objPointer = UnalignedHeapChunk.getObjectStart(uChunk); + Object obj = objPointer.toObject(); + if (ObjectHeaderImpl.isMarked(obj)) { + ObjectHeaderImpl.unsetMarkedAndKeepRememberedSetBit(obj); + RememberedSet.get().clearRememberedSet(uChunk); + UnalignedHeapChunk.walkObjectsInline(uChunk, fixupVisitor); + } else { + space.extractUnalignedHeapChunk(uChunk); + chunkReleaser.add(uChunk); + } + uChunk = next; } - timers.tenuredFixingRuntimeCodeCache.close(); } @NeverInline("Starting a stack walk in the caller frame. " + @@ -329,7 +370,6 @@ private void fixupStackReferences() { } private void compact(Timers timers) { - timers.tenuredCompactingChunks.open(); AlignedHeapChunk.AlignedHeader chunk = space.getFirstAlignedHeapChunk(); while (chunk.isNonNull()) { if (chunk.getShouldSweepInsteadOfCompact()) { @@ -341,31 +381,40 @@ private void compact(Timers timers) { } chunk = HeapChunk.getNext(chunk); } - timers.tenuredCompactingChunks.close(); - timers.tenuredCompactingUpdatingRemSet.open(); - chunk = space.getFirstAlignedHeapChunk(); - while (chunk.isNonNull()) { - /* - * Clears the card table (which currently contains the brick table) and updates the - * first object table. - * - * TODO: build the first object table during compaction (after processing a chunk and it - * is in the cache) or during fixup (in-flight with fixing references) or even planning. - */ - RememberedSet.get().enableRememberedSetForChunk(chunk); - chunk = HeapChunk.getNext(chunk); + Timer oldCompactionRememberedSetsTimer = timers.oldCompactionRememberedSets.open(); + try { + chunk = space.getFirstAlignedHeapChunk(); + while (chunk.isNonNull()) { + /* + * Clears the card table (which currently contains the brick table) and updates the + * first object table. + * + * GR-54022: we should be able to avoid this pass and build the first object tables + * during planning and reset card tables once we detect that we are finished with a + * chunk during compaction. The remembered set bits are already set after planning. + */ + if (!AlignedHeapChunk.isEmpty(chunk)) { + RememberedSet.get().enableRememberedSetForChunk(chunk); + } // empty chunks will be freed or reset before reuse, no need to reinitialize here + + chunk = HeapChunk.getNext(chunk); + } + } finally { + oldCompactionRememberedSetsTimer.close(); } - timers.tenuredCompactingUpdatingRemSet.close(); } - /** After the collection, releases those chunks which are empty (typically at the end). */ + /** + * At the end of the collection, adds empty aligned chunks to be released (typically at the + * end). Unaligned chunks have already been added in {@link #fixupReferencesBeforeCompaction}. + */ @Override void releaseSpaces(ChunkReleaser chunkReleaser) { AlignedHeapChunk.AlignedHeader aChunk = space.getFirstAlignedHeapChunk(); while (aChunk.isNonNull()) { AlignedHeapChunk.AlignedHeader next = HeapChunk.getNext(aChunk); - if (HeapChunk.getTopPointer(aChunk).equal(AlignedHeapChunk.getObjectsStart(aChunk))) { + if (AlignedHeapChunk.isEmpty(aChunk)) { space.extractAlignedHeapChunk(aChunk); chunkReleaser.add(aChunk); } @@ -379,9 +428,8 @@ void swapSpaces() { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isInSpace(Pointer ptr) { - return HeapImpl.findPointerInSpace(space, ptr); + return space.contains(ptr); } @Override @@ -414,18 +462,6 @@ UnsignedWord computeObjectBytes() { return space.computeObjectBytes(); } - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - AlignedHeapChunk.AlignedHeader requestAlignedChunk() { - assert VMOperation.isGCInProgress() : "Should only be called from the collector."; - AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); - if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { - throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); - } - RememberedSet.get().enableRememberedSetForChunk(chunk); - return chunk; - } - @Override boolean verifyRememberedSets() { return HeapVerifier.verifyRememberedSet(space); @@ -436,9 +472,20 @@ boolean verifySpaces() { return HeapVerifier.verifySpace(space); } + @Override + void checkSanityBeforeCollection() { + assert markStack.isEmpty(); + } + + @Override + void checkSanityAfterCollection() { + assert markStack.isEmpty(); + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void tearDown() { + markStack.tearDown(); space.tearDown(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java index f4ae3f11f4c2..9962dff2e554 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java @@ -24,9 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY; -import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; - import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -38,8 +35,6 @@ import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.thread.VMOperation; -import com.oracle.svm.core.util.VMError; /** * An OldGeneration has two Spaces, {@link #fromSpace} for existing objects, and {@link #toSpace} @@ -109,7 +104,7 @@ void releaseSpaces(ChunkReleaser chunkReleaser) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void beginPromotion(YoungGeneration youngGen, boolean incrementalGc) { + void beginPromotion(boolean incrementalGc) { if (incrementalGc) { emptyFromSpaceIntoToSpace(); } @@ -161,9 +156,8 @@ void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor) { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isInSpace(Pointer ptr) { - return HeapImpl.findPointerInSpace(fromSpace, ptr) || HeapImpl.findPointerInSpace(toSpace, ptr); + return fromSpace.contains(ptr) || toSpace.contains(ptr); } @Override @@ -216,18 +210,6 @@ UnsignedWord computeObjectBytes() { return fromSpace.computeObjectBytes().add(toSpace.computeObjectBytes()); } - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - AlignedHeapChunk.AlignedHeader requestAlignedChunk() { - assert VMOperation.isGCInProgress() : "Should only be called from the collector."; - AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); - if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { - throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); - } - RememberedSet.get().enableRememberedSetForChunk(chunk); - return chunk; - } - @Override void checkSanityBeforeCollection() { assert toSpace.isEmpty() : "toSpace should be empty before a collection."; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index b4c38a5ce446..a57c9c69ffc1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -999,7 +999,7 @@ private void blackenDirtyCardRoots() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void beginPromotion(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); - heap.getOldGeneration().beginPromotion(heap.getYoungGeneration(), isIncremental); + heap.getOldGeneration().beginPromotion(isIncremental); if (isIncremental) { heap.getYoungGeneration().beginPromotion(); } @@ -1084,7 +1084,7 @@ private void promotePinnedObject(PinnedObjectImpl pinned) { boolean isAligned = ObjectHeaderImpl.isAlignedObject(referent); Header originalChunk = getChunk(referent, isAligned); Space originalSpace = HeapChunk.getSpace(originalChunk); - if (originalSpace.isFromSpace() || originalSpace.isCompactingOldSpace()) { + if (originalSpace.isFromSpace() || (originalSpace.isCompactingOldSpace() && completeCollection)) { boolean promoted = false; if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { promoted = heap.getYoungGeneration().promotePinnedObject(referent, originalChunk, isAligned, originalSpace); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index adb808376dfa..ab2387f2be24 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -751,10 +751,10 @@ private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAcc if (allowJavaHeapAccess) { // Accessing spaces and chunks is safe if we prevent a GC. - if (isInYoungGen(ptr)) { + if (youngGeneration.isInSpace(ptr)) { log.string("points into the young generation"); return true; - } else if (isInOldGen(ptr)) { + } else if (oldGeneration.isInSpace(ptr)) { log.string("points into the old generation"); return true; } @@ -769,51 +769,7 @@ private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAcc } boolean isInHeap(Pointer ptr) { - return isInImageHeap(ptr) || isInYoungGen(ptr) || isInOldGen(ptr); - } - - @Uninterruptible(reason = "Prevent that chunks are freed.") - private boolean isInYoungGen(Pointer ptr) { - if (findPointerInSpace(youngGeneration.getEden(), ptr)) { - return true; - } - - for (int i = 0; i < youngGeneration.getMaxSurvivorSpaces(); i++) { - if (findPointerInSpace(youngGeneration.getSurvivorFromSpaceAt(i), ptr)) { - return true; - } - if (findPointerInSpace(youngGeneration.getSurvivorToSpaceAt(i), ptr)) { - return true; - } - } - return false; - } - - @Uninterruptible(reason = "Prevent that chunks are freed.") - private boolean isInOldGen(Pointer ptr) { - return oldGeneration.isInSpace(ptr); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static boolean findPointerInSpace(Space space, Pointer p) { - AlignedHeapChunk.AlignedHeader aChunk = space.getFirstAlignedHeapChunk(); - while (aChunk.isNonNull()) { - Pointer start = AlignedHeapChunk.getObjectsStart(aChunk); - if (start.belowOrEqual(p) && p.belowThan(HeapChunk.getTopPointer(aChunk))) { - return true; - } - aChunk = HeapChunk.getNext(aChunk); - } - - UnalignedHeapChunk.UnalignedHeader uChunk = space.getFirstUnalignedHeapChunk(); - while (uChunk.isNonNull()) { - Pointer start = UnalignedHeapChunk.getObjectStart(uChunk); - if (start.belowOrEqual(p) && p.belowThan(HeapChunk.getTopPointer(uChunk))) { - return true; - } - uChunk = HeapChunk.getNext(uChunk); - } - return false; + return isInImageHeap(ptr) || youngGeneration.isInSpace(ptr) || oldGeneration.isInSpace(ptr); } private static boolean printTlabInfo(Log log, Pointer ptr) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java index 07eb0b3d9009..87435a8de47e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java @@ -196,6 +196,11 @@ private static boolean verifyAlignedChunks(Space space, AlignedHeader firstAlign success = false; } + if (aChunk.getShouldSweepInsteadOfCompact()) { + Log.log().string("Aligned chunk ").zhex(aChunk).string(" is marked for sweeping while this should only be used during collections.").newline(); + success = false; + } + OBJECT_VERIFIER.initialize(aChunk, WordFactory.nullPointer()); AlignedHeapChunk.walkObjects(aChunk, OBJECT_VERIFIER); aChunk = HeapChunk.getNext(aChunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index ba40e2a73e34..8639528db37f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -358,14 +358,14 @@ public static boolean hasRememberedSet(UnsignedWord header) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setMarked(Object o) { - if (!SerialGCOptions.useCompactingOldGen()) { + if (!SerialGCOptions.useCompactingOldGen()) { // not guarantee(): always folds, prevent call throw VMError.shouldNotReachHere("Only for compacting GC."); } UnsignedWord header = readHeaderFromObject(o); assert header.and(FORWARDED_OR_MARKED2_BIT).equal(0) : "forwarded or already marked"; /* * The remembered bit is already set if the object was in the old generation before, or - * unset if it was only just absorbed from the young generation. + * unset if it was only just absorbed from the young generation, in which case we set it. */ writeHeaderToObject(o, header.or(MARKED_BITS)); } @@ -398,7 +398,7 @@ static boolean isPointerToForwardedObject(Pointer p) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isForwardedHeader(UnsignedWord header) { - return header.and(REMSET_OR_MARKED1_BIT.or(FORWARDED_OR_MARKED2_BIT)).equal(FORWARDED_OR_MARKED2_BIT); + return header.and(MARKED_BITS).equal(FORWARDED_OR_MARKED2_BIT); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 03b8a05d6f4d..22c43d93962a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -24,11 +24,17 @@ */ package com.oracle.svm.core.genscavenge; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; + import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; +import com.oracle.svm.core.genscavenge.remset.RememberedSet; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.util.VMError; public abstract class OldGeneration extends Generation { OldGeneration(String name) { @@ -36,7 +42,7 @@ public abstract class OldGeneration extends Generation { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - abstract void beginPromotion(YoungGeneration youngGen, boolean incrementalGc); + abstract void beginPromotion(boolean incrementalGc); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) abstract void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor); @@ -44,9 +50,6 @@ public abstract class OldGeneration extends Generation { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) abstract boolean scanGreyObjects(boolean incrementalGc); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - abstract AlignedHeapChunk.AlignedHeader requestAlignedChunk(); - abstract void sweepAndCompact(Timers timers, ChunkReleaser chunkReleaser); abstract void releaseSpaces(ChunkReleaser chunkReleaser); @@ -58,7 +61,6 @@ public abstract class OldGeneration extends Generation { abstract UnsignedWord computeObjectBytes(); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) abstract boolean isInSpace(Pointer ptr); abstract boolean verifyRememberedSets(); @@ -67,4 +69,15 @@ public abstract class OldGeneration extends Generation { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) abstract void tearDown(); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + AlignedHeapChunk.AlignedHeader requestAlignedChunk() { + assert VMOperation.isGCInProgress() : "Should only be called from the collector."; + AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); + if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { + throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); + } + RememberedSet.get().enableRememberedSetForChunk(chunk); + return chunk; + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 7108b4234e99..477fb6aadb4c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -250,8 +250,9 @@ private static boolean willSurviveThisCollection(Object obj) { } static void updateForwardedRefs() { - Reference current = rememberedRefsList; + assert SerialGCOptions.useCompactingOldGen(); + Reference current = rememberedRefsList; while (current != null) { // Get the next node (the last node has a cyclic reference to self). Reference next = ReferenceInternals.getNextDiscovered(current); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java index 908884aa5fbb..8e839de19ffc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java @@ -112,7 +112,7 @@ public Integer getValue(OptionValues values) { /** Query these options only through an appropriate method. */ public static class ConcealedOptions { @Option(help = "Collect old generation by compacting in-place instead of copying.", type = OptionType.Expert) // - public static final HostedOptionKey CompactingOldGen = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey CompactingOldGen = new HostedOptionKey<>(true, SerialGCOptions::validateCompactingOldGen); @Option(help = "Determines if a remembered set is used, which is necessary for collecting the young and old generation independently.", type = OptionType.Expert) // public static final HostedOptionKey UseRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); @@ -127,16 +127,11 @@ private static void serialGCOnly(OptionKey optionKey) { } } - @Fold - public static boolean useRememberedSet() { - return !SubstrateOptions.UseEpsilonGC.getValue() && ConcealedOptions.UseRememberedSet.getValue(); - } - - @Fold - public static boolean useCompactingOldGen() { - if (SubstrateOptions.UseEpsilonGC.getValue() || !ConcealedOptions.CompactingOldGen.getValue()) { - return false; + private static void validateCompactingOldGen(HostedOptionKey compactingOldGen) { + if (!compactingOldGen.getValue()) { + return; } + serialGCOnly(compactingOldGen); if (!useRememberedSet()) { throw UserError.abort("%s requires %s.", SubstrateOptionsParser.commandArgument(ConcealedOptions.CompactingOldGen, "+"), SubstrateOptionsParser.commandArgument(ConcealedOptions.UseRememberedSet, "+")); @@ -145,6 +140,15 @@ public static boolean useCompactingOldGen() { throw UserError.abort("%s requires %s.", SubstrateOptionsParser.commandArgument(ConcealedOptions.CompactingOldGen, "+"), SubstrateOptionsParser.commandArgument(SerialAndEpsilonGCOptions.AlignedHeapChunkSize, "")); } - return true; + } + + @Fold + public static boolean useRememberedSet() { + return !SubstrateOptions.UseEpsilonGC.getValue() && ConcealedOptions.UseRememberedSet.getValue(); + } + + @Fold + public static boolean useCompactingOldGen() { + return !SubstrateOptions.UseEpsilonGC.getValue() && ConcealedOptions.CompactingOldGen.getValue(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 3af207f13c9a..cdc9b6b11abb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -177,6 +177,17 @@ public boolean walkObjects(ObjectVisitor visitor) { return true; } + boolean walkAlignedHeapChunks(AlignedHeapChunk.Visitor visitor) { + AlignedHeapChunk.AlignedHeader chunk = getFirstAlignedHeapChunk(); + while (chunk.isNonNull()) { + if (!visitor.visitChunk(chunk)) { + return false; + } + chunk = HeapChunk.getNext(chunk); + } + return true; + } + public void logUsage(Log log, boolean logIfEmpty) { UnsignedWord chunkBytes; if (isEdenSpace() && !VMOperation.isGCInProgress()) { @@ -526,17 +537,6 @@ void absorb(Space src) { assert src.isEmpty(); } - boolean walkAlignedHeapChunks(AlignedHeapChunk.Visitor visitor) { - AlignedHeapChunk.AlignedHeader chunk = getFirstAlignedHeapChunk(); - while (chunk.isNonNull()) { - if (!visitor.visitChunk(chunk)) { - return false; - } - chunk = HeapChunk.getNext(chunk); - } - return true; - } - /** * This value is only updated during a GC. Be careful when calling this method during a GC as it * might wrongly include chunks that will be freed at the end of the GC. @@ -596,4 +596,25 @@ private UnsignedWord computeUnalignedObjectBytes() { } return result; } + + boolean contains(Pointer p) { + AlignedHeapChunk.AlignedHeader aChunk = getFirstAlignedHeapChunk(); + while (aChunk.isNonNull()) { + Pointer start = AlignedHeapChunk.getObjectsStart(aChunk); + if (start.belowOrEqual(p) && p.belowThan(HeapChunk.getTopPointer(aChunk))) { + return true; + } + aChunk = HeapChunk.getNext(aChunk); + } + + UnalignedHeapChunk.UnalignedHeader uChunk = getFirstUnalignedHeapChunk(); + while (uChunk.isNonNull()) { + Pointer start = UnalignedHeapChunk.getObjectStart(uChunk); + if (start.belowOrEqual(p) && p.belowThan(HeapChunk.getTopPointer(uChunk))) { + return true; + } + uChunk = HeapChunk.getNext(uChunk); + } + return false; + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index 74302ea7cfd0..6d34182d2194 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -122,17 +122,16 @@ final class Timers { final Timer promotePinnedObjects = new Timer("promotePinnedObjects"); final Timer rootScan = new Timer("rootScan"); final Timer scanGreyObjects = new Timer("scanGreyObjects"); - final Timer tenuredPlanning = new Timer("tenuredPlanning"); - final Timer tenuredFixing = new Timer("tenuredFixing"); - final Timer tenuredFixingAlignedChunks = new Timer("tenuredFixingAlignedChunks"); - final Timer tenuredFixingImageHeap = new Timer("tenuredFixingImageHeap"); - final Timer tenuredFixingThreadLocal = new Timer("tenuredFixingThreadLocal"); - final Timer tenuredFixingRuntimeCodeCache = new Timer("tenuredFixingRuntimeCodeCache"); - final Timer tenuredFixingStack = new Timer("tenuredFixingStack"); - final Timer tenuredFixingUnalignedChunks = new Timer("tenuredFixingUnalignedChunks"); - final Timer tenuredCompacting = new Timer("tenuredCompacting"); - final Timer tenuredCompactingChunks = new Timer("tenuredCompactingChunks"); - final Timer tenuredCompactingUpdatingRemSet = new Timer("tenuredCompactingUpdatingRemSet"); + final Timer oldPlanning = new Timer("oldPlanning"); + final Timer oldFixup = new Timer("oldFixup"); + final Timer oldFixupAlignedChunks = new Timer("oldFixupAlignedChunks"); + final Timer oldFixupImageHeap = new Timer("oldFixupImageHeap"); + final Timer oldFixupThreadLocals = new Timer("oldFixupThreadLocals"); + final Timer oldFixupRuntimeCodeCache = new Timer("oldFixupRuntimeCodeCache"); + final Timer oldFixupStack = new Timer("oldFixupStack"); + final Timer oldFixupUnalignedChunks = new Timer("oldFixupUnalignedChunks"); + final Timer oldCompaction = new Timer("oldCompaction"); + final Timer oldCompactionRememberedSets = new Timer("oldCompactionRememberedSets"); final Timer releaseSpaces = new Timer("releaseSpaces"); final Timer verifyAfter = new Timer("verifyAfter"); final Timer verifyBefore = new Timer("verifyBefore"); @@ -161,17 +160,16 @@ void resetAllExceptMutator() { blackenDirtyCardRoots.reset(); scanGreyObjects.reset(); if (SerialGCOptions.useCompactingOldGen()) { - tenuredPlanning.reset(); - tenuredFixing.reset(); - tenuredFixingAlignedChunks.reset(); - tenuredFixingImageHeap.reset(); - tenuredFixingThreadLocal.reset(); - tenuredFixingRuntimeCodeCache.reset(); - tenuredFixingStack.reset(); - tenuredFixingUnalignedChunks.reset(); - tenuredCompacting.reset(); - tenuredCompactingChunks.reset(); - tenuredCompactingUpdatingRemSet.reset(); + oldPlanning.reset(); + oldFixup.reset(); + oldFixupAlignedChunks.reset(); + oldFixupImageHeap.reset(); + oldFixupThreadLocals.reset(); + oldFixupRuntimeCodeCache.reset(); + oldFixupStack.reset(); + oldFixupUnalignedChunks.reset(); + oldCompaction.reset(); + oldCompactionRememberedSets.reset(); } cleanCodeCache.reset(); referenceObjects.reset(); @@ -199,17 +197,16 @@ void logAfterCollection(Log log) { logOneTimer(log, " ", blackenDirtyCardRoots); logOneTimer(log, " ", scanGreyObjects); if (SerialGCOptions.useCompactingOldGen()) { - logOneTimer(log, " ", tenuredPlanning); - logOneTimer(log, " ", tenuredFixing); - logOneTimer(log, " ", tenuredFixingAlignedChunks); - logOneTimer(log, " ", tenuredFixingImageHeap); - logOneTimer(log, " ", tenuredFixingThreadLocal); - logOneTimer(log, " ", tenuredFixingRuntimeCodeCache); - logOneTimer(log, " ", tenuredFixingStack); - logOneTimer(log, " ", tenuredFixingUnalignedChunks); - logOneTimer(log, " ", tenuredCompacting); - logOneTimer(log, " ", tenuredCompactingChunks); - logOneTimer(log, " ", tenuredCompactingUpdatingRemSet); + logOneTimer(log, " ", oldPlanning); + logOneTimer(log, " ", oldFixup); + logOneTimer(log, " ", oldFixupAlignedChunks); + logOneTimer(log, " ", oldFixupImageHeap); + logOneTimer(log, " ", oldFixupThreadLocals); + logOneTimer(log, " ", oldFixupRuntimeCodeCache); + logOneTimer(log, " ", oldFixupStack); + logOneTimer(log, " ", oldFixupUnalignedChunks); + logOneTimer(log, " ", oldCompaction); + logOneTimer(log, " ", oldCompactionRememberedSets); } logOneTimer(log, " ", cleanCodeCache); logOneTimer(log, " ", referenceObjects); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 6f8580fbdb88..847428e4ee2e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -26,6 +26,7 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -335,4 +336,19 @@ AlignedHeapChunk.AlignedHeader requestAlignedSurvivorChunk() { public void checkSanityAfterCollection() { assert eden.isEmpty() : "eden should be empty after a collection."; } + + boolean isInSpace(Pointer ptr) { + if (getEden().contains(ptr)) { + return true; + } + for (int i = 0; i < getMaxSurvivorSpaces(); i++) { + if (getSurvivorFromSpaceAt(i).contains(ptr)) { + return true; + } + if (getSurvivorToSpaceAt(i).contains(ptr)) { + return true; + } + } + return false; + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java index 915fd0faae79..f32f0fd5e50a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java @@ -27,14 +27,12 @@ import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import static jdk.vm.ci.code.CodeUtil.K; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.struct.UniqueLocationIdentity; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -42,12 +40,19 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.word.ObjectAccess; +/** + * LIFO stack for objects to visit during the mark phase. Without it, recursive calls could exhaust + * the {@linkplain com.oracle.svm.core.stack.StackOverflowCheck yellow zone stack space} during GC. + */ public final class MarkStack { - private static final int SEGMENT_SIZE = 64 * K - /* for any malloc overhead */ 32; + private static final int SEGMENT_SIZE = 64 * K - /* avoid potential malloc() overallocation */ 64; @Fold static int entriesPerSegment() { @@ -92,7 +97,7 @@ public Object pop() { Segment t = top; top = top.getNext(); cursor = entriesPerSegment(); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(t); + NullableNativeMemory.free(t); } else { // keep a single segment } @@ -104,7 +109,7 @@ public Object pop() { @AlwaysInline("GC performance") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public boolean isEmpty() { - assert cursor != 0 || top.getNext().isNull() : "should see cursor == 0 only with a single segment"; + assert cursor != 0 || top.isNull() || top.getNext().isNull() : "should see cursor == 0 only with a single segment (or none)"; return top.isNull() || cursor == 0; } @@ -123,7 +128,8 @@ interface Segment extends PointerBase { @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private static Segment allocateSegment(Segment next) { UnsignedWord size = WordFactory.unsigned(SEGMENT_SIZE); - Segment segment = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); + Segment segment = NullableNativeMemory.malloc(size, NmtCategory.GC); + VMError.guarantee(segment.isNonNull(), "Could not allocate mark stack memory: malloc() returned null."); segment.setNext(next); return segment; } @@ -134,4 +140,13 @@ private static UnsignedWord getOffsetAtIndex(int index) { int refSize = ConfigurationValues.getObjectLayout().getReferenceSize(); return WordFactory.unsigned(index).multiply(refSize).add(SizeOf.unsigned(Segment.class)); } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void tearDown() { + if (top.isNonNull()) { + assert top.getNext().isNull(); + NullableNativeMemory.free(top); + top = WordFactory.nullPointer(); + } + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectMoveInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectMoveInfo.java index 33503e44c558..4ff328417040 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectMoveInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectMoveInfo.java @@ -26,6 +26,7 @@ import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import com.oracle.svm.core.util.VMError; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -44,14 +45,18 @@ /** * {@link PlanningVisitor} decides where objects will be moved and uses the methods of this class to * store this information in a structure directly before each contiguous sequence of live objects, - * where there is always a sufficiently large gap formed by unreachable objects. This avoids - * reserving space in objects that is needed only for compaction, but also requires more passes over - * the heap and more expensive accesses to determine the new location of objects. + * where there is always a sufficiently large gap formed by unreachable objects (because the + * structure fits the minimum object size). This avoids reserving space in objects that is needed + * only for compaction, but also requires more passes over the heap and more expensive accesses to + * determine the new location of objects. *

- * The structure contains the following fields: + * The structure consists of the following fields, which are sized according to whether 8-byte or + * compressed 4-byte object references are in use, and in the latter case themselves use compression + * by shifting the (zero) object alignment bits. *

    *
  • New location:
    - * Provides the new address of the sequence of objects after compaction.
    • + * Provides the new address of the sequence of objects after compaction. This address can be outside + * of the current chunk. *
  • Size:
    * The number of live object bytes that the sequence consists of.
    • *
  • Next sequence offset:
    @@ -59,11 +64,10 @@ * This forms a singly-linked list over all object sequences (and their structures) in an aligned * chunk. An offset of 0 means that there are no more objects in the chunk.
    • *
- * - * Binary layout (sizes with 8-byte/4-byte object references): + * The binary layout is as follows, with sizes given for both 8-byte/4-byte object references. The + * fields are arranged so that accesses to them are aligned. * * 
- * With 8-byte object references:
  * ------------------------+======================+==============+=========================+-------------------
  *  ... gap (unused bytes) | new location (8B/4B) | size (4B/2B) | next seq offset (4B/2B) | live objects ...
  * ------------------------+======================+==============+=========================+-------------------
@@ -73,7 +77,7 @@
 public final class ObjectMoveInfo {
 
     /**
-     * The maximum size of aligned heap chunks, based on 2 bytes for gap size and next object
+     * The maximum size of aligned heap chunks, based on 2 bytes for the size and the next object
      * sequence offset and an object alignment of 8 bytes.
      */
     public static final int MAX_CHUNK_SIZE = ~(~0xffff * 8) + 1;
@@ -172,7 +176,7 @@ public static void walkObjects(AlignedHeapChunk.AlignedHeader chunkHeader, Objec
                 Object obj = p.toObject();
                 UnsignedWord objSize = LayoutEncoding.getSizeFromObjectInlineInGC(obj);
                 if (!visitor.visitObjectInline(obj)) {
-                    return;
+                    throw VMError.shouldNotReachHereAtRuntime();
                 }
                 p = p.add(objSize);
             }
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java
index 69bb6657ed94..161e3ab686b6 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java
@@ -62,7 +62,9 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole
         Object original = p.toObject();
         if (ObjectHeaderImpl.isAlignedObject(original)) {
             Pointer newLocation = ObjectMoveInfo.getNewObjectAddress(p);
-            assert newLocation.isNonNull() || holderObject == null || holderObject instanceof Reference;
+            assert newLocation.isNonNull() //
+                            || holderObject == null // references from CodeInfo, invalidated or weak
+                            || holderObject instanceof Reference; // cleared referent
 
             obj = newLocation.toObject();
             ReferenceAccess.singleton().writeObjectAt(objRef, obj, compressed);
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java
index f728c97c0427..27ef374685f1 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java
@@ -54,6 +54,11 @@ public final class PlanningVisitor implements AlignedHeapChunk.Visitor {
     public PlanningVisitor() {
     }
 
+    public void init(Space space) {
+        allocChunk = space.getFirstAlignedHeapChunk();
+        allocPointer = AlignedHeapChunk.getObjectsStart(allocChunk);
+    }
+
     @Override
     public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
         boolean sweeping = chunk.getShouldSweepInsteadOfCompact();
@@ -71,26 +76,26 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
 
         BrickTable.setEntry(chunk, brickIndex, objSeq);
 
-        Pointer p = AlignedHeapChunk.getObjectsStart(chunk);
+        Pointer p = objSeq;
         while (p.belowThan(initialTop)) {
             Word header = ObjectHeaderImpl.readHeaderFromPointer(p);
-            Object obj = p.toObject();
 
             UnsignedWord objSize;
             if (ObjectHeaderImpl.isForwardedHeader(header)) {
                 /*
                  * If an object was copied from a chunk that won't be swept and forwarding was put
-                 * in place, it was because we needed to add an identity hash code field.
+                 * in place, it was because we needed to add an identity hash code field to the
+                 * object, and we need the object's original size here.
                  */
                 assert !sweeping && ConfigurationValues.getObjectLayout().isIdentityHashFieldOptional();
                 Object forwardedObj = ObjectHeaderImpl.getObjectHeaderImpl().getForwardedObject(p, header);
                 objSize = LayoutEncoding.getSizeFromObjectWithoutOptionalIdHashFieldInGC(forwardedObj);
             } else {
-                objSize = LayoutEncoding.getSizeFromObjectInlineInGC(obj);
+                objSize = LayoutEncoding.getSizeFromObjectInlineInGC(p.toObject());
             }
 
             if (ObjectHeaderImpl.isMarkedHeader(header)) {
-                ObjectHeaderImpl.unsetMarkedAndKeepRememberedSetBit(obj);
+                ObjectHeaderImpl.unsetMarkedAndKeepRememberedSetBit(p.toObject());
 
                 /*
                  * Adding the optional identity hash field would increase an object's size, so we
@@ -111,6 +116,7 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
                 }
 
                 objSeqSize = objSeqSize.add(objSize);
+
             } else { // not marked, i.e. not alive and start of a gap of yet unknown size
                 if (objSeqSize.notEqual(0)) { // end of an object sequence
                     Pointer newAddress = sweeping ? objSeq : allocate(objSeqSize);
@@ -132,8 +138,7 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
         }
         assert gapSize.equal(0) || objSeqSize.equal(0);
 
-        /* A gap at the end of the chunk requires updating the chunk top. */
-        if (gapSize.notEqual(0)) {
+        if (gapSize.notEqual(0)) { // truncate gap at chunk end
             chunk.setTopOffset(chunk.getTopOffset().subtract(gapSize));
         } else if (objSeqSize.notEqual(0)) {
             Pointer newAddress = sweeping ? objSeq : allocate(objSeqSize);
@@ -147,6 +152,9 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
              * unused memory in the chunks before, but the order of objects must stay the same
              * across all chunks. If chunks end up completely empty however, they will be released
              * after compaction.
+             *
+             * GR-54021: it should be possible to avoid this limitation by sweeping chunks without
+             * ObjectMoveInfo and brick tables and potentially even do the sweeping right here.
              */
             this.allocChunk = chunk;
             this.allocPointer = HeapChunk.getTopPointer(chunk);
@@ -168,15 +176,11 @@ private Pointer allocate(UnsignedWord size) {
         if (allocPointer.aboveThan(AlignedHeapChunk.getObjectsEnd(allocChunk))) {
             allocChunk = HeapChunk.getNext(allocChunk);
             assert allocChunk.isNonNull();
+            assert !allocChunk.getShouldSweepInsteadOfCompact();
 
             p = AlignedHeapChunk.getObjectsStart(allocChunk);
             allocPointer = p.add(size);
         }
         return p;
     }
-
-    public void init(Space space) {
-        allocChunk = space.getFirstAlignedHeapChunk();
-        allocPointer = AlignedHeapChunk.getObjectsStart(allocChunk);
-    }
 }
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheFixupWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/RuntimeCodeCacheFixupWalker.java
similarity index 87%
rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheFixupWalker.java
rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/RuntimeCodeCacheFixupWalker.java
index 920f2519ae50..ae1b72d8685e 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheFixupWalker.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/RuntimeCodeCacheFixupWalker.java
@@ -22,7 +22,7 @@
  * or visit www.oracle.com if you need additional information or have any
  * questions.
  */
-package com.oracle.svm.core.genscavenge;
+package com.oracle.svm.core.genscavenge.compacting;
 
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
@@ -30,14 +30,15 @@
 import com.oracle.svm.core.code.CodeInfo;
 import com.oracle.svm.core.code.RuntimeCodeCache.CodeInfoVisitor;
 import com.oracle.svm.core.code.RuntimeCodeInfoAccess;
-import com.oracle.svm.core.genscavenge.compacting.ObjectRefFixupVisitor;
+import com.oracle.svm.core.genscavenge.SerialGCOptions;
 
 /** Before compaction, updates references from {@link CodeInfo} structures. */
-final class RuntimeCodeCacheFixupWalker implements CodeInfoVisitor {
+public final class RuntimeCodeCacheFixupWalker implements CodeInfoVisitor {
     private final ObjectRefFixupVisitor visitor;
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    RuntimeCodeCacheFixupWalker(ObjectRefFixupVisitor visitor) {
+    public RuntimeCodeCacheFixupWalker(ObjectRefFixupVisitor visitor) {
+        assert SerialGCOptions.useCompactingOldGen();
         this.visitor = visitor;
     }
 
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/SweepingVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/SweepingVisitor.java
index 8cf5c7c84eda..8f0c3d18040e 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/SweepingVisitor.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/SweepingVisitor.java
@@ -32,6 +32,7 @@
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatObjectNode;
+import com.oracle.svm.core.heap.FillerArray;
 import com.oracle.svm.core.heap.FillerObject;
 import com.oracle.svm.core.hub.LayoutEncoding;
 import com.oracle.svm.core.util.UnsignedUtils;
@@ -45,14 +46,18 @@
  * cannot encounter them (and their broken references).
  */
 public final class SweepingVisitor implements ObjectMoveInfo.Visitor {
+    private static final Class ARRAY_CLASS = FillerArray.class;
+    private static final JavaKind ARRAY_ELEMENT_KIND = JavaKind.Int;
+    private static final int ARRAY_ELEMENT_SIZE = ARRAY_ELEMENT_KIND.getByteCount();
+
     @Fold
-    static int byteArrayMinSize() {
-        return NumUtil.safeToInt(ConfigurationValues.getObjectLayout().getArraySize(JavaKind.Byte, 0, false));
+    static int arrayMinSize() {
+        return NumUtil.safeToInt(ConfigurationValues.getObjectLayout().getArraySize(ARRAY_ELEMENT_KIND, 0, false));
     }
 
     @Fold
-    static int byteArrayBaseOffset() {
-        return ConfigurationValues.getObjectLayout().getArrayBaseOffset(JavaKind.Byte);
+    static int arrayBaseOffset() {
+        return ConfigurationValues.getObjectLayout().getArrayBaseOffset(ARRAY_ELEMENT_KIND);
     }
 
     @Override
@@ -69,9 +74,9 @@ public boolean visit(Pointer objSeq, UnsignedWord size, Pointer newAddress, Poin
 
     private static void writeFillerObjectAt(Pointer p, UnsignedWord size) {
         assert size.aboveThan(0);
-        if (size.aboveOrEqual(byteArrayMinSize())) {
-            int length = UnsignedUtils.safeToInt(size.subtract(byteArrayBaseOffset()));
-            FormatArrayNode.formatArray(p, byte[].class, length, true, false, WITH_GARBAGE_IF_ASSERTIONS_ENABLED, false);
+        if (size.aboveOrEqual(arrayMinSize())) {
+            int length = UnsignedUtils.safeToInt(size.subtract(arrayBaseOffset()).unsignedDivide(ARRAY_ELEMENT_SIZE));
+            FormatArrayNode.formatArray(p, ARRAY_CLASS, length, true, false, WITH_GARBAGE_IF_ASSERTIONS_ENABLED, false);
         } else {
             FormatObjectNode.formatObject(p, FillerObject.class, true, WITH_GARBAGE_IF_ASSERTIONS_ENABLED, false);
         }
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java
index 60db455f6182..7efb7222bf00 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java
@@ -276,7 +276,7 @@ static UnsignedWord getCardTableLimitOffset() {
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    private static Pointer getCardTableStart(AlignedHeader chunk) {
+    static Pointer getCardTableStart(AlignedHeader chunk) {
         return getCardTableStart(HeapChunk.asPointer(chunk));
     }
 
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/BrickTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/BrickTable.java
index afc377237e7b..e6e61c8d3f05 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/BrickTable.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/BrickTable.java
@@ -90,7 +90,7 @@ public static void setEntry(AlignedHeapChunk.AlignedHeader chunk, UnsignedWord i
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     private static Pointer getBrickTableStart(AlignedHeapChunk.AlignedHeader chunk) {
-        return HeapChunk.asPointer(chunk).add(AlignedChunkRememberedSet.getCardTableStartOffset());
+        return AlignedChunkRememberedSet.getCardTableStart(chunk);
     }
 
     private BrickTable() {
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java
index d9fb3d7e1d3d..26aa7729e4bb 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java
@@ -45,8 +45,10 @@
 import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.core.util.UnsignedUtils;
 
+import jdk.graal.compiler.api.directives.GraalDirectives;
 import jdk.graal.compiler.core.common.SuppressFBWarnings;
 import jdk.graal.compiler.nodes.extended.BranchProbabilityNode;
+import jdk.graal.compiler.replacements.ReplacementsUtil;
 import jdk.graal.compiler.word.Word;
 
 /**
@@ -118,7 +120,13 @@ private static boolean isClean(Pointer table, UnsignedWord index) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private static int readEntry(Pointer table, UnsignedWord index) {
-        return table.readByte(index);
+        byte entry = table.readByte(index);
+        if (GraalDirectives.inIntrinsic()) {
+            ReplacementsUtil.dynamicAssert(entry == CLEAN_ENTRY || entry == DIRTY_ENTRY, "valid entry");
+        } else {
+            assert entry == CLEAN_ENTRY || entry == DIRTY_ENTRY;
+        }
+        return entry;
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)