From 1f567b6af3b0454b25c1ff61046ae3ab32fec816 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 16 Jul 2024 17:27:25 +0200 Subject: [PATCH 1/5] Improve serial GC heap verification. Other small GC-related cleanups and improvements. --- .../core/genscavenge/AlignedHeapChunk.java | 3 +- .../oracle/svm/core/genscavenge/GCImpl.java | 28 +++++++-- .../svm/core/genscavenge/HeapVerifier.java | 35 ++++++++--- .../svm/core/genscavenge/ImageHeapInfo.java | 10 +-- .../oracle/svm/core/genscavenge/Space.java | 3 +- .../genscavenge/ThreadLocalAllocation.java | 9 ++- .../remset/AlignedChunkRememberedSet.java | 21 +++++-- .../core/genscavenge/remset/CardTable.java | 61 +++++++++++++------ .../remset/CardTableBasedRememberedSet.java | 4 +- .../genscavenge/remset/NoRememberedSet.java | 2 +- .../genscavenge/remset/RememberedSet.java | 2 +- .../remset/UnalignedChunkRememberedSet.java | 7 ++- .../oracle/svm/core/heap/OutOfMemoryUtil.java | 12 +--- 13 files changed, 130 insertions(+), 67 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index f8d34e8e5dce..21217efa4443 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -92,7 +92,8 @@ public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) { } public static void reset(AlignedHeader chunk) { - initialize(chunk, HeapChunk.getEndOffset(chunk)); + assert HeapChunk.getEndOffset(chunk).rawValue() == SerialAndEpsilonGCOptions.AlignedHeapChunkSize.getValue(); + initialize(chunk, WordFactory.unsigned(SerialAndEpsilonGCOptions.AlignedHeapChunkSize.getValue())); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index c149d9fe455f..e551d6633621 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -28,8 +28,6 @@ import java.lang.ref.Reference; -import com.oracle.svm.core.deopt.DeoptimizationSlotPacking; -import com.oracle.svm.core.interpreter.InterpreterSupport; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -56,6 +54,7 @@ import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.code.RuntimeCodeInfoAccess; import com.oracle.svm.core.code.RuntimeCodeInfoMemory; +import com.oracle.svm.core.deopt.DeoptimizationSlotPacking; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -79,6 +78,7 @@ import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner; import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.interpreter.InterpreterSupport; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.jfr.JfrGCWhen; import com.oracle.svm.core.jfr.JfrTicks; @@ -235,9 +235,13 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) || GenScavengeMemoryPoolMXBeans.singleton().notifyBeforeCollection(); HeapImpl.getAccounting().notifyBeforeCollection(); + verifyBeforeGC(); + boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); data.setOutOfMemory(outOfMemory); + verifyAfterGC(); + HeapImpl.getAccounting().notifyAfterCollection(); GenScavengeMemoryPoolMXBeans.singleton().notifyAfterCollection(); @@ -313,11 +317,7 @@ private boolean doCollectOnce(GCCause cause, long requestingNanoTime, boolean co Timer collectionTimer = timers.collection.open(); try { - if (!followsIncremental) { // we would have verified the heap after the incremental GC - verifyBeforeGC(); - } doCollectCore(!complete); - verifyAfterGC(); if (complete) { lastWholeHeapExaminedTimeMillis = System.currentTimeMillis(); } @@ -337,6 +337,10 @@ private boolean doCollectOnce(GCCause cause, long requestingNanoTime, boolean co private void verifyBeforeGC() { if (SubstrateGCOptions.VerifyHeap.getValue() && SerialGCOptions.VerifyBeforeGC.getValue()) { + if (SubstrateGCOptions.VerboseGC.getValue()) { + printGCPrefixAndTime().string("Verifying Before GC ").newline(); + } + Timer verifyBeforeTimer = timers.verifyBefore.open(); try { boolean success = true; @@ -351,11 +355,19 @@ private void verifyBeforeGC() { } finally { verifyBeforeTimer.close(); } + + if (SubstrateGCOptions.VerboseGC.getValue()) { + printGCPrefixAndTime().string("Verifying Before GC ").rational(timers.verifyBefore.getMeasuredNanos(), TimeUtils.nanosPerMilli, 3).string("ms").newline(); + } } } private void verifyAfterGC() { if (SubstrateGCOptions.VerifyHeap.getValue() && SerialGCOptions.VerifyAfterGC.getValue()) { + if (SubstrateGCOptions.VerboseGC.getValue()) { + printGCPrefixAndTime().string("Verifying After GC ").newline(); + } + Timer verifyAfterTime = timers.verifyAfter.open(); try { boolean success = true; @@ -370,6 +382,10 @@ private void verifyAfterGC() { } finally { verifyAfterTime.close(); } + + if (SubstrateGCOptions.VerboseGC.getValue()) { + printGCPrefixAndTime().string("Verifying After GC ").rational(timers.verifyAfter.getMeasuredNanos(), TimeUtils.nanosPerMilli, 3).string("ms").newline(); + } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java index 87435a8de47e..d403f9c7185f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java @@ -121,7 +121,7 @@ private static boolean verifyRememberedSets() { * reasonable state. Now, we can verify the remembered sets without having to worry about * basic heap consistency. */ - if (!SerialGCOptions.useRememberedSet() || !SerialGCOptions.VerifyRememberedSet.getValue()) { + if (!SerialGCOptions.useRememberedSet()) { return true; } @@ -188,6 +188,7 @@ private static boolean verifyChunkList(Space space, String kind, HeapChunk.Heade private static boolean verifyAlignedChunks(Space space, AlignedHeader firstAlignedHeapChunk) { boolean success = true; + AlignedHeader aChunk = firstAlignedHeapChunk; while (aChunk.isNonNull()) { if (space != aChunk.getSpace()) { @@ -215,6 +216,7 @@ private static boolean verifyUnalignedChunks(Space space, UnalignedHeader firstU private static boolean verifyUnalignedChunks(Space space, UnalignedHeader firstUnalignedHeapChunk, UnalignedHeader lastUnalignedHeapChunk) { boolean success = true; + UnalignedHeader uChunk = firstUnalignedHeapChunk; while (uChunk.isNonNull()) { if (space != uChunk.getSpace()) { @@ -255,6 +257,11 @@ private static boolean verifyObject(Object obj, AlignedHeader aChunk, UnalignedH return false; } + if (!HeapImpl.getHeap().getObjectHeader().isEncodedObjectHeader(header)) { + Log.log().string("Object ").zhex(ptr).string(" does not have a valid hub: ").zhex(header).newline(); + return false; + } + if (ObjectHeaderImpl.isForwardedHeader(header)) { Log.log().string("Object ").zhex(ptr).string(" has a forwarded header: ").zhex(header).newline(); return false; @@ -311,12 +318,6 @@ private static boolean verifyObject(Object obj, AlignedHeader aChunk, UnalignedH } } - DynamicHub hub = KnownIntrinsics.readHub(obj); - if (!HeapImpl.getHeapImpl().isInImageHeap(hub)) { - Log.log().string("Object ").zhex(ptr).string(" references a hub that is not in the image heap: ").zhex(Word.objectToUntrackedPointer(hub)).newline(); - return false; - } - return verifyReferences(obj); } @@ -361,13 +362,31 @@ private static boolean verifyReference(Object parentObject, Pointer reference, P return false; } - if (!ObjectHeaderImpl.getObjectHeaderImpl().pointsToObjectHeader(referencedObject)) { + Word header = ObjectHeader.readHeaderFromPointer(referencedObject); + if (!ObjectHeaderImpl.getObjectHeaderImpl().isEncodedObjectHeader(header)) { Log.log().string("Object reference at ").zhex(reference).string(" does not point to a Java object or the object header of the Java object is invalid: ").zhex(referencedObject) .string(". "); printParent(parentObject); return false; } + if (ObjectHeaderImpl.isAlignedHeader(header)) { + AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(referencedObject); + if (referencedObject.belowThan(AlignedHeapChunk.getObjectsStart(chunk)) || referencedObject.aboveOrEqual(HeapChunk.getTopPointer(chunk))) { + Log.log().string("Object reference ").zhex(reference).string(" points to ").zhex(referencedObject).string(", which is outside the usable part of the corresponding aligned chunk."); + printParent(parentObject); + return false; + } + } else { + assert ObjectHeaderImpl.isUnalignedHeader(header); + UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunkFromObjectPointer(referencedObject); + if (referencedObject != UnalignedHeapChunk.getObjectStart(chunk)) { + Log.log().string("Object reference ").zhex(reference).string(" points to ").zhex(referencedObject).string(", which is outside the usable part of the corresponding unaligned chunk."); + printParent(parentObject); + return false; + } + } + return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java index d81eb32f5be6..0cfac0f24a95 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java @@ -157,19 +157,13 @@ public boolean isInReadOnlyHugePartition(Pointer ptr) { } /** - * This method only returns the correct result for pointers that point to the the start of an + * This method only returns the correct result for pointers that point to the start of an * object. This is sufficient for all our current use cases. This code must be as fast as * possible as the GC uses it for every visited reference. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isInImageHeap(Pointer objectPointer) { - boolean result; - if (objectPointer.isNull()) { - result = false; - } else { - result = objectPointer.aboveOrEqual(Word.objectToUntrackedPointer(firstObject)) && objectPointer.belowOrEqual(Word.objectToUntrackedPointer(lastObject)); - } - return result; + return objectPointer.aboveOrEqual(Word.objectToUntrackedPointer(firstObject)) && objectPointer.belowOrEqual(Word.objectToUntrackedPointer(lastObject)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index cdc9b6b11abb..12b145a9cc90 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -421,6 +421,7 @@ private Object copyAlignedObject(Object originalObj) { if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(header))) { addIdentityHashField = true; copySize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, true); + assert copySize.aboveOrEqual(originalSize); } } @@ -457,7 +458,7 @@ private Object copyAlignedObject(Object originalObj) { * and the first object table must be updated (even when copying from old to old). */ AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); - RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy, copySize); } } return copy; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index 02572680033b..c6f2c329c7d5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -44,6 +44,7 @@ import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode; @@ -72,6 +73,7 @@ import com.oracle.svm.core.threadlocal.FastThreadLocalBytes; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; +import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.api.replacements.Fold; @@ -425,11 +427,14 @@ private static UnsignedWord availableTlabMemory(Descriptor allocator) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void guaranteeZeroed(Pointer memory, UnsignedWord size) { + VMError.guarantee(UnsignedUtils.isAMultiple(size, WordFactory.unsigned(ConfigurationValues.getTarget().wordSize))); + Pointer pos = memory; Pointer end = memory.add(size); while (pos.belowThan(end)) { - VMError.guarantee(pos.readByte(0) == 0); - pos = pos.add(1); + Word v = pos.readWord(0); + VMError.guarantee(v.equal(0)); + pos = pos.add(ConfigurationValues.getTarget().wordSize); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index 558e1d046741..0a65ad6feb70 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -95,11 +95,14 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkP @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { + public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj, UnsignedWord objSize) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); - Pointer startOffset = Word.objectToUntrackedPointer(obj).subtract(objectsStart); - Pointer endOffset = LayoutEncoding.getObjectEndInGC(obj).subtract(objectsStart); + + Word objPtr = Word.objectToUntrackedPointer(obj); + UnsignedWord startOffset = objPtr.subtract(objectsStart); + UnsignedWord endOffset = startOffset.add(objSize); + FirstObjectTable.setTableForObject(fotStart, startOffset, endOffset); ObjectHeaderImpl.setRememberedSetBit(obj); } @@ -115,8 +118,9 @@ public static void enableRememberedSet(AlignedHeader chunk) { Pointer top = HeapChunk.getTopPointer(chunk); while (offset.belowThan(top)) { Object obj = offset.toObject(); - enableRememberedSetForObject(chunk, obj); - offset = offset.add(LayoutEncoding.getSizeFromObjectInGC(obj)); + UnsignedWord objSize = LayoutEncoding.getSizeFromObjectInGC(obj); + enableRememberedSetForObject(chunk, obj, objSize); + offset = offset.add(objSize); } } @@ -211,7 +215,7 @@ private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, public static boolean verify(AlignedHeader chunk) { boolean success = true; - success &= CardTable.verify(getCardTableStart(chunk), AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getTopPointer(chunk)); + success &= CardTable.verify(getCardTableStart(chunk), getCardTableEnd(chunk), AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getTopPointer(chunk)); success &= FirstObjectTable.verify(getFirstObjectTableStart(chunk), AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getTopPointer(chunk)); return success; } @@ -285,6 +289,11 @@ private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer getCardTableEnd(AlignedHeader chunk) { + return getCardTableStart(chunk).add(getCardTableSize()); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(AlignedHeader chunk) { return getFirstObjectTableStart(HeapChunk.asPointer(chunk)); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index 26aa7729e4bb..18a130b8a55f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -34,6 +34,7 @@ import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.HeapImpl; +import com.oracle.svm.core.genscavenge.SerialGCOptions; import com.oracle.svm.core.genscavenge.graal.BarrierSnippets; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; @@ -151,28 +152,42 @@ public static UnsignedWord indexLimitForMemorySize(UnsignedWord memorySize) { return CardTable.memoryOffsetToIndex(roundedMemory); } - public static boolean verify(Pointer cardTableStart, Pointer objectsStart, Pointer objectsLimit) { + public static boolean verify(Pointer cardTableStart, Pointer cardTableEnd, Pointer objectsStart, Pointer objectsLimit) { boolean success = true; - Pointer curPtr = objectsStart; - while (curPtr.belowThan(objectsLimit)) { - // As we only use imprecise card marking at the moment, only the card at the address of - // the object may be dirty. - Object obj = curPtr.toObject(); - UnsignedWord cardTableIndex = memoryOffsetToIndex(curPtr.subtract(objectsStart)); - if (isClean(cardTableStart, cardTableIndex)) { - CARD_TABLE_VERIFICATION_VISITOR.initialize(obj, cardTableStart, objectsStart); - InteriorObjRefWalker.walkObject(obj, CARD_TABLE_VERIFICATION_VISITOR); - success &= CARD_TABLE_VERIFICATION_VISITOR.success; - - DynamicHub hub = KnownIntrinsics.readHub(obj); - if (hub.isReferenceInstanceClass()) { - // The referent field of java.lang.Reference is excluded from the reference map, - // so we need to verify it separately. - Reference ref = (Reference) obj; - success &= verifyReferent(ref, cardTableStart, objectsStart); + if (SerialGCOptions.VerifyRememberedSet.getValue()) { + Pointer curPtr = objectsStart; + while (curPtr.belowThan(objectsLimit)) { + // As we only use imprecise card marking at the moment, only the card at the address + // of the object may be dirty. + Object obj = curPtr.toObject(); + UnsignedWord cardTableIndex = memoryOffsetToIndex(curPtr.subtract(objectsStart)); + if (isClean(cardTableStart, cardTableIndex)) { + CARD_TABLE_VERIFICATION_VISITOR.initialize(obj, cardTableStart, objectsStart); + InteriorObjRefWalker.walkObject(obj, CARD_TABLE_VERIFICATION_VISITOR); + success &= CARD_TABLE_VERIFICATION_VISITOR.success; + CARD_TABLE_VERIFICATION_VISITOR.reset(); + + DynamicHub hub = KnownIntrinsics.readHub(obj); + if (hub.isReferenceInstanceClass()) { + // The referent field of java.lang.Reference is excluded from the reference + // map, so we need to verify it separately. + Reference ref = (Reference) obj; + success &= verifyReferent(ref, cardTableStart, objectsStart); + } } + curPtr = LayoutEncoding.getObjectEndInGC(obj); + } + } else { + /* Do a basic sanity check of the card table data. */ + Pointer pos = cardTableStart; + while (pos.belowThan(cardTableEnd)) { + byte v = pos.readByte(0); + if (v != DIRTY_ENTRY && v != CLEAN_ENTRY) { + Log.log().string("Card at ").zhex(pos).string(" is neither dirty nor clean: ").zhex(v).newline(); + return false; + } + pos = pos.add(1); } - curPtr = LayoutEncoding.getObjectEndInGC(obj); } return success; } @@ -214,12 +229,20 @@ private static class CardTableVerificationVisitor implements ObjectReferenceVisi @SuppressWarnings("hiding") public void initialize(Object parentObject, Pointer cardTableStart, Pointer objectsStart) { + assert this.parentObject == null && this.cardTableStart.isNull() && this.objectsStart.isNull() && !this.success; this.parentObject = parentObject; this.cardTableStart = cardTableStart; this.objectsStart = objectsStart; this.success = true; } + public void reset() { + this.parentObject = null; + this.cardTableStart = WordFactory.nullPointer(); + this.objectsStart = WordFactory.nullPointer(); + this.success = false; + } + @Override @SuppressFBWarnings(value = {"NS_DANGEROUS_NON_SHORT_CIRCUIT"}, justification = "Non-short circuit logic is used on purpose here.") public boolean visitObjectReference(Pointer reference, boolean compressed, Object holderObject) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 3cdde75b7d36..4bb2a92def64 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -103,8 +103,8 @@ public void enableRememberedSetForChunk(UnalignedHeader chunk) { @Override @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { - AlignedChunkRememberedSet.enableRememberedSetForObject(chunk, obj); + public void enableRememberedSetForObject(AlignedHeader chunk, Object obj, UnsignedWord objSize) { + AlignedChunkRememberedSet.enableRememberedSetForObject(chunk, obj, objSize); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index ea156e60c228..d6d43393bdf7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -100,7 +100,7 @@ public void enableRememberedSetForChunk(UnalignedHeader chunk) { @Override @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { + public void enableRememberedSetForObject(AlignedHeader chunk, Object obj, UnsignedWord objSize) { // Nothing to do. } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java index dba9d1815c13..ac2a3424c8b8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java @@ -94,7 +94,7 @@ static RememberedSet get() { */ @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void enableRememberedSetForObject(AlignedHeader chunk, Object obj); + void enableRememberedSetForObject(AlignedHeader chunk, Object obj, UnsignedWord objSize); /** Clears the remembered set of an aligned chunk. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index 43adea759c2a..18856b644f49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -107,7 +107,7 @@ public static void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisi } public static boolean verify(UnalignedHeader chunk) { - return CardTable.verify(getCardTableStart(chunk), UnalignedHeapChunk.getObjectStart(chunk), HeapChunk.getTopPointer(chunk)); + return CardTable.verify(getCardTableStart(chunk), getCardTableEnd(chunk), UnalignedHeapChunk.getObjectStart(chunk), HeapChunk.getTopPointer(chunk)); } @Fold @@ -147,4 +147,9 @@ private static Pointer getCardTableStart(UnalignedHeader chunk) { private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer getCardTableEnd(UnalignedHeader chunk) { + return getCardTableStart(chunk).add(getCardTableSize()); + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index dd5a06d6a4a1..7536f4a2fed4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -24,11 +24,6 @@ */ package com.oracle.svm.core.heap; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.LogHandler; -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.impl.InternalPlatform; - import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; @@ -81,12 +76,7 @@ private static void reportOutOfMemoryError0(OutOfMemoryError error) { } if (SubstrateGCOptions.ReportFatalErrorOnOutOfMemoryError.getValue()) { - if (Platform.includedIn(InternalPlatform.NATIVE_ONLY.class)) { - Log.log().string("Reporting Fatal Error due to java.lang.OutOfMemoryError: ").exception(error); - } else { - Log.log().string("Reporting Fatal Error due to java.lang.OutOfMemoryError").newline(); - } - ImageSingletons.lookup(LogHandler.class).fatalError(); + throw VMError.shouldNotReachHere("reporting due to java.lang.OutOfMemoryError"); } } } From 2fc5e7a268e7aef1ae756bdd614983c7731ec19e Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 19 Jul 2024 09:46:56 +0200 Subject: [PATCH 2/5] Fix serial GC collection count and time so that it is consistent with -XX:+VerboseGC and emitted JFR events. --- .../svm/core/genscavenge/GCAccounting.java | 49 ++++++++----------- .../oracle/svm/core/genscavenge/GCImpl.java | 38 +++++++------- .../oracle/svm/core/genscavenge/Timers.java | 3 -- 3 files changed, 42 insertions(+), 48 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java index 4d83d99b2547..26ec582dfa0d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java @@ -119,7 +119,7 @@ public boolean hasLastIncrementalCollectionOverflowedSurvivors() { return lastIncrementalCollectionOverflowedSurvivors; } - void beforeCollection(boolean completeCollection) { + void beforeCollectOnce(boolean completeCollection) { /* Gather some space statistics. */ HeapImpl heap = HeapImpl.getHeapImpl(); YoungGeneration youngGen = heap.getYoungGeneration(); @@ -149,33 +149,7 @@ void onSurvivorOverflowed() { lastIncrementalCollectionOverflowedSurvivors = true; } - void afterCollection(boolean completeCollection, Timer collectionTimer) { - if (completeCollection) { - afterCompleteCollection(collectionTimer); - } else { - afterIncrementalCollection(collectionTimer); - } - } - - private void afterIncrementalCollection(Timer collectionTimer) { - /* - * Aggregating collection information is needed because any given collection policy may not - * be called for all collections, but may want to make decisions based on the aggregate - * values. - */ - incrementalCollectionCount += 1; - afterCollectionCommon(); - lastIncrementalCollectionPromotedChunkBytes = oldChunkBytesAfter.subtract(oldChunkBytesBefore); - incrementalCollectionTotalNanos += collectionTimer.getMeasuredNanos(); - } - - private void afterCompleteCollection(Timer collectionTimer) { - completeCollectionCount += 1; - afterCollectionCommon(); - completeCollectionTotalNanos += collectionTimer.getMeasuredNanos(); - } - - private void afterCollectionCommon() { + void afterCollectOnce(boolean completeCollection) { HeapImpl heap = HeapImpl.getHeapImpl(); YoungGeneration youngGen = heap.getYoungGeneration(); OldGeneration oldGen = heap.getOldGeneration(); @@ -207,5 +181,24 @@ private void afterCollectionCommon() { totalCollectedObjectBytes = totalCollectedObjectBytes.add(collectedObjectBytes); } } + + if (!completeCollection) { + /* + * Aggregating collection information is needed because any given collection policy may + * not be called for all collections, but may want to make decisions based on the + * aggregate values. + */ + lastIncrementalCollectionPromotedChunkBytes = oldChunkBytesAfter.subtract(oldChunkBytesBefore); + } + } + + void updateCollectionCountAndTime(boolean completeCollection, long collectionTime) { + if (completeCollection) { + completeCollectionCount += 1; + completeCollectionTotalNanos += collectionTime; + } else { + incrementalCollectionCount += 1; + incrementalCollectionTotalNanos += collectionTime; + } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index e551d6633621..5d249b49ea3c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -226,22 +226,30 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) || timers.mutator.closeAt(data.getRequestingNanoTime()); timers.resetAllExceptMutator(); + /* The type of collection will be determined later on. */ + completeCollection = false; JfrGCHeapSummaryEvent.emit(JfrGCWhen.BEFORE_GC); GCCause cause = GCCause.fromId(data.getCauseId()); printGCBefore(cause); - ThreadLocalAllocation.disableAndFlushForAllThreads(); - GenScavengeMemoryPoolMXBeans.singleton().notifyBeforeCollection(); - HeapImpl.getAccounting().notifyBeforeCollection(); + Timer collectionTimer = timers.collection.open(); + try { + ThreadLocalAllocation.disableAndFlushForAllThreads(); + GenScavengeMemoryPoolMXBeans.singleton().notifyBeforeCollection(); + HeapImpl.getAccounting().notifyBeforeCollection(); - verifyBeforeGC(); + verifyBeforeGC(); - boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); - data.setOutOfMemory(outOfMemory); + boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); + data.setOutOfMemory(outOfMemory); - verifyAfterGC(); + verifyAfterGC(); + } finally { + collectionTimer.close(); + } + accounting.updateCollectionCountAndTime(completeCollection, collectionTimer.getMeasuredNanos()); HeapImpl.getAccounting().notifyAfterCollection(); GenScavengeMemoryPoolMXBeans.singleton().notifyAfterCollection(); @@ -310,22 +318,18 @@ private boolean doCollectImpl(GCCause cause, long requestingNanoTime, boolean fo private boolean doCollectOnce(GCCause cause, long requestingNanoTime, boolean complete, boolean followsIncremental) { assert !followsIncremental || complete : "An incremental collection cannot be followed by another incremental collection"; + assert !completeCollection || complete : "After a complete collection, no further incremental collections may happen"; completeCollection = complete; - accounting.beforeCollection(completeCollection); + accounting.beforeCollectOnce(completeCollection); policy.onCollectionBegin(completeCollection, requestingNanoTime); - Timer collectionTimer = timers.collection.open(); - try { - doCollectCore(!complete); - if (complete) { - lastWholeHeapExaminedTimeMillis = System.currentTimeMillis(); - } - } finally { - collectionTimer.close(); + doCollectCore(!complete); + if (complete) { + lastWholeHeapExaminedTimeMillis = System.currentTimeMillis(); } - accounting.afterCollection(completeCollection, collectionTimer); + accounting.afterCollectOnce(completeCollection); policy.onCollectionEnd(completeCollection, cause); UnsignedWord usedBytes = getChunkBytes(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index 6d34182d2194..9dc8ca20c598 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -144,8 +144,6 @@ final class Timers { } void resetAllExceptMutator() { - Log trace = Log.noopLog(); - trace.string("[Timers.resetAllExceptMutator:"); verifyBefore.reset(); collection.reset(); rootScan.reset(); @@ -176,7 +174,6 @@ void resetAllExceptMutator() { releaseSpaces.reset(); verifyAfter.reset(); /* The mutator timer is *not* reset here. */ - trace.string("]").newline(); } void logAfterCollection(Log log) { From 9ce31e3d9d91473bb19a7124139646f8c159991c Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 23 Jul 2024 13:54:44 +0200 Subject: [PATCH 3/5] Split CommittedMemoryProvider. --- .../oracle/svm/core/genscavenge/GCImpl.java | 9 +- .../core/genscavenge/HeapChunkProvider.java | 14 +-- .../os/AbstractCommittedMemoryProvider.java | 45 +------- .../os/ChunkBasedCommittedMemoryProvider.java | 105 ++++++++++++++++++ .../svm/core/os/CommittedMemoryProvider.java | 31 ------ .../core/os/OSCommittedMemoryProvider.java | 2 +- 6 files changed, 120 insertions(+), 86 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ChunkBasedCommittedMemoryProvider.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 5d249b49ea3c..326a04831825 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -84,7 +84,7 @@ import com.oracle.svm.core.jfr.JfrTicks; import com.oracle.svm.core.jfr.events.AllocationRequiringGCEvent; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.os.CommittedMemoryProvider; +import com.oracle.svm.core.os.ChunkBasedCommittedMemoryProvider; import com.oracle.svm.core.snippets.ImplicitExceptions; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.stack.JavaFrame; @@ -252,6 +252,7 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) || accounting.updateCollectionCountAndTime(completeCollection, collectionTimer.getMeasuredNanos()); HeapImpl.getAccounting().notifyAfterCollection(); GenScavengeMemoryPoolMXBeans.singleton().notifyAfterCollection(); + ChunkBasedCommittedMemoryProvider.get().afterGarbageCollection(); printGCAfter(cause); JfrGCHeapSummaryEvent.emit(JfrGCWhen.AFTER_GC); @@ -284,7 +285,7 @@ private boolean collectImpl(GCCause cause, long requestingNanoTime, boolean forc private boolean doCollectImpl(GCCause cause, long requestingNanoTime, boolean forceFullGC, boolean forceNoIncremental) { checkSanityBeforeCollection(); - CommittedMemoryProvider.get().beforeGarbageCollection(); + ChunkBasedCommittedMemoryProvider.get().beforeGarbageCollection(); boolean incremental = !forceNoIncremental && !policy.shouldCollectCompletely(false); boolean outOfMemory = false; @@ -299,7 +300,7 @@ private boolean doCollectImpl(GCCause cause, long requestingNanoTime, boolean fo } if (!incremental || outOfMemory || forceFullGC || policy.shouldCollectCompletely(incremental)) { if (incremental) { // uncommit unaligned chunks - CommittedMemoryProvider.get().afterGarbageCollection(); + ChunkBasedCommittedMemoryProvider.get().uncommitUnusedMemory(); } long startTicks = JfrGCEvents.startGCPhasePause(); try { @@ -310,7 +311,7 @@ private boolean doCollectImpl(GCCause cause, long requestingNanoTime, boolean fo } HeapImpl.getChunkProvider().freeExcessAlignedChunks(); - CommittedMemoryProvider.get().afterGarbageCollection(); + ChunkBasedCommittedMemoryProvider.get().uncommitUnusedMemory(); checkSanityAfterCollection(); return outOfMemory; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index ec4bbfeac58f..f5dc7199eab6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -40,7 +40,7 @@ import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicUnsigned; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.os.CommittedMemoryProvider; +import com.oracle.svm.core.os.ChunkBasedCommittedMemoryProvider; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.UnsignedUtils; @@ -89,7 +89,7 @@ AlignedHeader produceAlignedChunk() { AlignedHeader result = popUnusedAlignedChunk(); if (result.isNull()) { /* Unused list was empty, need to allocate memory. */ - result = (AlignedHeader) CommittedMemoryProvider.get().allocateAlignedChunk(chunkSize, HeapParameters.getAlignedHeapChunkAlignment()); + result = (AlignedHeader) ChunkBasedCommittedMemoryProvider.get().allocateAlignedChunk(chunkSize, HeapParameters.getAlignedHeapChunkAlignment()); if (result.isNull()) { throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); } @@ -240,7 +240,7 @@ private void freeUnusedAlignedChunksAtSafepoint(UnsignedWord count) { UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize); - UnalignedHeader result = (UnalignedHeader) CommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize); + UnalignedHeader result = (UnalignedHeader) ChunkBasedCommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize); if (result.isNull()) { throw OutOfMemoryUtil.reportOutOfMemoryError(UNALIGNED_OUT_OF_MEMORY_ERROR); } @@ -249,14 +249,14 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { assert objectSize.belowOrEqual(HeapChunk.availableObjectMemory(result)) : "UnalignedHeapChunk insufficient for requested object"; /* Avoid zapping if unaligned chunks are pre-zeroed. */ - if (!CommittedMemoryProvider.get().areUnalignedChunksZeroed() && HeapParameters.getZapProducedHeapChunks()) { + if (!ChunkBasedCommittedMemoryProvider.get().areUnalignedChunksZeroed() && HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } return result; } public static boolean areUnalignedChunksZeroed() { - return CommittedMemoryProvider.get().areUnalignedChunksZeroed(); + return ChunkBasedCommittedMemoryProvider.get().areUnalignedChunksZeroed(); } /** @@ -306,12 +306,12 @@ static void freeUnalignedChunkList(UnalignedHeader first) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void freeAlignedChunk(AlignedHeader chunk) { - CommittedMemoryProvider.get().freeAlignedChunk(chunk, HeapParameters.getAlignedHeapChunkSize(), HeapParameters.getAlignedHeapChunkAlignment()); + ChunkBasedCommittedMemoryProvider.get().freeAlignedChunk(chunk, HeapParameters.getAlignedHeapChunkSize(), HeapParameters.getAlignedHeapChunkAlignment()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void freeUnalignedChunk(UnalignedHeader chunk) { - CommittedMemoryProvider.get().freeUnalignedChunk(chunk, unalignedChunkSize(chunk)); + ChunkBasedCommittedMemoryProvider.get().freeUnalignedChunk(chunk, unalignedChunkSize(chunk)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index c4d94e712589..f3a89590aede 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -38,12 +38,9 @@ import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointErrors; -import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.util.UnsignedUtils; -import jdk.graal.compiler.api.replacements.Fold; - public abstract class AbstractCommittedMemoryProvider implements CommittedMemoryProvider { @Uninterruptible(reason = "Still being initialized.") protected static int protectSingleIsolateImageHeap() { @@ -72,24 +69,13 @@ protected static int protectSingleIsolateImageHeap() { return CEntryPointErrors.NO_ERROR; } - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { - return allocate(nbytes, alignment, false); - } - - @Override - public Pointer allocateUnalignedChunk(UnsignedWord nbytes) { - return allocate(nbytes, getAlignmentForUnalignedChunks(), false); - } - @Override public Pointer allocateExecutableMemory(UnsignedWord nbytes, UnsignedWord alignment) { return allocate(nbytes, alignment, true); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable) { + protected Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable) { Pointer reserved = WordFactory.nullPointer(); if (!UnsignedUtils.isAMultiple(getGranularity(), alignment)) { reserved = VirtualMemoryProvider.get().reserve(size, alignment, executable); @@ -113,23 +99,6 @@ private Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean exec return committed; } - @Override - public boolean areUnalignedChunksZeroed() { - return false; - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void freeAlignedChunk(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) { - free(start, nbytes); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void freeUnalignedChunk(PointerBase start, UnsignedWord nbytes) { - free(start, nbytes); - } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void freeExecutableMemory(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) { @@ -137,22 +106,12 @@ public void freeExecutableMemory(PointerBase start, UnsignedWord nbytes, Unsigne } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void free(PointerBase start, UnsignedWord nbytes) { + protected void free(PointerBase start, UnsignedWord nbytes) { if (VirtualMemoryProvider.get().free(start, nbytes) == 0) { tracker.untrack(nbytes); } } - /** - * Unaligned chunks also need some minimal alignment - otherwise, the data in the chunk header - * or the Java heap object within the unaligned chunk would be misaligned. - */ - @Fold - protected static UnsignedWord getAlignmentForUnalignedChunks() { - int alignment = Math.max(ConfigurationValues.getTarget().wordSize, ConfigurationValues.getObjectLayout().getAlignment()); - return WordFactory.unsigned(alignment); - } - private final VirtualMemoryTracker tracker = new VirtualMemoryTracker(); public static class VirtualMemoryTracker { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ChunkBasedCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ChunkBasedCommittedMemoryProvider.java new file mode 100644 index 000000000000..369792e0e8bf --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ChunkBasedCommittedMemoryProvider.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.os; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.heap.RestrictHeapAccess; + +import jdk.graal.compiler.api.replacements.Fold; + +public abstract class ChunkBasedCommittedMemoryProvider extends AbstractCommittedMemoryProvider { + @Fold + public static ChunkBasedCommittedMemoryProvider get() { + return (ChunkBasedCommittedMemoryProvider) ImageSingletons.lookup(CommittedMemoryProvider.class); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { + return allocate(nbytes, alignment, false); + } + + public Pointer allocateUnalignedChunk(UnsignedWord nbytes) { + return allocate(nbytes, getAlignmentForUnalignedChunks(), false); + } + + /** + * This method returns {@code true} if the memory returned by {@link #allocateUnalignedChunk} is + * guaranteed to be zeroed. + */ + public boolean areUnalignedChunksZeroed() { + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void freeAlignedChunk(PointerBase start, UnsignedWord nbytes, @SuppressWarnings("unused") UnsignedWord alignment) { + free(start, nbytes); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void freeUnalignedChunk(PointerBase start, UnsignedWord nbytes) { + free(start, nbytes); + } + + /** + * Unaligned chunks also need some minimal alignment - otherwise, the data in the chunk header + * or the Java heap object within the unaligned chunk would be misaligned. + */ + @Fold + protected static UnsignedWord getAlignmentForUnalignedChunks() { + int alignment = Math.max(ConfigurationValues.getTarget().wordSize, ConfigurationValues.getObjectLayout().getAlignment()); + return WordFactory.unsigned(alignment); + } + + /** + * Called by the garbage collector before a collection is started, as an opportunity to perform + * lazy operations, sanity checks or clean-ups. + */ + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Called by the GC.") + public void beforeGarbageCollection() { + } + + /** + * Called by the garbage collector after a collection has ended, as an opportunity to perform + * lazy operations, sanity checks or clean-ups. + */ + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Called by the GC.") + public void afterGarbageCollection() { + } + + /** + * Called by the garbage collector to uncommit unused memory. Note that this method may be + * called multiple times during a single VM operation. + */ + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Called by the GC.") + public void uncommitUnusedMemory() { + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java index 41d3f8cf1ebf..c665f03c32b6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java @@ -72,39 +72,8 @@ default UnsignedWord getGranularity() { return VirtualMemoryProvider.get().getGranularity(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment); - - Pointer allocateUnalignedChunk(UnsignedWord nbytes); - Pointer allocateExecutableMemory(UnsignedWord nbytes, UnsignedWord alignment); - /** - * This method returns {@code true} if the memory returned by {@link #allocateUnalignedChunk} is - * guaranteed to be zeroed. - */ - boolean areUnalignedChunksZeroed(); - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void freeAlignedChunk(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment); - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void freeUnalignedChunk(PointerBase start, UnsignedWord nbytes); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void freeExecutableMemory(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment); - - /** - * Called by the garbage collector before a collection is started, as an opportunity to perform - * lazy operations, sanity checks or clean-ups. - */ - default void beforeGarbageCollection() { - } - - /** - * Called by the garbage collector after a collection has ended, as an opportunity to perform - * lazy operations, sanity checks or clean-ups. - */ - default void afterGarbageCollection() { - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java index b09b8bddcd04..9c9dc1909036 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java @@ -42,7 +42,7 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; -public class OSCommittedMemoryProvider extends AbstractCommittedMemoryProvider { +public class OSCommittedMemoryProvider extends ChunkBasedCommittedMemoryProvider { @Platforms(Platform.HOSTED_ONLY.class) public OSCommittedMemoryProvider() { } From 9c36e2935a1f2605b1d7cd345f57563d8ac70205 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 30 Jul 2024 11:55:05 +0200 Subject: [PATCH 4/5] Cleanups. --- .../core/genscavenge/AlignedHeapChunk.java | 5 +++-- .../svm/core/genscavenge/GCAccounting.java | 6 +++--- .../svm/core/genscavenge/HeapVerifier.java | 21 ++++++------------- .../genscavenge/ThreadLocalAllocation.java | 5 +++-- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 21217efa4443..70ae42d528a0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -92,8 +92,9 @@ public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) { } public static void reset(AlignedHeader chunk) { - assert HeapChunk.getEndOffset(chunk).rawValue() == SerialAndEpsilonGCOptions.AlignedHeapChunkSize.getValue(); - initialize(chunk, WordFactory.unsigned(SerialAndEpsilonGCOptions.AlignedHeapChunkSize.getValue())); + long alignedChunkSize = SerialAndEpsilonGCOptions.AlignedHeapChunkSize.getValue(); + assert HeapChunk.getEndOffset(chunk).rawValue() == alignedChunkSize; + initialize(chunk, WordFactory.unsigned(alignedChunkSize)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java index 26ec582dfa0d..ee06b99f1910 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java @@ -184,9 +184,9 @@ void afterCollectOnce(boolean completeCollection) { if (!completeCollection) { /* - * Aggregating collection information is needed because any given collection policy may - * not be called for all collections, but may want to make decisions based on the - * aggregate values. + * Aggregating collection information is needed because a collection policy might not be + * called for all collections, but may want to make decisions based on the aggregate + * values. */ lastIncrementalCollectionPromotedChunkBytes = oldChunkBytesAfter.subtract(oldChunkBytesBefore); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java index d403f9c7185f..0eed9da20e52 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java @@ -120,24 +120,15 @@ private static boolean verifyRememberedSets() { * After we are done with all other verifications, it is guaranteed that the heap is in a * reasonable state. Now, we can verify the remembered sets without having to worry about * basic heap consistency. - */ - if (!SerialGCOptions.useRememberedSet()) { - return true; - } - - /* + * * It would be nice to assert that all cards in the image heap and old generation are clean * after a garbage collection. For the image heap, it is pretty much impossible to do that * as the GC itself dirties the card table. For the old generation, it is also not possible * at the moment because the reference handling may result in dirty cards. */ - boolean success = true; RememberedSet rememberedSet = RememberedSet.get(); - /* - * For the image heap, we can't verify that all cards are clean after a GC because the GC - * itself may result in dirty cards. - */ + for (ImageHeapInfo info = HeapImpl.getFirstImageHeapInfo(); info != null; info = info.next) { success &= rememberedSet.verify(info.getFirstWritableAlignedChunk()); success &= rememberedSet.verify(info.getFirstWritableUnalignedChunk(), info.getLastWritableUnalignedChunk()); @@ -257,13 +248,13 @@ private static boolean verifyObject(Object obj, AlignedHeader aChunk, UnalignedH return false; } - if (!HeapImpl.getHeap().getObjectHeader().isEncodedObjectHeader(header)) { - Log.log().string("Object ").zhex(ptr).string(" does not have a valid hub: ").zhex(header).newline(); + if (ObjectHeaderImpl.isForwardedHeader(header)) { + Log.log().string("Object ").zhex(ptr).string(" has a forwarded header: ").zhex(header).newline(); return false; } - if (ObjectHeaderImpl.isForwardedHeader(header)) { - Log.log().string("Object ").zhex(ptr).string(" has a forwarded header: ").zhex(header).newline(); + if (!HeapImpl.getHeap().getObjectHeader().isEncodedObjectHeader(header)) { + Log.log().string("Object ").zhex(ptr).string(" does not have a valid hub: ").zhex(header).newline(); return false; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index c6f2c329c7d5..6ba691a3de37 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -427,14 +427,15 @@ private static UnsignedWord availableTlabMemory(Descriptor allocator) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void guaranteeZeroed(Pointer memory, UnsignedWord size) { - VMError.guarantee(UnsignedUtils.isAMultiple(size, WordFactory.unsigned(ConfigurationValues.getTarget().wordSize))); + int wordSize = ConfigurationValues.getTarget().wordSize; + VMError.guarantee(UnsignedUtils.isAMultiple(size, WordFactory.unsigned(wordSize))); Pointer pos = memory; Pointer end = memory.add(size); while (pos.belowThan(end)) { Word v = pos.readWord(0); VMError.guarantee(v.equal(0)); - pos = pos.add(ConfigurationValues.getTarget().wordSize); + pos = pos.add(wordSize); } } From 0555a4dcfef4538dc84e17b2190c0d4967569e2f Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 30 Jul 2024 15:21:19 +0200 Subject: [PATCH 5/5] Add option VerifyDuringGC. --- .../oracle/svm/core/genscavenge/GCImpl.java | 73 ++++++++----------- .../svm/core/genscavenge/HeapVerifier.java | 7 +- .../svm/core/genscavenge/SerialGCOptions.java | 3 + .../oracle/svm/core/genscavenge/Timers.java | 6 -- 4 files changed, 36 insertions(+), 53 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 326a04831825..fd77d1607339 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -25,6 +25,9 @@ package com.oracle.svm.core.genscavenge; import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static com.oracle.svm.core.genscavenge.HeapVerifier.Occasion.After; +import static com.oracle.svm.core.genscavenge.HeapVerifier.Occasion.Before; +import static com.oracle.svm.core.genscavenge.HeapVerifier.Occasion.During; import java.lang.ref.Reference; @@ -239,12 +242,12 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) || GenScavengeMemoryPoolMXBeans.singleton().notifyBeforeCollection(); HeapImpl.getAccounting().notifyBeforeCollection(); - verifyBeforeGC(); + verifyHeap(Before); boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); data.setOutOfMemory(outOfMemory); - verifyAfterGC(); + verifyHeap(After); } finally { collectionTimer.close(); } @@ -271,6 +274,7 @@ private boolean collectImpl(GCCause cause, long requestingNanoTime, boolean forc // objects ReferenceObjectProcessing.setSoftReferencesAreWeak(true); try { + verifyHeap(During); outOfMemory = doCollectImpl(cause, requestingNanoTime, true, true); } finally { ReferenceObjectProcessing.setSoftReferencesAreWeak(false); @@ -301,6 +305,7 @@ private boolean doCollectImpl(GCCause cause, long requestingNanoTime, boolean fo if (!incremental || outOfMemory || forceFullGC || policy.shouldCollectCompletely(incremental)) { if (incremental) { // uncommit unaligned chunks ChunkBasedCommittedMemoryProvider.get().uncommitUnusedMemory(); + verifyHeap(During); } long startTicks = JfrGCEvents.startGCPhasePause(); try { @@ -340,60 +345,40 @@ private boolean doCollectOnce(GCCause cause, long requestingNanoTime, boolean co return usedBytes.aboveThan(policy.getMaximumHeapSize()); // out of memory? } - private void verifyBeforeGC() { - if (SubstrateGCOptions.VerifyHeap.getValue() && SerialGCOptions.VerifyBeforeGC.getValue()) { + private void verifyHeap(HeapVerifier.Occasion occasion) { + if (SubstrateGCOptions.VerifyHeap.getValue() && shouldVerify(occasion)) { if (SubstrateGCOptions.VerboseGC.getValue()) { - printGCPrefixAndTime().string("Verifying Before GC ").newline(); + printGCPrefixAndTime().string("Verifying ").string(occasion.name()).string(" GC ").newline(); } - Timer verifyBeforeTimer = timers.verifyBefore.open(); - try { - boolean success = true; - success &= HeapVerifier.singleton().verify(HeapVerifier.Occasion.BEFORE_COLLECTION); - success &= StackVerifier.verifyAllThreads(); - - if (!success) { - String kind = getGCKind(); - Log.log().string("Heap verification failed before ").string(kind).string(" garbage collection.").newline(); - VMError.shouldNotReachHere("Heap verification failed"); - } - } finally { - verifyBeforeTimer.close(); - } - - if (SubstrateGCOptions.VerboseGC.getValue()) { - printGCPrefixAndTime().string("Verifying Before GC ").rational(timers.verifyBefore.getMeasuredNanos(), TimeUtils.nanosPerMilli, 3).string("ms").newline(); - } - } - } + long start = System.nanoTime(); - private void verifyAfterGC() { - if (SubstrateGCOptions.VerifyHeap.getValue() && SerialGCOptions.VerifyAfterGC.getValue()) { - if (SubstrateGCOptions.VerboseGC.getValue()) { - printGCPrefixAndTime().string("Verifying After GC ").newline(); - } + boolean success = true; + success &= HeapVerifier.singleton().verify(occasion); + success &= StackVerifier.verifyAllThreads(); - Timer verifyAfterTime = timers.verifyAfter.open(); - try { - boolean success = true; - success &= HeapVerifier.singleton().verify(HeapVerifier.Occasion.AFTER_COLLECTION); - success &= StackVerifier.verifyAllThreads(); - - if (!success) { - String kind = getGCKind(); - Log.log().string("Heap verification failed after ").string(kind).string(" garbage collection.").newline(); - VMError.shouldNotReachHere("Heap verification failed"); - } - } finally { - verifyAfterTime.close(); + if (!success) { + String kind = getGCKind(); + Log.log().string("Heap verification ").string(occasion.name()).string(" GC failed (").string(kind).string(" garbage collection)").newline(); + throw VMError.shouldNotReachHere("Heap verification failed"); } if (SubstrateGCOptions.VerboseGC.getValue()) { - printGCPrefixAndTime().string("Verifying After GC ").rational(timers.verifyAfter.getMeasuredNanos(), TimeUtils.nanosPerMilli, 3).string("ms").newline(); + printGCPrefixAndTime().string("Verifying ").string(occasion.name()).string(" GC ") + .rational(TimeUtils.nanoSecondsSince(start), TimeUtils.nanosPerMilli, 3).string("ms").newline(); } } } + private static boolean shouldVerify(HeapVerifier.Occasion occasion) { + return switch (occasion) { + case Before -> SerialGCOptions.VerifyBeforeGC.getValue(); + case During -> SerialGCOptions.VerifyDuringGC.getValue(); + case After -> SerialGCOptions.VerifyAfterGC.getValue(); + default -> throw VMError.shouldNotReachHere("Unexpected heap verification occasion."); + }; + } + private String getGCKind() { return isCompleteCollection() ? "complete" : "incremental"; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java index 0eed9da20e52..20bfd41ac7f2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java @@ -83,7 +83,7 @@ protected boolean verifyImageHeap() { private static boolean verifyYoungGeneration(Occasion occasion) { boolean success = true; YoungGeneration youngGeneration = HeapImpl.getHeapImpl().getYoungGeneration(); - if (occasion == HeapVerifier.Occasion.AFTER_COLLECTION) { + if (occasion == Occasion.During || occasion == Occasion.After) { Space eden = youngGeneration.getEden(); if (!eden.isEmpty()) { Log.log().string("Eden contains chunks after a collection: firstAlignedChunk: ").zhex(eden.getFirstAlignedHeapChunk()).string(", firstUnalignedChunk: ") @@ -431,7 +431,8 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } public enum Occasion { - BEFORE_COLLECTION, - AFTER_COLLECTION + Before, + During, + After } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java index 49d4be1b4092..02e26c1bd68a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java @@ -85,6 +85,9 @@ public Integer getValue(OptionValues values) { @Option(help = "Verify the heap before doing a garbage collection if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// public static final HostedOptionKey VerifyBeforeGC = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + @Option(help = "Verify the heap during a garbage collection if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// + public static final HostedOptionKey VerifyDuringGC = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + @Option(help = "Verify the heap after doing a garbage collection if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// public static final HostedOptionKey VerifyAfterGC = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index 9dc8ca20c598..00d0d8649462 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -133,8 +133,6 @@ final class Timers { final Timer oldCompaction = new Timer("oldCompaction"); final Timer oldCompactionRememberedSets = new Timer("oldCompactionRememberedSets"); final Timer releaseSpaces = new Timer("releaseSpaces"); - final Timer verifyAfter = new Timer("verifyAfter"); - final Timer verifyBefore = new Timer("verifyBefore"); final Timer walkThreadLocals = new Timer("walkThreadLocals"); final Timer walkRuntimeCodeCache = new Timer("walkRuntimeCodeCache"); final Timer cleanRuntimeCodeCache = new Timer("cleanRuntimeCodeCache"); @@ -144,7 +142,6 @@ final class Timers { } void resetAllExceptMutator() { - verifyBefore.reset(); collection.reset(); rootScan.reset(); scanFromRoots.reset(); @@ -172,7 +169,6 @@ void resetAllExceptMutator() { cleanCodeCache.reset(); referenceObjects.reset(); releaseSpaces.reset(); - verifyAfter.reset(); /* The mutator timer is *not* reset here. */ } @@ -181,7 +177,6 @@ void logAfterCollection(Log log) { log.newline(); log.string(" [GC nanoseconds:"); logOneTimer(log, " ", collection); - logOneTimer(log, " ", verifyBefore); logOneTimer(log, " ", rootScan); logOneTimer(log, " ", scanFromRoots); logOneTimer(log, " ", scanFromDirtyRoots); @@ -208,7 +203,6 @@ void logAfterCollection(Log log) { logOneTimer(log, " ", cleanCodeCache); logOneTimer(log, " ", referenceObjects); logOneTimer(log, " ", releaseSpaces); - logOneTimer(log, " ", verifyAfter); logGCLoad(log, " ", "GCLoad", collection, mutator); log.string("]"); }