From afe55d74a1cf4c40484861745ff1230932653149 Mon Sep 17 00:00:00 2001 From: Rudi Horn Date: Tue, 6 Feb 2024 11:41:02 +0100 Subject: [PATCH] [GR-51409] Image heap fallback to mremap --- .../oracle/svm/core/posix/headers/Errno.java | 5 +- .../posix/linux/LinuxImageHeapProvider.java | 140 ++++++++++++++++-- .../svm/core/posix/linux/LinuxLibCHelper.java | 33 +++++ .../com/oracle/svm/core/SubstrateOptions.java | 7 + .../core/c/function/CEntryPointErrors.java | 3 + .../core/imagelayer/ImageLayerSection.java | 10 +- .../imagelayer/ImageLayerSectionFeature.java | 13 +- .../src/varargs_helper.c | 13 +- 8 files changed, 207 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Errno.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Errno.java index 03e4627f62c2..9bc87159026e 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Errno.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Errno.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -61,6 +61,9 @@ public class Errno { @CConstant public static native int EEXIST(); + @CConstant + public static native int EINVAL(); + @CFunction public static native CCharPointer strerror(int errnum); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java index b9e0c4d0b51e..5e37e73f6a2d 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -58,6 +58,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.CGlobalData; import com.oracle.svm.core.c.CGlobalDataFactory; @@ -72,12 +73,14 @@ import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; import com.oracle.svm.core.posix.PosixUtils; +import com.oracle.svm.core.posix.headers.Errno; import com.oracle.svm.core.posix.headers.Fcntl; import com.oracle.svm.core.posix.headers.Unistd; import com.oracle.svm.core.util.PointerUtils; import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.nodes.PauseNode; import jdk.graal.compiler.word.Word; /** @@ -103,8 +106,11 @@ public class LinuxImageHeapProvider extends AbstractImageHeapProvider { private static final SignedWord UNASSIGNED_FD = signed(-1); private static final SignedWord CANNOT_OPEN_FD = signed(-2); - private static final CGlobalData CACHED_IMAGE_FDS = CGlobalDataFactory.createWord(UNASSIGNED_FD); - private static final CGlobalData CACHED_IMAGE_HEAP_OFFSETS = CGlobalDataFactory.createWord(); + private static final SignedWord COPY_RELOCATIONS_IN_PROGRESS = signed(-1); + + private static final CGlobalData CACHED_IMAGE_FD = CGlobalDataFactory.createWord(UNASSIGNED_FD); + private static final CGlobalData CACHED_IMAGE_HEAP_OFFSET = CGlobalDataFactory.createWord(); + private static final CGlobalData CACHED_IMAGE_HEAP_RELOCATIONS = CGlobalDataFactory.createWord(); private static final int MAX_PATHLEN = 4096; @@ -173,6 +179,7 @@ protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReserve while (currentSection.isNonNull()) { var cachedFDPointer = ImageLayerSection.getCachedImageFDs().get().addressOf(layerCount); var cachedOffsetsPointer = ImageLayerSection.getCachedImageHeapOffsets().get().addressOf(layerCount); + var cachedImageHeapRelocationsPtr = ImageLayerSection.getCachedImageHeapRelocations().get().addressOf(layerCount); Word heapBegin = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_BEGIN)); Word heapEnd = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_END)); Word heapRelocBegin = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_RELOCATABLE_BEGIN)); @@ -182,7 +189,7 @@ protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReserve Word heapWritableEnd = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_WRITEABLE_END)); result = initializeImageHeap(currentHeapStart, remainingSize, curEndPointer, - cachedFDPointer, cachedOffsetsPointer, MAGIC.get(), + cachedFDPointer, cachedOffsetsPointer, cachedImageHeapRelocationsPtr, MAGIC.get(), heapBegin, heapEnd, heapRelocBegin, heapAnyRelocPointer, heapRelocEnd, heapWritableBegin, heapWritableEnd); @@ -252,7 +259,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W remainingSize = remainingSize.subtract(imageHeapOffsetInAddressSpace); if (!ImageLayerBuildingSupport.buildingImageLayer()) { int result = initializeImageHeap(imageHeapStart, remainingSize, endPointer, - CACHED_IMAGE_FDS.get(), CACHED_IMAGE_HEAP_OFFSETS.get(), MAGIC.get(), + CACHED_IMAGE_FD.get(), CACHED_IMAGE_HEAP_OFFSET.get(), CACHED_IMAGE_HEAP_RELOCATIONS.get(), MAGIC.get(), IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get(), IMAGE_HEAP_RELOCATABLE_BEGIN.get(), IMAGE_HEAP_A_RELOCATABLE_POINTER.get(), IMAGE_HEAP_RELOCATABLE_END.get(), IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get()); @@ -267,6 +274,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W @Uninterruptible(reason = "Called during isolate initialization.") private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedSize, WordPointer endPointer, WordPointer cachedFd, WordPointer cachedOffsetInFile, + WordPointer cachedImageHeapRelocationsPtr, Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym) { assert heapBeginSym.belowOrEqual(heapWritableSym) && heapWritableSym.belowOrEqual(heapWritableEndSym) && heapWritableEndSym.belowOrEqual(heapEndSym); assert heapBeginSym.belowOrEqual(heapRelocsSym) && heapRelocsSym.belowOrEqual(heapRelocsEndSym) && heapRelocsEndSym.belowOrEqual(heapEndSym); @@ -304,7 +312,16 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS * heap must be in pristine condition for that). */ if (fd.equal(CANNOT_OPEN_FD)) { - return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym); + int result = initializeImageHeapWithMremap(imageHeap, imageHeapSize, pageSize, cachedImageHeapRelocationsPtr, heapBeginSym, heapRelocsSym, heapAnyRelocPointer, heapRelocsEndSym, + heapWritableSym, heapWritableEndSym); + if (result == CEntryPointErrors.MREMAP_NOT_SUPPORTED) { + /* + * MREMAP_DONTUNMAP is not supported, fall back to copying it from memory (the image + * heap must be in pristine condition for that). + */ + return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym); + } + return result; } // Create memory mappings from the image file. @@ -314,11 +331,109 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS return CEntryPointErrors.MAP_HEAP_FAILED; } + int result = copyRelocations(imageHeap, pageSize, heapBeginSym, heapRelocsSym, heapAnyRelocPointer, heapRelocsEndSym, WordFactory.nullPointer()); + if (result != CEntryPointErrors.NO_ERROR) { + return result; + } + + return unprotectWritablePages(imageHeap, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym); + } + + @Uninterruptible(reason = "Called during isolate initialization.") + private static int initializeImageHeapWithMremap(Pointer imageHeap, UnsignedWord imageHeapSizeInFile, UnsignedWord pageSize, WordPointer cachedImageHeapRelocationsPtr, Word heapBeginSym, + Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym) { + if (!SubstrateOptions.MremapImageHeap.getValue()) { + return CEntryPointErrors.MREMAP_NOT_SUPPORTED; + } + + Pointer cachedImageHeapRelocations = getCachedImageHeapRelocations((Pointer) cachedImageHeapRelocationsPtr, pageSize, heapRelocsSym, heapRelocsEndSym); + assert cachedImageHeapRelocations.notEqual(0); + if (cachedImageHeapRelocations.rawValue() < 0) { + return (int) -cachedImageHeapRelocations.rawValue(); // value is a negated error code + } + + // Map the image heap for the new isolate from the template + int mremapFlags = LinuxLibCHelper.MREMAP_FIXED() | LinuxLibCHelper.MREMAP_MAYMOVE() | LinuxLibCHelper.MREMAP_DONTUNMAP(); + PointerBase res = LinuxLibCHelper.NoTransitions.mremapP(heapBeginSym, imageHeapSizeInFile, imageHeapSizeInFile, mremapFlags, imageHeap); + if (res.notEqual(imageHeap)) { + return CEntryPointErrors.MAP_HEAP_FAILED; + } + + int result = copyRelocations(imageHeap, pageSize, heapBeginSym, heapRelocsSym, heapAnyRelocPointer, heapRelocsEndSym, cachedImageHeapRelocations); + if (result != CEntryPointErrors.NO_ERROR) { + return result; + } + + if (VirtualMemoryProvider.get().protect(imageHeap, imageHeapSizeInFile, Access.READ) != 0) { + return CEntryPointErrors.PROTECT_HEAP_FAILED; + } + + return unprotectWritablePages(imageHeap, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym); + } + + /** + * Returns a valid pointer if successful; otherwise, returns a negated + * {@linkplain CEntryPointErrors error code}. + */ + @Uninterruptible(reason = "Called during isolate initialization.") + private static Pointer getCachedImageHeapRelocations(Pointer cachedImageHeapRelocationsPtr, UnsignedWord pageSize, Word heapRelocsSym, Word heapRelocsEndSym) { + Pointer imageHeapRelocations = cachedImageHeapRelocationsPtr.readWord(0, LocationIdentity.ANY_LOCATION); + if (imageHeapRelocations.isNull() || imageHeapRelocations.equal(COPY_RELOCATIONS_IN_PROGRESS)) { + if (!cachedImageHeapRelocationsPtr.logicCompareAndSwapWord(0, WordFactory.nullPointer(), COPY_RELOCATIONS_IN_PROGRESS, LocationIdentity.ANY_LOCATION)) { + /* Wait for other thread to initialize heap relocations. */ + while ((imageHeapRelocations = cachedImageHeapRelocationsPtr.readWordVolatile(0, LocationIdentity.ANY_LOCATION)).equal(COPY_RELOCATIONS_IN_PROGRESS)) { + PauseNode.pause(); + } + } else { + /* + * This is the first time mapping the heap. Create a private copy of the relocated + * image heap symbols, as these may be reverted during subsequent mremaps. + */ + + Pointer linkedRelocsBoundary = roundDown(heapRelocsSym, pageSize); + UnsignedWord heapRelocsLength = roundUp(heapRelocsEndSym.subtract(linkedRelocsBoundary), pageSize); + int mremapFlags = LinuxLibCHelper.MREMAP_MAYMOVE() | LinuxLibCHelper.MREMAP_DONTUNMAP(); + imageHeapRelocations = LinuxLibCHelper.NoTransitions.mremapP(linkedRelocsBoundary, heapRelocsLength, heapRelocsLength, mremapFlags, WordFactory.nullPointer()); + + if (imageHeapRelocations.equal(-1)) { + if (LibC.errno() == Errno.EINVAL()) { + /* + * MREMAP_DONTUNMAP with non-anonymous mappings is only supported from + * kernel version 5.13 onwards, and fails with EINVAL otherwise. + * + * https://github.com/torvalds/linux/commit/ + * a4609387859f0281951f5e476d9f76d7fb9ab321 + */ + imageHeapRelocations = WordFactory.pointer(-CEntryPointErrors.MREMAP_NOT_SUPPORTED); + } else { + imageHeapRelocations = WordFactory.pointer(-CEntryPointErrors.MAP_HEAP_FAILED); + } + } else { + if (VirtualMemoryProvider.get().protect(imageHeapRelocations, heapRelocsLength, Access.READ) != 0) { + imageHeapRelocations = WordFactory.pointer(-CEntryPointErrors.PROTECT_HEAP_FAILED); + } + } + + cachedImageHeapRelocationsPtr.writeWordVolatile(0, imageHeapRelocations); + } + } + + assert imageHeapRelocations.isNonNull() && imageHeapRelocations.notEqual(COPY_RELOCATIONS_IN_PROGRESS); + return imageHeapRelocations; + } + + @Uninterruptible(reason = "Called during isolate initialization.") + private static int copyRelocations(Pointer imageHeap, UnsignedWord pageSize, Word heapBeginSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, + Pointer cachedRelocsBoundary) { if (heapAnyRelocPointer.isNonNull()) { - ComparableWord relocatedValue = heapAnyRelocPointer.readWord(0); + Pointer linkedRelocsBoundary = roundDown(heapRelocsSym, pageSize); + Pointer sourceRelocsBoundary = cachedRelocsBoundary; + if (sourceRelocsBoundary.isNull()) { + sourceRelocsBoundary = linkedRelocsBoundary; + } + ComparableWord relocatedValue = sourceRelocsBoundary.readWord(heapAnyRelocPointer.subtract(heapRelocsSym)); ComparableWord mappedValue = imageHeap.readWord(heapAnyRelocPointer.subtract(heapBeginSym)); if (relocatedValue.notEqual(mappedValue)) { - Pointer linkedRelocsBoundary = roundDown(heapRelocsSym, pageSize); UnsignedWord relocsAlignedSize = roundUp(heapRelocsEndSym.subtract(linkedRelocsBoundary), pageSize); Pointer relocsBoundary = imageHeap.add(linkedRelocsBoundary.subtract(heapBeginSym)); /* @@ -334,16 +449,19 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS if (committedRelocsBegin.isNull() || committedRelocsBegin != relocsBoundary) { return CEntryPointErrors.PROTECT_HEAP_FAILED; } - LibC.memcpy(relocsBoundary, linkedRelocsBoundary, relocsAlignedSize); + LibC.memcpy(relocsBoundary, sourceRelocsBoundary, relocsAlignedSize); if (VirtualMemoryProvider.get().protect(relocsBoundary, relocsAlignedSize, Access.READ) != 0) { return CEntryPointErrors.PROTECT_HEAP_FAILED; } } } + return CEntryPointErrors.NO_ERROR; + } + + @Uninterruptible(reason = "Called during isolate initialization.") + private static int unprotectWritablePages(Pointer imageHeap, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym) { /* - * Unprotect writable pages. - * * The last page might be shared with the subsequent read-only huge objects partition, in * which case we make some of its data writable, which we consider acceptable. */ diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxLibCHelper.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxLibCHelper.java index 9961b0ad3a64..06e550997b51 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxLibCHelper.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxLibCHelper.java @@ -24,15 +24,48 @@ */ package com.oracle.svm.core.posix.linux; +import com.oracle.svm.core.posix.headers.PosixDirectives; + +import org.graalvm.nativeimage.c.CContext; +import org.graalvm.nativeimage.c.constant.CConstant; import org.graalvm.nativeimage.c.function.CFunction; import org.graalvm.nativeimage.c.function.CFunction.Transition; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +import jdk.graal.compiler.api.replacements.Fold; + import org.graalvm.nativeimage.c.function.CLibrary; +// Checkstyle: stop + @CLibrary(value = "libchelper", requireStatic = true) +@CContext(PosixDirectives.class) public class LinuxLibCHelper { @CFunction(transition = Transition.NO_TRANSITION) public static native int getThreadId(); @CFunction(transition = Transition.NO_TRANSITION) public static native long getThreadUserTimeSlow(int tid); + + @CConstant + public static native int MREMAP_FIXED(); + + @CConstant + public static native int MREMAP_MAYMOVE(); + + @Fold + public static int MREMAP_DONTUNMAP() { + /* + * Hardcode this constant, as it was introduced in Linux 5.7 and may not always be available + * on a target system. If so, an mremap call fails with EINVAL. + */ + return 4; + } + + public static class NoTransitions { + @CFunction(transition = Transition.NO_TRANSITION) + public static native Pointer mremapP(PointerBase oldAddress, UnsignedWord oldSize, UnsignedWord newSize, int flags, PointerBase newAddress); + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 5892e0c57bad..233cf715fd4b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -1316,4 +1316,11 @@ public static class TruffleStableOptions { @Option(help = "Allow all instantiated types to be allocated via Unsafe.allocateInstance().", type = OptionType.Expert, // deprecated = true, deprecationMessage = "ThrowMissingRegistrationErrors is the preferred way of configuring this on a per-type level.") // public static final HostedOptionKey AllowUnsafeAllocationOfAllInstantiatedTypes = new HostedOptionKey<>(null); + + @Option(help = "Enable fallback to mremap for initializing the image heap.")// + public static final HostedOptionKey MremapImageHeap = new HostedOptionKey<>(true, key -> { + if (!Platform.includedIn(Platform.LINUX.class)) { + throw UserError.invalidOptionValue(key, key.getValue(), "Mapping the image heap with mremap() is only supported on Linux."); + } + }); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/CEntryPointErrors.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/CEntryPointErrors.java index 00c59cf5ded1..6a5ee2608944 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/CEntryPointErrors.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/CEntryPointErrors.java @@ -97,6 +97,9 @@ private CEntryPointErrors() { @Description("The image heap does not fit in the available address space.") // public static final int INSUFFICIENT_ADDRESS_SPACE = 802; + @Description("The operating system does not support mremap.") // + public static final int MREMAP_NOT_SUPPORTED = 803; + @Description("Setting the protection of the heap memory failed.") // public static final int PROTECT_HEAP_FAILED = 9; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java index 684c9960c412..6866127b1529 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java @@ -38,11 +38,14 @@ public abstract class ImageLayerSection implements LayeredImageSingleton { protected final CGlobalData initialSectionStart; protected final CGlobalData cachedImageFDs; protected final CGlobalData cachedImageHeapOffsets; + protected final CGlobalData cachedImageHeapRelocations; - protected ImageLayerSection(CGlobalData initialSectionStart, CGlobalData cachedImageFDs, CGlobalData cachedImageHeapOffsets) { + protected ImageLayerSection(CGlobalData initialSectionStart, CGlobalData cachedImageFDs, CGlobalData cachedImageHeapOffsets, + CGlobalData cachedImageHeapRelocations) { this.initialSectionStart = initialSectionStart; this.cachedImageFDs = cachedImageFDs; this.cachedImageHeapOffsets = cachedImageHeapOffsets; + this.cachedImageHeapRelocations = cachedImageHeapRelocations; } public enum SectionEntries { @@ -80,6 +83,11 @@ public static CGlobalData getCachedImageHeapOffsets() { return singleton().cachedImageHeapOffsets; } + @Fold + public static CGlobalData getCachedImageHeapRelocations() { + return singleton().cachedImageHeapRelocations; + } + protected abstract int getEntryOffsetInternal(SectionEntries entry); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ImageLayerSectionFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ImageLayerSectionFeature.java index 5b61fcad1fa4..2d30bf461fce 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ImageLayerSectionFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ImageLayerSectionFeature.java @@ -100,6 +100,7 @@ public final class ImageLayerSectionFeature implements InternalFeature, FeatureS private static final String CACHED_IMAGE_FDS_NAME = "__svm_layer_cached_image_fds"; private static final String CACHED_IMAGE_HEAP_OFFSETS_NAME = "__svm_layer_cached_image_heap_offsets"; + private static final String CACHED_IMAGE_HEAP_RELOCATIONS_NAME = "__svm_layer_cached_image_heap_relocations"; private static final SignedWord UNASSIGNED_FD = signed(-1); @@ -136,19 +137,23 @@ private static ImageLayerSectionImpl createImageLayerSection() { CGlobalData initialSectionStart = ImageLayerBuildingSupport.buildingInitialLayer() ? CGlobalDataFactory.forSymbol(getLayerName(DynamicImageLayerInfo.singleton().layerNumber)) : null; CGlobalData cachedImageFDs; CGlobalData cachedImageHeapOffsets; + CGlobalData cachedImageHeapRelocations; if (ImageLayerBuildingSupport.buildingInitialLayer()) { cachedImageFDs = CGlobalDataFactory.forSymbol(CACHED_IMAGE_FDS_NAME); cachedImageHeapOffsets = CGlobalDataFactory.forSymbol(CACHED_IMAGE_HEAP_OFFSETS_NAME); + cachedImageHeapRelocations = CGlobalDataFactory.forSymbol(CACHED_IMAGE_HEAP_RELOCATIONS_NAME); } else if (ImageLayerBuildingSupport.buildingApplicationLayer()) { cachedImageFDs = CGlobalDataFactory.createBytes(() -> createWords(DynamicImageLayerInfo.singleton().numLayers, UNASSIGNED_FD), CACHED_IMAGE_FDS_NAME); cachedImageHeapOffsets = CGlobalDataFactory.createBytes(() -> createWords(DynamicImageLayerInfo.singleton().numLayers, WordFactory.zero()), CACHED_IMAGE_HEAP_OFFSETS_NAME); + cachedImageHeapRelocations = CGlobalDataFactory.createBytes(() -> createWords(DynamicImageLayerInfo.singleton().numLayers, WordFactory.zero()), CACHED_IMAGE_HEAP_RELOCATIONS_NAME); } else { cachedImageFDs = null; cachedImageHeapOffsets = null; + cachedImageHeapRelocations = null; } - return new ImageLayerSectionImpl(initialSectionStart, cachedImageFDs, cachedImageHeapOffsets); + return new ImageLayerSectionImpl(initialSectionStart, cachedImageFDs, cachedImageHeapOffsets, cachedImageHeapRelocations); } @Override @@ -156,6 +161,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { if (ImageLayerBuildingSupport.buildingApplicationLayer()) { CGlobalDataFeature.singleton().registerWithGlobalSymbol(ImageLayerSection.getCachedImageFDs()); CGlobalDataFeature.singleton().registerWithGlobalSymbol(ImageLayerSection.getCachedImageHeapOffsets()); + CGlobalDataFeature.singleton().registerWithGlobalSymbol(ImageLayerSection.getCachedImageHeapRelocations()); } } @@ -240,8 +246,9 @@ public void beforeImageWrite(BeforeImageWriteAccess access) { private static class ImageLayerSectionImpl extends ImageLayerSection implements UnsavedSingleton { - ImageLayerSectionImpl(CGlobalData initialSectionStart, CGlobalData cachedImageFDs, CGlobalData cachedImageHeapOffsets) { - super(initialSectionStart, cachedImageFDs, cachedImageHeapOffsets); + ImageLayerSectionImpl(CGlobalData initialSectionStart, CGlobalData cachedImageFDs, CGlobalData cachedImageHeapOffsets, + CGlobalData cachedImageHeapRelocations) { + super(initialSectionStart, cachedImageFDs, cachedImageHeapOffsets, cachedImageHeapRelocations); } @Override diff --git a/substratevm/src/com.oracle.svm.native.libchelper/src/varargs_helper.c b/substratevm/src/com.oracle.svm.native.libchelper/src/varargs_helper.c index 68db4f8a7692..9a9dab8ec6ee 100644 --- a/substratevm/src/com.oracle.svm.native.libchelper/src/varargs_helper.c +++ b/substratevm/src/com.oracle.svm.native.libchelper/src/varargs_helper.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,11 @@ #ifndef _WIN64 +#ifdef __linux__ +#define _GNU_SOURCE +#include +#endif + #include #include @@ -53,4 +58,10 @@ int openatISII(int dirfd, const char *pathname, int flags, int mode) return openat(dirfd, pathname, flags, mode); } +#ifdef __linux__ +void *mremapP(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) { + return mremap(old_address, old_size, new_size, flags, new_address); +} +#endif + #endif