diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java index bd0c31201616..c047b80ba98a 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java @@ -5254,6 +5254,18 @@ public void clflush(AMD64Address adr) { emitOperandHelper(7, adr, 0); } + public void wrpkru() { + emitByte(0x0F); + emitByte(0x01); + emitByte(0xEF); + } + + public void rdpkru() { + emitByte(0x0F); + emitByte(0x01); + emitByte(0xEE); + } + public final void vpaddd(Register dst, Register nds, Register src, AVXKind.AVXSize size) { VexRVMOp.VPADDD.emit(this, size, dst, nds, src); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64LIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64LIRGenerator.java index b8ae88ca300b..1aea29ff5f6f 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64LIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64LIRGenerator.java @@ -1149,6 +1149,20 @@ public void emitSpeculationFence() { append(new AMD64LFenceOp()); } + @Override + public void emitProtectionKeyRegisterWrite(Value value) { + RegisterValue rax = AMD64.rax.asValue(value.getValueKind()); + emitMove(rax, value); + append(new AMD64WriteDataToUserPageKeyRegister(rax)); + } + + @Override + public Value emitProtectionKeyRegisterRead() { + AMD64ReadDataFromUserPageKeyRegister rdpkru = new AMD64ReadDataFromUserPageKeyRegister(); + append(rdpkru); + return emitReadRegister(AMD64.rax, rdpkru.retVal.getValueKind()); + } + @Override public void emitZeroMemory(Value address, Value length, boolean isAligned) { RegisterValue lengthReg = AMD64.rcx.asValue(length.getValueKind()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64ReadDataFromUserPageKeyRegister.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64ReadDataFromUserPageKeyRegister.java new file mode 100644 index 000000000000..23d5dd8e0677 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64ReadDataFromUserPageKeyRegister.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.core.amd64; + +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.core.common.LIRKind; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.Opcode; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.code.ValueUtil; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; + +/** + * Reads the value of PKRU into EAX and clears EDX. ECX must be 0 when RDPKRU is executed; + * otherwise, a general-protection exception (#GP) occurs. RDPKRU can be executed only if CR4.PKE = + * 1; otherwise, an invalid-opcode exception (#UD) occurs. Software can discover the value of + * CR4.PKE by examining CPUID.(EAX=07H,ECX=0H):ECX.OSPKE [bit 4]. On processors that support the + * Intel 64 Architecture, the high-order 32-bits of RCX are ignored and the high-order 32-bits of + * RDX and RAX are cleared. + */ +@Opcode("RDPKRU") +public class AMD64ReadDataFromUserPageKeyRegister extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64ReadDataFromUserPageKeyRegister.class); + + // the result of the rdpkru is in eax + @Def protected Value retVal; + + // edx will be cleared + @Temp({REG}) protected AllocatableValue edx; + + // ecx must be zero + @Temp({REG}) protected AllocatableValue zeroArg1; + + public AMD64ReadDataFromUserPageKeyRegister() { + super(TYPE); + this.retVal = AMD64.rax.asValue(LIRKind.value(AMD64Kind.DWORD)); + this.edx = AMD64.rdx.asValue(LIRKind.value(AMD64Kind.DWORD)); + this.zeroArg1 = AMD64.rcx.asValue(LIRKind.value(AMD64Kind.DWORD)); + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + masm.xorl(ValueUtil.asRegister(zeroArg1), ValueUtil.asRegister(zeroArg1)); + masm.rdpkru(); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64WriteDataToUserPageKeyRegister.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64WriteDataToUserPageKeyRegister.java new file mode 100644 index 000000000000..38b568c52628 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64WriteDataToUserPageKeyRegister.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.core.amd64; + +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.core.common.LIRKind; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.Opcode; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.code.RegisterValue; +import jdk.vm.ci.code.ValueUtil; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; + +/** + * Writes the value of EAX into PKRU. ECX and EDX must be 0 when WRPKRU is executed; otherwise, a + * general protection exception (#GP) occurs. WRPKRU can be executed only if CR4.PKE = 1; otherwise, + * an invalid-opcode exception (#UD) occurs. Software can discover the value of CR4.PKE by examining + * CPUID.(EAX=07H,ECX=0H):ECX.OSPKE [bit 4]. On processors that support the Intel 64 Architecture, + * the high-order 32-bits of RCX, RDX and RAX are ignored. + */ +@Opcode("WRPKRU") +public class AMD64WriteDataToUserPageKeyRegister extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64WriteDataToUserPageKeyRegister.class); + + // the argument to wrpkru is in eax + @Use protected Value arg; + // ecx and edx need to be zero + @Temp({REG}) protected AllocatableValue zeroArg1; + @Temp({REG}) protected AllocatableValue zeroArg2; + + public AMD64WriteDataToUserPageKeyRegister(RegisterValue arg) { + super(TYPE); + assert arg.getRegister().equals(AMD64.rax); + this.arg = arg; + zeroArg1 = AMD64.rcx.asValue(LIRKind.value(AMD64Kind.DWORD)); + zeroArg2 = AMD64.rdx.asValue(LIRKind.value(AMD64Kind.DWORD)); + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + masm.xorl(ValueUtil.asRegister(zeroArg1), ValueUtil.asRegister(zeroArg1)); + masm.xorl(ValueUtil.asRegister(zeroArg2), ValueUtil.asRegister(zeroArg2)); + masm.wrpkru(); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/ReadProtectionKeyRegisterNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/ReadProtectionKeyRegisterNode.java new file mode 100644 index 000000000000..99f19a98ef7a --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/ReadProtectionKeyRegisterNode.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.core.amd64; + +import jdk.graal.compiler.core.common.type.IntegerStamp; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.nodeinfo.NodeCycles; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodeinfo.NodeSize; +import jdk.graal.compiler.nodes.FixedWithNextNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; +import jdk.vm.ci.meta.Value; + +@NodeInfo(cycles = NodeCycles.CYCLES_1, size = NodeSize.SIZE_4) +public class ReadProtectionKeyRegisterNode extends FixedWithNextNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ReadProtectionKeyRegisterNode.class); + + public ReadProtectionKeyRegisterNode() { + super(TYPE, IntegerStamp.create(32)); + } + + @Override + public void generate(NodeLIRBuilderTool gen) { + Value result = gen.getLIRGeneratorTool().emitProtectionKeyRegisterRead(); + gen.setResult(this, result); + } + + @NodeIntrinsic + public static native int readProtectionKeyRegister(); +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/WriteProtectionKeyRegisterNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/WriteProtectionKeyRegisterNode.java new file mode 100644 index 000000000000..a09ba1da0d8c --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/WriteProtectionKeyRegisterNode.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.core.amd64; + +import jdk.graal.compiler.core.common.type.StampFactory; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.nodeinfo.NodeCycles; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodeinfo.NodeSize; +import jdk.graal.compiler.nodes.FixedWithNextNode; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; +import org.graalvm.word.WordBase; + +@NodeInfo(cycles = NodeCycles.CYCLES_16, size = NodeSize.SIZE_4) +public class WriteProtectionKeyRegisterNode extends FixedWithNextNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(WriteProtectionKeyRegisterNode.class); + + @Input protected ValueNode value; + + public WriteProtectionKeyRegisterNode(ValueNode value) { + super(TYPE, StampFactory.forVoid()); + this.value = value; + } + + @Override + public void generate(NodeLIRBuilderTool gen) { + gen.getLIRGeneratorTool().emitProtectionKeyRegisterWrite(gen.operand(value)); + } + + @NodeIntrinsic + public static native void writeProtectionKeyRegister(WordBase value); +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/LIRGeneratorTool.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/LIRGeneratorTool.java index 43f0a4494c90..a6d50fed904d 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/LIRGeneratorTool.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/LIRGeneratorTool.java @@ -569,6 +569,24 @@ default void emitConvertZeroToNull(AllocatableValue result, Value input) { */ void emitSpeculationFence(); + /** + * Write value to the protection key register. + * + * @param value to be written + */ + default void emitProtectionKeyRegisterWrite(Value value) { + throw new GraalError("Emitting code to write a value to the protection key register is not currently supported on %s", target().arch); + } + + /** + * Read contents of the protection key register. + * + * @return value read from the register + */ + default Value emitProtectionKeyRegisterRead() { + throw new GraalError("Emitting code to read the contents of the protection key register is not currently supported on %s", target().arch); + } + default VirtualStackSlot allocateStackMemory(int sizeInBytes, int alignmentInBytes) { return getResult().getFrameMapBuilder().allocateStackMemory(sizeInBytes, alignmentInBytes); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/extended/SpeculationFenceNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/extended/SpeculationFenceNode.java new file mode 100644 index 000000000000..244306facff2 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/extended/SpeculationFenceNode.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.extended; + +import jdk.graal.compiler.core.common.type.StampFactory; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.nodeinfo.NodeCycles; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodeinfo.NodeSize; +import jdk.graal.compiler.nodes.FixedWithNextNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +@NodeInfo(cycles = NodeCycles.CYCLES_4, size = NodeSize.SIZE_4) +public class SpeculationFenceNode extends FixedWithNextNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(SpeculationFenceNode.class); + + public SpeculationFenceNode() { + super(TYPE, StampFactory.forVoid()); + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + generator.getLIRGeneratorTool().emitSpeculationFence(); + } + + @NodeIntrinsic + public static native void memoryBarrier(); +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/CInterfaceWrapper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/CInterfaceWrapper.java new file mode 100644 index 000000000000..fa1ae13f4f63 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/CInterfaceWrapper.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.c; + +import com.oracle.svm.core.graal.replacements.SubstrateGraphKit; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode; +import jdk.vm.ci.meta.ResolvedJavaMethod; + +/** + * This wrapper exposes an interface that can be used to insert additional code around native + * accesses. For reads and writes, the CInterfaceReadNode and CInterfaceWriteNode can be + * conditionally replaced by another node. For function calls and entry points, the prologue and + * epilogue tagging functions can be used to add code before and after the method invocation. + */ +public interface CInterfaceWrapper { + /** + * This method is called in + * {@link com.oracle.svm.hosted.phases.CInterfaceInvocationPlugin#readPrimitive}. To indicate + * that the CInterfaceReadNode should be replaced, the callee adds the new node to the graph and + * returns it. + * + * @param b the context + * @param address base and offset for the primitive read + * @param stamp the type of the read + * @param method the original read method for the primitive object + * @return the node added to the graph in place of the CInterfaceReadNode, null otherwise. + */ + ValueNode replacePrimitiveRead(GraphBuilderContext b, OffsetAddressNode address, Stamp stamp, ResolvedJavaMethod method); + + /** + * This method is called in + * {@link com.oracle.svm.hosted.phases.CInterfaceInvocationPlugin#writePrimitive}. To indicate + * that the CInterfaceWriteNode should be replaced, the callee adds the new node to the graph + * and returns true. + * + * @param b the context + * @param address base and offset for the primitive write + * @param value the value to be written + * @param method the original write method for the primitive object + * @return true if the node should be replaced, false otherwise. + */ + boolean replacePrimitiveWrite(GraphBuilderContext b, OffsetAddressNode address, ValueNode value, ResolvedJavaMethod method); + + void tagCEntryPointPrologue(SubstrateGraphKit kit, ResolvedJavaMethod method); + + void tagCEntryPointEpilogue(SubstrateGraphKit kit, ResolvedJavaMethod method); + + void tagCFunctionCallPrologue(SubstrateGraphKit kit, ResolvedJavaMethod method); + + void tagCFunctionCallEpilogue(SubstrateGraphKit kit, ResolvedJavaMethod method); +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CCallStubMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CCallStubMethod.java index bdf52b2e741d..d8acd0017412 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CCallStubMethod.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CCallStubMethod.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.constant.CEnum; import org.graalvm.nativeimage.c.constant.CEnumLookup; @@ -37,6 +38,7 @@ import com.oracle.svm.common.meta.MultiMethod; import com.oracle.svm.core.util.UserError; import com.oracle.svm.hosted.annotation.CustomSubstitutionMethod; +import com.oracle.svm.hosted.c.CInterfaceWrapper; import com.oracle.svm.hosted.c.NativeLibraries; import com.oracle.svm.hosted.c.info.ElementInfo; import com.oracle.svm.hosted.c.info.EnumInfo; @@ -75,7 +77,17 @@ public StructuredGraph buildGraph(DebugContext debug, AnalysisMethod method, Hos AnalysisType[] paramTypes = method.toParameterList().toArray(AnalysisType[]::new); var signature = adaptSignatureAndConvertArguments(nativeLibraries, kit, method, method.getSignature().getReturnType(), paramTypes, arguments); state.clearLocals(); + + if (ImageSingletons.contains(CInterfaceWrapper.class)) { + ImageSingletons.lookup(CInterfaceWrapper.class).tagCFunctionCallPrologue(kit, method); + } + ValueNode returnValue = kit.createCFunctionCall(callAddress, arguments, signature, newThreadStatus, deoptimizationTarget); + + if (ImageSingletons.contains(CInterfaceWrapper.class)) { + ImageSingletons.lookup(CInterfaceWrapper.class).tagCFunctionCallEpilogue(kit, method); + } + returnValue = adaptReturnValue(method, nativeLibraries, kit, returnValue); kit.createReturn(returnValue, signature.getReturnKind()); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CEntryPointCallStubMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CEntryPointCallStubMethod.java index 3f378f124fb5..24027256ec0f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CEntryPointCallStubMethod.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CEntryPointCallStubMethod.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ import java.util.Iterator; import java.util.List; +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Isolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.c.constant.CEnum; @@ -58,6 +59,7 @@ import com.oracle.svm.core.graal.replacements.SubstrateGraphKit; import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.c.CInterfaceWrapper; import com.oracle.svm.hosted.c.NativeLibraries; import com.oracle.svm.hosted.c.info.ElementInfo; import com.oracle.svm.hosted.c.info.EnumInfo; @@ -159,6 +161,10 @@ public StructuredGraph buildGraph(DebugContext debug, AnalysisMethod method, Hos ValueNode[] args = kit.getInitialArguments().toArray(ValueNode.EMPTY_ARRAY); + if (ImageSingletons.contains(CInterfaceWrapper.class)) { + ImageSingletons.lookup(CInterfaceWrapper.class).tagCEntryPointPrologue(kit, method); + } + InvokeWithExceptionNode invokePrologue = generatePrologue(kit, parameterLoadTypes, targetMethod.getParameterAnnotations(), args); if (invokePrologue != null) { ResolvedJavaMethod prologueMethod = invokePrologue.callTarget().targetMethod(); @@ -218,16 +224,21 @@ public StructuredGraph buildGraph(DebugContext debug, AnalysisMethod method, Hos patchNodeSourcePosition(invoke); kit.exceptionPart(); ExceptionObjectNode exception = kit.exceptionObject(); - generateExceptionHandler(kit, exception, invoke.getStackKind()); + generateExceptionHandler(method, kit, exception, invoke.getStackKind()); kit.endInvokeWithException(); - generateEpilogueAndReturn(kit, invoke); + generateEpilogueAndReturn(method, kit, invoke); return kit.finalizeGraph(); } - private void generateEpilogueAndReturn(HostedGraphKit kit, ValueNode value) { + private void generateEpilogueAndReturn(ResolvedJavaMethod method, HostedGraphKit kit, ValueNode value) { ValueNode returnValue = adaptReturnValue(kit, value); generateEpilogue(kit); + + if (ImageSingletons.contains(CInterfaceWrapper.class)) { + ImageSingletons.lookup(CInterfaceWrapper.class).tagCEntryPointEpilogue(kit, method); + } + kit.createReturn(returnValue, returnValue.getStackKind()); } @@ -249,6 +260,10 @@ private StructuredGraph buildBuiltinGraph(DebugContext debug, AnalysisMethod met UserError.guarantee(entryPointData.getExceptionHandler() == CEntryPointData.DEFAULT_EXCEPTION_HANDLER, "@%s method declared as built-in must not have a custom exception handler: %s", CEntryPoint.class.getSimpleName(), aTargetMethod); + if (ImageSingletons.contains(CInterfaceWrapper.class)) { + ImageSingletons.lookup(CInterfaceWrapper.class).tagCEntryPointPrologue(kit, method); + } + ExecutionContextParameters executionContext = findExecutionContextParameters(kit, aTargetMethod.toParameterList(), aTargetMethod.getParameterAnnotations()); final CEntryPoint.Builtin builtin = entryPointData.getBuiltin(); @@ -304,9 +319,13 @@ private StructuredGraph buildBuiltinGraph(DebugContext debug, AnalysisMethod met kit.exceptionPart(); ExceptionObjectNode exception = kit.exceptionObject(); - generateExceptionHandler(kit, exception, invoke.getStackKind()); + generateExceptionHandler(method, kit, exception, invoke.getStackKind()); kit.endInvokeWithException(); + if (ImageSingletons.contains(CInterfaceWrapper.class)) { + ImageSingletons.lookup(CInterfaceWrapper.class).tagCEntryPointEpilogue(kit, method); + } + kit.createReturn(invoke, aTargetMethod.getSignature().getReturnKind()); return kit.finalizeGraph(); @@ -502,7 +521,7 @@ private ValueNode[] matchPrologueParameters(HostedGraphKit kit, List resultKind.getBitCount() && !readKind.isNumericFloat() && resultKind != JavaKind.Boolean) { readKind = resultKind; } - AddressNode offsetAddress = makeOffsetAddress(graph, args, accessorInfo, base, displacement, elementSize); + OffsetAddressNode offsetAddress = makeOffsetAddress(graph, args, accessorInfo, base, displacement, elementSize); LocationIdentity locationIdentity = makeLocationIdentity(b, method, args, accessorInfo); final Stamp stamp; if (readKind == JavaKind.Object) { @@ -195,7 +196,7 @@ private static boolean replaceAccessor(GraphBuilderContext b, AnalysisMethod met if (isPinnedObject) { node = b.add(new JavaReadNode(stamp, readKind, offsetAddress, locationIdentity, BarrierType.NONE, MemoryOrderMode.PLAIN, true)); } else { - ValueNode read = readPrimitive(b, offsetAddress, locationIdentity, stamp, accessorInfo); + ValueNode read = readPrimitive(b, offsetAddress, locationIdentity, stamp, accessorInfo, method); node = adaptPrimitiveType(graph, read, readKind, resultKind == JavaKind.Boolean ? resultKind : resultKind.getStackKind(), isUnsigned); } b.push(pushKind(method), node); @@ -205,13 +206,13 @@ private static boolean replaceAccessor(GraphBuilderContext b, AnalysisMethod met ValueNode value = args[accessorInfo.valueParameterNumber(true)]; JavaKind valueKind = value.getStackKind(); JavaKind writeKind = kindFromSize(elementSize, valueKind); - AddressNode offsetAddress = makeOffsetAddress(graph, args, accessorInfo, base, displacement, elementSize); + OffsetAddressNode offsetAddress = makeOffsetAddress(graph, args, accessorInfo, base, displacement, elementSize); LocationIdentity locationIdentity = makeLocationIdentity(b, method, args, accessorInfo); if (isPinnedObject) { b.add(new JavaWriteNode(writeKind, offsetAddress, locationIdentity, value, BarrierType.NONE, true)); } else { ValueNode adaptedValue = adaptPrimitiveType(graph, value, valueKind, writeKind, isUnsigned); - writePrimitive(b, offsetAddress, locationIdentity, adaptedValue, accessorInfo); + writePrimitive(b, offsetAddress, locationIdentity, adaptedValue, accessorInfo, method); } return true; } @@ -280,10 +281,10 @@ private static boolean replaceBitfieldAccessor(GraphBuilderContext b, AnalysisMe * Read the memory location. This is also necessary for writes, since we need to keep the * bits around the written bitfield unchanged. */ - AddressNode address = makeOffsetAddress(graph, args, accessorInfo, base, byteOffset, -1); + OffsetAddressNode address = makeOffsetAddress(graph, args, accessorInfo, base, byteOffset, -1); LocationIdentity locationIdentity = makeLocationIdentity(b, method, args, accessorInfo); Stamp stamp = IntegerStamp.create(memoryKind.getBitCount()); - ValueNode cur = readPrimitive(b, address, locationIdentity, stamp, accessorInfo); + ValueNode cur = readPrimitive(b, address, locationIdentity, stamp, accessorInfo, method); cur = adaptPrimitiveType(graph, cur, memoryKind, computeKind, true); switch (accessorInfo.getAccessorKind()) { @@ -328,7 +329,7 @@ private static boolean replaceBitfieldAccessor(GraphBuilderContext b, AnalysisMe /* Narrow value to the number of bits we need to write. */ cur = adaptPrimitiveType(graph, cur, computeKind, memoryKind, true); /* Perform the write (bitcount is taken from the stamp of the written value). */ - writePrimitive(b, address, locationIdentity, cur, accessorInfo); + writePrimitive(b, address, locationIdentity, cur, accessorInfo, method); return true; } default: @@ -336,7 +337,14 @@ private static boolean replaceBitfieldAccessor(GraphBuilderContext b, AnalysisMe } } - private static ValueNode readPrimitive(GraphBuilderContext b, AddressNode address, LocationIdentity locationIdentity, Stamp stamp, AccessorInfo accessorInfo) { + private static ValueNode readPrimitive(GraphBuilderContext b, OffsetAddressNode address, LocationIdentity locationIdentity, Stamp stamp, AccessorInfo accessorInfo, ResolvedJavaMethod method) { + if (ImageSingletons.contains(CInterfaceWrapper.class)) { + ValueNode replaced = ImageSingletons.lookup(CInterfaceWrapper.class).replacePrimitiveRead(b, address, stamp, method); + if (replaced != null) { + return replaced; + } + } + CInterfaceReadNode read = b.add(new CInterfaceReadNode(address, locationIdentity, stamp, BarrierType.NONE, MemoryOrderMode.PLAIN, accessName(accessorInfo))); /* * The read must not float outside its block otherwise it may float above an explicit zero @@ -346,7 +354,14 @@ private static ValueNode readPrimitive(GraphBuilderContext b, AddressNode addres return read; } - private static void writePrimitive(GraphBuilderContext b, AddressNode address, LocationIdentity locationIdentity, ValueNode value, AccessorInfo accessorInfo) { + private static void writePrimitive(GraphBuilderContext b, OffsetAddressNode address, LocationIdentity locationIdentity, ValueNode value, AccessorInfo accessorInfo, ResolvedJavaMethod method) { + if (ImageSingletons.contains(CInterfaceWrapper.class)) { + boolean replaced = ImageSingletons.lookup(CInterfaceWrapper.class).replacePrimitiveWrite(b, address, value, method); + if (replaced) { + return; + } + } + b.add(new CInterfaceWriteNode(address, locationIdentity, value, BarrierType.NONE, MemoryOrderMode.PLAIN, accessName(accessorInfo))); } @@ -373,7 +388,7 @@ private static ValueNode makeOffset(StructuredGraph graph, ValueNode[] args, Acc return offset; } - private static AddressNode makeOffsetAddress(StructuredGraph graph, ValueNode[] args, AccessorInfo accessorInfo, ValueNode base, int displacement, int indexScaling) { + private static OffsetAddressNode makeOffsetAddress(StructuredGraph graph, ValueNode[] args, AccessorInfo accessorInfo, ValueNode base, int displacement, int indexScaling) { return graph.addOrUniqueWithInputs(new OffsetAddressNode(base, makeOffset(graph, args, accessorInfo, displacement, indexScaling))); }