Skip to content

Commit

Permalink
type inference for atomic instructions
Browse files Browse the repository at this point in the history
  • Loading branch information
@VyacheslavLevytskyy
VyacheslavLevytskyy committed Jul 8, 2024
1 parent be1dd53 commit b6537c8
Show file tree
Hide file tree
Showing 5 changed files with 149 additions and 123 deletions.
58 changes: 49 additions & 9 deletions llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,15 +389,13 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited,
UnknownElemTypeI8);
} else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) {
Type *RefTy = deduceElementTypeHelper(Ref->getNewValOperand(), Visited,
UnknownElemTypeI8);
if (UnknownElemTypeI8 || !isUntypedPointerTy(RefTy))
Ty = RefTy;
Value *Op = Ref->getNewValOperand();
if (isPointerTy(Op->getType()))
Ty = deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8);
} else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) {
Type *RefTy = deduceElementTypeHelper(Ref->getValOperand(), Visited,
UnknownElemTypeI8);
if (UnknownElemTypeI8 || !isUntypedPointerTy(RefTy))
Ty = RefTy;
Value *Op = Ref->getValOperand();
if (isPointerTy(Op->getType()))
Ty = deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8);
} else if (auto *Ref = dyn_cast<PHINode>(I)) {
for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
Ty = deduceElementTypeByUsersDeep(Ref->getIncomingValue(i), Visited,
Expand Down Expand Up @@ -539,6 +537,19 @@ Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
return UnknownElemTypeI8 ? IntegerType::getInt8Ty(I->getContext()) : nullptr;
}

static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
Value *PointerOperand) {
Type *PointeeTy = GR->findDeducedElementType(PointerOperand);
if (PointeeTy && !isUntypedPointerTy(PointeeTy))
return nullptr;
auto *PtrTy = dyn_cast<PointerType>(I->getType());
if (!PtrTy)
return I->getType();
if (Type *NestedTy = GR->findDeducedElementType(I))
return getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
return nullptr;
}

// If the Instruction has Pointer operands with unresolved types, this function
// tries to deduce them. If the Instruction has Pointer operands with known
// types which differ from expected, this function tries to insert a bitcast to
Expand All @@ -561,14 +572,36 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I) {
if (!KnownElemTy)
return;
Ops.push_back(std::make_pair(Ref->getPointerOperand(), 0));
} else if (auto *Ref = dyn_cast<LoadInst>(I)) {
KnownElemTy = I->getType();
if (isUntypedPointerTy(KnownElemTy))
return;
Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
if (PointeeTy && !isUntypedPointerTy(PointeeTy))
return;
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
LoadInst::getPointerOperandIndex()));
} else if (auto *Ref = dyn_cast<StoreInst>(I)) {
KnownElemTy = Ref->getValueOperand()->getType();
if (isUntypedPointerTy(KnownElemTy))
return;
if (GR->findDeducedElementType(Ref->getPointerOperand()))
Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
if (PointeeTy && !isUntypedPointerTy(PointeeTy))
return;
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
StoreInst::getPointerOperandIndex()));
} else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) {
KnownElemTy = getAtomicElemTy(GR, I, Ref->getPointerOperand());
if (!KnownElemTy)
return;
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
AtomicCmpXchgInst::getPointerOperandIndex()));
} else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) {
KnownElemTy = getAtomicElemTy(GR, I, Ref->getPointerOperand());
if (!KnownElemTy)
return;
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
AtomicRMWInst::getPointerOperandIndex()));
} else if (auto *Ref = dyn_cast<SelectInst>(I)) {
if (!isPointerTy(I->getType()) ||
!(KnownElemTy = GR->findDeducedElementType(I)))
Expand Down Expand Up @@ -1512,6 +1545,7 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
for (auto &I : instructions(Func))
Worklist.push_back(&I);

// SmallVector<Instruction *> Postponed;
for (auto &I : Worklist) {
// Don't emit intrinsincs for convergence intrinsics.
if (isConvergenceIntrinsic(I))
Expand All @@ -1526,11 +1560,17 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
// already, and force it to be i8 if not
if (Postpone && !GR->findAssignPtrTypeInstr(I))
insertAssignPtrTypeIntrs(I, B, true);
// Postponed.push_back(I);
}

for (auto &I : instructions(Func))
deduceOperandElementType(&I);

// for (auto &I : Postponed)
// insertAssignPtrTypeIntrs(I, B, true);
// for (auto IB = Postponed.rbegin(), IE = Postponed.rend(); IB != IE; ++IB)
// insertAssignPtrTypeIntrs(*IB, B, true);

for (auto *I : Worklist) {
TrackConstants = true;
if (!I->getType()->isVoidTy() || isa<StoreInst>(I))
Expand Down
22 changes: 16 additions & 6 deletions llvm/test/CodeGen/SPIRV/instructions/atomic-ptr.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,33 @@
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s

; CHECK-DAG: %[[#CharTy:]] = OpTypeInt 8 0
; CHECK-DAG: %[[#LongTy:]] = OpTypeInt 64 0
; CHECK-DAG: %[[#PtrCharTy:]] = OpTypePointer CrossWorkgroup %[[#CharTy]]
; CHECK-DAG: %[[#PtrLongTy:]] = OpTypePointer CrossWorkgroup %[[#LongTy]]
; CHECK-DAG: %[[#IntTy:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Scope:]] = OpConstant %[[#IntTy]] 1
; CHECK-DAG: %[[#MemSem:]] = OpConstant %[[#IntTy]] 8
; CHECK-DAG: %[[#PtrPtrLongTy:]] = OpTypePointer CrossWorkgroup %[[#PtrLongTy]]

; CHECK: OpFunction
; CHECK: %[[#Arg1:]] = OpFunctionParameter %[[#PtrCharTy]]
; CHECK: %[[#Arg1:]] = OpFunctionParameter %[[#PtrPtrLongTy]]
; CHECK: %[[#Arg2:]] = OpFunctionParameter %[[#PtrLongTy]]
; CHECK: %[[#CastedArg1:]] = OpBitcast %[[#PtrPtrLongTy]] %[[#Arg1]]
; CHECK: OpAtomicExchange %[[#PtrLongTy]] %[[#CastedArg1]] %[[#Scope]] %[[#MemSem]] %[[#Arg2]]
; CHECK: OpAtomicExchange %[[#PtrLongTy]] %[[#Arg1]] %[[#Scope]] %[[#MemSem]] %[[#Arg2]]
; CHECK: OpFunctionEnd

define dso_local spir_func void @test_atomicrmw(ptr addrspace(1) %arg1, ptr addrspace(1) byval(i64) %arg_ptr) {
define dso_local spir_func void @test1(ptr addrspace(1) %arg1, ptr addrspace(1) byval(i64) %arg_ptr) {
entry:
%r = atomicrmw xchg ptr addrspace(1) %arg1, ptr addrspace(1) %arg_ptr acq_rel
ret void
}

; CHECK: OpFunction
; CHECK: %[[#Arg3:]] = OpFunctionParameter %[[#PtrLongTy]]
; CHECK: %[[#Arg4:]] = OpFunctionParameter %[[#LongTy]]
; CHECK: OpAtomicExchange %[[#LongTy]] %[[#Arg3]] %[[#Scope]] %[[#MemSem]] %[[#Arg4]]
; CHECK: OpFunctionEnd

define dso_local spir_func void @test2(ptr addrspace(1) %arg1, i64 %arg_ptr) {
entry:
%r = atomicrmw xchg ptr addrspace(1) %arg1, i64 %arg_ptr acq_rel
ret void
}
64 changes: 28 additions & 36 deletions llvm/test/CodeGen/SPIRV/instructions/atomic.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,18 @@
; CHECK-DAG: OpName [[XOR:%.*]] "test_xor"

; CHECK-DAG: [[I32Ty:%.*]] = OpTypeInt 32 0
; CHECK-DAG: [[PtrI32Ty:%.*]] = OpTypePointer Function [[I32Ty]]
; CHECK-DAG: [[I64Ty:%.*]] = OpTypeInt 64 0
;; Device scope is encoded with constant 1
; CHECK-DAG: [[SCOPE:%.*]] = OpConstant [[I32Ty]] 1
;; "monotonic" maps to the relaxed memory semantics, encoded with constant 0
; CHECK-DAG: [[RELAXED:%.*]] = OpConstantNull [[I32Ty]]

; CHECK: [[ADD]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_add(i32* %ptr, i32 %val) {
Expand All @@ -35,11 +35,10 @@ define i32 @test_add(i32* %ptr, i32 %val) {
}

; CHECK: [[SUB]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_sub(i32* %ptr, i32 %val) {
Expand All @@ -48,11 +47,10 @@ define i32 @test_sub(i32* %ptr, i32 %val) {
}

; CHECK: [[MIN]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_min(i32* %ptr, i32 %val) {
Expand All @@ -61,11 +59,10 @@ define i32 @test_min(i32* %ptr, i32 %val) {
}

; CHECK: [[MAX]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_max(i32* %ptr, i32 %val) {
Expand All @@ -74,11 +71,10 @@ define i32 @test_max(i32* %ptr, i32 %val) {
}

; CHECK: [[UMIN]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_umin(i32* %ptr, i32 %val) {
Expand All @@ -87,11 +83,10 @@ define i32 @test_umin(i32* %ptr, i32 %val) {
}

; CHECK: [[UMAX]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_umax(i32* %ptr, i32 %val) {
Expand All @@ -100,11 +95,10 @@ define i32 @test_umax(i32* %ptr, i32 %val) {
}

; CHECK: [[AND]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_and(i32* %ptr, i32 %val) {
Expand All @@ -113,11 +107,10 @@ define i32 @test_and(i32* %ptr, i32 %val) {
}

; CHECK: [[OR]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_or(i32* %ptr, i32 %val) {
Expand All @@ -126,11 +119,10 @@ define i32 @test_or(i32* %ptr, i32 %val) {
}

; CHECK: [[XOR]] = OpFunction [[I32Ty]]
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter [[PtrI32Ty]]
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter [[I32Ty]]
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_xor(i32* %ptr, i32 %val) {
Expand Down
Loading

0 comments on commit b6537c8

Please sign in to comment.