Skip to content

Commit

Permalink
[CodeGen] Emit a more efficient magic number multiplication for exact…
Browse files Browse the repository at this point in the history 
… udivs

Have simpler lowering for exact udivs in both SelectionDAG and GlobalISel.

The algorithm is the same between unsigned exact divs and signed divs save for arithmetic vs logical shift for even divisors, according to Hacker's Delight, 2nd Edition, page 242.
  • Loading branch information
@AreaZR
AreaZR committed Jul 15, 2024
1 parent 7ed4e9f commit 4d7d698
Show file tree
Hide file tree
Showing 5 changed files with 215 additions and 162 deletions.
62 changes: 59 additions & 3 deletions llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5183,8 +5183,35 @@ MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) {
KB ? KB->getKnownBits(LHS).countMinLeadingZeros() : 0;
auto &MIB = Builder;

bool UseSRL = false;
bool UseNPQ = false;
SmallVector<Register, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
SmallVector<Register, 16> Shifts, Factors;
auto *RHSDefInstr = cast<GenericMachineInstr>(getDefIgnoringCopies(RHS, MRI));
bool IsSplat = getIConstantSplatVal(*RHSDefInstr, MRI).has_value();

auto BuildExactUDIVPattern = [&](const Constant *C) {
// Don't recompute inverses for each splat element.
if (IsSplat && !Factors.empty()) {
Shifts.push_back(Shifts[0]);
Factors.push_back(Factors[0]);
return true;
}

auto *CI = cast<ConstantInt>(C);
APInt Divisor = CI->getValue();
unsigned Shift = Divisor.countr_zero();
if (Shift) {
Divisor.lshrInPlace(Shift);
UseSRL = true;
}

// Calculate the multiplicative inverse modulo BW.
APInt Factor = Divisor.multiplicativeInverse();
Shifts.push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));
Factors.push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));
return true;
};

auto BuildUDIVPattern = [&](const Constant *C) {
auto *CI = cast<ConstantInt>(C);
Expand Down Expand Up @@ -5231,6 +5258,29 @@ MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) {
return true;
};

if (MI.getFlag(MachineInstr::MIFlag::IsExact)) {
// Collect all magic values from the build vector.
bool Matched = matchUnaryPredicate(MRI, RHS, BuildExactUDIVPattern);
(void)Matched;
assert(Matched && "Expected unary predicate match to succeed");

Register Shift, Factor;
if (Ty.isVector()) {
Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);
Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);
} else {
Shift = Shifts[0];
Factor = Factors[0];
}

Register Res = LHS;

if (UseSRL)
Res = MIB.buildLShr(Ty, Res, Shift, MachineInstr::IsExact).getReg(0);

return MIB.buildMul(Ty, Res, Factor);
}

// Collect the shifts/magic values from each element.
bool Matched = matchUnaryPredicate(MRI, RHS, BuildUDIVPattern);
(void)Matched;
Expand Down Expand Up @@ -5283,9 +5333,6 @@ bool CombinerHelper::matchUDivByConst(MachineInstr &MI) {
Register Dst = MI.getOperand(0).getReg();
Register RHS = MI.getOperand(2).getReg();
LLT DstTy = MRI.getType(Dst);
auto *RHSDef = MRI.getVRegDef(RHS);
if (!isConstantOrConstantVector(*RHSDef, MRI))
return false;

auto &MF = *MI.getMF();
AttributeList Attr = MF.getFunction().getAttributes();
Expand All @@ -5300,6 +5347,15 @@ bool CombinerHelper::matchUDivByConst(MachineInstr &MI) {
if (MF.getFunction().hasMinSize())
return false;

if (MI.getFlag(MachineInstr::MIFlag::IsExact)) {
return matchUnaryPredicate(
MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
}

auto *RHSDef = MRI.getVRegDef(RHS);
if (!isConstantOrConstantVector(*RHSDef, MRI))
return false;

// Don't do this if the types are not going to be legal.
if (LI) {
if (!isLegalOrBeforeLegalizer({TargetOpcode::G_MUL, {DstTy, DstTy}}))
Expand Down
71 changes: 68 additions & 3 deletions llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6092,6 +6092,7 @@ void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,

/// Given an exact SDIV by a constant, create a multiplication
/// with the multiplicative inverse of the constant.
/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
const SDLoc &dl, SelectionDAG &DAG,
SmallVectorImpl<SDNode *> &Created) {
Expand Down Expand Up @@ -6141,10 +6142,7 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
}

SDValue Res = Op0;

// Shift the value upfront if it is even, so the LSB is one.
if (UseSRA) {
// TODO: For UDIV use SRL instead of SRA.
SDNodeFlags Flags;
Flags.setExact(true);
Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
Expand All @@ -6154,6 +6152,69 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
}

/// Given an exact UDIV by a constant, create a multiplication
/// with the multiplicative inverse of the constant.
/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N,
const SDLoc &dl, SelectionDAG &DAG,
SmallVectorImpl<SDNode *> &Created) {
EVT VT = N->getValueType(0);
EVT SVT = VT.getScalarType();
EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();

bool UseSRL = false;
SmallVector<SDValue, 16> Shifts, Factors;

auto BuildUDIVPattern = [&](ConstantSDNode *C) {
if (C->isZero())
return false;
APInt Divisor = C->getAPIntValue();
unsigned Shift = Divisor.countr_zero();
if (Shift) {
Divisor.lshrInPlace(Shift);
UseSRL = true;
}
// Calculate the multiplicative inverse modulo BW.
APInt Factor = Divisor.multiplicativeInverse();
Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
Factors.push_back(DAG.getConstant(Factor, dl, SVT));
return true;
};

SDValue Op1 = N->getOperand(1);

// Collect all magic values from the build vector.
if (!ISD::matchUnaryPredicate(Op1, BuildUDIVPattern))
return SDValue();

SDValue Shift, Factor;
if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
Shift = DAG.getBuildVector(ShVT, dl, Shifts);
Factor = DAG.getBuildVector(VT, dl, Factors);
} else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
assert(Shifts.size() == 1 && Factors.size() == 1 &&
"Expected matchUnaryPredicate to return one element for scalable "
"vectors");
Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
Factor = DAG.getSplatVector(VT, dl, Factors[0]);
} else {
assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
Shift = Shifts[0];
Factor = Factors[0];
}

SDValue Res = N->getOperand(0);
if (UseSRL) {
SDNodeFlags Flags;
Flags.setExact(true);
Res = DAG.getNode(ISD::SRL, dl, VT, Res, Shift, Flags);
Created.push_back(Res.getNode());
}

return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
}

SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
SelectionDAG &DAG,
SmallVectorImpl<SDNode *> &Created) const {
Expand Down Expand Up @@ -6413,6 +6474,10 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
return SDValue();
}

// If the udiv has an 'exact' bit we can use a simpler lowering.
if (N->getFlags().hasExact())
return BuildExactUDIV(*this, N, dl, DAG, Created);

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

Expand Down
33 changes: 10 additions & 23 deletions llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,21 +274,17 @@ define i32 @udiv_div_by_180_exact(i32 %x)
; SDAG-LABEL: udiv_div_by_180_exact:
; SDAG: // %bb.0:
; SDAG-NEXT: lsr w8, w0, #2
; SDAG-NEXT: mov w9, #27671 // =0x6c17
; SDAG-NEXT: movk w9, #5825, lsl #16
; SDAG-NEXT: umull x8, w8, w9
; SDAG-NEXT: lsr x0, x8, #34
; SDAG-NEXT: // kill: def $w0 killed $w0 killed $x0
; SDAG-NEXT: mov w9, #20389 // =0x4fa5
; SDAG-NEXT: movk w9, #42234, lsl #16
; SDAG-NEXT: mul w0, w8, w9
; SDAG-NEXT: ret
;
; GISEL-LABEL: udiv_div_by_180_exact:
; GISEL: // %bb.0:
; GISEL-NEXT: lsr w8, w0, #2
; GISEL-NEXT: mov w9, #27671 // =0x6c17
; GISEL-NEXT: movk w9, #5825, lsl #16
; GISEL-NEXT: umull x8, w8, w9
; GISEL-NEXT: lsr x8, x8, #32
; GISEL-NEXT: lsr w0, w8, #2
; GISEL-NEXT: mov w9, #20389 // =0x4fa5
; GISEL-NEXT: movk w9, #42234, lsl #16
; GISEL-NEXT: mul w0, w8, w9
; GISEL-NEXT: ret
{
%udiv = udiv exact i32 %x, 180
Expand All @@ -299,26 +295,17 @@ define <4 x i32> @udiv_div_by_104_exact(<4 x i32> %x)
; SDAG-LABEL: udiv_div_by_104_exact:
; SDAG: // %bb.0:
; SDAG-NEXT: adrp x8, .LCPI8_0
; SDAG-NEXT: ushr v0.4s, v0.4s, #3
; SDAG-NEXT: ldr q1, [x8, :lo12:.LCPI8_0]
; SDAG-NEXT: adrp x8, .LCPI8_1
; SDAG-NEXT: umull2 v2.2d, v0.4s, v1.4s
; SDAG-NEXT: umull v0.2d, v0.2s, v1.2s
; SDAG-NEXT: ldr q1, [x8, :lo12:.LCPI8_1]
; SDAG-NEXT: uzp2 v0.4s, v0.4s, v2.4s
; SDAG-NEXT: ushl v0.4s, v0.4s, v1.4s
; SDAG-NEXT: mul v0.4s, v0.4s, v1.4s
; SDAG-NEXT: ret
;
; GISEL-LABEL: udiv_div_by_104_exact:
; GISEL: // %bb.0:
; GISEL-NEXT: adrp x8, .LCPI8_1
; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI8_1]
; GISEL-NEXT: adrp x8, .LCPI8_0
; GISEL-NEXT: umull2 v2.2d, v0.4s, v1.4s
; GISEL-NEXT: umull v0.2d, v0.2s, v1.2s
; GISEL-NEXT: ushr v0.4s, v0.4s, #3
; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI8_0]
; GISEL-NEXT: neg v1.4s, v1.4s
; GISEL-NEXT: uzp2 v0.4s, v0.4s, v2.4s
; GISEL-NEXT: ushl v0.4s, v0.4s, v1.4s
; GISEL-NEXT: mul v0.4s, v0.4s, v1.4s
; GISEL-NEXT: ret
{
%udiv = udiv exact <4 x i32> %x, <i32 104, i32 72, i32 104, i32 72>
Expand Down
45 changes: 22 additions & 23 deletions llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.mir
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,11 +315,11 @@ body: |
; CHECK: liveins: $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1321528399
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[C]]
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UMULH]], [[C1]](s32)
; CHECK-NEXT: $w0 = COPY [[LSHR]](s32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -991146299
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = exact G_LSHR [[COPY]], [[C]](s32)
; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
; CHECK-NEXT: $w0 = COPY [[MUL]](s32)
; CHECK-NEXT: RET_ReallyLR implicit $w0
%0:_(s32) = COPY $w0
%1:_(s32) = G_CONSTANT i32 104
Expand Down Expand Up @@ -361,11 +361,11 @@ body: |
; CHECK: liveins: $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1321528399
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[C]]
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UMULH]], [[C1]](s32)
; CHECK-NEXT: $w0 = COPY [[LSHR]](s32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -991146299
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = exact G_LSHR [[COPY]], [[C]](s32)
; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
; CHECK-NEXT: $w0 = COPY [[MUL]](s32)
; CHECK-NEXT: RET_ReallyLR implicit $w0
%0:_(s32) = COPY $w0
%1:_(s32) = G_CONSTANT i32 104
Expand All @@ -384,15 +384,14 @@ body: |
; CHECK: liveins: $q0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1321528399
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -991146299
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 954437177
; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C2]](s32), [[C]](s32), [[C2]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C3]](s32), [[C1]](s32), [[C3]](s32)
; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<4 x s32>) = G_UMULH [[COPY]], [[BUILD_VECTOR]]
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<4 x s32>) = G_LSHR [[UMULH]], [[BUILD_VECTOR1]](<4 x s32>)
; CHECK-NEXT: $q0 = COPY [[LSHR]](<4 x s32>)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C2]](s32), [[C1]](s32), [[C2]](s32)
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<4 x s32>) = exact G_LSHR [[COPY]], [[BUILD_VECTOR]](<4 x s32>)
; CHECK-NEXT: [[MUL:%[0-9]+]]:_(<4 x s32>) = G_MUL [[LSHR]], [[BUILD_VECTOR1]]
; CHECK-NEXT: $q0 = COPY [[MUL]](<4 x s32>)
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:_(<4 x s32>) = COPY $q0
%c1:_(s32) = G_CONSTANT i32 104
Expand All @@ -413,13 +412,13 @@ body: |
; CHECK: liveins: $q0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1321528399
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -991146299
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C1]](s32), [[C1]](s32), [[C1]](s32)
; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(<4 x s32>) = G_UMULH [[COPY]], [[BUILD_VECTOR]]
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<4 x s32>) = G_LSHR [[UMULH]], [[BUILD_VECTOR1]](<4 x s32>)
; CHECK-NEXT: $q0 = COPY [[LSHR]](<4 x s32>)
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<4 x s32>) = exact G_LSHR [[COPY]], [[BUILD_VECTOR]](<4 x s32>)
; CHECK-NEXT: [[MUL:%[0-9]+]]:_(<4 x s32>) = G_MUL [[LSHR]], [[BUILD_VECTOR1]]
; CHECK-NEXT: $q0 = COPY [[MUL]](<4 x s32>)
; CHECK-NEXT: RET_ReallyLR implicit $q0
%0:_(<4 x s32>) = COPY $q0
%c1:_(s32) = G_CONSTANT i32 104
Expand Down
Loading

0 comments on commit 4d7d698

Please sign in to comment.