Skip to content

Commit

Permalink
[DAG][X86] expandABD - add branchless abds/abdu expansion for 0/-1 co…
Browse files Browse the repository at this point in the history
…mparison result cases

If the comparison results are allbits masks, we can expand as "abd(lhs, rhs) -> sub(cmpgt(lhs, rhs), xor(sub(lhs, rhs), cmpgt(lhs, rhs)))", replacing a sub+sub+select pattern with the simpler sub+xor+sub pattern.

This allows us to remove a lot of X86 specific legalization code, and will be useful in future generic expansion for the legalization work in llvm#92576

Alive2: https://alive2.llvm.org/ce/z/sj863C
  • Loading branch information
RKSimon committed May 20, 2024
1 parent 3efaf9c commit 4aab172
Show file tree
Hide file tree
Showing 8 changed files with 737 additions and 899 deletions.
13 changes: 11 additions & 2 deletions llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9194,11 +9194,20 @@ SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS),
DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS));

// abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
// abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
SDValue Cmp = DAG.getSetCC(dl, CCVT, LHS, RHS, CC);

// Branchless expansion iif cmp result is allbits:
// abd(lhs, rhs) -> sub(cmpgt(lhs, rhs), xor(sub(lhs, rhs), cmpgt(lhs, rhs)))
if (CCVT == VT && getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
SDValue Diff = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Diff, Cmp);
return DAG.getNode(ISD::SUB, dl, VT, Cmp, Xor);
}

// abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
// abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS),
DAG.getNode(ISD::SUB, dl, VT, RHS, LHS));
}
Expand Down
26 changes: 2 additions & 24 deletions llvm/lib/Target/X86/X86ISelLowering.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1108,13 +1108,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
}

setOperationAction(ISD::ABDU, MVT::v16i8, Custom);
setOperationAction(ISD::ABDS, MVT::v16i8, Custom);
setOperationAction(ISD::ABDU, MVT::v8i16, Custom);
setOperationAction(ISD::ABDS, MVT::v8i16, Custom);
setOperationAction(ISD::ABDU, MVT::v4i32, Custom);
setOperationAction(ISD::ABDS, MVT::v4i32, Custom);

setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
Expand All @@ -1135,6 +1128,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::CTPOP, VT, Custom);
setOperationAction(ISD::ABS, VT, Custom);
setOperationAction(ISD::ABDS, VT, Custom);
setOperationAction(ISD::ABDU, VT, Custom);

// The condition codes aren't legal in SSE/AVX and under AVX512 we use
// setcc all the way to isel and prefer SETGT in some isel patterns.
Expand Down Expand Up @@ -1336,11 +1331,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
setOperationAction(ISD::UMIN, MVT::v4i32, Legal);

for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
setOperationAction(ISD::ABDS, VT, Custom);
setOperationAction(ISD::ABDU, VT, Custom);
}

setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
setOperationAction(ISD::SADDSAT, MVT::v2i64, Custom);
setOperationAction(ISD::SSUBSAT, MVT::v2i64, Custom);
Expand Down Expand Up @@ -28421,18 +28411,6 @@ static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
}
}

// TODO: Move to TargetLowering expandABD().
if (!Subtarget.hasSSE41() &&
((IsSigned && VT == MVT::v16i8) || VT == MVT::v4i32)) {
SDValue LHS = DAG.getFreeze(Op.getOperand(0));
SDValue RHS = DAG.getFreeze(Op.getOperand(1));
ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
SDValue Cmp = DAG.getSetCC(dl, VT, LHS, RHS, CC);
SDValue Diff0 = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
SDValue Diff1 = DAG.getNode(ISD::SUB, dl, VT, RHS, LHS);
return getBitSelect(dl, VT, Diff0, Diff1, Cmp, DAG);
}

// Default to expand.
return SDValue();
}
Expand Down
Loading

0 comments on commit 4aab172

Please sign in to comment.