if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, ZeroOrAllOnes))
return {SPF_ABS, SPNB_NA, false};
+ // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
+ if (Pred == ICmpInst::ICMP_SGE && match(CmpRHS, ZeroOrOne))
+ return {SPF_ABS, SPNB_NA, false};
+
// (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
// (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, ZeroOrOne))
getShiftAmountTy(Temp.getValueType())));
}
- // Check to see if this is an integer abs.
- // select_cc setg[te] X, 0, X, -X ->
- // select_cc setgt X, -1, X, -X ->
- // select_cc setl[te] X, 0, -X, X ->
- // select_cc setlt X, 1, -X, X ->
- // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
- if (N1C) {
- ConstantSDNode *SubC = nullptr;
- if (((N1C->isNullValue() && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
- (N1C->isAllOnesValue() && CC == ISD::SETGT)) &&
- N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1))
- SubC = dyn_cast<ConstantSDNode>(N3.getOperand(0));
- else if (((N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE)) ||
- (N1C->isOne() && CC == ISD::SETLT)) &&
- N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1))
- SubC = dyn_cast<ConstantSDNode>(N2.getOperand(0));
-
- if (SubC && SubC->isNullValue() && CmpOpVT.isInteger()) {
- SDLoc DL(N0);
- SDValue Shift = DAG.getNode(ISD::SRA, DL, CmpOpVT, N0,
- DAG.getConstant(CmpOpVT.getSizeInBits() - 1,
- DL,
- getShiftAmountTy(CmpOpVT)));
- SDValue Add = DAG.getNode(ISD::ADD, DL, CmpOpVT, N0, Shift);
- AddToWorklist(Shift.getNode());
- AddToWorklist(Add.getNode());
- return DAG.getNode(ISD::XOR, DL, CmpOpVT, Add, Shift);
- }
- }
-
// select_cc seteq X, 0, sizeof(X), ctlz(X) -> ctlz(X)
// select_cc seteq X, 0, sizeof(X), ctlz_zero_undef(X) -> ctlz(X)
// select_cc seteq X, 0, sizeof(X), cttz(X) -> cttz(X)
case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
case ISD::SMULFIX:
case ISD::UMULFIX: Res = PromoteIntRes_MULFIX(N); break;
+ case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
case ISD::ATOMIC_LOAD:
Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
return SDValue(Res.getNode(), 0);
}
+SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
+ SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
+ return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
+}
+
SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
// Promote the overflow bit trivially.
if (ResNo == 1)
case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
+ case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
case ISD::CTLZ_ZERO_UNDEF:
case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
IsOpaque);
}
+void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
+ SDLoc dl(N);
+
+ // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
+ EVT VT = N->getValueType(0);
+ SDValue N0 = N->getOperand(0);
+ SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
+ DAG.getConstant(0, dl, VT), N0);
+ SDValue NegLo, NegHi;
+ SplitInteger(Neg, NegLo, NegHi);
+
+ GetExpandedInteger(N0, Lo, Hi);
+ EVT NVT = Lo.getValueType();
+ SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
+ DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
+ Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
+ Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
+}
+
void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
SDValue PromoteIntRes_MULFIX(SDNode *N);
SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N);
SDValue PromoteIntRes_VECREDUCE(SDNode *N);
+ SDValue PromoteIntRes_ABS(SDNode *N);
// Integer Operand Promotion.
bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);
void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi);
+ void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);
SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
break;
+ case ISD::ABS:
case ISD::BITREVERSE:
case ISD::BSWAP:
case ISD::CTLZ:
ISD::NodeType OpCode = Cond.getValueType().isVector() ?
ISD::VSELECT : ISD::SELECT;
+ bool IsUnaryAbs = false;
+
// Min/max matching is only viable if all output VTs are the same.
if (is_splat(ValueVTs)) {
EVT VT = ValueVTs[0];
break;
}
break;
+ case SPF_ABS:
+ IsUnaryAbs = true;
+ Opc = ISD::ABS;
+ break;
+ case SPF_NABS:
+ // TODO: we need to produce sub(0, abs(X)).
default: break;
}
- if (Opc != ISD::DELETED_NODE &&
+ if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
(TLI.isOperationLegalOrCustom(Opc, VT) ||
(UseScalarMinMax &&
TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
RHSVal = getValue(RHS);
BaseOps = {};
}
+
+ if (IsUnaryAbs) {
+ OpCode = Opc;
+ LHSVal = getValue(LHS);
+ BaseOps = {};
+ }
}
- for (unsigned i = 0; i != NumValues; ++i) {
- SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
- Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
- Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
- Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
- LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
- Ops);
+ if (IsUnaryAbs) {
+ for (unsigned i = 0; i != NumValues; ++i) {
+ Values[i] =
+ DAG.getNode(OpCode, getCurSDLoc(),
+ LHSVal.getNode()->getValueType(LHSVal.getResNo() + i),
+ SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
+ }
+ } else {
+ for (unsigned i = 0; i != NumValues; ++i) {
+ SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
+ Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
+ Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
+ Values[i] = DAG.getNode(
+ OpCode, getCurSDLoc(),
+ LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops);
+ }
}
setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
setPrefLoopAlignment(Subtarget->getPrefLoopAlignment());
setMinFunctionAlignment(Subtarget->isThumb() ? 1 : 2);
+
+ if (Subtarget->isThumb() || Subtarget->isThumb2())
+ setTargetDAGCombine(ISD::ABS);
}
bool ARMTargetLowering::useSoftFloat() const {
return;
case ISD::INTRINSIC_WO_CHAIN:
return ReplaceLongIntrinsic(N, Results, DAG);
+ case ISD::ABS:
+ lowerABS(N, Results, DAG);
+ return ;
+
}
if (Res.getNode())
Results.push_back(Res);
return SDValue();
}
+static SDValue PerformABSCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const ARMSubtarget *Subtarget) {
+ SDValue res;
+ SelectionDAG &DAG = DCI.DAG;
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+ if (!TLI.expandABS(N, res, DAG))
+ return SDValue();
+
+ return res;
+}
+
/// PerformADDECombine - Target-specific dag combine transform from
/// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or
/// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
DAGCombinerInfo &DCI) const {
switch (N->getOpcode()) {
default: break;
+ case ISD::ABS: return PerformABSCombine(N, DCI, Subtarget);
case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget);
case ARMISD::UMLAL: return PerformUMLALCombine(N, DCI.DAG, Subtarget);
case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
SDLoc(Op)).first;
}
+void ARMTargetLowering::lowerABS(SDNode *N, SmallVectorImpl<SDValue> &Results,
+ SelectionDAG &DAG) const {
+ EVT VT = N->getValueType(0);
+ assert(VT == MVT::i64 && "Unexpected type (!= i64) on ABS.");
+ MVT HalfT = MVT::i32;
+ SDLoc dl(N);
+ SDValue Hi, Lo, Tmp;
+
+ if (!isOperationLegalOrCustom(ISD::ADDCARRY, HalfT) ||
+ !isOperationLegalOrCustom(ISD::UADDO, HalfT))
+ return ;
+
+ unsigned OpTypeBits = HalfT.getScalarSizeInBits();
+ SDVTList VTList = DAG.getVTList(HalfT, MVT::i1);
+
+ Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0),
+ DAG.getConstant(0, dl, HalfT));
+ Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0),
+ DAG.getConstant(1, dl, HalfT));
+
+ Tmp = DAG.getNode(ISD::SRA, dl, HalfT, Hi,
+ DAG.getConstant(OpTypeBits - 1, dl,
+ getShiftAmountTy(HalfT, DAG.getDataLayout())));
+ Lo = DAG.getNode(ISD::UADDO, dl, VTList, Tmp, Lo);
+ Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Tmp, Hi,
+ SDValue(Lo.getNode(), 1));
+ Hi = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Hi);
+ Lo = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Lo);
+
+ Results.push_back(Lo);
+ Results.push_back(Hi);
+}
+
bool
ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
// The ARM target isn't yet aware of offsets.
SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
+ void lowerABS(SDNode *N, SmallVectorImpl<SDValue> &Results,
+ SelectionDAG &DAG) const;
unsigned getRegisterByName(const char* RegName, EVT VT,
SelectionDAG &DAG) const override;
// add/sub are legal for all supported vector VT's.
setOperationAction(ISD::ADD, VT, Legal);
setOperationAction(ISD::SUB, VT, Legal);
- setOperationAction(ISD::ABS, VT, Custom);
+ if (VT.getSizeInBits() == 128)
+ setOperationAction(ISD::ABS, VT, Custom);
// Vector instructions introduced in P8
if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) {
// Integer absolute.
if (Subtarget.hasCMov()) {
setOperationAction(ISD::ABS , MVT::i16 , Custom);
- setOperationAction(ISD::ABS , MVT::i32 , Custom);
- if (Subtarget.is64Bit())
- setOperationAction(ISD::ABS , MVT::i64 , Custom);
+ setOperationAction(ISD::ABS , MVT::i32 , Custom);
}
+ setOperationAction(ISD::ABS , MVT::i64 , Custom);
// Funnel shifts.
for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
Results.push_back(Res);
return;
}
+ case ISD::ABS: {
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ EVT VT = N->getValueType(0);
+ assert(VT == MVT::i64 && "Unexpected type (!= i64) on ABS.");
+ MVT HalfT = MVT::i32;
+ SDValue Lo, Hi, Tmp;
+ SDVTList VTList = DAG.getVTList(HalfT, MVT::i1);
+
+ Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0),
+ DAG.getConstant(0, dl, HalfT));
+ Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0),
+ DAG.getConstant(1, dl, HalfT));
+ Tmp = DAG.getNode(
+ ISD::SRA, dl, HalfT, Hi,
+ DAG.getConstant(HalfT.getSizeInBits() - 1, dl,
+ TLI.getShiftAmountTy(HalfT, DAG.getDataLayout())));
+ Lo = DAG.getNode(ISD::UADDO, dl, VTList, Tmp, Lo);
+ Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Tmp, Hi,
+ SDValue(Lo.getNode(), 1));
+ Hi = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Hi);
+ Lo = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Lo);
+ Results.push_back(Lo);
+ Results.push_back(Hi);
+ return;
+ }
case ISD::SETCC: {
// Widen v2i32 (setcc v2f32). This is really needed for AVX512VL when
// setCC result type is v2i1 because type legalzation will end up with
return SDValue();
}
-// Given a select, detect the following pattern:
-// 1: %2 = zext <N x i8> %0 to <N x i32>
-// 2: %3 = zext <N x i8> %1 to <N x i32>
-// 3: %4 = sub nsw <N x i32> %2, %3
-// 4: %5 = icmp sgt <N x i32> %4, [0 x N] or [-1 x N]
-// 5: %6 = sub nsw <N x i32> zeroinitializer, %4
-// 6: %7 = select <N x i1> %5, <N x i32> %4, <N x i32> %6
+// Given a ABS node, detect the following pattern:
+// (ABS (SUB (ZERO_EXTEND a), (ZERO_EXTEND b))).
// This is useful as it is the input into a SAD pattern.
-static bool detectZextAbsDiff(const SDValue &Select, SDValue &Op0,
- SDValue &Op1) {
- // Check the condition of the select instruction is greater-than.
- SDValue SetCC = Select->getOperand(0);
- if (SetCC.getOpcode() != ISD::SETCC)
- return false;
- ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get();
- if (CC != ISD::SETGT && CC != ISD::SETLT)
- return false;
-
- SDValue SelectOp1 = Select->getOperand(1);
- SDValue SelectOp2 = Select->getOperand(2);
-
- // The following instructions assume SelectOp1 is the subtraction operand
- // and SelectOp2 is the negation operand.
- // In the case of SETLT this is the other way around.
- if (CC == ISD::SETLT)
- std::swap(SelectOp1, SelectOp2);
-
- // The second operand of the select should be the negation of the first
- // operand, which is implemented as 0 - SelectOp1.
- if (!(SelectOp2.getOpcode() == ISD::SUB &&
- ISD::isBuildVectorAllZeros(SelectOp2.getOperand(0).getNode()) &&
- SelectOp2.getOperand(1) == SelectOp1))
- return false;
-
- // The first operand of SetCC is the first operand of the select, which is the
- // difference between the two input vectors.
- if (SetCC.getOperand(0) != SelectOp1)
- return false;
-
- // In SetLT case, The second operand of the comparison can be either 1 or 0.
- APInt SplatVal;
- if ((CC == ISD::SETLT) &&
- !((ISD::isConstantSplatVector(SetCC.getOperand(1).getNode(), SplatVal) &&
- SplatVal.isOneValue()) ||
- (ISD::isBuildVectorAllZeros(SetCC.getOperand(1).getNode()))))
- return false;
-
- // In SetGT case, The second operand of the comparison can be either -1 or 0.
- if ((CC == ISD::SETGT) &&
- !(ISD::isBuildVectorAllZeros(SetCC.getOperand(1).getNode()) ||
- ISD::isBuildVectorAllOnes(SetCC.getOperand(1).getNode())))
- return false;
-
- // The first operand of the select is the difference between the two input
- // vectors.
- if (SelectOp1.getOpcode() != ISD::SUB)
+static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
+ SDValue AbsOp1 = Abs->getOperand(0);
+ if (AbsOp1.getOpcode() != ISD::SUB)
return false;
- Op0 = SelectOp1.getOperand(0);
- Op1 = SelectOp1.getOperand(1);
+ Op0 = AbsOp1.getOperand(0);
+ Op1 = AbsOp1.getOperand(1);
// Check if the operands of the sub are zero-extended from vectors of i8.
if (Op0.getOpcode() != ISD::ZERO_EXTEND ||
// If there was a match, we want Root to be a select that is the root of an
// abs-diff pattern.
- if (!Root || (Root.getOpcode() != ISD::VSELECT))
+ if (!Root || Root.getOpcode() != ISD::ABS)
return SDValue();
// Check whether we have an abs-diff pattern feeding into the select.
return SDValue();
// We know N is a reduction add, which means one of its operands is a phi.
- // To match SAD, we need the other operand to be a vector select.
- if (Op0.getOpcode() != ISD::VSELECT)
+ // To match SAD, we need the other operand to be a ABS.
+ if (Op0.getOpcode() != ISD::ABS)
std::swap(Op0, Op1);
- if (Op0.getOpcode() != ISD::VSELECT)
+ if (Op0.getOpcode() != ISD::ABS)
return SDValue();
auto BuildPSADBW = [&](SDValue Op0, SDValue Op1) {
Op0 = BuildPSADBW(SadOp0, SadOp1);
// It's possible we have a sad on the other side too.
- if (Op1.getOpcode() == ISD::VSELECT &&
+ if (Op1.getOpcode() == ISD::ABS &&
detectZextAbsDiff(Op1, SadOp0, SadOp1)) {
Op1 = BuildPSADBW(SadOp0, SadOp1);
}
define i8 @test_i8(i8 %a) nounwind {
; CHECK-LABEL: test_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: sbfx w8, w0, #7, #1
-; CHECK-NEXT: add w9, w0, w8
-; CHECK-NEXT: eor w0, w9, w8
+; CHECK-NEXT: sxtb w8, w0
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: cneg w0, w8, mi
; CHECK-NEXT: ret
%tmp1neg = sub i8 0, %a
%b = icmp sgt i8 %a, -1
define i16 @test_i16(i16 %a) nounwind {
; CHECK-LABEL: test_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sbfx w8, w0, #15, #1
-; CHECK-NEXT: add w9, w0, w8
-; CHECK-NEXT: eor w0, w9, w8
+; CHECK-NEXT: sxth w8, w0
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: cneg w0, w8, mi
; CHECK-NEXT: ret
%tmp1neg = sub i16 0, %a
%b = icmp sgt i16 %a, -1
projects / llvm / commitdiff