From: Simon Pilgrim Date: Fri, 26 Apr 2019 11:45:54 +0000 (+0000) Subject: [X86][SSE] Pull out OR(EXTRACTELT(X,0),OR(EXTRACTELT(X,1),...)) matching code from... X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=edf11da4586c0d6ef0ca3f7a4428eb3531ebcd61;p=llvm [X86][SSE] Pull out OR(EXTRACTELT(X,0),OR(EXTRACTELT(X,1),...)) matching code from LowerVectorAllZeroTest Create a matchBitOpReduction helper that checks for the pattern with any opcode. First step towards reusing this code to recognize other scalar reduction patterns. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@359296 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 889a0111b87..d3d103a20bd 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -19206,36 +19206,25 @@ static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, DAG.getConstant(Cond, dl, MVT::i8), EFLAGS); } -// Check whether an OR'd tree is PTEST-able. -static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC, - const X86Subtarget &Subtarget, - SelectionDAG &DAG, - SDValue &X86CC) { - assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree."); - - if (!Subtarget.hasSSE41()) - return SDValue(); - - if (!Op->hasOneUse()) - return SDValue(); - - SDNode *N = Op.getNode(); - SDLoc DL(N); - +/// Helper for matching OR(EXTRACTELT(X,0),OR(EXTRACTELT(X,1),...)) +/// style scalarized (associative) reduction patterns. +static bool matchBitOpReduction(SDValue Op, ISD::NodeType BinOp, + SmallVectorImpl &SrcOps) { SmallVector Opnds; - DenseMap VecInMap; - SmallVector VecIns; + DenseMap SrcOpMap; EVT VT = MVT::Other; // Recognize a special case where a vector is casted into wide integer to // test all 0s. - Opnds.push_back(N->getOperand(0)); - Opnds.push_back(N->getOperand(1)); + assert(Op.getOpcode() == unsigned(BinOp) && + "Unexpected bit reduction opcode"); + Opnds.push_back(Op.getOperand(0)); + Opnds.push_back(Op.getOperand(1)); for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) { SmallVectorImpl::const_iterator I = Opnds.begin() + Slot; - // BFS traverse all OR'd operands. - if (I->getOpcode() == ISD::OR) { + // BFS traverse all BinOp operands. + if (I->getOpcode() == unsigned(BinOp)) { Opnds.push_back(I->getOperand(0)); Opnds.push_back(I->getOperand(1)); // Re-evaluate the number of nodes to be traversed. @@ -19245,42 +19234,63 @@ static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC, // Quit if a non-EXTRACT_VECTOR_ELT if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT) - return SDValue(); + return false; // Quit if without a constant index. SDValue Idx = I->getOperand(1); if (!isa(Idx)) - return SDValue(); + return false; - SDValue ExtractedFromVec = I->getOperand(0); - DenseMap::iterator M = VecInMap.find(ExtractedFromVec); - if (M == VecInMap.end()) { - VT = ExtractedFromVec.getValueType(); - // Quit if not 128/256-bit vector. - if (!VT.is128BitVector() && !VT.is256BitVector()) - return SDValue(); + SDValue Src = I->getOperand(0); + DenseMap::iterator M = SrcOpMap.find(Src); + if (M == SrcOpMap.end()) { + VT = Src.getValueType(); // Quit if not the same type. - if (VecInMap.begin() != VecInMap.end() && - VT != VecInMap.begin()->first.getValueType()) - return SDValue(); - M = VecInMap.insert(std::make_pair(ExtractedFromVec, 0)).first; - VecIns.push_back(ExtractedFromVec); + if (SrcOpMap.begin() != SrcOpMap.end() && + VT != SrcOpMap.begin()->first.getValueType()) + return false; + unsigned NumElts = VT.getVectorNumElements(); + APInt EltCount = APInt::getNullValue(NumElts); + M = SrcOpMap.insert(std::make_pair(Src, EltCount)).first; + SrcOps.push_back(Src); } - M->second |= 1U << cast(Idx)->getZExtValue(); + // Quit if element already used. + unsigned CIdx = cast(Idx)->getZExtValue(); + if (M->second[CIdx]) + return false; + M->second.setBit(CIdx); } - assert((VT.is128BitVector() || VT.is256BitVector()) && - "Not extracted from 128-/256-bit vector."); + // Quit if not all elements are used. + for (DenseMap::const_iterator I = SrcOpMap.begin(), + E = SrcOpMap.end(); + I != E; ++I) { + if (!I->second.isAllOnesValue()) + return false; + } - unsigned FullMask = (1U << VT.getVectorNumElements()) - 1U; + return true; +} - for (DenseMap::const_iterator - I = VecInMap.begin(), E = VecInMap.end(); I != E; ++I) { - // Quit if not all elements are used. - if (I->second != FullMask) - return SDValue(); - } +// Check whether an OR'd tree is PTEST-able. +static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC, + const X86Subtarget &Subtarget, + SelectionDAG &DAG, SDValue &X86CC) { + assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree."); + + if (!Subtarget.hasSSE41() || !Op->hasOneUse()) + return SDValue(); + SmallVector VecIns; + if (!matchBitOpReduction(Op, ISD::OR, VecIns)) + return SDValue(); + + // Quit if not 128/256-bit vector. + EVT VT = VecIns[0].getValueType(); + if (!VT.is128BitVector() && !VT.is256BitVector()) + return SDValue(); + + SDLoc DL(Op); MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; // Cast all vectors into TestVT for PTEST. @@ -19296,10 +19306,9 @@ static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC, VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS)); } - X86CC = DAG.getConstant(CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE, - DL, MVT::i8); - return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, - VecIns.back(), VecIns.back()); + X86CC = DAG.getConstant(CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE, DL, + MVT::i8); + return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, VecIns.back(), VecIns.back()); } /// return true if \c Op has a use that doesn't just read flags.