From: Craig Topper Date: Mon, 20 Jun 2016 04:00:55 +0000 (+0000) Subject: [X86] Pass the SDLoc and Mask ArrayRef down from lowerVectorShuffle through all of... X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=8de712d4c818c04b1e0670deb4a5168b8e683602;p=llvm [X86] Pass the SDLoc and Mask ArrayRef down from lowerVectorShuffle through all of the other routines instead of recreating them in the handlers for each type. NFC git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@273137 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 46773eb6045..076df6c756d 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -8626,10 +8626,10 @@ static SDValue lowerVectorShuffleAsBroadcast(const SDLoc &DL, MVT VT, // are much smaller to encode than a SHUFPS and an INSERTPS. We can also // perform INSERTPS if a single V1 element is out of place and all V2 // elements are zeroable. -static SDValue lowerVectorShuffleAsInsertPS(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerVectorShuffleAsInsertPS(const SDLoc &DL, + SDValue V1, SDValue V2, ArrayRef Mask, SelectionDAG &DAG) { - assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); @@ -8693,7 +8693,6 @@ static SDValue lowerVectorShuffleAsInsertPS(SDValue Op, SDValue V1, SDValue V2, assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!"); // Insert the V2 element into the desired position. - SDLoc DL(Op); return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, DAG.getConstant(InsertPSMask, DL, MVT::i8)); } @@ -8821,15 +8820,12 @@ static SDValue lowerVectorShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT, /// instructions will incur a domain crossing penalty on some chips though so /// it is better to avoid lowering through this for integer vectors where /// possible. -static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV2F64VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!"); assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!"); if (isSingleInputShuffleMask(Mask)) { @@ -8902,15 +8898,12 @@ static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// the integer unit to minimize domain crossing penalties. However, for blends /// it falls back to the floating point shuffle operation with appropriate bit /// casting. -static SDValue lowerV2I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV2I64VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!"); assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!"); if (isSingleInputShuffleMask(Mask)) { @@ -9118,15 +9111,12 @@ static SDValue lowerVectorShuffleWithSHUFPS(const SDLoc &DL, MVT VT, /// Uses instructions exclusively from the floating point unit to minimize /// domain crossing penalties, as these are sufficient to implement all v4f32 /// shuffles. -static SDValue lowerV4F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV4F32VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; }); @@ -9174,7 +9164,7 @@ static SDValue lowerV4F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, return Blend; // Use INSERTPS if we can complete the shuffle efficiently. - if (SDValue V = lowerVectorShuffleAsInsertPS(Op, V1, V2, Mask, DAG)) + if (SDValue V = lowerVectorShuffleAsInsertPS(DL, V1, V2, Mask, DAG)) return V; if (!isSingleSHUFPSMask(Mask)) @@ -9202,15 +9192,12 @@ static SDValue lowerV4F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// /// We try to handle these with integer-domain shuffles where we can, but for /// blends we use the floating point domain blend instructions. -static SDValue lowerV4I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV4I32VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!"); assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); // Whenever we can lower this as a zext, that instruction is strictly faster @@ -9833,16 +9820,12 @@ static SDValue lowerVectorShuffleAsBlendOfPSHUFBs( /// the two inputs, try to interleave them. Otherwise, blend the low and high /// halves of the inputs separately (making them have relatively few inputs) /// and then concatenate them. -static SDValue lowerV8I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV8I16VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!"); assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); // Whenever we can lower this as a zext, that instruction is strictly faster @@ -10020,15 +10003,12 @@ static int canLowerByDroppingEvenElements(ArrayRef Mask) { /// UNPCK to spread the i8 elements across two i16-element vectors, and uses /// the existing lowering for v8i16 blends on each half, finally PACK-ing them /// back together. -static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!"); assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); // Try to use shift instructions. @@ -10312,22 +10292,23 @@ static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// /// This routine breaks down the specific type of 128-bit shuffle and /// dispatches to the lowering routines accordingly. -static SDValue lower128BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, - MVT VT, const X86Subtarget &Subtarget, +static SDValue lower128BitVectorShuffle(const SDLoc &DL, ArrayRef Mask, + MVT VT, SDValue V1, SDValue V2, + const X86Subtarget &Subtarget, SelectionDAG &DAG) { switch (VT.SimpleTy) { case MVT::v2i64: - return lowerV2I64VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV2I64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v2f64: - return lowerV2F64VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV2F64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v4i32: - return lowerV4I32VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV4I32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v4f32: - return lowerV4F32VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV4F32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v8i16: - return lowerV8I16VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV8I16VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v16i8: - return lowerV16I8VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV16I8VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); default: llvm_unreachable("Unimplemented!"); @@ -11089,14 +11070,12 @@ static SDValue lowerVectorShuffleWithSHUFPD(const SDLoc &DL, MVT VT, /// /// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2 /// isn't available. -static SDValue lowerV4F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV4F64VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); SmallVector WidenedMask; @@ -11184,14 +11163,12 @@ static SDValue lowerV4F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// /// This routine is only called when we have AVX2 and thus a reasonable /// instruction set for v4i64 shuffling.. -static SDValue lowerV4I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV4I64VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); assert(Subtarget.hasAVX2() && "We can only lower v4i64 with AVX2!"); @@ -11264,14 +11241,12 @@ static SDValue lowerV4I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// /// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2 /// isn't available. -static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV8F32VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask, @@ -11356,14 +11331,12 @@ static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// /// This routine is only called when we have AVX2 and thus a reasonable /// instruction set for v8i32 shuffling.. -static SDValue lowerV8I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV8I32VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); assert(Subtarget.hasAVX2() && "We can only lower v8i32 with AVX2!"); @@ -11441,14 +11414,12 @@ static SDValue lowerV8I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// /// This routine is only called when we have AVX2 and thus a reasonable /// instruction set for v16i16 shuffling.. -static SDValue lowerV16I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV16I16VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); assert(Subtarget.hasAVX2() && "We can only lower v16i16 with AVX2!"); @@ -11524,14 +11495,12 @@ static SDValue lowerV16I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// /// This routine is only called when we have AVX2 and thus a reasonable /// instruction set for v32i8 shuffling.. -static SDValue lowerV32I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV32I8VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!"); assert(Subtarget.hasAVX2() && "We can only lower v32i8 with AVX2!"); @@ -11598,13 +11567,10 @@ static SDValue lowerV32I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// This routine either breaks down the specific type of a 256-bit x86 vector /// shuffle or splits it into two 128-bit shuffles and fuses the results back /// together based on the available instructions. -static SDValue lower256BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, - MVT VT, const X86Subtarget &Subtarget, +static SDValue lower256BitVectorShuffle(const SDLoc &DL, ArrayRef Mask, + MVT VT, SDValue V1, SDValue V2, + const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - // If we have a single input to the zero element, insert that into V1 if we // can do so cheaply. int NumElts = VT.getVectorNumElements(); @@ -11647,17 +11613,17 @@ static SDValue lower256BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, switch (VT.SimpleTy) { case MVT::v4f64: - return lowerV4F64VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV4F64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v4i64: - return lowerV4I64VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV4I64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v8f32: - return lowerV8F32VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV8F32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v8i32: - return lowerV8I32VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV8I32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v16i16: - return lowerV16I16VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV16I16VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v32i8: - return lowerV32I8VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV32I8VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); default: llvm_unreachable("Not a valid 256-bit x86 vector type!"); @@ -11729,14 +11695,12 @@ static SDValue lowerVectorShuffleWithPERMV(const SDLoc &DL, MVT VT, } /// \brief Handle lowering of 8-lane 64-bit floating point shuffles. -static SDValue lowerV8F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV8F64VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); if (SDValue Shuf128 = @@ -11751,14 +11715,12 @@ static SDValue lowerV8F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, } /// \brief Handle lowering of 16-lane 32-bit floating point shuffles. -static SDValue lowerV16F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV16F32VectorShuffle(SDLoc DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); if (SDValue Unpck = @@ -11769,14 +11731,12 @@ static SDValue lowerV16F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, } /// \brief Handle lowering of 8-lane 64-bit integer shuffles. -static SDValue lowerV8I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV8I64VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); if (SDValue Shuf128 = @@ -11815,14 +11775,12 @@ static SDValue lowerV8I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, } /// \brief Handle lowering of 16-lane 32-bit integer shuffles. -static SDValue lowerV16I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV16I32VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); // If the shuffle mask is repeated in each 128-bit lane we can use more @@ -11856,14 +11814,12 @@ static SDValue lowerV16I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, } /// \brief Handle lowering of 32-lane 16-bit integer shuffles. -static SDValue lowerV32I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV32I16VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!"); assert(Subtarget.hasBWI() && "We can only lower v32i16 with AVX-512-BWI!"); @@ -11892,14 +11848,12 @@ static SDValue lowerV32I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, } /// \brief Handle lowering of 64-lane 8-bit integer shuffles. -static SDValue lowerV64I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +static SDValue lowerV64I8VectorShuffle(const SDLoc &DL, ArrayRef Mask, + SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!"); assert(Subtarget.hasBWI() && "We can only lower v64i8 with AVX-512-BWI!"); @@ -11922,12 +11876,10 @@ static SDValue lowerV64I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, /// This routine either breaks down the specific type of a 512-bit x86 vector /// shuffle or splits it into two 256-bit shuffles and fuses the results back /// together based on the available instructions. -static SDValue lower512BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, - MVT VT, const X86Subtarget &Subtarget, +static SDValue lower512BitVectorShuffle(const SDLoc &DL, ArrayRef Mask, + MVT VT, SDValue V1, SDValue V2, + const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Subtarget.hasAVX512() && "Cannot lower 512-bit vectors w/ basic ISA!"); @@ -11942,17 +11894,17 @@ static SDValue lower512BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, // the requisite ISA extensions for that element type are available. switch (VT.SimpleTy) { case MVT::v8f64: - return lowerV8F64VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV8F64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v16f32: - return lowerV16F32VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV16F32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v8i64: - return lowerV8I64VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV8I64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v16i32: - return lowerV16I32VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV16I32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v32i16: - return lowerV32I16VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV32I16VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); case MVT::v64i8: - return lowerV64I8VectorShuffle(Op, V1, V2, Subtarget, DAG); + return lowerV64I8VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG); default: llvm_unreachable("Not a valid 512-bit x86 vector type!"); @@ -11963,12 +11915,10 @@ static SDValue lower512BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, // There is no a dedicated instruction on AVX-512 that shuffles the masks. // The only way to shuffle bits is to sign-extend the mask vector to SIMD // vector, shuffle and then truncate it back. -static SDValue lower1BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, - MVT VT, const X86Subtarget &Subtarget, +static SDValue lower1BitVectorShuffle(const SDLoc &DL, ArrayRef Mask, + MVT VT, SDValue V1, SDValue V2, + const X86Subtarget &Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); assert(Subtarget.hasAVX512() && "Cannot lower 512-bit vectors w/o basic ISA!"); MVT ExtVT; @@ -12028,7 +11978,7 @@ static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget, SDValue V2 = Op.getOperand(1); MVT VT = Op.getSimpleValueType(); int NumElements = VT.getVectorNumElements(); - SDLoc dl(Op); + SDLoc DL(Op); bool Is1BitVector = (VT.getVectorElementType() == MVT::i1); assert((VT.getSizeInBits() != 64 || Is1BitVector) && @@ -12055,7 +12005,7 @@ static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget, for (int &M : NewMask) if (M >= NumElements) M = -1; - return DAG.getVectorShuffle(VT, dl, V1, V2, NewMask); + return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); } // We actually see shuffles that are entirely re-arrangements of a set of @@ -12063,7 +12013,7 @@ static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget, // simple ones. Directly lower these as a buildvector of zeros. SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); if (Zeroable.all()) - return getZeroVector(VT, Subtarget, DAG, dl); + return getZeroVector(VT, Subtarget, DAG, DL); // Try to collapse shuffles into using a vector type with fewer elements but // wider element types. We cap this to not form integers or floating point @@ -12082,7 +12032,7 @@ static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget, V1 = DAG.getBitcast(NewVT, V1); V2 = DAG.getBitcast(NewVT, V2); return DAG.getBitcast( - VT, DAG.getVectorShuffle(NewVT, dl, V1, V2, WidenedMask)); + VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask)); } } @@ -12139,16 +12089,17 @@ static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget, // For each vector width, delegate to a specialized lowering routine. if (VT.is128BitVector()) - return lower128BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); + return lower128BitVectorShuffle(DL, Mask, VT, V1, V2, Subtarget, DAG); if (VT.is256BitVector()) - return lower256BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); + return lower256BitVectorShuffle(DL, Mask, VT, V1, V2, Subtarget, DAG); if (VT.is512BitVector()) - return lower512BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); + return lower512BitVectorShuffle(DL, Mask, VT, V1, V2, Subtarget, DAG); if (Is1BitVector) - return lower1BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); + return lower1BitVectorShuffle(DL, Mask, VT, V1, V2, Subtarget, DAG); + llvm_unreachable("Unimplemented!"); }