[DAGCombine] Slightly improve lowering of BUILD_VECTOR into a shuffle.

This handles the case of a BUILD_VECTOR being constructed out of elements extracted from a vector twice the size of the result vector. Previously this was always scalarized. Now, we try to construct a shuffle node that feeds on extract_subvectors.

This fixes PR15872 and provides a partial fix for PR21711.

Differential Revision: http://reviews.llvm.org/D6678

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@224429 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h
index 1f5c1fd71571bae567f4d9d67ab411adf73ffd8b..9c0189782249d8d959077e9872db6f9b3b8ceb3a 100644 (file)
--- a/include/llvm/Target/TargetLowering.h
+++ b/include/llvm/Target/TargetLowering.h
@@ -1531,6 +1531,15 @@ public:
                                                  Type *Ty) const {
     return false;
   }
+
+  /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
+  /// with this index. This is needed because EXTRACT_SUBVECTOR usually
+  /// has custom lowering that depends on the index of the first element,
+  /// and only the target knows which lowering is cheap.
+  virtual bool isExtractSubvectorCheap(EVT ResVT, unsigned Index) const {
+    return false;
+  }
+
   //===--------------------------------------------------------------------===//
   // Runtime Library hooks
   //

diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ed3d06cc9208885f654dd1ab8d898a77ca1277b2..8297e84146994389ea5eb6e628e4e36aadc3b8c8 100644 (file)
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -10783,9 +10783,6 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
       SDValue ExtVal = Extract.getOperand(1);
       unsigned ExtIndex = cast<ConstantSDNode>(ExtVal)->getZExtValue();
       if (Extract.getOperand(0) == VecIn1) {
-        if (ExtIndex > VT.getVectorNumElements())
-          return SDValue();
-
         Mask.push_back(ExtIndex);
         continue;
       }
@@ -10805,20 +10802,34 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
       if (VecIn2.getNode())
         return SDValue();
 
-      // We only support widening of vectors which are half the size of the
-      // output registers. For example XMM->YMM widening on X86 with AVX.
-      if (VecIn1.getValueType().getSizeInBits()*2 != VT.getSizeInBits())
-        return SDValue();
-
       // If the input vector type has a different base type to the output
       // vector type, bail out.
       if (VecIn1.getValueType().getVectorElementType() !=
           VT.getVectorElementType())
         return SDValue();
 
-      // Widen the input vector by adding undef values.
-      VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
-                           VecIn1, DAG.getUNDEF(VecIn1.getValueType()));
+      // If the input vector is too small, widen it.
+      // We only support widening of vectors which are half the size of the
+      // output registers. For example XMM->YMM widening on X86 with AVX.
+      EVT VecInT = VecIn1.getValueType();
+      if (VecInT.getSizeInBits() * 2 == VT.getSizeInBits()) {
+        // Widen the input vector by adding undef values.
+        VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
+                             VecIn1, DAG.getUNDEF(VecIn1.getValueType()));
+      } else if (VecInT.getSizeInBits() == VT.getSizeInBits() * 2) {
+        // If the input vector is too large, try to split it.
+        if (!TLI.isExtractSubvectorCheap(VT, VT.getVectorNumElements()))
+          return SDValue();
+        
+        // Try to replace VecIn1 with two extract_subvectors
+        // No need to update the masks, they should still be correct.
+        VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1, 
+          DAG.getConstant(VT.getVectorNumElements(), TLI.getVectorIdxTy()));
+        VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1,
+          DAG.getConstant(0, TLI.getVectorIdxTy()));
+        UsesZeroVector = false;
+      } else 
+        return SDValue();
     }
 
     if (UsesZeroVector)

diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index 2678ca645fd01872e37e0a47facdd215615b9380..6a6b20e81d92e878a23076e8387ddc3cf3e80467 100644 (file)
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -3851,6 +3851,14 @@ bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
   return true;
 }
 
+bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, 
+                                                unsigned Index) const {
+  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
+    return false;
+
+  return (Index == 0 || Index == ResVT.getVectorNumElements());
+}
+
 /// isUndefOrInRange - Return true if Val is undef or if its value falls within
 /// the specified range (L, H].
 static bool isUndefOrInRange(int Val, int Low, int Hi) {

diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index b793171e2c5e6bb26ea04adcf4f815293ceb4245..c5e3e14c545c5a0089eaefdf75662a716da902a4 100644 (file)
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -791,6 +791,10 @@ namespace llvm {
     bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                            Type *Ty) const override;
 
+    /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
+    /// with this index.
+    bool isExtractSubvectorCheap(EVT ResVT, unsigned Index) const override;
+
     /// Intel processors have a unified instruction and data cache
     const char * getClearCacheBuiltinName() const override {
       return nullptr; // nothing to do, move along.

diff --git a/test/CodeGen/X86/vec_extract-avx.ll b/test/CodeGen/X86/vec_extract-avx.ll
new file mode 100644 (file)
index 0000000..e5e7937
--- /dev/null
+++ b/test/CodeGen/X86/vec_extract-avx.ll
@@ -0,0 +1,82 @@
+target triple = "x86_64-unknown-unknown"\r
+\r
+; RUN: llc < %s -march=x86-64 -mattr=+avx | FileCheck %s\r
+\r
+; When extracting multiple consecutive elements from a larger\r
+; vector into a smaller one, do it efficiently. We should use\r
+; an EXTRACT_SUBVECTOR node internally rather than a bunch of\r
+; single element extractions. \r
+\r
+; Extracting the low elements only requires using the right kind of store.\r
+define void @low_v8f32_to_v4f32(<8 x float> %v, <4 x float>* %ptr) {\r
+  %ext0 = extractelement <8 x float> %v, i32 0\r
+  %ext1 = extractelement <8 x float> %v, i32 1\r
+  %ext2 = extractelement <8 x float> %v, i32 2\r
+  %ext3 = extractelement <8 x float> %v, i32 3\r
+  %ins0 = insertelement <4 x float> undef, float %ext0, i32 0\r
+  %ins1 = insertelement <4 x float> %ins0, float %ext1, i32 1\r
+  %ins2 = insertelement <4 x float> %ins1, float %ext2, i32 2\r
+  %ins3 = insertelement <4 x float> %ins2, float %ext3, i32 3\r
+  store <4 x float> %ins3, <4 x float>* %ptr, align 16\r
+  ret void\r
+\r
+; CHECK-LABEL: low_v8f32_to_v4f32\r
+; CHECK: vmovaps\r
+; CHECK-NEXT: vzeroupper\r
+; CHECK-NEXT: retq\r
+}\r
+\r
+; Extracting the high elements requires just one AVX instruction. \r
+define void @high_v8f32_to_v4f32(<8 x float> %v, <4 x float>* %ptr) {\r
+  %ext0 = extractelement <8 x float> %v, i32 4\r
+  %ext1 = extractelement <8 x float> %v, i32 5\r
+  %ext2 = extractelement <8 x float> %v, i32 6\r
+  %ext3 = extractelement <8 x float> %v, i32 7\r
+  %ins0 = insertelement <4 x float> undef, float %ext0, i32 0\r
+  %ins1 = insertelement <4 x float> %ins0, float %ext1, i32 1\r
+  %ins2 = insertelement <4 x float> %ins1, float %ext2, i32 2\r
+  %ins3 = insertelement <4 x float> %ins2, float %ext3, i32 3\r
+  store <4 x float> %ins3, <4 x float>* %ptr, align 16\r
+  ret void\r
+\r
+; CHECK-LABEL: high_v8f32_to_v4f32\r
+; CHECK: vextractf128\r
+; CHECK-NEXT: vzeroupper\r
+; CHECK-NEXT: retq\r
+}\r
+\r
+; Make sure element type doesn't alter the codegen. Note that\r
+; if we were actually using the vector in this function and\r
+; have AVX2, we should generate vextracti128 (the int version).\r
+define void @high_v8i32_to_v4i32(<8 x i32> %v, <4 x i32>* %ptr) {\r
+  %ext0 = extractelement <8 x i32> %v, i32 4\r
+  %ext1 = extractelement <8 x i32> %v, i32 5\r
+  %ext2 = extractelement <8 x i32> %v, i32 6\r
+  %ext3 = extractelement <8 x i32> %v, i32 7\r
+  %ins0 = insertelement <4 x i32> undef, i32 %ext0, i32 0\r
+  %ins1 = insertelement <4 x i32> %ins0, i32 %ext1, i32 1\r
+  %ins2 = insertelement <4 x i32> %ins1, i32 %ext2, i32 2\r
+  %ins3 = insertelement <4 x i32> %ins2, i32 %ext3, i32 3\r
+  store <4 x i32> %ins3, <4 x i32>* %ptr, align 16\r
+  ret void\r
+\r
+; CHECK-LABEL: high_v8i32_to_v4i32\r
+; CHECK: vextractf128\r
+; CHECK-NEXT: vzeroupper\r
+; CHECK-NEXT: retq\r
+}\r
+\r
+; Make sure that element size doesn't alter the codegen.\r
+define void @high_v4f64_to_v2f64(<4 x double> %v, <2 x double>* %ptr) {\r
+  %ext0 = extractelement <4 x double> %v, i32 2\r
+  %ext1 = extractelement <4 x double> %v, i32 3\r
+  %ins0 = insertelement <2 x double> undef, double %ext0, i32 0\r
+  %ins1 = insertelement <2 x double> %ins0, double %ext1, i32 1\r
+  store <2 x double> %ins1, <2 x double>* %ptr, align 16\r
+  ret void\r
+\r
+; CHECK-LABEL: high_v4f64_to_v2f64\r
+; CHECK: vextractf128\r
+; CHECK-NEXT: vzeroupper\r
+; CHECK-NEXT: retq\r
+}\r

diff --git a/test/CodeGen/X86/vector-shuffle-combining.ll b/test/CodeGen/X86/vector-shuffle-combining.ll
index 5f30891365d9c54296cdb858283b89d02641728e..e7bae3415bf91dfc118c3b8783509defdde0a6b1 100644 (file)
--- a/test/CodeGen/X86/vector-shuffle-combining.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining.ll
@@ -1552,6 +1552,37 @@ define <4 x i32> @combine_test20(<4 x i32> %a, <4 x i32> %b) {
   ret <4 x i32> %2
 }
 
+define <4 x i32> @combine_test21(<8 x i32> %a, <4 x i32>* %ptr) {
+; SSE-LABEL: combine_test21:
+; SSE:       # BB#0:
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpcklqdq  {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT:    punpckhqdq  {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE-NEXT:    movdqa %xmm2,
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: combine_test21:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpunpcklqdq  {{.*#+}} xmm2 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vpunpckhqdq  {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; AVX1-NEXT:    movdqa %xmm2,
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: combine_test21:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpunpcklqdq  {{.*#+}} xmm2 = xmm0[0],xmm1[0]
+; AVX2-NEXT:    vpunpckhqdq  {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; AVX2-NEXT:    movdqa %xmm2,
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %1 = shufflevector <8 x i32> %a, <8 x i32> %a, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  %2 = shufflevector <8 x i32> %a, <8 x i32> %a, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+  store <4 x i32> %1, <4 x i32>* %ptr, align 16
+  ret <4 x i32> %2
+}
 
 ; Check some negative cases.
 ; FIXME: Do any of these really make sense? Are they redundant with the above tests?