[X86][SSE] Add cost model for BSWAP of vectors

The BSWAP of vector types is quite efficiently implemented using vector shuffles on SSE/AVX targets, we should reflect the typical cost of this to encourage vectorization.

Differential Revision: http://reviews.llvm.org/D21521

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@273217 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/X86/X86TargetTransformInfo.cpp b/lib/Target/X86/X86TargetTransformInfo.cpp
index b5de8f947c906304d12bff855cc96d56411f2d36..338472adf0007c760c59a05bac87150572e849bb 100644 (file)
--- a/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -951,19 +951,33 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     { ISD::BITREVERSE, MVT::v4i64,   5 },
     { ISD::BITREVERSE, MVT::v8i32,   5 },
     { ISD::BITREVERSE, MVT::v16i16,  5 },
-    { ISD::BITREVERSE, MVT::v32i8,   5 }
+    { ISD::BITREVERSE, MVT::v32i8,   5 },
+    { ISD::BSWAP,      MVT::v4i64,   1 },
+    { ISD::BSWAP,      MVT::v8i32,   1 },
+    { ISD::BSWAP,      MVT::v16i16,  1 }
   };
   static const CostTblEntry AVX1CostTbl[] = {
     { ISD::BITREVERSE, MVT::v4i64,  10 },
     { ISD::BITREVERSE, MVT::v8i32,  10 },
     { ISD::BITREVERSE, MVT::v16i16, 10 },
-    { ISD::BITREVERSE, MVT::v32i8,  10 }
+    { ISD::BITREVERSE, MVT::v32i8,  10 },
+    { ISD::BSWAP,      MVT::v4i64,   4 },
+    { ISD::BSWAP,      MVT::v8i32,   4 },
+    { ISD::BSWAP,      MVT::v16i16,  4 }
   };
   static const CostTblEntry SSSE3CostTbl[] = {
     { ISD::BITREVERSE, MVT::v2i64,   5 },
     { ISD::BITREVERSE, MVT::v4i32,   5 },
     { ISD::BITREVERSE, MVT::v8i16,   5 },
-    { ISD::BITREVERSE, MVT::v16i8,   5 }
+    { ISD::BITREVERSE, MVT::v16i8,   5 },
+    { ISD::BSWAP,      MVT::v2i64,   1 },
+    { ISD::BSWAP,      MVT::v4i32,   1 },
+    { ISD::BSWAP,      MVT::v8i16,   1 }
+  };
+  static const CostTblEntry SSE2CostTbl[] = {
+    { ISD::BSWAP,      MVT::v2i64,   7 },
+    { ISD::BSWAP,      MVT::v4i32,   7 },
+    { ISD::BSWAP,      MVT::v8i16,   7 }
   };
 
   unsigned ISD = ISD::DELETED_NODE;
@@ -973,6 +987,9 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
   case Intrinsic::bitreverse:
     ISD = ISD::BITREVERSE;
     break;
+  case Intrinsic::bswap:
+    ISD = ISD::BSWAP;
+    break;
   }
 
   // Legalize the type.
@@ -996,6 +1013,10 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     if (const auto *Entry = CostTableLookup(SSSE3CostTbl, ISD, MTy))
       return LT.first * Entry->Cost;
 
+  if (ST->hasSSE2())
+    if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
+      return LT.first * Entry->Cost;
+
   return BaseT::getIntrinsicInstrCost(IID, RetTy, Tys, FMF);
 }
 

diff --git a/test/Analysis/CostModel/X86/bswap.ll b/test/Analysis/CostModel/X86/bswap.ll
index 8f4a52f90e4a80b56a9fa185e3362e3973176ee8..ecbe24c861cce01af4a0ab49952c9ab040b7d70f 100644 (file)
--- a/test/Analysis/CostModel/X86/bswap.ll
+++ b/test/Analysis/CostModel/X86/bswap.ll
@@ -1,8 +1,8 @@
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=pentium4 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=SSE2
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=SSE42
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core-avx2 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=XOP -check-prefix=XOPAVX
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=AVX -check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core-avx2 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=AVX -check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=XOP -check-prefix=XOPAVX1
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver4 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=XOP -check-prefix=XOPAVX2
 
 ; Verify the cost of vector bswap instructions.
@@ -17,66 +17,66 @@ declare <16 x i16> @llvm.bswap.v16i16(<16 x i16>)
 
 define <2 x i64> @var_bswap_v2i64(<2 x i64> %a) {
 ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v2i64':
-; SSE2: Found an estimated cost of 6 for instruction:   %bswap
-; SSE42: Found an estimated cost of 6 for instruction:   %bswap
-; AVX: Found an estimated cost of 6 for instruction:   %bswap
-; AVX2: Found an estimated cost of 6 for instruction:   %bswap
-; XOP: Found an estimated cost of 6 for instruction:   %bswap
+; SSE2: Found an estimated cost of 7 for instruction:   %bswap
+; SSE42: Found an estimated cost of 1 for instruction:   %bswap
+; AVX: Found an estimated cost of 1 for instruction:   %bswap
+; XOP: Found an estimated cost of 1 for instruction:   %bswap
   %bswap = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %a)
   ret <2 x i64> %bswap
 }
 
 define <4 x i64> @var_bswap_v4i64(<4 x i64> %a) {
 ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v4i64':
-; SSE2: Found an estimated cost of 12 for instruction:   %bswap
-; SSE42: Found an estimated cost of 12 for instruction:   %bswap
-; AVX: Found an estimated cost of 12 for instruction:   %bswap
-; AVX2: Found an estimated cost of 12 for instruction:   %bswap
-; XOP: Found an estimated cost of 12 for instruction:   %bswap
+; SSE2: Found an estimated cost of 14 for instruction:   %bswap
+; SSE42: Found an estimated cost of 2 for instruction:   %bswap
+; AVX1: Found an estimated cost of 4 for instruction:   %bswap
+; AVX2: Found an estimated cost of 1 for instruction:   %bswap
+; XOPAVX1: Found an estimated cost of 4 for instruction:   %bswap
+; XOPAVX2: Found an estimated cost of 1 for instruction:   %bswap
   %bswap = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> %a)
   ret <4 x i64> %bswap
 }
 
 define <4 x i32> @var_bswap_v4i32(<4 x i32> %a) {
 ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v4i32':
-; SSE2: Found an estimated cost of 12 for instruction:   %bswap
-; SSE42: Found an estimated cost of 12 for instruction:   %bswap
-; AVX: Found an estimated cost of 12 for instruction:   %bswap
-; AVX2: Found an estimated cost of 12 for instruction:   %bswap
-; XOP: Found an estimated cost of 12 for instruction:   %bswap
+; SSE2: Found an estimated cost of 7 for instruction:   %bswap
+; SSE42: Found an estimated cost of 1 for instruction:   %bswap
+; AVX: Found an estimated cost of 1 for instruction:   %bswap
+; XOP: Found an estimated cost of 1 for instruction:   %bswap
   %bswap = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %a)
   ret <4 x i32> %bswap
 }
 
 define <8 x i32> @var_bswap_v8i32(<8 x i32> %a) {
 ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v8i32':
-; SSE2: Found an estimated cost of 24 for instruction:   %bswap
-; SSE42: Found an estimated cost of 24 for instruction:   %bswap
-; AVX: Found an estimated cost of 24 for instruction:   %bswap
-; AVX2: Found an estimated cost of 24 for instruction:   %bswap
-; XOP: Found an estimated cost of 24 for instruction:   %bswap
+; SSE2: Found an estimated cost of 14 for instruction:   %bswap
+; SSE42: Found an estimated cost of 2 for instruction:   %bswap
+; AVX1: Found an estimated cost of 4 for instruction:   %bswap
+; AVX2: Found an estimated cost of 1 for instruction:   %bswap
+; XOPAVX1: Found an estimated cost of 4 for instruction:   %bswap
+; XOPAVX2: Found an estimated cost of 1 for instruction:   %bswap
   %bswap = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %a)
   ret <8 x i32> %bswap
 }
 
 define <8 x i16> @var_bswap_v8i16(<8 x i16> %a) {
 ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v8i16':
-; SSE2: Found an estimated cost of 24 for instruction:   %bswap
-; SSE42: Found an estimated cost of 24 for instruction:   %bswap
-; AVX: Found an estimated cost of 24 for instruction:   %bswap
-; AVX2: Found an estimated cost of 24 for instruction:   %bswap
-; XOP: Found an estimated cost of 24 for instruction:   %bswap
+; SSE2: Found an estimated cost of 7 for instruction:   %bswap
+; SSE42: Found an estimated cost of 1 for instruction:   %bswap
+; AVX: Found an estimated cost of 1 for instruction:   %bswap
+; XOP: Found an estimated cost of 1 for instruction:   %bswap
   %bswap = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %a)
   ret <8 x i16> %bswap
 }
 
 define <16 x i16> @var_bswap_v16i16(<16 x i16> %a) {
 ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v16i16':
-; SSE2: Found an estimated cost of 48 for instruction:   %bswap
-; SSE42: Found an estimated cost of 48 for instruction:   %bswap
-; AVX: Found an estimated cost of 48 for instruction:   %bswap
-; AVX2: Found an estimated cost of 48 for instruction:   %bswap
-; XOP: Found an estimated cost of 48 for instruction:   %bswap
+; SSE2: Found an estimated cost of 14 for instruction:   %bswap
+; SSE42: Found an estimated cost of 2 for instruction:   %bswap
+; AVX1: Found an estimated cost of 4 for instruction:   %bswap
+; AVX2: Found an estimated cost of 1 for instruction:   %bswap
+; XOPAVX1: Found an estimated cost of 4 for instruction:   %bswap
+; XOPAVX2: Found an estimated cost of 1 for instruction:   %bswap
   %bswap = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> %a)
   ret <16 x i16> %bswap
-}
\ No newline at end of file
+}

diff --git a/test/Analysis/CostModel/X86/scalarize.ll b/test/Analysis/CostModel/X86/scalarize.ll
index f124de50699ad67e0931f80b6835143c97392b4b..fd4e3005bd9f8b8027ce9ac7fc2082f2d55cf103 100644 (file)
--- a/test/Analysis/CostModel/X86/scalarize.ll
+++ b/test/Analysis/CostModel/X86/scalarize.ll
@@ -21,11 +21,11 @@ declare %i8 @llvm.cttz.v2i64(%i8)
 define void @test_scalarized_intrinsics() {
         %r1 = add %i8 undef, undef
 
-; CHECK32: cost of 12 {{.*}}bswap.v4i32
-; CHECK64: cost of 12 {{.*}}bswap.v4i32
+; CHECK32: cost of 1 {{.*}}bswap.v4i32
+; CHECK64: cost of 1 {{.*}}bswap.v4i32
         %r2 = call %i4 @llvm.bswap.v4i32(%i4 undef)
-; CHECK32: cost of 10 {{.*}}bswap.v2i64
-; CHECK64: cost of 6 {{.*}}bswap.v2i64
+; CHECK32: cost of 1 {{.*}}bswap.v2i64
+; CHECK64: cost of 1 {{.*}}bswap.v2i64
         %r3 = call %i8 @llvm.bswap.v2i64(%i8 undef)
 
 ; CHECK32: cost of 12 {{.*}}cttz.v4i32

diff --git a/test/Transforms/SLPVectorizer/X86/bswap.ll b/test/Transforms/SLPVectorizer/X86/bswap.ll
index 9105d5c0701bdd55b0071226723687fe9f2fbb34..79ce24f59119fed32b5237e491d2b667dbec3f1e 100644 (file)
--- a/test/Transforms/SLPVectorizer/X86/bswap.ll
+++ b/test/Transforms/SLPVectorizer/X86/bswap.ll
@@ -17,14 +17,20 @@ declare i32 @llvm.bswap.i32(i32)
 declare i16 @llvm.bswap.i16(i16)
 
 define void @bswap_2i64() #0 {
-; CHECK-LABEL: @bswap_2i64(
-; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
-; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
-; CHECK-NEXT:    [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]])
-; CHECK-NEXT:    [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]])
-; CHECK-NEXT:    store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
-; CHECK-NEXT:    store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @bswap_2i64(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]])
+; SSE-NEXT:    [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]])
+; SSE-NEXT:    store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bswap_2i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([4 x i64]* @src64 to <2 x i64>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> [[TMP1]])
+; AVX-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* bitcast ([4 x i64]* @dst64 to <2 x i64>*), align 8
+; AVX-NEXT:    ret void
 ;
   %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
   %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
@@ -36,20 +42,26 @@ define void @bswap_2i64() #0 {
 }
 
 define void @bswap_4i64() #0 {
-; CHECK-LABEL: @bswap_4i64(
-; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
-; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
-; CHECK-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
-; CHECK-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
-; CHECK-NEXT:    [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]])
-; CHECK-NEXT:    [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]])
-; CHECK-NEXT:    [[BSWAP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD2]])
-; CHECK-NEXT:    [[BSWAP3:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD3]])
-; CHECK-NEXT:    store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
-; CHECK-NEXT:    store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
-; CHECK-NEXT:    store i64 [[BSWAP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
-; CHECK-NEXT:    store i64 [[BSWAP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @bswap_4i64(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; SSE-NEXT:    [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]])
+; SSE-NEXT:    [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]])
+; SSE-NEXT:    [[BSWAP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD2]])
+; SSE-NEXT:    [[BSWAP3:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD3]])
+; SSE-NEXT:    store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; SSE-NEXT:    store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; SSE-NEXT:    store i64 [[BSWAP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; SSE-NEXT:    store i64 [[BSWAP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bswap_4i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([4 x i64]* @src64 to <4 x i64>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> [[TMP1]])
+; AVX-NEXT:    store <4 x i64> [[TMP2]], <4 x i64>* bitcast ([4 x i64]* @dst64 to <4 x i64>*), align 4
+; AVX-NEXT:    ret void
 ;
   %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
   %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
@@ -68,18 +80,9 @@ define void @bswap_4i64() #0 {
 
 define void @bswap_4i32() #0 {
 ; CHECK-LABEL: @bswap_4i32(
-; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
-; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
-; CHECK-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
-; CHECK-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
-; CHECK-NEXT:    [[BSWAP0:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD0]])
-; CHECK-NEXT:    [[BSWAP1:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD1]])
-; CHECK-NEXT:    [[BSWAP2:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD2]])
-; CHECK-NEXT:    [[BSWAP3:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD3]])
-; CHECK-NEXT:    store i32 [[BSWAP0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
-; CHECK-NEXT:    store i32 [[BSWAP1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
-; CHECK-NEXT:    store i32 [[BSWAP2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
-; CHECK-NEXT:    store i32 [[BSWAP3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4
 ; CHECK-NEXT:    ret void
 ;
   %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
@@ -98,32 +101,20 @@ define void @bswap_4i32() #0 {
 }
 
 define void @bswap_8i32() #0 {
-; CHECK-LABEL: @bswap_8i32(
-; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
-; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
-; CHECK-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
-; CHECK-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
-; CHECK-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
-; CHECK-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
-; CHECK-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
-; CHECK-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
-; CHECK-NEXT:    [[BSWAP0:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD0]])
-; CHECK-NEXT:    [[BSWAP1:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD1]])
-; CHECK-NEXT:    [[BSWAP2:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD2]])
-; CHECK-NEXT:    [[BSWAP3:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD3]])
-; CHECK-NEXT:    [[BSWAP4:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD4]])
-; CHECK-NEXT:    [[BSWAP5:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD5]])
-; CHECK-NEXT:    [[BSWAP6:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD6]])
-; CHECK-NEXT:    [[BSWAP7:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD7]])
-; CHECK-NEXT:    store i32 [[BSWAP0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
-; CHECK-NEXT:    store i32 [[BSWAP1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
-; CHECK-NEXT:    store i32 [[BSWAP2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
-; CHECK-NEXT:    store i32 [[BSWAP3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
-; CHECK-NEXT:    store i32 [[BSWAP4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
-; CHECK-NEXT:    store i32 [[BSWAP5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
-; CHECK-NEXT:    store i32 [[BSWAP6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
-; CHECK-NEXT:    store i32 [[BSWAP7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @bswap_8i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP2]])
+; SSE-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 2
+; SSE-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bswap_8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> [[TMP1]])
+; AVX-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX-NEXT:    ret void
 ;
   %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
   %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
@@ -154,30 +145,9 @@ define void @bswap_8i32() #0 {
 
 define void @bswap_8i16() #0 {
 ; CHECK-LABEL: @bswap_8i16(
-; CHECK-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
-; CHECK-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
-; CHECK-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
-; CHECK-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
-; CHECK-NEXT:    [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
-; CHECK-NEXT:    [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
-; CHECK-NEXT:    [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
-; CHECK-NEXT:    [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
-; CHECK-NEXT:    [[BSWAP0:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD0]])
-; CHECK-NEXT:    [[BSWAP1:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD1]])
-; CHECK-NEXT:    [[BSWAP2:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD2]])
-; CHECK-NEXT:    [[BSWAP3:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD3]])
-; CHECK-NEXT:    [[BSWAP4:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD4]])
-; CHECK-NEXT:    [[BSWAP5:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD5]])
-; CHECK-NEXT:    [[BSWAP6:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD6]])
-; CHECK-NEXT:    [[BSWAP7:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD7]])
-; CHECK-NEXT:    store i16 [[BSWAP0]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
-; CHECK-NEXT:    store i16 [[BSWAP1]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
-; CHECK-NEXT:    store i16 [[BSWAP2]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
-; CHECK-NEXT:    store i16 [[BSWAP3]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
-; CHECK-NEXT:    store i16 [[BSWAP4]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
-; CHECK-NEXT:    store i16 [[BSWAP5]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
-; CHECK-NEXT:    store i16 [[BSWAP6]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
-; CHECK-NEXT:    store i16 [[BSWAP7]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP1]])
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
 ; CHECK-NEXT:    ret void
 ;
   %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
@@ -208,56 +178,20 @@ define void @bswap_8i16() #0 {
 }
 
 define void @bswap_16i16() #0 {
-; CHECK-LABEL: @bswap_16i16(
-; CHECK-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
-; CHECK-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
-; CHECK-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
-; CHECK-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
-; CHECK-NEXT:    [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
-; CHECK-NEXT:    [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
-; CHECK-NEXT:    [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
-; CHECK-NEXT:    [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
-; CHECK-NEXT:    [[LD8:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8), align 2
-; CHECK-NEXT:    [[LD9:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 9), align 2
-; CHECK-NEXT:    [[LD10:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2
-; CHECK-NEXT:    [[LD11:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2
-; CHECK-NEXT:    [[LD12:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2
-; CHECK-NEXT:    [[LD13:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2
-; CHECK-NEXT:    [[LD14:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2
-; CHECK-NEXT:    [[LD15:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2
-; CHECK-NEXT:    [[BSWAP0:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD0]])
-; CHECK-NEXT:    [[BSWAP1:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD1]])
-; CHECK-NEXT:    [[BSWAP2:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD2]])
-; CHECK-NEXT:    [[BSWAP3:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD3]])
-; CHECK-NEXT:    [[BSWAP4:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD4]])
-; CHECK-NEXT:    [[BSWAP5:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD5]])
-; CHECK-NEXT:    [[BSWAP6:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD6]])
-; CHECK-NEXT:    [[BSWAP7:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD7]])
-; CHECK-NEXT:    [[BSWAP8:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD8]])
-; CHECK-NEXT:    [[BSWAP9:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD9]])
-; CHECK-NEXT:    [[BSWAP10:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD10]])
-; CHECK-NEXT:    [[BSWAP11:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD11]])
-; CHECK-NEXT:    [[BSWAP12:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD12]])
-; CHECK-NEXT:    [[BSWAP13:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD13]])
-; CHECK-NEXT:    [[BSWAP14:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD14]])
-; CHECK-NEXT:    [[BSWAP15:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD15]])
-; CHECK-NEXT:    store i16 [[BSWAP0]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
-; CHECK-NEXT:    store i16 [[BSWAP1]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
-; CHECK-NEXT:    store i16 [[BSWAP2]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
-; CHECK-NEXT:    store i16 [[BSWAP3]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
-; CHECK-NEXT:    store i16 [[BSWAP4]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
-; CHECK-NEXT:    store i16 [[BSWAP5]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
-; CHECK-NEXT:    store i16 [[BSWAP6]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
-; CHECK-NEXT:    store i16 [[BSWAP7]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
-; CHECK-NEXT:    store i16 [[BSWAP8]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8), align 2
-; CHECK-NEXT:    store i16 [[BSWAP9]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 9), align 2
-; CHECK-NEXT:    store i16 [[BSWAP10]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2
-; CHECK-NEXT:    store i16 [[BSWAP11]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2
-; CHECK-NEXT:    store i16 [[BSWAP12]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2
-; CHECK-NEXT:    store i16 [[BSWAP13]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2
-; CHECK-NEXT:    store i16 [[BSWAP14]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2
-; CHECK-NEXT:    store i16 [[BSWAP15]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @bswap_16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP2]])
+; SSE-NEXT:    store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bswap_16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> [[TMP1]])
+; AVX-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
 ;
   %ld0  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  0), align 2
   %ld1  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  1), align 2