From c11234753fec87e1eda13842f6161b031dadae7d Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 10 May 2017 21:29:33 +0000 Subject: [PATCH] AMDGPU: Make some packed shuffles free VOP3P instructions can encode access to either half of the register. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@302730 91177308-0d34-0410-b5e6-96231b3b80d8 --- .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 34 ++++++++- lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h | 3 + .../CostModel/AMDGPU/extractelement.ll | 74 ++++++++++++------- .../CostModel/AMDGPU/insertelement.ll | 43 +++++++---- .../CostModel/AMDGPU/shufflevector.ll | 43 +++++++++++ 5 files changed, 155 insertions(+), 42 deletions(-) create mode 100644 test/Analysis/CostModel/AMDGPU/shufflevector.ll diff --git a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index c9482c37ec8..beafebc1284 100644 --- a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -363,13 +363,22 @@ int AMDGPUTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy, unsigned Index) { switch (Opcode) { case Instruction::ExtractElement: - case Instruction::InsertElement: + case Instruction::InsertElement: { + unsigned EltSize + = DL.getTypeSizeInBits(cast(ValTy)->getElementType()); + if (EltSize < 32) { + if (EltSize == 16 && Index == 0 && ST->has16BitInsts()) + return 0; + return BaseT::getVectorInstrCost(Opcode, ValTy, Index); + } + // Extracts are just reads of a subregister, so are free. Inserts are // considered free because we don't want to have any cost for scalarizing // operations, and we don't have to copy into a different register class. // Dynamic indexing isn't free and is best avoided. return Index == ~0u ? 2 : 0; + } default: return BaseT::getVectorInstrCost(Opcode, ValTy, Index); } @@ -479,3 +488,26 @@ bool AMDGPUTTIImpl::isSourceOfDivergence(const Value *V) const { return false; } + +unsigned AMDGPUTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, + Type *SubTp) { + if (ST->hasVOP3PInsts()) { + VectorType *VT = cast(Tp); + if (VT->getNumElements() == 2 && + DL.getTypeSizeInBits(VT->getElementType()) == 16) { + // With op_sel VOP3P instructions freely can access the low half or high + // half of a register, so any swizzle is free. + + switch (Kind) { + case TTI::SK_Broadcast: + case TTI::SK_Reverse: + case TTI::SK_PermuteSingleSrc: + return 0; + default: + break; + } + } + } + + return BaseT::getShuffleCost(Kind, Tp, Index, SubTp); +} diff --git a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h index 71d6306bc1a..e0024e21e82 100644 --- a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h +++ b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h @@ -114,6 +114,9 @@ public: } unsigned getVectorSplitCost() { return 0; } + + unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, + Type *SubTp); }; } // end namespace llvm diff --git a/test/Analysis/CostModel/AMDGPU/extractelement.ll b/test/Analysis/CostModel/AMDGPU/extractelement.ll index 1efbb5873ac..54c8b6c5236 100644 --- a/test/Analysis/CostModel/AMDGPU/extractelement.ll +++ b/test/Analysis/CostModel/AMDGPU/extractelement.ll @@ -1,7 +1,9 @@ -; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s +; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa %s | FileCheck -check-prefixes=GCN,CI %s +; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,VI %s +; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX9 %s -; CHECK: 'extractelement_v2i32' -; CHECK: estimated cost of 0 for {{.*}} extractelement <2 x i32> +; GCN: 'extractelement_v2i32' +; GCN: estimated cost of 0 for {{.*}} extractelement <2 x i32> define amdgpu_kernel void @extractelement_v2i32(i32 addrspace(1)* %out, <2 x i32> addrspace(1)* %vaddr) { %vec = load <2 x i32>, <2 x i32> addrspace(1)* %vaddr %elt = extractelement <2 x i32> %vec, i32 1 @@ -9,8 +11,8 @@ define amdgpu_kernel void @extractelement_v2i32(i32 addrspace(1)* %out, <2 x i32 ret void } -; CHECK: 'extractelement_v2f32' -; CHECK: estimated cost of 0 for {{.*}} extractelement <2 x float> +; GCN: 'extractelement_v2f32' +; GCN: estimated cost of 0 for {{.*}} extractelement <2 x float> define amdgpu_kernel void @extractelement_v2f32(float addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr) { %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr %elt = extractelement <2 x float> %vec, i32 1 @@ -18,8 +20,8 @@ define amdgpu_kernel void @extractelement_v2f32(float addrspace(1)* %out, <2 x f ret void } -; CHECK: 'extractelement_v3i32' -; CHECK: estimated cost of 0 for {{.*}} extractelement <3 x i32> +; GCN: 'extractelement_v3i32' +; GCN: estimated cost of 0 for {{.*}} extractelement <3 x i32> define amdgpu_kernel void @extractelement_v3i32(i32 addrspace(1)* %out, <3 x i32> addrspace(1)* %vaddr) { %vec = load <3 x i32>, <3 x i32> addrspace(1)* %vaddr %elt = extractelement <3 x i32> %vec, i32 1 @@ -27,8 +29,8 @@ define amdgpu_kernel void @extractelement_v3i32(i32 addrspace(1)* %out, <3 x i32 ret void } -; CHECK: 'extractelement_v4i32' -; CHECK: estimated cost of 0 for {{.*}} extractelement <4 x i32> +; GCN: 'extractelement_v4i32' +; GCN: estimated cost of 0 for {{.*}} extractelement <4 x i32> define amdgpu_kernel void @extractelement_v4i32(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %vaddr) { %vec = load <4 x i32>, <4 x i32> addrspace(1)* %vaddr %elt = extractelement <4 x i32> %vec, i32 1 @@ -36,8 +38,8 @@ define amdgpu_kernel void @extractelement_v4i32(i32 addrspace(1)* %out, <4 x i32 ret void } -; CHECK: 'extractelement_v8i32' -; CHECK: estimated cost of 0 for {{.*}} extractelement <8 x i32> +; GCN: 'extractelement_v8i32' +; GCN: estimated cost of 0 for {{.*}} extractelement <8 x i32> define amdgpu_kernel void @extractelement_v8i32(i32 addrspace(1)* %out, <8 x i32> addrspace(1)* %vaddr) { %vec = load <8 x i32>, <8 x i32> addrspace(1)* %vaddr %elt = extractelement <8 x i32> %vec, i32 1 @@ -46,8 +48,8 @@ define amdgpu_kernel void @extractelement_v8i32(i32 addrspace(1)* %out, <8 x i32 } ; FIXME: Should be non-0 -; CHECK: 'extractelement_v8i32_dynindex' -; CHECK: estimated cost of 2 for {{.*}} extractelement <8 x i32> +; GCN: 'extractelement_v8i32_dynindex' +; GCN: estimated cost of 2 for {{.*}} extractelement <8 x i32> define amdgpu_kernel void @extractelement_v8i32_dynindex(i32 addrspace(1)* %out, <8 x i32> addrspace(1)* %vaddr, i32 %idx) { %vec = load <8 x i32>, <8 x i32> addrspace(1)* %vaddr %elt = extractelement <8 x i32> %vec, i32 %idx @@ -55,8 +57,8 @@ define amdgpu_kernel void @extractelement_v8i32_dynindex(i32 addrspace(1)* %out, ret void } -; CHECK: 'extractelement_v2i64' -; CHECK: estimated cost of 0 for {{.*}} extractelement <2 x i64> +; GCN: 'extractelement_v2i64' +; GCN: estimated cost of 0 for {{.*}} extractelement <2 x i64> define amdgpu_kernel void @extractelement_v2i64(i64 addrspace(1)* %out, <2 x i64> addrspace(1)* %vaddr) { %vec = load <2 x i64>, <2 x i64> addrspace(1)* %vaddr %elt = extractelement <2 x i64> %vec, i64 1 @@ -64,8 +66,8 @@ define amdgpu_kernel void @extractelement_v2i64(i64 addrspace(1)* %out, <2 x i64 ret void } -; CHECK: 'extractelement_v3i64' -; CHECK: estimated cost of 0 for {{.*}} extractelement <3 x i64> +; GCN: 'extractelement_v3i64' +; GCN: estimated cost of 0 for {{.*}} extractelement <3 x i64> define amdgpu_kernel void @extractelement_v3i64(i64 addrspace(1)* %out, <3 x i64> addrspace(1)* %vaddr) { %vec = load <3 x i64>, <3 x i64> addrspace(1)* %vaddr %elt = extractelement <3 x i64> %vec, i64 1 @@ -73,8 +75,8 @@ define amdgpu_kernel void @extractelement_v3i64(i64 addrspace(1)* %out, <3 x i64 ret void } -; CHECK: 'extractelement_v4i64' -; CHECK: estimated cost of 0 for {{.*}} extractelement <4 x i64> +; GCN: 'extractelement_v4i64' +; GCN: estimated cost of 0 for {{.*}} extractelement <4 x i64> define amdgpu_kernel void @extractelement_v4i64(i64 addrspace(1)* %out, <4 x i64> addrspace(1)* %vaddr) { %vec = load <4 x i64>, <4 x i64> addrspace(1)* %vaddr %elt = extractelement <4 x i64> %vec, i64 1 @@ -82,8 +84,8 @@ define amdgpu_kernel void @extractelement_v4i64(i64 addrspace(1)* %out, <4 x i64 ret void } -; CHECK: 'extractelement_v8i64' -; CHECK: estimated cost of 0 for {{.*}} extractelement <8 x i64> +; GCN: 'extractelement_v8i64' +; GCN: estimated cost of 0 for {{.*}} extractelement <8 x i64> define amdgpu_kernel void @extractelement_v8i64(i64 addrspace(1)* %out, <8 x i64> addrspace(1)* %vaddr) { %vec = load <8 x i64>, <8 x i64> addrspace(1)* %vaddr %elt = extractelement <8 x i64> %vec, i64 1 @@ -91,8 +93,8 @@ define amdgpu_kernel void @extractelement_v8i64(i64 addrspace(1)* %out, <8 x i64 ret void } -; CHECK: 'extractelement_v4i8' -; CHECK: estimated cost of 0 for {{.*}} extractelement <4 x i8> +; GCN: 'extractelement_v4i8' +; GCN: estimated cost of 1 for {{.*}} extractelement <4 x i8> define amdgpu_kernel void @extractelement_v4i8(i8 addrspace(1)* %out, <4 x i8> addrspace(1)* %vaddr) { %vec = load <4 x i8>, <4 x i8> addrspace(1)* %vaddr %elt = extractelement <4 x i8> %vec, i8 1 @@ -100,11 +102,31 @@ define amdgpu_kernel void @extractelement_v4i8(i8 addrspace(1)* %out, <4 x i8> a ret void } -; CHECK: 'extractelement_v2i16' -; CHECK: estimated cost of 0 for {{.*}} extractelement <2 x i16> -define amdgpu_kernel void @extractelement_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { +; GCN: 'extractelement_0_v2i16': +; CI: estimated cost of 1 for {{.*}} extractelement <2 x i16> %vec, i16 0 +; VI: estimated cost of 0 for {{.*}} extractelement <2 x i16> +; GFX9: estimated cost of 0 for {{.*}} extractelement <2 x i16> +define amdgpu_kernel void @extractelement_0_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { + %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr + %elt = extractelement <2 x i16> %vec, i16 0 + store i16 %elt, i16 addrspace(1)* %out + ret void +} + +; GCN: 'extractelement_1_v2i16': +; GCN: estimated cost of 1 for {{.*}} extractelement <2 x i16> +define amdgpu_kernel void @extractelement_1_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr %elt = extractelement <2 x i16> %vec, i16 1 store i16 %elt, i16 addrspace(1)* %out ret void } + +; GCN: 'extractelement_var_v2i16' +; GCN: estimated cost of 1 for {{.*}} extractelement <2 x i16> +define amdgpu_kernel void @extractelement_var_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, i32 %idx) { + %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr + %elt = extractelement <2 x i16> %vec, i32 %idx + store i16 %elt, i16 addrspace(1)* %out + ret void +} diff --git a/test/Analysis/CostModel/AMDGPU/insertelement.ll b/test/Analysis/CostModel/AMDGPU/insertelement.ll index 6f296a3e7a3..67ab2607acd 100644 --- a/test/Analysis/CostModel/AMDGPU/insertelement.ll +++ b/test/Analysis/CostModel/AMDGPU/insertelement.ll @@ -1,37 +1,50 @@ -; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s +; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa %s | FileCheck -check-prefixes=GCN,CI %s +; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,VI %s +; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX9 %s -; CHECK: 'insertelement_v2i32' -; CHECK: estimated cost of 0 for {{.*}} insertelement <2 x i32> +; GCN-LABEL: 'insertelement_v2i32' +; GCN: estimated cost of 0 for {{.*}} insertelement <2 x i32> define amdgpu_kernel void @insertelement_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %vaddr) { %vec = load <2 x i32>, <2 x i32> addrspace(1)* %vaddr - %insert = insertelement <2 x i32> %vec, i32 1, i32 123 + %insert = insertelement <2 x i32> %vec, i32 123, i32 1 store <2 x i32> %insert, <2 x i32> addrspace(1)* %out ret void } -; CHECK: 'insertelement_v2i64' -; CHECK: estimated cost of 0 for {{.*}} insertelement <2 x i64> +; GCN-LABEL: 'insertelement_v2i64' +; GCN: estimated cost of 0 for {{.*}} insertelement <2 x i64> define amdgpu_kernel void @insertelement_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %vaddr) { %vec = load <2 x i64>, <2 x i64> addrspace(1)* %vaddr - %insert = insertelement <2 x i64> %vec, i64 1, i64 123 + %insert = insertelement <2 x i64> %vec, i64 123, i64 1 store <2 x i64> %insert, <2 x i64> addrspace(1)* %out ret void } -; CHECK: 'insertelement_v2i16' -; CHECK: estimated cost of 0 for {{.*}} insertelement <2 x i16> -define amdgpu_kernel void @insertelement_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { +; GCN-LABEL: 'insertelement_0_v2i16' +; CI: estimated cost of 1 for {{.*}} insertelement <2 x i16> +; VI: estimated cost of 0 for {{.*}} insertelement <2 x i16> +; GFX9: estimated cost of 0 for {{.*}} insertelement <2 x i16> +define amdgpu_kernel void @insertelement_0_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr - %insert = insertelement <2 x i16> %vec, i16 1, i16 123 + %insert = insertelement <2 x i16> %vec, i16 123, i16 0 store <2 x i16> %insert, <2 x i16> addrspace(1)* %out ret void } -; CHECK: 'insertelement_v2i8' -; CHECK: estimated cost of 0 for {{.*}} insertelement <2 x i8> -define amdgpu_kernel void @insertelement_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %vaddr) { +; GCN-LABEL: 'insertelement_1_v2i16' +; GCN: estimated cost of 1 for {{.*}} insertelement <2 x i16> +define amdgpu_kernel void @insertelement_1_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { + %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr + %insert = insertelement <2 x i16> %vec, i16 123, i16 1 + store <2 x i16> %insert, <2 x i16> addrspace(1)* %out + ret void +} + +; GCN-LABEL: 'insertelement_1_v2i8' +; GCN: estimated cost of 1 for {{.*}} insertelement <2 x i8> +define amdgpu_kernel void @insertelement_1_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %vaddr) { %vec = load <2 x i8>, <2 x i8> addrspace(1)* %vaddr - %insert = insertelement <2 x i8> %vec, i8 1, i8 123 + %insert = insertelement <2 x i8> %vec, i8 123, i8 1 store <2 x i8> %insert, <2 x i8> addrspace(1)* %out ret void } diff --git a/test/Analysis/CostModel/AMDGPU/shufflevector.ll b/test/Analysis/CostModel/AMDGPU/shufflevector.ll new file mode 100644 index 00000000000..cc756c82fed --- /dev/null +++ b/test/Analysis/CostModel/AMDGPU/shufflevector.ll @@ -0,0 +1,43 @@ +; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GFX9,GCN %s +; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=VI,GCN %s + +; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> zeroinitializer +define amdgpu_kernel void @shufflevector_00_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { + %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr + %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> zeroinitializer + store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out + ret void +} + +; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> +define amdgpu_kernel void @shufflevector_01_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { + %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr + %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> + store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out + ret void +} + +; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> +define amdgpu_kernel void @shufflevector_10_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { + %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr + %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> + store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out + ret void +} + +; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> +define amdgpu_kernel void @shufflevector_11_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) { + %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr + %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> + store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out + ret void +} + +; GCN: estimated cost of 2 for {{.*}} shufflevector <2 x i16> %vec0, <2 x i16> %vec1, <2 x i32> +define amdgpu_kernel void @shufflevector_02_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr0, <2 x i16> addrspace(1)* %vaddr1) { + %vec0 = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr0 + %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr1 + %shuf = shufflevector <2 x i16> %vec0, <2 x i16> %vec1, <2 x i32> + store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out + ret void +} -- 2.40.0