From fcd77e8a04a3c73fcd0f6c8f292d3dfc8bc38f13 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 31 Aug 2017 23:53:50 +0000 Subject: [PATCH] AMDGPU: Fold clamp modifier for packed instructions git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@312297 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Target/AMDGPU/SIDefines.h | 21 ++- lib/Target/AMDGPU/SIFoldOperands.cpp | 23 ++- lib/Target/AMDGPU/SIInstrFormats.td | 27 +++- lib/Target/AMDGPU/SIInstrInfo.h | 16 +- lib/Target/AMDGPU/SIInstrInfo.td | 2 + lib/Target/AMDGPU/VOPInstructions.td | 4 + test/CodeGen/AMDGPU/clamp-modifier.ll | 202 ++++++++++++++++++++++++-- 7 files changed, 260 insertions(+), 35 deletions(-) diff --git a/lib/Target/AMDGPU/SIDefines.h b/lib/Target/AMDGPU/SIDefines.h index 500b25bbb04..98cf255d710 100644 --- a/lib/Target/AMDGPU/SIDefines.h +++ b/lib/Target/AMDGPU/SIDefines.h @@ -67,11 +67,22 @@ enum : uint64_t { SCALAR_STORE = UINT64_C(1) << 39, FIXED_SIZE = UINT64_C(1) << 40, VOPAsmPrefer32Bit = UINT64_C(1) << 41, - HasFPClamp = UINT64_C(1) << 42, - VOP3_OPSEL = UINT64_C(1) << 43, - maybeAtomic = UINT64_C(1) << 44, - F16_ZFILL = UINT64_C(1) << 45, - IntClamp = UINT64_C(1) << 46 + VOP3_OPSEL = UINT64_C(1) << 42, + maybeAtomic = UINT64_C(1) << 43, + F16_ZFILL = UINT64_C(1) << 44, + + // Is a clamp on FP type. + FPClamp = UINT64_C(1) << 45, + + // Is an integer clamp + IntClamp = UINT64_C(1) << 46, + + // Clamps lo component of register. + ClampLo = UINT64_C(1) << 47, + + // Clamps hi component of register. + // ClampLo and ClampHi set for packed clamp. + ClampHi = UINT64_C(1) << 48 }; // v_cmp_class_* etc. use a 10-bit mask for what operation is checked. diff --git a/lib/Target/AMDGPU/SIFoldOperands.cpp b/lib/Target/AMDGPU/SIFoldOperands.cpp index 0aad8f0843d..1a30a163e6d 100644 --- a/lib/Target/AMDGPU/SIFoldOperands.cpp +++ b/lib/Target/AMDGPU/SIFoldOperands.cpp @@ -728,7 +728,8 @@ const MachineOperand *SIFoldOperands::isClamp(const MachineInstr &MI) const { switch (Op) { case AMDGPU::V_MAX_F32_e64: case AMDGPU::V_MAX_F16_e64: - case AMDGPU::V_MAX_F64: { + case AMDGPU::V_MAX_F64: + case AMDGPU::V_PK_MAX_F16: { if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm()) return nullptr; @@ -741,9 +742,18 @@ const MachineOperand *SIFoldOperands::isClamp(const MachineInstr &MI) const { return nullptr; // Can't fold up if we have modifiers. - if (TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || - TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || - TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) + if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) + return nullptr; + + unsigned Src0Mods + = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm(); + unsigned Src1Mods + = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm(); + + // Having a 0 op_sel_hi would require swizzling the output in the source + // instruction, which we can't do. + unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1 : 0; + if (Src0Mods != UnsetMods && Src1Mods != UnsetMods) return nullptr; return Src0; } @@ -771,8 +781,11 @@ bool SIFoldOperands::tryFoldClamp(MachineInstr &MI) { return false; MachineInstr *Def = MRI->getVRegDef(ClampSrc->getReg()); - if (!TII->hasFPClamp(*Def)) + + // The type of clamp must be compatible. + if (TII->getClampMask(*Def) != TII->getClampMask(MI)) return false; + MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp); if (!DefClamp) return false; diff --git a/lib/Target/AMDGPU/SIInstrFormats.td b/lib/Target/AMDGPU/SIInstrFormats.td index b4b4eed9b43..faf14fff5b2 100644 --- a/lib/Target/AMDGPU/SIInstrFormats.td +++ b/lib/Target/AMDGPU/SIInstrFormats.td @@ -79,10 +79,6 @@ class InstSI _ArgVT> { field bit HasSDWAClamp = EmitDst; field bit HasFPClamp = BitAnd.ret, HasClamp>.ret; field bit HasIntClamp = !if(isFloatType.ret, 0, HasClamp); + field bit HasClampLo = HasClamp; + field bit HasClampHi = BitAnd.ret, HasClamp>.ret; field bit HasHigh = 0; field bit IsPacked = isPackedType.ret; diff --git a/lib/Target/AMDGPU/VOPInstructions.td b/lib/Target/AMDGPU/VOPInstructions.td index 823fb4c02c3..6f9f3339e6d 100644 --- a/lib/Target/AMDGPU/VOPInstructions.td +++ b/lib/Target/AMDGPU/VOPInstructions.td @@ -106,6 +106,10 @@ class VOP3_Pseudo pattern = [], let VOP3 = 1; let VALU = 1; let FPClamp = P.HasFPClamp; + let IntClamp = P.HasIntClamp; + let ClampLo = P.HasClampLo; + let ClampHi = P.HasClampHi; + let Uses = [EXEC]; let AsmVariantName = AMDGPUAsmVariants.VOP3; diff --git a/test/CodeGen/AMDGPU/clamp-modifier.ll b/test/CodeGen/AMDGPU/clamp-modifier.ll index c3a7d5e14d8..9c137101c56 100644 --- a/test/CodeGen/AMDGPU/clamp-modifier.ll +++ b/test/CodeGen/AMDGPU/clamp-modifier.ll @@ -1,8 +1,9 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s -; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89 %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9 %s ; GCN-LABEL: {{^}}v_clamp_add_src_f32: -; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]] +; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] ; GCN-NOT: [[A]] ; GCN: v_add_f32_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}} define amdgpu_kernel void @v_clamp_add_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 { @@ -18,7 +19,7 @@ define amdgpu_kernel void @v_clamp_add_src_f32(float addrspace(1)* %out, float a } ; GCN-LABEL: {{^}}v_clamp_multi_use_src_f32: -; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]] +; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] ; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}} ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[ADD]], [[ADD]] clamp{{$}} define amdgpu_kernel void @v_clamp_multi_use_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 { @@ -35,7 +36,7 @@ define amdgpu_kernel void @v_clamp_multi_use_src_f32(float addrspace(1)* %out, f } ; GCN-LABEL: {{^}}v_clamp_dbg_use_src_f32: -; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]] +; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] ; GCN-NOT: [[A]] ; GCN: v_add_f32_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}} define amdgpu_kernel void @v_clamp_dbg_use_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 { @@ -52,7 +53,7 @@ define amdgpu_kernel void @v_clamp_dbg_use_src_f32(float addrspace(1)* %out, flo } ; GCN-LABEL: {{^}}v_clamp_add_neg_src_f32: -; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]] +; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] ; GCN: v_floor_f32_e32 [[FLOOR:v[0-9]+]], [[A]] ; GCN: v_max_f32_e64 v{{[0-9]+}}, -[[FLOOR]], -[[FLOOR]] clamp{{$}} define amdgpu_kernel void @v_clamp_add_neg_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 { @@ -69,7 +70,7 @@ define amdgpu_kernel void @v_clamp_add_neg_src_f32(float addrspace(1)* %out, flo } ; GCN-LABEL: {{^}}v_non_clamp_max_f32: -; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]] +; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] ; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}} ; GCN: v_max_f32_e32 v{{[0-9]+}}, 0, [[ADD]]{{$}} define amdgpu_kernel void @v_non_clamp_max_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 { @@ -84,7 +85,7 @@ define amdgpu_kernel void @v_non_clamp_max_f32(float addrspace(1)* %out, float a } ; GCN-LABEL: {{^}}v_clamp_add_src_f32_denormals: -; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]] +; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] ; GCN: v_add_f32_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}} define amdgpu_kernel void @v_clamp_add_src_f32_denormals(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 { %tid = call i32 @llvm.amdgcn.workitem.id.x() @@ -99,8 +100,8 @@ define amdgpu_kernel void @v_clamp_add_src_f32_denormals(float addrspace(1)* %ou } ; GCN-LABEL: {{^}}v_clamp_add_src_f16_denorm: -; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]] -; VI: v_add_f16_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}} +; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]] +; GFX89: v_add_f16_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}} ; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]] ; SI: v_add_f32_e64 [[ADD:v[0-9]+]], [[CVT]], 1.0 clamp{{$}} @@ -118,9 +119,9 @@ define amdgpu_kernel void @v_clamp_add_src_f16_denorm(half addrspace(1)* %out, h } ; GCN-LABEL: {{^}}v_clamp_add_src_f16_no_denormals: -; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]] -; VI-NOT: [[A]] -; VI: v_add_f16_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}} +; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]] +; GFX89-NOT: [[A]] +; GFX89: v_add_f16_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}} ; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]] ; SI: v_add_f32_e64 [[ADD:v[0-9]+]], [[CVT]], 1.0 clamp{{$}} @@ -138,7 +139,7 @@ define amdgpu_kernel void @v_clamp_add_src_f16_no_denormals(half addrspace(1)* % } ; GCN-LABEL: {{^}}v_clamp_add_src_v2f32: -; GCN: {{buffer|flat}}_load_dwordx2 v{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}} +; GCN: {{buffer|flat|global}}_load_dwordx2 v{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}} ; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, v[[A]], 1.0 clamp{{$}} ; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, v[[B]], 1.0 clamp{{$}} define amdgpu_kernel void @v_clamp_add_src_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %aptr) #0 { @@ -154,7 +155,7 @@ define amdgpu_kernel void @v_clamp_add_src_v2f32(<2 x float> addrspace(1)* %out, } ; GCN-LABEL: {{^}}v_clamp_add_src_f64: -; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]] +; GCN: {{buffer|flat|global}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]] ; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, [[A]], 1.0 clamp{{$}} define amdgpu_kernel void @v_clamp_add_src_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 { %tid = call i32 @llvm.amdgcn.workitem.id.x() @@ -185,6 +186,173 @@ define amdgpu_kernel void @v_clamp_mac_to_mad(float addrspace(1)* %out, float ad ret void } + +; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm: +; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] +; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00 +; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]] clamp{{$}} +define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid + %a = load <2 x half>, <2 x half> addrspace(1)* %gep0 + %add = fadd <2 x half> %a, + %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %add, <2 x half> zeroinitializer) + %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> ) + store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_no_denormals: +; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] +; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00 +; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]] clamp{{$}} +define amdgpu_kernel void @v_clamp_add_src_v2f16_no_denormals(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #3 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid + %a = load <2 x half>, <2 x half> addrspace(1)* %gep0 + %add = fadd <2 x half> %a, + %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %add, <2 x half> zeroinitializer) + %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> ) + store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg: +; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] +; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00 +; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}} +; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_lo:[1,1] neg_hi:[1,1] clamp{{$}} +define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid + %a = load <2 x half>, <2 x half> addrspace(1)* %gep0 + %add = fadd <2 x half> %a, + %neg.add = fsub <2 x half> , %add + %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.add, <2 x half> zeroinitializer) + %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> ) + store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg_lo: +; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] +; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00 +; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}} +; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_lo:[1,1] clamp{{$}} +define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg_lo(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid + %a = load <2 x half>, <2 x half> addrspace(1)* %gep0 + %add = fadd <2 x half> %a, + %lo = extractelement <2 x half> %add, i32 0 + %neg.lo = fsub half -0.0, %lo + %neg.lo.add = insertelement <2 x half> %add, half %neg.lo, i32 0 + %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.lo.add, <2 x half> zeroinitializer) + %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> ) + store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg_hi: +; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] +; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00 +; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}} +; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_hi:[1,1] clamp{{$}} +define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg_hi(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid + %a = load <2 x half>, <2 x half> addrspace(1)* %gep0 + %add = fadd <2 x half> %a, + %hi = extractelement <2 x half> %add, i32 1 + %neg.hi = fsub half -0.0, %hi + %neg.hi.add = insertelement <2 x half> %add, half %neg.hi, i32 1 + %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.hi.add, <2 x half> zeroinitializer) + %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> ) + store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_shuf: +; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] +; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00 +; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}} +; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] op_sel:[1,1] op_sel_hi:[0,0] clamp{{$}} +define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_shuf(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid + %a = load <2 x half>, <2 x half> addrspace(1)* %gep0 + %add = fadd <2 x half> %a, + %shuf = shufflevector <2 x half> %add, <2 x half> undef, <2 x i32> + + %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %shuf, <2 x half> zeroinitializer) + %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> ) + store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}v_no_clamp_add_src_v2f16_f32_src: +; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] +; GFX9: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}} +; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}} +define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f32_src(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid + %a = load <2 x half>, <2 x half> addrspace(1)* %gep0 + %bc = bitcast <2 x half> %a to float + %f32.op = fadd float %bc, 1.0 + %f32.op.cast = bitcast float %f32.op to <2 x half> + %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %f32.op.cast, <2 x half> zeroinitializer) + %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> ) + store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}v_no_clamp_add_packed_src_f32: +; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]] +; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00 +; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}} +; GFX9: v_max_f32_e64 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}} +define amdgpu_kernel void @v_no_clamp_add_packed_src_f32(float addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid + %a = load <2 x half>, <2 x half> addrspace(1)* %gep0 + %add = fadd <2 x half> %a, + %bc.add = bitcast <2 x half> %add to float + %max = call float @llvm.maxnum.f32(float %bc.add, float 0.0) + %clamp = call float @llvm.minnum.f32(float %max, float 1.0) + store float %clamp, float addrspace(1)* %out.gep + ret void +} + +; Since the high bits are zeroed, it probably would be OK in this case +; to use clamp. +; GCN-LABEL: {{^}}v_no_clamp_add_src_v2f16_f16_src: +; GCN-DAG: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]] +; GFX9: v_add_f16_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}} +; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}} +define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f16_src(<2 x half> addrspace(1)* %out, half addrspace(1)* %aptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep0 = getelementptr half, half addrspace(1)* %aptr, i32 %tid + %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid + %a = load half, half addrspace(1)* %gep0 + %add = fadd half %a, 1.0 + %bc = bitcast half %add to i16 + %zext = zext i16 %bc to i32 + %v2f16 = bitcast i32 %zext to <2 x half> + %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %v2f16, <2 x half> zeroinitializer) + %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> ) + store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep + ret void +} + declare i32 @llvm.amdgcn.workitem.id.x() #1 declare float @llvm.fabs.f32(float) #1 declare float @llvm.floor.f32(float) #1 @@ -197,8 +365,12 @@ declare double @llvm.maxnum.f64(double, double) #1 declare half @llvm.fabs.f16(half) #1 declare half @llvm.minnum.f16(half, half) #1 declare half @llvm.maxnum.f16(half, half) #1 +declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>) #1 +declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>) #1 declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) #1 declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>) #1 + + declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1 attributes #0 = { nounwind } -- 2.50.1