AMDGPU: Fix i16 arithmetic pattern redundancy

There were 2 problems here. First, these patterns were duplicated to
handle the inverted shift operands instead of using the commuted
PatFrags.

Second, the point of the zext folding patterns don't apply to the
non-0ing high subtargets. They should be skipped instead of inserting
the extension. The zeroing high code would be emitted when necessary
anyway. This was also emitting unnecessary zexts in cases where the
high bits were undefined.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@374092 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/AMDGPU/VOP2Instructions.td b/lib/Target/AMDGPU/VOP2Instructions.td
index 154c712d1aa3345e3fa3fd8ea720d0b9f0e459e5..8a5847d6f5b016ba34abc18f1a89f1d6f598ed11 100644 (file)
--- a/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/lib/Target/AMDGPU/VOP2Instructions.td
@@ -608,9 +608,9 @@ def V_MADMK_F16 : VOP2_Pseudo <"v_madmk_f16", VOP_MADMK_F16, [], "">;
 defm V_LDEXP_F16 : VOP2Inst <"v_ldexp_f16", VOP_F16_F16_I32, AMDGPUldexp>;
 } // End FPDPRounding = 1
 
-defm V_LSHLREV_B16 : VOP2Inst <"v_lshlrev_b16", VOP_I16_I16_I16>;
-defm V_LSHRREV_B16 : VOP2Inst <"v_lshrrev_b16", VOP_I16_I16_I16>;
-defm V_ASHRREV_I16 : VOP2Inst <"v_ashrrev_i16", VOP_I16_I16_I16>;
+defm V_LSHLREV_B16 : VOP2Inst <"v_lshlrev_b16", VOP_I16_I16_I16, lshl_rev>;
+defm V_LSHRREV_B16 : VOP2Inst <"v_lshrrev_b16", VOP_I16_I16_I16, lshr_rev>;
+defm V_ASHRREV_I16 : VOP2Inst <"v_ashrrev_i16", VOP_I16_I16_I16, ashr_rev>;
 
 let isCommutable = 1 in {
 let FPDPRounding = 1 in {
@@ -620,16 +620,16 @@ defm V_SUBREV_F16 : VOP2Inst <"v_subrev_f16", VOP_F16_F16_F16, null_frag, "v_sub
 defm V_MUL_F16 : VOP2Inst <"v_mul_f16", VOP_F16_F16_F16, fmul>;
 def V_MADAK_F16 : VOP2_Pseudo <"v_madak_f16", VOP_MADAK_F16, [], "">;
 } // End FPDPRounding = 1
-defm V_ADD_U16 : VOP2Inst <"v_add_u16", VOP_I16_I16_I16>;
-defm V_SUB_U16 : VOP2Inst <"v_sub_u16" , VOP_I16_I16_I16>;
+defm V_ADD_U16 : VOP2Inst <"v_add_u16", VOP_I16_I16_I16, add>;
+defm V_SUB_U16 : VOP2Inst <"v_sub_u16" , VOP_I16_I16_I16, sub>;
 defm V_SUBREV_U16 : VOP2Inst <"v_subrev_u16", VOP_I16_I16_I16, null_frag, "v_sub_u16">;
-defm V_MUL_LO_U16 : VOP2Inst <"v_mul_lo_u16", VOP_I16_I16_I16>;
+defm V_MUL_LO_U16 : VOP2Inst <"v_mul_lo_u16", VOP_I16_I16_I16, mul>;
 defm V_MAX_F16 : VOP2Inst <"v_max_f16", VOP_F16_F16_F16, fmaxnum_like>;
 defm V_MIN_F16 : VOP2Inst <"v_min_f16", VOP_F16_F16_F16, fminnum_like>;
-defm V_MAX_U16 : VOP2Inst <"v_max_u16", VOP_I16_I16_I16>;
-defm V_MAX_I16 : VOP2Inst <"v_max_i16", VOP_I16_I16_I16>;
-defm V_MIN_U16 : VOP2Inst <"v_min_u16", VOP_I16_I16_I16>;
-defm V_MIN_I16 : VOP2Inst <"v_min_i16", VOP_I16_I16_I16>;
+defm V_MAX_U16 : VOP2Inst <"v_max_u16", VOP_I16_I16_I16, umax>;
+defm V_MAX_I16 : VOP2Inst <"v_max_i16", VOP_I16_I16_I16, smax>;
+defm V_MIN_U16 : VOP2Inst <"v_min_u16", VOP_I16_I16_I16, umin>;
+defm V_MIN_I16 : VOP2Inst <"v_min_i16", VOP_I16_I16_I16, smin>;
 
 let Constraints = "$vdst = $src2", DisableEncoding="$src2",
     isConvertibleToThreeAddress = 1 in {
@@ -722,53 +722,17 @@ defm V_PK_FMAC_F16 : VOP2Inst<"v_pk_fmac_f16", VOP_V2F16_V2F16_V2F16>;
 
 // Note: 16-bit instructions produce a 0 result in the high 16-bits
 // on GFX8 and GFX9 and preserve high 16 bits on GFX10+
-def ClearHI16 : OutPatFrag<(ops node:$op),
-                           (V_AND_B32_e64 $op, (V_MOV_B32_e32 (i32 0xffff)))>;
-
-multiclass Arithmetic_i16_Pats <SDPatternOperator op, Instruction inst,
-                                bit PreservesHI16 = 0> {
-
-def : GCNPat<
-  (op i16:$src0, i16:$src1),
-  !if(!eq(PreservesHI16,1), (ClearHI16 (inst $src0, $src1)), (inst $src0, $src1))
->;
+multiclass Arithmetic_i16_0Hi_Pats <SDPatternOperator op, Instruction inst> {
 
 def : GCNPat<
   (i32 (zext (op i16:$src0, i16:$src1))),
-  !if(!eq(PreservesHI16,1), (ClearHI16 (inst $src0, $src1)), (inst $src0, $src1))
+  (inst $src0, $src1)
 >;
 
 def : GCNPat<
   (i64 (zext (op i16:$src0, i16:$src1))),
    (REG_SEQUENCE VReg_64,
-     !if(!eq(PreservesHI16,1), (ClearHI16 (inst $src0, $src1)), (inst $src0, $src1)),
-     sub0,
-     (V_MOV_B32_e32 (i32 0)), sub1)
->;
-}
-
-multiclass Bits_OpsRev_i16_Pats <SDPatternOperator op, Instruction inst,
-                                 bit PreservesHI16 = 0> {
-
-def : GCNPat<
-  (op i16:$src0, i16:$src1),
-  !if(!eq(PreservesHI16,1), (ClearHI16 (inst VSrc_b32:$src1, VSrc_b32:$src0)),
-                                       (inst VSrc_b32:$src1, VSrc_b32:$src0))
->;
-
-def : GCNPat<
-  (i32 (zext (op i16:$src0, i16:$src1))),
-  !if(!eq(PreservesHI16,1), (ClearHI16 (inst VSrc_b32:$src1, VSrc_b32:$src0)),
-                                       (inst VSrc_b32:$src1, VSrc_b32:$src0))
->;
-
-
-def : GCNPat<
-  (i64 (zext (op i16:$src0, i16:$src1))),
-   (REG_SEQUENCE VReg_64,
-     !if(!eq(PreservesHI16,1), (ClearHI16 (inst VSrc_b32:$src1, VSrc_b32:$src0)),
-                                          (inst VSrc_b32:$src1, VSrc_b32:$src0)),
-     sub0,
+     (inst $src0, $src1), sub0,
      (V_MOV_B32_e32 (i32 0)), sub1)
 >;
 }
@@ -800,35 +764,16 @@ def : GCNPat <
 let Predicates = [Has16BitInsts] in {
 
 let Predicates = [Has16BitInsts, isGFX7GFX8GFX9] in {
-defm : Arithmetic_i16_Pats<add, V_ADD_U16_e64>;
-defm : Arithmetic_i16_Pats<mul, V_MUL_LO_U16_e64>;
-defm : Arithmetic_i16_Pats<sub, V_SUB_U16_e64>;
-defm : Arithmetic_i16_Pats<smin, V_MIN_I16_e64>;
-defm : Arithmetic_i16_Pats<smax, V_MAX_I16_e64>;
-defm : Arithmetic_i16_Pats<umin, V_MIN_U16_e64>;
-defm : Arithmetic_i16_Pats<umax, V_MAX_U16_e64>;
-}
-
-let Predicates = [Has16BitInsts, isGFX10Plus] in {
-defm : Arithmetic_i16_Pats<add, V_ADD_U16_e64,    1>;
-defm : Arithmetic_i16_Pats<mul, V_MUL_LO_U16_e64, 1>;
-defm : Arithmetic_i16_Pats<sub, V_SUB_U16_e64,    1>;
-defm : Arithmetic_i16_Pats<smin, V_MIN_I16_e64,   1>;
-defm : Arithmetic_i16_Pats<smax, V_MAX_I16_e64,   1>;
-defm : Arithmetic_i16_Pats<umin, V_MIN_U16_e64,   1>;
-defm : Arithmetic_i16_Pats<umax, V_MAX_U16_e64,   1>;
-}
-
-let Predicates = [Has16BitInsts, isGFX7GFX8GFX9] in {
-defm : Bits_OpsRev_i16_Pats<shl, V_LSHLREV_B16_e64>;
-defm : Bits_OpsRev_i16_Pats<srl, V_LSHRREV_B16_e64>;
-defm : Bits_OpsRev_i16_Pats<sra, V_ASHRREV_I16_e64>;
-}
-
-let Predicates = [Has16BitInsts, isGFX10Plus] in {
-defm : Bits_OpsRev_i16_Pats<shl, V_LSHLREV_B16_e64, 1>;
-defm : Bits_OpsRev_i16_Pats<srl, V_LSHRREV_B16_e64, 1>;
-defm : Bits_OpsRev_i16_Pats<sra, V_ASHRREV_I16_e64, 1>;
+defm : Arithmetic_i16_0Hi_Pats<add, V_ADD_U16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<mul, V_MUL_LO_U16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<sub, V_SUB_U16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<smin, V_MIN_I16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<smax, V_MAX_I16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<umin, V_MIN_U16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<umax, V_MAX_U16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<lshl_rev, V_LSHLREV_B16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<lshr_rev, V_LSHRREV_B16_e64>;
+defm : Arithmetic_i16_0Hi_Pats<ashr_rev, V_ASHRREV_I16_e64>;
 }
 
 def : ZExt_i16_i1_Pat<zext>;

diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
index e1ce7872e935597dbe6082c13306975c45df9572..e0f94a60496b45d52077ca689e3bfa95cc984917 100644 (file)
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
@@ -78,10 +78,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_ASHRREV_I16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:sgpr(s32) = COPY $sgpr0
     %2:vgpr(s16) = G_TRUNC %0
@@ -147,10 +145,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_ASHRREV_I16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -184,10 +180,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_ASHRREV_I16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: [[V_BFE_U32_:%[0-9]+]]:vgpr_32 = V_BFE_U32 [[V_AND_B32_e64_]], 0, 16, implicit $exec
+    ; GFX10: [[V_BFE_U32_:%[0-9]+]]:vgpr_32 = V_BFE_U32 [[V_ASHRREV_I16_e64_]], 0, 16, implicit $exec
     ; GFX10: S_ENDPGM 0, implicit [[V_BFE_U32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -329,10 +323,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_ASHRREV_I16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s16) = G_TRUNC %0

diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
index 65e9af7719665b4fa95e90bfe6d2a0babcf13a7f..95daab6495571f58a4af871f2eece843ac5724b0 100644 (file)
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
@@ -78,10 +78,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_LSHRREV_B16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:sgpr(s32) = COPY $sgpr0
     %2:vgpr(s16) = G_TRUNC %0
@@ -147,10 +145,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_LSHRREV_B16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -184,10 +180,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_LSHRREV_B16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: [[V_BFE_U32_:%[0-9]+]]:vgpr_32 = V_BFE_U32 [[V_AND_B32_e64_]], 0, 16, implicit $exec
+    ; GFX10: [[V_BFE_U32_:%[0-9]+]]:vgpr_32 = V_BFE_U32 [[V_LSHRREV_B16_e64_]], 0, 16, implicit $exec
     ; GFX10: S_ENDPGM 0, implicit [[V_BFE_U32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -329,10 +323,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_LSHRREV_B16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s16) = G_TRUNC %0

diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
index 07b4f9a3dcd9104ae5cc9520dae93d18a67b40d3..4ba49247afeb85b76f02d6a433dce6aba560505e 100644 (file)
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
@@ -78,10 +78,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_LSHLREV_B16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:sgpr(s32) = COPY $sgpr0
     %2:vgpr(s16) = G_TRUNC %0
@@ -147,10 +145,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_LSHLREV_B16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
     %2:vgpr(s16) = G_TRUNC %0
@@ -184,10 +180,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_LSHLREV_B16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: [[V_BFE_U32_:%[0-9]+]]:vgpr_32 = V_BFE_U32 [[V_AND_B32_e64_]], 0, 16, implicit $exec
+    ; GFX10: [[V_BFE_U32_:%[0-9]+]]:vgpr_32 = V_BFE_U32 [[V_LSHLREV_B16_e64_]], 0, 16, implicit $exec
     ; GFX10: S_ENDPGM 0, implicit [[V_BFE_U32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -329,10 +323,8 @@ body: |
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
     ; GFX10: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[V_LSHLREV_B16_e64_]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX10: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ; GFX10: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s16) = G_TRUNC %0

diff --git a/test/CodeGen/AMDGPU/idot2.ll b/test/CodeGen/AMDGPU/idot2.ll
index 200851712740bf840bb291765efcf009c588a9ce..74132defec7a93b64c4ca84eb498343769426c20 100644 (file)
--- a/test/CodeGen/AMDGPU/idot2.ll
+++ b/test/CodeGen/AMDGPU/idot2.ll
@@ -2775,7 +2775,6 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
@@ -2784,13 +2783,13 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
 ; GFX10-DL-NEXT:    global_load_ushort v2, v[2:3], off
-; GFX10-DL-NEXT:    global_load_ushort v7, v[0:1], off
+; GFX10-DL-NEXT:    global_load_ushort v0, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v1, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, v2
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v3, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, v0
 ; GFX10-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX10-DL-NEXT:    v_bfe_i32 v0, v7, 0, 8
+; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
 ; GFX10-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)

diff --git a/test/CodeGen/AMDGPU/idot4s.ll b/test/CodeGen/AMDGPU/idot4s.ll
index d9a705c735e622c20234262c28539ba96be4c689..698a620965affdeee5a5e96ef68dbcb5cc4e1fa8 100644 (file)
--- a/test/CodeGen/AMDGPU/idot4s.ll
+++ b/test/CodeGen/AMDGPU/idot4s.ll
@@ -841,7 +841,6 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
@@ -850,19 +849,19 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v3, s2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v4, s4
 ; GFX10-DL-NEXT:    s_sext_i32_i8 s0, s2
 ; GFX10-DL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX10-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
 ; GFX10-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GFX10-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
 ; GFX10-DL-NEXT:    s_bfe_i32 s4, s2, 0x80010
 ; GFX10-DL-NEXT:    s_bfe_i32 s5, s3, 0x80010
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v4, s0, s1, v4
 ; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 24
 ; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 24
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, v3, v2, v4
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
@@ -1057,16 +1056,16 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_bfe_i32 s1, s3, 0x80000
 ; GFX10-DL-NEXT:    s_lshr_b32 s4, s2, 16
 ; GFX10-DL-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, sext(s2), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 8, s2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v7, s0, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v6, s1, v2
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, sext(s3), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 8, s3
 ; GFX10-DL-NEXT:    s_bfe_i32 s0, s4, 0x80000
 ; GFX10-DL-NEXT:    s_bfe_i32 s1, s5, 0x80000
 ; GFX10-DL-NEXT:    v_lshl_or_b32 v4, v4, 16, v7
-; GFX10-DL-NEXT:    v_and_b32_sdwa v8, sext(s4), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 8, s4
 ; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v5, 16, v6
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, sext(s5), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v6, 8, s5
 ; GFX10-DL-NEXT:    v_and_b32_e32 v7, s1, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v2, s0, v2
 ; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5

diff --git a/test/CodeGen/AMDGPU/idot4u.ll b/test/CodeGen/AMDGPU/idot4u.ll
index 4c1dfb4a645aef687cdbb71bad2806d4625856d1..fddb14dbf9754c73ba752a82d62a3fd1ee59bda2 100644 (file)
--- a/test/CodeGen/AMDGPU/idot4u.ll
+++ b/test/CodeGen/AMDGPU/idot4u.ll
@@ -1738,28 +1738,30 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX10-DL-NEXT:    s_movk_i32 s3, 0xff
+; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_and_b32 s0, s3, s2
-; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
+; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, s4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    s_bfe_u32 s2, s3, 0x80010
-; GFX10-DL-NEXT:    s_bfe_u32 s5, s4, 0x80010
-; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s0, s1, v3
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 24
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v4, v2, v3
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s5, v2
+; GFX10-DL-NEXT:    s_and_b32 s0, s4, s3
+; GFX10-DL-NEXT:    s_and_b32 s1, s5, s3
+; GFX10-DL-NEXT:    v_mov_b32_e32 v4, s6
+; GFX10-DL-NEXT:    v_and_b32_sdwa v2, s2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-DL-NEXT:    v_and_b32_sdwa v3, s2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-DL-NEXT:    s_bfe_u32 s3, s4, 0x80010
+; GFX10-DL-NEXT:    s_bfe_u32 s2, s5, 0x80010
+; GFX10-DL-NEXT:    v_mad_u32_u24 v4, s0, s1, v4
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s5, 24
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v2, v3, v4
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
@@ -1938,9 +1940,9 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-DL-NEXT:    global_load_ushort v3, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, s2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v4, 8, s2
 ; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v5, 8, s3
 ; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v2, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX10-DL-NEXT:    s_lshr_b32 s0, s2, 16
 ; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 16
@@ -2150,40 +2152,36 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
-; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ubyte v3, v[0:1], off
+; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, s4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 16
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s3, s4
-; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v4, v5
-; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v6, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s1, s3
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, s0, s4
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v6, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v7, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_e32 v2, v4, v2
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s2
+; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v4, 8, s3
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s2, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s2, s3
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, v3, v4
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s0, s1
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 16
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v3
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 8, v4
+; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s4, s0
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v4, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v5
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v5
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                              <4 x i8> addrspace(1)* %src2,

diff --git a/test/CodeGen/AMDGPU/idot8s.ll b/test/CodeGen/AMDGPU/idot8s.ll
index a60bf13fab70d8ceaa0c1193074d59dc34f1ce8c..e41f5fe6b6a3bdabb312f56b5e4d96dbc514a3f2 100644 (file)
--- a/test/CodeGen/AMDGPU/idot8s.ll
+++ b/test/CodeGen/AMDGPU/idot8s.ll
@@ -473,49 +473,47 @@ define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ushort v3, v[0:1], off
+; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s0, s2, 12
-; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 12
-; GFX10-DL-NEXT:    s_bfe_i32 s5, s2, 0x40000
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 12
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s5, 12
 ; GFX10-DL-NEXT:    s_bfe_i32 s6, s4, 0x40000
-; GFX10-DL-NEXT:    s_bfe_i32 s7, s2, 0x40004
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s0
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX10-DL-NEXT:    s_bfe_i32 s0, s4, 0x40004
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s2, 0x40008
-; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40008
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v5, v5, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s9, s2, 0x40010
-; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x40010
-; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v6, s1, s8
+; GFX10-DL-NEXT:    s_bfe_i32 s7, s5, 0x40000
+; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40004
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX10-DL-NEXT:    s_bfe_i32 s9, s5, 0x40004
+; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x40008
+; GFX10-DL-NEXT:    s_bfe_i32 s11, s5, 0x40008
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v5
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s2, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s11, s2, 0x40018
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v2, v5, v2
+; GFX10-DL-NEXT:    s_bfe_i32 s0, s4, 0x40010
+; GFX10-DL-NEXT:    s_bfe_i32 s1, s5, 0x40010
+; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v5, s10, s11
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x40014
+; GFX10-DL-NEXT:    s_bfe_i32 s11, s5, 0x40014
 ; GFX10-DL-NEXT:    s_bfe_i32 s12, s4, 0x40018
-; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX10-DL-NEXT:    s_bfe_i32 s2, s5, 0x40018
 ; GFX10-DL-NEXT:    s_ashr_i32 s4, s4, 28
+; GFX10-DL-NEXT:    s_ashr_i32 s5, s5, 28
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s5, s6, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s7, s0, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v4, v2, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s9, s10, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s8, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s11, s12, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s6, s7, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s8, s9, v2
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s10, s11, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s12, s2, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
 ; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                        <8 x i4> addrspace(1)* %src2,
@@ -818,7 +816,6 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
 ; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
@@ -826,40 +823,38 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ubyte v3, v[0:1], off
+; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 12
 ; GFX10-DL-NEXT:    s_lshr_b32 s1, s5, 12
 ; GFX10-DL-NEXT:    s_bfe_i32 s6, s4, 0x40000
 ; GFX10-DL-NEXT:    s_bfe_i32 s7, s5, 0x40000
 ; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40004
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s0
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX10-DL-NEXT:    s_bfe_i32 s0, s5, 0x40004
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s4, 0x40008
-; GFX10-DL-NEXT:    s_bfe_i32 s9, s5, 0x40008
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v2, v5, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x40010
-; GFX10-DL-NEXT:    s_bfe_i32 s11, s5, 0x40010
-; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v5, s1, s9
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX10-DL-NEXT:    s_bfe_i32 s9, s5, 0x40004
+; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x40008
+; GFX10-DL-NEXT:    s_bfe_i32 s11, s5, 0x40008
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v2, 12, v2
-; GFX10-DL-NEXT:    s_bfe_i32 s1, s4, 0x40014
-; GFX10-DL-NEXT:    s_bfe_i32 s9, s5, 0x40014
+; GFX10-DL-NEXT:    s_bfe_i32 s0, s4, 0x40010
+; GFX10-DL-NEXT:    s_bfe_i32 s1, s5, 0x40010
+; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v5, s10, s11
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX10-DL-NEXT:    s_bfe_i32 s10, s4, 0x40014
+; GFX10-DL-NEXT:    s_bfe_i32 s11, s5, 0x40014
 ; GFX10-DL-NEXT:    s_bfe_i32 s12, s4, 0x40018
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v4, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX10-DL-NEXT:    s_bfe_i32 s2, s5, 0x40018
 ; GFX10-DL-NEXT:    s_ashr_i32 s4, s4, 28
 ; GFX10-DL-NEXT:    s_ashr_i32 s5, s5, 28
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s6, s7, v3
-; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s8, s0, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v4, v2, v3
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s6, s7, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s8, s9, v2
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s10, s11, v2
-; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s9, v2
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s12, s2, v2
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
@@ -2289,133 +2284,94 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ;
 ; GFX10-DL-LABEL: idot8_acc8_vecMul:
 ; GFX10-DL:       ; %bb.0: ; %entry
-; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
-; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
-; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX10-DL-NEXT:    global_load_ubyte v3, v[0:1], off
+; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
+; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
-; GFX10-DL-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT:    s_lshr_b32 s9, s0, 4
-; GFX10-DL-NEXT:    s_lshr_b32 s16, s1, 4
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s0
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX10-DL-NEXT:    s_lshr_b32 s10, s0, 8
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s9
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s16
-; GFX10-DL-NEXT:    s_lshr_b32 s11, s0, 12
-; GFX10-DL-NEXT:    s_lshr_b32 s17, s1, 8
-; GFX10-DL-NEXT:    s_lshr_b32 s18, s1, 12
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s10
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v5, v5, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v8, v8, v2
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s18
-; GFX10-DL-NEXT:    v_and_b32_e32 v15, v15, v2
-; GFX10-DL-NEXT:    s_lshr_b32 s7, s0, 24
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v23, 12, s11
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v31, 12, s17
-; GFX10-DL-NEXT:    v_and_b32_e32 v7, v7, v2
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 4
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s5, 4
+; GFX10-DL-NEXT:    s_lshr_b32 s6, s4, 12
+; GFX10-DL-NEXT:    s_lshr_b32 s7, s5, 12
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s6
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX10-DL-NEXT:    s_lshr_b32 s8, s4, 8
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s5, 8
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
-; GFX10-DL-NEXT:    v_and_b32_e32 v13, v13, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v6, v23, v2
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v5
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s8
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s0
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 12, v7
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, v3, v4
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v6, 12, v6
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 12, v8
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v15, 12, v15
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v27, 12, s7
-; GFX10-DL-NEXT:    v_and_b32_e32 v14, v31, v2
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v23, 12, v6
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v9
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v5, v7
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 20
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v10
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s5, 20
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, v6, v8
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v3
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s0
+; GFX10-DL-NEXT:    s_lshr_b32 s8, s5, 16
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s1
+; GFX10-DL-NEXT:    s_lshr_b32 s9, s5, 28
+; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    s_lshr_b32 s7, s4, 28
+; GFX10-DL-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v4, v9
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 8, v5
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s9
+; GFX10-DL-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX10-DL-NEXT:    s_lshr_b32 s1, s5, 24
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s7
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 12, v7
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v13, 12, v13
-; GFX10-DL-NEXT:    v_and_b32_e32 v10, v27, v2
-; GFX10-DL-NEXT:    s_lshr_b32 s5, s0, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s6, s0, 20
-; GFX10-DL-NEXT:    s_lshr_b32 s12, s1, 16
-; GFX10-DL-NEXT:    s_lshr_b32 s13, s1, 20
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, v4, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v5, v5, v2
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v27, 12, v14
-; GFX10-DL-NEXT:    v_and_b32_e32 v8, v8, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v15, v15, v2
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s6
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v4, v5
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s5
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v19, 12, s12
-; GFX10-DL-NEXT:    s_lshr_b32 s8, s0, 28
-; GFX10-DL-NEXT:    s_lshr_b32 s14, s1, 24
-; GFX10-DL-NEXT:    s_lshr_b32 s15, s1, 28
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v35, 12, s13
-; GFX10-DL-NEXT:    v_and_b32_e32 v6, v23, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v7, v7, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v5, v27, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v13, v13, v2
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v8, v8, v15
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s8
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s15
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v7, v5
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v17, 12, s14
-; GFX10-DL-NEXT:    v_and_b32_e32 v11, v11, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v12, v12, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v18, v35, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v19, v19, v2
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v15, v6, v13
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v4, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v8, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_e32 v9, v9, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v16, v16, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v17, v17, v2
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 12, v8
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s8
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s0
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s1
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v15, 12, v9
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v19, 12, v6
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v13
+; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v10, 12, v10
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v7, v8
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 12, v11
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v12
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v35, 12, v18
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v19, 12, v19
-; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v5, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v15, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v9
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v31, 12, v10
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v16, 12, v16
-; GFX10-DL-NEXT:    v_and_b32_e32 v7, v11, v2
-; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, s4, v4
-; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v17, 12, v17
-; GFX10-DL-NEXT:    v_and_b32_e32 v10, v12, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v11, v19, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v6, v35, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v8, v9, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v13, v16, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v9, v31, v2
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v10, v10, v11
-; GFX10-DL-NEXT:    v_and_b32_e32 v12, v17, v2
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v11, v7, v6
-; GFX10-DL-NEXT:    v_or_b32_e32 v5, v4, v5
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v8, v8, v13
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v9, v12
-; GFX10-DL-NEXT:    v_and_b32_sdwa v9, v10, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v10, 8, v5
-; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v7, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, v19, v10
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v15, v9
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v8, 8, v4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 8, v6
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v4, v3
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v11, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v8, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v10
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v7, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, s4, v4
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NEXT:    v_or_b32_e32 v2, v4, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v4
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v4, 8, v2
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v4
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v7
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v11, v12
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v8
+; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v5
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                              <8 x i4> addrspace(1)* %src2,

diff --git a/test/CodeGen/AMDGPU/idot8u.ll b/test/CodeGen/AMDGPU/idot8u.ll
index e80095b4899e040ca1cf33ef8dffcff5ecea6af0..0532042417da5039c52bcb168f6e16181fd39251 100644 (file)
--- a/test/CodeGen/AMDGPU/idot8u.ll
+++ b/test/CodeGen/AMDGPU/idot8u.ll
@@ -2550,7 +2550,6 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL:       ; %bb.0: ; %entry
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
 ; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2558,7 +2557,7 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-DL-NEXT:    global_load_ubyte v3, v[0:1], off
+; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_bfe_u32 s0, s4, 0x40004
 ; GFX10-DL-NEXT:    s_bfe_u32 s1, s5, 0x40004
@@ -2566,47 +2565,47 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_and_b32 s8, s5, 15
 ; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x4000c
 ; GFX10-DL-NEXT:    s_bfe_u32 s9, s5, 0x4000c
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s0, s1
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, s0, s1
 ; GFX10-DL-NEXT:    s_bfe_u32 s0, s4, 0x40008
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s6, s8
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s6, s8
 ; GFX10-DL-NEXT:    s_bfe_u32 s1, s5, 0x40008
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v11, s7, s9
-; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s7, s9
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v3
 ; GFX10-DL-NEXT:    s_bfe_u32 s6, s4, 0x40014
 ; GFX10-DL-NEXT:    s_lshr_b32 s7, s4, 28
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, s0, s1
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v11, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s0, s1
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 8, v5
+; GFX10-DL-NEXT:    v_or_b32_e32 v3, v4, v3
 ; GFX10-DL-NEXT:    s_bfe_u32 s0, s5, 0x40014
 ; GFX10-DL-NEXT:    s_lshr_b32 s9, s5, 28
 ; GFX10-DL-NEXT:    s_bfe_u32 s1, s4, 0x40010
-; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_and_b32_e32 v4, s2, v4
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v11, s6, s0
+; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s6, s0
 ; GFX10-DL-NEXT:    s_bfe_u32 s8, s5, 0x40010
 ; GFX10-DL-NEXT:    s_bfe_u32 s0, s4, 0x40018
 ; GFX10-DL-NEXT:    s_bfe_u32 s4, s5, 0x40018
-; GFX10-DL-NEXT:    v_or_b32_e32 v5, v4, v5
-; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v11, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, s1, s8
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v8, s7, s9
-; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v9, s0, s4
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v10, 8, v5
-; GFX10-DL-NEXT:    v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v8, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-DL-NEXT:    v_and_b32_e32 v6, s2, v6
-; GFX10-DL-NEXT:    v_or_b32_sdwa v7, v9, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-DL-NEXT:    v_or_b32_e32 v2, v6, v7
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v14, 8, v2
+; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 8, v5
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s1, s8
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, s7, s9
+; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v8, s0, s4
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v9, 8, v4
+; GFX10-DL-NEXT:    v_or_b32_e32 v5, v6, v5
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 8, v7
+; GFX10-DL-NEXT:    v_and_b32_e32 v5, s2, v5
+; GFX10-DL-NEXT:    v_or_b32_sdwa v6, v8, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-DL-NEXT:    v_or_b32_e32 v11, v5, v6
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v7, 8, v11
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v4, v3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v10
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v6
-; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v14
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v9
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v5
+; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v7
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX10-DL-NEXT:    s_endpgm
                                              <8 x i4> addrspace(1)* %src2,

diff --git a/test/CodeGen/AMDGPU/preserve-hi16.ll b/test/CodeGen/AMDGPU/preserve-hi16.ll
index 4eeb43e92094f5cb47eefce61f4647870e993294..a0b03363945698b36730e319a06c92d317644ac7 100644 (file)
--- a/test/CodeGen/AMDGPU/preserve-hi16.ll
+++ b/test/CodeGen/AMDGPU/preserve-hi16.ll
@@ -3,8 +3,8 @@
 
 ; GCN-LABEL: {{^}}shl_i16:
 ; GCN: v_lshlrev_b16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @shl_i16(i16 %x, i16 %y) {
   %res = shl i16 %x, %y
   ret i16 %res
@@ -12,8 +12,8 @@ define i16 @shl_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}lshr_i16:
 ; GCN: v_lshrrev_b16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @lshr_i16(i16 %x, i16 %y) {
   %res = lshr i16 %x, %y
   ret i16 %res
@@ -21,8 +21,8 @@ define i16 @lshr_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}ashr_i16:
 ; GCN: v_ashrrev_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @ashr_i16(i16 %x, i16 %y) {
   %res = ashr i16 %x, %y
   ret i16 %res
@@ -30,8 +30,8 @@ define i16 @ashr_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}add_u16:
 ; GCN: v_add_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @add_u16(i16 %x, i16 %y) {
   %res = add i16 %x, %y
   ret i16 %res
@@ -39,8 +39,8 @@ define i16 @add_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}sub_u16:
 ; GCN: v_sub_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @sub_u16(i16 %x, i16 %y) {
   %res = sub i16 %x, %y
   ret i16 %res
@@ -48,8 +48,8 @@ define i16 @sub_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}mul_lo_u16:
 ; GCN: v_mul_lo_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @mul_lo_u16(i16 %x, i16 %y) {
   %res = mul i16 %x, %y
   ret i16 %res
@@ -57,8 +57,8 @@ define i16 @mul_lo_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}min_u16:
 ; GCN: v_min_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @min_u16(i16 %x, i16 %y) {
   %cmp = icmp ule i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
@@ -67,8 +67,8 @@ define i16 @min_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}min_i16:
 ; GCN: v_min_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @min_i16(i16 %x, i16 %y) {
   %cmp = icmp sle i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
@@ -77,8 +77,8 @@ define i16 @min_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}max_u16:
 ; GCN: v_max_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @max_u16(i16 %x, i16 %y) {
   %cmp = icmp uge i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
@@ -87,10 +87,124 @@ define i16 @max_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}max_i16:
 ; GCN: v_max_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX9-NEXT: s_setpc_b64
-; GFX10: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GCN-NEXT: s_setpc_b64
 define i16 @max_i16(i16 %x, i16 %y) {
   %cmp = icmp sge i16 %x, %y
   %res = select i1 %cmp, i16 %x, i16 %y
   ret i16 %res
 }
+
+; GCN-LABEL: {{^}}shl_i16_zext_i32:
+; GCN: v_lshlrev_b16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @shl_i16_zext_i32(i16 %x, i16 %y) {
+  %res = shl i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}lshr_i16_zext_i32:
+; GCN: v_lshrrev_b16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @lshr_i16_zext_i32(i16 %x, i16 %y) {
+  %res = lshr i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}ashr_i16_zext_i32:
+; GCN: v_ashrrev_i16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @ashr_i16_zext_i32(i16 %x, i16 %y) {
+  %res = ashr i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}add_u16_zext_i32:
+; GCN: v_add_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @add_u16_zext_i32(i16 %x, i16 %y) {
+  %res = add i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}sub_u16_zext_i32:
+; GCN: v_sub_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @sub_u16_zext_i32(i16 %x, i16 %y) {
+  %res = sub i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}mul_lo_u16_zext_i32:
+; GCN: v_mul_lo_u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @mul_lo_u16_zext_i32(i16 %x, i16 %y) {
+  %res = mul i16 %x, %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}min_u16_zext_i32:
+; GCN: v_min_u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @min_u16_zext_i32(i16 %x, i16 %y) {
+  %cmp = icmp ule i16 %x, %y
+  %res = select i1 %cmp, i16 %x, i16 %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}min_i16_zext_i32:
+; GCN: v_min_i16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @min_i16_zext_i32(i16 %x, i16 %y) {
+  %cmp = icmp sle i16 %x, %y
+  %res = select i1 %cmp, i16 %x, i16 %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}max_u16_zext_i32:
+; GCN: v_max_u16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @max_u16_zext_i32(i16 %x, i16 %y) {
+  %cmp = icmp uge i16 %x, %y
+  %res = select i1 %cmp, i16 %x, i16 %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}
+
+; GCN-LABEL: {{^}}max_i16_zext_i32:
+; GCN: v_max_i16_e{{32|64}} [[OP:v[0-9]+]],
+; GFX10-NEXT: ; implicit-def: $vcc_hi
+; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
+; GCN-NEXT: s_setpc_b64
+define i32 @max_i16_zext_i32(i16 %x, i16 %y) {
+  %cmp = icmp sge i16 %x, %y
+  %res = select i1 %cmp, i16 %x, i16 %y
+  %zext = zext i16 %res to i32
+  ret i32 %zext
+}

diff --git a/test/CodeGen/AMDGPU/sdwa-peephole.ll b/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 92a3ab207b0ab0c5a20ebaece3eb7da5dd1c3aeb..84693f61aa61fc70f68fc138f44199a6df785833 100644 (file)
--- a/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -283,11 +283,8 @@ entry:
 
 ; GFX9: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 
-; GFX10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-
-
+; GFX10: v_lshlrev_b16_e64 v{{[0-9]+}}, 8, v
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 define amdgpu_kernel void @mul_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %ina, <2 x i8> addrspace(1)* %inb) {
 entry:
   %a = load <2 x i8>, <2 x i8> addrspace(1)* %ina, align 4
@@ -501,10 +498,10 @@ store_label:
 ;
 ; GFX89: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ;
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 
 define amdgpu_kernel void @pulled_out_test(<8 x i8> addrspace(1)* %sourceA, <8 x i8> addrspace(1)* %destValues) {
 entry: