[AVX512] Add missing patterns for non-temporal stores of 128/256-bit vXi8/vXi16/vXi32...

This caused regular stores to be emitted instead.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@268886 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/X86/X86InstrAVX512.td b/lib/Target/X86/X86InstrAVX512.td
index 242b21dc1775960c12dd2c38e7e63a61db9d9d32..2115ec2ce812c1534a5f2e780176b1cd5cbddd30 100644 (file)
--- a/lib/Target/X86/X86InstrAVX512.td
+++ b/lib/Target/X86/X86InstrAVX512.td
@@ -3214,6 +3214,22 @@ defm VMOVNTDQ : avx512_movnt_vl<0xE7, "vmovntdq", avx512vl_i64_info>, PD;
 defm VMOVNTPD : avx512_movnt_vl<0x2B, "vmovntpd", avx512vl_f64_info>, PD, VEX_W;
 defm VMOVNTPS : avx512_movnt_vl<0x2B, "vmovntps", avx512vl_f32_info>, PS;
 
+let Predicates = [HasVLX], AddedComplexity = 400 in {
+  def : Pat<(alignednontemporalstore (v8i32 VR256X:$src), addr:$dst),
+            (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
+  def : Pat<(alignednontemporalstore (v16i16 VR256X:$src), addr:$dst),
+            (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
+  def : Pat<(alignednontemporalstore (v32i8 VR256X:$src), addr:$dst),
+            (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
+
+  def : Pat<(alignednontemporalstore (v4i32 VR128X:$src), addr:$dst),
+            (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
+  def : Pat<(alignednontemporalstore (v8i16 VR128X:$src), addr:$dst),
+            (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
+  def : Pat<(alignednontemporalstore (v16i8 VR128X:$src), addr:$dst),
+            (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
+}
+
 //===----------------------------------------------------------------------===//
 // AVX-512 - Integer arithmetic
 //

diff --git a/test/CodeGen/X86/nontemporal-2.ll b/test/CodeGen/X86/nontemporal-2.ll
index 58c30f4b4d10a6629ead09f631a631c243518ecf..07d9fd8ffa5ac9662cd4b8c688635a8903b89776 100644 (file)
--- a/test/CodeGen/X86/nontemporal-2.ll
+++ b/test/CodeGen/X86/nontemporal-2.ll
@@ -1,6 +1,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s -check-prefix=CHECK -check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s -check-prefix=CHECK -check-prefix=AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s -check-prefix=CHECK -check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s -check-prefix=CHECK -check-prefix=VLX
 
 ; Make sure that we generate non-temporal stores for the test cases below.
 ; We use xorps for zeroing, so domain information isn't available anymore.
@@ -10,6 +11,7 @@ define void @test_zero_v4f32(<4 x float>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -19,6 +21,7 @@ define void @test_zero_v4i32(<4 x i32>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <4 x i32> zeroinitializer, <4 x i32>* %dst, align 16, !nontemporal !1
   store <4 x i32> zeroinitializer, <4 x i32>* %dst, align 16, !nontemporal !1
   ret void
@@ -29,6 +32,7 @@ define void @test_zero_v2f64(<2 x double>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <2 x double> zeroinitializer, <2 x double>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -38,6 +42,7 @@ define void @test_zero_v2i64(<2 x i64>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <2 x i64> zeroinitializer, <2 x i64>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -47,6 +52,7 @@ define void @test_zero_v8i16(<8 x i16>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <8 x i16> zeroinitializer, <8 x i16>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -56,6 +62,7 @@ define void @test_zero_v16i8(<16 x i8>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <16 x i8> zeroinitializer, <16 x i8>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -66,6 +73,7 @@ define void @test_zero_v8f32(<8 x float>* %dst) {
 ; CHECK-LABEL: test_zero_v8f32:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <8 x float> zeroinitializer, <8 x float>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -74,6 +82,7 @@ define void @test_zero_v8i32(<8 x i32>* %dst) {
 ; CHECK-LABEL: test_zero_v8i32:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <8 x i32> zeroinitializer, <8 x i32>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -82,6 +91,7 @@ define void @test_zero_v4f64(<4 x double>* %dst) {
 ; CHECK-LABEL: test_zero_v4f64:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <4 x double> zeroinitializer, <4 x double>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -90,6 +100,7 @@ define void @test_zero_v4i64(<4 x i64>* %dst) {
 ; CHECK-LABEL: test_zero_v4i64:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <4 x i64> zeroinitializer, <4 x i64>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -98,6 +109,7 @@ define void @test_zero_v16i16(<16 x i16>* %dst) {
 ; CHECK-LABEL: test_zero_v16i16:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <16 x i16> zeroinitializer, <16 x i16>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -106,6 +118,7 @@ define void @test_zero_v32i8(<32 x i8>* %dst) {
 ; CHECK-LABEL: test_zero_v32i8:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <32 x i8> zeroinitializer, <32 x i8>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -118,6 +131,7 @@ define void @test_arg_v4f32(<4 x float> %arg, <4 x float>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntps
   store <4 x float> %arg, <4 x float>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -127,6 +141,7 @@ define void @test_arg_v4i32(<4 x i32> %arg, <4 x i32>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <4 x i32> %arg, <4 x i32>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -136,6 +151,7 @@ define void @test_arg_v2f64(<2 x double> %arg, <2 x double>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntpd
   store <2 x double> %arg, <2 x double>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -145,6 +161,7 @@ define void @test_arg_v2i64(<2 x i64> %arg, <2 x i64>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <2 x i64> %arg, <2 x i64>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -154,6 +171,7 @@ define void @test_arg_v8i16(<8 x i16> %arg, <8 x i16>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <8 x i16> %arg, <8 x i16>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -163,6 +181,7 @@ define void @test_arg_v16i8(<16 x i8> %arg, <16 x i8>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntdq
   store <16 x i8> %arg, <16 x i8>* %dst, align 16, !nontemporal !1
   ret void
 }
@@ -173,6 +192,7 @@ define void @test_arg_v8f32(<8 x float> %arg, <8 x float>* %dst) {
 ; CHECK-LABEL: test_arg_v8f32:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntps %ymm
   store <8 x float> %arg, <8 x float>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -181,6 +201,7 @@ define void @test_arg_v8i32(<8 x i32> %arg, <8 x i32>* %dst) {
 ; CHECK-LABEL: test_arg_v8i32:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <8 x i32> %arg, <8 x i32>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -189,6 +210,7 @@ define void @test_arg_v4f64(<4 x double> %arg, <4 x double>* %dst) {
 ; CHECK-LABEL: test_arg_v4f64:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntpd %ymm
   store <4 x double> %arg, <4 x double>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -197,6 +219,7 @@ define void @test_arg_v4i64(<4 x i64> %arg, <4 x i64>* %dst) {
 ; CHECK-LABEL: test_arg_v4i64:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <4 x i64> %arg, <4 x i64>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -205,6 +228,7 @@ define void @test_arg_v16i16(<16 x i16> %arg, <16 x i16>* %dst) {
 ; CHECK-LABEL: test_arg_v16i16:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <16 x i16> %arg, <16 x i16>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -213,6 +237,7 @@ define void @test_arg_v32i8(<32 x i8> %arg, <32 x i8>* %dst) {
 ; CHECK-LABEL: test_arg_v32i8:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntdq %ymm
   store <32 x i8> %arg, <32 x i8>* %dst, align 32, !nontemporal !1
   ret void
 }
@@ -226,6 +251,7 @@ define void @test_op_v4f32(<4 x float> %a, <4 x float> %b, <4 x float>* %dst) {
 ; SSE: movntps
 ; AVX: vmovntps
 ; AVX2: vmovntps
+; VLX: vmovntps
   %r = fadd <4 x float> %a, %b
   store <4 x float> %r, <4 x float>* %dst, align 16, !nontemporal !1
   ret void
@@ -236,6 +262,7 @@ define void @test_op_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32>* %dst) {
 ; SSE: movntdq
 ; AVX: vmovntdq
 ; AVX2: vmovntdq
+; VLX: vmovntdq
   %r = add <4 x i32> %a, %b
   store <4 x i32> %r, <4 x i32>* %dst, align 16, !nontemporal !1
   ret void
@@ -246,6 +273,7 @@ define void @test_op_v2f64(<2 x double> %a, <2 x double> %b, <2 x double>* %dst)
 ; SSE: movntpd
 ; AVX: vmovntpd
 ; AVX2: vmovntpd
+; VLX: vmovntpd
   %r = fadd <2 x double> %a, %b
   store <2 x double> %r, <2 x double>* %dst, align 16, !nontemporal !1
   ret void
@@ -256,6 +284,7 @@ define void @test_op_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64>* %dst) {
 ; SSE: movntdq
 ; AVX: vmovntdq
 ; AVX2: vmovntdq
+; VLX: vmovntdq
   %r = add <2 x i64> %a, %b
   store <2 x i64> %r, <2 x i64>* %dst, align 16, !nontemporal !1
   ret void
@@ -266,6 +295,7 @@ define void @test_op_v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16>* %dst) {
 ; SSE: movntdq
 ; AVX: vmovntdq
 ; AVX2: vmovntdq
+; VLX: vmovntdq
   %r = add <8 x i16> %a, %b
   store <8 x i16> %r, <8 x i16>* %dst, align 16, !nontemporal !1
   ret void
@@ -276,6 +306,7 @@ define void @test_op_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8>* %dst) {
 ; SSE: movntdq
 ; AVX: vmovntdq
 ; AVX2: vmovntdq
+; VLX: vmovntdq
   %r = add <16 x i8> %a, %b
   store <16 x i8> %r, <16 x i8>* %dst, align 16, !nontemporal !1
   ret void
@@ -287,6 +318,7 @@ define void @test_op_v8f32(<8 x float> %a, <8 x float> %b, <8 x float>* %dst) {
 ; CHECK-LABEL: test_op_v8f32:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntps %ymm
+; VLX: vmovntps %ymm
   %r = fadd <8 x float> %a, %b
   store <8 x float> %r, <8 x float>* %dst, align 32, !nontemporal !1
   ret void
@@ -296,6 +328,7 @@ define void @test_op_v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %dst) {
 ; CHECK-LABEL: test_op_v8i32:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntdq %ymm
+; VLX: vmovntdq %ymm
   %r = add <8 x i32> %a, %b
   store <8 x i32> %r, <8 x i32>* %dst, align 32, !nontemporal !1
   ret void
@@ -305,6 +338,7 @@ define void @test_op_v4f64(<4 x double> %a, <4 x double> %b, <4 x double>* %dst)
 ; CHECK-LABEL: test_op_v4f64:
 ; AVX: vmovntpd %ymm
 ; AVX2: vmovntpd %ymm
+; VLX: vmovntpd %ymm
   %r = fadd <4 x double> %a, %b
   store <4 x double> %r, <4 x double>* %dst, align 32, !nontemporal !1
   ret void
@@ -314,6 +348,7 @@ define void @test_op_v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %dst) {
 ; CHECK-LABEL: test_op_v4i64:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntdq %ymm
+; VLX: vmovntdq %ymm
   %r = add <4 x i64> %a, %b
   store <4 x i64> %r, <4 x i64>* %dst, align 32, !nontemporal !1
   ret void
@@ -323,6 +358,7 @@ define void @test_op_v16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %dst) {
 ; CHECK-LABEL: test_op_v16i16:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntdq %ymm
+; VLX: vmovntdq %ymm
   %r = add <16 x i16> %a, %b
   store <16 x i16> %r, <16 x i16>* %dst, align 32, !nontemporal !1
   ret void
@@ -332,6 +368,7 @@ define void @test_op_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %dst) {
 ; CHECK-LABEL: test_op_v32i8:
 ; AVX: vmovntps %ymm
 ; AVX2: vmovntdq %ymm
+; VLX: vmovntdq %ymm
   %r = add <32 x i8> %a, %b
   store <32 x i8> %r, <32 x i8>* %dst, align 32, !nontemporal !1
   ret void
@@ -349,6 +386,8 @@ define void @test_unaligned_v8f32(<8 x float> %a, <8 x float> %b, <8 x float>* %
 ; AVX: vmovups %ymm
 ; AVX2-NOT: movnt
 ; AVX2: vmovups %ymm
+; VLX-NOT: movnt
+; VLX: vmovups %ymm
   %r = fadd <8 x float> %a, %b
   store <8 x float> %r, <8 x float>* %dst, align 16, !nontemporal !1
   ret void