[X86][FMA] Tag all FMA/FMA4 instructions with WriteFMA schedule class

As mentioned on PR17367, many instructions are missing scheduling tags preventing us from setting 'CompleteModel = 1' for better instruction analysis. This patch deals with FMA/FMA4 which is one of the bigger offenders (along with AVX512 in general).

Annoyingly all scheduler models need to define WriteFMA (now that its actually used), even for older targets without FMA/FMA4 support, but that is an existing problem shared by other schedule classes.

Differential Revision: https://reviews.llvm.org/D40351

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@319016 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/X86/X86InstrAVX512.td b/lib/Target/X86/X86InstrAVX512.td
index 1f2e7197ba7f0a27fc6e74d5dfed41d43a82ba60..c30faed2387091984241fd77167612c01b9bc1da 100644 (file)
--- a/lib/Target/X86/X86InstrAVX512.td
+++ b/lib/Target/X86/X86InstrAVX512.td
@@ -5790,13 +5790,13 @@ multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
           (ins _.RC:$src2, _.RC:$src3),
           OpcodeStr, "$src3, $src2", "$src2, $src3",
           (_.VT (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)), 1, 1>,
-         AVX512FMA3Base;
+          AVX512FMA3Base, Sched<[WriteFMA]>;
 
   defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
           (ins _.RC:$src2, _.MemOp:$src3),
           OpcodeStr, "$src3, $src2", "$src2, $src3",
           (_.VT (OpNode _.RC:$src2, _.RC:$src1, (_.LdFrag addr:$src3))), 1, 0>,
-          AVX512FMA3Base;
+          AVX512FMA3Base, Sched<[WriteFMA, ReadAfterLd]>;
 
   defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
             (ins _.RC:$src2, _.ScalarMemOp:$src3),
@@ -5804,7 +5804,7 @@ multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
             !strconcat("$src2, ${src3}", _.BroadcastStr ),
             (OpNode _.RC:$src2,
              _.RC:$src1,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))), 1, 0>,
-            AVX512FMA3Base, EVEX_B;
+            AVX512FMA3Base, EVEX_B, Sched<[WriteFMA, ReadAfterLd]>;
   }
 }
 
@@ -5815,7 +5815,7 @@ multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
           (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
           OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
           (_.VT ( OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i32 imm:$rc))), 1, 1>,
-          AVX512FMA3Base, EVEX_B, EVEX_RC;
+          AVX512FMA3Base, EVEX_B, EVEX_RC, Sched<[WriteFMA]>;
 }
 
 multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
@@ -5857,13 +5857,13 @@ multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
           (ins _.RC:$src2, _.RC:$src3),
           OpcodeStr, "$src3, $src2", "$src2, $src3",
           (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), 1, 1, vselect, 1>,
-         AVX512FMA3Base;
+         AVX512FMA3Base, Sched<[WriteFMA]>;
 
   defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
           (ins _.RC:$src2, _.MemOp:$src3),
           OpcodeStr, "$src3, $src2", "$src2, $src3",
           (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1)), 1, 0>,
-         AVX512FMA3Base;
+         AVX512FMA3Base, Sched<[WriteFMA, ReadAfterLd]>;
 
   defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
          (ins _.RC:$src2, _.ScalarMemOp:$src3),
@@ -5871,7 +5871,8 @@ multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
          "$src2, ${src3}"##_.BroadcastStr,
          (_.VT (OpNode _.RC:$src2,
                       (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
-                      _.RC:$src1)), 1, 0>, AVX512FMA3Base, EVEX_B;
+                      _.RC:$src1)), 1, 0>, AVX512FMA3Base, EVEX_B,
+         Sched<[WriteFMA, ReadAfterLd]>;
   }
 }
 
@@ -5883,7 +5884,7 @@ multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
           OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
           (_.VT ( OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 imm:$rc))), 1,
           1, vselect, 1>,
-          AVX512FMA3Base, EVEX_B, EVEX_RC;
+          AVX512FMA3Base, EVEX_B, EVEX_RC, Sched<[WriteFMA]>;
 }
 
 multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
@@ -5924,7 +5925,7 @@ multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
           (ins _.RC:$src2, _.RC:$src3),
           OpcodeStr, "$src3, $src2", "$src2, $src3",
           (_.VT (OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2)), 1, 1, vselect, 1>,
-         AVX512FMA3Base;
+         AVX512FMA3Base, Sched<[WriteFMA]>;
 
   // Pattern is 312 order so that the load is in a different place from the
   // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
@@ -5932,7 +5933,7 @@ multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
           (ins _.RC:$src2, _.MemOp:$src3),
           OpcodeStr, "$src3, $src2", "$src2, $src3",
           (_.VT (OpNode (_.LdFrag addr:$src3), _.RC:$src1, _.RC:$src2)), 1, 0>,
-         AVX512FMA3Base;
+         AVX512FMA3Base, Sched<[WriteFMA, ReadAfterLd]>;
 
   // Pattern is 312 order so that the load is in a different place from the
   // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
@@ -5941,7 +5942,8 @@ multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
          OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
          "$src2, ${src3}"##_.BroadcastStr,
          (_.VT (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
-                       _.RC:$src1, _.RC:$src2)), 1, 0>, AVX512FMA3Base, EVEX_B;
+                       _.RC:$src1, _.RC:$src2)), 1, 0>, AVX512FMA3Base, EVEX_B,
+         Sched<[WriteFMA, ReadAfterLd]>;
   }
 }
 
@@ -5953,7 +5955,7 @@ multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
           OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
           (_.VT ( OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2, (i32 imm:$rc))), 1,
           1, vselect, 1>,
-          AVX512FMA3Base, EVEX_B, EVEX_RC;
+          AVX512FMA3Base, EVEX_B, EVEX_RC, Sched<[WriteFMA]>;
 }
 
 multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
@@ -5994,28 +5996,30 @@ multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
 let Constraints = "$src1 = $dst", hasSideEffects = 0 in {
   defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
           (ins _.RC:$src2, _.RC:$src3), OpcodeStr,
-          "$src3, $src2", "$src2, $src3", RHS_VEC_r, 1, 1>, AVX512FMA3Base;
+          "$src3, $src2", "$src2, $src3", RHS_VEC_r, 1, 1>, AVX512FMA3Base,
+          Sched<[WriteFMA]>;
 
   defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
           (ins _.RC:$src2, _.IntScalarMemOp:$src3), OpcodeStr,
-          "$src3, $src2", "$src2, $src3", RHS_VEC_m, 1, 1>, AVX512FMA3Base;
+          "$src3, $src2", "$src2, $src3", RHS_VEC_m, 1, 1>, AVX512FMA3Base,
+          Sched<[WriteFMA, ReadAfterLd]>;
 
   defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
          (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
          OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", RHS_VEC_rb, 1, 1>,
-         AVX512FMA3Base, EVEX_B, EVEX_RC;
+         AVX512FMA3Base, EVEX_B, EVEX_RC, Sched<[WriteFMA, ReadAfterLd]>;
 
   let isCodeGenOnly = 1, isCommutable = 1 in {
     def r     : AVX512FMA3S<opc, MRMSrcReg, (outs _.FRC:$dst),
                      (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3),
                      !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                     !if(MaskOnlyReg, [], [RHS_r])>;
+                     !if(MaskOnlyReg, [], [RHS_r])>, Sched<[WriteFMA]>;
     def m     : AVX512FMA3S<opc, MRMSrcMem, (outs _.FRC:$dst),
                     (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3),
                     !strconcat(OpcodeStr,
                                "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                    [RHS_m]>;
+                    [RHS_m]>, Sched<[WriteFMA, ReadAfterLd]>;
   }// isCodeGenOnly = 1
 }// Constraints = "$src1 = $dst"
 }

diff --git a/lib/Target/X86/X86InstrFMA.td b/lib/Target/X86/X86InstrFMA.td
index dd6a61ddc3b260d786b9f42140cb3547c7527bcb..1b706674a4d07f8285e7f4a55699ebc3f6f9027e 100644 (file)
--- a/lib/Target/X86/X86InstrFMA.td
+++ b/lib/Target/X86/X86InstrFMA.td
@@ -41,7 +41,8 @@ multiclass fma3p_rm_213<bits<8> opc, string OpcodeStr, RegisterClass RC,
                    (ins RC:$src1, RC:$src2, RC:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set RC:$dst, (VT (Op RC:$src2, RC:$src1, RC:$src3)))]>;
+                   [(set RC:$dst, (VT (Op RC:$src2, RC:$src1, RC:$src3)))]>,
+                   Sched<[WriteFMA]>;
 
   let mayLoad = 1 in
   def m     : FMA3<opc, MRMSrcMem, (outs RC:$dst),
@@ -49,7 +50,8 @@ multiclass fma3p_rm_213<bits<8> opc, string OpcodeStr, RegisterClass RC,
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    [(set RC:$dst, (VT (Op RC:$src2, RC:$src1,
-                                          (MemFrag addr:$src3))))]>;
+                                          (MemFrag addr:$src3))))]>,
+                   Sched<[WriteFMA, ReadAfterLd]>;
 }
 
 multiclass fma3p_rm_231<bits<8> opc, string OpcodeStr, RegisterClass RC,
@@ -60,7 +62,7 @@ multiclass fma3p_rm_231<bits<8> opc, string OpcodeStr, RegisterClass RC,
                    (ins RC:$src1, RC:$src2, RC:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   []>;
+                   []>, Sched<[WriteFMA]>;
 
   let mayLoad = 1 in
   def m     : FMA3<opc, MRMSrcMem, (outs RC:$dst),
@@ -68,7 +70,7 @@ multiclass fma3p_rm_231<bits<8> opc, string OpcodeStr, RegisterClass RC,
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    [(set RC:$dst, (VT (Op RC:$src2, (MemFrag addr:$src3),
-                                          RC:$src1)))]>;
+                                          RC:$src1)))]>, Sched<[WriteFMA, ReadAfterLd]>;
 }
 
 multiclass fma3p_rm_132<bits<8> opc, string OpcodeStr, RegisterClass RC,
@@ -79,7 +81,7 @@ multiclass fma3p_rm_132<bits<8> opc, string OpcodeStr, RegisterClass RC,
                    (ins RC:$src1, RC:$src2, RC:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   []>;
+                   []>, Sched<[WriteFMA]>;
 
   // Pattern is 312 order so that the load is in a different place from the
   // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
@@ -89,7 +91,7 @@ multiclass fma3p_rm_132<bits<8> opc, string OpcodeStr, RegisterClass RC,
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    [(set RC:$dst, (VT (Op (MemFrag addr:$src3), RC:$src1,
-                                          RC:$src2)))]>;
+                                          RC:$src2)))]>, Sched<[WriteFMA, ReadAfterLd]>;
 }
 
 let Constraints = "$src1 = $dst", hasSideEffects = 0, isCommutable = 1 in
@@ -172,7 +174,8 @@ multiclass fma3s_rm_213<bits<8> opc, string OpcodeStr,
                 (ins RC:$src1, RC:$src2, RC:$src3),
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                [(set RC:$dst, (OpNode RC:$src2, RC:$src1, RC:$src3))]>;
+                [(set RC:$dst, (OpNode RC:$src2, RC:$src1, RC:$src3))]>,
+                Sched<[WriteFMA]>;
 
   let mayLoad = 1 in
   def m : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
@@ -180,7 +183,8 @@ multiclass fma3s_rm_213<bits<8> opc, string OpcodeStr,
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                 [(set RC:$dst,
-                  (OpNode RC:$src2, RC:$src1, (load addr:$src3)))]>;
+                  (OpNode RC:$src2, RC:$src1, (load addr:$src3)))]>,
+                Sched<[WriteFMA, ReadAfterLd]>;
 }
 
 multiclass fma3s_rm_231<bits<8> opc, string OpcodeStr,
@@ -191,7 +195,7 @@ multiclass fma3s_rm_231<bits<8> opc, string OpcodeStr,
                 (ins RC:$src1, RC:$src2, RC:$src3),
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                []>;
+                []>, Sched<[WriteFMA]>;
 
   let mayLoad = 1 in
   def m : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
@@ -199,7 +203,8 @@ multiclass fma3s_rm_231<bits<8> opc, string OpcodeStr,
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                 [(set RC:$dst,
-                  (OpNode RC:$src2, (load addr:$src3), RC:$src1))]>;
+                  (OpNode RC:$src2, (load addr:$src3), RC:$src1))]>,
+                Sched<[WriteFMA, ReadAfterLd]>;
 }
 
 multiclass fma3s_rm_132<bits<8> opc, string OpcodeStr,
@@ -210,7 +215,7 @@ multiclass fma3s_rm_132<bits<8> opc, string OpcodeStr,
                 (ins RC:$src1, RC:$src2, RC:$src3),
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                []>;
+                []>, Sched<[WriteFMA]>;
 
   // Pattern is 312 order so that the load is in a different place from the
   // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
@@ -220,7 +225,8 @@ multiclass fma3s_rm_132<bits<8> opc, string OpcodeStr,
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                 [(set RC:$dst,
-                  (OpNode (load addr:$src3), RC:$src1, RC:$src2))]>;
+                  (OpNode (load addr:$src3), RC:$src1, RC:$src2))]>,
+                Sched<[WriteFMA, ReadAfterLd]>;
 }
 
 let Constraints = "$src1 = $dst", isCommutable = 1, hasSideEffects = 0 in
@@ -257,14 +263,14 @@ multiclass fma3s_rm_int<bits<8> opc, string OpcodeStr,
                         (ins RC:$src1, RC:$src2, RC:$src3),
                         !strconcat(OpcodeStr,
                                    "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                        []>;
+                        []>, Sched<[WriteFMA]>;
 
   let mayLoad = 1 in
   def m_Int : FMA3S_Int<opc, MRMSrcMem, (outs RC:$dst),
                         (ins RC:$src1, RC:$src2, memopr:$src3),
                         !strconcat(OpcodeStr,
                                    "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                        []>;
+                        []>, Sched<[WriteFMA, ReadAfterLd]>;
 }
 
 // The FMA 213 form is created for lowering of scalar FMA intrinscis
@@ -360,26 +366,29 @@ multiclass fma4s<bits<8> opc, string OpcodeStr, RegisterClass RC,
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set RC:$dst,
-             (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>, VEX_W, VEX_LIG;
+             (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>, VEX_W, VEX_LIG,
+           Sched<[WriteFMA]>;
   def rm : FMA4S<opc, MRMSrcMemOp4, (outs RC:$dst),
            (ins RC:$src1, RC:$src2, x86memop:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set RC:$dst, (OpNode RC:$src1, RC:$src2,
-                           (mem_frag addr:$src3)))]>, VEX_W, VEX_LIG;
+                           (mem_frag addr:$src3)))]>, VEX_W, VEX_LIG,
+           Sched<[WriteFMA, ReadAfterLd]>;
   def mr : FMA4S<opc, MRMSrcMem, (outs RC:$dst),
            (ins RC:$src1, x86memop:$src2, RC:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set RC:$dst,
-             (OpNode RC:$src1, (mem_frag addr:$src2), RC:$src3))]>, VEX_LIG;
+             (OpNode RC:$src1, (mem_frag addr:$src2), RC:$src3))]>, VEX_LIG,
+           Sched<[WriteFMA, ReadAfterLd]>;
 // For disassembler
 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in
   def rr_REV : FMA4S<opc, MRMSrcReg, (outs RC:$dst),
                (ins RC:$src1, RC:$src2, RC:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>,
-               VEX_LIG, FoldGenData<NAME#rr>;
+               VEX_LIG, FoldGenData<NAME#rr>, Sched<[WriteFMA]>;
 }
 
 multiclass fma4s_int<bits<8> opc, string OpcodeStr, Operand memop,
@@ -391,26 +400,27 @@ let isCodeGenOnly = 1 in {
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                [(set VR128:$dst,
                  (VT (OpNode VR128:$src1, VR128:$src2, VR128:$src3)))]>, VEX_W,
-               VEX_LIG;
+               VEX_LIG, Sched<[WriteFMA]>;
   def rm_Int : FMA4S_Int<opc, MRMSrcMemOp4, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, memop:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                [(set VR128:$dst, (VT (OpNode VR128:$src1, VR128:$src2,
-                                  mem_cpat:$src3)))]>, VEX_W, VEX_LIG;
+                                  mem_cpat:$src3)))]>, VEX_W, VEX_LIG,
+               Sched<[WriteFMA, ReadAfterLd]>;
   def mr_Int : FMA4S_Int<opc, MRMSrcMem, (outs VR128:$dst),
                (ins VR128:$src1, memop:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                [(set VR128:$dst,
                  (VT (OpNode VR128:$src1, mem_cpat:$src2, VR128:$src3)))]>,
-               VEX_LIG;
+               VEX_LIG, Sched<[WriteFMA, ReadAfterLd]>;
 let hasSideEffects = 0 in
   def rr_Int_REV : FMA4S_Int<opc, MRMSrcReg, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-               []>, VEX_LIG, FoldGenData<NAME#rr_Int>; 
+               []>, VEX_LIG, FoldGenData<NAME#rr_Int>, Sched<[WriteFMA]>;
 } // isCodeGenOnly = 1
 }
 
@@ -424,19 +434,21 @@ multiclass fma4p<bits<8> opc, string OpcodeStr, SDNode OpNode,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR128:$dst,
              (OpVT128 (OpNode VR128:$src1, VR128:$src2, VR128:$src3)))]>,
-           VEX_W;
+           VEX_W, Sched<[WriteFMA]>;
   def rm : FMA4<opc, MRMSrcMemOp4, (outs VR128:$dst),
            (ins VR128:$src1, VR128:$src2, f128mem:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR128:$dst, (OpNode VR128:$src1, VR128:$src2,
-                              (ld_frag128 addr:$src3)))]>, VEX_W;
+                              (ld_frag128 addr:$src3)))]>, VEX_W,
+           Sched<[WriteFMA, ReadAfterLd]>;
   def mr : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
            (ins VR128:$src1, f128mem:$src2, VR128:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR128:$dst,
-             (OpNode VR128:$src1, (ld_frag128 addr:$src2), VR128:$src3))]>;
+             (OpNode VR128:$src1, (ld_frag128 addr:$src2), VR128:$src3))]>,
+           Sched<[WriteFMA, ReadAfterLd]>;
   let isCommutable = 1 in
   def Yrr : FMA4<opc, MRMSrcRegOp4, (outs VR256:$dst),
            (ins VR256:$src1, VR256:$src2, VR256:$src3),
@@ -444,31 +456,33 @@ multiclass fma4p<bits<8> opc, string OpcodeStr, SDNode OpNode,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR256:$dst,
              (OpVT256 (OpNode VR256:$src1, VR256:$src2, VR256:$src3)))]>,
-           VEX_W, VEX_L;
+           VEX_W, VEX_L, Sched<[WriteFMA]>;
   def Yrm : FMA4<opc, MRMSrcMemOp4, (outs VR256:$dst),
            (ins VR256:$src1, VR256:$src2, f256mem:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR256:$dst, (OpNode VR256:$src1, VR256:$src2,
-                              (ld_frag256 addr:$src3)))]>, VEX_W, VEX_L;
+                              (ld_frag256 addr:$src3)))]>, VEX_W, VEX_L,
+           Sched<[WriteFMA, ReadAfterLd]>;
   def Ymr : FMA4<opc, MRMSrcMem, (outs VR256:$dst),
            (ins VR256:$src1, f256mem:$src2, VR256:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR256:$dst, (OpNode VR256:$src1,
-                              (ld_frag256 addr:$src2), VR256:$src3))]>, VEX_L;
+                              (ld_frag256 addr:$src2), VR256:$src3))]>, VEX_L,
+           Sched<[WriteFMA, ReadAfterLd]>;
 // For disassembler
 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in {
   def rr_REV : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>,
-               FoldGenData<NAME#rr>;
+               Sched<[WriteFMA]>, FoldGenData<NAME#rr>;
   def Yrr_REV : FMA4<opc, MRMSrcReg, (outs VR256:$dst),
                 (ins VR256:$src1, VR256:$src2, VR256:$src3),
                 !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>,
-                VEX_L, FoldGenData<NAME#Yrr>;
+                VEX_L, Sched<[WriteFMA]>, FoldGenData<NAME#Yrr>;
 } // isCodeGenOnly = 1
 }
 

diff --git a/lib/Target/X86/X86SchedBroadwell.td b/lib/Target/X86/X86SchedBroadwell.td
index c70af22d060cc3b8ef8f4f4d8361bb55e94ddeb5..7fef01c72aafe7f7e5ed54283b4fab13b20ed210 100755 (executable)
--- a/lib/Target/X86/X86SchedBroadwell.td
+++ b/lib/Target/X86/X86SchedBroadwell.td
@@ -131,7 +131,7 @@ defm : BWWriteResPair<WriteFDiv,   BWPort0, 12>; // 10-14 cycles. // Floating po
 defm : BWWriteResPair<WriteFSqrt,  BWPort0, 15>; // Floating point square root.
 defm : BWWriteResPair<WriteFRcp,   BWPort0, 5>; // Floating point reciprocal estimate.
 defm : BWWriteResPair<WriteFRsqrt, BWPort0, 5>; // Floating point reciprocal square root estimate.
-// defm WriteFMA    : X86SchedWritePair; // Fused Multiply Add.
+defm : BWWriteResPair<WriteFMA,    BWPort01, 5>; // Fused Multiply Add.
 defm : BWWriteResPair<WriteFShuffle,  BWPort5,  1>; // Floating point vector shuffles.
 defm : BWWriteResPair<WriteFBlend,  BWPort015,  1>; // Floating point vector blends.
 def : WriteRes<WriteFVarBlend, [BWPort5]> { // Fp vector variable blends.             

diff --git a/lib/Target/X86/X86SchedHaswell.td b/lib/Target/X86/X86SchedHaswell.td
index c2b188b8846a54a8c2a68a95b6ba1c8be78d6fd5..5b9223432df237c2761e3c11d2d7f7ad8096956a 100644 (file)
--- a/lib/Target/X86/X86SchedHaswell.td
+++ b/lib/Target/X86/X86SchedHaswell.td
@@ -134,6 +134,7 @@ defm : HWWriteResPair<WriteFSqrt,  HWPort0, 15>;
 defm : HWWriteResPair<WriteCvtF2I, HWPort1, 3>;
 defm : HWWriteResPair<WriteCvtI2F, HWPort1, 4>;
 defm : HWWriteResPair<WriteCvtF2F, HWPort1, 3>;
+defm : HWWriteResPair<WriteFMA,   HWPort01, 5>;
 defm : HWWriteResPair<WriteFShuffle,  HWPort5,  1>;
 defm : HWWriteResPair<WriteFBlend,  HWPort015,  1>;
 defm : HWWriteResPair<WriteFShuffle256,  HWPort5,  3>;

diff --git a/lib/Target/X86/X86SchedSandyBridge.td b/lib/Target/X86/X86SchedSandyBridge.td
index c86c48ce06b81cdf9135f17bf1652eae9a6bdf44..c6c60bf03b2e9ab771c2120083155dca3585f76a 100644 (file)
--- a/lib/Target/X86/X86SchedSandyBridge.td
+++ b/lib/Target/X86/X86SchedSandyBridge.td
@@ -276,11 +276,12 @@ def : WriteRes<WriteMicrocoded, [SBPort015]> { let Latency = 100; }
 def : WriteRes<WriteFence, [SBPort23, SBPort4]>;
 def : WriteRes<WriteNop, []>;
 
-// AVX2 is not supported on that architecture, but we should define the basic
+// AVX2/FMA is not supported on that architecture, but we should define the basic
 // scheduling resources anyway.
 defm : SBWriteResPair<WriteFShuffle256, SBPort0,  1>;
 defm : SBWriteResPair<WriteShuffle256, SBPort0,  1>;
 defm : SBWriteResPair<WriteVarVecShift, SBPort0,  1>;
+defm : SBWriteResPair<WriteFMA, SBPort01,  5>;
 
 // Remaining SNB instrs.
 

diff --git a/lib/Target/X86/X86SchedSkylakeClient.td b/lib/Target/X86/X86SchedSkylakeClient.td
index aabb45be87c15cf200a191cc313d7874b58ebcf4..eeeffdf70083d8a47ffaf5760aca652dafd1e5b3 100644 (file)
--- a/lib/Target/X86/X86SchedSkylakeClient.td
+++ b/lib/Target/X86/X86SchedSkylakeClient.td
@@ -132,7 +132,7 @@ defm : SKLWriteResPair<WriteFDiv,   SKLPort0, 12>; // 10-14 cycles. // Floating
 defm : SKLWriteResPair<WriteFSqrt,  SKLPort0, 15>; // Floating point square root.
 defm : SKLWriteResPair<WriteFRcp,   SKLPort0, 5>; // Floating point reciprocal estimate.
 defm : SKLWriteResPair<WriteFRsqrt, SKLPort0, 5>; // Floating point reciprocal square root estimate.
-// defm WriteFMA    : X86SchedWritePair; // Fused Multiply Add.
+defm : SKLWriteResPair<WriteFMA,    SKLPort01, 4>; // Fused Multiply Add.
 defm : SKLWriteResPair<WriteFShuffle,  SKLPort5,  1>; // Floating point vector shuffles.
 defm : SKLWriteResPair<WriteFBlend,  SKLPort015,  1>; // Floating point vector blends.
 def : WriteRes<WriteFVarBlend, [SKLPort5]> { // Fp vector variable blends.            

diff --git a/lib/Target/X86/X86SchedSkylakeServer.td b/lib/Target/X86/X86SchedSkylakeServer.td
index 50f6379bad8434c4ff35b34ac6fb08f65127040d..8ba1ac027ce25f969e767082695d43004a22741a 100755 (executable)
--- a/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/lib/Target/X86/X86SchedSkylakeServer.td
@@ -132,7 +132,7 @@ defm : SKXWriteResPair<WriteFDiv,   SKXPort0, 12>; // 10-14 cycles. // Floating
 defm : SKXWriteResPair<WriteFSqrt,  SKXPort0, 15>; // Floating point square root.
 defm : SKXWriteResPair<WriteFRcp,   SKXPort0, 5>; // Floating point reciprocal estimate.
 defm : SKXWriteResPair<WriteFRsqrt, SKXPort0, 5>; // Floating point reciprocal square root estimate.
-// defm WriteFMA    : X86SchedWritePair; // Fused Multiply Add.
+defm : SKXWriteResPair<WriteFMA,  SKXPort015, 4>; // Fused Multiply Add.
 defm : SKXWriteResPair<WriteFShuffle,  SKXPort5,  1>; // Floating point vector shuffles.
 defm : SKXWriteResPair<WriteFBlend,  SKXPort015,  1>; // Floating point vector blends.
 def : WriteRes<WriteFVarBlend, [SKXPort5]> { // Fp vector variable blends.            

diff --git a/lib/Target/X86/X86ScheduleBtVer2.td b/lib/Target/X86/X86ScheduleBtVer2.td
index edfb58059e1c7757d0c67163fb673116351aeb52..a2f02962444cf6b0e5ae1e1593f6e35e5736dc55 100644 (file)
--- a/lib/Target/X86/X86ScheduleBtVer2.td
+++ b/lib/Target/X86/X86ScheduleBtVer2.td
@@ -192,6 +192,7 @@ defm : JWriteResIntPair<WriteJump,  JALU01, 1>;
 
 defm : JWriteResFpuPair<WriteFAdd,        JFPU0,  3>;
 defm : JWriteResFpuPair<WriteFMul,        JFPU1,  2>;
+defm : JWriteResFpuPair<WriteFMA,         JFPU1,  2>; // NOTE: Doesn't exist on Jaguar.
 defm : JWriteResFpuPair<WriteFRcp,        JFPU1,  2>;
 defm : JWriteResFpuPair<WriteFRsqrt,      JFPU1,  2>;
 defm : JWriteResFpuPair<WriteFShuffle,   JFPU01,  1>;

diff --git a/lib/Target/X86/X86ScheduleSLM.td b/lib/Target/X86/X86ScheduleSLM.td
index 03ed2db2350ddc5c99ef0376f89abd0b862fc22c..6a2a998b5ff3cf4dece6b709e7e151f1c0fce623 100644 (file)
--- a/lib/Target/X86/X86ScheduleSLM.td
+++ b/lib/Target/X86/X86ScheduleSLM.td
@@ -249,7 +249,7 @@ def : WriteRes<WriteMicrocoded, [FPC_RSV0]> { let Latency = 100; }
 def : WriteRes<WriteFence, [MEC_RSV]>;
 def : WriteRes<WriteNop, []>;
 
-// AVX is not supported on that architecture, but we should define the basic
+// AVX/FMA is not supported on that architecture, but we should define the basic
 // scheduling resources anyway.
 def  : WriteRes<WriteIMulH, [FPC_RSV0]>;
 defm : SMWriteResPair<WriteVarBlend, FPC_RSV0, 1>;
@@ -257,4 +257,5 @@ defm : SMWriteResPair<WriteFVarBlend, FPC_RSV0, 1>;
 defm : SMWriteResPair<WriteFShuffle256, FPC_RSV0,  1>;
 defm : SMWriteResPair<WriteShuffle256, FPC_RSV0,  1>;
 defm : SMWriteResPair<WriteVarVecShift, FPC_RSV0,  1>;
+defm : SMWriteResPair<WriteFMA, FPC_RSV0,  1>;
 } // SchedModel

diff --git a/lib/Target/X86/X86ScheduleZnver1.td b/lib/Target/X86/X86ScheduleZnver1.td
index c72880b7f65913a47ba81cf6f05b61eb2183ce40..5ebe8a28422ea913b2b1721363094f4a6c99fe38 100644 (file)
--- a/lib/Target/X86/X86ScheduleZnver1.td
+++ b/lib/Target/X86/X86ScheduleZnver1.td
@@ -175,6 +175,7 @@ defm : ZnWriteResFpuPair<WriteCvtF2I,    ZnFPU3,  5>;
 defm : ZnWriteResFpuPair<WriteFDiv,      ZnFPU3, 15>;
 defm : ZnWriteResFpuPair<WriteFShuffle,  ZnFPU12, 1>;
 defm : ZnWriteResFpuPair<WriteFMul,      ZnFPU0,  5>;
+defm : ZnWriteResFpuPair<WriteFMA,       ZnFPU03, 5>;
 defm : ZnWriteResFpuPair<WriteFRcp,      ZnFPU01, 5>;
 defm : ZnWriteResFpuPair<WriteFRsqrt,    ZnFPU01, 5>;
 defm : ZnWriteResFpuPair<WriteFSqrt,     ZnFPU3, 20>;