[SystemZ] Eliminate unnecessary serialization operations

We currently emit a serialization operation (bcr 14, 0) before every
atomic load and after every atomic store.  This is overly conservative.
The SystemZ architecture actually does not require any serialization
for atomic loads, and a serialization after an atomic store only if
we need to enforce sequential consistency.  This is what other compilers
for the platform implement as well.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@310093 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/SystemZ/SystemZISelLowering.cpp b/lib/Target/SystemZ/SystemZISelLowering.cpp
index c0e686e1370a487a3e0e6637d5430da64c8c5047..42d4a8d3185c5b57f9f60ddb9795dd68ad3b5c2a 100644 (file)
--- a/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -3348,28 +3348,28 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
   return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
 }
 
-// Op is an atomic load.  Lower it into a serialization followed
-// by a normal volatile load.
+// Op is an atomic load.  Lower it into a normal volatile load.
 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
                                                 SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
-  SDValue Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
-                                             MVT::Other, Node->getChain()), 0);
   return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
-                        Chain, Node->getBasePtr(),
+                        Node->getChain(), Node->getBasePtr(),
                         Node->getMemoryVT(), Node->getMemOperand());
 }
 
-// Op is an atomic store.  Lower it into a normal volatile store followed
-// by a serialization.
+// Op is an atomic store.  Lower it into a normal volatile store.
 SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
                                                  SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
                                     Node->getBasePtr(), Node->getMemoryVT(),
                                     Node->getMemOperand());
-  return SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op), MVT::Other,
-                                    Chain), 0);
+  // We have to enforce sequential consistency by performing a
+  // serialization operation after the store.
+  if (Node->getOrdering() == AtomicOrdering::SequentiallyConsistent)
+    Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
+                                       MVT::Other, Chain), 0);
+  return Chain;
 }
 
 // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation.  Lower the first

diff --git a/test/CodeGen/SystemZ/atomic-load-01.ll b/test/CodeGen/SystemZ/atomic-load-01.ll
index b2f4ebe6639e47cd99734a37ff754a9d867f1f2c..4e228ac668bd2365233a7d624734d7afd239a9fa 100644 (file)
--- a/test/CodeGen/SystemZ/atomic-load-01.ll
+++ b/test/CodeGen/SystemZ/atomic-load-01.ll
@@ -4,7 +4,6 @@
 
 define i8 @f1(i8 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lb %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i8 , i8 *%src seq_cst, align 1

diff --git a/test/CodeGen/SystemZ/atomic-load-02.ll b/test/CodeGen/SystemZ/atomic-load-02.ll
index b2b60f3d0160536461e282583876701d7f719cd6..44e24d3cca4bad747f369388990537f86f8f593b 100644 (file)
--- a/test/CodeGen/SystemZ/atomic-load-02.ll
+++ b/test/CodeGen/SystemZ/atomic-load-02.ll
@@ -4,7 +4,6 @@
 
 define i16 @f1(i16 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lh %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i16 , i16 *%src seq_cst, align 2

diff --git a/test/CodeGen/SystemZ/atomic-load-03.ll b/test/CodeGen/SystemZ/atomic-load-03.ll
index d83c430bd0af3811d68f2a16d65750ee3bb5a0bd..2f63bd652561f97f2e9ca99f5e7673db260848b6 100644 (file)
--- a/test/CodeGen/SystemZ/atomic-load-03.ll
+++ b/test/CodeGen/SystemZ/atomic-load-03.ll
@@ -4,7 +4,6 @@
 
 define i32 @f1(i32 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: l %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i32 , i32 *%src seq_cst, align 4

diff --git a/test/CodeGen/SystemZ/atomic-load-04.ll b/test/CodeGen/SystemZ/atomic-load-04.ll
index dc6b271e00e5a6528e41df6763d86005c6ded8b1..6dba26390ec37975f6cc3dd4354c41d6cfd7f11e 100644 (file)
--- a/test/CodeGen/SystemZ/atomic-load-04.ll
+++ b/test/CodeGen/SystemZ/atomic-load-04.ll
@@ -4,7 +4,6 @@
 
 define i64 @f1(i64 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lg %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i64 , i64 *%src seq_cst, align 8

diff --git a/test/CodeGen/SystemZ/atomic-store-01.ll b/test/CodeGen/SystemZ/atomic-store-01.ll
index 952e1a912168f23e6f9adaaddc7bdd2db789c068..617557fd1c28334b8f179d84de9bd9f72b13f83e 100644 (file)
--- a/test/CodeGen/SystemZ/atomic-store-01.ll
+++ b/test/CodeGen/SystemZ/atomic-store-01.ll
@@ -10,3 +10,12 @@ define void @f1(i8 %val, i8 *%src) {
   store atomic i8 %val, i8 *%src seq_cst, align 1
   ret void
 }
+
+define void @f2(i8 %val, i8 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: stc %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i8 %val, i8 *%src monotonic, align 1
+  ret void
+}

diff --git a/test/CodeGen/SystemZ/atomic-store-02.ll b/test/CodeGen/SystemZ/atomic-store-02.ll
index c9576e556566f4200f90f372d86e402e030d5a66..f23bac6828931a8e2c3f177cfe81577deeed2ebf 100644 (file)
--- a/test/CodeGen/SystemZ/atomic-store-02.ll
+++ b/test/CodeGen/SystemZ/atomic-store-02.ll
@@ -10,3 +10,12 @@ define void @f1(i16 %val, i16 *%src) {
   store atomic i16 %val, i16 *%src seq_cst, align 2
   ret void
 }
+
+define void @f2(i16 %val, i16 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: sth %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i16 %val, i16 *%src monotonic, align 2
+  ret void
+}

diff --git a/test/CodeGen/SystemZ/atomic-store-03.ll b/test/CodeGen/SystemZ/atomic-store-03.ll
index 459cb6a94e127dabe16c044dcb8b38dbec6367fb..2434bb27e7e4d150b42981b6e54a8164020ee64c 100644 (file)
--- a/test/CodeGen/SystemZ/atomic-store-03.ll
+++ b/test/CodeGen/SystemZ/atomic-store-03.ll
@@ -10,3 +10,12 @@ define void @f1(i32 %val, i32 *%src) {
   store atomic i32 %val, i32 *%src seq_cst, align 4
   ret void
 }
+
+define void @f2(i32 %val, i32 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: st %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i32 %val, i32 *%src monotonic, align 4
+  ret void
+}

diff --git a/test/CodeGen/SystemZ/atomic-store-04.ll b/test/CodeGen/SystemZ/atomic-store-04.ll
index 7f2406eb5468dd2139c9b5548ca14ac37fb369f5..8b04cdd20364f40971e761a705d0b2b4d8793c8e 100644 (file)
--- a/test/CodeGen/SystemZ/atomic-store-04.ll
+++ b/test/CodeGen/SystemZ/atomic-store-04.ll
@@ -10,3 +10,12 @@ define void @f1(i64 %val, i64 *%src) {
   store atomic i64 %val, i64 *%src seq_cst, align 8
   ret void
 }
+
+define void @f2(i64 %val, i64 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: stg %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i64 %val, i64 *%src monotonic, align 8
+  ret void
+}