[ManagedStatic] Reimplement double-checked locking with std::atomic.

This gets rid of the memory fence in the hot path (dereferencing the
ManagedStatic), trading for an extra mutex lock in the cold path (when
the ManagedStatic was uninitialized). Since this only happens on the
first accesses it shouldn't matter much. On strict architectures like
x86 this removes any atomic instructions from the hot path.

Also remove the tsan annotations, tsan knows how standard atomics work
so they should be unnecessary now.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@274131 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/Support/ManagedStatic.h b/include/llvm/Support/ManagedStatic.h
index 3a34bdbd01f9faed67888f013fdd94824d6a098a..ec8154b828e598f6f1e684b167f8227fe8d9e263 100644 (file)
--- a/include/llvm/Support/ManagedStatic.h
+++ b/include/llvm/Support/ManagedStatic.h
@@ -14,9 +14,9 @@
 #ifndef LLVM_SUPPORT_MANAGEDSTATIC_H
 #define LLVM_SUPPORT_MANAGEDSTATIC_H
 
-#include "llvm/Support/Atomic.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/Threading.h"
+#include <atomic>
+#include <cstddef>
 
 namespace llvm {
 
@@ -41,7 +41,7 @@ class ManagedStaticBase {
 protected:
   // This should only be used as a static variable, which guarantees that this
   // will be zero initialized.
-  mutable void *Ptr;
+  mutable std::atomic<void *> Ptr;
   mutable void (*DeleterFn)(void*);
   mutable const ManagedStaticBase *Next;
 
@@ -61,40 +61,26 @@ public:
 template<class C>
 class ManagedStatic : public ManagedStaticBase {
 public:
-
   // Accessors.
   C &operator*() {
-    void* tmp = Ptr;
-    if (llvm_is_multithreaded()) sys::MemoryFence();
-    if (!tmp) RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
-    TsanHappensAfter(this);
+    void *Tmp = Ptr.load(std::memory_order_acquire);
+    if (!Tmp)
+      RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
 
-    return *static_cast<C*>(Ptr);
+    return *static_cast<C *>(Ptr.load(std::memory_order_relaxed));
   }
-  C *operator->() {
-    void* tmp = Ptr;
-    if (llvm_is_multithreaded()) sys::MemoryFence();
-    if (!tmp) RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
-    TsanHappensAfter(this);
 
-    return static_cast<C*>(Ptr);
-  }
+  C *operator->() { return &**this; }
+
   const C &operator*() const {
-    void* tmp = Ptr;
-    if (llvm_is_multithreaded()) sys::MemoryFence();
-    if (!tmp) RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
-    TsanHappensAfter(this);
+    void *Tmp = Ptr.load(std::memory_order_acquire);
+    if (!Tmp)
+      RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
 
-    return *static_cast<C*>(Ptr);
+    return *static_cast<C *>(Ptr.load(std::memory_order_relaxed));
   }
-  const C *operator->() const {
-    void* tmp = Ptr;
-    if (llvm_is_multithreaded()) sys::MemoryFence();
-    if (!tmp) RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
-    TsanHappensAfter(this);
 
-    return static_cast<C*>(Ptr);
-  }
+  const C *operator->() const { return &**this; }
 };
 
 /// llvm_shutdown - Deallocate and destroy all ManagedStatic variables.

diff --git a/lib/Support/ManagedStatic.cpp b/lib/Support/ManagedStatic.cpp
index bc4fe954e22352497f777fa94557a2114be3c3f3..7dd31315f90daaca8c6903f45a16c1a2e635b4a4 100644 (file)
--- a/lib/Support/ManagedStatic.cpp
+++ b/lib/Support/ManagedStatic.cpp
@@ -13,7 +13,6 @@
 
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Config/config.h"
-#include "llvm/Support/Atomic.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/MutexGuard.h"
 #include "llvm/Support/Threading.h"
@@ -42,18 +41,10 @@ void ManagedStaticBase::RegisterManagedStatic(void *(*Creator)(),
   if (llvm_is_multithreaded()) {
     MutexGuard Lock(*getManagedStaticMutex());
 
-    if (!Ptr) {
-      void* tmp = Creator();
+    if (!Ptr.load(std::memory_order_relaxed)) {
+      void *Tmp = Creator();
 
-      TsanHappensBefore(this);
-      sys::MemoryFence();
-
-      // This write is racy against the first read in the ManagedStatic
-      // accessors. The race is benign because it does a second read after a
-      // memory fence, at which point it isn't possible to get a partial value.
-      TsanIgnoreWritesBegin();
-      Ptr = tmp;
-      TsanIgnoreWritesEnd();
+      Ptr.store(Tmp, std::memory_order_release);
       DeleterFn = Deleter;
       
       // Add to list of managed statics.