Use __builtin_expect in atomic_ops_malloc

* src/atomic_ops_malloc.c [HAVE_MMAP] (get_mmaped, SIZET_SAT_ADD,
AO_malloc_large): Use AO_EXPECT_FALSE.
* src/atomic_ops_malloc.c (get_chunk, msb, AO_malloc, AO_free):
Likewise.
* src/atomic_ops_malloc.c (AO_malloc_large): Reformat code.

diff --git a/src/atomic_ops_malloc.c b/src/atomic_ops_malloc.c
index 8b0f9634fb871e725a8419a70be919c914b5b1ab..f78ff4054f26fcb65008376b7bd1a2602029041b 100644 (file)
--- a/src/atomic_ops_malloc.c
+++ b/src/atomic_ops_malloc.c
@@ -133,8 +133,8 @@ static char *get_mmaped(size_t sz)
 # ifndef USE_MMAP_ANON
     close(zero_fd);
 # endif
-  if (result == MAP_FAILED)
-    result = 0;
+  if (AO_EXPECT_FALSE(result == MAP_FAILED))
+    result = NULL;
   return result;
 }
 
@@ -151,23 +151,24 @@ static char *get_mmaped(size_t sz)
 /* Saturated addition of size_t values.  Used to avoid value wrap       */
 /* around on overflow.  The arguments should have no side effects.      */
 #define SIZET_SAT_ADD(a, b) \
-                ((a) < AO_SIZE_MAX - (b) ? (a) + (b) : AO_SIZE_MAX)
+    (AO_EXPECT_FALSE((a) >= AO_SIZE_MAX - (b)) ? AO_SIZE_MAX : (a) + (b))
 
 /* Allocate an object of size (incl. header) of size > CHUNK_SIZE.      */
 /* sz includes space for an AO_t-sized header.                          */
 static char *
 AO_malloc_large(size_t sz)
 {
- char * result;
- /* The header will force us to waste ALIGNMENT bytes, incl. header.    */
- /* Round to multiple of CHUNK_SIZE.                                    */
- sz = SIZET_SAT_ADD(sz, ALIGNMENT + CHUNK_SIZE - 1) & ~(CHUNK_SIZE - 1);
- assert(sz > LOG_MAX_SIZE);
- result = get_mmaped(sz);
- if (result == 0) return 0;
- result += ALIGNMENT;
- ((AO_t *)result)[-1] = (AO_t)sz;
- return result;
+  char *result;
+  /* The header will force us to waste ALIGNMENT bytes, incl. header.   */
+  /* Round to multiple of CHUNK_SIZE.                                   */
+  sz = SIZET_SAT_ADD(sz, ALIGNMENT + CHUNK_SIZE - 1) & ~(CHUNK_SIZE - 1);
+  assert(sz > LOG_MAX_SIZE);
+  result = get_mmaped(sz);
+  if (AO_EXPECT_FALSE(NULL == result))
+    return NULL;
+  result += ALIGNMENT;
+  ((AO_t *)result)[-1] = (AO_t)sz;
+  return result;
 }
 
 static void
@@ -210,7 +211,8 @@ get_chunk(void)
                                     (AO_t)initial_ptr, (AO_t)my_chunk_ptr);
       }
 
-    if (my_chunk_ptr - AO_initial_heap > AO_INITIAL_HEAP_SIZE - CHUNK_SIZE)
+    if (AO_EXPECT_FALSE(my_chunk_ptr - AO_initial_heap
+                        > AO_INITIAL_HEAP_SIZE - CHUNK_SIZE))
       break;
     if (AO_compare_and_swap(&initial_heap_ptr, (AO_t)my_chunk_ptr,
                             (AO_t)(my_chunk_ptr + CHUNK_SIZE))) {
@@ -256,7 +258,7 @@ static unsigned msb(size_t s)
   if ((s & 0xff) != s) {
 #   if (__SIZEOF_SIZE_T__ == 8) && !defined(CPPCHECK)
       unsigned v = (unsigned)(s >> 32);
-      if (v != 0)
+      if (AO_EXPECT_FALSE(v != 0))
         {
           s = v;
           result += 32;
@@ -275,7 +277,7 @@ static unsigned msb(size_t s)
           result += 32;
         }
 #   endif /* !defined(__SIZEOF_SIZE_T__) */
-    if ((s >> 16) != 0)
+    if (AO_EXPECT_FALSE((s >> 16) != 0))
       {
         s >>= 16;
         result += 16;
@@ -301,15 +303,17 @@ AO_malloc(size_t sz)
   AO_t *result;
   unsigned log_sz;
 
-  if (sz > CHUNK_SIZE - sizeof(AO_t))
+  if (AO_EXPECT_FALSE(sz > CHUNK_SIZE - sizeof(AO_t)))
     return AO_malloc_large(sz);
   log_sz = msb(sz + (sizeof(AO_t) - 1));
   assert(log_sz <= LOG_MAX_SIZE);
   assert(((size_t)1 << log_sz) >= sz + sizeof(AO_t));
   result = AO_stack_pop(AO_free_list+log_sz);
-  while (0 == result) {
+  while (AO_EXPECT_FALSE(NULL == result)) {
     void * chunk = get_chunk();
-    if (0 == chunk) return 0;
+
+    if (AO_EXPECT_FALSE(NULL == chunk))
+      return NULL;
     add_chunk_as(chunk, log_sz);
     result = AO_stack_pop(AO_free_list+log_sz);
   }
@@ -331,7 +335,8 @@ AO_free(void *p)
   AO_t *base;
   int log_sz;
 
-  if (0 == p) return;
+  if (AO_EXPECT_FALSE(NULL == p))
+    return;
 
   base = (AO_t *)p - 1;
 # ifdef AO_THREAD_SANITIZER
@@ -343,8 +348,9 @@ AO_free(void *p)
     fprintf(stderr, "%p: AO_free(%p sz:%lu)\n", (void *)pthread_self(), p,
             log_sz > LOG_MAX_SIZE ? (unsigned)log_sz : 1UL << log_sz);
 # endif
-  if (log_sz > LOG_MAX_SIZE)
+  if (AO_EXPECT_FALSE(log_sz > LOG_MAX_SIZE)) {
     AO_free_large(p);
-  else
+  } else {
     AO_stack_push(AO_free_list + log_sz, base);
+  }
 }