Add AO_load primitives generalization based on CAS

* src/atomic_ops/generalize-small.template (AO_XSIZE_load_acquire,
AO_XSIZE_load, AO_XSIZE_load_read, AO_XSIZE_load_acquire_read,
AO_XSIZE_load_full, AO_XSIZE_load_dd_acquire_read,
AO_XSIZE_store_release, AO_XSIZE_store, AO_XSIZE_store_write,
AO_XSIZE_store_release_write, AO_XSIZE_store_full): Move primitive
down to be after CAS defintion section.
* src/atomic_ops/generalize-small.template (AO_XSIZE_load_read,
AO_XSIZE_load_full, AO_XSIZE_load_acquire, AO_XSIZE_load): Add
generalization template primitive based on CAS.
* src/atomic_ops/generalize-small.h: Regenerate.

diff --git a/src/atomic_ops/generalize-small.h b/src/atomic_ops/generalize-small.h
index 30339de1b5283460c0bd028e29e9fac2001c73db..2bdac014d94b8f38f146c858e7e85e87b4fbd3a5 100644 (file)
--- a/src/atomic_ops/generalize-small.h
+++ b/src/atomic_ops/generalize-small.h
@@ -20,142 +20,6 @@
  * SOFTWARE.
  */
 
-/* char_load */
-#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_acquire)
-# define AO_char_load_acquire(addr) AO_char_load_full(addr)
-# define AO_HAVE_char_load_acquire
-#endif
-
-#if defined(AO_HAVE_char_load_acquire) && !defined(AO_HAVE_char_load)
-# define AO_char_load(addr) AO_char_load_acquire(addr)
-# define AO_HAVE_char_load
-#endif
-
-#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_read)
-# define AO_char_load_read(addr) AO_char_load_full(addr)
-# define AO_HAVE_char_load_read
-#endif
-
-#if !defined(AO_HAVE_char_load_acquire_read) \
-    && defined(AO_HAVE_char_load_acquire)
-# define AO_char_load_acquire_read(addr) AO_char_load_acquire(addr)
-# define AO_HAVE_char_load_acquire_read
-#endif
-
-#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_load_acquire)
-  AO_INLINE unsigned/**/char
-  AO_char_load_acquire(const volatile unsigned/**/char *addr)
-  {
-    unsigned/**/char result = AO_char_load(addr);
-
-    /* Acquire barrier would be useless, since the load could be delayed    */
-    /* beyond it.                                                           */
-    AO_nop_full();
-    return result;
-  }
-# define AO_HAVE_char_load_acquire
-#endif
-
-#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_read) \
-    && !defined(AO_HAVE_char_load_read)
-  AO_INLINE unsigned/**/char
-  AO_char_load_read(const volatile unsigned/**/char *addr)
-  {
-    unsigned/**/char result = AO_char_load(addr);
-
-    /* Acquire barrier would be useless, since the load could be delayed    */
-    /* beyond it.                                                           */
-    AO_nop_read();
-    return result;
-  }
-# define AO_HAVE_char_load_read
-#endif
-
-#if defined(AO_HAVE_char_load_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_load_full)
-# define AO_char_load_full(addr) (AO_nop_full(), AO_char_load_acquire(addr))
-# define AO_HAVE_char_load_full
-#endif
-
-#if !defined(AO_HAVE_char_load_acquire_read) \
-    && defined(AO_HAVE_char_load_read)
-# define AO_char_load_acquire_read(addr) AO_char_load_read(addr)
-# define AO_HAVE_char_load_acquire_read
-#endif
-
-#if defined(AO_HAVE_char_load_acquire_read) && !defined(AO_HAVE_char_load)
-# define AO_char_load(addr) AO_char_load_acquire_read(addr)
-# define AO_HAVE_char_load
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_char_load_acquire_read)
-#   define AO_char_load_dd_acquire_read(addr) AO_char_load_acquire_read(addr)
-#   define AO_HAVE_char_load_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_char_load)
-#   define AO_char_load_dd_acquire_read(addr) AO_char_load(addr)
-#   define AO_HAVE_char_load_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* char_store */
-#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_release)
-# define AO_char_store_release(addr,val) AO_char_store_full(addr,val)
-# define AO_HAVE_char_store_release
-#endif
-
-#if defined(AO_HAVE_char_store_release) && !defined(AO_HAVE_char_store)
-# define AO_char_store(addr, val) AO_char_store_release(addr,val)
-# define AO_HAVE_char_store
-#endif
-
-#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_write)
-# define AO_char_store_write(addr,val) AO_char_store_full(addr,val)
-# define AO_HAVE_char_store_write
-#endif
-
-#if defined(AO_HAVE_char_store_release) \
-    && !defined(AO_HAVE_char_store_release_write)
-# define AO_char_store_release_write(addr, val) \
-                            AO_char_store_release(addr,val)
-# define AO_HAVE_char_store_release_write
-#endif
-
-#if defined(AO_HAVE_char_store_write) && !defined(AO_HAVE_char_store)
-# define AO_char_store(addr, val) AO_char_store_write(addr,val)
-# define AO_HAVE_char_store
-#endif
-
-#if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_store_release)
-# define AO_char_store_release(addr,val) \
-                                (AO_nop_full(), AO_char_store(addr,val))
-# define AO_HAVE_char_store_release
-#endif
-
-#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_char_store) \
-    && !defined(AO_HAVE_char_store_write)
-# define AO_char_store_write(addr, val) \
-                                (AO_nop_write(), AO_char_store(addr,val))
-# define AO_HAVE_char_store_write
-#endif
-
-#if defined(AO_HAVE_char_store_write) \
-    && !defined(AO_HAVE_char_store_release_write)
-# define AO_char_store_release_write(addr, val) AO_char_store_write(addr,val)
-# define AO_HAVE_char_store_release_write
-#endif
-
-#if defined(AO_HAVE_char_store_release) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_store_full)
-# define AO_char_store_full(addr, val) \
-                        (AO_char_store_release(addr, val), AO_nop_full())
-# define AO_HAVE_char_store_full
-#endif
-
 /* char_fetch_compare_and_swap */
 #if defined(AO_HAVE_char_fetch_compare_and_swap) \
     && defined(AO_HAVE_nop_full) \
@@ -388,163 +252,236 @@
 #   define AO_HAVE_char_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
-/*
- * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
 
-/* short_load */
-#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_acquire)
-# define AO_short_load_acquire(addr) AO_short_load_full(addr)
-# define AO_HAVE_short_load_acquire
+/* char_load */
+#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_acquire)
+# define AO_char_load_acquire(addr) AO_char_load_full(addr)
+# define AO_HAVE_char_load_acquire
 #endif
 
-#if defined(AO_HAVE_short_load_acquire) && !defined(AO_HAVE_short_load)
-# define AO_short_load(addr) AO_short_load_acquire(addr)
-# define AO_HAVE_short_load
+#if defined(AO_HAVE_char_load_acquire) && !defined(AO_HAVE_char_load)
+# define AO_char_load(addr) AO_char_load_acquire(addr)
+# define AO_HAVE_char_load
 #endif
 
-#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_read)
-# define AO_short_load_read(addr) AO_short_load_full(addr)
-# define AO_HAVE_short_load_read
+#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_read)
+# define AO_char_load_read(addr) AO_char_load_full(addr)
+# define AO_HAVE_char_load_read
 #endif
 
-#if !defined(AO_HAVE_short_load_acquire_read) \
-    && defined(AO_HAVE_short_load_acquire)
-# define AO_short_load_acquire_read(addr) AO_short_load_acquire(addr)
-# define AO_HAVE_short_load_acquire_read
+#if !defined(AO_HAVE_char_load_acquire_read) \
+    && defined(AO_HAVE_char_load_acquire)
+# define AO_char_load_acquire_read(addr) AO_char_load_acquire(addr)
+# define AO_HAVE_char_load_acquire_read
 #endif
 
-#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_load_acquire)
-  AO_INLINE unsigned/**/short
-  AO_short_load_acquire(const volatile unsigned/**/short *addr)
+#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_load_acquire)
+  AO_INLINE unsigned/**/char
+  AO_char_load_acquire(const volatile unsigned/**/char *addr)
   {
-    unsigned/**/short result = AO_short_load(addr);
+    unsigned/**/char result = AO_char_load(addr);
 
     /* Acquire barrier would be useless, since the load could be delayed    */
     /* beyond it.                                                           */
     AO_nop_full();
     return result;
   }
-# define AO_HAVE_short_load_acquire
+# define AO_HAVE_char_load_acquire
 #endif
 
-#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_read) \
-    && !defined(AO_HAVE_short_load_read)
-  AO_INLINE unsigned/**/short
-  AO_short_load_read(const volatile unsigned/**/short *addr)
+#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_read) \
+    && !defined(AO_HAVE_char_load_read)
+  AO_INLINE unsigned/**/char
+  AO_char_load_read(const volatile unsigned/**/char *addr)
   {
-    unsigned/**/short result = AO_short_load(addr);
+    unsigned/**/char result = AO_char_load(addr);
 
     /* Acquire barrier would be useless, since the load could be delayed    */
     /* beyond it.                                                           */
     AO_nop_read();
     return result;
   }
-# define AO_HAVE_short_load_read
+# define AO_HAVE_char_load_read
 #endif
 
-#if defined(AO_HAVE_short_load_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_load_full)
-# define AO_short_load_full(addr) (AO_nop_full(), AO_short_load_acquire(addr))
-# define AO_HAVE_short_load_full
+#if defined(AO_HAVE_char_load_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_load_full)
+# define AO_char_load_full(addr) (AO_nop_full(), AO_char_load_acquire(addr))
+# define AO_HAVE_char_load_full
 #endif
 
-#if !defined(AO_HAVE_short_load_acquire_read) \
-    && defined(AO_HAVE_short_load_read)
-# define AO_short_load_acquire_read(addr) AO_short_load_read(addr)
-# define AO_HAVE_short_load_acquire_read
-#endif
+#if defined(AO_HAVE_char_compare_and_swap_read) \
+    && !defined(AO_HAVE_char_load_read)
+# define AO_char_CAS_BASED_LOAD_READ
+  AO_INLINE unsigned/**/char
+  AO_char_load_read(const volatile unsigned/**/char *addr)
+  {
+    unsigned/**/char result;
 
-#if defined(AO_HAVE_short_load_acquire_read) && !defined(AO_HAVE_short_load)
-# define AO_short_load(addr) AO_short_load_acquire_read(addr)
-# define AO_HAVE_short_load
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_read(
+                                                (volatile unsigned/**/char *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_char_load_read
 #endif
 
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_short_load_acquire_read)
-#   define AO_short_load_dd_acquire_read(addr) AO_short_load_acquire_read(addr)
-#   define AO_HAVE_short_load_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_short_load)
-#   define AO_short_load_dd_acquire_read(addr) AO_short_load(addr)
-#   define AO_HAVE_short_load_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* short_store */
-#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_release)
-# define AO_short_store_release(addr,val) AO_short_store_full(addr,val)
-# define AO_HAVE_short_store_release
+#if !defined(AO_HAVE_char_load_acquire_read) \
+    && defined(AO_HAVE_char_load_read)
+# define AO_char_load_acquire_read(addr) AO_char_load_read(addr)
+# define AO_HAVE_char_load_acquire_read
 #endif
 
-#if defined(AO_HAVE_short_store_release) && !defined(AO_HAVE_short_store)
-# define AO_short_store(addr, val) AO_short_store_release(addr,val)
-# define AO_HAVE_short_store
+#if defined(AO_HAVE_char_load_acquire_read) && !defined(AO_HAVE_char_load) \
+    && (!defined(AO_char_CAS_BASED_LOAD_READ) \
+        || !defined(AO_HAVE_char_compare_and_swap))
+# define AO_char_load(addr) AO_char_load_acquire_read(addr)
+# define AO_HAVE_char_load
 #endif
 
-#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_write)
-# define AO_short_store_write(addr,val) AO_short_store_full(addr,val)
-# define AO_HAVE_short_store_write
-#endif
+#if defined(AO_HAVE_char_compare_and_swap_full) \
+    && !defined(AO_HAVE_char_load_full)
+  AO_INLINE unsigned/**/char
+  AO_char_load_full(const volatile unsigned/**/char *addr)
+  {
+    unsigned/**/char result;
 
-#if defined(AO_HAVE_short_store_release) \
-    && !defined(AO_HAVE_short_store_release_write)
-# define AO_short_store_release_write(addr, val) \
-                            AO_short_store_release(addr,val)
-# define AO_HAVE_short_store_release_write
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(
+                                                (volatile unsigned/**/char *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_char_load_full
 #endif
 
-#if defined(AO_HAVE_short_store_write) && !defined(AO_HAVE_short_store)
-# define AO_short_store(addr, val) AO_short_store_write(addr,val)
-# define AO_HAVE_short_store
+#if defined(AO_HAVE_char_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_char_load_acquire)
+  AO_INLINE unsigned/**/char
+  AO_char_load_acquire(const volatile unsigned/**/char *addr)
+  {
+    unsigned/**/char result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(
+                                                (volatile unsigned/**/char *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_char_load_acquire
 #endif
 
-#if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_store_release)
-# define AO_short_store_release(addr,val) \
-                                (AO_nop_full(), AO_short_store(addr,val))
-# define AO_HAVE_short_store_release
+#if defined(AO_HAVE_char_compare_and_swap) && !defined(AO_HAVE_char_load)
+  AO_INLINE unsigned/**/char
+  AO_char_load(const volatile unsigned/**/char *addr)
+  {
+    unsigned/**/char result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(
+                                                (volatile unsigned/**/char *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_char_load
 #endif
 
-#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_short_store) \
-    && !defined(AO_HAVE_short_store_write)
-# define AO_short_store_write(addr, val) \
-                                (AO_nop_write(), AO_short_store(addr,val))
-# define AO_HAVE_short_store_write
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_char_load_acquire_read)
+#   define AO_char_load_dd_acquire_read(addr) \
+                                AO_char_load_acquire_read(addr)
+#   define AO_HAVE_char_load_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_char_load)
+#   define AO_char_load_dd_acquire_read(addr) AO_char_load(addr)
+#   define AO_HAVE_char_load_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* char_store */
+#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_release)
+# define AO_char_store_release(addr, val) AO_char_store_full(addr, val)
+# define AO_HAVE_char_store_release
 #endif
 
-#if defined(AO_HAVE_short_store_write) \
-    && !defined(AO_HAVE_short_store_release_write)
-# define AO_short_store_release_write(addr, val) AO_short_store_write(addr,val)
-# define AO_HAVE_short_store_release_write
+#if defined(AO_HAVE_char_store_release) && !defined(AO_HAVE_char_store)
+# define AO_char_store(addr, val) AO_char_store_release(addr, val)
+# define AO_HAVE_char_store
 #endif
 
-#if defined(AO_HAVE_short_store_release) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_store_full)
-# define AO_short_store_full(addr, val) \
-                        (AO_short_store_release(addr, val), AO_nop_full())
-# define AO_HAVE_short_store_full
+#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_write)
+# define AO_char_store_write(addr, val) AO_char_store_full(addr, val)
+# define AO_HAVE_char_store_write
+#endif
+
+#if defined(AO_HAVE_char_store_release) \
+    && !defined(AO_HAVE_char_store_release_write)
+# define AO_char_store_release_write(addr, val) \
+                                AO_char_store_release(addr, val)
+# define AO_HAVE_char_store_release_write
+#endif
+
+#if defined(AO_HAVE_char_store_write) && !defined(AO_HAVE_char_store)
+# define AO_char_store(addr, val) AO_char_store_write(addr, val)
+# define AO_HAVE_char_store
+#endif
+
+#if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_store_release)
+# define AO_char_store_release(addr, val) \
+                                (AO_nop_full(), AO_char_store(addr, val))
+# define AO_HAVE_char_store_release
+#endif
+
+#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_char_store) \
+    && !defined(AO_HAVE_char_store_write)
+# define AO_char_store_write(addr, val) \
+                                (AO_nop_write(), AO_char_store(addr, val))
+# define AO_HAVE_char_store_write
+#endif
+
+#if defined(AO_HAVE_char_store_write) \
+    && !defined(AO_HAVE_char_store_release_write)
+# define AO_char_store_release_write(addr, val) \
+                                AO_char_store_write(addr, val)
+# define AO_HAVE_char_store_release_write
+#endif
+
+#if defined(AO_HAVE_char_store_release) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_store_full)
+# define AO_char_store_full(addr, val) \
+                                (AO_char_store_release(addr, val), \
+                                 AO_nop_full())
+# define AO_HAVE_char_store_full
 #endif
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
 
 /* short_fetch_compare_and_swap */
 #if defined(AO_HAVE_short_fetch_compare_and_swap) \
@@ -778,163 +715,236 @@
 #   define AO_HAVE_short_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
-/*
- * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
 
-/* int_load */
-#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire)
-# define AO_int_load_acquire(addr) AO_int_load_full(addr)
-# define AO_HAVE_int_load_acquire
+/* short_load */
+#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_acquire)
+# define AO_short_load_acquire(addr) AO_short_load_full(addr)
+# define AO_HAVE_short_load_acquire
 #endif
 
-#if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load)
-# define AO_int_load(addr) AO_int_load_acquire(addr)
-# define AO_HAVE_int_load
+#if defined(AO_HAVE_short_load_acquire) && !defined(AO_HAVE_short_load)
+# define AO_short_load(addr) AO_short_load_acquire(addr)
+# define AO_HAVE_short_load
 #endif
 
-#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read)
-# define AO_int_load_read(addr) AO_int_load_full(addr)
-# define AO_HAVE_int_load_read
+#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_read)
+# define AO_short_load_read(addr) AO_short_load_full(addr)
+# define AO_HAVE_short_load_read
 #endif
 
-#if !defined(AO_HAVE_int_load_acquire_read) \
-    && defined(AO_HAVE_int_load_acquire)
-# define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr)
-# define AO_HAVE_int_load_acquire_read
+#if !defined(AO_HAVE_short_load_acquire_read) \
+    && defined(AO_HAVE_short_load_acquire)
+# define AO_short_load_acquire_read(addr) AO_short_load_acquire(addr)
+# define AO_HAVE_short_load_acquire_read
 #endif
 
-#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_load_acquire)
-  AO_INLINE unsigned
-  AO_int_load_acquire(const volatile unsigned *addr)
+#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_load_acquire)
+  AO_INLINE unsigned/**/short
+  AO_short_load_acquire(const volatile unsigned/**/short *addr)
   {
-    unsigned result = AO_int_load(addr);
+    unsigned/**/short result = AO_short_load(addr);
 
     /* Acquire barrier would be useless, since the load could be delayed    */
     /* beyond it.                                                           */
     AO_nop_full();
     return result;
   }
-# define AO_HAVE_int_load_acquire
+# define AO_HAVE_short_load_acquire
 #endif
 
-#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \
-    && !defined(AO_HAVE_int_load_read)
-  AO_INLINE unsigned
-  AO_int_load_read(const volatile unsigned *addr)
+#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_read) \
+    && !defined(AO_HAVE_short_load_read)
+  AO_INLINE unsigned/**/short
+  AO_short_load_read(const volatile unsigned/**/short *addr)
   {
-    unsigned result = AO_int_load(addr);
+    unsigned/**/short result = AO_short_load(addr);
 
     /* Acquire barrier would be useless, since the load could be delayed    */
     /* beyond it.                                                           */
     AO_nop_read();
     return result;
   }
-# define AO_HAVE_int_load_read
+# define AO_HAVE_short_load_read
 #endif
 
-#if defined(AO_HAVE_int_load_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_load_full)
-# define AO_int_load_full(addr) (AO_nop_full(), AO_int_load_acquire(addr))
-# define AO_HAVE_int_load_full
+#if defined(AO_HAVE_short_load_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_load_full)
+# define AO_short_load_full(addr) (AO_nop_full(), AO_short_load_acquire(addr))
+# define AO_HAVE_short_load_full
 #endif
 
-#if !defined(AO_HAVE_int_load_acquire_read) \
-    && defined(AO_HAVE_int_load_read)
-# define AO_int_load_acquire_read(addr) AO_int_load_read(addr)
-# define AO_HAVE_int_load_acquire_read
+#if defined(AO_HAVE_short_compare_and_swap_read) \
+    && !defined(AO_HAVE_short_load_read)
+# define AO_short_CAS_BASED_LOAD_READ
+  AO_INLINE unsigned/**/short
+  AO_short_load_read(const volatile unsigned/**/short *addr)
+  {
+    unsigned/**/short result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_read(
+                                                (volatile unsigned/**/short *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_short_load_read
 #endif
 
-#if defined(AO_HAVE_int_load_acquire_read) && !defined(AO_HAVE_int_load)
-# define AO_int_load(addr) AO_int_load_acquire_read(addr)
-# define AO_HAVE_int_load
+#if !defined(AO_HAVE_short_load_acquire_read) \
+    && defined(AO_HAVE_short_load_read)
+# define AO_short_load_acquire_read(addr) AO_short_load_read(addr)
+# define AO_HAVE_short_load_acquire_read
+#endif
+
+#if defined(AO_HAVE_short_load_acquire_read) && !defined(AO_HAVE_short_load) \
+    && (!defined(AO_short_CAS_BASED_LOAD_READ) \
+        || !defined(AO_HAVE_short_compare_and_swap))
+# define AO_short_load(addr) AO_short_load_acquire_read(addr)
+# define AO_HAVE_short_load
+#endif
+
+#if defined(AO_HAVE_short_compare_and_swap_full) \
+    && !defined(AO_HAVE_short_load_full)
+  AO_INLINE unsigned/**/short
+  AO_short_load_full(const volatile unsigned/**/short *addr)
+  {
+    unsigned/**/short result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(
+                                                (volatile unsigned/**/short *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_short_load_full
+#endif
+
+#if defined(AO_HAVE_short_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_short_load_acquire)
+  AO_INLINE unsigned/**/short
+  AO_short_load_acquire(const volatile unsigned/**/short *addr)
+  {
+    unsigned/**/short result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(
+                                                (volatile unsigned/**/short *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_short_load_acquire
+#endif
+
+#if defined(AO_HAVE_short_compare_and_swap) && !defined(AO_HAVE_short_load)
+  AO_INLINE unsigned/**/short
+  AO_short_load(const volatile unsigned/**/short *addr)
+  {
+    unsigned/**/short result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(
+                                                (volatile unsigned/**/short *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_short_load
 #endif
 
 #ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_int_load_acquire_read)
-#   define AO_int_load_dd_acquire_read(addr) AO_int_load_acquire_read(addr)
-#   define AO_HAVE_int_load_dd_acquire_read
+# if defined(AO_HAVE_short_load_acquire_read)
+#   define AO_short_load_dd_acquire_read(addr) \
+                                AO_short_load_acquire_read(addr)
+#   define AO_HAVE_short_load_dd_acquire_read
 # endif
 #else
-# if defined(AO_HAVE_int_load)
-#   define AO_int_load_dd_acquire_read(addr) AO_int_load(addr)
-#   define AO_HAVE_int_load_dd_acquire_read
+# if defined(AO_HAVE_short_load)
+#   define AO_short_load_dd_acquire_read(addr) AO_short_load(addr)
+#   define AO_HAVE_short_load_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
 
-/* int_store */
-#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_release)
-# define AO_int_store_release(addr,val) AO_int_store_full(addr,val)
-# define AO_HAVE_int_store_release
+/* short_store */
+#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_release)
+# define AO_short_store_release(addr, val) AO_short_store_full(addr, val)
+# define AO_HAVE_short_store_release
 #endif
 
-#if defined(AO_HAVE_int_store_release) && !defined(AO_HAVE_int_store)
-# define AO_int_store(addr, val) AO_int_store_release(addr,val)
-# define AO_HAVE_int_store
+#if defined(AO_HAVE_short_store_release) && !defined(AO_HAVE_short_store)
+# define AO_short_store(addr, val) AO_short_store_release(addr, val)
+# define AO_HAVE_short_store
 #endif
 
-#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_write)
-# define AO_int_store_write(addr,val) AO_int_store_full(addr,val)
-# define AO_HAVE_int_store_write
+#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_write)
+# define AO_short_store_write(addr, val) AO_short_store_full(addr, val)
+# define AO_HAVE_short_store_write
 #endif
 
-#if defined(AO_HAVE_int_store_release) \
-    && !defined(AO_HAVE_int_store_release_write)
-# define AO_int_store_release_write(addr, val) \
-                            AO_int_store_release(addr,val)
-# define AO_HAVE_int_store_release_write
+#if defined(AO_HAVE_short_store_release) \
+    && !defined(AO_HAVE_short_store_release_write)
+# define AO_short_store_release_write(addr, val) \
+                                AO_short_store_release(addr, val)
+# define AO_HAVE_short_store_release_write
 #endif
 
-#if defined(AO_HAVE_int_store_write) && !defined(AO_HAVE_int_store)
-# define AO_int_store(addr, val) AO_int_store_write(addr,val)
-# define AO_HAVE_int_store
+#if defined(AO_HAVE_short_store_write) && !defined(AO_HAVE_short_store)
+# define AO_short_store(addr, val) AO_short_store_write(addr, val)
+# define AO_HAVE_short_store
 #endif
 
-#if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_store_release)
-# define AO_int_store_release(addr,val) \
-                                (AO_nop_full(), AO_int_store(addr,val))
-# define AO_HAVE_int_store_release
+#if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_store_release)
+# define AO_short_store_release(addr, val) \
+                                (AO_nop_full(), AO_short_store(addr, val))
+# define AO_HAVE_short_store_release
 #endif
 
-#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_int_store) \
-    && !defined(AO_HAVE_int_store_write)
-# define AO_int_store_write(addr, val) \
-                                (AO_nop_write(), AO_int_store(addr,val))
-# define AO_HAVE_int_store_write
+#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_short_store) \
+    && !defined(AO_HAVE_short_store_write)
+# define AO_short_store_write(addr, val) \
+                                (AO_nop_write(), AO_short_store(addr, val))
+# define AO_HAVE_short_store_write
 #endif
 
-#if defined(AO_HAVE_int_store_write) \
-    && !defined(AO_HAVE_int_store_release_write)
-# define AO_int_store_release_write(addr, val) AO_int_store_write(addr,val)
-# define AO_HAVE_int_store_release_write
+#if defined(AO_HAVE_short_store_write) \
+    && !defined(AO_HAVE_short_store_release_write)
+# define AO_short_store_release_write(addr, val) \
+                                AO_short_store_write(addr, val)
+# define AO_HAVE_short_store_release_write
 #endif
 
-#if defined(AO_HAVE_int_store_release) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_store_full)
-# define AO_int_store_full(addr, val) \
-                        (AO_int_store_release(addr, val), AO_nop_full())
-# define AO_HAVE_int_store_full
+#if defined(AO_HAVE_short_store_release) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_store_full)
+# define AO_short_store_full(addr, val) \
+                                (AO_short_store_release(addr, val), \
+                                 AO_nop_full())
+# define AO_HAVE_short_store_full
 #endif
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
 
 /* int_fetch_compare_and_swap */
 #if defined(AO_HAVE_int_fetch_compare_and_swap) \
@@ -1168,163 +1178,236 @@
 #   define AO_HAVE_int_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
-/*
- * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
 
-/* load */
-#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
-# define AO_load_acquire(addr) AO_load_full(addr)
-# define AO_HAVE_load_acquire
+/* int_load */
+#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire)
+# define AO_int_load_acquire(addr) AO_int_load_full(addr)
+# define AO_HAVE_int_load_acquire
 #endif
 
-#if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
-# define AO_load(addr) AO_load_acquire(addr)
-# define AO_HAVE_load
+#if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load)
+# define AO_int_load(addr) AO_int_load_acquire(addr)
+# define AO_HAVE_int_load
 #endif
 
-#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
-# define AO_load_read(addr) AO_load_full(addr)
-# define AO_HAVE_load_read
+#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read)
+# define AO_int_load_read(addr) AO_int_load_full(addr)
+# define AO_HAVE_int_load_read
 #endif
 
-#if !defined(AO_HAVE_load_acquire_read) \
-    && defined(AO_HAVE_load_acquire)
-# define AO_load_acquire_read(addr) AO_load_acquire(addr)
-# define AO_HAVE_load_acquire_read
+#if !defined(AO_HAVE_int_load_acquire_read) \
+    && defined(AO_HAVE_int_load_acquire)
+# define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr)
+# define AO_HAVE_int_load_acquire_read
 #endif
 
-#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_load_acquire)
-  AO_INLINE AO_t
-  AO_load_acquire(const volatile AO_t *addr)
+#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_load_acquire)
+  AO_INLINE unsigned
+  AO_int_load_acquire(const volatile unsigned *addr)
   {
-    AO_t result = AO_load(addr);
+    unsigned result = AO_int_load(addr);
 
     /* Acquire barrier would be useless, since the load could be delayed    */
     /* beyond it.                                                           */
     AO_nop_full();
     return result;
   }
-# define AO_HAVE_load_acquire
+# define AO_HAVE_int_load_acquire
 #endif
 
-#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \
-    && !defined(AO_HAVE_load_read)
-  AO_INLINE AO_t
-  AO_load_read(const volatile AO_t *addr)
+#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \
+    && !defined(AO_HAVE_int_load_read)
+  AO_INLINE unsigned
+  AO_int_load_read(const volatile unsigned *addr)
   {
-    AO_t result = AO_load(addr);
+    unsigned result = AO_int_load(addr);
 
     /* Acquire barrier would be useless, since the load could be delayed    */
     /* beyond it.                                                           */
     AO_nop_read();
     return result;
   }
-# define AO_HAVE_load_read
+# define AO_HAVE_int_load_read
 #endif
 
-#if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_load_full)
-# define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
-# define AO_HAVE_load_full
+#if defined(AO_HAVE_int_load_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_load_full)
+# define AO_int_load_full(addr) (AO_nop_full(), AO_int_load_acquire(addr))
+# define AO_HAVE_int_load_full
 #endif
 
-#if !defined(AO_HAVE_load_acquire_read) \
-    && defined(AO_HAVE_load_read)
-# define AO_load_acquire_read(addr) AO_load_read(addr)
-# define AO_HAVE_load_acquire_read
+#if defined(AO_HAVE_int_compare_and_swap_read) \
+    && !defined(AO_HAVE_int_load_read)
+# define AO_int_CAS_BASED_LOAD_READ
+  AO_INLINE unsigned
+  AO_int_load_read(const volatile unsigned *addr)
+  {
+    unsigned result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_read(
+                                                (volatile unsigned *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_int_load_read
 #endif
 
-#if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load)
-# define AO_load(addr) AO_load_acquire_read(addr)
-# define AO_HAVE_load
+#if !defined(AO_HAVE_int_load_acquire_read) \
+    && defined(AO_HAVE_int_load_read)
+# define AO_int_load_acquire_read(addr) AO_int_load_read(addr)
+# define AO_HAVE_int_load_acquire_read
+#endif
+
+#if defined(AO_HAVE_int_load_acquire_read) && !defined(AO_HAVE_int_load) \
+    && (!defined(AO_int_CAS_BASED_LOAD_READ) \
+        || !defined(AO_HAVE_int_compare_and_swap))
+# define AO_int_load(addr) AO_int_load_acquire_read(addr)
+# define AO_HAVE_int_load
+#endif
+
+#if defined(AO_HAVE_int_compare_and_swap_full) \
+    && !defined(AO_HAVE_int_load_full)
+  AO_INLINE unsigned
+  AO_int_load_full(const volatile unsigned *addr)
+  {
+    unsigned result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(
+                                                (volatile unsigned *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_int_load_full
+#endif
+
+#if defined(AO_HAVE_int_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_int_load_acquire)
+  AO_INLINE unsigned
+  AO_int_load_acquire(const volatile unsigned *addr)
+  {
+    unsigned result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(
+                                                (volatile unsigned *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_int_load_acquire
+#endif
+
+#if defined(AO_HAVE_int_compare_and_swap) && !defined(AO_HAVE_int_load)
+  AO_INLINE unsigned
+  AO_int_load(const volatile unsigned *addr)
+  {
+    unsigned result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(
+                                                (volatile unsigned *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_int_load
 #endif
 
 #ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_load_acquire_read)
-#   define AO_load_dd_acquire_read(addr) AO_load_acquire_read(addr)
-#   define AO_HAVE_load_dd_acquire_read
+# if defined(AO_HAVE_int_load_acquire_read)
+#   define AO_int_load_dd_acquire_read(addr) \
+                                AO_int_load_acquire_read(addr)
+#   define AO_HAVE_int_load_dd_acquire_read
 # endif
 #else
-# if defined(AO_HAVE_load)
-#   define AO_load_dd_acquire_read(addr) AO_load(addr)
-#   define AO_HAVE_load_dd_acquire_read
+# if defined(AO_HAVE_int_load)
+#   define AO_int_load_dd_acquire_read(addr) AO_int_load(addr)
+#   define AO_HAVE_int_load_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
 
-/* store */
-#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
-# define AO_store_release(addr,val) AO_store_full(addr,val)
-# define AO_HAVE_store_release
+/* int_store */
+#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_release)
+# define AO_int_store_release(addr, val) AO_int_store_full(addr, val)
+# define AO_HAVE_int_store_release
 #endif
 
-#if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
-# define AO_store(addr, val) AO_store_release(addr,val)
-# define AO_HAVE_store
+#if defined(AO_HAVE_int_store_release) && !defined(AO_HAVE_int_store)
+# define AO_int_store(addr, val) AO_int_store_release(addr, val)
+# define AO_HAVE_int_store
 #endif
 
-#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
-# define AO_store_write(addr,val) AO_store_full(addr,val)
-# define AO_HAVE_store_write
+#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_write)
+# define AO_int_store_write(addr, val) AO_int_store_full(addr, val)
+# define AO_HAVE_int_store_write
 #endif
 
-#if defined(AO_HAVE_store_release) \
-    && !defined(AO_HAVE_store_release_write)
-# define AO_store_release_write(addr, val) \
-                            AO_store_release(addr,val)
-# define AO_HAVE_store_release_write
+#if defined(AO_HAVE_int_store_release) \
+    && !defined(AO_HAVE_int_store_release_write)
+# define AO_int_store_release_write(addr, val) \
+                                AO_int_store_release(addr, val)
+# define AO_HAVE_int_store_release_write
 #endif
 
-#if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
-# define AO_store(addr, val) AO_store_write(addr,val)
-# define AO_HAVE_store
+#if defined(AO_HAVE_int_store_write) && !defined(AO_HAVE_int_store)
+# define AO_int_store(addr, val) AO_int_store_write(addr, val)
+# define AO_HAVE_int_store
 #endif
 
-#if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_store_release)
-# define AO_store_release(addr,val) \
-                                (AO_nop_full(), AO_store(addr,val))
-# define AO_HAVE_store_release
+#if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_store_release)
+# define AO_int_store_release(addr, val) \
+                                (AO_nop_full(), AO_int_store(addr, val))
+# define AO_HAVE_int_store_release
 #endif
 
-#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) \
-    && !defined(AO_HAVE_store_write)
-# define AO_store_write(addr, val) \
-                                (AO_nop_write(), AO_store(addr,val))
-# define AO_HAVE_store_write
+#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_int_store) \
+    && !defined(AO_HAVE_int_store_write)
+# define AO_int_store_write(addr, val) \
+                                (AO_nop_write(), AO_int_store(addr, val))
+# define AO_HAVE_int_store_write
 #endif
 
-#if defined(AO_HAVE_store_write) \
-    && !defined(AO_HAVE_store_release_write)
-# define AO_store_release_write(addr, val) AO_store_write(addr,val)
-# define AO_HAVE_store_release_write
+#if defined(AO_HAVE_int_store_write) \
+    && !defined(AO_HAVE_int_store_release_write)
+# define AO_int_store_release_write(addr, val) \
+                                AO_int_store_write(addr, val)
+# define AO_HAVE_int_store_release_write
 #endif
 
-#if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_store_full)
-# define AO_store_full(addr, val) \
-                        (AO_store_release(addr, val), AO_nop_full())
-# define AO_HAVE_store_full
+#if defined(AO_HAVE_int_store_release) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_store_full)
+# define AO_int_store_full(addr, val) \
+                                (AO_int_store_release(addr, val), \
+                                 AO_nop_full())
+# define AO_HAVE_int_store_full
 #endif
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
 
 /* fetch_compare_and_swap */
 #if defined(AO_HAVE_fetch_compare_and_swap) \
@@ -1558,3 +1641,212 @@
 #   define AO_HAVE_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
+
+/* load */
+#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
+# define AO_load_acquire(addr) AO_load_full(addr)
+# define AO_HAVE_load_acquire
+#endif
+
+#if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
+# define AO_load(addr) AO_load_acquire(addr)
+# define AO_HAVE_load
+#endif
+
+#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
+# define AO_load_read(addr) AO_load_full(addr)
+# define AO_HAVE_load_read
+#endif
+
+#if !defined(AO_HAVE_load_acquire_read) \
+    && defined(AO_HAVE_load_acquire)
+# define AO_load_acquire_read(addr) AO_load_acquire(addr)
+# define AO_HAVE_load_acquire_read
+#endif
+
+#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_load_acquire)
+  AO_INLINE AO_t
+  AO_load_acquire(const volatile AO_t *addr)
+  {
+    AO_t result = AO_load(addr);
+
+    /* Acquire barrier would be useless, since the load could be delayed    */
+    /* beyond it.                                                           */
+    AO_nop_full();
+    return result;
+  }
+# define AO_HAVE_load_acquire
+#endif
+
+#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \
+    && !defined(AO_HAVE_load_read)
+  AO_INLINE AO_t
+  AO_load_read(const volatile AO_t *addr)
+  {
+    AO_t result = AO_load(addr);
+
+    /* Acquire barrier would be useless, since the load could be delayed    */
+    /* beyond it.                                                           */
+    AO_nop_read();
+    return result;
+  }
+# define AO_HAVE_load_read
+#endif
+
+#if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_load_full)
+# define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
+# define AO_HAVE_load_full
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_read) \
+    && !defined(AO_HAVE_load_read)
+# define AO_CAS_BASED_LOAD_READ
+  AO_INLINE AO_t
+  AO_load_read(const volatile AO_t *addr)
+  {
+    AO_t result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_compare_and_swap_read(
+                                                (volatile AO_t *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_load_read
+#endif
+
+#if !defined(AO_HAVE_load_acquire_read) \
+    && defined(AO_HAVE_load_read)
+# define AO_load_acquire_read(addr) AO_load_read(addr)
+# define AO_HAVE_load_acquire_read
+#endif
+
+#if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load) \
+    && (!defined(AO_CAS_BASED_LOAD_READ) \
+        || !defined(AO_HAVE_compare_and_swap))
+# define AO_load(addr) AO_load_acquire_read(addr)
+# define AO_HAVE_load
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_full) \
+    && !defined(AO_HAVE_load_full)
+  AO_INLINE AO_t
+  AO_load_full(const volatile AO_t *addr)
+  {
+    AO_t result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(
+                                                (volatile AO_t *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_load_full
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_load_acquire)
+  AO_INLINE AO_t
+  AO_load_acquire(const volatile AO_t *addr)
+  {
+    AO_t result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(
+                                                (volatile AO_t *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_load_acquire
+#endif
+
+#if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_load)
+  AO_INLINE AO_t
+  AO_load(const volatile AO_t *addr)
+  {
+    AO_t result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_compare_and_swap(
+                                                (volatile AO_t *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_load
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_load_acquire_read)
+#   define AO_load_dd_acquire_read(addr) \
+                                AO_load_acquire_read(addr)
+#   define AO_HAVE_load_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_load)
+#   define AO_load_dd_acquire_read(addr) AO_load(addr)
+#   define AO_HAVE_load_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* store */
+#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
+# define AO_store_release(addr, val) AO_store_full(addr, val)
+# define AO_HAVE_store_release
+#endif
+
+#if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
+# define AO_store(addr, val) AO_store_release(addr, val)
+# define AO_HAVE_store
+#endif
+
+#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
+# define AO_store_write(addr, val) AO_store_full(addr, val)
+# define AO_HAVE_store_write
+#endif
+
+#if defined(AO_HAVE_store_release) \
+    && !defined(AO_HAVE_store_release_write)
+# define AO_store_release_write(addr, val) \
+                                AO_store_release(addr, val)
+# define AO_HAVE_store_release_write
+#endif
+
+#if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
+# define AO_store(addr, val) AO_store_write(addr, val)
+# define AO_HAVE_store
+#endif
+
+#if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_store_release)
+# define AO_store_release(addr, val) \
+                                (AO_nop_full(), AO_store(addr, val))
+# define AO_HAVE_store_release
+#endif
+
+#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) \
+    && !defined(AO_HAVE_store_write)
+# define AO_store_write(addr, val) \
+                                (AO_nop_write(), AO_store(addr, val))
+# define AO_HAVE_store_write
+#endif
+
+#if defined(AO_HAVE_store_write) \
+    && !defined(AO_HAVE_store_release_write)
+# define AO_store_release_write(addr, val) \
+                                AO_store_write(addr, val)
+# define AO_HAVE_store_release_write
+#endif
+
+#if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_store_full)
+# define AO_store_full(addr, val) \
+                                (AO_store_release(addr, val), \
+                                 AO_nop_full())
+# define AO_HAVE_store_full
+#endif

diff --git a/src/atomic_ops/generalize-small.template b/src/atomic_ops/generalize-small.template
index 8ac9cfd1b40f7fe33ea848dc576c8e257b0129ae..ae069019e7005d5843971d86604bba4927d20a09 100644 (file)
--- a/src/atomic_ops/generalize-small.template
+++ b/src/atomic_ops/generalize-small.template
@@ -20,142 +20,6 @@
  * SOFTWARE.
  */
 
-/* XSIZE_load */
-#if defined(AO_HAVE_XSIZE_load_full) && !defined(AO_HAVE_XSIZE_load_acquire)
-# define AO_XSIZE_load_acquire(addr) AO_XSIZE_load_full(addr)
-# define AO_HAVE_XSIZE_load_acquire
-#endif
-
-#if defined(AO_HAVE_XSIZE_load_acquire) && !defined(AO_HAVE_XSIZE_load)
-# define AO_XSIZE_load(addr) AO_XSIZE_load_acquire(addr)
-# define AO_HAVE_XSIZE_load
-#endif
-
-#if defined(AO_HAVE_XSIZE_load_full) && !defined(AO_HAVE_XSIZE_load_read)
-# define AO_XSIZE_load_read(addr) AO_XSIZE_load_full(addr)
-# define AO_HAVE_XSIZE_load_read
-#endif
-
-#if !defined(AO_HAVE_XSIZE_load_acquire_read) \
-    && defined(AO_HAVE_XSIZE_load_acquire)
-# define AO_XSIZE_load_acquire_read(addr) AO_XSIZE_load_acquire(addr)
-# define AO_HAVE_XSIZE_load_acquire_read
-#endif
-
-#if defined(AO_HAVE_XSIZE_load) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_load_acquire)
-  AO_INLINE XCTYPE
-  AO_XSIZE_load_acquire(const volatile XCTYPE *addr)
-  {
-    XCTYPE result = AO_XSIZE_load(addr);
-
-    /* Acquire barrier would be useless, since the load could be delayed    */
-    /* beyond it.                                                           */
-    AO_nop_full();
-    return result;
-  }
-# define AO_HAVE_XSIZE_load_acquire
-#endif
-
-#if defined(AO_HAVE_XSIZE_load) && defined(AO_HAVE_nop_read) \
-    && !defined(AO_HAVE_XSIZE_load_read)
-  AO_INLINE XCTYPE
-  AO_XSIZE_load_read(const volatile XCTYPE *addr)
-  {
-    XCTYPE result = AO_XSIZE_load(addr);
-
-    /* Acquire barrier would be useless, since the load could be delayed    */
-    /* beyond it.                                                           */
-    AO_nop_read();
-    return result;
-  }
-# define AO_HAVE_XSIZE_load_read
-#endif
-
-#if defined(AO_HAVE_XSIZE_load_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_load_full)
-# define AO_XSIZE_load_full(addr) (AO_nop_full(), AO_XSIZE_load_acquire(addr))
-# define AO_HAVE_XSIZE_load_full
-#endif
-
-#if !defined(AO_HAVE_XSIZE_load_acquire_read) \
-    && defined(AO_HAVE_XSIZE_load_read)
-# define AO_XSIZE_load_acquire_read(addr) AO_XSIZE_load_read(addr)
-# define AO_HAVE_XSIZE_load_acquire_read
-#endif
-
-#if defined(AO_HAVE_XSIZE_load_acquire_read) && !defined(AO_HAVE_XSIZE_load)
-# define AO_XSIZE_load(addr) AO_XSIZE_load_acquire_read(addr)
-# define AO_HAVE_XSIZE_load
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_XSIZE_load_acquire_read)
-#   define AO_XSIZE_load_dd_acquire_read(addr) AO_XSIZE_load_acquire_read(addr)
-#   define AO_HAVE_XSIZE_load_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_XSIZE_load)
-#   define AO_XSIZE_load_dd_acquire_read(addr) AO_XSIZE_load(addr)
-#   define AO_HAVE_XSIZE_load_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* XSIZE_store */
-#if defined(AO_HAVE_XSIZE_store_full) && !defined(AO_HAVE_XSIZE_store_release)
-# define AO_XSIZE_store_release(addr,val) AO_XSIZE_store_full(addr,val)
-# define AO_HAVE_XSIZE_store_release
-#endif
-
-#if defined(AO_HAVE_XSIZE_store_release) && !defined(AO_HAVE_XSIZE_store)
-# define AO_XSIZE_store(addr, val) AO_XSIZE_store_release(addr,val)
-# define AO_HAVE_XSIZE_store
-#endif
-
-#if defined(AO_HAVE_XSIZE_store_full) && !defined(AO_HAVE_XSIZE_store_write)
-# define AO_XSIZE_store_write(addr,val) AO_XSIZE_store_full(addr,val)
-# define AO_HAVE_XSIZE_store_write
-#endif
-
-#if defined(AO_HAVE_XSIZE_store_release) \
-    && !defined(AO_HAVE_XSIZE_store_release_write)
-# define AO_XSIZE_store_release_write(addr, val) \
-                            AO_XSIZE_store_release(addr,val)
-# define AO_HAVE_XSIZE_store_release_write
-#endif
-
-#if defined(AO_HAVE_XSIZE_store_write) && !defined(AO_HAVE_XSIZE_store)
-# define AO_XSIZE_store(addr, val) AO_XSIZE_store_write(addr,val)
-# define AO_HAVE_XSIZE_store
-#endif
-
-#if defined(AO_HAVE_XSIZE_store) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_store_release)
-# define AO_XSIZE_store_release(addr,val) \
-                                (AO_nop_full(), AO_XSIZE_store(addr,val))
-# define AO_HAVE_XSIZE_store_release
-#endif
-
-#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_XSIZE_store) \
-    && !defined(AO_HAVE_XSIZE_store_write)
-# define AO_XSIZE_store_write(addr, val) \
-                                (AO_nop_write(), AO_XSIZE_store(addr,val))
-# define AO_HAVE_XSIZE_store_write
-#endif
-
-#if defined(AO_HAVE_XSIZE_store_write) \
-    && !defined(AO_HAVE_XSIZE_store_release_write)
-# define AO_XSIZE_store_release_write(addr, val) AO_XSIZE_store_write(addr,val)
-# define AO_HAVE_XSIZE_store_release_write
-#endif
-
-#if defined(AO_HAVE_XSIZE_store_release) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_store_full)
-# define AO_XSIZE_store_full(addr, val) \
-                        (AO_XSIZE_store_release(addr, val), AO_nop_full())
-# define AO_HAVE_XSIZE_store_full
-#endif
-
 /* XSIZE_fetch_compare_and_swap */
 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap) \
     && defined(AO_HAVE_nop_full) \
@@ -388,3 +252,212 @@
 #   define AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
+
+/* XSIZE_load */
+#if defined(AO_HAVE_XSIZE_load_full) && !defined(AO_HAVE_XSIZE_load_acquire)
+# define AO_XSIZE_load_acquire(addr) AO_XSIZE_load_full(addr)
+# define AO_HAVE_XSIZE_load_acquire
+#endif
+
+#if defined(AO_HAVE_XSIZE_load_acquire) && !defined(AO_HAVE_XSIZE_load)
+# define AO_XSIZE_load(addr) AO_XSIZE_load_acquire(addr)
+# define AO_HAVE_XSIZE_load
+#endif
+
+#if defined(AO_HAVE_XSIZE_load_full) && !defined(AO_HAVE_XSIZE_load_read)
+# define AO_XSIZE_load_read(addr) AO_XSIZE_load_full(addr)
+# define AO_HAVE_XSIZE_load_read
+#endif
+
+#if !defined(AO_HAVE_XSIZE_load_acquire_read) \
+    && defined(AO_HAVE_XSIZE_load_acquire)
+# define AO_XSIZE_load_acquire_read(addr) AO_XSIZE_load_acquire(addr)
+# define AO_HAVE_XSIZE_load_acquire_read
+#endif
+
+#if defined(AO_HAVE_XSIZE_load) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_load_acquire)
+  AO_INLINE XCTYPE
+  AO_XSIZE_load_acquire(const volatile XCTYPE *addr)
+  {
+    XCTYPE result = AO_XSIZE_load(addr);
+
+    /* Acquire barrier would be useless, since the load could be delayed    */
+    /* beyond it.                                                           */
+    AO_nop_full();
+    return result;
+  }
+# define AO_HAVE_XSIZE_load_acquire
+#endif
+
+#if defined(AO_HAVE_XSIZE_load) && defined(AO_HAVE_nop_read) \
+    && !defined(AO_HAVE_XSIZE_load_read)
+  AO_INLINE XCTYPE
+  AO_XSIZE_load_read(const volatile XCTYPE *addr)
+  {
+    XCTYPE result = AO_XSIZE_load(addr);
+
+    /* Acquire barrier would be useless, since the load could be delayed    */
+    /* beyond it.                                                           */
+    AO_nop_read();
+    return result;
+  }
+# define AO_HAVE_XSIZE_load_read
+#endif
+
+#if defined(AO_HAVE_XSIZE_load_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_load_full)
+# define AO_XSIZE_load_full(addr) (AO_nop_full(), AO_XSIZE_load_acquire(addr))
+# define AO_HAVE_XSIZE_load_full
+#endif
+
+#if defined(AO_HAVE_XSIZE_compare_and_swap_read) \
+    && !defined(AO_HAVE_XSIZE_load_read)
+# define AO_XSIZE_CAS_BASED_LOAD_READ
+  AO_INLINE XCTYPE
+  AO_XSIZE_load_read(const volatile XCTYPE *addr)
+  {
+    XCTYPE result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_read(
+                                                (volatile XCTYPE *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_XSIZE_load_read
+#endif
+
+#if !defined(AO_HAVE_XSIZE_load_acquire_read) \
+    && defined(AO_HAVE_XSIZE_load_read)
+# define AO_XSIZE_load_acquire_read(addr) AO_XSIZE_load_read(addr)
+# define AO_HAVE_XSIZE_load_acquire_read
+#endif
+
+#if defined(AO_HAVE_XSIZE_load_acquire_read) && !defined(AO_HAVE_XSIZE_load) \
+    && (!defined(AO_XSIZE_CAS_BASED_LOAD_READ) \
+        || !defined(AO_HAVE_XSIZE_compare_and_swap))
+# define AO_XSIZE_load(addr) AO_XSIZE_load_acquire_read(addr)
+# define AO_HAVE_XSIZE_load
+#endif
+
+#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
+    && !defined(AO_HAVE_XSIZE_load_full)
+  AO_INLINE XCTYPE
+  AO_XSIZE_load_full(const volatile XCTYPE *addr)
+  {
+    XCTYPE result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(
+                                                (volatile XCTYPE *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_XSIZE_load_full
+#endif
+
+#if defined(AO_HAVE_XSIZE_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_XSIZE_load_acquire)
+  AO_INLINE XCTYPE
+  AO_XSIZE_load_acquire(const volatile XCTYPE *addr)
+  {
+    XCTYPE result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_acquire(
+                                                (volatile XCTYPE *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_XSIZE_load_acquire
+#endif
+
+#if defined(AO_HAVE_XSIZE_compare_and_swap) && !defined(AO_HAVE_XSIZE_load)
+  AO_INLINE XCTYPE
+  AO_XSIZE_load(const volatile XCTYPE *addr)
+  {
+    XCTYPE result;
+
+    do {
+      result = *addr;
+    } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap(
+                                                (volatile XCTYPE *)addr,
+                                                result, result)));
+    return result;
+  }
+# define AO_HAVE_XSIZE_load
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_XSIZE_load_acquire_read)
+#   define AO_XSIZE_load_dd_acquire_read(addr) \
+                                AO_XSIZE_load_acquire_read(addr)
+#   define AO_HAVE_XSIZE_load_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_XSIZE_load)
+#   define AO_XSIZE_load_dd_acquire_read(addr) AO_XSIZE_load(addr)
+#   define AO_HAVE_XSIZE_load_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* XSIZE_store */
+#if defined(AO_HAVE_XSIZE_store_full) && !defined(AO_HAVE_XSIZE_store_release)
+# define AO_XSIZE_store_release(addr, val) AO_XSIZE_store_full(addr, val)
+# define AO_HAVE_XSIZE_store_release
+#endif
+
+#if defined(AO_HAVE_XSIZE_store_release) && !defined(AO_HAVE_XSIZE_store)
+# define AO_XSIZE_store(addr, val) AO_XSIZE_store_release(addr, val)
+# define AO_HAVE_XSIZE_store
+#endif
+
+#if defined(AO_HAVE_XSIZE_store_full) && !defined(AO_HAVE_XSIZE_store_write)
+# define AO_XSIZE_store_write(addr, val) AO_XSIZE_store_full(addr, val)
+# define AO_HAVE_XSIZE_store_write
+#endif
+
+#if defined(AO_HAVE_XSIZE_store_release) \
+    && !defined(AO_HAVE_XSIZE_store_release_write)
+# define AO_XSIZE_store_release_write(addr, val) \
+                                AO_XSIZE_store_release(addr, val)
+# define AO_HAVE_XSIZE_store_release_write
+#endif
+
+#if defined(AO_HAVE_XSIZE_store_write) && !defined(AO_HAVE_XSIZE_store)
+# define AO_XSIZE_store(addr, val) AO_XSIZE_store_write(addr, val)
+# define AO_HAVE_XSIZE_store
+#endif
+
+#if defined(AO_HAVE_XSIZE_store) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_store_release)
+# define AO_XSIZE_store_release(addr, val) \
+                                (AO_nop_full(), AO_XSIZE_store(addr, val))
+# define AO_HAVE_XSIZE_store_release
+#endif
+
+#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_XSIZE_store) \
+    && !defined(AO_HAVE_XSIZE_store_write)
+# define AO_XSIZE_store_write(addr, val) \
+                                (AO_nop_write(), AO_XSIZE_store(addr, val))
+# define AO_HAVE_XSIZE_store_write
+#endif
+
+#if defined(AO_HAVE_XSIZE_store_write) \
+    && !defined(AO_HAVE_XSIZE_store_release_write)
+# define AO_XSIZE_store_release_write(addr, val) \
+                                AO_XSIZE_store_write(addr, val)
+# define AO_HAVE_XSIZE_store_release_write
+#endif
+
+#if defined(AO_HAVE_XSIZE_store_release) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_store_full)
+# define AO_XSIZE_store_full(addr, val) \
+                                (AO_XSIZE_store_release(addr, val), \
+                                 AO_nop_full())
+# define AO_HAVE_XSIZE_store_full
+#endif