From 53a14c1d6eede42614ac0f7cd335cb76faf6e145 Mon Sep 17 00:00:00 2001 From: Ivan Maidanski Date: Wed, 9 Nov 2011 21:04:37 +0400 Subject: [PATCH] Add AO_GENERALIZE_ASM_BOOL_CAS new macro to allow AO_compare_and_swap definition via AO_fetch_compare_and_swap instead of own ASM-based one * src/atomic_ops/sysdeps/armcc/arm_v6.h (AO_compare_and_swap): Do not define if AO_GENERALIZE_ASM_BOOL_CAS. * src/atomic_ops/sysdeps/gcc/arm.h (AO_compare_and_swap): Likewise. * src/atomic_ops/sysdeps/gcc/hexagon.h (AO_compare_and_swap): Likewise. * src/atomic_ops/sysdeps/gcc/powerpc.h (AO_compare_and_swap, AO_compare_and_swap_acquire, AO_compare_and_swap_release, AO_compare_and_swap_full): Likewise. * src/atomic_ops/sysdeps/gcc/x86.h (AO_compare_and_swap_full): Likewise. * src/atomic_ops/sysdeps/gcc/x86_64.h (AO_compare_and_swap_full): Likewise. * src/atomic_ops/sysdeps/sunc/x86.h (AO_compare_and_swap_full): Likewise. * src/atomic_ops/sysdeps/sunc/x86_64.h (AO_compare_and_swap_full): Likewise. --- src/atomic_ops/sysdeps/armcc/arm_v6.h | 39 +++--- src/atomic_ops/sysdeps/gcc/arm.h | 48 ++++---- src/atomic_ops/sysdeps/gcc/hexagon.h | 48 ++++---- src/atomic_ops/sysdeps/gcc/powerpc.h | 168 ++++++++++++++------------ src/atomic_ops/sysdeps/gcc/x86.h | 35 +++--- src/atomic_ops/sysdeps/gcc/x86_64.h | 35 +++--- src/atomic_ops/sysdeps/sunc/x86.h | 25 ++-- src/atomic_ops/sysdeps/sunc/x86_64.h | 25 ++-- 8 files changed, 224 insertions(+), 199 deletions(-) diff --git a/src/atomic_ops/sysdeps/armcc/arm_v6.h b/src/atomic_ops/sysdeps/armcc/arm_v6.h index c2c9b1d..0ef50c3 100644 --- a/src/atomic_ops/sysdeps/armcc/arm_v6.h +++ b/src/atomic_ops/sysdeps/armcc/arm_v6.h @@ -175,28 +175,29 @@ __asm { #define AO_HAVE_fetch_and_sub1 /* NEC LE-IT: compare and swap */ -/* Returns nonzero if the comparison succeeded. */ -AO_INLINE int -AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) -{ - AO_t result,tmp; - -retry: -__asm__ { - mov result, #2 - ldrex tmp, [addr] - teq tmp, old_val +#ifndef AO_GENERALIZE_ASM_BOOL_CAS + /* Returns nonzero if the comparison succeeded. */ + AO_INLINE int + AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) + { + AO_t result, tmp; + + retry: + __asm__ { + mov result, #2 + ldrex tmp, [addr] + teq tmp, old_val # ifdef __thumb__ it eq # endif - strexeq result, new_val, [addr] - teq result, #1 - beq retry - } - - return !(result&2); -} -#define AO_HAVE_compare_and_swap + strexeq result, new_val, [addr] + teq result, #1 + beq retry + } + return !(result&2); + } +# define AO_HAVE_compare_and_swap +#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ AO_INLINE AO_t AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) diff --git a/src/atomic_ops/sysdeps/gcc/arm.h b/src/atomic_ops/sysdeps/gcc/arm.h index 4466c5b..4f14271 100644 --- a/src/atomic_ops/sysdeps/gcc/arm.h +++ b/src/atomic_ops/sysdeps/gcc/arm.h @@ -240,30 +240,32 @@ AO_fetch_and_sub1(volatile AO_t *p) #define AO_HAVE_fetch_and_sub1 /* NEC LE-IT: compare and swap */ -/* Returns nonzero if the comparison succeeded. */ -AO_INLINE int -AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) -{ - AO_t result, tmp; +#ifndef AO_GENERALIZE_ASM_BOOL_CAS + /* Returns nonzero if the comparison succeeded. */ + AO_INLINE int + AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) + { + AO_t result, tmp; - __asm__ __volatile__("@AO_compare_and_swap\n" - AO_THUMB_GO_ARM - "1: mov %0, #2\n" /* store a flag */ - " ldrex %1, [%3]\n" /* get original */ - " teq %1, %4\n" /* see if match */ -# ifdef __thumb2__ - " it eq\n" -# endif - " strexeq %0, %5, [%3]\n" /* store new one if matched */ - " teq %0, #1\n" - " beq 1b\n" /* if update failed, repeat */ - AO_THUMB_RESTORE_MODE - : "=&r"(result), "=&r"(tmp), "+m"(*addr) - : "r"(addr), "r"(old_val), "r"(new_val) - : AO_THUMB_SWITCH_CLOBBERS "cc"); - return !(result&2); /* if succeded, return 1, else 0 */ -} -#define AO_HAVE_compare_and_swap + __asm__ __volatile__("@AO_compare_and_swap\n" + AO_THUMB_GO_ARM + "1: mov %0, #2\n" /* store a flag */ + " ldrex %1, [%3]\n" /* get original */ + " teq %1, %4\n" /* see if match */ +# ifdef __thumb2__ + " it eq\n" +# endif + " strexeq %0, %5, [%3]\n" /* store new one if matched */ + " teq %0, #1\n" + " beq 1b\n" /* if update failed, repeat */ + AO_THUMB_RESTORE_MODE + : "=&r"(result), "=&r"(tmp), "+m"(*addr) + : "r"(addr), "r"(old_val), "r"(new_val) + : AO_THUMB_SWITCH_CLOBBERS "cc"); + return !(result&2); /* if succeded, return 1, else 0 */ + } +# define AO_HAVE_compare_and_swap +#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ AO_INLINE AO_t AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) diff --git a/src/atomic_ops/sysdeps/gcc/hexagon.h b/src/atomic_ops/sysdeps/gcc/hexagon.h index fed10e0..6d25797 100644 --- a/src/atomic_ops/sysdeps/gcc/hexagon.h +++ b/src/atomic_ops/sysdeps/gcc/hexagon.h @@ -68,29 +68,31 @@ AO_test_and_set(volatile AO_TS_t *addr) } #define AO_HAVE_test_and_set -AO_INLINE int -AO_compare_and_swap(volatile AO_t *addr, AO_t old, AO_t new_val) -{ - AO_t __oldval; - int result = 0; - __asm__ __volatile__( - "1:\n" - " %0 = memw_locked(%3);\n" /* load and reserve */ - " {\n" - " p2 = cmp.eq(%0,%4);\n" /* if load is not equal to */ - " if (!p2.new) jump:nt 2f; \n" /* old, fail */ - " }\n" - " memw_locked(%3,p1) = %5;\n" /* else store conditional */ - " if (!p1) jump 1b;\n" /* retry if lost reservation */ - " %1 = #1\n" /* success, result = 1 */ - "2:\n" - : "=&r" (__oldval), "+r" (result), "+m"(*addr) - : "r" (addr), "r" (old), "r" (new_val) - : "p1", "p2", "memory" - ); - return result; -} -#define AO_HAVE_compare_and_swap +#ifndef AO_GENERALIZE_ASM_BOOL_CAS + AO_INLINE int + AO_compare_and_swap(volatile AO_t *addr, AO_t old, AO_t new_val) + { + AO_t __oldval; + int result = 0; + __asm__ __volatile__( + "1:\n" + " %0 = memw_locked(%3);\n" /* load and reserve */ + " {\n" + " p2 = cmp.eq(%0,%4);\n" /* if load is not equal to */ + " if (!p2.new) jump:nt 2f; \n" /* old, fail */ + " }\n" + " memw_locked(%3,p1) = %5;\n" /* else store conditional */ + " if (!p1) jump 1b;\n" /* retry if lost reservation */ + " %1 = #1\n" /* success, result = 1 */ + "2:\n" + : "=&r" (__oldval), "+r" (result), "+m"(*addr) + : "r" (addr), "r" (old), "r" (new_val) + : "p1", "p2", "memory" + ); + return result; + } +# define AO_HAVE_compare_and_swap +#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ AO_INLINE AO_t AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) diff --git a/src/atomic_ops/sysdeps/gcc/powerpc.h b/src/atomic_ops/sysdeps/gcc/powerpc.h index 36cb613..cc1aaa6 100644 --- a/src/atomic_ops/sysdeps/gcc/powerpc.h +++ b/src/atomic_ops/sysdeps/gcc/powerpc.h @@ -1,7 +1,7 @@ /* * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. + * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. * * * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED @@ -166,93 +166,101 @@ AO_test_and_set_full(volatile AO_TS_t *addr) { } #define AO_HAVE_test_and_set_full -AO_INLINE int -AO_compare_and_swap(volatile AO_t *addr, AO_t old, AO_t new_val) { - AO_t oldval; - int result = 0; -#if defined(__powerpc64__) || defined(__ppc64__) || defined(__64BIT__) -/* FIXME: Completely untested. */ - __asm__ __volatile__( - "1:ldarx %0,0,%2\n" /* load and reserve */ - "cmpd %0, %4\n" /* if load is not equal to */ - "bne 2f\n" /* old, fail */ - "stdcx. %3,0,%2\n" /* else store conditional */ - "bne- 1b\n" /* retry if lost reservation */ - "li %1,1\n" /* result = 1; */ - "2:\n" - : "=&r"(oldval), "=&r"(result) - : "r"(addr), "r"(new_val), "r"(old), "1"(result) - : "memory", "cr0"); -#else - __asm__ __volatile__( - "1:lwarx %0,0,%2\n" /* load and reserve */ - "cmpw %0, %4\n" /* if load is not equal to */ - "bne 2f\n" /* old, fail */ - "stwcx. %3,0,%2\n" /* else store conditional */ - "bne- 1b\n" /* retry if lost reservation */ - "li %1,1\n" /* result = 1; */ - "2:\n" - : "=&r"(oldval), "=&r"(result) - : "r"(addr), "r"(new_val), "r"(old), "1"(result) - : "memory", "cr0"); -#endif - return result; -} -#define AO_HAVE_compare_and_swap +#ifndef AO_GENERALIZE_ASM_BOOL_CAS -AO_INLINE int -AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val) { - int result = AO_compare_and_swap(addr, old, new_val); - AO_lwsync(); - return result; -} -#define AO_HAVE_compare_and_swap_acquire + AO_INLINE int + AO_compare_and_swap(volatile AO_t *addr, AO_t old, AO_t new_val) + { + AO_t oldval; + int result = 0; +# if defined(__powerpc64__) || defined(__ppc64__) || defined(__64BIT__) + /* FIXME: Completely untested. */ + __asm__ __volatile__( + "1:ldarx %0,0,%2\n" /* load and reserve */ + "cmpd %0, %4\n" /* if load is not equal to */ + "bne 2f\n" /* old, fail */ + "stdcx. %3,0,%2\n" /* else store conditional */ + "bne- 1b\n" /* retry if lost reservation */ + "li %1,1\n" /* result = 1; */ + "2:\n" + : "=&r"(oldval), "=&r"(result) + : "r"(addr), "r"(new_val), "r"(old), "1"(result) + : "memory", "cr0"); +# else + __asm__ __volatile__( + "1:lwarx %0,0,%2\n" /* load and reserve */ + "cmpw %0, %4\n" /* if load is not equal to */ + "bne 2f\n" /* old, fail */ + "stwcx. %3,0,%2\n" /* else store conditional */ + "bne- 1b\n" /* retry if lost reservation */ + "li %1,1\n" /* result = 1; */ + "2:\n" + : "=&r"(oldval), "=&r"(result) + : "r"(addr), "r"(new_val), "r"(old), "1"(result) + : "memory", "cr0"); +# endif + return result; + } +# define AO_HAVE_compare_and_swap -AO_INLINE int -AO_compare_and_swap_release(volatile AO_t *addr, AO_t old, AO_t new_val) { - AO_lwsync(); - return AO_compare_and_swap(addr, old, new_val); -} -#define AO_HAVE_compare_and_swap_release + AO_INLINE int + AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val) + { + int result = AO_compare_and_swap(addr, old, new_val); + AO_lwsync(); + return result; + } +# define AO_HAVE_compare_and_swap_acquire -AO_INLINE int -AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) { - int result; - AO_lwsync(); - result = AO_compare_and_swap(addr, old, new_val); - AO_lwsync(); - return result; -} -#define AO_HAVE_compare_and_swap_full + AO_INLINE int + AO_compare_and_swap_release(volatile AO_t *addr, AO_t old, AO_t new_val) + { + AO_lwsync(); + return AO_compare_and_swap(addr, old, new_val); + } +# define AO_HAVE_compare_and_swap_release + + AO_INLINE int + AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) + { + int result; + AO_lwsync(); + result = AO_compare_and_swap(addr, old, new_val); + AO_lwsync(); + return result; + } +# define AO_HAVE_compare_and_swap_full + +#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ AO_INLINE AO_t AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) { AO_t fetched_val; -#if defined(__powerpc64__) || defined(__ppc64__) || defined(__64BIT__) -/* FIXME: Completely untested. */ - __asm__ __volatile__( - "1:ldarx %0,0,%1\n" /* load and reserve */ - "cmpd %0, %3\n" /* if load is not equal to */ - "bne 2f\n" /* old_val, fail */ - "stdcx. %2,0,%1\n" /* else store conditional */ - "bne- 1b\n" /* retry if lost reservation */ - "2:\n" - : "=&r"(fetched_val), - : "r"(addr), "r"(new_val), "r"(old_val) - : "memory", "cr0"); -#else - __asm__ __volatile__( - "1:lwarx %0,0,%1\n" /* load and reserve */ - "cmpw %0, %3\n" /* if load is not equal to */ - "bne 2f\n" /* old_val, fail */ - "stwcx. %2,0,%1\n" /* else store conditional */ - "bne- 1b\n" /* retry if lost reservation */ - "2:\n" - : "=&r"(fetched_val), - : "r"(addr), "r"(new_val), "r"(old_val) - : "memory", "cr0"); -#endif + /* FIXME: Completely untested. */ +# if defined(__powerpc64__) || defined(__ppc64__) || defined(__64BIT__) + __asm__ __volatile__( + "1:ldarx %0,0,%1\n" /* load and reserve */ + "cmpd %0, %3\n" /* if load is not equal to */ + "bne 2f\n" /* old_val, fail */ + "stdcx. %2,0,%1\n" /* else store conditional */ + "bne- 1b\n" /* retry if lost reservation */ + "2:\n" + : "=&r"(fetched_val), + : "r"(addr), "r"(new_val), "r"(old_val) + : "memory", "cr0"); +# else + __asm__ __volatile__( + "1:lwarx %0,0,%1\n" /* load and reserve */ + "cmpw %0, %3\n" /* if load is not equal to */ + "bne 2f\n" /* old_val, fail */ + "stwcx. %2,0,%1\n" /* else store conditional */ + "bne- 1b\n" /* retry if lost reservation */ + "2:\n" + : "=&r"(fetched_val), + : "r"(addr), "r"(new_val), "r"(old_val) + : "memory", "cr0"); +# endif return fetched_val; } #define AO_HAVE_fetch_compare_and_swap diff --git a/src/atomic_ops/sysdeps/gcc/x86.h b/src/atomic_ops/sysdeps/gcc/x86.h index 44e815a..29aae1f 100644 --- a/src/atomic_ops/sysdeps/gcc/x86.h +++ b/src/atomic_ops/sysdeps/gcc/x86.h @@ -129,25 +129,28 @@ AO_test_and_set_full(volatile AO_TS_t *addr) } #define AO_HAVE_test_and_set_full -/* Returns nonzero if the comparison succeeded. */ -AO_INLINE int -AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) -{ -# ifdef AO_USE_SYNC_CAS_BUILTIN - return (int)__sync_bool_compare_and_swap(addr, old, new_val - /* empty protection list */); +#ifndef AO_GENERALIZE_ASM_BOOL_CAS + /* Returns nonzero if the comparison succeeded. */ + AO_INLINE int + AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) + { +# ifdef AO_USE_SYNC_CAS_BUILTIN + return (int)__sync_bool_compare_and_swap(addr, old, new_val + /* empty protection list */); /* Note: an empty list of variables protected by the */ /* memory barrier should mean all globally accessible */ /* variables are protected. */ -# else - char result; - __asm__ __volatile__("lock; cmpxchgl %3, %0; setz %1" - : "=m" (*addr), "=a" (result) - : "m" (*addr), "r" (new_val), "a" (old) : "memory"); - return (int)result; -# endif -} -#define AO_HAVE_compare_and_swap_full +# else + char result; + __asm__ __volatile__("lock; cmpxchgl %3, %0; setz %1" + : "=m" (*addr), "=a" (result) + : "m" (*addr), "r" (new_val), "a" (old) + : "memory"); + return (int)result; +# endif + } +# define AO_HAVE_compare_and_swap_full +#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ AO_INLINE AO_t AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val, diff --git a/src/atomic_ops/sysdeps/gcc/x86_64.h b/src/atomic_ops/sysdeps/gcc/x86_64.h index 7e869c1..062eeb5 100644 --- a/src/atomic_ops/sysdeps/gcc/x86_64.h +++ b/src/atomic_ops/sysdeps/gcc/x86_64.h @@ -126,22 +126,25 @@ AO_test_and_set_full(volatile AO_TS_t *addr) } #define AO_HAVE_test_and_set_full -/* Returns nonzero if the comparison succeeded. */ -AO_INLINE int -AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) -{ -# ifdef AO_USE_SYNC_CAS_BUILTIN - return (int)__sync_bool_compare_and_swap(addr, old, new_val - /* empty protection list */); -# else - char result; - __asm__ __volatile__("lock; cmpxchgq %3, %0; setz %1" - : "=m" (*addr), "=a" (result) - : "m" (*addr), "r" (new_val), "a" (old) : "memory"); - return (int) result; -# endif -} -#define AO_HAVE_compare_and_swap_full +#ifndef AO_GENERALIZE_ASM_BOOL_CAS + /* Returns nonzero if the comparison succeeded. */ + AO_INLINE int + AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) + { +# ifdef AO_USE_SYNC_CAS_BUILTIN + return (int)__sync_bool_compare_and_swap(addr, old, new_val + /* empty protection list */); +# else + char result; + __asm__ __volatile__("lock; cmpxchgq %3, %0; setz %1" + : "=m" (*addr), "=a" (result) + : "m" (*addr), "r" (new_val), "a" (old) + : "memory"); + return (int)result; +# endif + } +# define AO_HAVE_compare_and_swap_full +#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ AO_INLINE AO_t AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val, diff --git a/src/atomic_ops/sysdeps/sunc/x86.h b/src/atomic_ops/sysdeps/sunc/x86.h index 114e5fa..f76c944 100644 --- a/src/atomic_ops/sysdeps/sunc/x86.h +++ b/src/atomic_ops/sysdeps/sunc/x86.h @@ -130,17 +130,20 @@ AO_test_and_set_full (volatile AO_TS_t *addr) } #define AO_HAVE_test_and_set_full -/* Returns nonzero if the comparison succeeded. */ -AO_INLINE int -AO_compare_and_swap_full (volatile AO_t *addr, AO_t old, AO_t new_val) -{ - char result; - __asm__ __volatile__ ("lock; cmpxchgl %2, %0; setz %1" - : "=m"(*addr), "=a"(result) - : "r" (new_val), "a"(old) : "memory"); - return (int) result; -} -#define AO_HAVE_compare_and_swap_full +#ifndef AO_GENERALIZE_ASM_BOOL_CAS + /* Returns nonzero if the comparison succeeded. */ + AO_INLINE int + AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) + { + char result; + __asm__ __volatile__ ("lock; cmpxchgl %2, %0; setz %1" + : "=m"(*addr), "=a"(result) + : "r" (new_val), "a"(old) + : "memory"); + return (int) result; + } +# define AO_HAVE_compare_and_swap_full +#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ AO_INLINE AO_t AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val, diff --git a/src/atomic_ops/sysdeps/sunc/x86_64.h b/src/atomic_ops/sysdeps/sunc/x86_64.h index 0a718be..8162345 100644 --- a/src/atomic_ops/sysdeps/sunc/x86_64.h +++ b/src/atomic_ops/sysdeps/sunc/x86_64.h @@ -130,17 +130,20 @@ AO_test_and_set_full (volatile AO_TS_t *addr) } #define AO_HAVE_test_and_set_full -/* Returns nonzero if the comparison succeeded. */ -AO_INLINE int -AO_compare_and_swap_full (volatile AO_t *addr, AO_t old, AO_t new_val) -{ - char result; - __asm__ __volatile__ ("lock; cmpxchgq %2, %0; setz %1" - : "=m"(*addr), "=a"(result) - : "r" (new_val), "a"(old) : "memory"); - return (int) result; -} -#define AO_HAVE_compare_and_swap_full +#ifndef AO_GENERALIZE_ASM_BOOL_CAS + /* Returns nonzero if the comparison succeeded. */ + AO_INLINE int + AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) + { + char result; + __asm__ __volatile__ ("lock; cmpxchgq %2, %0; setz %1" + : "=m"(*addr), "=a"(result) + : "r" (new_val), "a"(old) + : "memory"); + return (int)result; + } +# define AO_HAVE_compare_and_swap_full +#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ AO_INLINE AO_t AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val, -- 2.49.0