Merge "libc: provide atomic operations will full barriers for NDK apps."
diff --git a/libc/Android.mk b/libc/Android.mk
index 410f242..207a133 100644
--- a/libc/Android.mk
+++ b/libc/Android.mk
@@ -346,10 +346,11 @@
arch-arm/bionic/__get_sp.S \
arch-arm/bionic/_exit_with_stack_teardown.S \
arch-arm/bionic/_setjmp.S \
- arch-arm/bionic/atomics_arm.S \
+ arch-arm/bionic/atomics_arm.c \
arch-arm/bionic/clone.S \
arch-arm/bionic/eabi.c \
arch-arm/bionic/ffs.S \
+ arch-arm/bionic/futex_arm.S \
arch-arm/bionic/kill.S \
arch-arm/bionic/libgcc_compat.c \
arch-arm/bionic/tkill.S \
@@ -395,9 +396,9 @@
arch-x86/bionic/__get_sp.S \
arch-x86/bionic/__get_tls.c \
arch-x86/bionic/__set_tls.c \
- arch-x86/bionic/atomics_x86.S \
arch-x86/bionic/clone.S \
arch-x86/bionic/_exit_with_stack_teardown.S \
+ arch-x86/bionic/futex_x86.S \
arch-x86/bionic/setjmp.S \
arch-x86/bionic/_setjmp.S \
arch-x86/bionic/sigsetjmp.S \
diff --git a/libc/arch-arm/bionic/atomics_arm.S b/libc/arch-arm/bionic/atomics_arm.S
deleted file mode 100644
index 4d9cbcf..0000000
--- a/libc/arch-arm/bionic/atomics_arm.S
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Copyright (C) 2008 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-#include <sys/linux-syscalls.h>
-#include <machine/asm.h>
-#include <machine/cpu-features.h>
-
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-
-#if defined(__ARM_HAVE_LDREX_STREX)
-/*
- * ===========================================================================
- * ARMv6+ implementation
- * ===========================================================================
- */
-
-/* r0(addr) -> r0(old) */
-ENTRY(__atomic_dec)
- mov r1, r0 @ copy addr so we don't clobber it
-1: ldrex r0, [r1] @ load current value into r0
- sub r2, r0, #1 @ generate new value into r2
- strex r3, r2, [r1] @ try to store new value; result in r3
- cmp r3, #0 @ success?
- bxeq lr @ yes, return
- b 1b @ no, retry
-END(__atomic_dec)
-
-/* r0(addr) -> r0(old) */
-ENTRY(__atomic_inc)
- mov r1, r0
-1: ldrex r0, [r1]
- add r2, r0, #1
- strex r3, r2, [r1]
- cmp r3, #0
- bxeq lr
- b 1b
-END(__atomic_inc)
-
-/* r0(old) r1(new) r2(addr) -> r0(zero_if_succeeded) */
-ENTRY(__atomic_cmpxchg)
-1: mov ip, #2 @ ip=2 means "new != old"
- ldrex r3, [r2] @ load current value into r3
- teq r0, r3 @ new == old?
- strexeq ip, r1, [r2] @ yes, try store, set ip to 0 or 1
- teq ip, #1 @ strex failure?
- beq 1b @ yes, retry
- mov r0, ip @ return 0 on success, 2 on failure
- bx lr
-END(__atomic_cmpxchg)
-
-/* r0(new) r1(addr) -> r0(old) */
-ENTRY(__atomic_swap)
-1: ldrex r2, [r1]
- strex r3, r0, [r1]
- teq r3, #0
- bne 1b
- mov r0, r2
- bx lr
-END(__atomic_swap)
-
-#else /*not defined __ARM_HAVE_LDREX_STREX*/
-/*
- * ===========================================================================
- * Pre-ARMv6 implementation
- * ===========================================================================
- */
-
- /* int __kernel_cmpxchg(int oldval, int newval, int* ptr) */
- .equ kernel_cmpxchg, 0xFFFF0FC0
- .equ kernel_atomic_base, 0xFFFF0FFF
-
-/* r0(addr) -> r0(old) */
-ENTRY(__atomic_dec)
- .save {r4, lr}
- stmdb sp!, {r4, lr}
- mov r2, r0
-1: @ atomic_dec
- ldr r0, [r2]
- mov r3, #kernel_atomic_base
- add lr, pc, #4
- sub r1, r0, #1
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
- bcc 1b
- add r0, r1, #1
- ldmia sp!, {r4, lr}
- bx lr
-END(__atomic_dec)
-
-/* r0(addr) -> r0(old) */
-ENTRY(__atomic_inc)
- .save {r4, lr}
- stmdb sp!, {r4, lr}
- mov r2, r0
-1: @ atomic_inc
- ldr r0, [r2]
- mov r3, #kernel_atomic_base
- add lr, pc, #4
- add r1, r0, #1
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
- bcc 1b
- sub r0, r1, #1
- ldmia sp!, {r4, lr}
- bx lr
-END(__atomic_inc)
-
-/* r0(old) r1(new) r2(addr) -> r0(zero_if_succeeded) */
-ENTRY(__atomic_cmpxchg)
- .save {r4, lr}
- stmdb sp!, {r4, lr}
- mov r4, r0 /* r4 = save oldvalue */
-1: @ atomic_cmpxchg
- mov r3, #kernel_atomic_base
- add lr, pc, #4
- mov r0, r4 /* r0 = oldvalue */
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
- bcs 2f /* swap was made. we're good, return. */
- ldr r3, [r2] /* swap not made, see if it's because *ptr!=oldvalue */
- cmp r3, r4
- beq 1b
-2: @ atomic_cmpxchg
- ldmia sp!, {r4, lr}
- bx lr
-END(__atomic_cmpxchg)
-
-/* r0(new) r1(addr) -> r0(old) */
-ENTRY(__atomic_swap)
- swp r0, r0, [r1]
- bx lr
-END(__atomic_swap)
-
-#endif /*not defined __ARM_HAVE_LDREX_STREX*/
-
-
-/* __futex_wait(*ftx, val, *timespec) */
-/* __futex_wake(*ftx, counter) */
-/* __futex_syscall3(*ftx, op, val) */
-/* __futex_syscall4(*ftx, op, val, *timespec) */
-
-.global __futex_wait
-.type __futex_wait, %function
-
-.global __futex_wake
-.type __futex_wake, %function
-
-.global __futex_syscall3
-.type __futex_syscall3, %function
-
-.global __futex_syscall4
-.type __futex_syscall4, %function
-
-#if __ARM_EABI__
-
-ENTRY(__futex_syscall3)
- stmdb sp!, {r4, r7}
- .save {r4, r7}
- ldr r7, =__NR_futex
- swi #0
- ldmia sp!, {r4, r7}
- bx lr
-END(__futex_syscall3)
-
-ENTRY(__futex_wait)
- stmdb sp!, {r4, r7}
- .save {r4, r7}
- mov r3, r2
- mov r2, r1
- mov r1, #FUTEX_WAIT
- ldr r7, =__NR_futex
- swi #0
- ldmia sp!, {r4, r7}
- bx lr
-END(__futex_wait)
-
-ENTRY(__futex_wake)
- .save {r4, r7}
- stmdb sp!, {r4, r7}
- mov r2, r1
- mov r1, #FUTEX_WAKE
- ldr r7, =__NR_futex
- swi #0
- ldmia sp!, {r4, r7}
- bx lr
-END(__futex_wake)
-
-#else
-
-ENTRY(__futex_syscall3)
- swi #__NR_futex
- bx lr
-END(__futex_syscall3)
-
-ENTRY(__futex_wait)
- mov r3, r2
- mov r2, r1
- mov r1, #FUTEX_WAIT
- swi #__NR_futex
- bx lr
-END(__futex_wait)
-
-ENTRY(__futex_wake)
- mov r2, r1
- mov r1, #FUTEX_WAKE
- swi #__NR_futex
- bx lr
-END(__futex_wake)
-
-#endif
-
-ENTRY(__futex_syscall4)
- b __futex_syscall3
-END(__futex_syscall4)
diff --git a/libc/arch-arm/bionic/atomics_arm.c b/libc/arch-arm/bionic/atomics_arm.c
new file mode 100644
index 0000000..1d7a1e1
--- /dev/null
+++ b/libc/arch-arm/bionic/atomics_arm.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+/* The purpose of this file is to export a small set of atomic-related
+ * functions from the C library, to ensure binary ABI compatibility for
+ * the NDK.
+ *
+ * These functions were initially exposed by the NDK through <sys/atomics.h>,
+ * which was unfortunate because their implementation didn't provide any
+ * memory barriers at all.
+ *
+ * This wasn't a problem for the platform code that used them, because it
+ * used explicit barrier instructions around them. On the other hand, it means
+ * that any NDK-generated machine code that linked against them would not
+ * perform correctly when running on multi-core devices.
+ *
+ * To fix this, the platform code was first modified to not use any of these
+ * functions (everything is now inlined through assembly statements, see
+ * libc/private/bionic_arm_inline.h and the headers it includes.
+ *
+ * The functions here are thus only for the benefit of NDK applications,
+ * and now includes full memory barriers to prevent any random memory ordering
+ * issue from cropping.
+ *
+ * Note that we also provide an updated <sys/atomics.h> header that defines
+ * always_inlined versions of the functions that use the GCC builtin
+ * intrinsics to perform the same thing.
+ *
+ * NOTE: There is no need for a similar file for non-ARM platforms.
+ */
+
+/* DO NOT INCLUDE <sys/atomics.h> HERE ! */
+
+int
+__android_cmpxchg(int old, int _new, volatile int *ptr)
+{
+ /* We must return 0 on success */
+ return __sync_val_compare_and_swap(ptr, old, _new) != old;
+}
+
+int
+__atomic_swap(int _new, volatile int *ptr)
+{
+ int prev;
+ do {
+ prev = *ptr;
+ } while (__sync_val_compare_and_swap(ptr, prev, _new) != prev);
+ return prev;
+}
+
+int
+__atomic_dec(volatile int *ptr)
+{
+ return __sync_fetch_and_sub (ptr, 1);
+}
+
+int
+__atomic_inc(volatile int *ptr)
+{
+ return __sync_fetch_and_add (ptr, 1);
+}
diff --git a/libc/arch-arm/bionic/futex_arm.S b/libc/arch-arm/bionic/futex_arm.S
new file mode 100644
index 0000000..7041663
--- /dev/null
+++ b/libc/arch-arm/bionic/futex_arm.S
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include <sys/linux-syscalls.h>
+#include <machine/asm.h>
+#include <machine/cpu-features.h>
+
+#define FUTEX_WAIT 0
+#define FUTEX_WAKE 1
+
+/* __futex_wait(*ftx, val, *timespec) */
+/* __futex_wake(*ftx, counter) */
+/* __futex_syscall3(*ftx, op, val) */
+/* __futex_syscall4(*ftx, op, val, *timespec) */
+
+.global __futex_wait
+.type __futex_wait, %function
+
+.global __futex_wake
+.type __futex_wake, %function
+
+.global __futex_syscall3
+.type __futex_syscall3, %function
+
+.global __futex_syscall4
+.type __futex_syscall4, %function
+
+#if __ARM_EABI__
+
+ENTRY(__futex_syscall3)
+ stmdb sp!, {r4, r7}
+ .save {r4, r7}
+ ldr r7, =__NR_futex
+ swi #0
+ ldmia sp!, {r4, r7}
+ bx lr
+END(__futex_syscall3)
+
+ENTRY(__futex_wait)
+ stmdb sp!, {r4, r7}
+ .save {r4, r7}
+ mov r3, r2
+ mov r2, r1
+ mov r1, #FUTEX_WAIT
+ ldr r7, =__NR_futex
+ swi #0
+ ldmia sp!, {r4, r7}
+ bx lr
+END(__futex_wait)
+
+ENTRY(__futex_wake)
+ .save {r4, r7}
+ stmdb sp!, {r4, r7}
+ mov r2, r1
+ mov r1, #FUTEX_WAKE
+ ldr r7, =__NR_futex
+ swi #0
+ ldmia sp!, {r4, r7}
+ bx lr
+END(__futex_wake)
+
+#else
+
+ENTRY(__futex_syscall3)
+ swi #__NR_futex
+ bx lr
+END(__futex_syscall3)
+
+ENTRY(__futex_wait)
+ mov r3, r2
+ mov r2, r1
+ mov r1, #FUTEX_WAIT
+ swi #__NR_futex
+ bx lr
+END(__futex_wait)
+
+ENTRY(__futex_wake)
+ mov r2, r1
+ mov r1, #FUTEX_WAKE
+ swi #__NR_futex
+ bx lr
+END(__futex_wake)
+
+#endif
+
+ENTRY(__futex_syscall4)
+ b __futex_syscall3
+END(__futex_syscall4)
diff --git a/libc/arch-x86/bionic/atomics_x86.S b/libc/arch-x86/bionic/futex_x86.S
similarity index 100%
rename from libc/arch-x86/bionic/atomics_x86.S
rename to libc/arch-x86/bionic/futex_x86.S
diff --git a/libc/arch-x86/include/sys/atomics.h b/libc/arch-x86/include/sys/atomics.h
deleted file mode 100644
index 7aed3ae..0000000
--- a/libc/arch-x86/include/sys/atomics.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-#ifndef _SYS_ATOMICS_H
-#define _SYS_ATOMICS_H
-
-#include <sys/cdefs.h>
-#include <sys/time.h>
-
-__BEGIN_DECLS
-
-static inline __attribute__((always_inline)) int
-__atomic_cmpxchg(int old, int _new, volatile int *ptr)
-{
- return !__sync_bool_compare_and_swap (ptr, old, _new);
-}
-
-static inline __attribute__((always_inline)) int
-__atomic_swap(int _new, volatile int *ptr)
-{
- return __sync_lock_test_and_set(ptr, _new);
-}
-
-static inline __attribute__((always_inline)) int
-__atomic_dec(volatile int *ptr)
-{
- return __sync_fetch_and_sub (ptr, 1);
-}
-
-static inline __attribute__((always_inline)) int
-__atomic_inc(volatile int *ptr)
-{
- return __sync_fetch_and_add (ptr, 1);
-}
-
-int __futex_wait(volatile void *ftx, int val, const struct timespec *timeout);
-int __futex_wake(volatile void *ftx, int count);
-
-__END_DECLS
-
-#endif /* _SYS_ATOMICS_H */
diff --git a/libc/include/sys/atomics.h b/libc/include/sys/atomics.h
index d3fa145..3ada8de 100644
--- a/libc/include/sys/atomics.h
+++ b/libc/include/sys/atomics.h
@@ -33,10 +33,48 @@
__BEGIN_DECLS
-extern int __atomic_cmpxchg(int old, int _new, volatile int *ptr);
-extern int __atomic_swap(int _new, volatile int *ptr);
-extern int __atomic_dec(volatile int *ptr);
-extern int __atomic_inc(volatile int *ptr);
+/* Note: atomic operations that were exported by the C library didn't
+ * provide any memory barriers, which created potential issues on
+ * multi-core devices. We now define them as inlined calls to
+ * GCC sync builtins, which always provide a full barrier.
+ *
+ * NOTE: The C library still exports atomic functions by the same
+ * name to ensure ABI stability for existing NDK machine code.
+ *
+ * If you are an NDK developer, we encourage you to rebuild your
+ * unmodified sources against this header as soon as possible.
+ */
+#define __ATOMIC_INLINE__ static __inline__ __attribute__((always_inline))
+
+__ATOMIC_INLINE__ int
+__atomic_cmpxchg(int old, int _new, volatile int *ptr)
+{
+ /* We must return 0 on success */
+ return __sync_val_compare_and_swap(ptr, old, _new) != old;
+}
+
+__ATOMIC_INLINE__ int
+__atomic_swap(int _new, volatile int *ptr)
+{
+ int prev;
+ do {
+ prev = *ptr;
+ } while (__sync_val_compare_and_swap(ptr, prev, _new) != prev);
+ return prev;
+}
+
+__ATOMIC_INLINE__ int
+__atomic_dec(volatile int *ptr)
+{
+ return __sync_fetch_and_sub (ptr, 1);
+}
+
+__ATOMIC_INLINE__ int
+__atomic_inc(volatile int *ptr)
+{
+ return __sync_fetch_and_add (ptr, 1);
+}
+
int __futex_wait(volatile void *ftx, int val, const struct timespec *timeout);
int __futex_wake(volatile void *ftx, int count);