Define inline atomic operations for x86 and ARM.
This change moves the ARM definitions into GCC extended inline
assembler. In addition, the same set of x86 definitions are now
shared among all x86 targets.
Change-Id: I6e5aa3a413d0af2acbe5d32994983d35a01fdcb3
diff --git a/libcutils/Android.mk b/libcutils/Android.mk
index 5b05a1e..778b5bd 100644
--- a/libcutils/Android.mk
+++ b/libcutils/Android.mk
@@ -26,7 +26,7 @@
commonSources := \
array.c \
hashmap.c \
- atomic.c \
+ atomic.c.arm \
native_handle.c \
buffer.c \
socket_inaddr_any_server.c \
@@ -112,7 +112,7 @@
LOCAL_SRC_FILES := $(commonSources) ashmem-dev.c mq.c
ifeq ($(TARGET_ARCH),arm)
-LOCAL_SRC_FILES += memset32.S atomic-android-arm.S
+LOCAL_SRC_FILES += memset32.S
else # !arm
ifeq ($(TARGET_ARCH),sh)
LOCAL_SRC_FILES += memory.c atomic-android-sh.c
diff --git a/libcutils/atomic-android-arm.S b/libcutils/atomic-android-arm.S
deleted file mode 100644
index d8ee15c..0000000
--- a/libcutils/atomic-android-arm.S
+++ /dev/null
@@ -1,561 +0,0 @@
-/*
- * Copyright (C) 2005 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <machine/cpu-features.h>
-
- .text
- .align
-
- .global android_atomic_acquire_load
- .type android_atomic_acquire_load, %function
- .global android_atomic_release_load
- .type android_atomic_release_load, %function
-
- .global android_atomic_acquire_store
- .type android_atomic_acquire_store, %function
- .global android_atomic_release_store
- .type android_atomic_release_store, %function
-
- .global android_atomic_inc
- .type android_atomic_inc, %function
- .global android_atomic_dec
- .type android_atomic_dec, %function
-
- .global android_atomic_add
- .type android_atomic_add, %function
- .global android_atomic_and
- .type android_atomic_and, %function
- .global android_atomic_or
- .type android_atomic_or, %function
-
- .global android_atomic_release_swap
- .type android_atomic_release_swap, %function
- .global android_atomic_acquire_swap
- .type android_atomic_acquire_swap, %function
-
- .global android_atomic_release_cas
- .type android_atomic_release_cas, %function
- .global android_atomic_acquire_cas
- .type android_atomic_acquire_cas, %function
-
-/* must be on or off; cannot be left undefined */
-#if !defined(ANDROID_SMP)
-# error "ANDROID_SMP not defined"
-#endif
-
-
-#if defined(__ARM_HAVE_LDREX_STREX)
-/*
- * ===========================================================================
- * ARMv6+ implementation
- * ===========================================================================
- *
- * These functions use the LDREX/STREX instructions to perform atomic
- * operations ("LL/SC" approach). On an SMP build they will include
- * an appropriate memory barrier.
- */
-
-/* generate the memory barrier instruction when the build requires it */
-#if ANDROID_SMP == 1
-# if defined(__ARM_HAVE_DMB)
-# define SMP_DMB dmb
-# else
- /* Data Memory Barrier operation, initated by writing a value into a
- specific register with the Move to Coprocessor instruction. We
- arbitrarily use r0 here. */
-# define SMP_DMB mcr p15, 0, r0, c7, c10, 5
-# endif
-#else
-# define SMP_DMB
-#endif
-
-/*
- * Sidebar: do we need to use the -EX instructions for atomic load/store?
- *
- * Consider the following situation (time advancing downward):
- *
- * P1 P2
- * val = LDREX(mem)
- * val = val + 1
- * STR(mem, otherval)
- * STREX(mem, val)
- *
- * If these instructions issue on separate cores, the STREX will correctly
- * fail because of the intervening store from the other core. If this same
- * sequence of instructions executes in two threads on the same core, the
- * STREX will incorrectly succeed.
- *
- * There are two ways to fix this:
- * (1) Use LDREX/STREX for the atomic store operations. This doesn't
- * prevent the program from doing a non-exclusive store, but at least
- * this way if they always use atomic ops to access the memory location
- * there won't be any problems.
- * (2) Have the kernel clear the LDREX reservation on thread context switch.
- * This will sometimes clear the reservation unnecessarily, but guarantees
- * correct behavior.
- *
- * The Android kernel performs a CLREX (v7) or dummy STREX (pre-v7), so we
- * can get away with a non-exclusive store here.
- *
- * -----
- *
- * It's worth noting that using non-exclusive LDR and STR means the "load"
- * and "store" operations aren't quite the same as read-modify-write or
- * swap operations. By definition those must read and write memory in a
- * in a way that is coherent across all cores, whereas our non-exclusive
- * load and store have no such requirement.
- *
- * In practice this doesn't matter, because the only guarantees we make
- * about who sees what when are tied to the acquire/release semantics.
- * Other cores may not see our atomic releasing store as soon as they would
- * if the code used LDREX/STREX, but a store-release operation doesn't make
- * any guarantees as to how soon the store will be visible. It's allowable
- * for operations that happen later in program order to become visible
- * before the store. For an acquring store we issue a full barrier after
- * the STREX, ensuring that other processors see events in the proper order.
- */
-
-/*
- * android_atomic_acquire_load / android_atomic_release_load
- * input: r0 = address
- * output: r0 = value
- */
-android_atomic_acquire_load:
- .fnstart
- ldr r0, [r0]
- SMP_DMB
- bx lr
- .fnend
-
-android_atomic_release_load:
- .fnstart
- SMP_DMB
- ldr r0, [r0]
- bx lr
- .fnend
-
-
-/*
- * android_atomic_acquire_store / android_atomic_release_store
- * input: r0 = value, r1 = address
- * output: void
- */
-android_atomic_acquire_store:
- .fnstart
- str r0, [r1]
- SMP_DMB
- bx lr
- .fnend
-
-android_atomic_release_store:
- .fnstart
- SMP_DMB
- str r0, [r1]
- bx lr
- .fnend
-
-/*
- * Common sequence for read-modify-write operations.
- *
- * input: r1 = address
- * output: r0 = original value, returns to caller
- */
- .macro RMWEX op, arg
-1: ldrex r0, [r1] @ load current value into r0
- \op r2, r0, \arg @ generate new value into r2
- strex r3, r2, [r1] @ try to store new value; result in r3
- cmp r3, #0 @ success?
- bxeq lr @ yes, return
- b 1b @ no, retry
- .endm
-
-
-/*
- * android_atomic_inc
- * input: r0 = address
- * output: r0 = old value
- */
-android_atomic_inc:
- .fnstart
- SMP_DMB
- mov r1, r0
- RMWEX add, #1
- .fnend
-
-
-/*
- * android_atomic_dec
- * input: r0 = address
- * output: r0 = old value
- */
-android_atomic_dec:
- .fnstart
- SMP_DMB
- mov r1, r0
- RMWEX sub, #1
- .fnend
-
-
-/*
- * android_atomic_add
- * input: r0 = value, r1 = address
- * output: r0 = old value
- */
-android_atomic_add:
- .fnstart
- SMP_DMB
- mov ip, r0
- RMWEX add, ip
- .fnend
-
-
-/*
- * android_atomic_and
- * input: r0 = value, r1 = address
- * output: r0 = old value
- */
-android_atomic_and:
- .fnstart
- SMP_DMB
- mov ip, r0
- RMWEX and, ip
- .fnend
-
-
-/*
- * android_atomic_or
- * input: r0 = value, r1 = address
- * output: r0 = old value
- */
-android_atomic_or:
- .fnstart
- SMP_DMB
- mov ip, r0
- RMWEX orr, ip
- .fnend
-
-
-/*
- * android_atomic_acquire_swap / android_atomic_release_swap
- * input: r0 = value, r1 = address
- * output: r0 = old value
- */
-android_atomic_acquire_swap:
- .fnstart
-1: ldrex r2, [r1] @ load current value into r2
- strex r3, r0, [r1] @ store new value
- teq r3, #0 @ strex success?
- bne 1b @ no, loop
- mov r0, r2 @ return old value
- SMP_DMB
- bx lr
- .fnend
-
-android_atomic_release_swap:
- .fnstart
- SMP_DMB
-1: ldrex r2, [r1]
- strex r3, r0, [r1]
- teq r3, #0
- bne 1b
- mov r0, r2
- bx lr
- .fnend
-
-
-/*
- * android_atomic_acquire_cas / android_atomic_release_cas
- * input: r0 = oldvalue, r1 = newvalue, r2 = address
- * output: r0 = 0 (xchg done) or non-zero (xchg not done)
- */
-android_atomic_acquire_cas:
- .fnstart
-1: mov ip, #2 @ ip=2 means "new != old"
- ldrex r3, [r2] @ load current value into r3
- teq r0, r3 @ new == old?
- strexeq ip, r1, [r2] @ yes, try store, set ip to 0 or 1
- teq ip, #1 @ strex failure?
- beq 1b @ yes, retry
- mov r0, ip @ return 0 on success, 2 on failure
- SMP_DMB
- bx lr
- .fnend
-
-android_atomic_release_cas:
- .fnstart
- SMP_DMB
-1: mov ip, #2
- ldrex r3, [r2]
- teq r0, r3
- strexeq ip, r1, [r2]
- teq ip, #1
- beq 1b
- mov r0, ip
- bx lr
- .fnend
-
-
-#else /*not defined __ARM_HAVE_LDREX_STREX*/
-/*
- * ===========================================================================
- * Pre-ARMv6 implementation
- * ===========================================================================
- *
- * These functions call through the kernel cmpxchg facility, or use the
- * (now deprecated) SWP instruction. They are not SMP-safe.
- */
-#if ANDROID_SMP == 1
-# error "SMP defined, but LDREX/STREX not available"
-#endif
-
-/*
- * int __kernel_cmpxchg(int oldval, int newval, int *ptr)
- * clobbered: r3, ip, flags
- * return 0 if a swap was made, non-zero otherwise.
- */
- .equ kernel_cmpxchg, 0xFFFF0FC0
- .equ kernel_atomic_base, 0xFFFF0FFF
-
-
-/*
- * android_atomic_acquire_load / android_atomic_release_load
- * input: r0 = address
- * output: r0 = value
- */
-android_atomic_acquire_load:
-android_atomic_release_load:
- .fnstart
- ldr r0, [r0]
- bx lr
- .fnend
-
-
-/*
- * android_atomic_acquire_store / android_atomic_release_store
- * input: r0 = value, r1 = address
- * output: void
- */
-android_atomic_acquire_store:
-android_atomic_release_store:
- .fnstart
- str r0, [r1]
- bx lr
- .fnend
-
-
-/*
- * android_atomic_inc
- * input: r0 = address
- * output: r0 = old value
- */
-android_atomic_inc:
- .fnstart
- .save {r4, lr}
- stmdb sp!, {r4, lr}
- mov r2, r0
-1: @ android_atomic_inc
- ldr r0, [r2]
- mov r3, #kernel_atomic_base
-#ifdef __ARM_HAVE_PC_INTERWORK
- add lr, pc, #4
- add r1, r0, #1
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
-#else
- add r1, r0, #1
- add r3, r3, #(kernel_cmpxchg - kernel_atomic_base)
- mov lr, pc
- bx r3
-#endif
- bcc 1b
- sub r0, r1, #1
- ldmia sp!, {r4, lr}
- bx lr
- .fnend
-
-
-/*
- * android_atomic_dec
- * input: r0 = address
- * output: r0 = old value
- */
-android_atomic_dec:
- .fnstart
- .save {r4, lr}
- stmdb sp!, {r4, lr}
- mov r2, r0
-1: @ android_atomic_dec
- ldr r0, [r2]
- mov r3, #kernel_atomic_base
-#ifdef __ARM_HAVE_PC_INTERWORK
- add lr, pc, #4
- sub r1, r0, #1
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
-#else
- sub r1, r0, #1
- add r3, r3, #(kernel_cmpxchg - kernel_atomic_base)
- mov lr, pc
- bx r3
-#endif
- bcc 1b
- add r0, r1, #1
- ldmia sp!, {r4, lr}
- bx lr
- .fnend
-
-
-/*
- * android_atomic_add
- * input: r0 = value, r1 = address
- * output: r0 = old value
- */
-android_atomic_add:
- .fnstart
- .save {r4, lr}
- stmdb sp!, {r4, lr}
- mov r2, r1
- mov r4, r0
-1: @ android_atomic_add
- ldr r0, [r2]
- mov r3, #kernel_atomic_base
-#ifdef __ARM_HAVE_PC_INTERWORK
- add lr, pc, #4
- add r1, r0, r4
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
-#else
- add r1, r0, r4
- add r3, r3, #(kernel_cmpxchg - kernel_atomic_base)
- mov lr, pc
- bx r3
-#endif
- bcc 1b
- sub r0, r1, r4
- ldmia sp!, {r4, lr}
- bx lr
- .fnend
-
-
-/*
- * android_atomic_and
- * input: r0 = value, r1 = address
- * output: r0 = old value
- */
-android_atomic_and:
- .fnstart
- .save {r4, r5, ip, lr} /* include ip for 64-bit stack alignment */
- stmdb sp!, {r4, r5, ip, lr}
- mov r2, r1 /* r2 = address */
- mov r4, r0 /* r4 = the value */
-1: @ android_atomic_and
- ldr r0, [r2] /* r0 = address[0] */
- mov r3, #kernel_atomic_base
-#ifdef __ARM_HAVE_PC_INTERWORK
- add lr, pc, #8
- mov r5, r0 /* r5 = save address[0] */
- and r1, r0, r4 /* r1 = new value */
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base) /* call cmpxchg() */
-#else
- mov r5, r0 /* r5 = save address[0] */
- and r1, r0, r4 /* r1 = new value */
- add r3, r3, #(kernel_cmpxchg - kernel_atomic_base) /* call cmpxchg() */
- mov lr, pc
- bx r3
-#endif
- bcc 1b
- mov r0, r5
- ldmia sp!, {r4, r5, ip, lr}
- bx lr
- .fnend
-
-
-/*
- * android_atomic_or
- * input: r0 = value, r1 = address
- * output: r0 = old value
- */
-android_atomic_or:
- .fnstart
- .save {r4, r5, ip, lr} /* include ip for 64-bit stack alignment */
- stmdb sp!, {r4, r5, ip, lr}
- mov r2, r1 /* r2 = address */
- mov r4, r0 /* r4 = the value */
-1: @ android_atomic_or
- ldr r0, [r2] /* r0 = address[0] */
- mov r3, #kernel_atomic_base
-#ifdef __ARM_HAVE_PC_INTERWORK
- add lr, pc, #8
- mov r5, r0 /* r5 = save address[0] */
- orr r1, r0, r4 /* r1 = new value */
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base) /* call cmpxchg() */
-#else
- mov r5, r0 /* r5 = save address[0] */
- orr r1, r0, r4 /* r1 = new value */
- add r3, r3, #(kernel_cmpxchg - kernel_atomic_base) /* call cmpxchg() */
- mov lr, pc
- bx r3
-#endif
- bcc 1b
- mov r0, r5
- ldmia sp!, {r4, r5, ip, lr}
- bx lr
- .fnend
-
-
-/*
- * android_atomic_acquire_swap / android_atomic_release_swap
- * input: r0 = value, r1 = address
- * output: r0 = old value
- */
-android_atomic_acquire_swap:
-android_atomic_release_swap:
- .fnstart
- swp r0, r0, [r1]
- bx lr
- .fnend
-
-
-/*
- * android_atomic_acquire_cas / android_atomic_release_cas
- * input: r0 = oldvalue, r1 = newvalue, r2 = address
- * output: r0 = 0 (xchg done) or non-zero (xchg not done)
- */
-android_atomic_acquire_cas:
-android_atomic_release_cas:
- .fnstart
- .save {r4, lr}
- stmdb sp!, {r4, lr}
- mov r4, r0 /* r4 = save oldvalue */
-1: @ android_atomic_cmpxchg
- mov r3, #kernel_atomic_base
-#ifdef __ARM_HAVE_PC_INTERWORK
- add lr, pc, #4
- mov r0, r4 /* r0 = oldvalue */
- add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
-#else
- mov r0, r4 /* r0 = oldvalue */
- add r3, r3, #(kernel_cmpxchg - kernel_atomic_base)
- mov lr, pc
- bx r3
-#endif
- bcs 2f /* swap was made. we're good, return. */
- ldr r3, [r2] /* swap not made, see if it's because *ptr!=oldvalue */
- cmp r3, r4
- beq 1b
-2: @ android_atomic_cmpxchg
- ldmia sp!, {r4, lr}
- bx lr
- .fnend
-
-#endif /*not defined __ARM_HAVE_LDREX_STREX*/
diff --git a/libcutils/atomic.c b/libcutils/atomic.c
index 4cefa6b..f6cd8b0 100644
--- a/libcutils/atomic.c
+++ b/libcutils/atomic.c
@@ -14,207 +14,6 @@
* limitations under the License.
*/
-#include <cutils/atomic.h>
+#define inline
+
#include <cutils/atomic-inline.h>
-#ifdef HAVE_WIN32_THREADS
-#include <windows.h>
-#else
-#include <sched.h>
-#endif
-
-/*****************************************************************************/
-#if defined(HAVE_MACOSX_IPC)
-
-#include <libkern/OSAtomic.h>
-
-int32_t android_atomic_acquire_load(volatile int32_t* addr) {
- int32_t value = *addr;
- OSMemoryBarrier();
- return value;
-}
-
-int32_t android_atomic_release_load(volatile int32_t* addr) {
- OSMemoryBarrier();
- return *addr;
-}
-
-void android_atomic_acquire_store(int32_t value, volatile int32_t* addr) {
- *addr = value;
- OSMemoryBarrier();
-}
-
-void android_atomic_release_store(int32_t value, volatile int32_t* addr) {
- OSMemoryBarrier();
- *addr = value;
-}
-
-int32_t android_atomic_inc(volatile int32_t* addr) {
- return OSAtomicIncrement32Barrier((int32_t*)addr)-1;
-}
-
-int32_t android_atomic_dec(volatile int32_t* addr) {
- return OSAtomicDecrement32Barrier((int32_t*)addr)+1;
-}
-
-int32_t android_atomic_add(int32_t value, volatile int32_t* addr) {
- return OSAtomicAdd32Barrier(value, (int32_t*)addr)-value;
-}
-
-int32_t android_atomic_and(int32_t value, volatile int32_t* addr) {
- return OSAtomicAnd32OrigBarrier(value, (int32_t*)addr);
-}
-
-int32_t android_atomic_or(int32_t value, volatile int32_t* addr) {
- return OSAtomicOr32OrigBarrier(value, (int32_t*)addr);
-}
-
-int32_t android_atomic_acquire_swap(int32_t value, volatile int32_t* addr) {
- int32_t oldValue;
- do {
- oldValue = *addr;
- } while (android_atomic_acquire_cas(oldValue, value, addr));
- return oldValue;
-}
-
-int32_t android_atomic_release_swap(int32_t value, volatile int32_t* addr) {
- int32_t oldValue;
- do {
- oldValue = *addr;
- } while (android_atomic_release_cas(oldValue, value, addr));
- return oldValue;
-}
-
-int android_atomic_release_cas(int32_t oldvalue, int32_t newvalue, volatile int32_t* addr) {
- /* OS X CAS returns zero on failure; invert to return zero on success */
- return OSAtomicCompareAndSwap32Barrier(oldvalue, newvalue, (int32_t*)addr) == 0;
-}
-
-int android_atomic_acquire_cas(int32_t oldvalue, int32_t newvalue,
- volatile int32_t* addr) {
- int result = (OSAtomicCompareAndSwap32(oldvalue, newvalue, (int32_t*)addr) == 0);
- if (result == 0) {
- /* success, perform barrier */
- OSMemoryBarrier();
- }
- return result;
-}
-
-/*****************************************************************************/
-#elif defined(__i386__) || defined(__x86_64__)
-
-int32_t android_atomic_acquire_load(volatile int32_t* addr) {
- int32_t value = *addr;
- ANDROID_MEMBAR_FULL();
- return value;
-}
-
-int32_t android_atomic_release_load(volatile int32_t* addr) {
- ANDROID_MEMBAR_FULL();
- return *addr;
-}
-
-void android_atomic_acquire_store(int32_t value, volatile int32_t* addr) {
- *addr = value;
- ANDROID_MEMBAR_FULL();
-}
-
-void android_atomic_release_store(int32_t value, volatile int32_t* addr) {
- ANDROID_MEMBAR_FULL();
- *addr = value;
-}
-
-int32_t android_atomic_inc(volatile int32_t* addr) {
- return android_atomic_add(1, addr);
-}
-
-int32_t android_atomic_dec(volatile int32_t* addr) {
- return android_atomic_add(-1, addr);
-}
-
-int32_t android_atomic_add(int32_t value, volatile int32_t* addr) {
- int32_t oldValue;
- do {
- oldValue = *addr;
- } while (android_atomic_release_cas(oldValue, oldValue+value, addr));
- return oldValue;
-}
-
-int32_t android_atomic_and(int32_t value, volatile int32_t* addr) {
- int32_t oldValue;
- do {
- oldValue = *addr;
- } while (android_atomic_release_cas(oldValue, oldValue&value, addr));
- return oldValue;
-}
-
-int32_t android_atomic_or(int32_t value, volatile int32_t* addr) {
- int32_t oldValue;
- do {
- oldValue = *addr;
- } while (android_atomic_release_cas(oldValue, oldValue|value, addr));
- return oldValue;
-}
-
-/* returns 0 on successful swap */
-static inline int cas(int32_t oldvalue, int32_t newvalue,
- volatile int32_t* addr) {
- int xchg;
- asm volatile
- (
- " lock; cmpxchg %%ecx, (%%edx);"
- " setne %%al;"
- " andl $1, %%eax"
- : "=a" (xchg)
- : "a" (oldvalue), "c" (newvalue), "d" (addr)
- );
- return xchg;
-}
-
-int32_t android_atomic_acquire_swap(int32_t value, volatile int32_t* addr) {
- int32_t oldValue;
- do {
- oldValue = *addr;
- } while (cas(oldValue, value, addr));
- ANDROID_MEMBAR_FULL();
- return oldValue;
-}
-
-int32_t android_atomic_release_swap(int32_t value, volatile int32_t* addr) {
- ANDROID_MEMBAR_FULL();
- int32_t oldValue;
- do {
- oldValue = *addr;
- } while (cas(oldValue, value, addr));
- return oldValue;
-}
-
-int android_atomic_acquire_cas(int32_t oldvalue, int32_t newvalue,
- volatile int32_t* addr) {
- int xchg = cas(oldvalue, newvalue, addr);
- if (xchg == 0)
- ANDROID_MEMBAR_FULL();
- return xchg;
-}
-
-int android_atomic_release_cas(int32_t oldvalue, int32_t newvalue,
- volatile int32_t* addr) {
- ANDROID_MEMBAR_FULL();
- int xchg = cas(oldvalue, newvalue, addr);
- return xchg;
-}
-
-
-/*****************************************************************************/
-#elif __arm__
-// implementation for ARM is in atomic-android-arm.s.
-
-/*****************************************************************************/
-#elif __sh__
-// implementation for SuperH is in atomic-android-sh.c.
-
-#else
-
-#error "Unsupported atomic operations for this platform"
-
-#endif
-