libcutils/atomic-android-arm.S

/*
 * Copyright (C) 2005 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <machine/cpu-features.h>

    .text
    .align

    .global android_atomic_acquire_load
    .type android_atomic_acquire_load, %function
    .global android_atomic_release_load
    .type android_atomic_release_load, %function

    .global android_atomic_acquire_store
    .type android_atomic_acquire_store, %function
    .global android_atomic_release_store
    .type android_atomic_release_store, %function

    .global android_atomic_inc
    .type android_atomic_inc, %function
    .global android_atomic_dec
    .type android_atomic_dec, %function

    .global android_atomic_add
    .type android_atomic_add, %function
    .global android_atomic_and
    .type android_atomic_and, %function
    .global android_atomic_or
    .type android_atomic_or, %function

    .global android_atomic_release_swap
    .type android_atomic_release_swap, %function
    .global android_atomic_acquire_swap
    .type android_atomic_acquire_swap, %function

    .global android_atomic_release_cas
    .type android_atomic_release_cas, %function
    .global android_atomic_acquire_cas
    .type android_atomic_acquire_cas, %function

/* must be on or off; cannot be left undefined */
#if !defined(ANDROID_SMP)
# error "ANDROID_SMP not defined"
#endif


#if defined(__ARM_HAVE_LDREX_STREX)
/*
 * ===========================================================================
 *      ARMv6+ implementation
 * ===========================================================================
 *
 * These functions use the LDREX/STREX instructions to perform atomic
 * operations ("LL/SC" approach).  On an SMP build they will include
 * an appropriate memory barrier.
 */

/* generate the memory barrier instruction when the build requires it */
#if ANDROID_SMP == 1
# if defined(__ARM_HAVE_DMB)
#  define SMP_DMB dmb
# else
   /* Data Memory Barrier operation, initated by writing a value into a
      specific register with the Move to Coprocessor instruction.  We
      arbitrarily use r0 here. */
#  define SMP_DMB mcr p15, 0, r0, c7, c10, 5
# endif
#else
# define SMP_DMB
#endif

/*
 * Sidebar: do we need to use the -EX instructions for atomic load/store?
 *
 * Consider the following situation (time advancing downward):
 *
 * P1                  P2
 *  val = LDREX(mem)
 *  val = val + 1
 *                      STR(mem, otherval)
 *  STREX(mem, val)
 *
 * If these instructions issue on separate cores, the STREX will correctly
 * fail because of the intervening store from the other core.  If this same
 * sequence of instructions executes in two threads on the same core, the
 * STREX will incorrectly succeed.
 *
 * There are two ways to fix this:
 * (1) Use LDREX/STREX for the atomic store operations.  This doesn't
 *   prevent the program from doing a non-exclusive store, but at least
 *   this way if they always use atomic ops to access the memory location
 *   there won't be any problems.
 * (2) Have the kernel clear the LDREX reservation on thread context switch.
 *  This will sometimes clear the reservation unnecessarily, but guarantees
 *  correct behavior.
 *
 * The Android kernel performs a CLREX (v7) or dummy STREX (pre-v7), so we
 * can get away with a non-exclusive store here.
 *
 * -----
 *
 * It's worth noting that using non-exclusive LDR and STR means the "load"
 * and "store" operations aren't quite the same as read-modify-write or
 * swap operations.  By definition those must read and write memory in a
 * in a way that is coherent across all cores, whereas our non-exclusive
 * load and store have no such requirement.
 *
 * In practice this doesn't matter, because the only guarantees we make
 * about who sees what when are tied to the acquire/release semantics.
 * Other cores may not see our atomic releasing store as soon as they would
 * if the code used LDREX/STREX, but a store-release operation doesn't make
 * any guarantees as to how soon the store will be visible.  It's allowable
 * for operations that happen later in program order to become visible
 * before the store.  For an acquring store we issue a full barrier after
 * the STREX, ensuring that other processors see events in the proper order.
 */

/*
 * android_atomic_acquire_load / android_atomic_release_load
 * input: r0 = address
 * output: r0 = value
 */
android_atomic_acquire_load:
    .fnstart
    ldr     r0, [r0]
    SMP_DMB
    bx      lr
    .fnend

android_atomic_release_load:
    .fnstart
    SMP_DMB
    ldr     r0, [r0]
    bx      lr
    .fnend


/*
 * android_atomic_acquire_store / android_atomic_release_store
 * input: r0 = value, r1 = address
 * output: void
 */
android_atomic_acquire_store:
    .fnstart
    str     r0, [r1]
    SMP_DMB
    bx      lr
    .fnend

android_atomic_release_store:
    .fnstart
    SMP_DMB
    str     r0, [r1]
    bx      lr
    .fnend

/*
 * Common sequence for read-modify-write operations.
 *
 * input: r1 = address
 * output: r0 = original value, returns to caller
 */
    .macro  RMWEX   op, arg
1:  ldrex   r0, [r1]                    @ load current value into r0
    \op     r2, r0, \arg                @ generate new value into r2
    strex   r3, r2, [r1]                @ try to store new value; result in r3
    cmp     r3, #0                      @ success?
    bxeq    lr                          @ yes, return
    b       1b                          @ no, retry
    .endm


/*
 * android_atomic_inc
 * input: r0 = address
 * output: r0 = old value
 */
android_atomic_inc:
    .fnstart
    SMP_DMB
    mov     r1, r0
    RMWEX   add, #1
    .fnend


/*
 * android_atomic_dec
 * input: r0 = address
 * output: r0 = old value
 */
android_atomic_dec:
    .fnstart
    SMP_DMB
    mov     r1, r0
    RMWEX   sub, #1
    .fnend


/*
 * android_atomic_add
 * input: r0 = value, r1 = address
 * output: r0 = old value
 */
android_atomic_add:
    .fnstart
    SMP_DMB
    mov     ip, r0
    RMWEX   add, ip
    .fnend


/*
 * android_atomic_and
 * input: r0 = value, r1 = address
 * output: r0 = old value
 */
android_atomic_and:
    .fnstart
    SMP_DMB
    mov     ip, r0
    RMWEX   and, ip
    .fnend


/*
 * android_atomic_or
 * input: r0 = value, r1 = address
 * output: r0 = old value
 */
android_atomic_or:
    .fnstart
    SMP_DMB
    mov     ip, r0
    RMWEX   orr, ip
    .fnend


/*
 * android_atomic_acquire_swap / android_atomic_release_swap
 * input: r0 = value, r1 = address
 * output: r0 = old value
 */
android_atomic_acquire_swap:
    .fnstart
1:  ldrex   r2, [r1]                    @ load current value into r2
    strex   r3, r0, [r1]                @ store new value
    teq     r3, #0                      @ strex success?
    bne     1b                          @ no, loop
    mov     r0, r2                      @ return old value
    SMP_DMB
    bx      lr
    .fnend

android_atomic_release_swap:
    .fnstart
    SMP_DMB
1:  ldrex   r2, [r1]
    strex   r3, r0, [r1]
    teq     r3, #0
    bne     1b
    mov     r0, r2
    bx      lr
    .fnend


/*
 * android_atomic_acquire_cas / android_atomic_release_cas
 * input: r0 = oldvalue, r1 = newvalue, r2 = address
 * output: r0 = 0 (xchg done) or non-zero (xchg not done)
 */
android_atomic_acquire_cas:
    .fnstart
1:  mov     ip, #2                      @ ip=2 means "new != old"
    ldrex   r3, [r2]                    @ load current value into r3
    teq     r0, r3                      @ new == old?
    strexeq ip, r1, [r2]                @ yes, try store, set ip to 0 or 1
    teq     ip, #1                      @ strex failure?
    beq     1b                          @ yes, retry
    mov     r0, ip                      @ return 0 on success, 2 on failure
    SMP_DMB
    bx      lr
    .fnend

android_atomic_release_cas:
    .fnstart
    SMP_DMB
1:  mov     ip, #2
    ldrex   r3, [r2]
    teq     r0, r3
    strexeq ip, r1, [r2]
    teq     ip, #1
    beq     1b
    mov     r0, ip
    bx      lr
    .fnend


#else /*not defined __ARM_HAVE_LDREX_STREX*/
/*
 * ===========================================================================
 *      Pre-ARMv6 implementation
 * ===========================================================================
 *
 * These functions call through the kernel cmpxchg facility, or use the
 * (now deprecated) SWP instruction.  They are not SMP-safe.
 */
#if ANDROID_SMP == 1
# error "SMP defined, but LDREX/STREX not available"
#endif

/*
 * int __kernel_cmpxchg(int oldval, int newval, int *ptr)
 * clobbered: r3, ip, flags
 * return 0 if a swap was made, non-zero otherwise.
 */ 
   .equ     kernel_cmpxchg, 0xFFFF0FC0
   .equ     kernel_atomic_base, 0xFFFF0FFF


/*
 * android_atomic_acquire_load / android_atomic_release_load
 * input: r0 = address
 * output: r0 = value
 */
android_atomic_acquire_load:
android_atomic_release_load:
    .fnstart
    ldr     r0, [r0]
    bx      lr
    .fnend


/*
 * android_atomic_acquire_store / android_atomic_release_store
 * input: r0 = value, r1 = address
 * output: void
 */
android_atomic_acquire_store:
android_atomic_release_store:
    .fnstart
    str     r0, [r1]
    bx      lr
    .fnend


/*
 * android_atomic_inc
 * input: r0 = address
 * output: r0 = old value
 */
android_atomic_inc:
    .fnstart
    .save {r4, lr}
    stmdb   sp!, {r4, lr}
    mov     r2, r0
1: @ android_atomic_inc
    ldr     r0, [r2]
    mov     r3, #kernel_atomic_base
#ifdef __ARM_HAVE_PC_INTERWORK
    add     lr, pc, #4
    add     r1, r0, #1
    add     pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
#else
    add     r1, r0, #1
    add     r3, r3, #(kernel_cmpxchg - kernel_atomic_base)
    mov     lr, pc
    bx      r3
#endif
    bcc     1b
    sub     r0, r1, #1
    ldmia   sp!, {r4, lr}
    bx      lr
    .fnend


/*
 * android_atomic_dec
 * input: r0 = address
 * output: r0 = old value
 */
android_atomic_dec:
    .fnstart
    .save {r4, lr}
    stmdb   sp!, {r4, lr}
    mov     r2, r0
1: @ android_atomic_dec
    ldr     r0, [r2]
    mov     r3, #kernel_atomic_base
#ifdef __ARM_HAVE_PC_INTERWORK
    add     lr, pc, #4
    sub     r1, r0, #1
    add     pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
#else
    sub     r1, r0, #1
    add     r3, r3, #(kernel_cmpxchg - kernel_atomic_base)
    mov     lr, pc
    bx      r3
#endif
    bcc     1b
    add     r0, r1, #1
    ldmia   sp!, {r4, lr}
    bx      lr
    .fnend


/*
 * android_atomic_add
 * input: r0 = value, r1 = address
 * output: r0 = old value
 */
android_atomic_add:
    .fnstart
    .save {r4, lr}
    stmdb   sp!, {r4, lr}
    mov     r2, r1
    mov     r4, r0
1: @ android_atomic_add
    ldr     r0, [r2]
    mov     r3, #kernel_atomic_base
#ifdef __ARM_HAVE_PC_INTERWORK
    add     lr, pc, #4
    add     r1, r0, r4
    add     pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
#else
    add     r1, r0, r4
    add     r3, r3, #(kernel_cmpxchg - kernel_atomic_base)
    mov     lr, pc
    bx      r3
#endif
    bcc     1b
    sub     r0, r1, r4
    ldmia   sp!, {r4, lr}
    bx      lr
    .fnend


/*
 * android_atomic_and
 * input: r0 = value, r1 = address
 * output: r0 = old value
 */
android_atomic_and:
    .fnstart
    .save {r4, r5, ip, lr}      /* include ip for 64-bit stack alignment */
    stmdb   sp!, {r4, r5, ip, lr}
    mov     r2, r1              /* r2 = address */
    mov     r4, r0              /* r4 = the value */
1: @ android_atomic_and
    ldr     r0, [r2]            /* r0 = address[0] */
    mov     r3, #kernel_atomic_base
#ifdef __ARM_HAVE_PC_INTERWORK
    add     lr, pc, #8
    mov     r5, r0              /* r5 = save address[0] */
    and     r1, r0, r4          /* r1 = new value */
    add     pc, r3, #(kernel_cmpxchg - kernel_atomic_base)  /* call cmpxchg() */
#else
    mov     r5, r0              /* r5 = save address[0] */
    and     r1, r0, r4          /* r1 = new value */
    add     r3, r3, #(kernel_cmpxchg - kernel_atomic_base)  /* call cmpxchg() */
    mov     lr, pc
    bx      r3
#endif
    bcc     1b
    mov     r0, r5
    ldmia   sp!, {r4, r5, ip, lr}
    bx      lr
    .fnend


/*
 * android_atomic_or
 * input: r0 = value, r1 = address
 * output: r0 = old value
 */
android_atomic_or:
    .fnstart
    .save {r4, r5, ip, lr}      /* include ip for 64-bit stack alignment */
    stmdb   sp!, {r4, r5, ip, lr}
    mov     r2, r1              /* r2 = address */
    mov     r4, r0              /* r4 = the value */
1: @ android_atomic_or
    ldr     r0, [r2]            /* r0 = address[0] */
    mov     r3, #kernel_atomic_base
#ifdef __ARM_HAVE_PC_INTERWORK
    add     lr, pc, #8
    mov     r5, r0              /* r5 = save address[0] */
    orr     r1, r0, r4          /* r1 = new value */
    add     pc, r3, #(kernel_cmpxchg - kernel_atomic_base)  /* call cmpxchg() */
#else
    mov     r5, r0              /* r5 = save address[0] */
    orr     r1, r0, r4          /* r1 = new value */
    add     r3, r3, #(kernel_cmpxchg - kernel_atomic_base)  /* call cmpxchg() */
    mov     lr, pc
    bx      r3
#endif
    bcc     1b
    mov     r0, r5
    ldmia   sp!, {r4, r5, ip, lr}
    bx      lr
    .fnend


/*
 * android_atomic_acquire_swap / android_atomic_release_swap
 * input: r0 = value, r1 = address
 * output: r0 = old value
 */
android_atomic_acquire_swap:
android_atomic_release_swap:
    .fnstart
    swp     r0, r0, [r1]
    bx      lr
    .fnend


/*
 * android_atomic_acquire_cas / android_atomic_release_cas
 * input: r0 = oldvalue, r1 = newvalue, r2 = address
 * output: r0 = 0 (xchg done) or non-zero (xchg not done)
 */
android_atomic_acquire_cas:
android_atomic_release_cas:
    .fnstart
    .save {r4, lr}
    stmdb   sp!, {r4, lr}
    mov     r4, r0          /* r4 = save oldvalue */
1: @ android_atomic_cmpxchg
    mov     r3, #kernel_atomic_base
#ifdef __ARM_HAVE_PC_INTERWORK
    add     lr, pc, #4
    mov     r0, r4          /* r0 = oldvalue */
    add     pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
#else
    mov     r0, r4          /* r0 = oldvalue */
    add     r3, r3, #(kernel_cmpxchg - kernel_atomic_base)
    mov     lr, pc
    bx      r3
#endif
    bcs     2f              /* swap was made. we're good, return. */
    ldr     r3, [r2]        /* swap not made, see if it's because *ptr!=oldvalue */
    cmp     r3, r4
    beq     1b
2: @ android_atomic_cmpxchg
    ldmia   sp!, {r4, lr}
    bx      lr
    .fnend

#endif /*not defined __ARM_HAVE_LDREX_STREX*/