Implement arm64 TLSDESC
Each TLSDESC relocation relocates a 2-word descriptor in the GOT that
contains:
- the address of a TLS resolver function
- an argument to pass (indirectly) to the resolver function
(Specifically, the address of the 2-word descriptor is passed to the
resolver.)
The loader resolves R_GENERIC_TLSDESC relocations using one of three
resolver functions that it defines:
- tlsdesc_resolver_static
- tlsdesc_resolver_dynamic
- tlsdesc_resolver_unresolved_weak
The resolver functions are written in assembly because they have a
restrictive calling convention. They're only allowed to modify x0 and
(apparently) the condition codes.
For a relocation to memory in static TLS (i.e. the executable or an solib
loaded initially), the loader uses a simple resolver function,
tlsdesc_resolver_static, that returns the static offset it receives from
the loader.
For relocations to dynamic TLS memory (i.e. memory in a dlopen'ed solib),
the loader uses tlsdesc_resolver_dynamic, which allocates TLS memory on
demand. It inlines the fast path of __tls_get_addr, then falls back to
__tls_get_addr when it needs to allocate memory. The loader handles these
dynamic TLS relocations in two passes:
- In the first pass, it allocates a table of TlsDynamicResolverArg
objects, one per dynamic TLSDESC relocation.
- In the second pass, once the table is finalized, it writes the
addresses of the TlsDynamicResolverArg objects into the TLSDESC
relocations.
tlsdesc_resolver_unresolved_weak returns a negated thread pointer so that
taking the address of an unresolved weak TLS symbols produces NULL.
The loader handles R_GENERIC_TLSDESC in a target-independent way, but
only for arm64, because Bionic has only implemented the resolver functions
for arm64.
Bug: http://b/78026329
Test: bionic unit tests
Test: check that backtrace works inside a resolver function and inside
__tls_get_addr called from a resolver
(gdbclient.py, b __tls_get_addr, bt)
Merged-In: I752e59ff986292449892c449dad2546e6f0ff7b6
Change-Id: I752e59ff986292449892c449dad2546e6f0ff7b6
diff --git a/linker/arch/arm64/tlsdesc_resolver.S b/linker/arch/arm64/tlsdesc_resolver.S
new file mode 100644
index 0000000..ef46839
--- /dev/null
+++ b/linker/arch/arm64/tlsdesc_resolver.S
@@ -0,0 +1,203 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <private/bionic_asm.h>
+#include <private/bionic_asm_tls.h>
+
+.globl __tls_get_addr
+
+// These resolver functions must preserve every register except x0. They set x0
+// to the offset of the TLS symbol relative to the thread pointer.
+
+ENTRY_PRIVATE(tlsdesc_resolver_static)
+ ldr x0, [x0, #8]
+ ret
+END(tlsdesc_resolver_static)
+
+ENTRY_PRIVATE(tlsdesc_resolver_dynamic)
+ stp x19, x20, [sp, #-32]!
+ .cfi_def_cfa_offset 32
+ .cfi_rel_offset x19, 0
+ .cfi_rel_offset x20, 8
+ stp x21, x22, [sp, #16]
+ .cfi_rel_offset x21, 16
+ .cfi_rel_offset x22, 24
+
+ mrs x19, tpidr_el0 // __get_tls()
+ ldr x20, [x19, #(TLS_SLOT_DTV * 8)]
+ ldr x21, [x20] // TlsDtv::generation
+
+ ldr x0, [x0, #8] // TlsDynamicResolverArg*
+ ldr x22, [x0] // TlsDynamicResolverArg::generation
+
+ cmp x21, x22
+ b.lo .fallback
+
+ ldr x21, [x0, #8] // TlsIndex::module
+ ldr x22, [x0, #16] // TlsIndex::offset
+ ldr x21, [x20, x21, lsl #3] // TlsDtv::modules[module]
+ cbz x21, .fallback
+ add x0, x21, x22
+ sub x0, x0, x19
+
+ ldp x21, x22, [sp, #16]
+ .cfi_remember_state
+ .cfi_restore x21
+ .cfi_restore x22
+ ldp x19, x20, [sp], #32
+ .cfi_adjust_cfa_offset -32
+ .cfi_restore x19
+ .cfi_restore x20
+ ret
+
+.fallback:
+ .cfi_restore_state
+ ldp x21, x22, [sp, #16]
+ .cfi_restore x21
+ .cfi_restore x22
+ ldp x19, x20, [sp], #32
+ .cfi_adjust_cfa_offset -32
+ .cfi_restore x19
+ .cfi_restore x20
+ b tlsdesc_resolver_dynamic_slow_path
+END(tlsdesc_resolver_dynamic)
+
+#define SAVE_REG(x, slot) \
+ str x, [sp, #((slot) * 8)]; \
+ .cfi_rel_offset x, (slot) * 8; \
+
+#define SAVE_GPR_PAIR(x, y, slot) \
+ stp x, y, [sp, #((slot) * 8)]; \
+ .cfi_rel_offset x, (slot) * 8; \
+ .cfi_rel_offset y, ((slot) + 1) * 8; \
+
+#define SAVE_VEC_PAIR(x, y, slot) \
+ stp x, y, [sp, #((slot) * 8)]; \
+ .cfi_rel_offset x, (slot) * 8; \
+ .cfi_rel_offset y, ((slot) + 2) * 8; \
+
+#define RESTORE_REG(x, slot) \
+ ldr x, [sp, #((slot) * 8)]; \
+ .cfi_restore x; \
+
+#define RESTORE_REG_PAIR(x, y, slot) \
+ ldp x, y, [sp, #((slot) * 8)]; \
+ .cfi_restore x; \
+ .cfi_restore y; \
+
+// On entry, x0 is the address of a TlsDynamicResolverArg object rather than
+// the TlsDescriptor address passed to the original resolver function.
+ENTRY_PRIVATE(tlsdesc_resolver_dynamic_slow_path)
+ sub sp, sp, #(8 * 84)
+ .cfi_def_cfa_offset (8 * 84)
+ SAVE_GPR_PAIR(x29, x30, 0)
+ mov x29, sp
+
+ // Avoid leaking the contents of the shadow call stack register (x18) into
+ // memory. x19 through x29 are callee-save registers, so we do not need to
+ // save them.
+ SAVE_GPR_PAIR(x1, x2, 2)
+ SAVE_GPR_PAIR(x3, x4, 4)
+ SAVE_GPR_PAIR(x5, x6, 6)
+ SAVE_GPR_PAIR(x7, x8, 8)
+ SAVE_GPR_PAIR(x9, x10, 10)
+ SAVE_GPR_PAIR(x11, x12, 12)
+ SAVE_GPR_PAIR(x13, x14, 14)
+ SAVE_GPR_PAIR(x15, x16, 16)
+ SAVE_REG(x17, 18)
+
+ SAVE_VEC_PAIR(q0, q1, 20)
+ SAVE_VEC_PAIR(q2, q3, 24)
+ SAVE_VEC_PAIR(q4, q5, 28)
+ SAVE_VEC_PAIR(q6, q7, 32)
+ SAVE_VEC_PAIR(q8, q9, 36)
+ SAVE_VEC_PAIR(q10, q11, 40)
+ SAVE_VEC_PAIR(q12, q13, 44)
+ SAVE_VEC_PAIR(q14, q15, 48)
+ SAVE_VEC_PAIR(q16, q17, 52)
+ SAVE_VEC_PAIR(q18, q19, 56)
+ SAVE_VEC_PAIR(q20, q21, 60)
+ SAVE_VEC_PAIR(q22, q23, 64)
+ SAVE_VEC_PAIR(q24, q25, 68)
+ SAVE_VEC_PAIR(q26, q27, 72)
+ SAVE_VEC_PAIR(q28, q29, 76)
+ SAVE_VEC_PAIR(q30, q31, 80)
+
+ add x0, x0, #8
+ bl __tls_get_addr
+ mrs x1, tpidr_el0 // __get_tls()
+ sub x0, x0, x1
+
+ RESTORE_REG_PAIR(q30, q31, 80)
+ RESTORE_REG_PAIR(q28, q29, 76)
+ RESTORE_REG_PAIR(q26, q27, 72)
+ RESTORE_REG_PAIR(q24, q25, 68)
+ RESTORE_REG_PAIR(q22, q23, 64)
+ RESTORE_REG_PAIR(q20, q21, 60)
+ RESTORE_REG_PAIR(q18, q19, 56)
+ RESTORE_REG_PAIR(q16, q17, 52)
+ RESTORE_REG_PAIR(q14, q15, 48)
+ RESTORE_REG_PAIR(q12, q13, 44)
+ RESTORE_REG_PAIR(q10, q11, 40)
+ RESTORE_REG_PAIR(q8, q9, 36)
+ RESTORE_REG_PAIR(q6, q7, 32)
+ RESTORE_REG_PAIR(q4, q5, 28)
+ RESTORE_REG_PAIR(q2, q3, 24)
+ RESTORE_REG_PAIR(q0, q1, 20)
+
+ RESTORE_REG(x17, 18)
+ RESTORE_REG_PAIR(x15, x16, 16)
+ RESTORE_REG_PAIR(x13, x14, 14)
+ RESTORE_REG_PAIR(x11, x12, 12)
+ RESTORE_REG_PAIR(x9, x10, 10)
+ RESTORE_REG_PAIR(x7, x8, 8)
+ RESTORE_REG_PAIR(x5, x6, 6)
+ RESTORE_REG_PAIR(x3, x4, 4)
+ RESTORE_REG_PAIR(x1, x2, 2)
+
+ RESTORE_REG_PAIR(x29, x30, 0)
+ add sp, sp, #(8 * 84)
+ .cfi_def_cfa_offset 0
+ ret
+END(tlsdesc_resolver_dynamic_slow_path)
+
+// The address of an unresolved weak TLS symbol evaluates to NULL with TLSDESC.
+// The value returned by this function is added to the thread pointer, so return
+// a negated thread pointer to cancel it out.
+ENTRY_PRIVATE(tlsdesc_resolver_unresolved_weak)
+ str x19, [sp, #-16]!
+ .cfi_def_cfa_offset 16
+ .cfi_rel_offset x19, 0
+ ldr x19, [x0, #8]
+ mrs x0, tpidr_el0 // __get_tls()
+ sub x0, x19, x0
+ ldr x19, [sp], #16
+ .cfi_def_cfa_offset 0
+ .cfi_restore x19
+ ret
+END(tlsdesc_resolver_unresolved_weak)