blob: fa1af768e811f76e1aba6d44389c1faad0a2b6ee [file] [log] [blame]
Ryan Prichard45d13492019-01-03 02:51:30 -08001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#pragma once
30
Ryan Prichard48097552019-01-06 18:24:10 -080031#include <link.h>
Ryan Pricharde5e69e02019-01-01 18:53:48 -080032#include <pthread.h>
33#include <stdatomic.h>
Ryan Prichard45d13492019-01-03 02:51:30 -080034#include <stdint.h>
Ryan Prichard48097552019-01-06 18:24:10 -080035#include <sys/cdefs.h>
36
Ryan Prichard16455b52019-01-18 01:00:59 -080037__LIBC_HIDDEN__ extern _Atomic(size_t) __libc_tls_generation_copy;
38
Ryan Prichard48097552019-01-06 18:24:10 -080039struct TlsSegment {
40 size_t size = 0;
41 size_t alignment = 1;
42 const void* init_ptr = ""; // Field is non-null even when init_size is 0.
43 size_t init_size = 0;
44};
45
46__LIBC_HIDDEN__ bool __bionic_get_tls_segment(const ElfW(Phdr)* phdr_table, size_t phdr_count,
Ryan Prichard19883502019-01-16 23:13:38 -080047 ElfW(Addr) load_bias, TlsSegment* out);
48
49__LIBC_HIDDEN__ bool __bionic_check_tls_alignment(size_t* alignment);
Ryan Prichard45d13492019-01-03 02:51:30 -080050
51struct StaticTlsLayout {
52 constexpr StaticTlsLayout() {}
53
54private:
55 size_t offset_ = 0;
56 size_t alignment_ = 1;
57 bool overflowed_ = false;
58
59 // Offsets to various Bionic TLS structs from the beginning of static TLS.
60 size_t offset_bionic_tcb_ = SIZE_MAX;
61 size_t offset_bionic_tls_ = SIZE_MAX;
62
63public:
64 size_t offset_bionic_tcb() const { return offset_bionic_tcb_; }
65 size_t offset_bionic_tls() const { return offset_bionic_tls_; }
Ryan Prichardfb8730d2019-01-15 00:11:37 -080066 size_t offset_thread_pointer() const;
Ryan Prichard45d13492019-01-03 02:51:30 -080067
68 size_t size() const { return offset_; }
69 size_t alignment() const { return alignment_; }
70 bool overflowed() const { return overflowed_; }
71
Ryan Prichard977e47d2019-01-14 21:52:14 -080072 size_t reserve_exe_segment_and_tcb(const TlsSegment* exe_segment, const char* progname);
Ryan Prichard45d13492019-01-03 02:51:30 -080073 void reserve_bionic_tls();
Ryan Prichard977e47d2019-01-14 21:52:14 -080074 size_t reserve_solib_segment(const TlsSegment& segment) {
75 return reserve(segment.size, segment.alignment);
76 }
Ryan Prichard45d13492019-01-03 02:51:30 -080077 void finish_layout();
78
79private:
80 size_t reserve(size_t size, size_t alignment);
81
82 template <typename T> size_t reserve_type() {
83 return reserve(sizeof(T), alignof(T));
84 }
85
86 size_t round_up_with_overflow_check(size_t value, size_t alignment);
87};
Ryan Pricharde5e69e02019-01-01 18:53:48 -080088
Ryan Prichard16455b52019-01-18 01:00:59 -080089static constexpr size_t kTlsGenerationNone = 0;
90static constexpr size_t kTlsGenerationFirst = 1;
91
92// The first ELF TLS module has ID 1. Zero is reserved for the first word of
93// the DTV, a generation count. Unresolved weak symbols also use module ID 0.
94static constexpr size_t kTlsUninitializedModuleId = 0;
95
96static inline size_t __tls_module_id_to_idx(size_t id) { return id - 1; }
97static inline size_t __tls_module_idx_to_id(size_t idx) { return idx + 1; }
98
Ryan Pricharde5e69e02019-01-01 18:53:48 -080099// A descriptor for a single ELF TLS module.
100struct TlsModule {
101 TlsSegment segment;
102
103 // Offset into the static TLS block or SIZE_MAX for a dynamic module.
104 size_t static_offset = SIZE_MAX;
105
106 // The generation in which this module was loaded. Dynamic TLS lookups use
107 // this field to detect when a module has been unloaded.
Ryan Prichard16455b52019-01-18 01:00:59 -0800108 size_t first_generation = kTlsGenerationNone;
Ryan Pricharde5e69e02019-01-01 18:53:48 -0800109
110 // Used by the dynamic linker to track the associated soinfo* object.
111 void* soinfo_ptr = nullptr;
112};
113
114// Table of the ELF TLS modules. Either the dynamic linker or the static
115// initialization code prepares this table, and it's then used during thread
116// creation and for dynamic TLS lookups.
117struct TlsModules {
118 constexpr TlsModules() {}
119
Ryan Prichard16455b52019-01-18 01:00:59 -0800120 // A pointer to the TLS generation counter in libc.so. The counter is
121 // incremented each time an solib is loaded or unloaded.
122 _Atomic(size_t) generation = kTlsGenerationFirst;
123 _Atomic(size_t) *generation_libc_so = nullptr;
Ryan Pricharde5e69e02019-01-01 18:53:48 -0800124
125 // Access to the TlsModule[] table requires taking this lock.
126 pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER;
127
128 // Pointer to a block of TlsModule objects. The first module has ID 1 and
129 // is stored at index 0 in this table.
130 size_t module_count = 0;
131 TlsModule* module_table = nullptr;
132};
Ryan Prichard361c1b42019-01-15 13:45:27 -0800133
134void __init_static_tls(void* static_tls);
Ryan Prichard16455b52019-01-18 01:00:59 -0800135
136// Dynamic Thread Vector. Each thread has a different DTV. For each module
137// (executable or solib), the DTV has a pointer to that module's TLS memory. The
138// DTV is initially empty and is allocated on-demand. It grows as more modules
139// are dlopen'ed. See https://www.akkadia.org/drepper/tls.pdf.
140//
141// The layout of the DTV is specified in various documents, but it is not part
142// of Bionic's public ABI. A compiler can't generate code to access it directly,
143// because it can't access libc's global generation counter.
144struct TlsDtv {
145 // Number of elements in this object's modules field.
146 size_t count;
147
148 // A pointer to an older TlsDtv object that should be freed when the thread
149 // exits. The objects aren't immediately freed because a DTV could be
150 // reallocated by a signal handler that interrupted __tls_get_addr's fast
151 // path.
152 TlsDtv* next;
153
154 // The DTV slot points at this field, which allows omitting an add instruction
155 // on the fast path for a TLS lookup. The arm64 tlsdesc_resolver.S depends on
156 // the layout of fields past this point.
157 size_t generation;
158 void* modules[];
159};
160
161struct TlsIndex {
162 size_t module_id;
163 size_t offset;
164};
165
166#if defined(__i386__)
167#define TLS_GET_ADDR_CCONV __attribute__((regparm(1)))
168#define TLS_GET_ADDR ___tls_get_addr
169#else
170#define TLS_GET_ADDR_CCONV
171#define TLS_GET_ADDR __tls_get_addr
172#endif
173
174extern "C" void* TLS_GET_ADDR(const TlsIndex* ti) TLS_GET_ADDR_CCONV;
175
176struct bionic_tcb;
177void __free_dynamic_tls(bionic_tcb* tcb);