Record TLS modules and layout static TLS memory
Bug: http://b/78026329
Test: bionic unit tests
Change-Id: Ibf1bf5ec864c7830e4cd1cb882842b644e6182ae
diff --git a/linker/linker.cpp b/linker/linker.cpp
index 1f259e1..38f15c3 100644
--- a/linker/linker.cpp
+++ b/linker/linker.cpp
@@ -65,8 +65,10 @@
#include "linker_phdr.h"
#include "linker_relocs.h"
#include "linker_reloc_iterators.h"
+#include "linker_tls.h"
#include "linker_utils.h"
+#include "private/bionic_globals.h"
#include "android-base/macros.h"
#include "android-base/strings.h"
#include "android-base/stringprintf.h"
@@ -1652,6 +1654,7 @@
if (!si->is_linked() && !si->prelink_image()) {
return false;
}
+ register_soinfo_tls(si);
}
// Step 4: Construct the global group. Note: DF_1_GLOBAL bit of a library is
@@ -1887,6 +1890,7 @@
si->get_realpath(),
si);
notify_gdb_of_unload(si);
+ unregister_soinfo_tls(si);
get_cfi_shadow()->BeforeUnload(si);
soinfo_free(si);
}
@@ -3073,6 +3077,12 @@
&ARM_exidx, &ARM_exidx_count);
#endif
+ TlsSegment tls_segment;
+ if (__bionic_get_tls_segment(phdr, phnum, load_bias, get_realpath(), &tls_segment)) {
+ tls_ = std::make_unique<soinfo_tls>();
+ tls_->segment = tls_segment;
+ }
+
// Extract useful information from dynamic section.
// Note that: "Except for the DT_NULL element at the end of the array,
// and the relative order of DT_NEEDED elements, entries may appear in any order."
diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp
index 9b4ce47..b0c27dc 100644
--- a/linker/linker_main.cpp
+++ b/linker/linker_main.cpp
@@ -415,6 +415,8 @@
}
}
+ linker_setup_exe_static_tls(g_argv[0]);
+
// Load ld_preloads and dependencies.
std::vector<const char*> needed_library_name_list;
size_t ld_preloads_count = 0;
@@ -452,8 +454,7 @@
si->increment_ref_count();
}
- layout_linker_static_tls();
-
+ linker_finalize_static_tls();
__libc_init_main_thread_final();
if (!get_cfi_shadow()->InitialLinkDone(solist)) __linker_cannot_link(g_argv[0]);
diff --git a/linker/linker_soinfo.cpp b/linker/linker_soinfo.cpp
index 93079ca..dcc6bf3 100644
--- a/linker/linker_soinfo.cpp
+++ b/linker/linker_soinfo.cpp
@@ -628,6 +628,10 @@
return secondary_namespaces_;
}
+soinfo_tls* soinfo::get_tls() const {
+ return has_min_version(5) ? tls_.get() : nullptr;
+}
+
ElfW(Addr) soinfo::resolve_symbol_address(const ElfW(Sym)* s) const {
if (ELF_ST_TYPE(s->st_info) == STT_GNU_IFUNC) {
return call_ifunc_resolver(s->st_value + load_bias);
diff --git a/linker/linker_soinfo.h b/linker/linker_soinfo.h
index 44bff28..87e385d 100644
--- a/linker/linker_soinfo.h
+++ b/linker/linker_soinfo.h
@@ -30,8 +30,10 @@
#include <link.h>
+#include <memory>
#include <string>
+#include "private/bionic_elf_tls.h"
#include "linker_namespaces.h"
#define FLAG_LINKED 0x00000001
@@ -61,7 +63,7 @@
// unset.
#define FLAG_NEW_SOINFO 0x40000000 // new soinfo format
-#define SOINFO_VERSION 4
+#define SOINFO_VERSION 5
typedef void (*linker_dtor_function_t)();
typedef void (*linker_ctor_function_t)(int, char**, char**);
@@ -100,6 +102,17 @@
// TODO(dimitry): remove reference from soinfo member functions to this class.
class VersionTracker;
+// The first ELF TLS module has ID 1. Zero is reserved for the first word of
+// the DTV, a generation count, and unresolved weak symbols also use module
+// ID 0.
+static constexpr size_t kUninitializedModuleId = 0;
+
+struct soinfo_tls {
+ TlsSegment segment;
+ size_t module_id = kUninitializedModuleId;
+ TlsModule* module = nullptr;
+};
+
#if defined(__work_around_b_24465209__)
#define SOINFO_NAME_LEN 128
#endif
@@ -284,6 +297,8 @@
void add_secondary_namespace(android_namespace_t* secondary_ns);
android_namespace_list_t& get_secondary_namespaces();
+ soinfo_tls* get_tls() const;
+
void set_mapped_by_caller(bool reserved_map);
bool is_mapped_by_caller() const;
@@ -366,6 +381,9 @@
// version >= 4
ElfW(Relr)* relr_;
size_t relr_count_;
+
+ // version >= 5
+ std::unique_ptr<soinfo_tls> tls_;
};
// This function is used by dlvsym() to calculate hash of sym_ver
diff --git a/linker/linker_tls.cpp b/linker/linker_tls.cpp
index c0ebf25..a50d926 100644
--- a/linker/linker_tls.cpp
+++ b/linker/linker_tls.cpp
@@ -28,20 +28,75 @@
#include "linker_tls.h"
+#include <vector>
+
+#include "private/ScopedRWLock.h"
#include "private/bionic_defs.h"
#include "private/bionic_elf_tls.h"
#include "private/bionic_globals.h"
#include "private/linker_native_bridge.h"
+#include "linker_main.h"
+#include "linker_soinfo.h"
+
+static bool g_static_tls_finished;
+static std::vector<TlsModule> g_tls_modules;
+
+static size_t get_unused_module_index() {
+ for (size_t i = 0; i < g_tls_modules.size(); ++i) {
+ if (g_tls_modules[i].soinfo_ptr == nullptr) {
+ return i;
+ }
+ }
+ g_tls_modules.push_back({});
+ __libc_shared_globals()->tls_modules.module_count = g_tls_modules.size();
+ __libc_shared_globals()->tls_modules.module_table = g_tls_modules.data();
+ return g_tls_modules.size() - 1;
+}
+
+static void register_tls_module(soinfo* si, size_t static_offset) {
+ // The global TLS module table points at the std::vector of modules declared
+ // in this file, so acquire a write lock before modifying the std::vector.
+ ScopedWriteLock locker(&__libc_shared_globals()->tls_modules.rwlock);
+
+ size_t module_idx = get_unused_module_index();
+ TlsModule* module = &g_tls_modules[module_idx];
+
+ soinfo_tls* si_tls = si->get_tls();
+ si_tls->module_id = module_idx + 1;
+ si_tls->module = module;
+
+ *module = {
+ .segment = si_tls->segment,
+ .static_offset = static_offset,
+ .first_generation = ++__libc_shared_globals()->tls_modules.generation,
+ .soinfo_ptr = si,
+ };
+}
+
+static void unregister_tls_module(soinfo* si) {
+ ScopedWriteLock locker(&__libc_shared_globals()->tls_modules.rwlock);
+
+ soinfo_tls* si_tls = si->get_tls();
+ CHECK(si_tls->module->static_offset == SIZE_MAX);
+ CHECK(si_tls->module->soinfo_ptr == si);
+ *si_tls->module = {};
+ si_tls->module_id = kUninitializedModuleId;
+ si_tls->module = nullptr;
+}
__BIONIC_WEAK_FOR_NATIVE_BRIDGE
extern "C" void __linker_reserve_bionic_tls_in_static_tls() {
__libc_shared_globals()->static_tls_layout.reserve_bionic_tls();
}
-// Stub for linker static TLS layout.
-void layout_linker_static_tls() {
+void linker_setup_exe_static_tls(const char* progname) {
+ soinfo* somain = solist_get_somain();
StaticTlsLayout& layout = __libc_shared_globals()->static_tls_layout;
- layout.reserve_exe_segment_and_tcb(nullptr);
+ if (somain->get_tls() == nullptr) {
+ layout.reserve_exe_segment_and_tcb(nullptr, progname);
+ } else {
+ register_tls_module(somain, layout.reserve_exe_segment_and_tcb(&somain->get_tls()->segment, progname));
+ }
// The pthread key data is located at the very front of bionic_tls. As a
// temporary workaround, allocate bionic_tls just after the thread pointer so
@@ -49,8 +104,32 @@
// small enough. Specifically, Golang scans forward 384 words from the TP on
// ARM.
// - http://b/118381796
- // - https://groups.google.com/d/msg/golang-dev/yVrkFnYrYPE/2G3aFzYqBgAJ
+ // - https://github.com/golang/go/issues/29674
__linker_reserve_bionic_tls_in_static_tls();
+}
- layout.finish_layout();
+void linker_finalize_static_tls() {
+ g_static_tls_finished = true;
+ __libc_shared_globals()->static_tls_layout.finish_layout();
+}
+
+void register_soinfo_tls(soinfo* si) {
+ soinfo_tls* si_tls = si->get_tls();
+ if (si_tls == nullptr || si_tls->module_id != kUninitializedModuleId) {
+ return;
+ }
+ size_t static_offset = SIZE_MAX;
+ if (!g_static_tls_finished) {
+ StaticTlsLayout& layout = __libc_shared_globals()->static_tls_layout;
+ static_offset = layout.reserve_solib_segment(si_tls->segment);
+ }
+ register_tls_module(si, static_offset);
+}
+
+void unregister_soinfo_tls(soinfo* si) {
+ soinfo_tls* si_tls = si->get_tls();
+ if (si_tls == nullptr || si_tls->module_id == kUninitializedModuleId) {
+ return;
+ }
+ return unregister_tls_module(si);
}
diff --git a/linker/linker_tls.h b/linker/linker_tls.h
index 2f0a57d..4f48337 100644
--- a/linker/linker_tls.h
+++ b/linker/linker_tls.h
@@ -28,4 +28,10 @@
#pragma once
-void layout_linker_static_tls();
+struct soinfo;
+
+void linker_setup_exe_static_tls(const char* progname);
+void linker_finalize_static_tls();
+
+void register_soinfo_tls(soinfo* si);
+void unregister_soinfo_tls(soinfo* si);