Initialize static TLS memory using module list
This implementation simply iterates over each static TLS module and
copies its initialization image into a new thread's static TLS block.
Bug: http://b/78026329
Test: bionic unit tests
Change-Id: Ib7edb665271a07010bc68e306feb5df422f2f9e6
diff --git a/libc/bionic/__libc_init_main_thread.cpp b/libc/bionic/__libc_init_main_thread.cpp
index 2b90c90..6279e65 100644
--- a/libc/bionic/__libc_init_main_thread.cpp
+++ b/libc/bionic/__libc_init_main_thread.cpp
@@ -126,6 +126,7 @@
auto new_tcb = reinterpret_cast<bionic_tcb*>(mapping.static_tls + layout.offset_bionic_tcb());
auto new_tls = reinterpret_cast<bionic_tls*>(mapping.static_tls + layout.offset_bionic_tls());
+ __init_static_tls(mapping.static_tls);
new_tcb->copy_from_bootstrap(temp_tcb);
new_tls->copy_from_bootstrap(temp_tls);
__init_tcb(new_tcb, &main_thread);
diff --git a/libc/bionic/bionic_elf_tls.cpp b/libc/bionic/bionic_elf_tls.cpp
index a54f29e..19efff2 100644
--- a/libc/bionic/bionic_elf_tls.cpp
+++ b/libc/bionic/bionic_elf_tls.cpp
@@ -29,9 +29,12 @@
#include "private/bionic_elf_tls.h"
#include <async_safe/log.h>
+#include <string.h>
#include <sys/param.h>
#include <unistd.h>
+#include "private/ScopedRWLock.h"
+#include "private/bionic_globals.h"
#include "private/bionic_macros.h"
#include "private/bionic_tls.h"
@@ -147,3 +150,33 @@
if (value < old_value) overflowed_ = true;
return value;
}
+
+// Copy each TLS module's initialization image into a newly-allocated block of
+// static TLS memory. To reduce dirty pages, this function only writes to pages
+// within the static TLS that need initialization. The memory should already be
+// zero-initialized on entry.
+void __init_static_tls(void* static_tls) {
+ // The part of the table we care about (i.e. static TLS modules) never changes
+ // after startup, but we still need the mutex because the table could grow,
+ // moving the initial part. If this locking is too slow, we can duplicate the
+ // static part of the table.
+ TlsModules& modules = __libc_shared_globals()->tls_modules;
+ ScopedReadLock locker(&modules.rwlock);
+
+ for (size_t i = 0; i < modules.module_count; ++i) {
+ TlsModule& module = modules.module_table[i];
+ if (module.static_offset == SIZE_MAX) {
+ // All of the static modules come before all of the dynamic modules, so
+ // once we see the first dynamic module, we're done.
+ break;
+ }
+ if (module.segment.init_size == 0) {
+ // Skip the memcpy call for TLS segments with no initializer, which is
+ // common.
+ continue;
+ }
+ memcpy(static_cast<char*>(static_tls) + module.static_offset,
+ module.segment.init_ptr,
+ module.segment.init_size);
+ }
+}
diff --git a/libc/bionic/pthread_create.cpp b/libc/bionic/pthread_create.cpp
index 8676a45..31e0378 100644
--- a/libc/bionic/pthread_create.cpp
+++ b/libc/bionic/pthread_create.cpp
@@ -288,7 +288,8 @@
auto tcb = reinterpret_cast<bionic_tcb*>(mapping.static_tls + layout.offset_bionic_tcb());
auto tls = reinterpret_cast<bionic_tls*>(mapping.static_tls + layout.offset_bionic_tls());
- // (Re)initialize TLS pointers.
+ // Initialize TLS memory.
+ __init_static_tls(mapping.static_tls);
__init_tcb(tcb, thread);
__init_tcb_stack_guard(tcb);
__init_bionic_tls_ptrs(tcb, tls);
diff --git a/libc/private/bionic_elf_tls.h b/libc/private/bionic_elf_tls.h
index e95560b..fee821e 100644
--- a/libc/private/bionic_elf_tls.h
+++ b/libc/private/bionic_elf_tls.h
@@ -115,3 +115,5 @@
size_t module_count = 0;
TlsModule* module_table = nullptr;
};
+
+void __init_static_tls(void* static_tls);