Use ifuncs in the linker
Using ifuncs allows the linker to select faster versions of libc functions
like strcmp, making linking faster.
The linker continues to first initialize TLS, then call the ifunc
resolvers. There are small amounts of code in Bionic that need to avoid
calling functions selected using ifuncs (generally string.h APIs). I've
tried to compile those pieces with -ffreestanding. Maybe it's unnecessary,
but maybe it could help avoid compiler-inserted memset calls, and maybe
it will be useful later on.
The ifuncs are called in a special early pass using special
__rel[a]_iplt_start / __rel[a]_iplt_end symbols. The linker will encounter
the ifuncs again as R_*_IRELATIVE dynamic relocations, so they're skipped
on the second pass.
Break linker_main.cpp into its own liblinker_main library so it can be
compiled with -ffreestanding.
On walleye, this change fixes a recent 2.3% linker64 start-up time
regression (156.6ms -> 160.2ms), but it also helps the 32-bit time by
about 1.9% on the same benchmark. I'm measuring the run-time using a
synthetic benchmark based on loading libandroid_servers.so.
Test: bionic unit tests, manual benchmarking
Bug: none
Merged-In: Ieb9446c2df13a66fc0d377596756becad0af6995
Change-Id: Ieb9446c2df13a66fc0d377596756becad0af6995
(cherry picked from commit 772bcbb0c2f7a87b18021849528240ef0c617d94)
diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp
index 3b950a3..264923f 100644
--- a/linker/linker_main.cpp
+++ b/linker/linker_main.cpp
@@ -40,6 +40,8 @@
#include "linker_tls.h"
#include "linker_utils.h"
+#include "private/bionic_auxv.h"
+#include "private/bionic_call_ifunc_resolver.h"
#include "private/bionic_globals.h"
#include "private/bionic_tls.h"
#include "private/KernelArgumentBlock.h"
@@ -565,6 +567,32 @@
}
}
+// TODO: There is a similar ifunc resolver calling loop in libc_init_static.cpp, but that version
+// uses weak symbols, which don't work in the linker prior to its relocation. This version also
+// supports a load bias. When we stop supporting the gold linker in the NDK, then maybe we can use
+// non-weak definitions and merge the two loops.
+#if defined(USE_RELA)
+extern __LIBC_HIDDEN__ ElfW(Rela) __rela_iplt_start[], __rela_iplt_end[];
+
+static void call_ifunc_resolvers(ElfW(Addr) load_bias) {
+ for (ElfW(Rela) *r = __rela_iplt_start; r != __rela_iplt_end; ++r) {
+ ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset + load_bias);
+ ElfW(Addr) resolver = r->r_addend + load_bias;
+ *offset = __bionic_call_ifunc_resolver(resolver);
+ }
+}
+#else
+extern __LIBC_HIDDEN__ ElfW(Rel) __rel_iplt_start[], __rel_iplt_end[];
+
+static void call_ifunc_resolvers(ElfW(Addr) load_bias) {
+ for (ElfW(Rel) *r = __rel_iplt_start; r != __rel_iplt_end; ++r) {
+ ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset + load_bias);
+ ElfW(Addr) resolver = *offset + load_bias;
+ *offset = __bionic_call_ifunc_resolver(resolver);
+ }
+}
+#endif
+
// Detect an attempt to run the linker on itself. e.g.:
// /system/bin/linker64 /system/bin/linker64
// Use priority-1 to run this constructor before other constructors.
@@ -616,11 +644,15 @@
ElfW(Ehdr)* elf_hdr = reinterpret_cast<ElfW(Ehdr)*>(linker_addr);
ElfW(Phdr)* phdr = reinterpret_cast<ElfW(Phdr)*>(linker_addr + elf_hdr->e_phoff);
+ // string.h functions must not be used prior to calling the linker's ifunc resolvers.
+ const ElfW(Addr) load_bias = get_elf_exec_load_bias(elf_hdr);
+ call_ifunc_resolvers(load_bias);
+
soinfo tmp_linker_so(nullptr, nullptr, nullptr, 0, 0);
tmp_linker_so.base = linker_addr;
tmp_linker_so.size = phdr_table_get_load_size(phdr, elf_hdr->e_phnum);
- tmp_linker_so.load_bias = get_elf_exec_load_bias(elf_hdr);
+ tmp_linker_so.load_bias = load_bias;
tmp_linker_so.dynamic = nullptr;
tmp_linker_so.phdr = phdr;
tmp_linker_so.phnum = elf_hdr->e_phnum;