libc: implement kernel vdso syscalls for i386
This patch uses __kernel_vsyscall instead of "int 0x80"
as the syscall entry point. AT_SYSINFO points to
an adapter to mask the arch specific difference and gives a
performance boost on i386 architecture.
Bug: http://b/27533895
Change-ID: Ib340c604d02c6c25714a95793737e3cfdc3fc5d7
Signed-off-by: Mingwei Shi <mingwei.shi@intel.com>
(cherry picked from commit be910529322b461148debefd50b9e0d67ae84f8e)
diff --git a/linker/linker.cpp b/linker/linker.cpp
index 1ad000c..4591498 100644
--- a/linker/linker.cpp
+++ b/linker/linker.cpp
@@ -44,6 +44,7 @@
#include <vector>
// Private C library headers.
+#include "private/bionic_globals.h"
#include "private/bionic_tls.h"
#include "private/KernelArgumentBlock.h"
#include "private/ScopedPthreadMutexLocker.h"
@@ -66,6 +67,8 @@
extern void __libc_init_globals(KernelArgumentBlock&);
extern void __libc_init_AT_SECURE(KernelArgumentBlock&);
+extern "C" void _start();
+
// Override macros to use C++ style casts.
#undef ELF_ST_TYPE
#define ELF_ST_TYPE(x) (static_cast<uint32_t>(x) & 0xf)
@@ -3966,10 +3969,9 @@
}
#endif
- /* We can also turn on GNU RELRO protection */
- if (phdr_table_protect_gnu_relro(phdr, phnum, load_bias) < 0) {
- DL_ERR("can't enable GNU RELRO protection for \"%s\": %s",
- get_realpath(), strerror(errno));
+ // We can also turn on GNU RELRO protection if we're not linking the dynamic linker
+ // itself --- it can't make system calls yet, and will have to call protect_relro later.
+ if (!is_linker() && !protect_relro()) {
return false;
}
@@ -3994,6 +3996,15 @@
return true;
}
+bool soinfo::protect_relro() {
+ if (phdr_table_protect_gnu_relro(phdr, phnum, load_bias) < 0) {
+ DL_ERR("can't enable GNU RELRO protection for \"%s\": %s",
+ get_realpath(), strerror(errno));
+ return false;
+ }
+ return true;
+}
+
/*
* This function add vdso to internal dso list.
* It helps to stack unwinding through signal handlers.
@@ -4296,8 +4307,9 @@
return 0;
}
-extern "C" int __set_tls(void*);
-extern "C" void _start();
+static void __linker_cannot_link(KernelArgumentBlock& args) {
+ __libc_fatal("CANNOT LINK EXECUTABLE \"%s\": %s", args.argv[0], linker_get_error_buffer());
+}
/*
* This is the entry point for the linker, called from begin.S. This
@@ -4311,9 +4323,6 @@
extern "C" ElfW(Addr) __linker_init(void* raw_args) {
KernelArgumentBlock args(raw_args);
- void* tls[BIONIC_TLS_SLOTS];
- __set_tls(tls);
-
ElfW(Addr) linker_addr = args.getauxval(AT_BASE);
ElfW(Addr) entry_point = args.getauxval(AT_ENTRY);
ElfW(Ehdr)* elf_hdr = reinterpret_cast<ElfW(Ehdr)*>(linker_addr);
@@ -4342,18 +4351,33 @@
linker_so.phnum = elf_hdr->e_phnum;
linker_so.set_linker_flag();
+ // Prelink the linker so we can access linker globals.
+ if (!linker_so.prelink_image()) __linker_cannot_link(args);
+
// This might not be obvious... The reasons why we pass g_empty_list
// in place of local_group here are (1) we do not really need it, because
// linker is built with DT_SYMBOLIC and therefore relocates its symbols against
// itself without having to look into local_group and (2) allocators
// are not yet initialized, and therefore we cannot use linked_list.push_*
// functions at this point.
- if (!(linker_so.prelink_image() && linker_so.link_image(g_empty_list, g_empty_list, nullptr))) {
- __libc_fatal("CANNOT LINK EXECUTABLE \"%s\": %s", args.argv[0], linker_get_error_buffer());
- }
+ if (!linker_so.link_image(g_empty_list, g_empty_list, nullptr)) __linker_cannot_link(args);
+#if defined(__i386__)
+ // On x86, we can't make system calls before this point.
+ // We can't move this up because this needs to assign to a global.
+ // Note that until we call __libc_init_main_thread below we have
+ // no TLS, so you shouldn't make a system call that can fail, because
+ // it will SEGV when it tries to set errno.
+ __libc_init_sysinfo(args);
+#endif
+
+ // Initialize the main thread (including TLS, so system calls really work).
__libc_init_main_thread(args);
+ // We didn't protect the linker's RELRO pages in link_image because we
+ // couldn't make system calls on x86 at that point, but we can now...
+ if (!linker_so.protect_relro()) __linker_cannot_link(args);
+
// Initialize the linker's static libc's globals
__libc_init_globals(args);