Re-do static function dispatch.
Until now we had ifuncs for the string/memory routines in dynamic
executables but trivial one-line assembler stubs branching to the
default implementation in all cases for static executables. I did
recently experiment with running the ifuncs earlier in static
executables, but that actually broke one of our tests --- although our
internal ifuncs are built very carefully, an ifunc built as a test ends
up with shadow call stack trying to write and read x18 in the prolog and
epilog, but the shadow call stack (and x18) isn't set up yet, leading to
a crash (before any of the usual signal handlers have been installed,
so without any useful diagnostics).
While as far as I can tell, basically no-one outside of libc actually
uses ifuncs (in part because they're not available on non-Linux systems,
and if you're going to have to have a different implementation for other
OSes, you usually end up preferring to just use the same implementation
everywhere), *and* this only affects static executables, which are also
barely used in practice, I still don't fancy the app compat risk of
just saying "meh, anyone with ifuncs will have to make sure they're safe".
Which leads us to this change, which is basically a poor man's ifunc,
with function-scoped static function pointers as a poor man's GOT.
(Which means that, while not ideal performance-wise, this isn't actually
much worse than a dynamic executable.)
This change also factors out the typedefs, since there were a handful
of mistakes in some of the copies.
Test: run the static benchmarks on various architectures
Change-Id: Iec945b710174bebaa27cf71ed0101e7f5d769697
diff --git a/libc/private/bionic_ifuncs.h b/libc/private/bionic_ifuncs.h
index e6b349a..b31c903 100644
--- a/libc/private/bionic_ifuncs.h
+++ b/libc/private/bionic_ifuncs.h
@@ -31,6 +31,8 @@
#include <stdint.h>
#include <sys/ifunc.h>
+#include <private/bionic_call_ifunc_resolver.h>
+
#if defined(__aarch64__)
#define IFUNC_ARGS (uint64_t hwcap __attribute__((unused)), \
__ifunc_arg_t* arg __attribute__((unused)))
@@ -40,14 +42,6 @@
#define IFUNC_ARGS ()
#endif
-// We can't have HWASAN enabled in resolvers because they may be called before HWASAN is
-// initialized.
-#define DEFINE_IFUNC_FOR(name) \
- name##_func name __attribute__((ifunc(#name "_resolver"))); \
- __attribute__((visibility("hidden"))) \
- __attribute__((no_sanitize("hwaddress"))) \
- name##_func* name##_resolver IFUNC_ARGS
-
#define DECLARE_FUNC(type, name) \
__attribute__((visibility("hidden"))) \
type name
@@ -56,3 +50,137 @@
DECLARE_FUNC(type, name); \
return name; \
}
+
+#if defined(BIONIC_DYNAMIC_DISPATCH)
+
+// We can't have HWASAN enabled in resolvers because they may be called before
+// HWASAN is initialized.
+#define DEFINE_IFUNC_FOR(name) \
+ name##_func_t name __attribute__((ifunc(#name "_resolver"))); \
+ __attribute__((visibility("hidden"))) \
+ __attribute__((no_sanitize("hwaddress"))) name##_func_t* name##_resolver IFUNC_ARGS
+
+#define DEFINE_STATIC_SHIM(x)
+
+#elif defined(BIONIC_STATIC_DISPATCH)
+
+#define DEFINE_IFUNC_FOR(name) \
+ name##_func_t* name##_resolver IFUNC_ARGS; \
+ __attribute__((visibility("hidden"))) \
+ __attribute__((no_sanitize("hwaddress"))) name##_func_t* name##_resolver IFUNC_ARGS
+
+#define DEFINE_STATIC_SHIM(x) x
+
+#define FORWARD(name) \
+ static name##_func_t* fn = reinterpret_cast<name##_func_t*>( \
+ __bionic_call_ifunc_resolver(reinterpret_cast<ElfW(Addr)>(name##_resolver))); \
+ return fn
+
+#else
+#error neither dynamic nor static dispatch?!
+#endif
+
+typedef void* memchr_func_t(const void*, int, size_t);
+#define MEMCHR_SHIM() \
+ DEFINE_STATIC_SHIM(void* memchr(const void* src, int ch, size_t n) { \
+ FORWARD(memchr)(src, ch, n); \
+ })
+
+typedef int memcmp_func_t(const void*, const void*, size_t);
+#define MEMCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int memcmp(const void* lhs, const void* rhs, size_t n) { \
+ FORWARD(memcmp)(lhs, rhs, n); \
+ })
+
+typedef void* memcpy_func_t(void*, const void*, size_t);
+#define MEMCPY_SHIM() \
+ DEFINE_STATIC_SHIM(void* memcpy(void* dst, const void* src, size_t n) { \
+ FORWARD(memcpy)(dst, src, n); \
+ })
+
+typedef void* memmove_func_t(void*, const void*, size_t);
+#define MEMMOVE_SHIM() \
+ DEFINE_STATIC_SHIM(void* memmove(void* dst, const void* src, size_t n) { \
+ FORWARD(memmove)(dst, src, n); \
+ })
+
+typedef int memrchr_func_t(const void*, int, size_t);
+#define MEMRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(int memrchr(const void* src, int ch, size_t n) { \
+ FORWARD(memrchr)(src, ch, n); \
+ })
+
+typedef void* memset_func_t(void*, int, size_t);
+#define MEMSET_SHIM() \
+ DEFINE_STATIC_SHIM(void* memset(void* dst, int ch, size_t n) { FORWARD(memset)(dst, ch, n); })
+
+typedef void* __memset_chk_func_t(void*, int, size_t, size_t);
+#define __MEMSET_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(void* __memset_chk(void* dst, int ch, size_t n, size_t n2) { \
+ FORWARD(__memset_chk)(dst, ch, n, n2); \
+ })
+
+typedef char* stpcpy_func_t(char*, const char*);
+#define STPCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* stpcpy(char* dst, const char* src) { FORWARD(stpcpy)(dst, src); })
+
+typedef char* strcat_func_t(char*, const char*);
+#define STRCAT_SHIM() \
+ DEFINE_STATIC_SHIM(char* strcat(char* dst, const char* src) { FORWARD(strcat)(dst, src); })
+
+typedef char* __strcat_chk_func_t(char*, const char*, size_t);
+#define __STRCAT_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(char* __strcat_chk(char* dst, const char* src, size_t dst_buf_size) { \
+ FORWARD(__strcat_chk)(dst, src, dst_buf_size); \
+ })
+
+typedef char* strchr_func_t(const char*, int);
+#define STRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(char* strchr(const char* src, int ch) { FORWARD(strchr)(src, ch); })
+
+typedef char* strchrnul_func_t(const char*, int);
+#define STRCHRNUL_SHIM() \
+ DEFINE_STATIC_SHIM(char* strchrnul(const char* src, int ch) { FORWARD(strchrnul)(src, ch); })
+
+typedef int strcmp_func_t(const char*, const char*);
+#define STRCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int strcmp(char* lhs, const char* rhs) { FORWARD(strcmp)(lhs, rhs); })
+
+typedef char* strcpy_func_t(char*, const char*);
+#define STRCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* strcpy(char* dst, const char* src) { FORWARD(strcpy)(dst, src); })
+
+typedef char* __strcpy_chk_func_t(char*, const char*, size_t);
+#define __STRCPY_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(char* __strcpy_chk(char* dst, const char* src, size_t dst_len) { \
+ FORWARD(__strcpy_chk)(dst, src, dst_len); \
+ })
+
+typedef size_t strlen_func_t(const char*);
+#define STRLEN_SHIM() DEFINE_STATIC_SHIM(size_t strlen(const char* s) { FORWARD(strlen)(s); })
+
+typedef char* strncat_func_t(char*, const char*, size_t);
+#define STRNCAT_SHIM() \
+ DEFINE_STATIC_SHIM(char* strncat(char* dst, const char* src, size_t n) { \
+ FORWARD(strncat)(dst, src, n); \
+ })
+
+typedef int strncmp_func_t(const char*, const char*, size_t);
+#define STRNCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int strncmp(const char* lhs, const char* rhs, size_t n) { \
+ FORWARD(strncmp)(lhs, rhs, n); \
+ })
+
+typedef char* strncpy_func_t(char*, const char*, size_t);
+#define STRNCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* strncpy(char* dst, const char* src, size_t n) { \
+ FORWARD(strncpy)(dst, src, n); \
+ })
+
+typedef size_t strnlen_func_t(const char*, size_t);
+#define STRNLEN_SHIM() \
+ DEFINE_STATIC_SHIM(size_t strnlen(const char* s, size_t n) { FORWARD(strnlen)(s, n); })
+
+typedef char* strrchr_func_t(const char*, int);
+#define STRRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(char* strrchr(const char* src, int ch) { FORWARD(strrchr)(src, ch); })