Re-do static function dispatch.
Until now we had ifuncs for the string/memory routines in dynamic
executables but trivial one-line assembler stubs branching to the
default implementation in all cases for static executables. I did
recently experiment with running the ifuncs earlier in static
executables, but that actually broke one of our tests --- although our
internal ifuncs are built very carefully, an ifunc built as a test ends
up with shadow call stack trying to write and read x18 in the prolog and
epilog, but the shadow call stack (and x18) isn't set up yet, leading to
a crash (before any of the usual signal handlers have been installed,
so without any useful diagnostics).
While as far as I can tell, basically no-one outside of libc actually
uses ifuncs (in part because they're not available on non-Linux systems,
and if you're going to have to have a different implementation for other
OSes, you usually end up preferring to just use the same implementation
everywhere), *and* this only affects static executables, which are also
barely used in practice, I still don't fancy the app compat risk of
just saying "meh, anyone with ifuncs will have to make sure they're safe".
Which leads us to this change, which is basically a poor man's ifunc,
with function-scoped static function pointers as a poor man's GOT.
(Which means that, while not ideal performance-wise, this isn't actually
much worse than a dynamic executable.)
This change also factors out the typedefs, since there were a handful
of mistakes in some of the copies.
Test: run the static benchmarks on various architectures
Change-Id: Iec945b710174bebaa27cf71ed0101e7f5d769697
diff --git a/libc/arch-arm64/dynamic_function_dispatch.cpp b/libc/arch-arm64/dynamic_function_dispatch.cpp
index f9e4263..29e47b3 100644
--- a/libc/arch-arm64/dynamic_function_dispatch.cpp
+++ b/libc/arch-arm64/dynamic_function_dispatch.cpp
@@ -28,148 +28,147 @@
#include <private/bionic_ifuncs.h>
#include <stddef.h>
-#include <sys/auxv.h>
static inline bool __bionic_is_oryon(unsigned long hwcap) {
- if (!(hwcap & HWCAP_CPUID)) return false;
+ if (!(hwcap & HWCAP_CPUID)) return false;
- // Extract the implementor and variant bits from MIDR_EL1.
- // https://www.kernel.org/doc/html/latest/arch/arm64/cpu-feature-registers.html#list-of-registers-with-visible-features
- unsigned long midr;
- __asm__ __volatile__("mrs %0, MIDR_EL1" : "=r"(midr));
- uint16_t cpu = (midr >> 20) & 0xfff;
+ // Extract the implementor and variant bits from MIDR_EL1.
+ // https://www.kernel.org/doc/html/latest/arch/arm64/cpu-feature-registers.html#list-of-registers-with-visible-features
+ unsigned long midr;
+ __asm__ __volatile__("mrs %0, MIDR_EL1" : "=r"(midr));
+ uint16_t cpu = (midr >> 20) & 0xfff;
- auto make_cpu = [](unsigned implementor, unsigned variant) {
- return (implementor << 4) | variant;
- };
+ auto make_cpu = [](unsigned implementor, unsigned variant) {
+ return (implementor << 4) | variant;
+ };
- // Check for implementor Qualcomm's variants 0x1..0x5 (Oryon).
- return cpu >= make_cpu('Q', 0x1) && cpu <= make_cpu('Q', 0x5);
+ // Check for implementor Qualcomm's variants 0x1..0x5 (Oryon).
+ return cpu >= make_cpu('Q', 0x1) && cpu <= make_cpu('Q', 0x5);
}
extern "C" {
-typedef void* memchr_func(const void*, int, size_t);
DEFINE_IFUNC_FOR(memchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(memchr_func, __memchr_aarch64_mte);
- } else {
- RETURN_FUNC(memchr_func, __memchr_aarch64);
- }
-}
-
-typedef int memcmp_func(const void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memcmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(memcmp_func, __memcmp_aarch64);
-}
-
-typedef void* memcpy_func(void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memcpy) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memcpy_func, __memmove_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64_nt);
- } else if (arg->_hwcap & HWCAP_ASIMD) {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64_simd);
- } else {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64);
- }
-}
-
-typedef void* memmove_func(void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memmove) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memmove_func, __memmove_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memcpy_func, __memmove_aarch64_nt);
- } else if (arg->_hwcap & HWCAP_ASIMD) {
- RETURN_FUNC(memmove_func, __memmove_aarch64_simd);
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(memchr_func_t, __memchr_aarch64_mte);
} else {
- RETURN_FUNC(memmove_func, __memmove_aarch64);
+ RETURN_FUNC(memchr_func_t, __memchr_aarch64);
}
}
+MEMCHR_SHIM()
-typedef int memrchr_func(const void*, int, size_t);
+DEFINE_IFUNC_FOR(memcmp) {
+ // TODO: enable the SVE version.
+ RETURN_FUNC(memcmp_func_t, __memcmp_aarch64);
+}
+MEMCMP_SHIM()
+
+DEFINE_IFUNC_FOR(memcpy) {
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memcpy_func_t, __memmove_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64_nt);
+ } else if (arg->_hwcap & HWCAP_ASIMD) {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64_simd);
+ } else {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64);
+ }
+}
+MEMCPY_SHIM()
+
+DEFINE_IFUNC_FOR(memmove) {
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_nt);
+ } else if (arg->_hwcap & HWCAP_ASIMD) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_simd);
+ } else {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64);
+ }
+}
+MEMMOVE_SHIM()
+
DEFINE_IFUNC_FOR(memrchr) {
- RETURN_FUNC(memrchr_func, __memrchr_aarch64);
+ RETURN_FUNC(memrchr_func_t, __memrchr_aarch64);
}
+MEMRCHR_SHIM()
-typedef int memset_func(void*, int, size_t);
DEFINE_IFUNC_FOR(memset) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memset_func, __memset_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memset_func, __memset_aarch64_nt);
- } else {
- RETURN_FUNC(memset_func, __memset_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memset_func_t, __memset_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memset_func_t, __memset_aarch64_nt);
+ } else {
+ RETURN_FUNC(memset_func_t, __memset_aarch64);
+ }
}
+MEMSET_SHIM()
-typedef char* stpcpy_func(char*, const char*, size_t);
DEFINE_IFUNC_FOR(stpcpy) {
- // TODO: enable the SVE version.
- RETURN_FUNC(stpcpy_func, __stpcpy_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(stpcpy_func_t, __stpcpy_aarch64);
}
+STPCPY_SHIM()
-typedef char* strchr_func(const char*, int);
DEFINE_IFUNC_FOR(strchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strchr_func, __strchr_aarch64_mte);
- } else {
- RETURN_FUNC(strchr_func, __strchr_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strchr_func_t, __strchr_aarch64_mte);
+ } else {
+ RETURN_FUNC(strchr_func_t, __strchr_aarch64);
+ }
}
+STRCHR_SHIM()
-typedef char* strchrnul_func(const char*, int);
DEFINE_IFUNC_FOR(strchrnul) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strchrnul_func, __strchrnul_aarch64_mte);
- } else {
- RETURN_FUNC(strchrnul_func, __strchrnul_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strchrnul_func_t, __strchrnul_aarch64_mte);
+ } else {
+ RETURN_FUNC(strchrnul_func_t, __strchrnul_aarch64);
+ }
}
+STRCHRNUL_SHIM()
-typedef int strcmp_func(const char*, const char*);
DEFINE_IFUNC_FOR(strcmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strcmp_func, __strcmp_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strcmp_func_t, __strcmp_aarch64);
}
+STRCMP_SHIM()
-typedef char* strcpy_func(char*, const char*);
DEFINE_IFUNC_FOR(strcpy) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strcpy_func, __strcpy_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strcpy_func_t, __strcpy_aarch64);
}
+STRCPY_SHIM()
-typedef size_t strlen_func(const char*);
DEFINE_IFUNC_FOR(strlen) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strlen_func, __strlen_aarch64_mte);
- } else {
- RETURN_FUNC(strlen_func, __strlen_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strlen_func_t, __strlen_aarch64_mte);
+ } else {
+ RETURN_FUNC(strlen_func_t, __strlen_aarch64);
+ }
}
+STRLEN_SHIM()
-typedef int strncmp_func(const char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strncmp_func, __strncmp_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strncmp_func_t, __strncmp_aarch64);
}
+STRNCMP_SHIM()
-typedef size_t strnlen_func(const char*, size_t);
DEFINE_IFUNC_FOR(strnlen) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strnlen_func, __strnlen_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strnlen_func_t, __strnlen_aarch64);
}
+STRNLEN_SHIM()
-typedef char* strrchr_func(const char*, int);
DEFINE_IFUNC_FOR(strrchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strrchr_func, __strrchr_aarch64_mte);
- } else {
- RETURN_FUNC(strrchr_func, __strrchr_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strrchr_func_t, __strrchr_aarch64_mte);
+ } else {
+ RETURN_FUNC(strrchr_func_t, __strrchr_aarch64);
+ }
}
+STRRCHR_SHIM()
} // extern "C"