Re-do static function dispatch.
Until now we had ifuncs for the string/memory routines in dynamic
executables but trivial one-line assembler stubs branching to the
default implementation in all cases for static executables. I did
recently experiment with running the ifuncs earlier in static
executables, but that actually broke one of our tests --- although our
internal ifuncs are built very carefully, an ifunc built as a test ends
up with shadow call stack trying to write and read x18 in the prolog and
epilog, but the shadow call stack (and x18) isn't set up yet, leading to
a crash (before any of the usual signal handlers have been installed,
so without any useful diagnostics).
While as far as I can tell, basically no-one outside of libc actually
uses ifuncs (in part because they're not available on non-Linux systems,
and if you're going to have to have a different implementation for other
OSes, you usually end up preferring to just use the same implementation
everywhere), *and* this only affects static executables, which are also
barely used in practice, I still don't fancy the app compat risk of
just saying "meh, anyone with ifuncs will have to make sure they're safe".
Which leads us to this change, which is basically a poor man's ifunc,
with function-scoped static function pointers as a poor man's GOT.
(Which means that, while not ideal performance-wise, this isn't actually
much worse than a dynamic executable.)
This change also factors out the typedefs, since there were a handful
of mistakes in some of the copies.
Test: run the static benchmarks on various architectures
Change-Id: Iec945b710174bebaa27cf71ed0101e7f5d769697
diff --git a/libc/Android.bp b/libc/Android.bp
index 42ffd37..fd681d5 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -1416,43 +1416,13 @@
}
// ========================================================
-// libc_static_dispatch.a --- libc.a ifuncs
+// libc_static_dispatch.a/libc_dynamic_dispatch.a --- string/memory "ifuncs"
+// (Actually ifuncs for libc.so, but a home-grown alternative for libc.a.)
// ========================================================
-cc_library_static {
+
+cc_defaults {
+ name: "libc_dispatch_defaults",
defaults: ["libc_defaults"],
- name: "libc_static_dispatch",
-
- arch: {
- x86_64: {
- srcs: ["arch-x86_64/static_function_dispatch.S"],
- },
- x86: {
- srcs: ["arch-x86/static_function_dispatch.S"],
- },
- arm: {
- srcs: ["arch-arm/static_function_dispatch.S"],
- },
- arm64: {
- srcs: ["arch-arm64/static_function_dispatch.S"],
- },
- riscv64: {
- srcs: ["arch-riscv64/static_function_dispatch.S"],
- },
- },
-}
-
-// ========================================================
-// libc_dynamic_dispatch.a --- libc.so ifuncs
-// ========================================================
-cc_library_static {
- defaults: ["libc_defaults"],
- name: "libc_dynamic_dispatch",
-
- cflags: [
- "-ffreestanding",
- "-fno-stack-protector",
- "-fno-jump-tables",
- ],
arch: {
x86_64: {
srcs: ["arch-x86_64/dynamic_function_dispatch.cpp"],
@@ -1470,6 +1440,30 @@
srcs: ["arch-riscv64/dynamic_function_dispatch.cpp"],
},
},
+ // Prevent the compiler from inserting calls to libc/taking the address of
+ // a jump table from within an ifunc (or, in the static case, code that
+ // can be executed arbitrarily early).
+ cflags: [
+ "-ffreestanding",
+ "-fno-stack-protector",
+ "-fno-jump-tables",
+ ],
+}
+
+cc_library_static {
+ name: "libc_static_dispatch",
+ defaults: ["libc_dispatch_defaults"],
+ cflags: [
+ "-DBIONIC_STATIC_DISPATCH",
+ ],
+}
+
+cc_library_static {
+ name: "libc_dynamic_dispatch",
+ defaults: ["libc_dispatch_defaults"],
+ cflags: [
+ "-DBIONIC_DYNAMIC_DISPATCH",
+ ],
}
// ========================================================
diff --git a/libc/arch-arm/dynamic_function_dispatch.cpp b/libc/arch-arm/dynamic_function_dispatch.cpp
index 1d2f38f..f984421 100644
--- a/libc/arch-arm/dynamic_function_dispatch.cpp
+++ b/libc/arch-arm/dynamic_function_dispatch.cpp
@@ -27,27 +27,26 @@
*/
#include <fcntl.h>
-#include <sys/syscall.h>
-
#include <private/bionic_ifuncs.h>
+#include <sys/syscall.h>
extern "C" {
enum CpuVariant {
- kUnknown = 0,
- kGeneric,
- kCortexA7,
- kCortexA9,
- kCortexA53,
- kCortexA55,
- kKrait,
- kKryo,
+ kUnknown = 0,
+ kGeneric,
+ kCortexA7,
+ kCortexA9,
+ kCortexA53,
+ kCortexA55,
+ kKrait,
+ kKryo,
};
static constexpr int MAX_CPU_NAME_LEN = 12;
struct CpuVariantNames {
- alignas(alignof(int)) char name[MAX_CPU_NAME_LEN];
- CpuVariant variant;
+ alignas(alignof(int)) char name[MAX_CPU_NAME_LEN];
+ CpuVariant variant;
};
static constexpr CpuVariantNames cpu_variant_names[] = {
@@ -66,227 +65,237 @@
};
static long ifunc_open(const char* pathname) {
- register long r0 __asm__("r0") = AT_FDCWD;
- register long r1 __asm__("r1") = reinterpret_cast<long>(pathname);
- register long r2 __asm__("r2") = O_RDONLY;
- register long r3 __asm__("r3") = 0;
- register long r7 __asm__("r7") = __NR_openat;
- __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r7));
- return r0;
+ register long r0 __asm__("r0") = AT_FDCWD;
+ register long r1 __asm__("r1") = reinterpret_cast<long>(pathname);
+ register long r2 __asm__("r2") = O_RDONLY;
+ register long r3 __asm__("r3") = 0;
+ register long r7 __asm__("r7") = __NR_openat;
+ __asm__ volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r7));
+ return r0;
}
static ssize_t ifunc_read(int fd, void* buf, size_t count) {
- register long r0 __asm__("r0") = fd;
- register long r1 __asm__("r1") = reinterpret_cast<long>(buf);
- register long r2 __asm__("r2") = count;
- register long r7 __asm__("r7") = __NR_read;
- __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r1), "r"(r2), "r"(r7) : "memory");
- return r0;
+ register long r0 __asm__("r0") = fd;
+ register long r1 __asm__("r1") = reinterpret_cast<long>(buf);
+ register long r2 __asm__("r2") = count;
+ register long r7 __asm__("r7") = __NR_read;
+ __asm__ volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r0), "r"(r1), "r"(r2), "r"(r7)
+ : "memory");
+ return r0;
}
static int ifunc_close(int fd) {
- register long r0 __asm__("r0") = fd;
- register long r7 __asm__("r7") = __NR_close;
- __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r7));
- return r0;
+ register long r0 __asm__("r0") = fd;
+ register long r7 __asm__("r7") = __NR_close;
+ __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r7));
+ return r0;
}
static bool is_same_name(const char* a, const char* b) {
- static_assert(MAX_CPU_NAME_LEN % sizeof(int) == 0, "");
- const int* ia = reinterpret_cast<const int*>(a);
- const int* ib = reinterpret_cast<const int*>(b);
- for (size_t i = 0; i < MAX_CPU_NAME_LEN / sizeof(int); ++i) {
- if (ia[i] != ib[i]) {
- return false;
- }
+ static_assert(MAX_CPU_NAME_LEN % sizeof(int) == 0, "");
+ const int* ia = reinterpret_cast<const int*>(a);
+ const int* ib = reinterpret_cast<const int*>(b);
+ for (size_t i = 0; i < MAX_CPU_NAME_LEN / sizeof(int); ++i) {
+ if (ia[i] != ib[i]) {
+ return false;
}
- return true;
+ }
+ return true;
}
static CpuVariant init_cpu_variant() {
- int fd = ifunc_open("/dev/cpu_variant:arm");
- if (fd < 0) return kGeneric;
+ int fd = ifunc_open("/dev/cpu_variant:arm");
+ if (fd < 0) return kGeneric;
- alignas(alignof(int)) char name[MAX_CPU_NAME_LEN] = {};
+ alignas(alignof(int)) char name[MAX_CPU_NAME_LEN] = {};
- int bytes_read, total_read = 0;
- while (total_read < MAX_CPU_NAME_LEN - 1 &&
- (bytes_read = ifunc_read(fd, name + total_read,
- MAX_CPU_NAME_LEN - 1 - total_read)) > 0) {
- total_read += bytes_read;
- }
- ifunc_close(fd);
+ int bytes_read, total_read = 0;
+ while (total_read < MAX_CPU_NAME_LEN - 1 &&
+ (bytes_read = ifunc_read(fd, name + total_read,
+ MAX_CPU_NAME_LEN - 1 - total_read)) > 0) {
+ total_read += bytes_read;
+ }
+ ifunc_close(fd);
- if (bytes_read != 0) {
- // The file is too big. We haven't reach the end. Or maybe there is an
- // error when reading.
- return kGeneric;
- }
- name[total_read] = 0;
-
- const CpuVariantNames* cpu_variant = cpu_variant_names;
- while (cpu_variant->variant != kUnknown) {
- if (is_same_name(cpu_variant->name, name)) {
- return cpu_variant->variant;
- }
- cpu_variant++;
- }
+ if (bytes_read != 0) {
+ // The file is too big. We haven't reach the end. Or maybe there is an
+ // error when reading.
return kGeneric;
+ }
+ name[total_read] = 0;
+
+ const CpuVariantNames* cpu_variant = cpu_variant_names;
+ while (cpu_variant->variant != kUnknown) {
+ if (is_same_name(cpu_variant->name, name)) {
+ return cpu_variant->variant;
+ }
+ cpu_variant++;
+ }
+ return kGeneric;
}
static CpuVariant get_cpu_variant() {
- static CpuVariant cpu_variant = kUnknown;
- if (cpu_variant == kUnknown) {
- cpu_variant = init_cpu_variant();
- }
- return cpu_variant;
+ static CpuVariant cpu_variant = kUnknown;
+ if (cpu_variant == kUnknown) {
+ cpu_variant = init_cpu_variant();
+ }
+ return cpu_variant;
}
-typedef void* memmove_func(void* __dst, const void* __src, size_t __n);
DEFINE_IFUNC_FOR(memmove) {
- RETURN_FUNC(memmove_func, memmove_a15);
+ RETURN_FUNC(memmove_func_t, memmove_a15);
}
+MEMMOVE_SHIM()
-typedef void* memcpy_func(void*, const void*, size_t);
DEFINE_IFUNC_FOR(memcpy) {
- return memmove_resolver(hwcap);
+ return memmove_resolver(hwcap);
}
+MEMCPY_SHIM()
-typedef void* __memcpy_func(void*, const void*, size_t);
+// On arm32, __memcpy() is not publicly exposed, but gets called by memmove()
+// in cases where the copy is known to be overlap-safe.
+typedef void* __memcpy_func_t(void*, const void*, size_t);
DEFINE_IFUNC_FOR(__memcpy) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- RETURN_FUNC(__memcpy_func, __memcpy_a7);
- case kCortexA9:
- RETURN_FUNC(__memcpy_func, __memcpy_a9);
- case kKrait:
- RETURN_FUNC(__memcpy_func, __memcpy_krait);
- case kCortexA53:
- RETURN_FUNC(__memcpy_func, __memcpy_a53);
- case kCortexA55:
- RETURN_FUNC(__memcpy_func, __memcpy_a55);
- case kKryo:
- RETURN_FUNC(__memcpy_func, __memcpy_kryo);
- default:
- RETURN_FUNC(__memcpy_func, __memcpy_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a7);
+ case kCortexA9:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a9);
+ case kKrait:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_krait);
+ case kCortexA53:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a53);
+ case kCortexA55:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a55);
+ case kKryo:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_kryo);
+ default:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a15);
+ }
}
+DEFINE_STATIC_SHIM(void* __memcpy(void* dst, const void* src, size_t n) {
+ FORWARD(__memcpy)(dst, src, n);
+})
-typedef void* __memset_chk_func(void* s, int c, size_t n, size_t n2);
DEFINE_IFUNC_FOR(__memset_chk) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- case kCortexA53:
- case kCortexA55:
- case kKryo:
- RETURN_FUNC(__memset_chk_func, __memset_chk_a7);
- case kCortexA9:
- RETURN_FUNC(__memset_chk_func, __memset_chk_a9);
- case kKrait:
- RETURN_FUNC(__memset_chk_func, __memset_chk_krait);
- default:
- RETURN_FUNC(__memset_chk_func, __memset_chk_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ case kCortexA53:
+ case kCortexA55:
+ case kKryo:
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_a7);
+ case kCortexA9:
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_a9);
+ case kKrait:
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_krait);
+ default:
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_a15);
+ }
}
+__MEMSET_CHK_SHIM()
-typedef void* memset_func(void* __dst, int __ch, size_t __n);
DEFINE_IFUNC_FOR(memset) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- case kCortexA53:
- case kCortexA55:
- case kKryo:
- RETURN_FUNC(memset_func, memset_a7);
- case kCortexA9:
- RETURN_FUNC(memset_func, memset_a9);
- case kKrait:
- RETURN_FUNC(memset_func, memset_krait);
- default:
- RETURN_FUNC(memset_func, memset_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ case kCortexA53:
+ case kCortexA55:
+ case kKryo:
+ RETURN_FUNC(memset_func_t, memset_a7);
+ case kCortexA9:
+ RETURN_FUNC(memset_func_t, memset_a9);
+ case kKrait:
+ RETURN_FUNC(memset_func_t, memset_krait);
+ default:
+ RETURN_FUNC(memset_func_t, memset_a15);
+ }
}
+MEMSET_SHIM()
-typedef char* strcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcpy) {
- switch(get_cpu_variant()) {
- case kCortexA9:
- RETURN_FUNC(strcpy_func, strcpy_a9);
- default:
- RETURN_FUNC(strcpy_func, strcpy_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA9:
+ RETURN_FUNC(strcpy_func_t, strcpy_a9);
+ default:
+ RETURN_FUNC(strcpy_func_t, strcpy_a15);
+ }
}
+STRCPY_SHIM()
-typedef char* __strcpy_chk_func(char* dst, const char* src, size_t dst_len);
DEFINE_IFUNC_FOR(__strcpy_chk) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a7);
- case kCortexA9:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a9);
- case kKrait:
- case kKryo:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_krait);
- case kCortexA53:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a53);
- case kCortexA55:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a55);
- default:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a7);
+ case kCortexA9:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a9);
+ case kKrait:
+ case kKryo:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_krait);
+ case kCortexA53:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a53);
+ case kCortexA55:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a55);
+ default:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a15);
+ }
}
+__STRCPY_CHK_SHIM()
-typedef char* stpcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(stpcpy) {
- switch(get_cpu_variant()) {
- case kCortexA9:
- RETURN_FUNC(stpcpy_func, stpcpy_a9);
- default:
- RETURN_FUNC(stpcpy_func, stpcpy_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA9:
+ RETURN_FUNC(stpcpy_func_t, stpcpy_a9);
+ default:
+ RETURN_FUNC(stpcpy_func_t, stpcpy_a15);
+ }
}
+STPCPY_SHIM()
-typedef char* strcat_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcat) {
- switch(get_cpu_variant()) {
- case kCortexA9:
- RETURN_FUNC(strcat_func, strcat_a9);
- default:
- RETURN_FUNC(strcat_func, strcat_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA9:
+ RETURN_FUNC(strcat_func_t, strcat_a9);
+ default:
+ RETURN_FUNC(strcat_func_t, strcat_a15);
+ }
}
+STRCAT_SHIM()
-typedef char* __strcat_chk_func(char* dst, const char* src, size_t dst_buf_size);
DEFINE_IFUNC_FOR(__strcat_chk) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a7);
- case kCortexA9:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a9);
- case kKrait:
- case kKryo:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_krait);
- case kCortexA53:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a53);
- case kCortexA55:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a55);
- default:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a7);
+ case kCortexA9:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a9);
+ case kKrait:
+ case kKryo:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_krait);
+ case kCortexA53:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a53);
+ case kCortexA55:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a55);
+ default:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a15);
+ }
}
+__STRCAT_CHK_SHIM()
-typedef int strcmp_func(const char* __lhs, const char* __rhs);
DEFINE_IFUNC_FOR(strcmp) {
- RETURN_FUNC(strcmp_func, strcmp_a15);
+ RETURN_FUNC(strcmp_func_t, strcmp_a15);
}
+STRCMP_SHIM()
-typedef size_t strlen_func(const char* __s);
DEFINE_IFUNC_FOR(strlen) {
- switch(get_cpu_variant()) {
- case kCortexA9:
- RETURN_FUNC(strlen_func, strlen_a9);
- default:
- RETURN_FUNC(strlen_func, strlen_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA9:
+ RETURN_FUNC(strlen_func_t, strlen_a9);
+ default:
+ RETURN_FUNC(strlen_func_t, strlen_a15);
+ }
}
+STRLEN_SHIM()
} // extern "C"
diff --git a/libc/arch-arm/static_function_dispatch.S b/libc/arch-arm/static_function_dispatch.S
deleted file mode 100644
index a8235c2..0000000
--- a/libc/arch-arm/static_function_dispatch.S
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- b impl; \
-END(name)
-
-FUNCTION_DELEGATE(memmove, memmove_generic)
-FUNCTION_DELEGATE(memcpy, memmove_generic)
-FUNCTION_DELEGATE(memset, memset_generic)
-FUNCTION_DELEGATE(__memset_chk, __memset_chk_generic)
-FUNCTION_DELEGATE(strcpy, strcpy_generic)
-FUNCTION_DELEGATE(__strcpy_chk, __strcpy_chk_generic)
-FUNCTION_DELEGATE(stpcpy, stpcpy_generic)
-FUNCTION_DELEGATE(strcat, strcat_generic)
-FUNCTION_DELEGATE(__strcat_chk, __strcat_chk_generic)
-FUNCTION_DELEGATE(strcmp, strcmp_generic)
-FUNCTION_DELEGATE(strlen, strlen_generic)
diff --git a/libc/arch-arm64/dynamic_function_dispatch.cpp b/libc/arch-arm64/dynamic_function_dispatch.cpp
index f9e4263..29e47b3 100644
--- a/libc/arch-arm64/dynamic_function_dispatch.cpp
+++ b/libc/arch-arm64/dynamic_function_dispatch.cpp
@@ -28,148 +28,147 @@
#include <private/bionic_ifuncs.h>
#include <stddef.h>
-#include <sys/auxv.h>
static inline bool __bionic_is_oryon(unsigned long hwcap) {
- if (!(hwcap & HWCAP_CPUID)) return false;
+ if (!(hwcap & HWCAP_CPUID)) return false;
- // Extract the implementor and variant bits from MIDR_EL1.
- // https://www.kernel.org/doc/html/latest/arch/arm64/cpu-feature-registers.html#list-of-registers-with-visible-features
- unsigned long midr;
- __asm__ __volatile__("mrs %0, MIDR_EL1" : "=r"(midr));
- uint16_t cpu = (midr >> 20) & 0xfff;
+ // Extract the implementor and variant bits from MIDR_EL1.
+ // https://www.kernel.org/doc/html/latest/arch/arm64/cpu-feature-registers.html#list-of-registers-with-visible-features
+ unsigned long midr;
+ __asm__ __volatile__("mrs %0, MIDR_EL1" : "=r"(midr));
+ uint16_t cpu = (midr >> 20) & 0xfff;
- auto make_cpu = [](unsigned implementor, unsigned variant) {
- return (implementor << 4) | variant;
- };
+ auto make_cpu = [](unsigned implementor, unsigned variant) {
+ return (implementor << 4) | variant;
+ };
- // Check for implementor Qualcomm's variants 0x1..0x5 (Oryon).
- return cpu >= make_cpu('Q', 0x1) && cpu <= make_cpu('Q', 0x5);
+ // Check for implementor Qualcomm's variants 0x1..0x5 (Oryon).
+ return cpu >= make_cpu('Q', 0x1) && cpu <= make_cpu('Q', 0x5);
}
extern "C" {
-typedef void* memchr_func(const void*, int, size_t);
DEFINE_IFUNC_FOR(memchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(memchr_func, __memchr_aarch64_mte);
- } else {
- RETURN_FUNC(memchr_func, __memchr_aarch64);
- }
-}
-
-typedef int memcmp_func(const void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memcmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(memcmp_func, __memcmp_aarch64);
-}
-
-typedef void* memcpy_func(void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memcpy) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memcpy_func, __memmove_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64_nt);
- } else if (arg->_hwcap & HWCAP_ASIMD) {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64_simd);
- } else {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64);
- }
-}
-
-typedef void* memmove_func(void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memmove) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memmove_func, __memmove_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memcpy_func, __memmove_aarch64_nt);
- } else if (arg->_hwcap & HWCAP_ASIMD) {
- RETURN_FUNC(memmove_func, __memmove_aarch64_simd);
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(memchr_func_t, __memchr_aarch64_mte);
} else {
- RETURN_FUNC(memmove_func, __memmove_aarch64);
+ RETURN_FUNC(memchr_func_t, __memchr_aarch64);
}
}
+MEMCHR_SHIM()
-typedef int memrchr_func(const void*, int, size_t);
+DEFINE_IFUNC_FOR(memcmp) {
+ // TODO: enable the SVE version.
+ RETURN_FUNC(memcmp_func_t, __memcmp_aarch64);
+}
+MEMCMP_SHIM()
+
+DEFINE_IFUNC_FOR(memcpy) {
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memcpy_func_t, __memmove_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64_nt);
+ } else if (arg->_hwcap & HWCAP_ASIMD) {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64_simd);
+ } else {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64);
+ }
+}
+MEMCPY_SHIM()
+
+DEFINE_IFUNC_FOR(memmove) {
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_nt);
+ } else if (arg->_hwcap & HWCAP_ASIMD) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_simd);
+ } else {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64);
+ }
+}
+MEMMOVE_SHIM()
+
DEFINE_IFUNC_FOR(memrchr) {
- RETURN_FUNC(memrchr_func, __memrchr_aarch64);
+ RETURN_FUNC(memrchr_func_t, __memrchr_aarch64);
}
+MEMRCHR_SHIM()
-typedef int memset_func(void*, int, size_t);
DEFINE_IFUNC_FOR(memset) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memset_func, __memset_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memset_func, __memset_aarch64_nt);
- } else {
- RETURN_FUNC(memset_func, __memset_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memset_func_t, __memset_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memset_func_t, __memset_aarch64_nt);
+ } else {
+ RETURN_FUNC(memset_func_t, __memset_aarch64);
+ }
}
+MEMSET_SHIM()
-typedef char* stpcpy_func(char*, const char*, size_t);
DEFINE_IFUNC_FOR(stpcpy) {
- // TODO: enable the SVE version.
- RETURN_FUNC(stpcpy_func, __stpcpy_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(stpcpy_func_t, __stpcpy_aarch64);
}
+STPCPY_SHIM()
-typedef char* strchr_func(const char*, int);
DEFINE_IFUNC_FOR(strchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strchr_func, __strchr_aarch64_mte);
- } else {
- RETURN_FUNC(strchr_func, __strchr_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strchr_func_t, __strchr_aarch64_mte);
+ } else {
+ RETURN_FUNC(strchr_func_t, __strchr_aarch64);
+ }
}
+STRCHR_SHIM()
-typedef char* strchrnul_func(const char*, int);
DEFINE_IFUNC_FOR(strchrnul) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strchrnul_func, __strchrnul_aarch64_mte);
- } else {
- RETURN_FUNC(strchrnul_func, __strchrnul_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strchrnul_func_t, __strchrnul_aarch64_mte);
+ } else {
+ RETURN_FUNC(strchrnul_func_t, __strchrnul_aarch64);
+ }
}
+STRCHRNUL_SHIM()
-typedef int strcmp_func(const char*, const char*);
DEFINE_IFUNC_FOR(strcmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strcmp_func, __strcmp_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strcmp_func_t, __strcmp_aarch64);
}
+STRCMP_SHIM()
-typedef char* strcpy_func(char*, const char*);
DEFINE_IFUNC_FOR(strcpy) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strcpy_func, __strcpy_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strcpy_func_t, __strcpy_aarch64);
}
+STRCPY_SHIM()
-typedef size_t strlen_func(const char*);
DEFINE_IFUNC_FOR(strlen) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strlen_func, __strlen_aarch64_mte);
- } else {
- RETURN_FUNC(strlen_func, __strlen_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strlen_func_t, __strlen_aarch64_mte);
+ } else {
+ RETURN_FUNC(strlen_func_t, __strlen_aarch64);
+ }
}
+STRLEN_SHIM()
-typedef int strncmp_func(const char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strncmp_func, __strncmp_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strncmp_func_t, __strncmp_aarch64);
}
+STRNCMP_SHIM()
-typedef size_t strnlen_func(const char*, size_t);
DEFINE_IFUNC_FOR(strnlen) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strnlen_func, __strnlen_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strnlen_func_t, __strnlen_aarch64);
}
+STRNLEN_SHIM()
-typedef char* strrchr_func(const char*, int);
DEFINE_IFUNC_FOR(strrchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strrchr_func, __strrchr_aarch64_mte);
- } else {
- RETURN_FUNC(strrchr_func, __strrchr_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strrchr_func_t, __strrchr_aarch64_mte);
+ } else {
+ RETURN_FUNC(strrchr_func_t, __strrchr_aarch64);
+ }
}
+STRRCHR_SHIM()
} // extern "C"
diff --git a/libc/arch-arm64/static_function_dispatch.S b/libc/arch-arm64/static_function_dispatch.S
deleted file mode 100644
index 18c3783..0000000
--- a/libc/arch-arm64/static_function_dispatch.S
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (C) 2019 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- b impl; \
-END(name)
-
-FUNCTION_DELEGATE(memchr, __memchr_aarch64_mte)
-FUNCTION_DELEGATE(memcmp, __memcmp_aarch64)
-FUNCTION_DELEGATE(memcpy, __memcpy_aarch64)
-FUNCTION_DELEGATE(memmove, __memmove_aarch64)
-FUNCTION_DELEGATE(memrchr, __memrchr_aarch64)
-FUNCTION_DELEGATE(memset, __memset_aarch64)
-FUNCTION_DELEGATE(stpcpy, __stpcpy_aarch64)
-FUNCTION_DELEGATE(strchr, __strchr_aarch64_mte)
-FUNCTION_DELEGATE(strchrnul, __strchrnul_aarch64_mte)
-FUNCTION_DELEGATE(strcmp, __strcmp_aarch64)
-FUNCTION_DELEGATE(strcpy, __strcpy_aarch64)
-FUNCTION_DELEGATE(strlen, __strlen_aarch64_mte)
-FUNCTION_DELEGATE(strrchr, __strrchr_aarch64_mte)
-FUNCTION_DELEGATE(strncmp, __strncmp_aarch64)
-FUNCTION_DELEGATE(strnlen, __strnlen_aarch64)
-
-NOTE_GNU_PROPERTY()
diff --git a/libc/arch-riscv64/dynamic_function_dispatch.cpp b/libc/arch-riscv64/dynamic_function_dispatch.cpp
index bb2ba51..ce6c028 100644
--- a/libc/arch-riscv64/dynamic_function_dispatch.cpp
+++ b/libc/arch-riscv64/dynamic_function_dispatch.cpp
@@ -27,12 +27,11 @@
*/
#include <fcntl.h>
+#include <private/bionic_ifuncs.h>
#include <stddef.h>
#include <sys/syscall.h>
#include <unistd.h>
-#include <private/bionic_ifuncs.h>
-
extern "C" {
static inline __always_inline int ifunc_faccessat(int dir_fd, const char* path, int mode) {
@@ -53,94 +52,94 @@
return result;
}
-typedef void* memchr_func(const void*, int, size_t);
DEFINE_IFUNC_FOR(memchr) {
- if (have_fast_v()) RETURN_FUNC(memchr_func, memchr_v);
- RETURN_FUNC(memchr_func, memchr_gc);
+ if (have_fast_v()) RETURN_FUNC(memchr_func_t, memchr_v);
+ RETURN_FUNC(memchr_func_t, memchr_gc);
}
+MEMCHR_SHIM()
-typedef int memcmp_func(const void*, const void*, size_t);
DEFINE_IFUNC_FOR(memcmp) {
- if (have_fast_v()) RETURN_FUNC(memcmp_func, memcmp_v);
- RETURN_FUNC(memcmp_func, memcmp_gc);
+ if (have_fast_v()) RETURN_FUNC(memcmp_func_t, memcmp_v);
+ RETURN_FUNC(memcmp_func_t, memcmp_gc);
}
+MEMCMP_SHIM()
-typedef void* memcpy_func(void*, const void*, size_t);
DEFINE_IFUNC_FOR(memcpy) {
- if (have_fast_v()) RETURN_FUNC(memcpy_func, memcpy_v);
- RETURN_FUNC(memcpy_func, memcpy_gc);
+ if (have_fast_v()) RETURN_FUNC(memcpy_func_t, memcpy_v);
+ RETURN_FUNC(memcpy_func_t, memcpy_gc);
}
+MEMCPY_SHIM()
-typedef void* memmove_func(void*, const void*, size_t);
DEFINE_IFUNC_FOR(memmove) {
- if (have_fast_v()) RETURN_FUNC(memmove_func, memmove_v);
- RETURN_FUNC(memmove_func, memmove_gc);
+ if (have_fast_v()) RETURN_FUNC(memmove_func_t, memmove_v);
+ RETURN_FUNC(memmove_func_t, memmove_gc);
}
+MEMMOVE_SHIM()
-typedef void* memset_func(void*, int, size_t);
DEFINE_IFUNC_FOR(memset) {
- if (have_fast_v()) RETURN_FUNC(memset_func, memset_v);
- RETURN_FUNC(memset_func, memset_gc);
+ if (have_fast_v()) RETURN_FUNC(memset_func_t, memset_v);
+ RETURN_FUNC(memset_func_t, memset_gc);
}
+MEMSET_SHIM()
-typedef char* stpcpy_func(char*, const char*);
DEFINE_IFUNC_FOR(stpcpy) {
- if (have_fast_v()) RETURN_FUNC(stpcpy_func, stpcpy_v);
- RETURN_FUNC(stpcpy_func, stpcpy_gc);
+ if (have_fast_v()) RETURN_FUNC(stpcpy_func_t, stpcpy_v);
+ RETURN_FUNC(stpcpy_func_t, stpcpy_gc);
}
+STPCPY_SHIM()
-typedef char* strcat_func(char*, const char*);
DEFINE_IFUNC_FOR(strcat) {
- if (have_fast_v()) RETURN_FUNC(strcat_func, strcat_v);
- RETURN_FUNC(strcat_func, strcat_gc);
+ if (have_fast_v()) RETURN_FUNC(strcat_func_t, strcat_v);
+ RETURN_FUNC(strcat_func_t, strcat_gc);
}
+STRCAT_SHIM()
-typedef char* strchr_func(const char*, int);
DEFINE_IFUNC_FOR(strchr) {
- if (have_fast_v()) RETURN_FUNC(strchr_func, strchr_v);
- RETURN_FUNC(strchr_func, strchr_gc);
+ if (have_fast_v()) RETURN_FUNC(strchr_func_t, strchr_v);
+ RETURN_FUNC(strchr_func_t, strchr_gc);
}
+STRCHR_SHIM()
-typedef int strcmp_func(const char*, const char*);
DEFINE_IFUNC_FOR(strcmp) {
- if (have_fast_v()) RETURN_FUNC(strcmp_func, strcmp_v);
- RETURN_FUNC(strcmp_func, strcmp_gc);
+ if (have_fast_v()) RETURN_FUNC(strcmp_func_t, strcmp_v);
+ RETURN_FUNC(strcmp_func_t, strcmp_gc);
}
+STRCMP_SHIM()
-typedef char* strcpy_func(char*, const char*);
DEFINE_IFUNC_FOR(strcpy) {
- if (have_fast_v()) RETURN_FUNC(strcpy_func, strcpy_v);
- RETURN_FUNC(strcpy_func, strcpy_gc);
+ if (have_fast_v()) RETURN_FUNC(strcpy_func_t, strcpy_v);
+ RETURN_FUNC(strcpy_func_t, strcpy_gc);
}
+STRCPY_SHIM()
-typedef size_t strlen_func(const char*);
DEFINE_IFUNC_FOR(strlen) {
- if (have_fast_v()) RETURN_FUNC(strlen_func, strlen_v);
- RETURN_FUNC(strlen_func, strlen_gc);
+ if (have_fast_v()) RETURN_FUNC(strlen_func_t, strlen_v);
+ RETURN_FUNC(strlen_func_t, strlen_gc);
}
+STRLEN_SHIM()
-typedef char* strncat_func(char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncat) {
- if (have_fast_v()) RETURN_FUNC(strncat_func, strncat_v);
- RETURN_FUNC(strncat_func, strncat_gc);
+ if (have_fast_v()) RETURN_FUNC(strncat_func_t, strncat_v);
+ RETURN_FUNC(strncat_func_t, strncat_gc);
}
+STRNCAT_SHIM()
-typedef int strncmp_func(const char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncmp) {
- if (have_fast_v()) RETURN_FUNC(strncmp_func, strncmp_v);
- RETURN_FUNC(strncmp_func, strncmp_gc);
+ if (have_fast_v()) RETURN_FUNC(strncmp_func_t, strncmp_v);
+ RETURN_FUNC(strncmp_func_t, strncmp_gc);
}
+STRNCMP_SHIM()
-typedef char* strncpy_func(char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncpy) {
- if (have_fast_v()) RETURN_FUNC(strncpy_func, strncpy_v);
- RETURN_FUNC(strncpy_func, strncpy_gc);
+ if (have_fast_v()) RETURN_FUNC(strncpy_func_t, strncpy_v);
+ RETURN_FUNC(strncpy_func_t, strncpy_gc);
}
+STRNCPY_SHIM()
-typedef size_t strnlen_func(const char*, size_t);
DEFINE_IFUNC_FOR(strnlen) {
- if (have_fast_v()) RETURN_FUNC(strnlen_func, strnlen_v);
- RETURN_FUNC(strnlen_func, strnlen_gc);
+ if (have_fast_v()) RETURN_FUNC(strnlen_func_t, strnlen_v);
+ RETURN_FUNC(strnlen_func_t, strnlen_gc);
}
+STRNLEN_SHIM()
} // extern "C"
diff --git a/libc/arch-riscv64/static_function_dispatch.S b/libc/arch-riscv64/static_function_dispatch.S
deleted file mode 100644
index accfec8..0000000
--- a/libc/arch-riscv64/static_function_dispatch.S
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (C) 2023 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- j impl; \
-END(name)
-
-// TODO: switch to the V variants when qemu is fixed.
-FUNCTION_DELEGATE(memchr, memchr_gc)
-FUNCTION_DELEGATE(memcmp, memcmp_gc)
-FUNCTION_DELEGATE(memcpy, memcpy_gc)
-FUNCTION_DELEGATE(memmove, memmove_gc)
-FUNCTION_DELEGATE(memset, memset_gc)
-FUNCTION_DELEGATE(stpcpy, stpcpy_gc)
-FUNCTION_DELEGATE(strcat, strcat_gc)
-FUNCTION_DELEGATE(strchr, strchr_gc)
-FUNCTION_DELEGATE(strcmp, strcmp_gc)
-FUNCTION_DELEGATE(strcpy, strcpy_gc)
-FUNCTION_DELEGATE(strlen, strlen_gc)
-FUNCTION_DELEGATE(strncat, strncat_gc)
-FUNCTION_DELEGATE(strncmp, strncmp_gc)
-FUNCTION_DELEGATE(strncpy, strncpy_gc)
-FUNCTION_DELEGATE(strnlen, strnlen_gc)
diff --git a/libc/arch-x86/dynamic_function_dispatch.cpp b/libc/arch-x86/dynamic_function_dispatch.cpp
index 38d8a0a..9d7a4c9 100644
--- a/libc/arch-x86/dynamic_function_dispatch.cpp
+++ b/libc/arch-x86/dynamic_function_dispatch.cpp
@@ -26,129 +26,150 @@
* SUCH DAMAGE.
*/
-#include <stddef.h>
-
#include <private/bionic_ifuncs.h>
+#include <stddef.h>
extern "C" {
-typedef int memcmp_func(const void* __lhs, const void* __rhs, size_t __n);
DEFINE_IFUNC_FOR(memcmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memcmp_func, memcmp_atom);
- if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(memcmp_func, memcmp_sse4);
- RETURN_FUNC(memcmp_func, memcmp_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(memcmp_func_t, memcmp_atom);
+ if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(memcmp_func_t, memcmp_sse4);
+ RETURN_FUNC(memcmp_func_t, memcmp_generic);
}
+MEMCMP_SHIM()
-typedef void* memset_func(void* __dst, int __ch, size_t __n);
DEFINE_IFUNC_FOR(memset) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memset_func, memset_atom);
- RETURN_FUNC(memset_func, memset_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(memset_func_t, memset_atom);
+ RETURN_FUNC(memset_func_t, memset_generic);
}
+MEMSET_SHIM()
-typedef void* __memset_chk_func(void *s, int c, size_t n, size_t n2);
DEFINE_IFUNC_FOR(__memset_chk) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(__memset_chk_func, __memset_chk_atom);
- RETURN_FUNC(__memset_chk_func, __memset_chk_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(__memset_chk_func_t, __memset_chk_atom);
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_generic);
}
+__MEMSET_CHK_SHIM()
-typedef void* memmove_func(void* __dst, const void* __src, size_t __n);
DEFINE_IFUNC_FOR(memmove) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memmove_func, memmove_atom);
- RETURN_FUNC(memmove_func, memmove_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(memmove_func_t, memmove_atom);
+ RETURN_FUNC(memmove_func_t, memmove_generic);
}
+MEMMOVE_SHIM()
-typedef void* memcpy_func(void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memcpy) {
- return memmove_resolver();
-}
+DEFINE_IFUNC_FOR(memcpy) { return memmove_resolver(); }
+MEMCPY_SHIM()
-typedef char* strcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strcpy_func, strcpy_atom);
- RETURN_FUNC(strcpy_func, strcpy_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(strcpy_func_t, strcpy_atom);
+ RETURN_FUNC(strcpy_func_t, strcpy_generic);
}
+STRCPY_SHIM()
-typedef char* strncpy_func(char* __dst, const char* __src, size_t __n);
DEFINE_IFUNC_FOR(strncpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strncpy_func, strncpy_atom);
- RETURN_FUNC(strncpy_func, strncpy_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(strncpy_func_t, strncpy_atom);
+ RETURN_FUNC(strncpy_func_t, strncpy_generic);
}
+STRNCPY_SHIM()
-typedef size_t strlen_func(const char* __s);
DEFINE_IFUNC_FOR(strlen) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strlen_func, strlen_atom);
- RETURN_FUNC(strlen_func, strlen_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(strlen_func_t, strlen_atom);
+ RETURN_FUNC(strlen_func_t, strlen_generic);
}
+STRLEN_SHIM()
-typedef int wmemcmp_func(const wchar_t* __lhs, const wchar_t* __rhs, size_t __n);
-DEFINE_IFUNC_FOR(wmemcmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(wmemcmp_func, wmemcmp_sse4);
- if (__builtin_cpu_is("atom")) RETURN_FUNC(wmemcmp_func, wmemcmp_atom);
- RETURN_FUNC(wmemcmp_func, wmemcmp_freebsd);
-}
-
-typedef int strcmp_func(const char* __lhs, const char* __rhs);
DEFINE_IFUNC_FOR(strcmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcmp_func, strcmp_ssse3);
- RETURN_FUNC(strcmp_func, strcmp_generic);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcmp_func_t, strcmp_ssse3);
+ RETURN_FUNC(strcmp_func_t, strcmp_generic);
}
+STRCMP_SHIM()
-typedef int strncmp_func(const char* __lhs, const char* __rhs, size_t __n);
DEFINE_IFUNC_FOR(strncmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strncmp_func, strncmp_ssse3);
- RETURN_FUNC(strncmp_func, strncmp_generic);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3"))
+ RETURN_FUNC(strncmp_func_t, strncmp_ssse3);
+ RETURN_FUNC(strncmp_func_t, strncmp_generic);
}
+STRNCMP_SHIM()
-typedef char* strcat_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcat) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcat_func, strcat_ssse3);
- RETURN_FUNC(strcat_func, strcat_generic);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcat_func_t, strcat_ssse3);
+ RETURN_FUNC(strcat_func_t, strcat_generic);
}
+STRCAT_SHIM()
-typedef char* strncat_func(char* __dst, const char* __src, size_t __n);
DEFINE_IFUNC_FOR(strncat) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strncat_func, strncat_ssse3);
- RETURN_FUNC(strncat_func, strncat_openbsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strncat_func_t, strncat_ssse3);
+ RETURN_FUNC(strncat_func_t, strncat_openbsd);
}
+STRNCAT_SHIM()
-typedef size_t strlcat_func(char *dst, const char *src, size_t dsize);
+typedef size_t strlcat_func_t(char*, const char*, size_t);
DEFINE_IFUNC_FOR(strlcat) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcat_func, strlcat_ssse3);
- RETURN_FUNC(strlcat_func, strlcat_openbsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcat_func_t, strlcat_ssse3);
+ RETURN_FUNC(strlcat_func_t, strlcat_openbsd);
}
+DEFINE_STATIC_SHIM(size_t strlcat(char* dst, const char* src, size_t n) {
+ FORWARD(strlcat)(dst, src, n);
+})
-typedef size_t strlcpy_func(char *dst, const char *src, size_t dsize);
+typedef size_t strlcpy_func_t(char*, const char*, size_t);
DEFINE_IFUNC_FOR(strlcpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcpy_func, strlcpy_ssse3);
- RETURN_FUNC(strlcpy_func, strlcpy_openbsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcpy_func_t, strlcpy_ssse3);
+ RETURN_FUNC(strlcpy_func_t, strlcpy_openbsd);
}
+DEFINE_STATIC_SHIM(size_t strlcpy(char* dst, const char* src, size_t n) {
+ FORWARD(strlcpy)(dst, src, n);
+})
-typedef wchar_t* wcscat_func(wchar_t *s1, const wchar_t *s2);
+typedef wchar_t* wcscat_func_t(wchar_t*, const wchar_t*);
DEFINE_IFUNC_FOR(wcscat) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscat_func, wcscat_ssse3);
- RETURN_FUNC(wcscat_func, wcscat_freebsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscat_func_t, wcscat_ssse3);
+ RETURN_FUNC(wcscat_func_t, wcscat_freebsd);
}
+DEFINE_STATIC_SHIM(wchar_t* wcscat(wchar_t* dst, const wchar_t* src) {
+ FORWARD(wcscat)(dst, src);
+})
-typedef wchar_t* wcscpy_func(wchar_t *s1, const wchar_t *s2);
+typedef wchar_t* wcscpy_func_t(wchar_t*, const wchar_t*);
DEFINE_IFUNC_FOR(wcscpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscpy_func, wcscpy_ssse3);
- RETURN_FUNC(wcscpy_func, wcscpy_freebsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscpy_func_t, wcscpy_ssse3);
+ RETURN_FUNC(wcscpy_func_t, wcscpy_freebsd);
}
+DEFINE_STATIC_SHIM(wchar_t* wcscpy(wchar_t* dst, const wchar_t* src) {
+ FORWARD(wcscpy)(dst, src);
+})
+
+typedef int wmemcmp_func_t(const wchar_t*, const wchar_t*, size_t);
+DEFINE_IFUNC_FOR(wmemcmp) {
+ __builtin_cpu_init();
+ if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(wmemcmp_func_t, wmemcmp_sse4);
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(wmemcmp_func_t, wmemcmp_atom);
+ RETURN_FUNC(wmemcmp_func_t, wmemcmp_freebsd);
+}
+DEFINE_STATIC_SHIM(int wmemcmp(const wchar_t* lhs, const wchar_t* rhs, size_t n) {
+ FORWARD(wmemcmp)(lhs, rhs, n);
+})
} // extern "C"
diff --git a/libc/arch-x86/static_function_dispatch.S b/libc/arch-x86/static_function_dispatch.S
deleted file mode 100644
index 7e8e63d..0000000
--- a/libc/arch-x86/static_function_dispatch.S
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- jmp impl; \
-END(name)
-
-FUNCTION_DELEGATE(memcmp, memcmp_generic)
-FUNCTION_DELEGATE(memset, memset_generic)
-FUNCTION_DELEGATE(__memset_chk, __memset_chk_generic)
-FUNCTION_DELEGATE(memcpy, memmove_generic)
-FUNCTION_DELEGATE(memmove, memmove_generic)
-FUNCTION_DELEGATE(strcpy, strcpy_generic)
-FUNCTION_DELEGATE(strncpy, strncpy_generic)
-FUNCTION_DELEGATE(strlen, strlen_generic)
-FUNCTION_DELEGATE(strcmp, strcmp_generic)
-FUNCTION_DELEGATE(strncmp, strncmp_generic)
-FUNCTION_DELEGATE(strcat, strcat_generic)
-FUNCTION_DELEGATE(wmemcmp, wmemcmp_freebsd)
-FUNCTION_DELEGATE(wcscat, wcscat_freebsd)
-FUNCTION_DELEGATE(strncat, strncat_openbsd)
-FUNCTION_DELEGATE(strlcat, strlcat_openbsd)
-FUNCTION_DELEGATE(strlcpy, strlcpy_openbsd)
-FUNCTION_DELEGATE(wcscpy, wcscpy_freebsd)
diff --git a/libc/arch-x86_64/dynamic_function_dispatch.cpp b/libc/arch-x86_64/dynamic_function_dispatch.cpp
index c846ded..cbe68a3 100644
--- a/libc/arch-x86_64/dynamic_function_dispatch.cpp
+++ b/libc/arch-x86_64/dynamic_function_dispatch.cpp
@@ -32,18 +32,18 @@
extern "C" {
-typedef int memset_func(void* __dst, int __ch, size_t __n);
DEFINE_IFUNC_FOR(memset) {
__builtin_cpu_init();
- if (__builtin_cpu_supports("avx2")) RETURN_FUNC(memset_func, memset_avx2);
- RETURN_FUNC(memset_func, memset_generic);
+ if (__builtin_cpu_supports("avx2")) RETURN_FUNC(memset_func_t, memset_avx2);
+ RETURN_FUNC(memset_func_t, memset_generic);
}
+MEMSET_SHIM()
-typedef void* __memset_chk_func(void* s, int c, size_t n, size_t n2);
DEFINE_IFUNC_FOR(__memset_chk) {
__builtin_cpu_init();
- if (__builtin_cpu_supports("avx2")) RETURN_FUNC(__memset_chk_func, __memset_chk_avx2);
- RETURN_FUNC(__memset_chk_func, __memset_chk_generic);
+ if (__builtin_cpu_supports("avx2")) RETURN_FUNC(__memset_chk_func_t, __memset_chk_avx2);
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_generic);
}
+__MEMSET_CHK_SHIM()
} // extern "C"
diff --git a/libc/arch-x86_64/static_function_dispatch.S b/libc/arch-x86_64/static_function_dispatch.S
deleted file mode 100644
index 93ff5f2..0000000
--- a/libc/arch-x86_64/static_function_dispatch.S
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2022 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- jmp impl; \
-END(name)
-
-FUNCTION_DELEGATE(memset, memset_generic)
-FUNCTION_DELEGATE(__memset_chk, __memset_chk_generic)
diff --git a/libc/bionic/getauxval.cpp b/libc/bionic/getauxval.cpp
index e188d73..51ff949 100644
--- a/libc/bionic/getauxval.cpp
+++ b/libc/bionic/getauxval.cpp
@@ -26,14 +26,12 @@
* SUCH DAMAGE.
*/
-#include <stddef.h>
-#include <sys/cdefs.h>
-#include <sys/auxv.h>
-#include <private/bionic_auxv.h>
-#include <private/bionic_globals.h>
-#include <private/bionic_ifuncs.h>
#include <elf.h>
#include <errno.h>
+#include <private/bionic_auxv.h>
+#include <private/bionic_globals.h>
+#include <stddef.h>
+#include <sys/auxv.h>
// This function needs to be safe to call before TLS is set up, so it can't
// access errno or the stack protector.
diff --git a/libc/private/bionic_ifuncs.h b/libc/private/bionic_ifuncs.h
index e6b349a..b31c903 100644
--- a/libc/private/bionic_ifuncs.h
+++ b/libc/private/bionic_ifuncs.h
@@ -31,6 +31,8 @@
#include <stdint.h>
#include <sys/ifunc.h>
+#include <private/bionic_call_ifunc_resolver.h>
+
#if defined(__aarch64__)
#define IFUNC_ARGS (uint64_t hwcap __attribute__((unused)), \
__ifunc_arg_t* arg __attribute__((unused)))
@@ -40,14 +42,6 @@
#define IFUNC_ARGS ()
#endif
-// We can't have HWASAN enabled in resolvers because they may be called before HWASAN is
-// initialized.
-#define DEFINE_IFUNC_FOR(name) \
- name##_func name __attribute__((ifunc(#name "_resolver"))); \
- __attribute__((visibility("hidden"))) \
- __attribute__((no_sanitize("hwaddress"))) \
- name##_func* name##_resolver IFUNC_ARGS
-
#define DECLARE_FUNC(type, name) \
__attribute__((visibility("hidden"))) \
type name
@@ -56,3 +50,137 @@
DECLARE_FUNC(type, name); \
return name; \
}
+
+#if defined(BIONIC_DYNAMIC_DISPATCH)
+
+// We can't have HWASAN enabled in resolvers because they may be called before
+// HWASAN is initialized.
+#define DEFINE_IFUNC_FOR(name) \
+ name##_func_t name __attribute__((ifunc(#name "_resolver"))); \
+ __attribute__((visibility("hidden"))) \
+ __attribute__((no_sanitize("hwaddress"))) name##_func_t* name##_resolver IFUNC_ARGS
+
+#define DEFINE_STATIC_SHIM(x)
+
+#elif defined(BIONIC_STATIC_DISPATCH)
+
+#define DEFINE_IFUNC_FOR(name) \
+ name##_func_t* name##_resolver IFUNC_ARGS; \
+ __attribute__((visibility("hidden"))) \
+ __attribute__((no_sanitize("hwaddress"))) name##_func_t* name##_resolver IFUNC_ARGS
+
+#define DEFINE_STATIC_SHIM(x) x
+
+#define FORWARD(name) \
+ static name##_func_t* fn = reinterpret_cast<name##_func_t*>( \
+ __bionic_call_ifunc_resolver(reinterpret_cast<ElfW(Addr)>(name##_resolver))); \
+ return fn
+
+#else
+#error neither dynamic nor static dispatch?!
+#endif
+
+typedef void* memchr_func_t(const void*, int, size_t);
+#define MEMCHR_SHIM() \
+ DEFINE_STATIC_SHIM(void* memchr(const void* src, int ch, size_t n) { \
+ FORWARD(memchr)(src, ch, n); \
+ })
+
+typedef int memcmp_func_t(const void*, const void*, size_t);
+#define MEMCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int memcmp(const void* lhs, const void* rhs, size_t n) { \
+ FORWARD(memcmp)(lhs, rhs, n); \
+ })
+
+typedef void* memcpy_func_t(void*, const void*, size_t);
+#define MEMCPY_SHIM() \
+ DEFINE_STATIC_SHIM(void* memcpy(void* dst, const void* src, size_t n) { \
+ FORWARD(memcpy)(dst, src, n); \
+ })
+
+typedef void* memmove_func_t(void*, const void*, size_t);
+#define MEMMOVE_SHIM() \
+ DEFINE_STATIC_SHIM(void* memmove(void* dst, const void* src, size_t n) { \
+ FORWARD(memmove)(dst, src, n); \
+ })
+
+typedef int memrchr_func_t(const void*, int, size_t);
+#define MEMRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(int memrchr(const void* src, int ch, size_t n) { \
+ FORWARD(memrchr)(src, ch, n); \
+ })
+
+typedef void* memset_func_t(void*, int, size_t);
+#define MEMSET_SHIM() \
+ DEFINE_STATIC_SHIM(void* memset(void* dst, int ch, size_t n) { FORWARD(memset)(dst, ch, n); })
+
+typedef void* __memset_chk_func_t(void*, int, size_t, size_t);
+#define __MEMSET_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(void* __memset_chk(void* dst, int ch, size_t n, size_t n2) { \
+ FORWARD(__memset_chk)(dst, ch, n, n2); \
+ })
+
+typedef char* stpcpy_func_t(char*, const char*);
+#define STPCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* stpcpy(char* dst, const char* src) { FORWARD(stpcpy)(dst, src); })
+
+typedef char* strcat_func_t(char*, const char*);
+#define STRCAT_SHIM() \
+ DEFINE_STATIC_SHIM(char* strcat(char* dst, const char* src) { FORWARD(strcat)(dst, src); })
+
+typedef char* __strcat_chk_func_t(char*, const char*, size_t);
+#define __STRCAT_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(char* __strcat_chk(char* dst, const char* src, size_t dst_buf_size) { \
+ FORWARD(__strcat_chk)(dst, src, dst_buf_size); \
+ })
+
+typedef char* strchr_func_t(const char*, int);
+#define STRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(char* strchr(const char* src, int ch) { FORWARD(strchr)(src, ch); })
+
+typedef char* strchrnul_func_t(const char*, int);
+#define STRCHRNUL_SHIM() \
+ DEFINE_STATIC_SHIM(char* strchrnul(const char* src, int ch) { FORWARD(strchrnul)(src, ch); })
+
+typedef int strcmp_func_t(const char*, const char*);
+#define STRCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int strcmp(char* lhs, const char* rhs) { FORWARD(strcmp)(lhs, rhs); })
+
+typedef char* strcpy_func_t(char*, const char*);
+#define STRCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* strcpy(char* dst, const char* src) { FORWARD(strcpy)(dst, src); })
+
+typedef char* __strcpy_chk_func_t(char*, const char*, size_t);
+#define __STRCPY_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(char* __strcpy_chk(char* dst, const char* src, size_t dst_len) { \
+ FORWARD(__strcpy_chk)(dst, src, dst_len); \
+ })
+
+typedef size_t strlen_func_t(const char*);
+#define STRLEN_SHIM() DEFINE_STATIC_SHIM(size_t strlen(const char* s) { FORWARD(strlen)(s); })
+
+typedef char* strncat_func_t(char*, const char*, size_t);
+#define STRNCAT_SHIM() \
+ DEFINE_STATIC_SHIM(char* strncat(char* dst, const char* src, size_t n) { \
+ FORWARD(strncat)(dst, src, n); \
+ })
+
+typedef int strncmp_func_t(const char*, const char*, size_t);
+#define STRNCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int strncmp(const char* lhs, const char* rhs, size_t n) { \
+ FORWARD(strncmp)(lhs, rhs, n); \
+ })
+
+typedef char* strncpy_func_t(char*, const char*, size_t);
+#define STRNCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* strncpy(char* dst, const char* src, size_t n) { \
+ FORWARD(strncpy)(dst, src, n); \
+ })
+
+typedef size_t strnlen_func_t(const char*, size_t);
+#define STRNLEN_SHIM() \
+ DEFINE_STATIC_SHIM(size_t strnlen(const char* s, size_t n) { FORWARD(strnlen)(s, n); })
+
+typedef char* strrchr_func_t(const char*, int);
+#define STRRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(char* strrchr(const char* src, int ch) { FORWARD(strrchr)(src, ch); })