Merge "No need to require API 23 for strerror_l()." into main
diff --git a/libc/Android.bp b/libc/Android.bp
index 64785a5..2141e67 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -1414,43 +1414,13 @@
}
// ========================================================
-// libc_static_dispatch.a --- libc.a ifuncs
+// libc_static_dispatch.a/libc_dynamic_dispatch.a --- string/memory "ifuncs"
+// (Actually ifuncs for libc.so, but a home-grown alternative for libc.a.)
// ========================================================
-cc_library_static {
+
+cc_defaults {
+ name: "libc_dispatch_defaults",
defaults: ["libc_defaults"],
- name: "libc_static_dispatch",
-
- arch: {
- x86_64: {
- srcs: ["arch-x86_64/static_function_dispatch.S"],
- },
- x86: {
- srcs: ["arch-x86/static_function_dispatch.S"],
- },
- arm: {
- srcs: ["arch-arm/static_function_dispatch.S"],
- },
- arm64: {
- srcs: ["arch-arm64/static_function_dispatch.S"],
- },
- riscv64: {
- srcs: ["arch-riscv64/static_function_dispatch.S"],
- },
- },
-}
-
-// ========================================================
-// libc_dynamic_dispatch.a --- libc.so ifuncs
-// ========================================================
-cc_library_static {
- defaults: ["libc_defaults"],
- name: "libc_dynamic_dispatch",
-
- cflags: [
- "-ffreestanding",
- "-fno-stack-protector",
- "-fno-jump-tables",
- ],
arch: {
x86_64: {
srcs: ["arch-x86_64/dynamic_function_dispatch.cpp"],
@@ -1468,6 +1438,30 @@
srcs: ["arch-riscv64/dynamic_function_dispatch.cpp"],
},
},
+ // Prevent the compiler from inserting calls to libc/taking the address of
+ // a jump table from within an ifunc (or, in the static case, code that
+ // can be executed arbitrarily early).
+ cflags: [
+ "-ffreestanding",
+ "-fno-stack-protector",
+ "-fno-jump-tables",
+ ],
+}
+
+cc_library_static {
+ name: "libc_static_dispatch",
+ defaults: ["libc_dispatch_defaults"],
+ cflags: [
+ "-DBIONIC_STATIC_DISPATCH",
+ ],
+}
+
+cc_library_static {
+ name: "libc_dynamic_dispatch",
+ defaults: ["libc_dispatch_defaults"],
+ cflags: [
+ "-DBIONIC_DYNAMIC_DISPATCH",
+ ],
}
// ========================================================
@@ -2227,6 +2221,12 @@
visibility: [
"//packages/modules/Virtualization/libs/libvmbase",
],
+
+ // b/358211032: This library gets linked into a rust rlib. Disable LTO
+ // until cross-language lto is supported.
+ lto: {
+ never: true,
+ },
}
// ========================================================
diff --git a/libc/arch-arm/dynamic_function_dispatch.cpp b/libc/arch-arm/dynamic_function_dispatch.cpp
index 1d2f38f..f984421 100644
--- a/libc/arch-arm/dynamic_function_dispatch.cpp
+++ b/libc/arch-arm/dynamic_function_dispatch.cpp
@@ -27,27 +27,26 @@
*/
#include <fcntl.h>
-#include <sys/syscall.h>
-
#include <private/bionic_ifuncs.h>
+#include <sys/syscall.h>
extern "C" {
enum CpuVariant {
- kUnknown = 0,
- kGeneric,
- kCortexA7,
- kCortexA9,
- kCortexA53,
- kCortexA55,
- kKrait,
- kKryo,
+ kUnknown = 0,
+ kGeneric,
+ kCortexA7,
+ kCortexA9,
+ kCortexA53,
+ kCortexA55,
+ kKrait,
+ kKryo,
};
static constexpr int MAX_CPU_NAME_LEN = 12;
struct CpuVariantNames {
- alignas(alignof(int)) char name[MAX_CPU_NAME_LEN];
- CpuVariant variant;
+ alignas(alignof(int)) char name[MAX_CPU_NAME_LEN];
+ CpuVariant variant;
};
static constexpr CpuVariantNames cpu_variant_names[] = {
@@ -66,227 +65,237 @@
};
static long ifunc_open(const char* pathname) {
- register long r0 __asm__("r0") = AT_FDCWD;
- register long r1 __asm__("r1") = reinterpret_cast<long>(pathname);
- register long r2 __asm__("r2") = O_RDONLY;
- register long r3 __asm__("r3") = 0;
- register long r7 __asm__("r7") = __NR_openat;
- __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r7));
- return r0;
+ register long r0 __asm__("r0") = AT_FDCWD;
+ register long r1 __asm__("r1") = reinterpret_cast<long>(pathname);
+ register long r2 __asm__("r2") = O_RDONLY;
+ register long r3 __asm__("r3") = 0;
+ register long r7 __asm__("r7") = __NR_openat;
+ __asm__ volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r7));
+ return r0;
}
static ssize_t ifunc_read(int fd, void* buf, size_t count) {
- register long r0 __asm__("r0") = fd;
- register long r1 __asm__("r1") = reinterpret_cast<long>(buf);
- register long r2 __asm__("r2") = count;
- register long r7 __asm__("r7") = __NR_read;
- __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r1), "r"(r2), "r"(r7) : "memory");
- return r0;
+ register long r0 __asm__("r0") = fd;
+ register long r1 __asm__("r1") = reinterpret_cast<long>(buf);
+ register long r2 __asm__("r2") = count;
+ register long r7 __asm__("r7") = __NR_read;
+ __asm__ volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r0), "r"(r1), "r"(r2), "r"(r7)
+ : "memory");
+ return r0;
}
static int ifunc_close(int fd) {
- register long r0 __asm__("r0") = fd;
- register long r7 __asm__("r7") = __NR_close;
- __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r7));
- return r0;
+ register long r0 __asm__("r0") = fd;
+ register long r7 __asm__("r7") = __NR_close;
+ __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r7));
+ return r0;
}
static bool is_same_name(const char* a, const char* b) {
- static_assert(MAX_CPU_NAME_LEN % sizeof(int) == 0, "");
- const int* ia = reinterpret_cast<const int*>(a);
- const int* ib = reinterpret_cast<const int*>(b);
- for (size_t i = 0; i < MAX_CPU_NAME_LEN / sizeof(int); ++i) {
- if (ia[i] != ib[i]) {
- return false;
- }
+ static_assert(MAX_CPU_NAME_LEN % sizeof(int) == 0, "");
+ const int* ia = reinterpret_cast<const int*>(a);
+ const int* ib = reinterpret_cast<const int*>(b);
+ for (size_t i = 0; i < MAX_CPU_NAME_LEN / sizeof(int); ++i) {
+ if (ia[i] != ib[i]) {
+ return false;
}
- return true;
+ }
+ return true;
}
static CpuVariant init_cpu_variant() {
- int fd = ifunc_open("/dev/cpu_variant:arm");
- if (fd < 0) return kGeneric;
+ int fd = ifunc_open("/dev/cpu_variant:arm");
+ if (fd < 0) return kGeneric;
- alignas(alignof(int)) char name[MAX_CPU_NAME_LEN] = {};
+ alignas(alignof(int)) char name[MAX_CPU_NAME_LEN] = {};
- int bytes_read, total_read = 0;
- while (total_read < MAX_CPU_NAME_LEN - 1 &&
- (bytes_read = ifunc_read(fd, name + total_read,
- MAX_CPU_NAME_LEN - 1 - total_read)) > 0) {
- total_read += bytes_read;
- }
- ifunc_close(fd);
+ int bytes_read, total_read = 0;
+ while (total_read < MAX_CPU_NAME_LEN - 1 &&
+ (bytes_read = ifunc_read(fd, name + total_read,
+ MAX_CPU_NAME_LEN - 1 - total_read)) > 0) {
+ total_read += bytes_read;
+ }
+ ifunc_close(fd);
- if (bytes_read != 0) {
- // The file is too big. We haven't reach the end. Or maybe there is an
- // error when reading.
- return kGeneric;
- }
- name[total_read] = 0;
-
- const CpuVariantNames* cpu_variant = cpu_variant_names;
- while (cpu_variant->variant != kUnknown) {
- if (is_same_name(cpu_variant->name, name)) {
- return cpu_variant->variant;
- }
- cpu_variant++;
- }
+ if (bytes_read != 0) {
+ // The file is too big. We haven't reach the end. Or maybe there is an
+ // error when reading.
return kGeneric;
+ }
+ name[total_read] = 0;
+
+ const CpuVariantNames* cpu_variant = cpu_variant_names;
+ while (cpu_variant->variant != kUnknown) {
+ if (is_same_name(cpu_variant->name, name)) {
+ return cpu_variant->variant;
+ }
+ cpu_variant++;
+ }
+ return kGeneric;
}
static CpuVariant get_cpu_variant() {
- static CpuVariant cpu_variant = kUnknown;
- if (cpu_variant == kUnknown) {
- cpu_variant = init_cpu_variant();
- }
- return cpu_variant;
+ static CpuVariant cpu_variant = kUnknown;
+ if (cpu_variant == kUnknown) {
+ cpu_variant = init_cpu_variant();
+ }
+ return cpu_variant;
}
-typedef void* memmove_func(void* __dst, const void* __src, size_t __n);
DEFINE_IFUNC_FOR(memmove) {
- RETURN_FUNC(memmove_func, memmove_a15);
+ RETURN_FUNC(memmove_func_t, memmove_a15);
}
+MEMMOVE_SHIM()
-typedef void* memcpy_func(void*, const void*, size_t);
DEFINE_IFUNC_FOR(memcpy) {
- return memmove_resolver(hwcap);
+ return memmove_resolver(hwcap);
}
+MEMCPY_SHIM()
-typedef void* __memcpy_func(void*, const void*, size_t);
+// On arm32, __memcpy() is not publicly exposed, but gets called by memmove()
+// in cases where the copy is known to be overlap-safe.
+typedef void* __memcpy_func_t(void*, const void*, size_t);
DEFINE_IFUNC_FOR(__memcpy) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- RETURN_FUNC(__memcpy_func, __memcpy_a7);
- case kCortexA9:
- RETURN_FUNC(__memcpy_func, __memcpy_a9);
- case kKrait:
- RETURN_FUNC(__memcpy_func, __memcpy_krait);
- case kCortexA53:
- RETURN_FUNC(__memcpy_func, __memcpy_a53);
- case kCortexA55:
- RETURN_FUNC(__memcpy_func, __memcpy_a55);
- case kKryo:
- RETURN_FUNC(__memcpy_func, __memcpy_kryo);
- default:
- RETURN_FUNC(__memcpy_func, __memcpy_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a7);
+ case kCortexA9:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a9);
+ case kKrait:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_krait);
+ case kCortexA53:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a53);
+ case kCortexA55:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a55);
+ case kKryo:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_kryo);
+ default:
+ RETURN_FUNC(__memcpy_func_t, __memcpy_a15);
+ }
}
+DEFINE_STATIC_SHIM(void* __memcpy(void* dst, const void* src, size_t n) {
+ FORWARD(__memcpy)(dst, src, n);
+})
-typedef void* __memset_chk_func(void* s, int c, size_t n, size_t n2);
DEFINE_IFUNC_FOR(__memset_chk) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- case kCortexA53:
- case kCortexA55:
- case kKryo:
- RETURN_FUNC(__memset_chk_func, __memset_chk_a7);
- case kCortexA9:
- RETURN_FUNC(__memset_chk_func, __memset_chk_a9);
- case kKrait:
- RETURN_FUNC(__memset_chk_func, __memset_chk_krait);
- default:
- RETURN_FUNC(__memset_chk_func, __memset_chk_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ case kCortexA53:
+ case kCortexA55:
+ case kKryo:
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_a7);
+ case kCortexA9:
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_a9);
+ case kKrait:
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_krait);
+ default:
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_a15);
+ }
}
+__MEMSET_CHK_SHIM()
-typedef void* memset_func(void* __dst, int __ch, size_t __n);
DEFINE_IFUNC_FOR(memset) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- case kCortexA53:
- case kCortexA55:
- case kKryo:
- RETURN_FUNC(memset_func, memset_a7);
- case kCortexA9:
- RETURN_FUNC(memset_func, memset_a9);
- case kKrait:
- RETURN_FUNC(memset_func, memset_krait);
- default:
- RETURN_FUNC(memset_func, memset_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ case kCortexA53:
+ case kCortexA55:
+ case kKryo:
+ RETURN_FUNC(memset_func_t, memset_a7);
+ case kCortexA9:
+ RETURN_FUNC(memset_func_t, memset_a9);
+ case kKrait:
+ RETURN_FUNC(memset_func_t, memset_krait);
+ default:
+ RETURN_FUNC(memset_func_t, memset_a15);
+ }
}
+MEMSET_SHIM()
-typedef char* strcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcpy) {
- switch(get_cpu_variant()) {
- case kCortexA9:
- RETURN_FUNC(strcpy_func, strcpy_a9);
- default:
- RETURN_FUNC(strcpy_func, strcpy_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA9:
+ RETURN_FUNC(strcpy_func_t, strcpy_a9);
+ default:
+ RETURN_FUNC(strcpy_func_t, strcpy_a15);
+ }
}
+STRCPY_SHIM()
-typedef char* __strcpy_chk_func(char* dst, const char* src, size_t dst_len);
DEFINE_IFUNC_FOR(__strcpy_chk) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a7);
- case kCortexA9:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a9);
- case kKrait:
- case kKryo:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_krait);
- case kCortexA53:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a53);
- case kCortexA55:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a55);
- default:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a7);
+ case kCortexA9:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a9);
+ case kKrait:
+ case kKryo:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_krait);
+ case kCortexA53:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a53);
+ case kCortexA55:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a55);
+ default:
+ RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a15);
+ }
}
+__STRCPY_CHK_SHIM()
-typedef char* stpcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(stpcpy) {
- switch(get_cpu_variant()) {
- case kCortexA9:
- RETURN_FUNC(stpcpy_func, stpcpy_a9);
- default:
- RETURN_FUNC(stpcpy_func, stpcpy_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA9:
+ RETURN_FUNC(stpcpy_func_t, stpcpy_a9);
+ default:
+ RETURN_FUNC(stpcpy_func_t, stpcpy_a15);
+ }
}
+STPCPY_SHIM()
-typedef char* strcat_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcat) {
- switch(get_cpu_variant()) {
- case kCortexA9:
- RETURN_FUNC(strcat_func, strcat_a9);
- default:
- RETURN_FUNC(strcat_func, strcat_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA9:
+ RETURN_FUNC(strcat_func_t, strcat_a9);
+ default:
+ RETURN_FUNC(strcat_func_t, strcat_a15);
+ }
}
+STRCAT_SHIM()
-typedef char* __strcat_chk_func(char* dst, const char* src, size_t dst_buf_size);
DEFINE_IFUNC_FOR(__strcat_chk) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a7);
- case kCortexA9:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a9);
- case kKrait:
- case kKryo:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_krait);
- case kCortexA53:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a53);
- case kCortexA55:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a55);
- default:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA7:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a7);
+ case kCortexA9:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a9);
+ case kKrait:
+ case kKryo:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_krait);
+ case kCortexA53:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a53);
+ case kCortexA55:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a55);
+ default:
+ RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a15);
+ }
}
+__STRCAT_CHK_SHIM()
-typedef int strcmp_func(const char* __lhs, const char* __rhs);
DEFINE_IFUNC_FOR(strcmp) {
- RETURN_FUNC(strcmp_func, strcmp_a15);
+ RETURN_FUNC(strcmp_func_t, strcmp_a15);
}
+STRCMP_SHIM()
-typedef size_t strlen_func(const char* __s);
DEFINE_IFUNC_FOR(strlen) {
- switch(get_cpu_variant()) {
- case kCortexA9:
- RETURN_FUNC(strlen_func, strlen_a9);
- default:
- RETURN_FUNC(strlen_func, strlen_a15);
- }
+ switch (get_cpu_variant()) {
+ case kCortexA9:
+ RETURN_FUNC(strlen_func_t, strlen_a9);
+ default:
+ RETURN_FUNC(strlen_func_t, strlen_a15);
+ }
}
+STRLEN_SHIM()
} // extern "C"
diff --git a/libc/arch-arm/static_function_dispatch.S b/libc/arch-arm/static_function_dispatch.S
deleted file mode 100644
index a8235c2..0000000
--- a/libc/arch-arm/static_function_dispatch.S
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- b impl; \
-END(name)
-
-FUNCTION_DELEGATE(memmove, memmove_generic)
-FUNCTION_DELEGATE(memcpy, memmove_generic)
-FUNCTION_DELEGATE(memset, memset_generic)
-FUNCTION_DELEGATE(__memset_chk, __memset_chk_generic)
-FUNCTION_DELEGATE(strcpy, strcpy_generic)
-FUNCTION_DELEGATE(__strcpy_chk, __strcpy_chk_generic)
-FUNCTION_DELEGATE(stpcpy, stpcpy_generic)
-FUNCTION_DELEGATE(strcat, strcat_generic)
-FUNCTION_DELEGATE(__strcat_chk, __strcat_chk_generic)
-FUNCTION_DELEGATE(strcmp, strcmp_generic)
-FUNCTION_DELEGATE(strlen, strlen_generic)
diff --git a/libc/arch-arm64/dynamic_function_dispatch.cpp b/libc/arch-arm64/dynamic_function_dispatch.cpp
index f9e4263..29e47b3 100644
--- a/libc/arch-arm64/dynamic_function_dispatch.cpp
+++ b/libc/arch-arm64/dynamic_function_dispatch.cpp
@@ -28,148 +28,147 @@
#include <private/bionic_ifuncs.h>
#include <stddef.h>
-#include <sys/auxv.h>
static inline bool __bionic_is_oryon(unsigned long hwcap) {
- if (!(hwcap & HWCAP_CPUID)) return false;
+ if (!(hwcap & HWCAP_CPUID)) return false;
- // Extract the implementor and variant bits from MIDR_EL1.
- // https://www.kernel.org/doc/html/latest/arch/arm64/cpu-feature-registers.html#list-of-registers-with-visible-features
- unsigned long midr;
- __asm__ __volatile__("mrs %0, MIDR_EL1" : "=r"(midr));
- uint16_t cpu = (midr >> 20) & 0xfff;
+ // Extract the implementor and variant bits from MIDR_EL1.
+ // https://www.kernel.org/doc/html/latest/arch/arm64/cpu-feature-registers.html#list-of-registers-with-visible-features
+ unsigned long midr;
+ __asm__ __volatile__("mrs %0, MIDR_EL1" : "=r"(midr));
+ uint16_t cpu = (midr >> 20) & 0xfff;
- auto make_cpu = [](unsigned implementor, unsigned variant) {
- return (implementor << 4) | variant;
- };
+ auto make_cpu = [](unsigned implementor, unsigned variant) {
+ return (implementor << 4) | variant;
+ };
- // Check for implementor Qualcomm's variants 0x1..0x5 (Oryon).
- return cpu >= make_cpu('Q', 0x1) && cpu <= make_cpu('Q', 0x5);
+ // Check for implementor Qualcomm's variants 0x1..0x5 (Oryon).
+ return cpu >= make_cpu('Q', 0x1) && cpu <= make_cpu('Q', 0x5);
}
extern "C" {
-typedef void* memchr_func(const void*, int, size_t);
DEFINE_IFUNC_FOR(memchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(memchr_func, __memchr_aarch64_mte);
- } else {
- RETURN_FUNC(memchr_func, __memchr_aarch64);
- }
-}
-
-typedef int memcmp_func(const void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memcmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(memcmp_func, __memcmp_aarch64);
-}
-
-typedef void* memcpy_func(void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memcpy) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memcpy_func, __memmove_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64_nt);
- } else if (arg->_hwcap & HWCAP_ASIMD) {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64_simd);
- } else {
- RETURN_FUNC(memcpy_func, __memcpy_aarch64);
- }
-}
-
-typedef void* memmove_func(void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memmove) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memmove_func, __memmove_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memcpy_func, __memmove_aarch64_nt);
- } else if (arg->_hwcap & HWCAP_ASIMD) {
- RETURN_FUNC(memmove_func, __memmove_aarch64_simd);
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(memchr_func_t, __memchr_aarch64_mte);
} else {
- RETURN_FUNC(memmove_func, __memmove_aarch64);
+ RETURN_FUNC(memchr_func_t, __memchr_aarch64);
}
}
+MEMCHR_SHIM()
-typedef int memrchr_func(const void*, int, size_t);
+DEFINE_IFUNC_FOR(memcmp) {
+ // TODO: enable the SVE version.
+ RETURN_FUNC(memcmp_func_t, __memcmp_aarch64);
+}
+MEMCMP_SHIM()
+
+DEFINE_IFUNC_FOR(memcpy) {
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memcpy_func_t, __memmove_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64_nt);
+ } else if (arg->_hwcap & HWCAP_ASIMD) {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64_simd);
+ } else {
+ RETURN_FUNC(memcpy_func_t, __memcpy_aarch64);
+ }
+}
+MEMCPY_SHIM()
+
+DEFINE_IFUNC_FOR(memmove) {
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_nt);
+ } else if (arg->_hwcap & HWCAP_ASIMD) {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64_simd);
+ } else {
+ RETURN_FUNC(memmove_func_t, __memmove_aarch64);
+ }
+}
+MEMMOVE_SHIM()
+
DEFINE_IFUNC_FOR(memrchr) {
- RETURN_FUNC(memrchr_func, __memrchr_aarch64);
+ RETURN_FUNC(memrchr_func_t, __memrchr_aarch64);
}
+MEMRCHR_SHIM()
-typedef int memset_func(void*, int, size_t);
DEFINE_IFUNC_FOR(memset) {
- if (arg->_hwcap2 & HWCAP2_MOPS) {
- RETURN_FUNC(memset_func, __memset_aarch64_mops);
- } else if (__bionic_is_oryon(arg->_hwcap)) {
- RETURN_FUNC(memset_func, __memset_aarch64_nt);
- } else {
- RETURN_FUNC(memset_func, __memset_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MOPS) {
+ RETURN_FUNC(memset_func_t, __memset_aarch64_mops);
+ } else if (__bionic_is_oryon(arg->_hwcap)) {
+ RETURN_FUNC(memset_func_t, __memset_aarch64_nt);
+ } else {
+ RETURN_FUNC(memset_func_t, __memset_aarch64);
+ }
}
+MEMSET_SHIM()
-typedef char* stpcpy_func(char*, const char*, size_t);
DEFINE_IFUNC_FOR(stpcpy) {
- // TODO: enable the SVE version.
- RETURN_FUNC(stpcpy_func, __stpcpy_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(stpcpy_func_t, __stpcpy_aarch64);
}
+STPCPY_SHIM()
-typedef char* strchr_func(const char*, int);
DEFINE_IFUNC_FOR(strchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strchr_func, __strchr_aarch64_mte);
- } else {
- RETURN_FUNC(strchr_func, __strchr_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strchr_func_t, __strchr_aarch64_mte);
+ } else {
+ RETURN_FUNC(strchr_func_t, __strchr_aarch64);
+ }
}
+STRCHR_SHIM()
-typedef char* strchrnul_func(const char*, int);
DEFINE_IFUNC_FOR(strchrnul) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strchrnul_func, __strchrnul_aarch64_mte);
- } else {
- RETURN_FUNC(strchrnul_func, __strchrnul_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strchrnul_func_t, __strchrnul_aarch64_mte);
+ } else {
+ RETURN_FUNC(strchrnul_func_t, __strchrnul_aarch64);
+ }
}
+STRCHRNUL_SHIM()
-typedef int strcmp_func(const char*, const char*);
DEFINE_IFUNC_FOR(strcmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strcmp_func, __strcmp_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strcmp_func_t, __strcmp_aarch64);
}
+STRCMP_SHIM()
-typedef char* strcpy_func(char*, const char*);
DEFINE_IFUNC_FOR(strcpy) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strcpy_func, __strcpy_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strcpy_func_t, __strcpy_aarch64);
}
+STRCPY_SHIM()
-typedef size_t strlen_func(const char*);
DEFINE_IFUNC_FOR(strlen) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strlen_func, __strlen_aarch64_mte);
- } else {
- RETURN_FUNC(strlen_func, __strlen_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strlen_func_t, __strlen_aarch64_mte);
+ } else {
+ RETURN_FUNC(strlen_func_t, __strlen_aarch64);
+ }
}
+STRLEN_SHIM()
-typedef int strncmp_func(const char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncmp) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strncmp_func, __strncmp_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strncmp_func_t, __strncmp_aarch64);
}
+STRNCMP_SHIM()
-typedef size_t strnlen_func(const char*, size_t);
DEFINE_IFUNC_FOR(strnlen) {
- // TODO: enable the SVE version.
- RETURN_FUNC(strnlen_func, __strnlen_aarch64);
+ // TODO: enable the SVE version.
+ RETURN_FUNC(strnlen_func_t, __strnlen_aarch64);
}
+STRNLEN_SHIM()
-typedef char* strrchr_func(const char*, int);
DEFINE_IFUNC_FOR(strrchr) {
- if (arg->_hwcap2 & HWCAP2_MTE) {
- RETURN_FUNC(strrchr_func, __strrchr_aarch64_mte);
- } else {
- RETURN_FUNC(strrchr_func, __strrchr_aarch64);
- }
+ if (arg->_hwcap2 & HWCAP2_MTE) {
+ RETURN_FUNC(strrchr_func_t, __strrchr_aarch64_mte);
+ } else {
+ RETURN_FUNC(strrchr_func_t, __strrchr_aarch64);
+ }
}
+STRRCHR_SHIM()
} // extern "C"
diff --git a/libc/arch-arm64/static_function_dispatch.S b/libc/arch-arm64/static_function_dispatch.S
deleted file mode 100644
index 18c3783..0000000
--- a/libc/arch-arm64/static_function_dispatch.S
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (C) 2019 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- b impl; \
-END(name)
-
-FUNCTION_DELEGATE(memchr, __memchr_aarch64_mte)
-FUNCTION_DELEGATE(memcmp, __memcmp_aarch64)
-FUNCTION_DELEGATE(memcpy, __memcpy_aarch64)
-FUNCTION_DELEGATE(memmove, __memmove_aarch64)
-FUNCTION_DELEGATE(memrchr, __memrchr_aarch64)
-FUNCTION_DELEGATE(memset, __memset_aarch64)
-FUNCTION_DELEGATE(stpcpy, __stpcpy_aarch64)
-FUNCTION_DELEGATE(strchr, __strchr_aarch64_mte)
-FUNCTION_DELEGATE(strchrnul, __strchrnul_aarch64_mte)
-FUNCTION_DELEGATE(strcmp, __strcmp_aarch64)
-FUNCTION_DELEGATE(strcpy, __strcpy_aarch64)
-FUNCTION_DELEGATE(strlen, __strlen_aarch64_mte)
-FUNCTION_DELEGATE(strrchr, __strrchr_aarch64_mte)
-FUNCTION_DELEGATE(strncmp, __strncmp_aarch64)
-FUNCTION_DELEGATE(strnlen, __strnlen_aarch64)
-
-NOTE_GNU_PROPERTY()
diff --git a/libc/arch-riscv64/dynamic_function_dispatch.cpp b/libc/arch-riscv64/dynamic_function_dispatch.cpp
index bb2ba51..ce6c028 100644
--- a/libc/arch-riscv64/dynamic_function_dispatch.cpp
+++ b/libc/arch-riscv64/dynamic_function_dispatch.cpp
@@ -27,12 +27,11 @@
*/
#include <fcntl.h>
+#include <private/bionic_ifuncs.h>
#include <stddef.h>
#include <sys/syscall.h>
#include <unistd.h>
-#include <private/bionic_ifuncs.h>
-
extern "C" {
static inline __always_inline int ifunc_faccessat(int dir_fd, const char* path, int mode) {
@@ -53,94 +52,94 @@
return result;
}
-typedef void* memchr_func(const void*, int, size_t);
DEFINE_IFUNC_FOR(memchr) {
- if (have_fast_v()) RETURN_FUNC(memchr_func, memchr_v);
- RETURN_FUNC(memchr_func, memchr_gc);
+ if (have_fast_v()) RETURN_FUNC(memchr_func_t, memchr_v);
+ RETURN_FUNC(memchr_func_t, memchr_gc);
}
+MEMCHR_SHIM()
-typedef int memcmp_func(const void*, const void*, size_t);
DEFINE_IFUNC_FOR(memcmp) {
- if (have_fast_v()) RETURN_FUNC(memcmp_func, memcmp_v);
- RETURN_FUNC(memcmp_func, memcmp_gc);
+ if (have_fast_v()) RETURN_FUNC(memcmp_func_t, memcmp_v);
+ RETURN_FUNC(memcmp_func_t, memcmp_gc);
}
+MEMCMP_SHIM()
-typedef void* memcpy_func(void*, const void*, size_t);
DEFINE_IFUNC_FOR(memcpy) {
- if (have_fast_v()) RETURN_FUNC(memcpy_func, memcpy_v);
- RETURN_FUNC(memcpy_func, memcpy_gc);
+ if (have_fast_v()) RETURN_FUNC(memcpy_func_t, memcpy_v);
+ RETURN_FUNC(memcpy_func_t, memcpy_gc);
}
+MEMCPY_SHIM()
-typedef void* memmove_func(void*, const void*, size_t);
DEFINE_IFUNC_FOR(memmove) {
- if (have_fast_v()) RETURN_FUNC(memmove_func, memmove_v);
- RETURN_FUNC(memmove_func, memmove_gc);
+ if (have_fast_v()) RETURN_FUNC(memmove_func_t, memmove_v);
+ RETURN_FUNC(memmove_func_t, memmove_gc);
}
+MEMMOVE_SHIM()
-typedef void* memset_func(void*, int, size_t);
DEFINE_IFUNC_FOR(memset) {
- if (have_fast_v()) RETURN_FUNC(memset_func, memset_v);
- RETURN_FUNC(memset_func, memset_gc);
+ if (have_fast_v()) RETURN_FUNC(memset_func_t, memset_v);
+ RETURN_FUNC(memset_func_t, memset_gc);
}
+MEMSET_SHIM()
-typedef char* stpcpy_func(char*, const char*);
DEFINE_IFUNC_FOR(stpcpy) {
- if (have_fast_v()) RETURN_FUNC(stpcpy_func, stpcpy_v);
- RETURN_FUNC(stpcpy_func, stpcpy_gc);
+ if (have_fast_v()) RETURN_FUNC(stpcpy_func_t, stpcpy_v);
+ RETURN_FUNC(stpcpy_func_t, stpcpy_gc);
}
+STPCPY_SHIM()
-typedef char* strcat_func(char*, const char*);
DEFINE_IFUNC_FOR(strcat) {
- if (have_fast_v()) RETURN_FUNC(strcat_func, strcat_v);
- RETURN_FUNC(strcat_func, strcat_gc);
+ if (have_fast_v()) RETURN_FUNC(strcat_func_t, strcat_v);
+ RETURN_FUNC(strcat_func_t, strcat_gc);
}
+STRCAT_SHIM()
-typedef char* strchr_func(const char*, int);
DEFINE_IFUNC_FOR(strchr) {
- if (have_fast_v()) RETURN_FUNC(strchr_func, strchr_v);
- RETURN_FUNC(strchr_func, strchr_gc);
+ if (have_fast_v()) RETURN_FUNC(strchr_func_t, strchr_v);
+ RETURN_FUNC(strchr_func_t, strchr_gc);
}
+STRCHR_SHIM()
-typedef int strcmp_func(const char*, const char*);
DEFINE_IFUNC_FOR(strcmp) {
- if (have_fast_v()) RETURN_FUNC(strcmp_func, strcmp_v);
- RETURN_FUNC(strcmp_func, strcmp_gc);
+ if (have_fast_v()) RETURN_FUNC(strcmp_func_t, strcmp_v);
+ RETURN_FUNC(strcmp_func_t, strcmp_gc);
}
+STRCMP_SHIM()
-typedef char* strcpy_func(char*, const char*);
DEFINE_IFUNC_FOR(strcpy) {
- if (have_fast_v()) RETURN_FUNC(strcpy_func, strcpy_v);
- RETURN_FUNC(strcpy_func, strcpy_gc);
+ if (have_fast_v()) RETURN_FUNC(strcpy_func_t, strcpy_v);
+ RETURN_FUNC(strcpy_func_t, strcpy_gc);
}
+STRCPY_SHIM()
-typedef size_t strlen_func(const char*);
DEFINE_IFUNC_FOR(strlen) {
- if (have_fast_v()) RETURN_FUNC(strlen_func, strlen_v);
- RETURN_FUNC(strlen_func, strlen_gc);
+ if (have_fast_v()) RETURN_FUNC(strlen_func_t, strlen_v);
+ RETURN_FUNC(strlen_func_t, strlen_gc);
}
+STRLEN_SHIM()
-typedef char* strncat_func(char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncat) {
- if (have_fast_v()) RETURN_FUNC(strncat_func, strncat_v);
- RETURN_FUNC(strncat_func, strncat_gc);
+ if (have_fast_v()) RETURN_FUNC(strncat_func_t, strncat_v);
+ RETURN_FUNC(strncat_func_t, strncat_gc);
}
+STRNCAT_SHIM()
-typedef int strncmp_func(const char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncmp) {
- if (have_fast_v()) RETURN_FUNC(strncmp_func, strncmp_v);
- RETURN_FUNC(strncmp_func, strncmp_gc);
+ if (have_fast_v()) RETURN_FUNC(strncmp_func_t, strncmp_v);
+ RETURN_FUNC(strncmp_func_t, strncmp_gc);
}
+STRNCMP_SHIM()
-typedef char* strncpy_func(char*, const char*, size_t);
DEFINE_IFUNC_FOR(strncpy) {
- if (have_fast_v()) RETURN_FUNC(strncpy_func, strncpy_v);
- RETURN_FUNC(strncpy_func, strncpy_gc);
+ if (have_fast_v()) RETURN_FUNC(strncpy_func_t, strncpy_v);
+ RETURN_FUNC(strncpy_func_t, strncpy_gc);
}
+STRNCPY_SHIM()
-typedef size_t strnlen_func(const char*, size_t);
DEFINE_IFUNC_FOR(strnlen) {
- if (have_fast_v()) RETURN_FUNC(strnlen_func, strnlen_v);
- RETURN_FUNC(strnlen_func, strnlen_gc);
+ if (have_fast_v()) RETURN_FUNC(strnlen_func_t, strnlen_v);
+ RETURN_FUNC(strnlen_func_t, strnlen_gc);
}
+STRNLEN_SHIM()
} // extern "C"
diff --git a/libc/arch-riscv64/static_function_dispatch.S b/libc/arch-riscv64/static_function_dispatch.S
deleted file mode 100644
index accfec8..0000000
--- a/libc/arch-riscv64/static_function_dispatch.S
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (C) 2023 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- j impl; \
-END(name)
-
-// TODO: switch to the V variants when qemu is fixed.
-FUNCTION_DELEGATE(memchr, memchr_gc)
-FUNCTION_DELEGATE(memcmp, memcmp_gc)
-FUNCTION_DELEGATE(memcpy, memcpy_gc)
-FUNCTION_DELEGATE(memmove, memmove_gc)
-FUNCTION_DELEGATE(memset, memset_gc)
-FUNCTION_DELEGATE(stpcpy, stpcpy_gc)
-FUNCTION_DELEGATE(strcat, strcat_gc)
-FUNCTION_DELEGATE(strchr, strchr_gc)
-FUNCTION_DELEGATE(strcmp, strcmp_gc)
-FUNCTION_DELEGATE(strcpy, strcpy_gc)
-FUNCTION_DELEGATE(strlen, strlen_gc)
-FUNCTION_DELEGATE(strncat, strncat_gc)
-FUNCTION_DELEGATE(strncmp, strncmp_gc)
-FUNCTION_DELEGATE(strncpy, strncpy_gc)
-FUNCTION_DELEGATE(strnlen, strnlen_gc)
diff --git a/libc/arch-x86/dynamic_function_dispatch.cpp b/libc/arch-x86/dynamic_function_dispatch.cpp
index 38d8a0a..9d7a4c9 100644
--- a/libc/arch-x86/dynamic_function_dispatch.cpp
+++ b/libc/arch-x86/dynamic_function_dispatch.cpp
@@ -26,129 +26,150 @@
* SUCH DAMAGE.
*/
-#include <stddef.h>
-
#include <private/bionic_ifuncs.h>
+#include <stddef.h>
extern "C" {
-typedef int memcmp_func(const void* __lhs, const void* __rhs, size_t __n);
DEFINE_IFUNC_FOR(memcmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memcmp_func, memcmp_atom);
- if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(memcmp_func, memcmp_sse4);
- RETURN_FUNC(memcmp_func, memcmp_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(memcmp_func_t, memcmp_atom);
+ if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(memcmp_func_t, memcmp_sse4);
+ RETURN_FUNC(memcmp_func_t, memcmp_generic);
}
+MEMCMP_SHIM()
-typedef void* memset_func(void* __dst, int __ch, size_t __n);
DEFINE_IFUNC_FOR(memset) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memset_func, memset_atom);
- RETURN_FUNC(memset_func, memset_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(memset_func_t, memset_atom);
+ RETURN_FUNC(memset_func_t, memset_generic);
}
+MEMSET_SHIM()
-typedef void* __memset_chk_func(void *s, int c, size_t n, size_t n2);
DEFINE_IFUNC_FOR(__memset_chk) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(__memset_chk_func, __memset_chk_atom);
- RETURN_FUNC(__memset_chk_func, __memset_chk_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(__memset_chk_func_t, __memset_chk_atom);
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_generic);
}
+__MEMSET_CHK_SHIM()
-typedef void* memmove_func(void* __dst, const void* __src, size_t __n);
DEFINE_IFUNC_FOR(memmove) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memmove_func, memmove_atom);
- RETURN_FUNC(memmove_func, memmove_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(memmove_func_t, memmove_atom);
+ RETURN_FUNC(memmove_func_t, memmove_generic);
}
+MEMMOVE_SHIM()
-typedef void* memcpy_func(void*, const void*, size_t);
-DEFINE_IFUNC_FOR(memcpy) {
- return memmove_resolver();
-}
+DEFINE_IFUNC_FOR(memcpy) { return memmove_resolver(); }
+MEMCPY_SHIM()
-typedef char* strcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strcpy_func, strcpy_atom);
- RETURN_FUNC(strcpy_func, strcpy_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(strcpy_func_t, strcpy_atom);
+ RETURN_FUNC(strcpy_func_t, strcpy_generic);
}
+STRCPY_SHIM()
-typedef char* strncpy_func(char* __dst, const char* __src, size_t __n);
DEFINE_IFUNC_FOR(strncpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strncpy_func, strncpy_atom);
- RETURN_FUNC(strncpy_func, strncpy_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(strncpy_func_t, strncpy_atom);
+ RETURN_FUNC(strncpy_func_t, strncpy_generic);
}
+STRNCPY_SHIM()
-typedef size_t strlen_func(const char* __s);
DEFINE_IFUNC_FOR(strlen) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strlen_func, strlen_atom);
- RETURN_FUNC(strlen_func, strlen_generic);
+ __builtin_cpu_init();
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(strlen_func_t, strlen_atom);
+ RETURN_FUNC(strlen_func_t, strlen_generic);
}
+STRLEN_SHIM()
-typedef int wmemcmp_func(const wchar_t* __lhs, const wchar_t* __rhs, size_t __n);
-DEFINE_IFUNC_FOR(wmemcmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(wmemcmp_func, wmemcmp_sse4);
- if (__builtin_cpu_is("atom")) RETURN_FUNC(wmemcmp_func, wmemcmp_atom);
- RETURN_FUNC(wmemcmp_func, wmemcmp_freebsd);
-}
-
-typedef int strcmp_func(const char* __lhs, const char* __rhs);
DEFINE_IFUNC_FOR(strcmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcmp_func, strcmp_ssse3);
- RETURN_FUNC(strcmp_func, strcmp_generic);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcmp_func_t, strcmp_ssse3);
+ RETURN_FUNC(strcmp_func_t, strcmp_generic);
}
+STRCMP_SHIM()
-typedef int strncmp_func(const char* __lhs, const char* __rhs, size_t __n);
DEFINE_IFUNC_FOR(strncmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strncmp_func, strncmp_ssse3);
- RETURN_FUNC(strncmp_func, strncmp_generic);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3"))
+ RETURN_FUNC(strncmp_func_t, strncmp_ssse3);
+ RETURN_FUNC(strncmp_func_t, strncmp_generic);
}
+STRNCMP_SHIM()
-typedef char* strcat_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcat) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcat_func, strcat_ssse3);
- RETURN_FUNC(strcat_func, strcat_generic);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcat_func_t, strcat_ssse3);
+ RETURN_FUNC(strcat_func_t, strcat_generic);
}
+STRCAT_SHIM()
-typedef char* strncat_func(char* __dst, const char* __src, size_t __n);
DEFINE_IFUNC_FOR(strncat) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strncat_func, strncat_ssse3);
- RETURN_FUNC(strncat_func, strncat_openbsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strncat_func_t, strncat_ssse3);
+ RETURN_FUNC(strncat_func_t, strncat_openbsd);
}
+STRNCAT_SHIM()
-typedef size_t strlcat_func(char *dst, const char *src, size_t dsize);
+typedef size_t strlcat_func_t(char*, const char*, size_t);
DEFINE_IFUNC_FOR(strlcat) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcat_func, strlcat_ssse3);
- RETURN_FUNC(strlcat_func, strlcat_openbsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcat_func_t, strlcat_ssse3);
+ RETURN_FUNC(strlcat_func_t, strlcat_openbsd);
}
+DEFINE_STATIC_SHIM(size_t strlcat(char* dst, const char* src, size_t n) {
+ FORWARD(strlcat)(dst, src, n);
+})
-typedef size_t strlcpy_func(char *dst, const char *src, size_t dsize);
+typedef size_t strlcpy_func_t(char*, const char*, size_t);
DEFINE_IFUNC_FOR(strlcpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcpy_func, strlcpy_ssse3);
- RETURN_FUNC(strlcpy_func, strlcpy_openbsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcpy_func_t, strlcpy_ssse3);
+ RETURN_FUNC(strlcpy_func_t, strlcpy_openbsd);
}
+DEFINE_STATIC_SHIM(size_t strlcpy(char* dst, const char* src, size_t n) {
+ FORWARD(strlcpy)(dst, src, n);
+})
-typedef wchar_t* wcscat_func(wchar_t *s1, const wchar_t *s2);
+typedef wchar_t* wcscat_func_t(wchar_t*, const wchar_t*);
DEFINE_IFUNC_FOR(wcscat) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscat_func, wcscat_ssse3);
- RETURN_FUNC(wcscat_func, wcscat_freebsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscat_func_t, wcscat_ssse3);
+ RETURN_FUNC(wcscat_func_t, wcscat_freebsd);
}
+DEFINE_STATIC_SHIM(wchar_t* wcscat(wchar_t* dst, const wchar_t* src) {
+ FORWARD(wcscat)(dst, src);
+})
-typedef wchar_t* wcscpy_func(wchar_t *s1, const wchar_t *s2);
+typedef wchar_t* wcscpy_func_t(wchar_t*, const wchar_t*);
DEFINE_IFUNC_FOR(wcscpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscpy_func, wcscpy_ssse3);
- RETURN_FUNC(wcscpy_func, wcscpy_freebsd);
+ __builtin_cpu_init();
+ // TODO: ssse3 is required by our x86 abi!
+ if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscpy_func_t, wcscpy_ssse3);
+ RETURN_FUNC(wcscpy_func_t, wcscpy_freebsd);
}
+DEFINE_STATIC_SHIM(wchar_t* wcscpy(wchar_t* dst, const wchar_t* src) {
+ FORWARD(wcscpy)(dst, src);
+})
+
+typedef int wmemcmp_func_t(const wchar_t*, const wchar_t*, size_t);
+DEFINE_IFUNC_FOR(wmemcmp) {
+ __builtin_cpu_init();
+ if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(wmemcmp_func_t, wmemcmp_sse4);
+ if (__builtin_cpu_is("atom")) RETURN_FUNC(wmemcmp_func_t, wmemcmp_atom);
+ RETURN_FUNC(wmemcmp_func_t, wmemcmp_freebsd);
+}
+DEFINE_STATIC_SHIM(int wmemcmp(const wchar_t* lhs, const wchar_t* rhs, size_t n) {
+ FORWARD(wmemcmp)(lhs, rhs, n);
+})
} // extern "C"
diff --git a/libc/arch-x86/static_function_dispatch.S b/libc/arch-x86/static_function_dispatch.S
deleted file mode 100644
index 7e8e63d..0000000
--- a/libc/arch-x86/static_function_dispatch.S
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- jmp impl; \
-END(name)
-
-FUNCTION_DELEGATE(memcmp, memcmp_generic)
-FUNCTION_DELEGATE(memset, memset_generic)
-FUNCTION_DELEGATE(__memset_chk, __memset_chk_generic)
-FUNCTION_DELEGATE(memcpy, memmove_generic)
-FUNCTION_DELEGATE(memmove, memmove_generic)
-FUNCTION_DELEGATE(strcpy, strcpy_generic)
-FUNCTION_DELEGATE(strncpy, strncpy_generic)
-FUNCTION_DELEGATE(strlen, strlen_generic)
-FUNCTION_DELEGATE(strcmp, strcmp_generic)
-FUNCTION_DELEGATE(strncmp, strncmp_generic)
-FUNCTION_DELEGATE(strcat, strcat_generic)
-FUNCTION_DELEGATE(wmemcmp, wmemcmp_freebsd)
-FUNCTION_DELEGATE(wcscat, wcscat_freebsd)
-FUNCTION_DELEGATE(strncat, strncat_openbsd)
-FUNCTION_DELEGATE(strlcat, strlcat_openbsd)
-FUNCTION_DELEGATE(strlcpy, strlcpy_openbsd)
-FUNCTION_DELEGATE(wcscpy, wcscpy_freebsd)
diff --git a/libc/arch-x86_64/dynamic_function_dispatch.cpp b/libc/arch-x86_64/dynamic_function_dispatch.cpp
index c846ded..cbe68a3 100644
--- a/libc/arch-x86_64/dynamic_function_dispatch.cpp
+++ b/libc/arch-x86_64/dynamic_function_dispatch.cpp
@@ -32,18 +32,18 @@
extern "C" {
-typedef int memset_func(void* __dst, int __ch, size_t __n);
DEFINE_IFUNC_FOR(memset) {
__builtin_cpu_init();
- if (__builtin_cpu_supports("avx2")) RETURN_FUNC(memset_func, memset_avx2);
- RETURN_FUNC(memset_func, memset_generic);
+ if (__builtin_cpu_supports("avx2")) RETURN_FUNC(memset_func_t, memset_avx2);
+ RETURN_FUNC(memset_func_t, memset_generic);
}
+MEMSET_SHIM()
-typedef void* __memset_chk_func(void* s, int c, size_t n, size_t n2);
DEFINE_IFUNC_FOR(__memset_chk) {
__builtin_cpu_init();
- if (__builtin_cpu_supports("avx2")) RETURN_FUNC(__memset_chk_func, __memset_chk_avx2);
- RETURN_FUNC(__memset_chk_func, __memset_chk_generic);
+ if (__builtin_cpu_supports("avx2")) RETURN_FUNC(__memset_chk_func_t, __memset_chk_avx2);
+ RETURN_FUNC(__memset_chk_func_t, __memset_chk_generic);
}
+__MEMSET_CHK_SHIM()
} // extern "C"
diff --git a/libc/arch-x86_64/static_function_dispatch.S b/libc/arch-x86_64/static_function_dispatch.S
deleted file mode 100644
index 93ff5f2..0000000
--- a/libc/arch-x86_64/static_function_dispatch.S
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2022 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#define FUNCTION_DELEGATE(name, impl) \
-ENTRY(name); \
- jmp impl; \
-END(name)
-
-FUNCTION_DELEGATE(memset, memset_generic)
-FUNCTION_DELEGATE(__memset_chk, __memset_chk_generic)
diff --git a/libc/bionic/getauxval.cpp b/libc/bionic/getauxval.cpp
index e188d73..51ff949 100644
--- a/libc/bionic/getauxval.cpp
+++ b/libc/bionic/getauxval.cpp
@@ -26,14 +26,12 @@
* SUCH DAMAGE.
*/
-#include <stddef.h>
-#include <sys/cdefs.h>
-#include <sys/auxv.h>
-#include <private/bionic_auxv.h>
-#include <private/bionic_globals.h>
-#include <private/bionic_ifuncs.h>
#include <elf.h>
#include <errno.h>
+#include <private/bionic_auxv.h>
+#include <private/bionic_globals.h>
+#include <stddef.h>
+#include <sys/auxv.h>
// This function needs to be safe to call before TLS is set up, so it can't
// access errno or the stack protector.
diff --git a/libc/bionic/libc_init_static.cpp b/libc/bionic/libc_init_static.cpp
index ac97376..f8f7d2a 100644
--- a/libc/bionic/libc_init_static.cpp
+++ b/libc/bionic/libc_init_static.cpp
@@ -30,6 +30,7 @@
#include <elf.h>
#include <errno.h>
#include <malloc.h>
+#include <signal.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
@@ -293,6 +294,28 @@
return level;
}
+static void __enable_mte_signal_handler(int, siginfo_t* info, void*) {
+ if (info->si_code != SI_TIMER) {
+ async_safe_format_log(ANDROID_LOG_ERROR, "libc", "Got BIONIC_ENABLE_MTE not from SI_TIMER");
+ return;
+ }
+ int tagged_addr_ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
+ if (tagged_addr_ctrl < 0) {
+ async_safe_fatal("failed to PR_GET_TAGGED_ADDR_CTRL: %m");
+ }
+ if ((tagged_addr_ctrl & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE) {
+ return;
+ }
+ async_safe_format_log(ANDROID_LOG_INFO, "libc",
+ "Re-enabling MTE, value: %x (tagged_addr_ctrl %lu)",
+ info->si_value.sival_int, info->si_value.sival_int & PR_MTE_TCF_MASK);
+ tagged_addr_ctrl =
+ (tagged_addr_ctrl & ~PR_MTE_TCF_MASK) | (info->si_value.sival_int & PR_MTE_TCF_MASK);
+ if (prctl(PR_SET_TAGGED_ADDR_CTRL, tagged_addr_ctrl, 0, 0, 0) < 0) {
+ async_safe_fatal("failed to PR_SET_TAGGED_ADDR_CTRL %d: %m", tagged_addr_ctrl);
+ }
+}
+
static int64_t __get_memtag_upgrade_secs() {
char* env = getenv("BIONIC_MEMTAG_UPGRADE_SECS");
if (!env) return 0;
@@ -366,7 +389,10 @@
async_safe_fatal("error: failed to set PROT_MTE on main thread stack: %m");
}
}
-
+ struct sigaction action = {};
+ action.sa_flags = SA_SIGINFO | SA_RESTART;
+ action.sa_sigaction = __enable_mte_signal_handler;
+ sigaction(BIONIC_ENABLE_MTE, &action, nullptr);
return;
}
}
diff --git a/libc/private/bionic_ifuncs.h b/libc/private/bionic_ifuncs.h
index e6b349a..b31c903 100644
--- a/libc/private/bionic_ifuncs.h
+++ b/libc/private/bionic_ifuncs.h
@@ -31,6 +31,8 @@
#include <stdint.h>
#include <sys/ifunc.h>
+#include <private/bionic_call_ifunc_resolver.h>
+
#if defined(__aarch64__)
#define IFUNC_ARGS (uint64_t hwcap __attribute__((unused)), \
__ifunc_arg_t* arg __attribute__((unused)))
@@ -40,14 +42,6 @@
#define IFUNC_ARGS ()
#endif
-// We can't have HWASAN enabled in resolvers because they may be called before HWASAN is
-// initialized.
-#define DEFINE_IFUNC_FOR(name) \
- name##_func name __attribute__((ifunc(#name "_resolver"))); \
- __attribute__((visibility("hidden"))) \
- __attribute__((no_sanitize("hwaddress"))) \
- name##_func* name##_resolver IFUNC_ARGS
-
#define DECLARE_FUNC(type, name) \
__attribute__((visibility("hidden"))) \
type name
@@ -56,3 +50,137 @@
DECLARE_FUNC(type, name); \
return name; \
}
+
+#if defined(BIONIC_DYNAMIC_DISPATCH)
+
+// We can't have HWASAN enabled in resolvers because they may be called before
+// HWASAN is initialized.
+#define DEFINE_IFUNC_FOR(name) \
+ name##_func_t name __attribute__((ifunc(#name "_resolver"))); \
+ __attribute__((visibility("hidden"))) \
+ __attribute__((no_sanitize("hwaddress"))) name##_func_t* name##_resolver IFUNC_ARGS
+
+#define DEFINE_STATIC_SHIM(x)
+
+#elif defined(BIONIC_STATIC_DISPATCH)
+
+#define DEFINE_IFUNC_FOR(name) \
+ name##_func_t* name##_resolver IFUNC_ARGS; \
+ __attribute__((visibility("hidden"))) \
+ __attribute__((no_sanitize("hwaddress"))) name##_func_t* name##_resolver IFUNC_ARGS
+
+#define DEFINE_STATIC_SHIM(x) x
+
+#define FORWARD(name) \
+ static name##_func_t* fn = reinterpret_cast<name##_func_t*>( \
+ __bionic_call_ifunc_resolver(reinterpret_cast<ElfW(Addr)>(name##_resolver))); \
+ return fn
+
+#else
+#error neither dynamic nor static dispatch?!
+#endif
+
+typedef void* memchr_func_t(const void*, int, size_t);
+#define MEMCHR_SHIM() \
+ DEFINE_STATIC_SHIM(void* memchr(const void* src, int ch, size_t n) { \
+ FORWARD(memchr)(src, ch, n); \
+ })
+
+typedef int memcmp_func_t(const void*, const void*, size_t);
+#define MEMCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int memcmp(const void* lhs, const void* rhs, size_t n) { \
+ FORWARD(memcmp)(lhs, rhs, n); \
+ })
+
+typedef void* memcpy_func_t(void*, const void*, size_t);
+#define MEMCPY_SHIM() \
+ DEFINE_STATIC_SHIM(void* memcpy(void* dst, const void* src, size_t n) { \
+ FORWARD(memcpy)(dst, src, n); \
+ })
+
+typedef void* memmove_func_t(void*, const void*, size_t);
+#define MEMMOVE_SHIM() \
+ DEFINE_STATIC_SHIM(void* memmove(void* dst, const void* src, size_t n) { \
+ FORWARD(memmove)(dst, src, n); \
+ })
+
+typedef int memrchr_func_t(const void*, int, size_t);
+#define MEMRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(int memrchr(const void* src, int ch, size_t n) { \
+ FORWARD(memrchr)(src, ch, n); \
+ })
+
+typedef void* memset_func_t(void*, int, size_t);
+#define MEMSET_SHIM() \
+ DEFINE_STATIC_SHIM(void* memset(void* dst, int ch, size_t n) { FORWARD(memset)(dst, ch, n); })
+
+typedef void* __memset_chk_func_t(void*, int, size_t, size_t);
+#define __MEMSET_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(void* __memset_chk(void* dst, int ch, size_t n, size_t n2) { \
+ FORWARD(__memset_chk)(dst, ch, n, n2); \
+ })
+
+typedef char* stpcpy_func_t(char*, const char*);
+#define STPCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* stpcpy(char* dst, const char* src) { FORWARD(stpcpy)(dst, src); })
+
+typedef char* strcat_func_t(char*, const char*);
+#define STRCAT_SHIM() \
+ DEFINE_STATIC_SHIM(char* strcat(char* dst, const char* src) { FORWARD(strcat)(dst, src); })
+
+typedef char* __strcat_chk_func_t(char*, const char*, size_t);
+#define __STRCAT_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(char* __strcat_chk(char* dst, const char* src, size_t dst_buf_size) { \
+ FORWARD(__strcat_chk)(dst, src, dst_buf_size); \
+ })
+
+typedef char* strchr_func_t(const char*, int);
+#define STRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(char* strchr(const char* src, int ch) { FORWARD(strchr)(src, ch); })
+
+typedef char* strchrnul_func_t(const char*, int);
+#define STRCHRNUL_SHIM() \
+ DEFINE_STATIC_SHIM(char* strchrnul(const char* src, int ch) { FORWARD(strchrnul)(src, ch); })
+
+typedef int strcmp_func_t(const char*, const char*);
+#define STRCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int strcmp(char* lhs, const char* rhs) { FORWARD(strcmp)(lhs, rhs); })
+
+typedef char* strcpy_func_t(char*, const char*);
+#define STRCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* strcpy(char* dst, const char* src) { FORWARD(strcpy)(dst, src); })
+
+typedef char* __strcpy_chk_func_t(char*, const char*, size_t);
+#define __STRCPY_CHK_SHIM() \
+ DEFINE_STATIC_SHIM(char* __strcpy_chk(char* dst, const char* src, size_t dst_len) { \
+ FORWARD(__strcpy_chk)(dst, src, dst_len); \
+ })
+
+typedef size_t strlen_func_t(const char*);
+#define STRLEN_SHIM() DEFINE_STATIC_SHIM(size_t strlen(const char* s) { FORWARD(strlen)(s); })
+
+typedef char* strncat_func_t(char*, const char*, size_t);
+#define STRNCAT_SHIM() \
+ DEFINE_STATIC_SHIM(char* strncat(char* dst, const char* src, size_t n) { \
+ FORWARD(strncat)(dst, src, n); \
+ })
+
+typedef int strncmp_func_t(const char*, const char*, size_t);
+#define STRNCMP_SHIM() \
+ DEFINE_STATIC_SHIM(int strncmp(const char* lhs, const char* rhs, size_t n) { \
+ FORWARD(strncmp)(lhs, rhs, n); \
+ })
+
+typedef char* strncpy_func_t(char*, const char*, size_t);
+#define STRNCPY_SHIM() \
+ DEFINE_STATIC_SHIM(char* strncpy(char* dst, const char* src, size_t n) { \
+ FORWARD(strncpy)(dst, src, n); \
+ })
+
+typedef size_t strnlen_func_t(const char*, size_t);
+#define STRNLEN_SHIM() \
+ DEFINE_STATIC_SHIM(size_t strnlen(const char* s, size_t n) { FORWARD(strnlen)(s, n); })
+
+typedef char* strrchr_func_t(const char*, int);
+#define STRRCHR_SHIM() \
+ DEFINE_STATIC_SHIM(char* strrchr(const char* src, int ch) { FORWARD(strrchr)(src, ch); })