Add tagged pointers to bionic.
This patch introduces tagged pointers to bionic. We add a static tag to
all pointers on arm64 compatible platforms (needs requisite
top-byte-ignore hardware feature and relevant kernel patches).
We dynamically detect TBI-compatible devices (a device with the TBI feature and
kernel support) at process start time, and insert an implementation-dependent
tag into the top byte of the pointer for all heap allocations. We then check
that the tag has not been truncated when deallocating the memory.
If an application incorrectly writes to the top byte of the pointer, we
terminate the process at time of detection. This will allow MTE-incompatible
applications to be caught early.
Bug: 135754954
Bug: 147147490
Test: cd bionic && atest .
Change-Id: I6e5b809fc81f55dd517f845eaf20f3c0ebd4d86e
diff --git a/libc/bionic/malloc_common.cpp b/libc/bionic/malloc_common.cpp
index a0da3db..199d7fc 100644
--- a/libc/bionic/malloc_common.cpp
+++ b/libc/bionic/malloc_common.cpp
@@ -43,6 +43,7 @@
#include "malloc_common.h"
#include "malloc_limit.h"
+#include "malloc_tagged_pointers.h"
// =============================================================================
// Global variables instantations.
@@ -61,17 +62,18 @@
extern "C" void* calloc(size_t n_elements, size_t elem_size) {
auto dispatch_table = GetDispatchTable();
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->calloc(n_elements, elem_size);
+ return MaybeTagPointer(dispatch_table->calloc(n_elements, elem_size));
}
void* result = Malloc(calloc)(n_elements, elem_size);
if (__predict_false(result == nullptr)) {
warning_log("calloc(%zu, %zu) failed: returning null pointer", n_elements, elem_size);
}
- return result;
+ return MaybeTagPointer(result);
}
extern "C" void free(void* mem) {
auto dispatch_table = GetDispatchTable();
+ mem = MaybeUntagAndCheckPointer(mem);
if (__predict_false(dispatch_table != nullptr)) {
dispatch_table->free(mem);
} else {
@@ -105,18 +107,22 @@
extern "C" void* malloc(size_t bytes) {
auto dispatch_table = GetDispatchTable();
+ void *result;
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->malloc(bytes);
+ result = dispatch_table->malloc(bytes);
+ } else {
+ result = Malloc(malloc)(bytes);
}
- void* result = Malloc(malloc)(bytes);
if (__predict_false(result == nullptr)) {
warning_log("malloc(%zu) failed: returning null pointer", bytes);
+ return nullptr;
}
- return result;
+ return MaybeTagPointer(result);
}
extern "C" size_t malloc_usable_size(const void* mem) {
auto dispatch_table = GetDispatchTable();
+ mem = MaybeUntagAndCheckPointer(mem);
if (__predict_false(dispatch_table != nullptr)) {
return dispatch_table->malloc_usable_size(mem);
}
@@ -126,45 +132,52 @@
extern "C" void* memalign(size_t alignment, size_t bytes) {
auto dispatch_table = GetDispatchTable();
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->memalign(alignment, bytes);
+ return MaybeTagPointer(dispatch_table->memalign(alignment, bytes));
}
void* result = Malloc(memalign)(alignment, bytes);
if (__predict_false(result == nullptr)) {
warning_log("memalign(%zu, %zu) failed: returning null pointer", alignment, bytes);
}
- return result;
+ return MaybeTagPointer(result);
}
extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) {
auto dispatch_table = GetDispatchTable();
+ int result;
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->posix_memalign(memptr, alignment, size);
+ result = dispatch_table->posix_memalign(memptr, alignment, size);
+ } else {
+ result = Malloc(posix_memalign)(memptr, alignment, size);
}
- return Malloc(posix_memalign)(memptr, alignment, size);
+ if (result == 0) {
+ *memptr = MaybeTagPointer(*memptr);
+ }
+ return result;
}
extern "C" void* aligned_alloc(size_t alignment, size_t size) {
auto dispatch_table = GetDispatchTable();
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->aligned_alloc(alignment, size);
+ return MaybeTagPointer(dispatch_table->aligned_alloc(alignment, size));
}
void* result = Malloc(aligned_alloc)(alignment, size);
if (__predict_false(result == nullptr)) {
warning_log("aligned_alloc(%zu, %zu) failed: returning null pointer", alignment, size);
}
- return result;
+ return MaybeTagPointer(result);
}
extern "C" __attribute__((__noinline__)) void* realloc(void* old_mem, size_t bytes) {
auto dispatch_table = GetDispatchTable();
+ old_mem = MaybeUntagAndCheckPointer(old_mem);
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->realloc(old_mem, bytes);
+ return MaybeTagPointer(dispatch_table->realloc(old_mem, bytes));
}
void* result = Malloc(realloc)(old_mem, bytes);
if (__predict_false(result == nullptr && bytes != 0)) {
warning_log("realloc(%p, %zu) failed: returning null pointer", old_mem, bytes);
}
- return result;
+ return MaybeTagPointer(result);
}
extern "C" void* reallocarray(void* old_mem, size_t item_count, size_t item_size) {
@@ -182,42 +195,66 @@
extern "C" void* pvalloc(size_t bytes) {
auto dispatch_table = GetDispatchTable();
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->pvalloc(bytes);
+ return MaybeTagPointer(dispatch_table->pvalloc(bytes));
}
void* result = Malloc(pvalloc)(bytes);
if (__predict_false(result == nullptr)) {
warning_log("pvalloc(%zu) failed: returning null pointer", bytes);
}
- return result;
+ return MaybeTagPointer(result);
}
extern "C" void* valloc(size_t bytes) {
auto dispatch_table = GetDispatchTable();
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->valloc(bytes);
+ return MaybeTagPointer(dispatch_table->valloc(bytes));
}
void* result = Malloc(valloc)(bytes);
if (__predict_false(result == nullptr)) {
warning_log("valloc(%zu) failed: returning null pointer", bytes);
}
- return result;
+ return MaybeTagPointer(result);
}
#endif
// =============================================================================
+struct CallbackWrapperArg {
+ void (*callback)(uintptr_t base, size_t size, void* arg);
+ void* arg;
+};
+
+void CallbackWrapper(uintptr_t base, size_t size, void* arg) {
+ CallbackWrapperArg* wrapper_arg = reinterpret_cast<CallbackWrapperArg*>(arg);
+ wrapper_arg->callback(
+ reinterpret_cast<uintptr_t>(MaybeTagPointer(reinterpret_cast<void*>(base))),
+ size, wrapper_arg->arg);
+}
+
// =============================================================================
// Exported for use by libmemunreachable.
// =============================================================================
// Calls callback for every allocation in the anonymous heap mapping
-// [base, base+size). Must be called between malloc_disable and malloc_enable.
+// [base, base+size). Must be called between malloc_disable and malloc_enable.
+// `base` in this can take either a tagged or untagged pointer, but we always
+// provide a tagged pointer to the `base` argument of `callback` if the kernel
+// supports tagged pointers.
extern "C" int malloc_iterate(uintptr_t base, size_t size,
void (*callback)(uintptr_t base, size_t size, void* arg), void* arg) {
auto dispatch_table = GetDispatchTable();
+ // Wrap the malloc_iterate callback we were provided, in order to provide
+ // pointer tagging support.
+ CallbackWrapperArg wrapper_arg;
+ wrapper_arg.callback = callback;
+ wrapper_arg.arg = arg;
+ uintptr_t untagged_base =
+ reinterpret_cast<uintptr_t>(UntagPointer(reinterpret_cast<void*>(base)));
if (__predict_false(dispatch_table != nullptr)) {
- return dispatch_table->malloc_iterate(base, size, callback, arg);
+ return dispatch_table->malloc_iterate(
+ untagged_base, size, CallbackWrapper, &wrapper_arg);
}
- return Malloc(malloc_iterate)(base, size, callback, arg);
+ return Malloc(malloc_iterate)(
+ untagged_base, size, CallbackWrapper, &wrapper_arg);
}
// Disable calls to malloc so malloc_iterate gets a consistent view of