libc/bionic/malloc_common.cpp

/*
 * Copyright (C) 2009 The Android Open Source Project
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

// Contains a thin layer that calls whatever real native allocator
// has been defined. For the libc shared library, this allows the
// implementation of a debug malloc that can intercept all of the allocation
// calls and add special debugging code to attempt to catch allocation
// errors. All of the debugging code is implemented in a separate shared
// library that is only loaded when the property "libc.debug.malloc.options"
// is set to a non-zero value.

#include <errno.h>
#include <stdint.h>
#include <stdio.h>

#include <platform/bionic/malloc.h>
#include <private/ScopedPthreadMutexLocker.h>
#include <private/bionic_config.h>
#include <private/bionic_defs.h>

#include "gwp_asan_wrappers.h"
#include "heap_tagging.h"
#include "heap_zero_init.h"
#include "malloc_common.h"
#include "malloc_limit.h"
#include "malloc_tagged_pointers.h"

// =============================================================================
// Global variables instantations.
// =============================================================================

// Malloc hooks globals.
void* (*volatile __malloc_hook)(size_t, const void*);
void* (*volatile __realloc_hook)(void*, size_t, const void*);
void (*volatile __free_hook)(void*, const void*);
void* (*volatile __memalign_hook)(size_t, size_t, const void*);
// =============================================================================

// =============================================================================
// Allocation functions
// =============================================================================
extern "C" void* calloc(size_t n_elements, size_t elem_size) {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return MaybeTagPointer(dispatch_table->calloc(n_elements, elem_size));
  }
  void* result = Malloc(calloc)(n_elements, elem_size);
  if (__predict_false(result == nullptr)) {
    warning_log("calloc(%zu, %zu) failed: returning null pointer", n_elements, elem_size);
  }
  return MaybeTagPointer(result);
}

extern "C" void free(void* mem) {
  auto dispatch_table = GetDispatchTable();
  mem = MaybeUntagAndCheckPointer(mem);
  if (__predict_false(dispatch_table != nullptr)) {
    dispatch_table->free(mem);
  } else {
    Malloc(free)(mem);
  }
}

extern "C" struct mallinfo mallinfo() {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return dispatch_table->mallinfo();
  }
  return Malloc(mallinfo)();
}

extern "C" int malloc_info(int options, FILE* fp) {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return dispatch_table->malloc_info(options, fp);
  }
  return Malloc(malloc_info)(options, fp);
}

extern "C" int mallopt(int param, int value) {
  // Some are handled by libc directly rather than by the allocator.
  if (param == M_BIONIC_SET_HEAP_TAGGING_LEVEL) {
    ScopedPthreadMutexLocker locker(&g_heap_tagging_lock);
    return SetHeapTaggingLevel(static_cast<HeapTaggingLevel>(value));
  }
  if (param == M_BIONIC_ZERO_INIT) {
    return SetHeapZeroInitialize(value);
  }

  // The rest we pass on...
  int retval;
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    retval = dispatch_table->mallopt(param, value);
  } else {
    retval = Malloc(mallopt)(param, value);
  }

  // Track the M_DECAY_TIME mallopt calls.
  if (param == M_DECAY_TIME && retval == 1) {
    __libc_globals.mutate([value](libc_globals* globals) {
      if (value <= 0) {
        atomic_store(&globals->decay_time_enabled, false);
      } else {
        atomic_store(&globals->decay_time_enabled, true);
      }
    });
  }
  return retval;
}

extern "C" void* malloc(size_t bytes) {
  auto dispatch_table = GetDispatchTable();
  void *result;
  if (__predict_false(dispatch_table != nullptr)) {
    result = dispatch_table->malloc(bytes);
  } else {
    result = Malloc(malloc)(bytes);
  }
  if (__predict_false(result == nullptr)) {
    warning_log("malloc(%zu) failed: returning null pointer", bytes);
    return nullptr;
  }
  return MaybeTagPointer(result);
}

extern "C" size_t malloc_usable_size(const void* mem) {
  auto dispatch_table = GetDispatchTable();
  mem = MaybeUntagAndCheckPointer(mem);
  if (__predict_false(dispatch_table != nullptr)) {
    return dispatch_table->malloc_usable_size(mem);
  }
  return Malloc(malloc_usable_size)(mem);
}

extern "C" void* memalign(size_t alignment, size_t bytes) {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return MaybeTagPointer(dispatch_table->memalign(alignment, bytes));
  }
  void* result = Malloc(memalign)(alignment, bytes);
  if (__predict_false(result == nullptr)) {
    warning_log("memalign(%zu, %zu) failed: returning null pointer", alignment, bytes);
  }
  return MaybeTagPointer(result);
}

extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) {
  auto dispatch_table = GetDispatchTable();
  int result;
  if (__predict_false(dispatch_table != nullptr)) {
    result = dispatch_table->posix_memalign(memptr, alignment, size);
  } else {
    result = Malloc(posix_memalign)(memptr, alignment, size);
  }
  if (result == 0) {
    *memptr = MaybeTagPointer(*memptr);
  }
  return result;
}

extern "C" void* aligned_alloc(size_t alignment, size_t size) {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return MaybeTagPointer(dispatch_table->aligned_alloc(alignment, size));
  }
  void* result = Malloc(aligned_alloc)(alignment, size);
  if (__predict_false(result == nullptr)) {
    warning_log("aligned_alloc(%zu, %zu) failed: returning null pointer", alignment, size);
  }
  return MaybeTagPointer(result);
}

extern "C" __attribute__((__noinline__)) void* realloc(void* old_mem, size_t bytes) {
  auto dispatch_table = GetDispatchTable();
  old_mem = MaybeUntagAndCheckPointer(old_mem);
  if (__predict_false(dispatch_table != nullptr)) {
    return MaybeTagPointer(dispatch_table->realloc(old_mem, bytes));
  }
  void* result = Malloc(realloc)(old_mem, bytes);
  if (__predict_false(result == nullptr && bytes != 0)) {
    warning_log("realloc(%p, %zu) failed: returning null pointer", old_mem, bytes);
  }
  return MaybeTagPointer(result);
}

extern "C" void* reallocarray(void* old_mem, size_t item_count, size_t item_size) {
  size_t new_size;
  if (__builtin_mul_overflow(item_count, item_size, &new_size)) {
    warning_log("reallocaray(%p, %zu, %zu) failed: returning null pointer",
                old_mem, item_count, item_size);
    errno = ENOMEM;
    return nullptr;
  }
  return realloc(old_mem, new_size);
}

#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
extern "C" void* pvalloc(size_t bytes) {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return MaybeTagPointer(dispatch_table->pvalloc(bytes));
  }
  void* result = Malloc(pvalloc)(bytes);
  if (__predict_false(result == nullptr)) {
    warning_log("pvalloc(%zu) failed: returning null pointer", bytes);
  }
  return MaybeTagPointer(result);
}

extern "C" void* valloc(size_t bytes) {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return MaybeTagPointer(dispatch_table->valloc(bytes));
  }
  void* result = Malloc(valloc)(bytes);
  if (__predict_false(result == nullptr)) {
    warning_log("valloc(%zu) failed: returning null pointer", bytes);
  }
  return MaybeTagPointer(result);
}
#endif
// =============================================================================

struct CallbackWrapperArg {
  void (*callback)(uintptr_t base, size_t size, void* arg);
  void* arg;
};

void CallbackWrapper(uintptr_t base, size_t size, void* arg) {
  CallbackWrapperArg* wrapper_arg = reinterpret_cast<CallbackWrapperArg*>(arg);
  wrapper_arg->callback(
    reinterpret_cast<uintptr_t>(MaybeTagPointer(reinterpret_cast<void*>(base))),
    size, wrapper_arg->arg);
}

// =============================================================================
// Exported for use by libmemunreachable.
// =============================================================================

// Calls callback for every allocation in the anonymous heap mapping
// [base, base+size). Must be called between malloc_disable and malloc_enable.
// `base` in this can take either a tagged or untagged pointer, but we always
// provide a tagged pointer to the `base` argument of `callback` if the kernel
// supports tagged pointers.
extern "C" int malloc_iterate(uintptr_t base, size_t size,
    void (*callback)(uintptr_t base, size_t size, void* arg), void* arg) {
  auto dispatch_table = GetDispatchTable();
  // Wrap the malloc_iterate callback we were provided, in order to provide
  // pointer tagging support.
  CallbackWrapperArg wrapper_arg;
  wrapper_arg.callback = callback;
  wrapper_arg.arg = arg;
  uintptr_t untagged_base =
      reinterpret_cast<uintptr_t>(UntagPointer(reinterpret_cast<void*>(base)));
  if (__predict_false(dispatch_table != nullptr)) {
    return dispatch_table->malloc_iterate(
      untagged_base, size, CallbackWrapper, &wrapper_arg);
  }
  return Malloc(malloc_iterate)(
    untagged_base, size, CallbackWrapper, &wrapper_arg);
}

// Disable calls to malloc so malloc_iterate gets a consistent view of
// allocated memory.
extern "C" void malloc_disable() {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return dispatch_table->malloc_disable();
  }
  return Malloc(malloc_disable)();
}

// Re-enable calls to malloc after a previous call to malloc_disable.
extern "C" void malloc_enable() {
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    return dispatch_table->malloc_enable();
  }
  return Malloc(malloc_enable)();
}

#if defined(LIBC_STATIC)
extern "C" ssize_t malloc_backtrace(void*, uintptr_t*, size_t) {
  return 0;
}
#endif

#if __has_feature(hwaddress_sanitizer)
// FIXME: implement these in HWASan allocator.
extern "C" int __sanitizer_malloc_iterate(uintptr_t base __unused, size_t size __unused,
                                          void (*callback)(uintptr_t base, size_t size, void* arg)
                                              __unused,
                                          void* arg __unused) {
  return 0;
}

extern "C" void __sanitizer_malloc_disable() {
}

extern "C" void __sanitizer_malloc_enable() {
}

extern "C" int __sanitizer_malloc_info(int, FILE*) {
  errno = ENOTSUP;
  return -1;
}
#endif
// =============================================================================

static constexpr MallocDispatch __libc_malloc_default_dispatch __attribute__((unused)) = {
  Malloc(calloc),
  Malloc(free),
  Malloc(mallinfo),
  Malloc(malloc),
  Malloc(malloc_usable_size),
  Malloc(memalign),
  Malloc(posix_memalign),
#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
  Malloc(pvalloc),
#endif
  Malloc(realloc),
#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
  Malloc(valloc),
#endif
  Malloc(malloc_iterate),
  Malloc(malloc_disable),
  Malloc(malloc_enable),
  Malloc(mallopt),
  Malloc(aligned_alloc),
  Malloc(malloc_info),
};

const MallocDispatch* NativeAllocatorDispatch() {
  return &__libc_malloc_default_dispatch;
}

#if !defined(LIBC_STATIC)
void MallocInitImpl(libc_globals* globals);
#endif

// Initializes memory allocation framework.
// This routine is called from __libc_init routines in libc_init_dynamic.cpp
// and libc_init_static.cpp.
__BIONIC_WEAK_FOR_NATIVE_BRIDGE
__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals) {
#if !defined(LIBC_STATIC)
  MallocInitImpl(globals);
#endif
  const char* value = getenv("MALLOC_USE_APP_DEFAULTS");
  if (value == nullptr || value[0] == '\0') {
    return;
  }

  // Normal apps currently turn off zero init for performance reasons.
  SetHeapZeroInitialize(false);

  // Do not call mallopt directly since that will try and lock the globals
  // data structure.
  int retval;
  auto dispatch_table = GetDispatchTable();
  if (__predict_false(dispatch_table != nullptr)) {
    retval = dispatch_table->mallopt(M_DECAY_TIME, 1);
  } else {
    retval = Malloc(mallopt)(M_DECAY_TIME, 1);
  }
  if (retval == 1) {
    globals->decay_time_enabled = true;
  }
}