blob: 528ffb01ded20b7fbfb7b788f6537579521d48d9 [file] [log] [blame]
Christopher Ferris63860cb2015-11-16 17:30:32 -08001/*
2 * Copyright (C) 2009 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29// Contains a thin layer that calls whatever real native allocator
30// has been defined. For the libc shared library, this allows the
31// implementation of a debug malloc that can intercept all of the allocation
32// calls and add special debugging code to attempt to catch allocation
33// errors. All of the debugging code is implemented in a separate shared
34// library that is only loaded when the property "libc.debug.malloc.options"
35// is set to a non-zero value. There are two functions exported to
36// allow ddms, or other external users to get information from the debug
37// allocation.
38// get_malloc_leak_info: Returns information about all of the known native
39// allocations that are currently in use.
40// free_malloc_leak_info: Frees the data allocated by the call to
41// get_malloc_leak_info.
Christopher Ferris2e1a40a2018-06-13 10:46:34 -070042// write_malloc_leak_info: Writes the leak info data to a file.
Christopher Ferris63860cb2015-11-16 17:30:32 -080043
Colin Cross869691c2016-01-29 12:48:18 -080044#include <pthread.h>
Florian Mayerf7f71e32018-08-31 15:36:48 -070045#include <stdatomic.h>
Colin Cross869691c2016-01-29 12:48:18 -080046
Christopher Ferris63860cb2015-11-16 17:30:32 -080047#include <private/bionic_config.h>
48#include <private/bionic_globals.h>
49#include <private/bionic_malloc_dispatch.h>
50
Evgenii Stepanovbe551f52018-08-13 16:46:15 -070051#if __has_feature(hwaddress_sanitizer)
52// FIXME: implement these in HWASan allocator.
53extern "C" int __sanitizer_iterate(uintptr_t base __unused, size_t size __unused,
54 void (*callback)(uintptr_t base, size_t size, void* arg) __unused,
55 void* arg __unused) {
56 return 0;
57}
58
59extern "C" void __sanitizer_malloc_disable() {
60}
61
62extern "C" void __sanitizer_malloc_enable() {
63}
64#include <sanitizer/hwasan_interface.h>
65#define Malloc(function) __sanitizer_ ## function
66
67#else // __has_feature(hwaddress_sanitizer)
Christopher Ferris63860cb2015-11-16 17:30:32 -080068#include "jemalloc.h"
69#define Malloc(function) je_ ## function
Evgenii Stepanovbe551f52018-08-13 16:46:15 -070070#endif
Christopher Ferris63860cb2015-11-16 17:30:32 -080071
Florian Mayerf7f71e32018-08-31 15:36:48 -070072template <typename T>
73static T* RemoveConst(const T* x) {
74 return const_cast<T*>(x);
75}
76
77// RemoveConst is a workaround for bug in current libcxx. Fix in
78// https://reviews.llvm.org/D47613
79#define atomic_load_explicit_const(obj, order) atomic_load_explicit(RemoveConst(obj), order)
80
81static constexpr memory_order default_read_memory_order = memory_order_acquire;
82
Christopher Ferris63860cb2015-11-16 17:30:32 -080083static constexpr MallocDispatch __libc_malloc_default_dispatch
84 __attribute__((unused)) = {
85 Malloc(calloc),
86 Malloc(free),
87 Malloc(mallinfo),
88 Malloc(malloc),
89 Malloc(malloc_usable_size),
90 Malloc(memalign),
91 Malloc(posix_memalign),
92#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
93 Malloc(pvalloc),
94#endif
95 Malloc(realloc),
96#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
97 Malloc(valloc),
98#endif
Colin Cross869691c2016-01-29 12:48:18 -080099 Malloc(iterate),
100 Malloc(malloc_disable),
101 Malloc(malloc_enable),
Christopher Ferrisa1c0d2f2017-05-15 15:50:19 -0700102 Malloc(mallopt),
Christopher Ferriscae21a92018-02-05 18:14:55 -0800103 Malloc(aligned_alloc),
Christopher Ferris63860cb2015-11-16 17:30:32 -0800104 };
105
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800106// Malloc hooks.
107void* (*volatile __malloc_hook)(size_t, const void*);
108void* (*volatile __realloc_hook)(void*, size_t, const void*);
109void (*volatile __free_hook)(void*, const void*);
110void* (*volatile __memalign_hook)(size_t, size_t, const void*);
111
Christopher Ferris63860cb2015-11-16 17:30:32 -0800112// In a VM process, this is set to 1 after fork()ing out of zygote.
113int gMallocLeakZygoteChild = 0;
114
115// =============================================================================
116// Allocation functions
117// =============================================================================
118extern "C" void* calloc(size_t n_elements, size_t elem_size) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700119 auto _calloc = atomic_load_explicit_const(
120 &__libc_globals->malloc_dispatch.calloc,
121 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800122 if (__predict_false(_calloc != nullptr)) {
123 return _calloc(n_elements, elem_size);
124 }
125 return Malloc(calloc)(n_elements, elem_size);
126}
127
128extern "C" void free(void* mem) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700129 auto _free = atomic_load_explicit_const(
130 &__libc_globals->malloc_dispatch.free,
131 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800132 if (__predict_false(_free != nullptr)) {
133 _free(mem);
134 } else {
135 Malloc(free)(mem);
136 }
137}
138
139extern "C" struct mallinfo mallinfo() {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700140 auto _mallinfo = atomic_load_explicit_const(
141 &__libc_globals->malloc_dispatch.mallinfo,
142 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800143 if (__predict_false(_mallinfo != nullptr)) {
144 return _mallinfo();
145 }
146 return Malloc(mallinfo)();
147}
148
Christopher Ferrisa1c0d2f2017-05-15 15:50:19 -0700149extern "C" int mallopt(int param, int value) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700150 auto _mallopt = atomic_load_explicit_const(
151 &__libc_globals->malloc_dispatch.mallopt,
152 default_read_memory_order);
Christopher Ferrisa1c0d2f2017-05-15 15:50:19 -0700153 if (__predict_false(_mallopt != nullptr)) {
154 return _mallopt(param, value);
155 }
156 return Malloc(mallopt)(param, value);
157}
158
Christopher Ferris63860cb2015-11-16 17:30:32 -0800159extern "C" void* malloc(size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700160 auto _malloc = atomic_load_explicit_const(
161 &__libc_globals->malloc_dispatch.malloc,
162 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800163 if (__predict_false(_malloc != nullptr)) {
164 return _malloc(bytes);
165 }
166 return Malloc(malloc)(bytes);
167}
168
169extern "C" size_t malloc_usable_size(const void* mem) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700170 auto _malloc_usable_size = atomic_load_explicit_const(
171 &__libc_globals->malloc_dispatch.malloc_usable_size,
172 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800173 if (__predict_false(_malloc_usable_size != nullptr)) {
174 return _malloc_usable_size(mem);
175 }
176 return Malloc(malloc_usable_size)(mem);
177}
178
179extern "C" void* memalign(size_t alignment, size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700180 auto _memalign = atomic_load_explicit_const(
181 &__libc_globals->malloc_dispatch.memalign,
182 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800183 if (__predict_false(_memalign != nullptr)) {
184 return _memalign(alignment, bytes);
185 }
186 return Malloc(memalign)(alignment, bytes);
187}
188
189extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700190 auto _posix_memalign = atomic_load_explicit_const(
191 &__libc_globals->malloc_dispatch.posix_memalign,
192 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800193 if (__predict_false(_posix_memalign != nullptr)) {
194 return _posix_memalign(memptr, alignment, size);
195 }
196 return Malloc(posix_memalign)(memptr, alignment, size);
197}
198
Christopher Ferriscae21a92018-02-05 18:14:55 -0800199extern "C" void* aligned_alloc(size_t alignment, size_t size) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700200 auto _aligned_alloc = atomic_load_explicit_const(
201 &__libc_globals->malloc_dispatch.aligned_alloc,
202 default_read_memory_order);
Christopher Ferriscae21a92018-02-05 18:14:55 -0800203 if (__predict_false(_aligned_alloc != nullptr)) {
204 return _aligned_alloc(alignment, size);
205 }
206 return Malloc(aligned_alloc)(alignment, size);
207}
208
Christopher Ferris63860cb2015-11-16 17:30:32 -0800209extern "C" void* realloc(void* old_mem, size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700210 auto _realloc = atomic_load_explicit_const(
211 &__libc_globals->malloc_dispatch.realloc,
212 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800213 if (__predict_false(_realloc != nullptr)) {
214 return _realloc(old_mem, bytes);
215 }
216 return Malloc(realloc)(old_mem, bytes);
217}
218
Elliott Hughesb1770852018-09-18 12:52:42 -0700219extern "C" void* reallocarray(void* old_mem, size_t item_count, size_t item_size) {
220 size_t new_size;
221 if (__builtin_mul_overflow(item_count, item_size, &new_size)) {
222 errno = ENOMEM;
223 return nullptr;
224 }
225 return realloc(old_mem, new_size);
226}
227
Christopher Ferris63860cb2015-11-16 17:30:32 -0800228#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
229extern "C" void* pvalloc(size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700230 auto _pvalloc = atomic_load_explicit_const(
231 &__libc_globals->malloc_dispatch.pvalloc,
232 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800233 if (__predict_false(_pvalloc != nullptr)) {
234 return _pvalloc(bytes);
235 }
236 return Malloc(pvalloc)(bytes);
237}
238
239extern "C" void* valloc(size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700240 auto _valloc = atomic_load_explicit_const(
241 &__libc_globals->malloc_dispatch.valloc,
242 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800243 if (__predict_false(_valloc != nullptr)) {
244 return _valloc(bytes);
245 }
246 return Malloc(valloc)(bytes);
247}
248#endif
249
250// We implement malloc debugging only in libc.so, so the code below
251// must be excluded if we compile this file for static libc.a
252#if !defined(LIBC_STATIC)
253
254#include <dlfcn.h>
Florian Mayer4e28ea12018-11-22 17:34:34 +0000255#include <fcntl.h>
Christopher Ferris63860cb2015-11-16 17:30:32 -0800256#include <stdio.h>
257#include <stdlib.h>
Florian Mayer4e28ea12018-11-22 17:34:34 +0000258#include <unistd.h>
Christopher Ferris63860cb2015-11-16 17:30:32 -0800259
Christopher Ferris7a3681e2017-04-24 17:48:32 -0700260#include <async_safe/log.h>
Christopher Ferris63860cb2015-11-16 17:30:32 -0800261#include <sys/system_properties.h>
262
263extern "C" int __cxa_atexit(void (*func)(void *), void *arg, void *dso);
264
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800265static const char* HOOKS_SHARED_LIB = "libc_malloc_hooks.so";
266static const char* HOOKS_PROPERTY_ENABLE = "libc.debug.hooks.enable";
267static const char* HOOKS_ENV_ENABLE = "LIBC_HOOKS_ENABLE";
268
Christopher Ferris63860cb2015-11-16 17:30:32 -0800269static const char* DEBUG_SHARED_LIB = "libc_malloc_debug.so";
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800270static const char* DEBUG_PROPERTY_OPTIONS = "libc.debug.malloc.options";
271static const char* DEBUG_PROPERTY_PROGRAM = "libc.debug.malloc.program";
272static const char* DEBUG_ENV_OPTIONS = "LIBC_DEBUG_MALLOC_OPTIONS";
Christopher Ferris63860cb2015-11-16 17:30:32 -0800273
Florian Mayerf7f71e32018-08-31 15:36:48 -0700274static const char* HEAPPROFD_SHARED_LIB = "heapprofd_client.so";
275static const char* HEAPPROFD_PREFIX = "heapprofd";
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000276static const char* HEAPPROFD_PROPERTY_ENABLE = "heapprofd.enable";
Florian Mayerf7f71e32018-08-31 15:36:48 -0700277static const int HEAPPROFD_SIGNAL = __SIGRTMIN + 4;
278
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800279enum FunctionEnum : uint8_t {
280 FUNC_INITIALIZE,
281 FUNC_FINALIZE,
282 FUNC_GET_MALLOC_LEAK_INFO,
283 FUNC_FREE_MALLOC_LEAK_INFO,
284 FUNC_MALLOC_BACKTRACE,
Christopher Ferris2e1a40a2018-06-13 10:46:34 -0700285 FUNC_WRITE_LEAK_INFO,
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800286 FUNC_LAST,
287};
288static void* g_functions[FUNC_LAST];
Christopher Ferris63860cb2015-11-16 17:30:32 -0800289
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800290typedef void (*finalize_func_t)();
291typedef bool (*init_func_t)(const MallocDispatch*, int*, const char*);
292typedef void (*get_malloc_leak_info_func_t)(uint8_t**, size_t*, size_t*, size_t*, size_t*);
293typedef void (*free_malloc_leak_info_func_t)(uint8_t*);
Christopher Ferris2e1a40a2018-06-13 10:46:34 -0700294typedef bool (*write_malloc_leak_info_func_t)(FILE*);
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800295typedef ssize_t (*malloc_backtrace_func_t)(void*, uintptr_t*, size_t);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800296
297// =============================================================================
298// Log functions
299// =============================================================================
300#define error_log(format, ...) \
Christopher Ferris7a3681e2017-04-24 17:48:32 -0700301 async_safe_format_log(ANDROID_LOG_ERROR, "libc", (format), ##__VA_ARGS__ )
Christopher Ferris63860cb2015-11-16 17:30:32 -0800302#define info_log(format, ...) \
Christopher Ferris7a3681e2017-04-24 17:48:32 -0700303 async_safe_format_log(ANDROID_LOG_INFO, "libc", (format), ##__VA_ARGS__ )
Christopher Ferris63860cb2015-11-16 17:30:32 -0800304// =============================================================================
305
306// =============================================================================
307// Exported for use by ddms.
308// =============================================================================
309
310// Retrieve native heap information.
311//
312// "*info" is set to a buffer we allocate
313// "*overall_size" is set to the size of the "info" buffer
314// "*info_size" is set to the size of a single entry
315// "*total_memory" is set to the sum of all allocations we're tracking; does
316// not include heap overhead
317// "*backtrace_size" is set to the maximum number of entries in the back trace
318extern "C" void get_malloc_leak_info(uint8_t** info, size_t* overall_size,
319 size_t* info_size, size_t* total_memory, size_t* backtrace_size) {
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800320 void* func = g_functions[FUNC_GET_MALLOC_LEAK_INFO];
321 if (func == nullptr) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800322 return;
323 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800324 reinterpret_cast<get_malloc_leak_info_func_t>(func)(info, overall_size, info_size, total_memory,
325 backtrace_size);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800326}
327
328extern "C" void free_malloc_leak_info(uint8_t* info) {
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800329 void* func = g_functions[FUNC_FREE_MALLOC_LEAK_INFO];
330 if (func == nullptr) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800331 return;
332 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800333 reinterpret_cast<free_malloc_leak_info_func_t>(func)(info);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800334}
Colin Cross869691c2016-01-29 12:48:18 -0800335
Christopher Ferris2e1a40a2018-06-13 10:46:34 -0700336extern "C" void write_malloc_leak_info(FILE* fp) {
337 if (fp == nullptr) {
338 error_log("write_malloc_leak_info called with a nullptr");
339 return;
340 }
341
342 void* func = g_functions[FUNC_WRITE_LEAK_INFO];
343 bool written = false;
344 if (func != nullptr) {
345 written = reinterpret_cast<write_malloc_leak_info_func_t>(func)(fp);
346 }
347
348 if (!written) {
349 fprintf(fp, "Native heap dump not available. To enable, run these commands (requires root):\n");
350 fprintf(fp, "# adb shell stop\n");
351 fprintf(fp, "# adb shell setprop libc.debug.malloc.options backtrace\n");
352 fprintf(fp, "# adb shell start\n");
353 }
354}
355
Christopher Ferris63860cb2015-11-16 17:30:32 -0800356// =============================================================================
357
358template<typename FunctionType>
Florian Mayerf7f71e32018-08-31 15:36:48 -0700359static bool InitMallocFunction(void* malloc_impl_handler, _Atomic(FunctionType)* func, const char* prefix, const char* suffix) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800360 char symbol[128];
361 snprintf(symbol, sizeof(symbol), "%s_%s", prefix, suffix);
362 *func = reinterpret_cast<FunctionType>(dlsym(malloc_impl_handler, symbol));
363 if (*func == nullptr) {
364 error_log("%s: dlsym(\"%s\") failed", getprogname(), symbol);
365 return false;
366 }
367 return true;
368}
369
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800370static bool InitMallocFunctions(void* impl_handler, MallocDispatch* table, const char* prefix) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700371 if (!InitMallocFunction<MallocFree>(impl_handler, &table->free, prefix, "free")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800372 return false;
373 }
Florian Mayerf7f71e32018-08-31 15:36:48 -0700374 if (!InitMallocFunction<MallocCalloc>(impl_handler, &table->calloc, prefix, "calloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800375 return false;
376 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800377 if (!InitMallocFunction<MallocMallinfo>(impl_handler, &table->mallinfo, prefix, "mallinfo")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800378 return false;
379 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800380 if (!InitMallocFunction<MallocMallopt>(impl_handler, &table->mallopt, prefix, "mallopt")) {
Christopher Ferrisa1c0d2f2017-05-15 15:50:19 -0700381 return false;
382 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800383 if (!InitMallocFunction<MallocMalloc>(impl_handler, &table->malloc, prefix, "malloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800384 return false;
385 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800386 if (!InitMallocFunction<MallocMallocUsableSize>(impl_handler, &table->malloc_usable_size, prefix,
387 "malloc_usable_size")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800388 return false;
389 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800390 if (!InitMallocFunction<MallocMemalign>(impl_handler, &table->memalign, prefix, "memalign")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800391 return false;
392 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800393 if (!InitMallocFunction<MallocPosixMemalign>(impl_handler, &table->posix_memalign, prefix,
394 "posix_memalign")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800395 return false;
396 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800397 if (!InitMallocFunction<MallocAlignedAlloc>(impl_handler, &table->aligned_alloc,
Christopher Ferriscae21a92018-02-05 18:14:55 -0800398 prefix, "aligned_alloc")) {
399 return false;
400 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800401 if (!InitMallocFunction<MallocRealloc>(impl_handler, &table->realloc, prefix, "realloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800402 return false;
403 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800404 if (!InitMallocFunction<MallocIterate>(impl_handler, &table->iterate, prefix, "iterate")) {
Colin Cross869691c2016-01-29 12:48:18 -0800405 return false;
406 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800407 if (!InitMallocFunction<MallocMallocDisable>(impl_handler, &table->malloc_disable, prefix,
408 "malloc_disable")) {
Colin Cross869691c2016-01-29 12:48:18 -0800409 return false;
410 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800411 if (!InitMallocFunction<MallocMallocEnable>(impl_handler, &table->malloc_enable, prefix,
412 "malloc_enable")) {
Colin Cross869691c2016-01-29 12:48:18 -0800413 return false;
414 }
Christopher Ferris63860cb2015-11-16 17:30:32 -0800415#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800416 if (!InitMallocFunction<MallocPvalloc>(impl_handler, &table->pvalloc, prefix, "pvalloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800417 return false;
418 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800419 if (!InitMallocFunction<MallocValloc>(impl_handler, &table->valloc, prefix, "valloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800420 return false;
421 }
422#endif
423
424 return true;
425}
426
427static void malloc_fini_impl(void*) {
428 // Our BSD stdio implementation doesn't close the standard streams,
429 // it only flushes them. Other unclosed FILE*s will show up as
430 // malloc leaks, but to avoid the standard streams showing up in
431 // leak reports, close them here.
432 fclose(stdin);
433 fclose(stdout);
434 fclose(stderr);
435
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800436 reinterpret_cast<finalize_func_t>(g_functions[FUNC_FINALIZE])();
437}
438
439static bool CheckLoadMallocHooks(char** options) {
440 char* env = getenv(HOOKS_ENV_ENABLE);
441 if ((env == nullptr || env[0] == '\0' || env[0] == '0') &&
442 (__system_property_get(HOOKS_PROPERTY_ENABLE, *options) == 0 || *options[0] == '\0' || *options[0] == '0')) {
443 return false;
444 }
445 *options = nullptr;
446 return true;
447}
448
449static bool CheckLoadMallocDebug(char** options) {
450 // If DEBUG_MALLOC_ENV_OPTIONS is set then it overrides the system properties.
451 char* env = getenv(DEBUG_ENV_OPTIONS);
452 if (env == nullptr || env[0] == '\0') {
453 if (__system_property_get(DEBUG_PROPERTY_OPTIONS, *options) == 0 || *options[0] == '\0') {
454 return false;
455 }
456
457 // Check to see if only a specific program should have debug malloc enabled.
458 char program[PROP_VALUE_MAX];
459 if (__system_property_get(DEBUG_PROPERTY_PROGRAM, program) != 0 &&
460 strstr(getprogname(), program) == nullptr) {
461 return false;
462 }
463 } else {
464 *options = env;
465 }
466 return true;
467}
468
Florian Mayer4e28ea12018-11-22 17:34:34 +0000469static bool GetHeapprofdProgramProperty(char* data, size_t size) {
470 constexpr char prefix[] = "heapprofd.enable.";
471 // - 1 to skip nullbyte, which we will write later.
472 constexpr size_t prefix_size = sizeof(prefix) - 1;
473 if (size < prefix_size) {
474 error_log("%s: Overflow constructing heapprofd property", getprogname());
475 return false;
476 }
477 memcpy(data, prefix, prefix_size);
478
479 int fd = open("/proc/self/cmdline", O_RDONLY | O_CLOEXEC);
480 if (fd == -1) {
481 error_log("%s: Failed to open /proc/self/cmdline", getprogname());
482 return false;
483 }
484 char cmdline[128];
485 ssize_t rd = read(fd, cmdline, sizeof(cmdline) - 1);
486 close(fd);
487 if (rd == -1) {
488 error_log("%s: Failed to read /proc/self/cmdline", getprogname());
489 return false;
490 }
491 cmdline[rd] = '\0';
492 char* first_arg = static_cast<char*>(memchr(cmdline, '\0', rd));
493 if (first_arg == nullptr || first_arg == cmdline + size - 1) {
494 error_log("%s: Overflow reading cmdline", getprogname());
495 return false;
496 }
497 // For consistency with what we do with Java app cmdlines, trim everything
498 // after the @ sign of the first arg.
499 char* first_at = static_cast<char*>(memchr(cmdline, '@', rd));
500 if (first_at != nullptr && first_at < first_arg) {
501 *first_at = '\0';
502 first_arg = first_at;
503 }
504
505 char* start = static_cast<char*>(memrchr(cmdline, '/', first_arg - cmdline));
506 if (start == first_arg) {
507 // The first argument ended in a slash.
508 error_log("%s: cmdline ends in /", getprogname());
509 return false;
510 } else if (start == nullptr) {
511 start = cmdline;
512 } else {
513 // Skip the /.
514 start++;
515 }
516
517 size_t name_size = static_cast<size_t>(first_arg - start);
518 if (name_size >= size - prefix_size) {
519 error_log("%s: overflow constructing heapprofd property.", getprogname());
520 return false;
521 }
522 // + 1 to also copy the trailing null byte.
523 memcpy(data + prefix_size, start, name_size + 1);
524 return true;
525}
526
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000527static bool CheckLoadHeapprofd() {
528 // First check for heapprofd.enable. If it is set to "all", enable
529 // heapprofd for all processes. Otherwise, check heapprofd.enable.${prog},
530 // if it is set and not 0, enable heap profiling for this process.
531 char property_value[PROP_VALUE_MAX];
532 if (__system_property_get(HEAPPROFD_PROPERTY_ENABLE, property_value) == 0) {
533 return false;
534 }
535 if (strcmp(property_value, "all") == 0) {
536 return true;
537 }
538
539 char program_property[128];
Florian Mayer4e28ea12018-11-22 17:34:34 +0000540 if (!GetHeapprofdProgramProperty(program_property,
541 sizeof(program_property))) {
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000542 return false;
543 }
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000544 if (__system_property_get(program_property, property_value) == 0) {
545 return false;
546 }
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000547 return program_property[0] != '\0';
548}
549
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800550static void ClearGlobalFunctions() {
551 for (size_t i = 0; i < FUNC_LAST; i++) {
552 g_functions[i] = nullptr;
553 }
554}
555
556static void* LoadSharedLibrary(const char* shared_lib, const char* prefix, MallocDispatch* dispatch_table) {
557 void* impl_handle = dlopen(shared_lib, RTLD_NOW | RTLD_LOCAL);
558 if (impl_handle == nullptr) {
559 error_log("%s: Unable to open shared library %s: %s", getprogname(), shared_lib, dlerror());
560 return nullptr;
561 }
562
563 static constexpr const char* names[] = {
564 "initialize",
565 "finalize",
566 "get_malloc_leak_info",
567 "free_malloc_leak_info",
568 "malloc_backtrace",
Christopher Ferris2e1a40a2018-06-13 10:46:34 -0700569 "write_malloc_leak_info",
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800570 };
571 for (size_t i = 0; i < FUNC_LAST; i++) {
572 char symbol[128];
573 snprintf(symbol, sizeof(symbol), "%s_%s", prefix, names[i]);
574 g_functions[i] = dlsym(impl_handle, symbol);
575 if (g_functions[i] == nullptr) {
576 error_log("%s: %s routine not found in %s", getprogname(), symbol, shared_lib);
577 dlclose(impl_handle);
578 ClearGlobalFunctions();
579 return nullptr;
580 }
581 }
582
583 if (!InitMallocFunctions(impl_handle, dispatch_table, prefix)) {
584 dlclose(impl_handle);
585 ClearGlobalFunctions();
586 return nullptr;
587 }
588
589 return impl_handle;
Christopher Ferris63860cb2015-11-16 17:30:32 -0800590}
591
Florian Mayer176a4752018-10-23 11:48:34 +0100592// A function pointer to heapprofds init function. Used to re-initialize
593// heapprofd. This will start a new profiling session and tear down the old
594// one in case it is still active.
595static _Atomic init_func_t g_heapprofd_init_func = nullptr;
596
Florian Mayerf7f71e32018-08-31 15:36:48 -0700597static void install_hooks(libc_globals* globals, const char* options,
598 const char* prefix, const char* shared_lib) {
Florian Mayer176a4752018-10-23 11:48:34 +0100599 init_func_t init_func = atomic_load(&g_heapprofd_init_func);
600 if (init_func != nullptr) {
601 init_func(&__libc_malloc_default_dispatch, &gMallocLeakZygoteChild, options);
602 info_log("%s: malloc %s re-enabled", getprogname(), prefix);
603 return;
604 }
605
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800606 MallocDispatch dispatch_table;
607 void* impl_handle = LoadSharedLibrary(shared_lib, prefix, &dispatch_table);
608 if (impl_handle == nullptr) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800609 return;
610 }
Florian Mayer176a4752018-10-23 11:48:34 +0100611 init_func = reinterpret_cast<init_func_t>(g_functions[FUNC_INITIALIZE]);
Tamas Berghammerac81fe82016-08-26 15:54:59 +0100612 if (!init_func(&__libc_malloc_default_dispatch, &gMallocLeakZygoteChild, options)) {
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800613 dlclose(impl_handle);
614 ClearGlobalFunctions();
Christopher Ferris63860cb2015-11-16 17:30:32 -0800615 return;
616 }
617
Florian Mayer176a4752018-10-23 11:48:34 +0100618 atomic_store(&g_heapprofd_init_func, init_func);
Florian Mayere965bcd2018-11-23 15:35:42 +0000619 // We assign free first explicitly to prevent the case where we observe a
620 // alloc, but miss the corresponding free because of initialization order.
621 //
622 // This is safer than relying on the declaration order inside
623 // MallocDispatch at the cost of an extra atomic pointer write on
624 // initialization.
625 atomic_store(&globals->malloc_dispatch.free, dispatch_table.free);
626 // The struct gets assigned elementwise and each of the elements is an
627 // _Atomic. Assigning to an _Atomic is an atomic_store operation.
628 // The assignment is done in declaration order.
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800629 globals->malloc_dispatch = dispatch_table;
Christopher Ferris63860cb2015-11-16 17:30:32 -0800630
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800631 info_log("%s: malloc %s enabled", getprogname(), prefix);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800632
633 // Use atexit to trigger the cleanup function. This avoids a problem
634 // where another atexit function is used to cleanup allocated memory,
635 // but the finalize function was already called. This particular error
636 // seems to be triggered by a zygote spawned process calling exit.
637 int ret_value = __cxa_atexit(malloc_fini_impl, nullptr, nullptr);
638 if (ret_value != 0) {
639 error_log("failed to set atexit cleanup function: %d", ret_value);
640 }
641}
642
Florian Mayerf7f71e32018-08-31 15:36:48 -0700643extern "C" void InstallInitHeapprofdHook(int);
644
645// Initializes memory allocation framework once per process.
646static void malloc_init_impl(libc_globals* globals) {
647 struct sigaction action = {};
648 action.sa_handler = InstallInitHeapprofdHook;
649 sigaction(HEAPPROFD_SIGNAL, &action, nullptr);
650
651 const char* prefix;
652 const char* shared_lib;
653 char prop[PROP_VALUE_MAX];
654 char* options = prop;
655 // Prefer malloc debug since it existed first and is a more complete
656 // malloc interceptor than the hooks.
657 if (CheckLoadMallocDebug(&options)) {
658 prefix = "debug";
659 shared_lib = DEBUG_SHARED_LIB;
660 } else if (CheckLoadMallocHooks(&options)) {
661 prefix = "hooks";
662 shared_lib = HOOKS_SHARED_LIB;
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000663 } else if (CheckLoadHeapprofd()) {
664 prefix = "heapprofd";
665 shared_lib = HEAPPROFD_SHARED_LIB;
Florian Mayerf7f71e32018-08-31 15:36:48 -0700666 } else {
667 return;
668 }
669 install_hooks(globals, options, prefix, shared_lib);
670}
671
Christopher Ferris63860cb2015-11-16 17:30:32 -0800672// Initializes memory allocation framework.
673// This routine is called from __libc_init routines in libc_init_dynamic.cpp.
674__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals) {
675 malloc_init_impl(globals);
676}
Florian Mayerf7f71e32018-08-31 15:36:48 -0700677
678// The logic for triggering heapprofd below is as following.
679// 1. HEAPPROFD_SIGNAL is received by the process.
Florian Mayer176a4752018-10-23 11:48:34 +0100680// 2. If neither InitHeapprofd nor InitHeapprofdHook are currently installed
681// (g_heapprofd_init_hook_installed is false), InitHeapprofdHook is
682// installed and g_heapprofd_init_in_progress is set to true.
683//
684// On the next subsequent malloc, InitHeapprofdHook is called and
685// 3a. If the signal is currently being handled (g_heapprofd_init_in_progress
Florian Mayerf7f71e32018-08-31 15:36:48 -0700686// is true), no action is taken.
Florian Mayer176a4752018-10-23 11:48:34 +0100687// 3b. Otherwise, The signal handler (InstallInitHeapprofdHook) installs a
Florian Mayerf7f71e32018-08-31 15:36:48 -0700688// temporary malloc hook (InitHeapprofdHook).
Florian Mayer176a4752018-10-23 11:48:34 +0100689// 4. When this hook gets run the first time, it uninstalls itself and spawns
Florian Mayerf7f71e32018-08-31 15:36:48 -0700690// a thread running InitHeapprofd that loads heapprofd.so and installs the
691// hooks within.
Florian Mayer176a4752018-10-23 11:48:34 +0100692// 5. g_heapprofd_init_in_progress and g_heapprofd_init_hook_installed are
693// reset to false so heapprofd can be reinitialized. Reinitialization
694// means that a new profiling session is started and any still active is
695// torn down.
Florian Mayerf7f71e32018-08-31 15:36:48 -0700696//
697// This roundabout way is needed because we are running non AS-safe code, so
698// we cannot run it directly in the signal handler. The other approach of
699// running a standby thread and signalling through write(2) and read(2) would
700// significantly increase the number of active threads in the system.
701
702static _Atomic bool g_heapprofd_init_in_progress = false;
Florian Mayer176a4752018-10-23 11:48:34 +0100703static _Atomic bool g_heapprofd_init_hook_installed = false;
Florian Mayerf7f71e32018-08-31 15:36:48 -0700704
705static void* InitHeapprofd(void*) {
706 __libc_globals.mutate([](libc_globals* globals) {
707 install_hooks(globals, nullptr, HEAPPROFD_PREFIX, HEAPPROFD_SHARED_LIB);
708 });
709 atomic_store(&g_heapprofd_init_in_progress, false);
Florian Mayer176a4752018-10-23 11:48:34 +0100710 // Allow to install hook again to re-initialize heap profiling after the
711 // current session finished.
712 atomic_store(&g_heapprofd_init_hook_installed, false);
Florian Mayerf7f71e32018-08-31 15:36:48 -0700713 return nullptr;
714}
715
716static void* InitHeapprofdHook(size_t bytes) {
Florian Mayer176a4752018-10-23 11:48:34 +0100717 if (!atomic_exchange(&g_heapprofd_init_hook_installed, true)) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700718 __libc_globals.mutate([](libc_globals* globals) {
719 atomic_store(&globals->malloc_dispatch.malloc, nullptr);
720 });
721
722 pthread_t thread_id;
723 if (pthread_create(&thread_id, nullptr, InitHeapprofd, nullptr) == -1)
724 error_log("%s: heapprofd: failed to pthread_create.", getprogname());
725 else if (pthread_detach(thread_id) == -1)
726 error_log("%s: heapprofd: failed to pthread_detach", getprogname());
727 if (pthread_setname_np(thread_id, "heapprofdinit") == -1)
728 error_log("%s: heapprod: failed to pthread_setname_np", getprogname());
729 }
730 return Malloc(malloc)(bytes);
731}
732
733extern "C" void InstallInitHeapprofdHook(int) {
734 if (!atomic_exchange(&g_heapprofd_init_in_progress, true)) {
735 __libc_globals.mutate([](libc_globals* globals) {
Florian Mayere965bcd2018-11-23 15:35:42 +0000736 atomic_store(&globals->malloc_dispatch.malloc, InitHeapprofdHook);
Florian Mayerf7f71e32018-08-31 15:36:48 -0700737 });
738 }
739}
740
Christopher Ferris63860cb2015-11-16 17:30:32 -0800741#endif // !LIBC_STATIC
Colin Cross869691c2016-01-29 12:48:18 -0800742
743// =============================================================================
744// Exported for use by libmemunreachable.
745// =============================================================================
746
747// Calls callback for every allocation in the anonymous heap mapping
748// [base, base+size). Must be called between malloc_disable and malloc_enable.
749extern "C" int malloc_iterate(uintptr_t base, size_t size,
750 void (*callback)(uintptr_t base, size_t size, void* arg), void* arg) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700751 auto _iterate = atomic_load_explicit_const(
752 &__libc_globals->malloc_dispatch.iterate,
753 default_read_memory_order);
Colin Cross869691c2016-01-29 12:48:18 -0800754 if (__predict_false(_iterate != nullptr)) {
755 return _iterate(base, size, callback, arg);
756 }
757 return Malloc(iterate)(base, size, callback, arg);
758}
759
760// Disable calls to malloc so malloc_iterate gets a consistent view of
761// allocated memory.
762extern "C" void malloc_disable() {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700763 auto _malloc_disable = atomic_load_explicit_const(
764 & __libc_globals->malloc_dispatch.malloc_disable,
765 default_read_memory_order);
Colin Cross869691c2016-01-29 12:48:18 -0800766 if (__predict_false(_malloc_disable != nullptr)) {
767 return _malloc_disable();
768 }
769 return Malloc(malloc_disable)();
770}
771
772// Re-enable calls to malloc after a previous call to malloc_disable.
773extern "C" void malloc_enable() {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700774 auto _malloc_enable = atomic_load_explicit_const(
775 &__libc_globals->malloc_dispatch.malloc_enable,
776 default_read_memory_order);
Colin Cross869691c2016-01-29 12:48:18 -0800777 if (__predict_false(_malloc_enable != nullptr)) {
778 return _malloc_enable();
779 }
780 return Malloc(malloc_enable)();
781}
Colin Cross2d4721c2016-02-02 11:57:54 -0800782
783#ifndef LIBC_STATIC
784extern "C" ssize_t malloc_backtrace(void* pointer, uintptr_t* frames, size_t frame_count) {
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800785 void* func = g_functions[FUNC_MALLOC_BACKTRACE];
786 if (func == nullptr) {
Colin Cross2d4721c2016-02-02 11:57:54 -0800787 return 0;
788 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800789 return reinterpret_cast<malloc_backtrace_func_t>(func)(pointer, frames, frame_count);
Colin Cross2d4721c2016-02-02 11:57:54 -0800790}
791#else
792extern "C" ssize_t malloc_backtrace(void*, uintptr_t*, size_t) {
793 return 0;
794}
795#endif