blob: 4cc5df9d08c4045934e42ca12de753a5db585f58 [file] [log] [blame]
Christopher Ferris63860cb2015-11-16 17:30:32 -08001/*
2 * Copyright (C) 2009 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29// Contains a thin layer that calls whatever real native allocator
30// has been defined. For the libc shared library, this allows the
31// implementation of a debug malloc that can intercept all of the allocation
32// calls and add special debugging code to attempt to catch allocation
33// errors. All of the debugging code is implemented in a separate shared
34// library that is only loaded when the property "libc.debug.malloc.options"
35// is set to a non-zero value. There are two functions exported to
36// allow ddms, or other external users to get information from the debug
37// allocation.
38// get_malloc_leak_info: Returns information about all of the known native
39// allocations that are currently in use.
40// free_malloc_leak_info: Frees the data allocated by the call to
41// get_malloc_leak_info.
Christopher Ferris2e1a40a2018-06-13 10:46:34 -070042// write_malloc_leak_info: Writes the leak info data to a file.
Christopher Ferris63860cb2015-11-16 17:30:32 -080043
Colin Cross869691c2016-01-29 12:48:18 -080044#include <pthread.h>
Florian Mayerf7f71e32018-08-31 15:36:48 -070045#include <stdatomic.h>
Colin Cross869691c2016-01-29 12:48:18 -080046
dimitry5332af62018-12-04 14:03:13 +010047#include <private/bionic_defs.h>
Christopher Ferris63860cb2015-11-16 17:30:32 -080048#include <private/bionic_config.h>
49#include <private/bionic_globals.h>
50#include <private/bionic_malloc_dispatch.h>
51
Evgenii Stepanovbe551f52018-08-13 16:46:15 -070052#if __has_feature(hwaddress_sanitizer)
53// FIXME: implement these in HWASan allocator.
54extern "C" int __sanitizer_iterate(uintptr_t base __unused, size_t size __unused,
55 void (*callback)(uintptr_t base, size_t size, void* arg) __unused,
56 void* arg __unused) {
57 return 0;
58}
59
60extern "C" void __sanitizer_malloc_disable() {
61}
62
63extern "C" void __sanitizer_malloc_enable() {
64}
65#include <sanitizer/hwasan_interface.h>
66#define Malloc(function) __sanitizer_ ## function
67
68#else // __has_feature(hwaddress_sanitizer)
Christopher Ferris63860cb2015-11-16 17:30:32 -080069#include "jemalloc.h"
70#define Malloc(function) je_ ## function
Evgenii Stepanovbe551f52018-08-13 16:46:15 -070071#endif
Christopher Ferris63860cb2015-11-16 17:30:32 -080072
Florian Mayerf7f71e32018-08-31 15:36:48 -070073template <typename T>
74static T* RemoveConst(const T* x) {
75 return const_cast<T*>(x);
76}
77
78// RemoveConst is a workaround for bug in current libcxx. Fix in
79// https://reviews.llvm.org/D47613
80#define atomic_load_explicit_const(obj, order) atomic_load_explicit(RemoveConst(obj), order)
81
82static constexpr memory_order default_read_memory_order = memory_order_acquire;
83
Christopher Ferris63860cb2015-11-16 17:30:32 -080084static constexpr MallocDispatch __libc_malloc_default_dispatch
85 __attribute__((unused)) = {
86 Malloc(calloc),
87 Malloc(free),
88 Malloc(mallinfo),
89 Malloc(malloc),
90 Malloc(malloc_usable_size),
91 Malloc(memalign),
92 Malloc(posix_memalign),
93#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
94 Malloc(pvalloc),
95#endif
96 Malloc(realloc),
97#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
98 Malloc(valloc),
99#endif
Colin Cross869691c2016-01-29 12:48:18 -0800100 Malloc(iterate),
101 Malloc(malloc_disable),
102 Malloc(malloc_enable),
Christopher Ferrisa1c0d2f2017-05-15 15:50:19 -0700103 Malloc(mallopt),
Christopher Ferriscae21a92018-02-05 18:14:55 -0800104 Malloc(aligned_alloc),
Christopher Ferris63860cb2015-11-16 17:30:32 -0800105 };
106
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800107// Malloc hooks.
108void* (*volatile __malloc_hook)(size_t, const void*);
109void* (*volatile __realloc_hook)(void*, size_t, const void*);
110void (*volatile __free_hook)(void*, const void*);
111void* (*volatile __memalign_hook)(size_t, size_t, const void*);
112
Christopher Ferris63860cb2015-11-16 17:30:32 -0800113// In a VM process, this is set to 1 after fork()ing out of zygote.
114int gMallocLeakZygoteChild = 0;
115
116// =============================================================================
117// Allocation functions
118// =============================================================================
119extern "C" void* calloc(size_t n_elements, size_t elem_size) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700120 auto _calloc = atomic_load_explicit_const(
121 &__libc_globals->malloc_dispatch.calloc,
122 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800123 if (__predict_false(_calloc != nullptr)) {
124 return _calloc(n_elements, elem_size);
125 }
126 return Malloc(calloc)(n_elements, elem_size);
127}
128
129extern "C" void free(void* mem) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700130 auto _free = atomic_load_explicit_const(
131 &__libc_globals->malloc_dispatch.free,
132 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800133 if (__predict_false(_free != nullptr)) {
134 _free(mem);
135 } else {
136 Malloc(free)(mem);
137 }
138}
139
140extern "C" struct mallinfo mallinfo() {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700141 auto _mallinfo = atomic_load_explicit_const(
142 &__libc_globals->malloc_dispatch.mallinfo,
143 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800144 if (__predict_false(_mallinfo != nullptr)) {
145 return _mallinfo();
146 }
147 return Malloc(mallinfo)();
148}
149
Christopher Ferrisa1c0d2f2017-05-15 15:50:19 -0700150extern "C" int mallopt(int param, int value) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700151 auto _mallopt = atomic_load_explicit_const(
152 &__libc_globals->malloc_dispatch.mallopt,
153 default_read_memory_order);
Christopher Ferrisa1c0d2f2017-05-15 15:50:19 -0700154 if (__predict_false(_mallopt != nullptr)) {
155 return _mallopt(param, value);
156 }
157 return Malloc(mallopt)(param, value);
158}
159
Christopher Ferris63860cb2015-11-16 17:30:32 -0800160extern "C" void* malloc(size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700161 auto _malloc = atomic_load_explicit_const(
162 &__libc_globals->malloc_dispatch.malloc,
163 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800164 if (__predict_false(_malloc != nullptr)) {
165 return _malloc(bytes);
166 }
167 return Malloc(malloc)(bytes);
168}
169
170extern "C" size_t malloc_usable_size(const void* mem) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700171 auto _malloc_usable_size = atomic_load_explicit_const(
172 &__libc_globals->malloc_dispatch.malloc_usable_size,
173 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800174 if (__predict_false(_malloc_usable_size != nullptr)) {
175 return _malloc_usable_size(mem);
176 }
177 return Malloc(malloc_usable_size)(mem);
178}
179
180extern "C" void* memalign(size_t alignment, size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700181 auto _memalign = atomic_load_explicit_const(
182 &__libc_globals->malloc_dispatch.memalign,
183 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800184 if (__predict_false(_memalign != nullptr)) {
185 return _memalign(alignment, bytes);
186 }
187 return Malloc(memalign)(alignment, bytes);
188}
189
190extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700191 auto _posix_memalign = atomic_load_explicit_const(
192 &__libc_globals->malloc_dispatch.posix_memalign,
193 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800194 if (__predict_false(_posix_memalign != nullptr)) {
195 return _posix_memalign(memptr, alignment, size);
196 }
197 return Malloc(posix_memalign)(memptr, alignment, size);
198}
199
Christopher Ferriscae21a92018-02-05 18:14:55 -0800200extern "C" void* aligned_alloc(size_t alignment, size_t size) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700201 auto _aligned_alloc = atomic_load_explicit_const(
202 &__libc_globals->malloc_dispatch.aligned_alloc,
203 default_read_memory_order);
Christopher Ferriscae21a92018-02-05 18:14:55 -0800204 if (__predict_false(_aligned_alloc != nullptr)) {
205 return _aligned_alloc(alignment, size);
206 }
207 return Malloc(aligned_alloc)(alignment, size);
208}
209
Christopher Ferris63860cb2015-11-16 17:30:32 -0800210extern "C" void* realloc(void* old_mem, size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700211 auto _realloc = atomic_load_explicit_const(
212 &__libc_globals->malloc_dispatch.realloc,
213 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800214 if (__predict_false(_realloc != nullptr)) {
215 return _realloc(old_mem, bytes);
216 }
217 return Malloc(realloc)(old_mem, bytes);
218}
219
Elliott Hughesb1770852018-09-18 12:52:42 -0700220extern "C" void* reallocarray(void* old_mem, size_t item_count, size_t item_size) {
221 size_t new_size;
222 if (__builtin_mul_overflow(item_count, item_size, &new_size)) {
223 errno = ENOMEM;
224 return nullptr;
225 }
226 return realloc(old_mem, new_size);
227}
228
Christopher Ferris63860cb2015-11-16 17:30:32 -0800229#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
230extern "C" void* pvalloc(size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700231 auto _pvalloc = atomic_load_explicit_const(
232 &__libc_globals->malloc_dispatch.pvalloc,
233 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800234 if (__predict_false(_pvalloc != nullptr)) {
235 return _pvalloc(bytes);
236 }
237 return Malloc(pvalloc)(bytes);
238}
239
240extern "C" void* valloc(size_t bytes) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700241 auto _valloc = atomic_load_explicit_const(
242 &__libc_globals->malloc_dispatch.valloc,
243 default_read_memory_order);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800244 if (__predict_false(_valloc != nullptr)) {
245 return _valloc(bytes);
246 }
247 return Malloc(valloc)(bytes);
248}
249#endif
250
251// We implement malloc debugging only in libc.so, so the code below
252// must be excluded if we compile this file for static libc.a
253#if !defined(LIBC_STATIC)
254
255#include <dlfcn.h>
Florian Mayer4e28ea12018-11-22 17:34:34 +0000256#include <fcntl.h>
Christopher Ferris63860cb2015-11-16 17:30:32 -0800257#include <stdio.h>
258#include <stdlib.h>
Florian Mayer4e28ea12018-11-22 17:34:34 +0000259#include <unistd.h>
Christopher Ferris63860cb2015-11-16 17:30:32 -0800260
Christopher Ferris7a3681e2017-04-24 17:48:32 -0700261#include <async_safe/log.h>
Christopher Ferris63860cb2015-11-16 17:30:32 -0800262#include <sys/system_properties.h>
263
264extern "C" int __cxa_atexit(void (*func)(void *), void *arg, void *dso);
265
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800266static const char* HOOKS_SHARED_LIB = "libc_malloc_hooks.so";
267static const char* HOOKS_PROPERTY_ENABLE = "libc.debug.hooks.enable";
268static const char* HOOKS_ENV_ENABLE = "LIBC_HOOKS_ENABLE";
269
Christopher Ferris63860cb2015-11-16 17:30:32 -0800270static const char* DEBUG_SHARED_LIB = "libc_malloc_debug.so";
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800271static const char* DEBUG_PROPERTY_OPTIONS = "libc.debug.malloc.options";
272static const char* DEBUG_PROPERTY_PROGRAM = "libc.debug.malloc.program";
273static const char* DEBUG_ENV_OPTIONS = "LIBC_DEBUG_MALLOC_OPTIONS";
Christopher Ferris63860cb2015-11-16 17:30:32 -0800274
Florian Mayerf7f71e32018-08-31 15:36:48 -0700275static const char* HEAPPROFD_SHARED_LIB = "heapprofd_client.so";
276static const char* HEAPPROFD_PREFIX = "heapprofd";
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000277static const char* HEAPPROFD_PROPERTY_ENABLE = "heapprofd.enable";
Florian Mayerf7f71e32018-08-31 15:36:48 -0700278static const int HEAPPROFD_SIGNAL = __SIGRTMIN + 4;
279
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800280enum FunctionEnum : uint8_t {
281 FUNC_INITIALIZE,
282 FUNC_FINALIZE,
283 FUNC_GET_MALLOC_LEAK_INFO,
284 FUNC_FREE_MALLOC_LEAK_INFO,
285 FUNC_MALLOC_BACKTRACE,
Christopher Ferris2e1a40a2018-06-13 10:46:34 -0700286 FUNC_WRITE_LEAK_INFO,
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800287 FUNC_LAST,
288};
289static void* g_functions[FUNC_LAST];
Christopher Ferris63860cb2015-11-16 17:30:32 -0800290
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800291typedef void (*finalize_func_t)();
292typedef bool (*init_func_t)(const MallocDispatch*, int*, const char*);
293typedef void (*get_malloc_leak_info_func_t)(uint8_t**, size_t*, size_t*, size_t*, size_t*);
294typedef void (*free_malloc_leak_info_func_t)(uint8_t*);
Christopher Ferris2e1a40a2018-06-13 10:46:34 -0700295typedef bool (*write_malloc_leak_info_func_t)(FILE*);
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800296typedef ssize_t (*malloc_backtrace_func_t)(void*, uintptr_t*, size_t);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800297
298// =============================================================================
299// Log functions
300// =============================================================================
301#define error_log(format, ...) \
Christopher Ferris7a3681e2017-04-24 17:48:32 -0700302 async_safe_format_log(ANDROID_LOG_ERROR, "libc", (format), ##__VA_ARGS__ )
Christopher Ferris63860cb2015-11-16 17:30:32 -0800303#define info_log(format, ...) \
Christopher Ferris7a3681e2017-04-24 17:48:32 -0700304 async_safe_format_log(ANDROID_LOG_INFO, "libc", (format), ##__VA_ARGS__ )
Christopher Ferris63860cb2015-11-16 17:30:32 -0800305// =============================================================================
306
307// =============================================================================
308// Exported for use by ddms.
309// =============================================================================
310
311// Retrieve native heap information.
312//
313// "*info" is set to a buffer we allocate
314// "*overall_size" is set to the size of the "info" buffer
315// "*info_size" is set to the size of a single entry
316// "*total_memory" is set to the sum of all allocations we're tracking; does
317// not include heap overhead
318// "*backtrace_size" is set to the maximum number of entries in the back trace
319extern "C" void get_malloc_leak_info(uint8_t** info, size_t* overall_size,
320 size_t* info_size, size_t* total_memory, size_t* backtrace_size) {
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800321 void* func = g_functions[FUNC_GET_MALLOC_LEAK_INFO];
322 if (func == nullptr) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800323 return;
324 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800325 reinterpret_cast<get_malloc_leak_info_func_t>(func)(info, overall_size, info_size, total_memory,
326 backtrace_size);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800327}
328
329extern "C" void free_malloc_leak_info(uint8_t* info) {
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800330 void* func = g_functions[FUNC_FREE_MALLOC_LEAK_INFO];
331 if (func == nullptr) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800332 return;
333 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800334 reinterpret_cast<free_malloc_leak_info_func_t>(func)(info);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800335}
Colin Cross869691c2016-01-29 12:48:18 -0800336
Christopher Ferris2e1a40a2018-06-13 10:46:34 -0700337extern "C" void write_malloc_leak_info(FILE* fp) {
338 if (fp == nullptr) {
339 error_log("write_malloc_leak_info called with a nullptr");
340 return;
341 }
342
343 void* func = g_functions[FUNC_WRITE_LEAK_INFO];
344 bool written = false;
345 if (func != nullptr) {
346 written = reinterpret_cast<write_malloc_leak_info_func_t>(func)(fp);
347 }
348
349 if (!written) {
350 fprintf(fp, "Native heap dump not available. To enable, run these commands (requires root):\n");
351 fprintf(fp, "# adb shell stop\n");
352 fprintf(fp, "# adb shell setprop libc.debug.malloc.options backtrace\n");
353 fprintf(fp, "# adb shell start\n");
354 }
355}
356
Christopher Ferris63860cb2015-11-16 17:30:32 -0800357// =============================================================================
358
359template<typename FunctionType>
Florian Mayerf7f71e32018-08-31 15:36:48 -0700360static bool InitMallocFunction(void* malloc_impl_handler, _Atomic(FunctionType)* func, const char* prefix, const char* suffix) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800361 char symbol[128];
362 snprintf(symbol, sizeof(symbol), "%s_%s", prefix, suffix);
363 *func = reinterpret_cast<FunctionType>(dlsym(malloc_impl_handler, symbol));
364 if (*func == nullptr) {
365 error_log("%s: dlsym(\"%s\") failed", getprogname(), symbol);
366 return false;
367 }
368 return true;
369}
370
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800371static bool InitMallocFunctions(void* impl_handler, MallocDispatch* table, const char* prefix) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700372 if (!InitMallocFunction<MallocFree>(impl_handler, &table->free, prefix, "free")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800373 return false;
374 }
Florian Mayerf7f71e32018-08-31 15:36:48 -0700375 if (!InitMallocFunction<MallocCalloc>(impl_handler, &table->calloc, prefix, "calloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800376 return false;
377 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800378 if (!InitMallocFunction<MallocMallinfo>(impl_handler, &table->mallinfo, prefix, "mallinfo")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800379 return false;
380 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800381 if (!InitMallocFunction<MallocMallopt>(impl_handler, &table->mallopt, prefix, "mallopt")) {
Christopher Ferrisa1c0d2f2017-05-15 15:50:19 -0700382 return false;
383 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800384 if (!InitMallocFunction<MallocMalloc>(impl_handler, &table->malloc, prefix, "malloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800385 return false;
386 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800387 if (!InitMallocFunction<MallocMallocUsableSize>(impl_handler, &table->malloc_usable_size, prefix,
388 "malloc_usable_size")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800389 return false;
390 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800391 if (!InitMallocFunction<MallocMemalign>(impl_handler, &table->memalign, prefix, "memalign")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800392 return false;
393 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800394 if (!InitMallocFunction<MallocPosixMemalign>(impl_handler, &table->posix_memalign, prefix,
395 "posix_memalign")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800396 return false;
397 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800398 if (!InitMallocFunction<MallocAlignedAlloc>(impl_handler, &table->aligned_alloc,
Christopher Ferriscae21a92018-02-05 18:14:55 -0800399 prefix, "aligned_alloc")) {
400 return false;
401 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800402 if (!InitMallocFunction<MallocRealloc>(impl_handler, &table->realloc, prefix, "realloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800403 return false;
404 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800405 if (!InitMallocFunction<MallocIterate>(impl_handler, &table->iterate, prefix, "iterate")) {
Colin Cross869691c2016-01-29 12:48:18 -0800406 return false;
407 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800408 if (!InitMallocFunction<MallocMallocDisable>(impl_handler, &table->malloc_disable, prefix,
409 "malloc_disable")) {
Colin Cross869691c2016-01-29 12:48:18 -0800410 return false;
411 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800412 if (!InitMallocFunction<MallocMallocEnable>(impl_handler, &table->malloc_enable, prefix,
413 "malloc_enable")) {
Colin Cross869691c2016-01-29 12:48:18 -0800414 return false;
415 }
Christopher Ferris63860cb2015-11-16 17:30:32 -0800416#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800417 if (!InitMallocFunction<MallocPvalloc>(impl_handler, &table->pvalloc, prefix, "pvalloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800418 return false;
419 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800420 if (!InitMallocFunction<MallocValloc>(impl_handler, &table->valloc, prefix, "valloc")) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800421 return false;
422 }
423#endif
424
425 return true;
426}
427
428static void malloc_fini_impl(void*) {
429 // Our BSD stdio implementation doesn't close the standard streams,
430 // it only flushes them. Other unclosed FILE*s will show up as
431 // malloc leaks, but to avoid the standard streams showing up in
432 // leak reports, close them here.
433 fclose(stdin);
434 fclose(stdout);
435 fclose(stderr);
436
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800437 reinterpret_cast<finalize_func_t>(g_functions[FUNC_FINALIZE])();
438}
439
440static bool CheckLoadMallocHooks(char** options) {
441 char* env = getenv(HOOKS_ENV_ENABLE);
442 if ((env == nullptr || env[0] == '\0' || env[0] == '0') &&
443 (__system_property_get(HOOKS_PROPERTY_ENABLE, *options) == 0 || *options[0] == '\0' || *options[0] == '0')) {
444 return false;
445 }
446 *options = nullptr;
447 return true;
448}
449
450static bool CheckLoadMallocDebug(char** options) {
451 // If DEBUG_MALLOC_ENV_OPTIONS is set then it overrides the system properties.
452 char* env = getenv(DEBUG_ENV_OPTIONS);
453 if (env == nullptr || env[0] == '\0') {
454 if (__system_property_get(DEBUG_PROPERTY_OPTIONS, *options) == 0 || *options[0] == '\0') {
455 return false;
456 }
457
458 // Check to see if only a specific program should have debug malloc enabled.
459 char program[PROP_VALUE_MAX];
460 if (__system_property_get(DEBUG_PROPERTY_PROGRAM, program) != 0 &&
461 strstr(getprogname(), program) == nullptr) {
462 return false;
463 }
464 } else {
465 *options = env;
466 }
467 return true;
468}
469
Florian Mayer4e28ea12018-11-22 17:34:34 +0000470static bool GetHeapprofdProgramProperty(char* data, size_t size) {
471 constexpr char prefix[] = "heapprofd.enable.";
472 // - 1 to skip nullbyte, which we will write later.
473 constexpr size_t prefix_size = sizeof(prefix) - 1;
474 if (size < prefix_size) {
475 error_log("%s: Overflow constructing heapprofd property", getprogname());
476 return false;
477 }
478 memcpy(data, prefix, prefix_size);
479
480 int fd = open("/proc/self/cmdline", O_RDONLY | O_CLOEXEC);
481 if (fd == -1) {
482 error_log("%s: Failed to open /proc/self/cmdline", getprogname());
483 return false;
484 }
485 char cmdline[128];
486 ssize_t rd = read(fd, cmdline, sizeof(cmdline) - 1);
487 close(fd);
488 if (rd == -1) {
489 error_log("%s: Failed to read /proc/self/cmdline", getprogname());
490 return false;
491 }
492 cmdline[rd] = '\0';
493 char* first_arg = static_cast<char*>(memchr(cmdline, '\0', rd));
494 if (first_arg == nullptr || first_arg == cmdline + size - 1) {
495 error_log("%s: Overflow reading cmdline", getprogname());
496 return false;
497 }
498 // For consistency with what we do with Java app cmdlines, trim everything
499 // after the @ sign of the first arg.
500 char* first_at = static_cast<char*>(memchr(cmdline, '@', rd));
501 if (first_at != nullptr && first_at < first_arg) {
502 *first_at = '\0';
503 first_arg = first_at;
504 }
505
506 char* start = static_cast<char*>(memrchr(cmdline, '/', first_arg - cmdline));
507 if (start == first_arg) {
508 // The first argument ended in a slash.
509 error_log("%s: cmdline ends in /", getprogname());
510 return false;
511 } else if (start == nullptr) {
512 start = cmdline;
513 } else {
514 // Skip the /.
515 start++;
516 }
517
518 size_t name_size = static_cast<size_t>(first_arg - start);
519 if (name_size >= size - prefix_size) {
520 error_log("%s: overflow constructing heapprofd property.", getprogname());
521 return false;
522 }
523 // + 1 to also copy the trailing null byte.
524 memcpy(data + prefix_size, start, name_size + 1);
525 return true;
526}
527
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000528static bool CheckLoadHeapprofd() {
529 // First check for heapprofd.enable. If it is set to "all", enable
530 // heapprofd for all processes. Otherwise, check heapprofd.enable.${prog},
531 // if it is set and not 0, enable heap profiling for this process.
532 char property_value[PROP_VALUE_MAX];
533 if (__system_property_get(HEAPPROFD_PROPERTY_ENABLE, property_value) == 0) {
534 return false;
535 }
536 if (strcmp(property_value, "all") == 0) {
537 return true;
538 }
539
540 char program_property[128];
Florian Mayer4e28ea12018-11-22 17:34:34 +0000541 if (!GetHeapprofdProgramProperty(program_property,
542 sizeof(program_property))) {
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000543 return false;
544 }
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000545 if (__system_property_get(program_property, property_value) == 0) {
546 return false;
547 }
Florian Mayer0dbe6d12018-11-08 11:25:49 +0000548 return program_property[0] != '\0';
549}
550
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800551static void ClearGlobalFunctions() {
552 for (size_t i = 0; i < FUNC_LAST; i++) {
553 g_functions[i] = nullptr;
554 }
555}
556
557static void* LoadSharedLibrary(const char* shared_lib, const char* prefix, MallocDispatch* dispatch_table) {
558 void* impl_handle = dlopen(shared_lib, RTLD_NOW | RTLD_LOCAL);
559 if (impl_handle == nullptr) {
560 error_log("%s: Unable to open shared library %s: %s", getprogname(), shared_lib, dlerror());
561 return nullptr;
562 }
563
564 static constexpr const char* names[] = {
565 "initialize",
566 "finalize",
567 "get_malloc_leak_info",
568 "free_malloc_leak_info",
569 "malloc_backtrace",
Christopher Ferris2e1a40a2018-06-13 10:46:34 -0700570 "write_malloc_leak_info",
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800571 };
572 for (size_t i = 0; i < FUNC_LAST; i++) {
573 char symbol[128];
574 snprintf(symbol, sizeof(symbol), "%s_%s", prefix, names[i]);
575 g_functions[i] = dlsym(impl_handle, symbol);
576 if (g_functions[i] == nullptr) {
577 error_log("%s: %s routine not found in %s", getprogname(), symbol, shared_lib);
578 dlclose(impl_handle);
579 ClearGlobalFunctions();
580 return nullptr;
581 }
582 }
583
584 if (!InitMallocFunctions(impl_handle, dispatch_table, prefix)) {
585 dlclose(impl_handle);
586 ClearGlobalFunctions();
587 return nullptr;
588 }
589
590 return impl_handle;
Christopher Ferris63860cb2015-11-16 17:30:32 -0800591}
592
Florian Mayer176a4752018-10-23 11:48:34 +0100593// A function pointer to heapprofds init function. Used to re-initialize
594// heapprofd. This will start a new profiling session and tear down the old
595// one in case it is still active.
596static _Atomic init_func_t g_heapprofd_init_func = nullptr;
597
Florian Mayerf7f71e32018-08-31 15:36:48 -0700598static void install_hooks(libc_globals* globals, const char* options,
599 const char* prefix, const char* shared_lib) {
Florian Mayer176a4752018-10-23 11:48:34 +0100600 init_func_t init_func = atomic_load(&g_heapprofd_init_func);
601 if (init_func != nullptr) {
602 init_func(&__libc_malloc_default_dispatch, &gMallocLeakZygoteChild, options);
603 info_log("%s: malloc %s re-enabled", getprogname(), prefix);
604 return;
605 }
606
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800607 MallocDispatch dispatch_table;
608 void* impl_handle = LoadSharedLibrary(shared_lib, prefix, &dispatch_table);
609 if (impl_handle == nullptr) {
Christopher Ferris63860cb2015-11-16 17:30:32 -0800610 return;
611 }
Florian Mayer176a4752018-10-23 11:48:34 +0100612 init_func = reinterpret_cast<init_func_t>(g_functions[FUNC_INITIALIZE]);
Tamas Berghammerac81fe82016-08-26 15:54:59 +0100613 if (!init_func(&__libc_malloc_default_dispatch, &gMallocLeakZygoteChild, options)) {
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800614 dlclose(impl_handle);
615 ClearGlobalFunctions();
Christopher Ferris63860cb2015-11-16 17:30:32 -0800616 return;
617 }
618
Florian Mayer176a4752018-10-23 11:48:34 +0100619 atomic_store(&g_heapprofd_init_func, init_func);
Florian Mayere965bcd2018-11-23 15:35:42 +0000620 // We assign free first explicitly to prevent the case where we observe a
621 // alloc, but miss the corresponding free because of initialization order.
622 //
623 // This is safer than relying on the declaration order inside
624 // MallocDispatch at the cost of an extra atomic pointer write on
625 // initialization.
626 atomic_store(&globals->malloc_dispatch.free, dispatch_table.free);
627 // The struct gets assigned elementwise and each of the elements is an
628 // _Atomic. Assigning to an _Atomic is an atomic_store operation.
629 // The assignment is done in declaration order.
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800630 globals->malloc_dispatch = dispatch_table;
Christopher Ferris63860cb2015-11-16 17:30:32 -0800631
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800632 info_log("%s: malloc %s enabled", getprogname(), prefix);
Christopher Ferris63860cb2015-11-16 17:30:32 -0800633
634 // Use atexit to trigger the cleanup function. This avoids a problem
635 // where another atexit function is used to cleanup allocated memory,
636 // but the finalize function was already called. This particular error
637 // seems to be triggered by a zygote spawned process calling exit.
638 int ret_value = __cxa_atexit(malloc_fini_impl, nullptr, nullptr);
639 if (ret_value != 0) {
640 error_log("failed to set atexit cleanup function: %d", ret_value);
641 }
642}
643
Florian Mayerf7f71e32018-08-31 15:36:48 -0700644// The logic for triggering heapprofd below is as following.
645// 1. HEAPPROFD_SIGNAL is received by the process.
Florian Mayer176a4752018-10-23 11:48:34 +0100646// 2. If neither InitHeapprofd nor InitHeapprofdHook are currently installed
647// (g_heapprofd_init_hook_installed is false), InitHeapprofdHook is
648// installed and g_heapprofd_init_in_progress is set to true.
649//
650// On the next subsequent malloc, InitHeapprofdHook is called and
651// 3a. If the signal is currently being handled (g_heapprofd_init_in_progress
Florian Mayerf7f71e32018-08-31 15:36:48 -0700652// is true), no action is taken.
Florian Mayer176a4752018-10-23 11:48:34 +0100653// 3b. Otherwise, The signal handler (InstallInitHeapprofdHook) installs a
Florian Mayerf7f71e32018-08-31 15:36:48 -0700654// temporary malloc hook (InitHeapprofdHook).
Florian Mayer176a4752018-10-23 11:48:34 +0100655// 4. When this hook gets run the first time, it uninstalls itself and spawns
Florian Mayerf7f71e32018-08-31 15:36:48 -0700656// a thread running InitHeapprofd that loads heapprofd.so and installs the
657// hooks within.
Florian Mayer176a4752018-10-23 11:48:34 +0100658// 5. g_heapprofd_init_in_progress and g_heapprofd_init_hook_installed are
659// reset to false so heapprofd can be reinitialized. Reinitialization
660// means that a new profiling session is started and any still active is
661// torn down.
Florian Mayerf7f71e32018-08-31 15:36:48 -0700662//
663// This roundabout way is needed because we are running non AS-safe code, so
664// we cannot run it directly in the signal handler. The other approach of
665// running a standby thread and signalling through write(2) and read(2) would
666// significantly increase the number of active threads in the system.
667
668static _Atomic bool g_heapprofd_init_in_progress = false;
Florian Mayer176a4752018-10-23 11:48:34 +0100669static _Atomic bool g_heapprofd_init_hook_installed = false;
Florian Mayerf7f71e32018-08-31 15:36:48 -0700670
Florian Mayer3a538a42018-12-20 11:23:50 +0000671extern "C" void InstallInitHeapprofdHook(int);
672
673// Initializes memory allocation framework once per process.
674static void malloc_init_impl(libc_globals* globals) {
675 struct sigaction action = {};
676 action.sa_handler = InstallInitHeapprofdHook;
677 sigaction(HEAPPROFD_SIGNAL, &action, nullptr);
678
679 const char* prefix;
680 const char* shared_lib;
681 char prop[PROP_VALUE_MAX];
682 char* options = prop;
683 // Prefer malloc debug since it existed first and is a more complete
684 // malloc interceptor than the hooks.
685 if (CheckLoadMallocDebug(&options)) {
686 prefix = "debug";
687 shared_lib = DEBUG_SHARED_LIB;
688 } else if (CheckLoadMallocHooks(&options)) {
689 prefix = "hooks";
690 shared_lib = HOOKS_SHARED_LIB;
691 } else if (CheckLoadHeapprofd()) {
692 prefix = "heapprofd";
693 shared_lib = HEAPPROFD_SHARED_LIB;
694 } else {
695 return;
696 }
697 if (!atomic_exchange(&g_heapprofd_init_in_progress, true)) {
698 install_hooks(globals, options, prefix, shared_lib);
699 atomic_store(&g_heapprofd_init_in_progress, false);
700 }
701}
702
703// Initializes memory allocation framework.
704// This routine is called from __libc_init routines in libc_init_dynamic.cpp.
705__BIONIC_WEAK_FOR_NATIVE_BRIDGE
706__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals) {
707 malloc_init_impl(globals);
708}
709
Florian Mayerf7f71e32018-08-31 15:36:48 -0700710static void* InitHeapprofd(void*) {
711 __libc_globals.mutate([](libc_globals* globals) {
712 install_hooks(globals, nullptr, HEAPPROFD_PREFIX, HEAPPROFD_SHARED_LIB);
713 });
714 atomic_store(&g_heapprofd_init_in_progress, false);
Florian Mayer176a4752018-10-23 11:48:34 +0100715 // Allow to install hook again to re-initialize heap profiling after the
716 // current session finished.
717 atomic_store(&g_heapprofd_init_hook_installed, false);
Florian Mayerf7f71e32018-08-31 15:36:48 -0700718 return nullptr;
719}
720
721static void* InitHeapprofdHook(size_t bytes) {
Florian Mayer176a4752018-10-23 11:48:34 +0100722 if (!atomic_exchange(&g_heapprofd_init_hook_installed, true)) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700723 __libc_globals.mutate([](libc_globals* globals) {
724 atomic_store(&globals->malloc_dispatch.malloc, nullptr);
725 });
726
727 pthread_t thread_id;
728 if (pthread_create(&thread_id, nullptr, InitHeapprofd, nullptr) == -1)
729 error_log("%s: heapprofd: failed to pthread_create.", getprogname());
730 else if (pthread_detach(thread_id) == -1)
731 error_log("%s: heapprofd: failed to pthread_detach", getprogname());
732 if (pthread_setname_np(thread_id, "heapprofdinit") == -1)
733 error_log("%s: heapprod: failed to pthread_setname_np", getprogname());
734 }
735 return Malloc(malloc)(bytes);
736}
737
738extern "C" void InstallInitHeapprofdHook(int) {
739 if (!atomic_exchange(&g_heapprofd_init_in_progress, true)) {
740 __libc_globals.mutate([](libc_globals* globals) {
Florian Mayere965bcd2018-11-23 15:35:42 +0000741 atomic_store(&globals->malloc_dispatch.malloc, InitHeapprofdHook);
Florian Mayerf7f71e32018-08-31 15:36:48 -0700742 });
743 }
744}
745
Christopher Ferris63860cb2015-11-16 17:30:32 -0800746#endif // !LIBC_STATIC
Colin Cross869691c2016-01-29 12:48:18 -0800747
748// =============================================================================
749// Exported for use by libmemunreachable.
750// =============================================================================
751
752// Calls callback for every allocation in the anonymous heap mapping
753// [base, base+size). Must be called between malloc_disable and malloc_enable.
754extern "C" int malloc_iterate(uintptr_t base, size_t size,
755 void (*callback)(uintptr_t base, size_t size, void* arg), void* arg) {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700756 auto _iterate = atomic_load_explicit_const(
757 &__libc_globals->malloc_dispatch.iterate,
758 default_read_memory_order);
Colin Cross869691c2016-01-29 12:48:18 -0800759 if (__predict_false(_iterate != nullptr)) {
760 return _iterate(base, size, callback, arg);
761 }
762 return Malloc(iterate)(base, size, callback, arg);
763}
764
765// Disable calls to malloc so malloc_iterate gets a consistent view of
766// allocated memory.
767extern "C" void malloc_disable() {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700768 auto _malloc_disable = atomic_load_explicit_const(
769 & __libc_globals->malloc_dispatch.malloc_disable,
770 default_read_memory_order);
Colin Cross869691c2016-01-29 12:48:18 -0800771 if (__predict_false(_malloc_disable != nullptr)) {
772 return _malloc_disable();
773 }
774 return Malloc(malloc_disable)();
775}
776
777// Re-enable calls to malloc after a previous call to malloc_disable.
778extern "C" void malloc_enable() {
Florian Mayerf7f71e32018-08-31 15:36:48 -0700779 auto _malloc_enable = atomic_load_explicit_const(
780 &__libc_globals->malloc_dispatch.malloc_enable,
781 default_read_memory_order);
Colin Cross869691c2016-01-29 12:48:18 -0800782 if (__predict_false(_malloc_enable != nullptr)) {
783 return _malloc_enable();
784 }
785 return Malloc(malloc_enable)();
786}
Colin Cross2d4721c2016-02-02 11:57:54 -0800787
788#ifndef LIBC_STATIC
789extern "C" ssize_t malloc_backtrace(void* pointer, uintptr_t* frames, size_t frame_count) {
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800790 void* func = g_functions[FUNC_MALLOC_BACKTRACE];
791 if (func == nullptr) {
Colin Cross2d4721c2016-02-02 11:57:54 -0800792 return 0;
793 }
Christopher Ferrisdb478a62018-02-07 18:42:14 -0800794 return reinterpret_cast<malloc_backtrace_func_t>(func)(pointer, frames, frame_count);
Colin Cross2d4721c2016-02-02 11:57:54 -0800795}
796#else
797extern "C" ssize_t malloc_backtrace(void*, uintptr_t*, size_t) {
798 return 0;
799}
800#endif