blob: 83de03f30a603bb1f066f2e0751677d70b343aa2 [file] [log] [blame]
Jeff Brown053b8652012-06-06 16:25:03 -07001/*
Mark Salyzyn706fad22013-12-12 12:21:20 -08002 * Copyright (C) 2012-2014 The Android Open Source Project
Jeff Brown053b8652012-06-06 16:25:03 -07003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Jeff Brown053b8652012-06-06 16:25:03 -070017#include <dirent.h>
Kévin PETIT4bb47722013-12-18 16:44:24 +000018#include <errno.h>
19#include <fcntl.h>
20#include <inttypes.h>
21#include <signal.h>
22#include <stddef.h>
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
Jeff Brown053b8652012-06-06 16:25:03 -070026#include <time.h>
27#include <sys/ptrace.h>
Kévin PETIT4bb47722013-12-18 16:44:24 +000028#include <sys/socket.h>
Jeff Brown053b8652012-06-06 16:25:03 -070029#include <sys/stat.h>
Kévin PETIT4bb47722013-12-18 16:44:24 +000030#include <sys/un.h>
Jeff Brown053b8652012-06-06 16:25:03 -070031
32#include <private/android_filesystem_config.h>
33
Mark Salyzyna63f9272013-11-22 10:53:34 -080034#include <log/log.h>
Colin Cross9227bd32013-07-23 16:59:20 -070035#include <log/logger.h>
Jeff Brown053b8652012-06-06 16:25:03 -070036#include <cutils/properties.h>
37
Christopher Ferris20303f82014-01-10 16:33:16 -080038#include <backtrace/Backtrace.h>
Christopher Ferris46756822014-01-14 20:16:30 -080039#include <backtrace/BacktraceMap.h>
Jeff Brown053b8652012-06-06 16:25:03 -070040
rpcraigf1186f32012-07-19 09:38:06 -040041#include <selinux/android.h>
rpcraigf1186f32012-07-19 09:38:06 -040042
Christopher Ferris20303f82014-01-10 16:33:16 -080043#include <UniquePtr.h>
44
Jeff Brown053b8652012-06-06 16:25:03 -070045#include "machine.h"
46#include "tombstone.h"
Christopher Ferris365e4ae2013-10-02 12:26:48 -070047#include "backtrace.h"
Jeff Brown053b8652012-06-06 16:25:03 -070048
Jeff Brown053b8652012-06-06 16:25:03 -070049#define STACK_WORDS 16
50
51#define MAX_TOMBSTONES 10
52#define TOMBSTONE_DIR "/data/tombstones"
Kévin PETIT4bb47722013-12-18 16:44:24 +000053#define TOMBSTONE_TEMPLATE (TOMBSTONE_DIR"/tombstone_%02d")
Jeff Brown053b8652012-06-06 16:25:03 -070054
Christopher Ferris20303f82014-01-10 16:33:16 -080055// Must match the path defined in NativeCrashListener.java
Christopher Tateded2e5a2013-03-19 13:12:23 -070056#define NCRASH_SOCKET_PATH "/data/system/ndebugsocket"
57
Jeff Brown053b8652012-06-06 16:25:03 -070058static bool signal_has_address(int sig) {
Christopher Ferris20303f82014-01-10 16:33:16 -080059 switch (sig) {
Jeff Brown053b8652012-06-06 16:25:03 -070060 case SIGILL:
Christopher Ferris20303f82014-01-10 16:33:16 -080061 case SIGFPE:
62 case SIGSEGV:
63 case SIGBUS:
64 return true;
65 default:
66 return false;
67 }
68}
69
70static const char* get_signame(int sig) {
71 switch(sig) {
72 case SIGILL: return "SIGILL";
73 case SIGABRT: return "SIGABRT";
74 case SIGBUS: return "SIGBUS";
75 case SIGFPE: return "SIGFPE";
76 case SIGSEGV: return "SIGSEGV";
77 case SIGPIPE: return "SIGPIPE";
78#ifdef SIGSTKFLT
79 case SIGSTKFLT: return "SIGSTKFLT";
80#endif
81 case SIGSTOP: return "SIGSTOP";
82 default: return "?";
83 }
84}
85
86static const char* get_sigcode(int signo, int code) {
87 // Try the signal-specific codes...
88 switch (signo) {
89 case SIGILL:
90 switch (code) {
Jeff Brown053b8652012-06-06 16:25:03 -070091 case ILL_ILLOPC: return "ILL_ILLOPC";
92 case ILL_ILLOPN: return "ILL_ILLOPN";
93 case ILL_ILLADR: return "ILL_ILLADR";
94 case ILL_ILLTRP: return "ILL_ILLTRP";
95 case ILL_PRVOPC: return "ILL_PRVOPC";
96 case ILL_PRVREG: return "ILL_PRVREG";
97 case ILL_COPROC: return "ILL_COPROC";
98 case ILL_BADSTK: return "ILL_BADSTK";
Christopher Ferris20303f82014-01-10 16:33:16 -080099 }
Elliott Hughesbd395b92014-04-24 13:53:22 -0700100 static_assert(NSIGILL == ILL_BADSTK, "missing ILL_* si_code");
Christopher Ferris20303f82014-01-10 16:33:16 -0800101 break;
Jeff Brown053b8652012-06-06 16:25:03 -0700102 case SIGBUS:
Christopher Ferris20303f82014-01-10 16:33:16 -0800103 switch (code) {
Jeff Brown053b8652012-06-06 16:25:03 -0700104 case BUS_ADRALN: return "BUS_ADRALN";
105 case BUS_ADRERR: return "BUS_ADRERR";
106 case BUS_OBJERR: return "BUS_OBJERR";
Elliott Hughesbd395b92014-04-24 13:53:22 -0700107 case BUS_MCEERR_AR: return "BUS_MCEERR_AR";
108 case BUS_MCEERR_AO: return "BUS_MCEERR_AO";
Christopher Ferris20303f82014-01-10 16:33:16 -0800109 }
Elliott Hughesbd395b92014-04-24 13:53:22 -0700110 static_assert(NSIGBUS == BUS_MCEERR_AO, "missing BUS_* si_code");
Christopher Ferris20303f82014-01-10 16:33:16 -0800111 break;
Jeff Brown053b8652012-06-06 16:25:03 -0700112 case SIGFPE:
Christopher Ferris20303f82014-01-10 16:33:16 -0800113 switch (code) {
Jeff Brown053b8652012-06-06 16:25:03 -0700114 case FPE_INTDIV: return "FPE_INTDIV";
115 case FPE_INTOVF: return "FPE_INTOVF";
116 case FPE_FLTDIV: return "FPE_FLTDIV";
117 case FPE_FLTOVF: return "FPE_FLTOVF";
118 case FPE_FLTUND: return "FPE_FLTUND";
119 case FPE_FLTRES: return "FPE_FLTRES";
120 case FPE_FLTINV: return "FPE_FLTINV";
121 case FPE_FLTSUB: return "FPE_FLTSUB";
Christopher Ferris20303f82014-01-10 16:33:16 -0800122 }
Elliott Hughesbd395b92014-04-24 13:53:22 -0700123 static_assert(NSIGFPE == FPE_FLTSUB, "missing FPE_* si_code");
Christopher Ferris20303f82014-01-10 16:33:16 -0800124 break;
Jeff Brown053b8652012-06-06 16:25:03 -0700125 case SIGSEGV:
Christopher Ferris20303f82014-01-10 16:33:16 -0800126 switch (code) {
Jeff Brown053b8652012-06-06 16:25:03 -0700127 case SEGV_MAPERR: return "SEGV_MAPERR";
128 case SEGV_ACCERR: return "SEGV_ACCERR";
Christopher Ferris20303f82014-01-10 16:33:16 -0800129 }
Elliott Hughesbd395b92014-04-24 13:53:22 -0700130 static_assert(NSIGSEGV == SEGV_ACCERR, "missing SEGV_* si_code");
Christopher Ferris20303f82014-01-10 16:33:16 -0800131 break;
Elliott Hughes8f7d4432012-12-10 10:29:05 -0800132 case SIGTRAP:
Christopher Ferris20303f82014-01-10 16:33:16 -0800133 switch (code) {
Elliott Hughes8f7d4432012-12-10 10:29:05 -0800134 case TRAP_BRKPT: return "TRAP_BRKPT";
135 case TRAP_TRACE: return "TRAP_TRACE";
Elliott Hughesbd395b92014-04-24 13:53:22 -0700136 case TRAP_BRANCH: return "TRAP_BRANCH";
137 case TRAP_HWBKPT: return "TRAP_HWBKPT";
Christopher Ferris20303f82014-01-10 16:33:16 -0800138 }
Elliott Hughesbd395b92014-04-24 13:53:22 -0700139 static_assert(NSIGTRAP == TRAP_HWBKPT, "missing TRAP_* si_code");
Christopher Ferris20303f82014-01-10 16:33:16 -0800140 break;
141 }
142 // Then the other codes...
143 switch (code) {
144 case SI_USER: return "SI_USER";
Christopher Ferris20303f82014-01-10 16:33:16 -0800145 case SI_KERNEL: return "SI_KERNEL";
Christopher Ferris20303f82014-01-10 16:33:16 -0800146 case SI_QUEUE: return "SI_QUEUE";
147 case SI_TIMER: return "SI_TIMER";
148 case SI_MESGQ: return "SI_MESGQ";
Elliott Hughes8f7d4432012-12-10 10:29:05 -0800149 case SI_ASYNCIO: return "SI_ASYNCIO";
Christopher Ferris20303f82014-01-10 16:33:16 -0800150 case SI_SIGIO: return "SI_SIGIO";
Christopher Ferris20303f82014-01-10 16:33:16 -0800151 case SI_TKILL: return "SI_TKILL";
Elliott Hughesbd395b92014-04-24 13:53:22 -0700152 case SI_DETHREAD: return "SI_DETHREAD";
Christopher Ferris20303f82014-01-10 16:33:16 -0800153 }
154 // Then give up...
155 return "?";
Jeff Brown053b8652012-06-06 16:25:03 -0700156}
157
Christopher Ferris20303f82014-01-10 16:33:16 -0800158static void dump_revision_info(log_t* log) {
159 char revision[PROPERTY_VALUE_MAX];
Ben Chengd7760c12012-09-19 16:04:01 -0700160
Christopher Ferris20303f82014-01-10 16:33:16 -0800161 property_get("ro.revision", revision, "unknown");
Ben Chengd7760c12012-09-19 16:04:01 -0700162
Christopher Ferris20303f82014-01-10 16:33:16 -0800163 _LOG(log, SCOPE_AT_FAULT, "Revision: '%s'\n", revision);
Ben Chengd7760c12012-09-19 16:04:01 -0700164}
165
Christopher Ferris20303f82014-01-10 16:33:16 -0800166static void dump_build_info(log_t* log) {
167 char fingerprint[PROPERTY_VALUE_MAX];
Jeff Brown053b8652012-06-06 16:25:03 -0700168
Christopher Ferris20303f82014-01-10 16:33:16 -0800169 property_get("ro.build.fingerprint", fingerprint, "unknown");
Jeff Brown053b8652012-06-06 16:25:03 -0700170
Christopher Ferris20303f82014-01-10 16:33:16 -0800171 _LOG(log, SCOPE_AT_FAULT, "Build fingerprint: '%s'\n", fingerprint);
Jeff Brown053b8652012-06-06 16:25:03 -0700172}
173
Christopher Ferris20303f82014-01-10 16:33:16 -0800174static void dump_fault_addr(log_t* log, pid_t tid, int sig) {
175 siginfo_t si;
Jeff Brown053b8652012-06-06 16:25:03 -0700176
Christopher Ferris20303f82014-01-10 16:33:16 -0800177 memset(&si, 0, sizeof(si));
178 if (ptrace(PTRACE_GETSIGINFO, tid, 0, &si)){
179 _LOG(log, SCOPE_AT_FAULT, "cannot get siginfo: %s\n", strerror(errno));
180 } else if (signal_has_address(sig)) {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400181 _LOG(log, SCOPE_AT_FAULT, "signal %d (%s), code %d (%s), fault addr %" PRIPTR "\n",
Christopher Ferris20303f82014-01-10 16:33:16 -0800182 sig, get_signame(sig), si.si_code, get_sigcode(sig, si.si_code),
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400183 reinterpret_cast<uintptr_t>(si.si_addr));
Christopher Ferris20303f82014-01-10 16:33:16 -0800184 } else {
185 _LOG(log, SCOPE_AT_FAULT, "signal %d (%s), code %d (%s), fault addr --------\n",
186 sig, get_signame(sig), si.si_code, get_sigcode(sig, si.si_code));
187 }
Jeff Brown053b8652012-06-06 16:25:03 -0700188}
189
Christopher Ferris365e4ae2013-10-02 12:26:48 -0700190static void dump_thread_info(log_t* log, pid_t pid, pid_t tid, int scope_flags) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800191 char path[64];
192 char threadnamebuf[1024];
193 char* threadname = NULL;
194 FILE *fp;
Jeff Brown053b8652012-06-06 16:25:03 -0700195
Christopher Ferris20303f82014-01-10 16:33:16 -0800196 snprintf(path, sizeof(path), "/proc/%d/comm", tid);
197 if ((fp = fopen(path, "r"))) {
198 threadname = fgets(threadnamebuf, sizeof(threadnamebuf), fp);
199 fclose(fp);
200 if (threadname) {
201 size_t len = strlen(threadname);
202 if (len && threadname[len - 1] == '\n') {
203 threadname[len - 1] = '\0';
204 }
205 }
206 }
207
208 if (IS_AT_FAULT(scope_flags)) {
209 char procnamebuf[1024];
210 char* procname = NULL;
211
212 snprintf(path, sizeof(path), "/proc/%d/cmdline", pid);
Jeff Brown053b8652012-06-06 16:25:03 -0700213 if ((fp = fopen(path, "r"))) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800214 procname = fgets(procnamebuf, sizeof(procnamebuf), fp);
215 fclose(fp);
Jeff Brown053b8652012-06-06 16:25:03 -0700216 }
217
Christopher Ferris20303f82014-01-10 16:33:16 -0800218 _LOG(log, SCOPE_AT_FAULT, "pid: %d, tid: %d, name: %s >>> %s <<<\n", pid, tid,
219 threadname ? threadname : "UNKNOWN", procname ? procname : "UNKNOWN");
220 } else {
221 _LOG(log, 0, "pid: %d, tid: %d, name: %s\n", pid, tid, threadname ? threadname : "UNKNOWN");
222 }
223}
Jeff Brown053b8652012-06-06 16:25:03 -0700224
Christopher Ferris20303f82014-01-10 16:33:16 -0800225static void dump_stack_segment(
226 Backtrace* backtrace, log_t* log, int scope_flags, uintptr_t* sp, size_t words, int label) {
227 for (size_t i = 0; i < words; i++) {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400228 word_t stack_content;
Christopher Ferris20303f82014-01-10 16:33:16 -0800229 if (!backtrace->ReadWord(*sp, &stack_content)) {
230 break;
231 }
232
Christopher Ferris46756822014-01-14 20:16:30 -0800233 const backtrace_map_t* map = backtrace->FindMap(stack_content);
234 const char* map_name;
235 if (!map) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800236 map_name = "";
Christopher Ferris46756822014-01-14 20:16:30 -0800237 } else {
238 map_name = map->name.c_str();
Christopher Ferris20303f82014-01-10 16:33:16 -0800239 }
240 uintptr_t offset = 0;
241 std::string func_name(backtrace->GetFunctionName(stack_content, &offset));
242 if (!func_name.empty()) {
243 if (!i && label >= 0) {
244 if (offset) {
Elliott Hughesf7b4b512014-01-31 23:13:55 -0800245 _LOG(log, scope_flags, " #%02d %" PRIPTR " %" PRIPTR " %s (%s+%" PRIuPTR ")\n",
Christopher Ferris20303f82014-01-10 16:33:16 -0800246 label, *sp, stack_content, map_name, func_name.c_str(), offset);
247 } else {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400248 _LOG(log, scope_flags, " #%02d %" PRIPTR " %" PRIPTR " %s (%s)\n",
Christopher Ferris20303f82014-01-10 16:33:16 -0800249 label, *sp, stack_content, map_name, func_name.c_str());
Jeff Brown053b8652012-06-06 16:25:03 -0700250 }
Christopher Ferris20303f82014-01-10 16:33:16 -0800251 } else {
252 if (offset) {
Elliott Hughesf7b4b512014-01-31 23:13:55 -0800253 _LOG(log, scope_flags, " %" PRIPTR " %" PRIPTR " %s (%s+%" PRIuPTR ")\n",
Christopher Ferris20303f82014-01-10 16:33:16 -0800254 *sp, stack_content, map_name, func_name.c_str(), offset);
255 } else {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400256 _LOG(log, scope_flags, " %" PRIPTR " %" PRIPTR " %s (%s)\n",
Christopher Ferris20303f82014-01-10 16:33:16 -0800257 *sp, stack_content, map_name, func_name.c_str());
258 }
259 }
Jeff Brown053b8652012-06-06 16:25:03 -0700260 } else {
Christopher Ferris20303f82014-01-10 16:33:16 -0800261 if (!i && label >= 0) {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400262 _LOG(log, scope_flags, " #%02d %" PRIPTR " %" PRIPTR " %s\n",
Christopher Ferris20303f82014-01-10 16:33:16 -0800263 label, *sp, stack_content, map_name);
264 } else {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400265 _LOG(log, scope_flags, " %" PRIPTR " %" PRIPTR " %s\n",
Christopher Ferris20303f82014-01-10 16:33:16 -0800266 *sp, stack_content, map_name);
267 }
Jeff Brown053b8652012-06-06 16:25:03 -0700268 }
Christopher Ferris20303f82014-01-10 16:33:16 -0800269
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400270 *sp += sizeof(word_t);
Christopher Ferris20303f82014-01-10 16:33:16 -0800271 }
Jeff Brown053b8652012-06-06 16:25:03 -0700272}
273
Christopher Ferris20303f82014-01-10 16:33:16 -0800274static void dump_stack(Backtrace* backtrace, log_t* log, int scope_flags) {
275 size_t first = 0, last;
276 for (size_t i = 0; i < backtrace->NumFrames(); i++) {
277 const backtrace_frame_data_t* frame = backtrace->GetFrame(i);
278 if (frame->sp) {
279 if (!first) {
280 first = i+1;
281 }
282 last = i;
Jeff Brown053b8652012-06-06 16:25:03 -0700283 }
Christopher Ferris20303f82014-01-10 16:33:16 -0800284 }
285 if (!first) {
286 return;
287 }
288 first--;
289
290 scope_flags |= SCOPE_SENSITIVE;
291
292 // Dump a few words before the first frame.
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400293 word_t sp = backtrace->GetFrame(first)->sp - STACK_WORDS * sizeof(word_t);
Christopher Ferris20303f82014-01-10 16:33:16 -0800294 dump_stack_segment(backtrace, log, scope_flags, &sp, STACK_WORDS, -1);
295
296 // Dump a few words from all successive frames.
297 // Only log the first 3 frames, put the rest in the tombstone.
298 for (size_t i = first; i <= last; i++) {
299 const backtrace_frame_data_t* frame = backtrace->GetFrame(i);
300 if (sp != frame->sp) {
301 _LOG(log, scope_flags, " ........ ........\n");
302 sp = frame->sp;
303 }
304 if (i - first == 3) {
305 scope_flags &= (~SCOPE_AT_FAULT);
306 }
307 if (i == last) {
308 dump_stack_segment(backtrace, log, scope_flags, &sp, STACK_WORDS, i);
309 if (sp < frame->sp + frame->stack_size) {
310 _LOG(log, scope_flags, " ........ ........\n");
311 }
312 } else {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400313 size_t words = frame->stack_size / sizeof(word_t);
Christopher Ferris20303f82014-01-10 16:33:16 -0800314 if (words == 0) {
315 words = 1;
316 } else if (words > STACK_WORDS) {
317 words = STACK_WORDS;
318 }
319 dump_stack_segment(backtrace, log, scope_flags, &sp, words, i);
320 }
321 }
Jeff Brown053b8652012-06-06 16:25:03 -0700322}
323
Christopher Ferris20303f82014-01-10 16:33:16 -0800324static void dump_backtrace_and_stack(Backtrace* backtrace, log_t* log, int scope_flags) {
325 if (backtrace->NumFrames()) {
326 _LOG(log, scope_flags, "\nbacktrace:\n");
327 dump_backtrace_to_log(backtrace, log, scope_flags, " ");
Jeff Brown053b8652012-06-06 16:25:03 -0700328
Christopher Ferris20303f82014-01-10 16:33:16 -0800329 _LOG(log, scope_flags, "\nstack:\n");
330 dump_stack(backtrace, log, scope_flags);
331 }
Jeff Brown053b8652012-06-06 16:25:03 -0700332}
333
Christopher Ferris46756822014-01-14 20:16:30 -0800334static void dump_map(log_t* log, const backtrace_map_t* map, const char* what, int scope_flags) {
335 if (map != NULL) {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400336 _LOG(log, scope_flags, " %" PRIPTR "-%" PRIPTR " %c%c%c %s\n", map->start, map->end,
Christopher Ferris46756822014-01-14 20:16:30 -0800337 (map->flags & PROT_READ) ? 'r' : '-', (map->flags & PROT_WRITE) ? 'w' : '-',
338 (map->flags & PROT_EXEC) ? 'x' : '-', map->name.c_str());
Christopher Ferris20303f82014-01-10 16:33:16 -0800339 } else {
340 _LOG(log, scope_flags, " (no %s)\n", what);
341 }
Elliott Hughesd1420be2013-01-03 13:39:57 -0800342}
343
Christopher Ferris46756822014-01-14 20:16:30 -0800344static void dump_nearby_maps(BacktraceMap* map, log_t* log, pid_t tid, int scope_flags) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800345 scope_flags |= SCOPE_SENSITIVE;
346 siginfo_t si;
347 memset(&si, 0, sizeof(si));
348 if (ptrace(PTRACE_GETSIGINFO, tid, 0, &si)) {
349 _LOG(log, scope_flags, "cannot get siginfo for %d: %s\n", tid, strerror(errno));
350 return;
351 }
352 if (!signal_has_address(si.si_signo)) {
353 return;
354 }
355
Christopher Ferris46756822014-01-14 20:16:30 -0800356 uintptr_t addr = reinterpret_cast<uintptr_t>(si.si_addr);
Christopher Ferris20303f82014-01-10 16:33:16 -0800357 addr &= ~0xfff; // round to 4K page boundary
358 if (addr == 0) { // null-pointer deref
359 return;
360 }
361
Kévin PETITabc60c22013-12-19 12:36:59 +0000362 _LOG(log, scope_flags, "\nmemory map around fault addr %" PRIPTR ":\n",
Christopher Ferris20303f82014-01-10 16:33:16 -0800363 reinterpret_cast<uintptr_t>(si.si_addr));
364
365 // Search for a match, or for a hole where the match would be. The list
366 // is backward from the file content, so it starts at high addresses.
Christopher Ferris46756822014-01-14 20:16:30 -0800367 const backtrace_map_t* cur_map = NULL;
368 const backtrace_map_t* next_map = NULL;
369 const backtrace_map_t* prev_map = NULL;
370 for (BacktraceMap::const_iterator it = map->begin(); it != map->end(); ++it) {
371 if (addr >= it->start && addr < it->end) {
372 cur_map = &*it;
373 if (it != map->begin()) {
374 prev_map = &*(it-1);
375 }
376 if (++it != map->end()) {
377 next_map = &*it;
378 }
Christopher Ferris20303f82014-01-10 16:33:16 -0800379 break;
Jeff Brown053b8652012-06-06 16:25:03 -0700380 }
Christopher Ferris20303f82014-01-10 16:33:16 -0800381 }
Jeff Brown053b8652012-06-06 16:25:03 -0700382
Christopher Ferris46756822014-01-14 20:16:30 -0800383 // Show the map address in ascending order (like /proc/pid/maps).
384 dump_map(log, prev_map, "map below", scope_flags);
385 dump_map(log, cur_map, "map for address", scope_flags);
386 dump_map(log, next_map, "map above", scope_flags);
Jeff Brown053b8652012-06-06 16:25:03 -0700387}
388
Christopher Ferris20303f82014-01-10 16:33:16 -0800389static void dump_thread(
390 Backtrace* backtrace, log_t* log, int scope_flags, int* total_sleep_time_usec) {
391 wait_for_stop(backtrace->Tid(), total_sleep_time_usec);
Jeff Brown053b8652012-06-06 16:25:03 -0700392
Christopher Ferris20303f82014-01-10 16:33:16 -0800393 dump_registers(log, backtrace->Tid(), scope_flags);
394 dump_backtrace_and_stack(backtrace, log, scope_flags);
395 if (IS_AT_FAULT(scope_flags)) {
396 dump_memory_and_code(log, backtrace->Tid(), scope_flags);
Christopher Ferris46756822014-01-14 20:16:30 -0800397 dump_nearby_maps(backtrace->GetMap(), log, backtrace->Tid(), scope_flags);
Christopher Ferris20303f82014-01-10 16:33:16 -0800398 }
Jeff Brown053b8652012-06-06 16:25:03 -0700399}
400
Christopher Ferris20303f82014-01-10 16:33:16 -0800401// Return true if some thread is not detached cleanly
Christopher Ferris365e4ae2013-10-02 12:26:48 -0700402static bool dump_sibling_thread_report(
Christopher Ferris46756822014-01-14 20:16:30 -0800403 log_t* log, pid_t pid, pid_t tid, int* total_sleep_time_usec, BacktraceMap* map) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800404 char task_path[64];
405 snprintf(task_path, sizeof(task_path), "/proc/%d/task", pid);
Jeff Brown053b8652012-06-06 16:25:03 -0700406
Christopher Ferris20303f82014-01-10 16:33:16 -0800407 DIR* d = opendir(task_path);
408 // Bail early if the task directory cannot be opened
409 if (d == NULL) {
410 XLOG("Cannot open /proc/%d/task\n", pid);
411 return false;
412 }
413
414 bool detach_failed = false;
415 struct dirent* de;
416 while ((de = readdir(d)) != NULL) {
417 // Ignore "." and ".."
418 if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) {
419 continue;
Jeff Brown053b8652012-06-06 16:25:03 -0700420 }
421
Christopher Ferris20303f82014-01-10 16:33:16 -0800422 // The main thread at fault has been handled individually
423 char* end;
424 pid_t new_tid = strtoul(de->d_name, &end, 10);
425 if (*end || new_tid == tid) {
426 continue;
Jeff Brown053b8652012-06-06 16:25:03 -0700427 }
428
Christopher Ferris20303f82014-01-10 16:33:16 -0800429 // Skip this thread if cannot ptrace it
430 if (ptrace(PTRACE_ATTACH, new_tid, 0, 0) < 0) {
431 continue;
432 }
433
434 _LOG(log, 0, "--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
435 dump_thread_info(log, pid, new_tid, 0);
436
Christopher Ferris46756822014-01-14 20:16:30 -0800437 UniquePtr<Backtrace> backtrace(Backtrace::Create(pid, new_tid, map));
Christopher Ferris20303f82014-01-10 16:33:16 -0800438 if (backtrace->Unwind(0)) {
439 dump_thread(backtrace.get(), log, 0, total_sleep_time_usec);
440 }
441
442 if (ptrace(PTRACE_DETACH, new_tid, 0, 0) != 0) {
443 LOG("ptrace detach from %d failed: %s\n", new_tid, strerror(errno));
444 detach_failed = true;
445 }
446 }
447
448 closedir(d);
449 return detach_failed;
Jeff Brown053b8652012-06-06 16:25:03 -0700450}
451
Christopher Ferris20303f82014-01-10 16:33:16 -0800452// Reads the contents of the specified log device, filters out the entries
453// that don't match the specified pid, and writes them to the tombstone file.
454//
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800455// If "tail" is non-zero, log the last "tail" number of lines.
456static void dump_log_file(
457 log_t* log, pid_t pid, const char* filename, unsigned int tail) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800458 bool first = true;
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800459 struct logger_list* logger_list;
Jeff Brown053b8652012-06-06 16:25:03 -0700460
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800461 logger_list = android_logger_list_open(
462 android_name_to_log_id(filename), O_RDONLY | O_NONBLOCK, tail, pid);
Jeff Brown053b8652012-06-06 16:25:03 -0700463
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800464 if (!logger_list) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800465 XLOG("Unable to open %s: %s\n", filename, strerror(errno));
466 return;
467 }
468
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800469 struct log_msg log_entry;
Christopher Ferris20303f82014-01-10 16:33:16 -0800470
471 while (true) {
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800472 ssize_t actual = android_logger_list_read(logger_list, &log_entry);
473 struct logger_entry* entry;
474
Christopher Ferris20303f82014-01-10 16:33:16 -0800475 if (actual < 0) {
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800476 if (actual == -EINTR) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800477 // interrupted by signal, retry
478 continue;
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800479 } else if (actual == -EAGAIN) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800480 // non-blocking EOF; we're done
481 break;
482 } else {
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800483 _LOG(log, 0, "Error while reading log: %s\n", strerror(-actual));
Christopher Ferris20303f82014-01-10 16:33:16 -0800484 break;
485 }
486 } else if (actual == 0) {
Mark Salyzyn22b5cef2013-11-22 10:53:34 -0800487 _LOG(log, 0, "Got zero bytes while reading log: %s\n",
488 strerror(errno));
Christopher Ferris20303f82014-01-10 16:33:16 -0800489 break;
Jeff Brown053b8652012-06-06 16:25:03 -0700490 }
491
Christopher Ferris20303f82014-01-10 16:33:16 -0800492 // NOTE: if you XLOG something here, this will spin forever,
493 // because you will be writing as fast as you're reading. Any
494 // high-frequency debug diagnostics should just be written to
495 // the tombstone file.
Jeff Brown053b8652012-06-06 16:25:03 -0700496
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800497 entry = &log_entry.entry_v1;
Mark Salyzyna63f9272013-11-22 10:53:34 -0800498
Christopher Ferris20303f82014-01-10 16:33:16 -0800499 if (first) {
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800500 _LOG(log, 0, "--------- %slog %s\n", tail ? "tail end of " : "", filename);
Christopher Ferris20303f82014-01-10 16:33:16 -0800501 first = false;
Jeff Brown053b8652012-06-06 16:25:03 -0700502 }
503
Christopher Ferris20303f82014-01-10 16:33:16 -0800504 // Msg format is: <priority:1><tag:N>\0<message:N>\0
505 //
506 // We want to display it in the same format as "logcat -v threadtime"
507 // (although in this case the pid is redundant).
Christopher Ferris20303f82014-01-10 16:33:16 -0800508 static const char* kPrioChars = "!.VDIWEFS";
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800509 unsigned hdr_size = log_entry.entry.hdr_size;
510 if (!hdr_size) {
511 hdr_size = sizeof(log_entry.entry_v1);
512 }
513 char* msg = reinterpret_cast<char*>(log_entry.buf) + hdr_size;
514 unsigned char prio = msg[0];
515 char* tag = msg + 1;
516 msg = tag + strlen(tag) + 1;
Christopher Ferris20303f82014-01-10 16:33:16 -0800517
518 // consume any trailing newlines
Mark Salyzyn706fad22013-12-12 12:21:20 -0800519 char* nl = msg + strlen(msg) - 1;
520 while (nl >= msg && *nl == '\n') {
Mark Salyzyn8e4cdc12014-01-28 13:51:46 -0800521 *nl-- = '\0';
Christopher Ferris20303f82014-01-10 16:33:16 -0800522 }
523
524 char prioChar = (prio < strlen(kPrioChars) ? kPrioChars[prio] : '?');
525
526 char timeBuf[32];
527 time_t sec = static_cast<time_t>(entry->sec);
528 struct tm tmBuf;
529 struct tm* ptm;
530 ptm = localtime_r(&sec, &tmBuf);
531 strftime(timeBuf, sizeof(timeBuf), "%m-%d %H:%M:%S", ptm);
532
Mark Salyzyn706fad22013-12-12 12:21:20 -0800533 // Look for line breaks ('\n') and display each text line
534 // on a separate line, prefixed with the header, like logcat does.
535 do {
Mark Salyzynfca0bd12013-12-12 12:21:20 -0800536 nl = strchr(msg, '\n');
537 if (nl) {
538 *nl = '\0';
539 ++nl;
540 }
Mark Salyzyn706fad22013-12-12 12:21:20 -0800541
Mark Salyzynfca0bd12013-12-12 12:21:20 -0800542 _LOG(log, 0, "%s.%03d %5d %5d %c %-8s: %s\n",
543 timeBuf, entry->nsec / 1000000, entry->pid, entry->tid,
544 prioChar, tag, msg);
Mark Salyzyn706fad22013-12-12 12:21:20 -0800545 } while ((msg = nl));
Christopher Ferris20303f82014-01-10 16:33:16 -0800546 }
Jeff Brown053b8652012-06-06 16:25:03 -0700547
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800548 android_logger_list_free(logger_list);
Jeff Brown053b8652012-06-06 16:25:03 -0700549}
550
Christopher Ferris20303f82014-01-10 16:33:16 -0800551// Dumps the logs generated by the specified pid to the tombstone, from both
552// "system" and "main" log devices. Ideally we'd interleave the output.
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800553static void dump_logs(log_t* log, pid_t pid, unsigned int tail) {
554 dump_log_file(log, pid, "system", tail);
555 dump_log_file(log, pid, "main", tail);
Jeff Brown053b8652012-06-06 16:25:03 -0700556}
557
Christopher Ferris20303f82014-01-10 16:33:16 -0800558static void dump_abort_message(Backtrace* backtrace, log_t* log, uintptr_t address) {
Elliott Hughese5f8a692013-04-04 13:52:01 -0700559 if (address == 0) {
560 return;
561 }
562
563 address += sizeof(size_t); // Skip the buffer length.
564
565 char msg[512];
566 memset(msg, 0, sizeof(msg));
567 char* p = &msg[0];
568 while (p < &msg[sizeof(msg)]) {
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400569 word_t data;
570 size_t len = sizeof(word_t);
Christopher Ferris20303f82014-01-10 16:33:16 -0800571 if (!backtrace->ReadWord(address, &data)) {
Elliott Hughese5f8a692013-04-04 13:52:01 -0700572 break;
573 }
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400574 address += sizeof(word_t);
Elliott Hughese5f8a692013-04-04 13:52:01 -0700575
Pavel Chupinc6c194c2013-11-21 23:17:20 +0400576 while (len > 0 && (*p++ = (data >> (sizeof(word_t) - len) * 8) & 0xff) != 0)
577 len--;
Elliott Hughese5f8a692013-04-04 13:52:01 -0700578 }
579 msg[sizeof(msg) - 1] = '\0';
580
Christopher Tate7716aef2013-04-02 14:00:27 -0700581 _LOG(log, SCOPE_AT_FAULT, "Abort message: '%s'\n", msg);
Elliott Hughese5f8a692013-04-04 13:52:01 -0700582}
583
Christopher Ferris20303f82014-01-10 16:33:16 -0800584// Dumps all information about the specified pid to the tombstone.
Elliott Hughese5f8a692013-04-04 13:52:01 -0700585static bool dump_crash(log_t* log, pid_t pid, pid_t tid, int signal, uintptr_t abort_msg_address,
Christopher Ferris20303f82014-01-10 16:33:16 -0800586 bool dump_sibling_threads, int* total_sleep_time_usec) {
587 // don't copy log messages to tombstone unless this is a dev device
588 char value[PROPERTY_VALUE_MAX];
589 property_get("ro.debuggable", value, "0");
590 bool want_logs = (value[0] == '1');
Jeff Brown053b8652012-06-06 16:25:03 -0700591
Christopher Ferris20303f82014-01-10 16:33:16 -0800592 if (log->amfd >= 0) {
593 // Activity Manager protocol: binary 32-bit network-byte-order ints for the
594 // pid and signal number, followed by the raw text of the dump, culminating
595 // in a zero byte that marks end-of-data.
596 uint32_t datum = htonl(pid);
597 TEMP_FAILURE_RETRY( write(log->amfd, &datum, 4) );
598 datum = htonl(signal);
599 TEMP_FAILURE_RETRY( write(log->amfd, &datum, 4) );
600 }
Christopher Tateded2e5a2013-03-19 13:12:23 -0700601
Christopher Ferris20303f82014-01-10 16:33:16 -0800602 _LOG(log, SCOPE_AT_FAULT,
603 "*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***\n");
604 dump_build_info(log);
605 dump_revision_info(log);
606 dump_thread_info(log, pid, tid, SCOPE_AT_FAULT);
607 if (signal) {
608 dump_fault_addr(log, tid, signal);
609 }
Jeff Brown053b8652012-06-06 16:25:03 -0700610
Christopher Ferrisdf290612014-01-22 19:21:07 -0800611 UniquePtr<BacktraceMap> map(BacktraceMap::Create(pid));
612 UniquePtr<Backtrace> backtrace(Backtrace::Create(pid, tid, map.get()));
Christopher Ferris20303f82014-01-10 16:33:16 -0800613 if (backtrace->Unwind(0)) {
614 dump_abort_message(backtrace.get(), log, abort_msg_address);
615 dump_thread(backtrace.get(), log, SCOPE_AT_FAULT, total_sleep_time_usec);
616 }
Jeff Brown053b8652012-06-06 16:25:03 -0700617
Christopher Ferris20303f82014-01-10 16:33:16 -0800618 if (want_logs) {
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800619 dump_logs(log, pid, 5);
Christopher Ferris20303f82014-01-10 16:33:16 -0800620 }
Jeff Brown053b8652012-06-06 16:25:03 -0700621
Christopher Ferris20303f82014-01-10 16:33:16 -0800622 bool detach_failed = false;
623 if (dump_sibling_threads) {
Christopher Ferrisdf290612014-01-22 19:21:07 -0800624 detach_failed = dump_sibling_thread_report(log, pid, tid, total_sleep_time_usec, map.get());
Christopher Ferris20303f82014-01-10 16:33:16 -0800625 }
Christopher Ferris98464972014-01-06 19:16:33 -0800626
Christopher Ferris20303f82014-01-10 16:33:16 -0800627 if (want_logs) {
Christopher Ferris4167d7e2014-01-13 18:12:04 -0800628 dump_logs(log, pid, 0);
Christopher Ferris20303f82014-01-10 16:33:16 -0800629 }
Christopher Tateded2e5a2013-03-19 13:12:23 -0700630
Christopher Ferris20303f82014-01-10 16:33:16 -0800631 // send EOD to the Activity Manager, then wait for its ack to avoid racing ahead
632 // and killing the target out from under it
633 if (log->amfd >= 0) {
634 uint8_t eodMarker = 0;
635 TEMP_FAILURE_RETRY( write(log->amfd, &eodMarker, 1) );
636 // 3 sec timeout reading the ack; we're fine if that happens
637 TEMP_FAILURE_RETRY( read(log->amfd, &eodMarker, 1) );
638 }
639
640 return detach_failed;
Jeff Brown053b8652012-06-06 16:25:03 -0700641}
642
Christopher Ferris20303f82014-01-10 16:33:16 -0800643// find_and_open_tombstone - find an available tombstone slot, if any, of the
644// form tombstone_XX where XX is 00 to MAX_TOMBSTONES-1, inclusive. If no
645// file is available, we reuse the least-recently-modified file.
646//
647// Returns the path of the tombstone file, allocated using malloc(). Caller must free() it.
648static char* find_and_open_tombstone(int* fd) {
Christopher Ferris68bd59f2014-02-07 13:25:26 -0800649 // In a single pass, find an available slot and, in case none
Christopher Ferris20303f82014-01-10 16:33:16 -0800650 // exist, find and record the least-recently-modified file.
651 char path[128];
Christopher Ferris68bd59f2014-02-07 13:25:26 -0800652 int oldest = -1;
653 struct stat oldest_sb;
Christopher Ferris20303f82014-01-10 16:33:16 -0800654 for (int i = 0; i < MAX_TOMBSTONES; i++) {
Kévin PETIT4bb47722013-12-18 16:44:24 +0000655 snprintf(path, sizeof(path), TOMBSTONE_TEMPLATE, i);
Jeff Brown053b8652012-06-06 16:25:03 -0700656
Christopher Ferris68bd59f2014-02-07 13:25:26 -0800657 struct stat sb;
Christopher Ferris20303f82014-01-10 16:33:16 -0800658 if (!stat(path, &sb)) {
Christopher Ferris68bd59f2014-02-07 13:25:26 -0800659 if (oldest < 0 || sb.st_mtime < oldest_sb.st_mtime) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800660 oldest = i;
Christopher Ferris68bd59f2014-02-07 13:25:26 -0800661 oldest_sb.st_mtime = sb.st_mtime;
Christopher Ferris20303f82014-01-10 16:33:16 -0800662 }
663 continue;
Jeff Brown053b8652012-06-06 16:25:03 -0700664 }
Christopher Ferris20303f82014-01-10 16:33:16 -0800665 if (errno != ENOENT)
666 continue;
Jeff Brown053b8652012-06-06 16:25:03 -0700667
Christopher Ferris20303f82014-01-10 16:33:16 -0800668 *fd = open(path, O_CREAT | O_EXCL | O_WRONLY, 0600);
669 if (*fd < 0)
670 continue; // raced ?
671
Jeff Brown053b8652012-06-06 16:25:03 -0700672 fchown(*fd, AID_SYSTEM, AID_SYSTEM);
673 return strdup(path);
Christopher Ferris20303f82014-01-10 16:33:16 -0800674 }
675
Christopher Ferris68bd59f2014-02-07 13:25:26 -0800676 if (oldest < 0) {
677 LOG("Failed to find a valid tombstone, default to using tombstone 0.\n");
678 oldest = 0;
679 }
680
Christopher Ferris20303f82014-01-10 16:33:16 -0800681 // we didn't find an available file, so we clobber the oldest one
Kévin PETIT4bb47722013-12-18 16:44:24 +0000682 snprintf(path, sizeof(path), TOMBSTONE_TEMPLATE, oldest);
Christopher Ferris20303f82014-01-10 16:33:16 -0800683 *fd = open(path, O_CREAT | O_TRUNC | O_WRONLY, 0600);
684 if (*fd < 0) {
685 LOG("failed to open tombstone file '%s': %s\n", path, strerror(errno));
686 return NULL;
687 }
688 fchown(*fd, AID_SYSTEM, AID_SYSTEM);
689 return strdup(path);
Jeff Brown053b8652012-06-06 16:25:03 -0700690}
691
Christopher Tateded2e5a2013-03-19 13:12:23 -0700692static int activity_manager_connect() {
Christopher Ferris20303f82014-01-10 16:33:16 -0800693 int amfd = socket(PF_UNIX, SOCK_STREAM, 0);
694 if (amfd >= 0) {
695 struct sockaddr_un address;
696 int err;
Christopher Tateded2e5a2013-03-19 13:12:23 -0700697
Christopher Ferris20303f82014-01-10 16:33:16 -0800698 memset(&address, 0, sizeof(address));
699 address.sun_family = AF_UNIX;
700 strncpy(address.sun_path, NCRASH_SOCKET_PATH, sizeof(address.sun_path));
701 err = TEMP_FAILURE_RETRY(connect(
702 amfd, reinterpret_cast<struct sockaddr*>(&address), sizeof(address)));
703 if (!err) {
704 struct timeval tv;
705 memset(&tv, 0, sizeof(tv));
706 tv.tv_sec = 1; // tight leash
707 err = setsockopt(amfd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
708 if (!err) {
709 tv.tv_sec = 3; // 3 seconds on handshake read
710 err = setsockopt(amfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
711 }
Christopher Tateded2e5a2013-03-19 13:12:23 -0700712 }
Christopher Ferris20303f82014-01-10 16:33:16 -0800713 if (err) {
714 close(amfd);
715 amfd = -1;
716 }
717 }
Christopher Tateded2e5a2013-03-19 13:12:23 -0700718
Christopher Ferris20303f82014-01-10 16:33:16 -0800719 return amfd;
Christopher Tateded2e5a2013-03-19 13:12:23 -0700720}
721
Christopher Ferris20303f82014-01-10 16:33:16 -0800722char* engrave_tombstone(
723 pid_t pid, pid_t tid, int signal, uintptr_t abort_msg_address, bool dump_sibling_threads,
724 bool quiet, bool* detach_failed, int* total_sleep_time_usec) {
Kévin PETIT4bb47722013-12-18 16:44:24 +0000725 if ((mkdir(TOMBSTONE_DIR, 0755) == -1) && (errno != EEXIST)) {
726 LOG("failed to create %s: %s\n", TOMBSTONE_DIR, strerror(errno));
727 }
728
729 if (chown(TOMBSTONE_DIR, AID_SYSTEM, AID_SYSTEM) == -1) {
730 LOG("failed to change ownership of %s: %s\n", TOMBSTONE_DIR, strerror(errno));
731 }
Jeff Brown053b8652012-06-06 16:25:03 -0700732
Stephen Smalley27a93652014-02-07 09:14:13 -0500733 if (selinux_android_restorecon(TOMBSTONE_DIR, 0) == -1) {
Christopher Ferris20303f82014-01-10 16:33:16 -0800734 *detach_failed = false;
735 return NULL;
736 }
rpcraigf1186f32012-07-19 09:38:06 -0400737
Christopher Ferris20303f82014-01-10 16:33:16 -0800738 int fd;
739 char* path = find_and_open_tombstone(&fd);
740 if (!path) {
741 *detach_failed = false;
742 return NULL;
743 }
Jeff Brown053b8652012-06-06 16:25:03 -0700744
Christopher Ferris20303f82014-01-10 16:33:16 -0800745 log_t log;
746 log.tfd = fd;
747 log.amfd = activity_manager_connect();
748 log.quiet = quiet;
749 *detach_failed = dump_crash(
750 &log, pid, tid, signal, abort_msg_address, dump_sibling_threads, total_sleep_time_usec);
Jeff Brown053b8652012-06-06 16:25:03 -0700751
Christopher Ferris20303f82014-01-10 16:33:16 -0800752 close(log.amfd);
753 close(fd);
754 return path;
Jeff Brown053b8652012-06-06 16:25:03 -0700755}