Add support for tombstone symbolization to pbtombstone.
This patch teaches pbtombstone to use llvm-symbolizer to symbolize
stack traces and augment the protobuf tombstones with the symbol
information, before printing tombstones with the symbolized stack
traces included.
The main advantage of adding this information to the tombstone
as opposed to having developers use the stack tool is that stack
does not print all of the information in the original tombstone,
which means that both reports may be required to understand a crash.
Furthermore, stack traces printed by stack are not correlated with
the stack traces in the tombstone, making the report harder to read,
especially with GWP-ASan and MTE which may produce multiple stack
traces for the crashing thread.
Although we could teach stack to print more information, this would
continue to be fragile because stack relies on parsing textual
tombstones. Switching stack to read proto tombstones would be
tantamount to a full rewrite and would require duplicating the C++
proto-to-text logic that we already have in Python. It seems better
to reuse the C++ code for the proto-based symbolization tool.
llvm-symbolizer will look up the symbol files by build ID using a
.build-id directory following the standard here:
https://fedoraproject.org/wiki/RolandMcGrath/BuildID
It will look for .build-id directories under paths specified with
--debug-file-directory, which pbtombstone will pass through to
llvm-symbolizer using its own --debug-file-directory flag. The
intent is that tools for platform developers will pass the flag
--debug-file-directory $ANDROID_PRODUCT_OUT/symbols to pbtombstone.
Soong will start creating .build-id under symbols after a corresponding
Soong CL lands.
Bug: 328531087
Change-Id: Ia4676821cf980c69487cf11aefa2a02dc0c1626f
diff --git a/debuggerd/Android.bp b/debuggerd/Android.bp
index d4dc9a3..3257a2c 100644
--- a/debuggerd/Android.bp
+++ b/debuggerd/Android.bp
@@ -333,7 +333,10 @@
name: "pbtombstone",
host_supported: true,
defaults: ["debuggerd_defaults"],
- srcs: ["pbtombstone.cpp"],
+ srcs: [
+ "pbtombstone.cpp",
+ "tombstone_symbolize.cpp",
+ ],
static_libs: [
"libbase",
"libdebuggerd_tombstone_proto_to_text",
diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h b/debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h
index 515a15f..2de9723 100644
--- a/debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h
+++ b/debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h
@@ -19,8 +19,10 @@
#include <functional>
#include <string>
+class BacktraceFrame;
class Tombstone;
bool tombstone_proto_to_text(
const Tombstone& tombstone,
- std::function<void(const std::string& line, bool should_log)> callback);
+ std::function<void(const std::string& line, bool should_log)> callback,
+ std::function<void(const BacktraceFrame& frame)> symbolize);
diff --git a/debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp b/debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp
index 3fdb71d..aad209a 100644
--- a/debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp
+++ b/debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp
@@ -61,12 +61,16 @@
void ProtoToString() {
text_ = "";
- EXPECT_TRUE(
- tombstone_proto_to_text(*tombstone_, [this](const std::string& line, bool should_log) {
+ EXPECT_TRUE(tombstone_proto_to_text(
+ *tombstone_,
+ [this](const std::string& line, bool should_log) {
if (should_log) {
text_ += "LOG ";
}
text_ += line + '\n';
+ },
+ [&](const BacktraceFrame& frame) {
+ text_ += "SYMBOLIZE " + frame.build_id() + " " + std::to_string(frame.pc()) + "\n";
}));
}
@@ -163,3 +167,11 @@
EXPECT_MATCH(text_, "stack_record fp:0x1 tag:0xb pc:foo\\.so\\+0x567 \\(BuildId: ABC123\\)");
EXPECT_MATCH(text_, "stack_record fp:0x2 tag:0xc pc:bar\\.so\\+0x678");
}
+
+TEST_F(TombstoneProtoToTextTest, symbolize) {
+ BacktraceFrame* frame = main_thread_->add_current_backtrace();
+ frame->set_pc(12345);
+ frame->set_build_id("0123456789abcdef");
+ ProtoToString();
+ EXPECT_MATCH(text_, "\\(BuildId: 0123456789abcdef\\)\\nSYMBOLIZE 0123456789abcdef 12345\\n");
+}
diff --git a/debuggerd/libdebuggerd/tombstone.cpp b/debuggerd/libdebuggerd/tombstone.cpp
index d483b98..30c6fe4 100644
--- a/debuggerd/libdebuggerd/tombstone.cpp
+++ b/debuggerd/libdebuggerd/tombstone.cpp
@@ -146,7 +146,10 @@
log.tfd = output_fd.get();
log.amfd_data = amfd_data;
- tombstone_proto_to_text(tombstone, [&log](const std::string& line, bool should_log) {
- _LOG(&log, should_log ? logtype::HEADER : logtype::LOGS, "%s\n", line.c_str());
- });
+ tombstone_proto_to_text(
+ tombstone,
+ [&log](const std::string& line, bool should_log) {
+ _LOG(&log, should_log ? logtype::HEADER : logtype::LOGS, "%s\n", line.c_str());
+ },
+ [](const BacktraceFrame&) {});
}
diff --git a/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp b/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp
index 611e237..fedafc0 100644
--- a/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp
+++ b/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp
@@ -41,6 +41,7 @@
#define CBL(...) CB(true, __VA_ARGS__)
#define CBS(...) CB(false, __VA_ARGS__)
using CallbackType = std::function<void(const std::string& line, bool should_log)>;
+using SymbolizeCallbackType = std::function<void(const BacktraceFrame& frame)>;
#define DESCRIBE_FLAG(flag) \
if (value & flag) { \
@@ -184,7 +185,8 @@
print_register_row(callback, word_size, special_row, should_log);
}
-static void print_backtrace(CallbackType callback, const Tombstone& tombstone,
+static void print_backtrace(CallbackType callback, SymbolizeCallbackType symbolize,
+ const Tombstone& tombstone,
const google::protobuf::RepeatedPtrField<BacktraceFrame>& backtrace,
bool should_log) {
int index = 0;
@@ -209,11 +211,14 @@
}
line += function + build_id;
CB(should_log, "%s", line.c_str());
+
+ symbolize(frame);
}
}
-static void print_thread_backtrace(CallbackType callback, const Tombstone& tombstone,
- const Thread& thread, bool should_log) {
+static void print_thread_backtrace(CallbackType callback, SymbolizeCallbackType symbolize,
+ const Tombstone& tombstone, const Thread& thread,
+ bool should_log) {
CBS("");
CB(should_log, "%d total frames", thread.current_backtrace().size());
CB(should_log, "backtrace:");
@@ -221,7 +226,7 @@
CB(should_log, " NOTE: %s",
android::base::Join(thread.backtrace_note(), "\n NOTE: ").c_str());
}
- print_backtrace(callback, tombstone, thread.current_backtrace(), should_log);
+ print_backtrace(callback, symbolize, tombstone, thread.current_backtrace(), should_log);
}
static void print_thread_memory_dump(CallbackType callback, const Tombstone& tombstone,
@@ -274,10 +279,11 @@
}
}
-static void print_thread(CallbackType callback, const Tombstone& tombstone, const Thread& thread) {
+static void print_thread(CallbackType callback, SymbolizeCallbackType symbolize,
+ const Tombstone& tombstone, const Thread& thread) {
print_thread_header(callback, tombstone, thread, false);
print_thread_registers(callback, tombstone, thread, false);
- print_thread_backtrace(callback, tombstone, thread, false);
+ print_thread_backtrace(callback, symbolize, tombstone, thread, false);
print_thread_memory_dump(callback, tombstone, thread);
}
@@ -433,8 +439,8 @@
return oct_encoded;
}
-static void print_main_thread(CallbackType callback, const Tombstone& tombstone,
- const Thread& thread) {
+static void print_main_thread(CallbackType callback, SymbolizeCallbackType symbolize,
+ const Tombstone& tombstone, const Thread& thread) {
print_thread_header(callback, tombstone, thread, true);
const Signal& signal_info = tombstone.signal_info();
@@ -488,7 +494,7 @@
CBL(" in this process. The stack trace below is the first system call or context");
CBL(" switch that was executed after the memory corruption happened.");
}
- print_thread_backtrace(callback, tombstone, thread, true);
+ print_thread_backtrace(callback, symbolize, tombstone, thread, true);
if (tombstone.causes_size() > 1) {
CBS("");
@@ -521,13 +527,13 @@
if (heap_object.deallocation_backtrace_size() != 0) {
CBS("");
CBL("deallocated by thread %" PRIu64 ":", heap_object.deallocation_tid());
- print_backtrace(callback, tombstone, heap_object.deallocation_backtrace(), true);
+ print_backtrace(callback, symbolize, tombstone, heap_object.deallocation_backtrace(), true);
}
if (heap_object.allocation_backtrace_size() != 0) {
CBS("");
CBL("allocated by thread %" PRIu64 ":", heap_object.allocation_tid());
- print_backtrace(callback, tombstone, heap_object.allocation_backtrace(), true);
+ print_backtrace(callback, symbolize, tombstone, heap_object.allocation_backtrace(), true);
}
}
}
@@ -576,8 +582,9 @@
}
}
-static void print_guest_thread(CallbackType callback, const Tombstone& tombstone,
- const Thread& guest_thread, pid_t tid, bool should_log) {
+static void print_guest_thread(CallbackType callback, SymbolizeCallbackType symbolize,
+ const Tombstone& tombstone, const Thread& guest_thread, pid_t tid,
+ bool should_log) {
CBS("--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---");
CBS("Guest thread information for tid: %d", tid);
print_thread_registers(callback, tombstone, guest_thread, should_log);
@@ -585,12 +592,13 @@
CBS("");
CB(true, "%d total frames", guest_thread.current_backtrace().size());
CB(true, "backtrace:");
- print_backtrace(callback, tombstone, guest_thread.current_backtrace(), should_log);
+ print_backtrace(callback, symbolize, tombstone, guest_thread.current_backtrace(), should_log);
print_thread_memory_dump(callback, tombstone, guest_thread);
}
-bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback) {
+bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback,
+ SymbolizeCallbackType symbolize) {
CBL("*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***");
CBL("Build fingerprint: '%s'", tombstone.build_fingerprint().c_str());
CBL("Revision: '%s'", tombstone.revision().c_str());
@@ -618,14 +626,15 @@
const auto& main_thread = main_thread_it->second;
- print_main_thread(callback, tombstone, main_thread);
+ print_main_thread(callback, symbolize, tombstone, main_thread);
print_logs(callback, tombstone, 50);
const auto& guest_threads = tombstone.guest_threads();
auto main_guest_thread_it = guest_threads.find(tombstone.tid());
if (main_guest_thread_it != threads.end()) {
- print_guest_thread(callback, tombstone, main_guest_thread_it->second, tombstone.tid(), true);
+ print_guest_thread(callback, symbolize, tombstone, main_guest_thread_it->second,
+ tombstone.tid(), true);
}
// protobuf's map is unordered, so sort the keys first.
@@ -638,10 +647,10 @@
for (const auto& tid : thread_ids) {
CBS("--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---");
- print_thread(callback, tombstone, threads.find(tid)->second);
+ print_thread(callback, symbolize, tombstone, threads.find(tid)->second);
auto guest_thread_it = guest_threads.find(tid);
if (guest_thread_it != guest_threads.end()) {
- print_guest_thread(callback, tombstone, guest_thread_it->second, tid, false);
+ print_guest_thread(callback, symbolize, tombstone, guest_thread_it->second, tid, false);
}
}
diff --git a/debuggerd/pbtombstone.cpp b/debuggerd/pbtombstone.cpp
index dcb7c6c..0902b38 100644
--- a/debuggerd/pbtombstone.cpp
+++ b/debuggerd/pbtombstone.cpp
@@ -16,32 +16,55 @@
#include <err.h>
#include <fcntl.h>
+#include <getopt.h>
#include <stdio.h>
#include <unistd.h>
+#include <string>
+#include <vector>
+
#include <android-base/unique_fd.h>
#include <libdebuggerd/tombstone_proto_to_text.h>
#include "tombstone.pb.h"
+#include "tombstone_symbolize.h"
using android::base::unique_fd;
[[noreturn]] void usage(bool error) {
- fprintf(stderr, "usage: pbtombstone TOMBSTONE.PB\n");
+ fprintf(stderr, "usage: pbtombstone [OPTION] TOMBSTONE.PB\n");
fprintf(stderr, "Convert a protobuf tombstone to text.\n");
+ fprintf(stderr, "Arguments:\n");
+ fprintf(stderr, " -h, --help print this message\n");
+ fprintf(stderr, " --debug-file-directory PATH specify the path to a symbols directory\n");
exit(error);
}
-int main(int argc, const char* argv[]) {
- if (argc != 2) {
+int main(int argc, char* argv[]) {
+ std::vector<std::string> debug_file_directories;
+ static struct option long_options[] = {
+ {"debug-file-directory", required_argument, 0, 0},
+ {"help", no_argument, 0, 'h'},
+ {},
+ };
+ int c;
+ while ((c = getopt_long(argc, argv, "h", long_options, 0)) != -1) {
+ switch (c) {
+ case 0:
+ debug_file_directories.push_back(optarg);
+ break;
+
+ case 'h':
+ usage(false);
+ break;
+ }
+ }
+
+ if (optind != argc-1) {
usage(true);
}
- if (strcmp("-h", argv[1]) == 0 || strcmp("--help", argv[1]) == 0) {
- usage(false);
- }
-
- unique_fd fd(open(argv[1], O_RDONLY | O_CLOEXEC));
+ unique_fd fd(open(argv[optind], O_RDONLY | O_CLOEXEC));
if (fd == -1) {
err(1, "failed to open tombstone '%s'", argv[1]);
}
@@ -51,8 +74,11 @@
err(1, "failed to parse tombstone");
}
+ Symbolizer sym;
+ sym.Start(debug_file_directories);
bool result = tombstone_proto_to_text(
- tombstone, [](const std::string& line, bool) { printf("%s\n", line.c_str()); });
+ tombstone, [](const std::string& line, bool) { printf("%s\n", line.c_str()); },
+ [&](const BacktraceFrame& frame) { symbolize_backtrace_frame(frame, sym); });
if (!result) {
errx(1, "tombstone was malformed");
diff --git a/debuggerd/tombstone_symbolize.cpp b/debuggerd/tombstone_symbolize.cpp
new file mode 100644
index 0000000..07735d0
--- /dev/null
+++ b/debuggerd/tombstone_symbolize.cpp
@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "tombstone_symbolize.h"
+
+#include <fcntl.h>
+#include <inttypes.h>
+#include <unistd.h>
+
+#include <string>
+#include <vector>
+
+#include "android-base/stringprintf.h"
+#include "android-base/unique_fd.h"
+
+#include "tombstone.pb.h"
+
+using android::base::StringPrintf;
+using android::base::unique_fd;
+
+bool Symbolizer::Start(const std::vector<std::string>& debug_file_directories) {
+ unique_fd parent_in, parent_out, child_in, child_out;
+ if (!Pipe(&parent_in, &child_out) || !Pipe(&child_in, &parent_out)) {
+ return false;
+ }
+
+ std::vector<const char *> args;
+ args.push_back("llvm-symbolizer");
+ for (const std::string &dir : debug_file_directories) {
+ args.push_back("--debug-file-directory");
+ args.push_back(dir.c_str());
+ }
+ args.push_back(0);
+
+ int pid = fork();
+ if (pid == -1) {
+ return false;
+ } else if (pid == 0) {
+ parent_in.reset();
+ parent_out.reset();
+
+ dup2(child_in.get(), STDIN_FILENO);
+ dup2(child_out.get(), STDOUT_FILENO);
+
+ execvp("llvm-symbolizer", const_cast<char *const *>(args.data()));
+
+ fprintf(stderr, "unable to start llvm-symbolizer: %s\n", strerror(errno));
+ _exit(1);
+ } else {
+ child_in.reset();
+ child_out.reset();
+
+ // TODO: Check that llvm-symbolizer started up successfully.
+ // There used to be an easy way to do this, but it was removed in:
+ // https://github.com/llvm/llvm-project/commit/1792852f86dc75efa1f44d46b1a0daf386d64afa
+
+ in_fd = std::move(parent_in);
+ out_fd = std::move(parent_out);
+ return true;
+ }
+}
+
+std::string Symbolizer::read_response() {
+ std::string resp;
+
+ while (resp.size() < 2 || resp[resp.size() - 2] != '\n' || resp[resp.size() - 1] != '\n') {
+ char buf[4096];
+ ssize_t size = read(in_fd, buf, 4096);
+ if (size <= 0) {
+ return "";
+ }
+ resp.append(buf, size);
+ }
+
+ return resp;
+}
+
+std::vector<Symbolizer::Frame> Symbolizer::SymbolizeCode(std::string path, uint64_t rel_pc) {
+ std::string request = StringPrintf("CODE %s 0x%" PRIx64 "\n", path.c_str(), rel_pc);
+ if (write(out_fd, request.c_str(), request.size()) != static_cast<ssize_t>(request.size())) {
+ return {};
+ }
+
+ std::string response = read_response();
+ if (response.empty()) {
+ return {};
+ }
+
+ std::vector<Symbolizer::Frame> frames;
+
+ size_t frame_start = 0;
+ while (frame_start < response.size() - 1) {
+ Symbolizer::Frame frame;
+
+ size_t second_line_start = response.find('\n', frame_start) + 1;
+ if (second_line_start == std::string::npos + 1) {
+ return {};
+ }
+
+ size_t third_line_start = response.find('\n', second_line_start) + 1;
+ if (third_line_start == std::string::npos + 1) {
+ return {};
+ }
+
+ frame.function_name = response.substr(frame_start, second_line_start - frame_start - 1);
+
+ size_t column_number_start = response.rfind(':', third_line_start);
+ if (column_number_start == std::string::npos) {
+ return {};
+ }
+
+ size_t line_number_start = response.rfind(':', column_number_start - 1);
+ if (line_number_start == std::string::npos) {
+ return {};
+ }
+
+ frame.file = response.substr(second_line_start, line_number_start - second_line_start);
+
+ errno = 0;
+ frame.line = strtoull(response.c_str() + line_number_start + 1, 0, 10);
+ frame.column = strtoull(response.c_str() + column_number_start + 1, 0, 10);
+ if (errno != 0) {
+ return {};
+ }
+
+ frames.push_back(frame);
+
+ frame_start = third_line_start;
+ }
+
+ if (frames.size() == 1 && frames[0].file == "??") {
+ return {};
+ }
+
+ return frames;
+}
+
+void symbolize_backtrace_frame(const BacktraceFrame& frame, Symbolizer& sym) {
+ if (frame.build_id().empty()) {
+ return;
+ }
+
+ for (Symbolizer::Frame f : sym.SymbolizeCode("BUILDID:" + frame.build_id(), frame.rel_pc())) {
+ printf(" %s:%" PRId64 ":%" PRId64 " (%s)\n", f.file.c_str(), f.line, f.column,
+ f.function_name.c_str());
+ }
+}
diff --git a/debuggerd/tombstone_symbolize.h b/debuggerd/tombstone_symbolize.h
new file mode 100644
index 0000000..c22d677
--- /dev/null
+++ b/debuggerd/tombstone_symbolize.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "android-base/unique_fd.h"
+
+class BacktraceFrame;
+
+class Symbolizer {
+ android::base::unique_fd in_fd, out_fd;
+
+ std::string read_response();
+
+ public:
+ bool Start(const std::vector<std::string>& debug_file_directories);
+
+ struct Frame {
+ std::string function_name, file;
+ uint64_t line, column;
+ };
+
+ std::vector<Frame> SymbolizeCode(std::string path, uint64_t rel_pc);
+};
+
+void symbolize_backtrace_frame(const BacktraceFrame& frame, Symbolizer& sym);