Add a config option to save the metrics data locally.
Usually after config owner calls statsd to get data, statsd will remove that data
forever. This cl adds an option to save historical data locally so that when a bugreport
is taken, it contains historical statsd metrics for debugging.
+ All existing guardrail on storage still apply, and local history data has lower
priority when the guardrail is hit. On top of that, statsd will remove history
files that are more than 2 days old.
+ Also fixed issues inside StorageManager (e.g.,
using string::compare to sort by timestamp is problematic, redundant file size read, etc)
Test: statsd_test and manually
Fix: 126562025
Change-Id: I5238b9fd3069cbef5bcccabd2212bc4d6c734702
diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp
index df84b6a..a9f5208e 100644
--- a/cmds/statsd/src/StatsLogProcessor.cpp
+++ b/cmds/statsd/src/StatsLogProcessor.cpp
@@ -77,7 +77,6 @@
#define NS_PER_HOUR 3600 * NS_PER_SEC
-#define STATS_DATA_DIR "/data/misc/stats-data"
#define STATS_ACTIVE_METRIC_DIR "/data/misc/stats-active-metric"
// Cool down period for writing data to disk to avoid overwriting files.
@@ -106,6 +105,19 @@
StatsLogProcessor::~StatsLogProcessor() {
}
+static void flushProtoToBuffer(ProtoOutputStream& proto, vector<uint8_t>* outData) {
+ outData->clear();
+ outData->resize(proto.size());
+ size_t pos = 0;
+ sp<android::util::ProtoReader> reader = proto.data();
+ while (reader->readBuffer() != NULL) {
+ size_t toRead = reader->currentToRead();
+ std::memcpy(&((*outData)[pos]), reader->readBuffer(), toRead);
+ pos += toRead;
+ reader->move(toRead);
+ }
+}
+
void StatsLogProcessor::onAnomalyAlarmFired(
const int64_t& timestampNs,
unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet) {
@@ -366,25 +378,29 @@
proto->end(configKeyToken);
// End of ConfigKey.
+ bool keepFile = false;
+ auto it = mMetricsManagers.find(key);
+ if (it != mMetricsManagers.end() && it->second->shouldPersistLocalHistory()) {
+ keepFile = true;
+ }
+
// Then, check stats-data directory to see there's any file containing
// ConfigMetricsReport from previous shutdowns to concatenate to reports.
- StorageManager::appendConfigMetricsReport(key, proto, erase_data);
+ StorageManager::appendConfigMetricsReport(
+ key, proto, erase_data && !keepFile /* should remove file after appending it */,
+ dumpReportReason == ADB_DUMP /*if caller is adb*/);
- auto it = mMetricsManagers.find(key);
if (it != mMetricsManagers.end()) {
// This allows another broadcast to be sent within the rate-limit period if we get close to
// filling the buffer again soon.
mLastBroadcastTimes.erase(key);
- // Start of ConfigMetricsReport (reports).
- uint64_t reportsToken =
- proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS);
- onConfigMetricsReportLocked(key, dumpTimeStampNs,
- include_current_partial_bucket,
- erase_data, dumpReportReason,
- dumpLatency, proto);
- proto->end(reportsToken);
- // End of ConfigMetricsReport (reports).
+ vector<uint8_t> buffer;
+ onConfigMetricsReportLocked(key, dumpTimeStampNs, include_current_partial_bucket,
+ erase_data, dumpReportReason, dumpLatency,
+ false /* is this data going to be saved on disk */, &buffer);
+ proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS,
+ reinterpret_cast<char*>(buffer.data()), buffer.size());
} else {
ALOGW("Config source %s does not exist", key.ToString().c_str());
}
@@ -404,16 +420,8 @@
dumpReportReason, dumpLatency, &proto);
if (outData != nullptr) {
- outData->clear();
- outData->resize(proto.size());
- size_t pos = 0;
- sp<android::util::ProtoReader> reader = proto.data();
- while (reader->readBuffer() != NULL) {
- size_t toRead = reader->currentToRead();
- std::memcpy(&((*outData)[pos]), reader->readBuffer(), toRead);
- pos += toRead;
- reader->move(toRead);
- }
+ flushProtoToBuffer(proto, outData);
+ VLOG("output data size %zu", outData->size());
}
StatsdStats::getInstance().noteMetricsReportSent(key, proto.size());
@@ -422,13 +430,11 @@
/*
* onConfigMetricsReportLocked dumps serialized ConfigMetricsReport into outData.
*/
-void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,
- const int64_t dumpTimeStampNs,
- const bool include_current_partial_bucket,
- const bool erase_data,
- const DumpReportReason dumpReportReason,
- const DumpLatency dumpLatency,
- ProtoOutputStream* proto) {
+void StatsLogProcessor::onConfigMetricsReportLocked(
+ const ConfigKey& key, const int64_t dumpTimeStampNs,
+ const bool include_current_partial_bucket, const bool erase_data,
+ const DumpReportReason dumpReportReason, const DumpLatency dumpLatency,
+ const bool dataSavedOnDisk, vector<uint8_t>* buffer) {
// We already checked whether key exists in mMetricsManagers in
// WriteDataToDisk.
auto it = mMetricsManagers.find(key);
@@ -440,35 +446,46 @@
std::set<string> str_set;
+ ProtoOutputStream tempProto;
// First, fill in ConfigMetricsReport using current data on memory, which
// starts from filling in StatsLogReport's.
- it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket,
- erase_data, dumpLatency, &str_set, proto);
+ it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, erase_data,
+ dumpLatency, &str_set, &tempProto);
// Fill in UidMap if there is at least one metric to report.
// This skips the uid map if it's an empty config.
if (it->second->getNumMetrics() > 0) {
- uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP);
+ uint64_t uidMapToken = tempProto.start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP);
mUidMap->appendUidMap(
dumpTimeStampNs, key, it->second->hashStringInReport() ? &str_set : nullptr,
- it->second->versionStringsInReport(), it->second->installerInReport(), proto);
- proto->end(uidMapToken);
+ it->second->versionStringsInReport(), it->second->installerInReport(), &tempProto);
+ tempProto.end(uidMapToken);
}
// Fill in the timestamps.
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
- (long long)lastReportTimeNs);
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
- (long long)dumpTimeStampNs);
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS,
- (long long)lastReportWallClockNs);
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
- (long long)getWallClockNs());
+ tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
+ (long long)lastReportTimeNs);
+ tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
+ (long long)dumpTimeStampNs);
+ tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS,
+ (long long)lastReportWallClockNs);
+ tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
+ (long long)getWallClockNs());
// Dump report reason
- proto->write(FIELD_TYPE_INT32 | FIELD_ID_DUMP_REPORT_REASON, dumpReportReason);
+ tempProto.write(FIELD_TYPE_INT32 | FIELD_ID_DUMP_REPORT_REASON, dumpReportReason);
for (const auto& str : str_set) {
- proto->write(FIELD_TYPE_STRING | FIELD_COUNT_REPEATED | FIELD_ID_STRINGS, str);
+ tempProto.write(FIELD_TYPE_STRING | FIELD_COUNT_REPEATED | FIELD_ID_STRINGS, str);
+ }
+
+ flushProtoToBuffer(tempProto, buffer);
+
+ // save buffer to disk if needed
+ if (erase_data && !dataSavedOnDisk && it->second->shouldPersistLocalHistory()) {
+ VLOG("save history to disk");
+ string file_name = StorageManager::getDataHistoryFileName((long)getWallClockSec(),
+ key.GetUid(), key.GetId());
+ StorageManager::writeFile(file_name.c_str(), buffer->data(), buffer->size());
}
}
@@ -584,18 +601,14 @@
!mMetricsManagers.find(key)->second->shouldWriteToDisk()) {
return;
}
- ProtoOutputStream proto;
+ vector<uint8_t> buffer;
onConfigMetricsReportLocked(key, timestampNs, true /* include_current_partial_bucket*/,
- true /* erase_data */, dumpReportReason, dumpLatency, &proto);
- string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,
- (long)getWallClockSec(), key.GetUid(), (long long)key.GetId());
- android::base::unique_fd fd(open(file_name.c_str(),
- O_WRONLY | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR));
- if (fd == -1) {
- ALOGE("Attempt to write %s but failed", file_name.c_str());
- return;
- }
- proto.flush(fd.get());
+ true /* erase_data */, dumpReportReason, dumpLatency, true,
+ &buffer);
+ string file_name =
+ StorageManager::getDataFileName((long)getWallClockSec(), key.GetUid(), key.GetId());
+ StorageManager::writeFile(file_name.c_str(), buffer.data(), buffer.size());
+
// We were able to write the ConfigMetricsReport to disk, so we should trigger collection ASAP.
mOnDiskDataConfigs.insert(key);
}
diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h
index 305a4ce..f4db0af 100644
--- a/cmds/statsd/src/StatsLogProcessor.h
+++ b/cmds/statsd/src/StatsLogProcessor.h
@@ -164,12 +164,13 @@
const DumpReportReason dumpReportReason,
const DumpLatency dumpLatency);
- void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs,
- const bool include_current_partial_bucket,
- const bool erase_data,
- const DumpReportReason dumpReportReason,
- const DumpLatency dumpLatency,
- util::ProtoOutputStream* proto);
+ void onConfigMetricsReportLocked(
+ const ConfigKey& key, const int64_t dumpTimeStampNs,
+ const bool include_current_partial_bucket, const bool erase_data,
+ const DumpReportReason dumpReportReason, const DumpLatency dumpLatency,
+ /*if dataSavedToDisk is true, it indicates the caller will write the data to disk
+ (e.g., before reboot). So no need to further persist local history.*/
+ const bool dataSavedToDisk, vector<uint8_t>* proto);
/* Check if we should send a broadcast if approaching memory limits and if we're over, we
* actually delete the data. */
diff --git a/cmds/statsd/src/guardrail/StatsdStats.h b/cmds/statsd/src/guardrail/StatsdStats.h
index 88ecccc..194001a1 100644
--- a/cmds/statsd/src/guardrail/StatsdStats.h
+++ b/cmds/statsd/src/guardrail/StatsdStats.h
@@ -145,6 +145,9 @@
// Maximum age (30 days) that files on disk can exist in seconds.
static const int kMaxAgeSecond = 60 * 60 * 24 * 30;
+ // Maximum age (2 days) that local history files on disk can exist in seconds.
+ static const int kMaxLocalHistoryAgeSecond = 60 * 60 * 24 * 2;
+
// Maximum number of files (1000) that can be in stats directory on disk.
static const int kMaxFileNumber = 1000;
diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp
index 095f9dd..6a55289 100644
--- a/cmds/statsd/src/metrics/MetricsManager.cpp
+++ b/cmds/statsd/src/metrics/MetricsManager.cpp
@@ -65,7 +65,8 @@
mTtlNs(config.has_ttl_in_seconds() ? config.ttl_in_seconds() * NS_PER_SEC : -1),
mTtlEndNs(-1),
mLastReportTimeNs(currentTimeNs),
- mLastReportWallClockNs(getWallClockNs()) {
+ mLastReportWallClockNs(getWallClockNs()),
+ mShouldPersistHistory(config.persist_locally()) {
// Init the ttl end timestamp.
refreshTtl(timeBaseNs);
diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h
index d317f8e..00ae3b7 100644
--- a/cmds/statsd/src/metrics/MetricsManager.h
+++ b/cmds/statsd/src/metrics/MetricsManager.h
@@ -78,6 +78,10 @@
return mNoReportMetricIds.size() != mAllMetricProducers.size();
}
+ bool shouldPersistLocalHistory() const {
+ return mShouldPersistHistory;
+ }
+
void dumpStates(FILE* out, bool verbose);
inline bool isInTtl(const int64_t timestampNs) const {
@@ -184,6 +188,8 @@
// Contains the annotations passed in with StatsdConfig.
std::list<std::pair<const int64_t, const int32_t>> mAnnotations;
+ const bool mShouldPersistHistory;
+
// To guard access to mAllowedLogSources
mutable std::mutex mAllowedLogSourcesMutex;
diff --git a/cmds/statsd/src/statsd_config.proto b/cmds/statsd/src/statsd_config.proto
index 257e65e..2260b9b 100644
--- a/cmds/statsd/src/statsd_config.proto
+++ b/cmds/statsd/src/statsd_config.proto
@@ -439,6 +439,8 @@
optional bool installer_in_metric_report = 19;
+ optional bool persist_locally = 20 [default = false];
+
// Field number 1000 is reserved for later use.
reserved 1000;
}
diff --git a/cmds/statsd/src/storage/StorageManager.cpp b/cmds/statsd/src/storage/StorageManager.cpp
index cf8b974..0a9161d 100644
--- a/cmds/statsd/src/storage/StorageManager.cpp
+++ b/cmds/statsd/src/storage/StorageManager.cpp
@@ -56,9 +56,31 @@
using android::base::StringPrintf;
using std::unique_ptr;
-// Returns array of int64_t which contains timestamp in seconds, uid, and
-// configID.
-static void parseFileName(char* name, int64_t* result) {
+struct FileName {
+ int64_t mTimestampSec;
+ int mUid;
+ int64_t mConfigId;
+ bool mIsHistory;
+ string getFullFileName(const char* path) {
+ return StringPrintf("%s/%lld_%d_%lld%s", path, (long long)mTimestampSec, (int)mUid,
+ (long long)mConfigId, (mIsHistory ? "_history" : ""));
+ };
+};
+
+string StorageManager::getDataFileName(long wallClockSec, int uid, int64_t id) {
+ return StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, wallClockSec, uid,
+ (long long)id);
+}
+
+string StorageManager::getDataHistoryFileName(long wallClockSec, int uid, int64_t id) {
+ return StringPrintf("%s/%ld_%d_%lld_history", STATS_DATA_DIR, wallClockSec, uid,
+ (long long)id);
+}
+
+// Returns array of int64_t which contains timestamp in seconds, uid,
+// configID and whether the file is a local history file.
+static void parseFileName(char* name, FileName* output) {
+ int64_t result[3];
int index = 0;
char* substr = strtok(name, "_");
while (substr != nullptr && index < 3) {
@@ -72,11 +94,12 @@
if (index < 3) {
result[0] = -1;
}
-}
-static string getFilePath(const char* path, int64_t timestamp, int64_t uid, int64_t configID) {
- return StringPrintf("%s/%lld_%d_%lld", path, (long long)timestamp, (int)uid,
- (long long)configID);
+ output->mTimestampSec = result[0];
+ output->mUid = result[1];
+ output->mConfigId = result[2];
+ // check if the file is a local history.
+ output->mIsHistory = (substr != nullptr && strcmp("history", substr) == 0);
}
void StorageManager::writeFile(const char* file, const void* buffer, int numBytes) {
@@ -88,14 +111,13 @@
trimToFit(STATS_SERVICE_DIR);
trimToFit(STATS_DATA_DIR);
- int result = write(fd, buffer, numBytes);
- if (result == numBytes) {
+ if (android::base::WriteFully(fd, buffer, numBytes)) {
VLOG("Successfully wrote %s", file);
} else {
- VLOG("Failed to write %s", file);
+ ALOGE("Failed to write %s", file);
}
- result = fchown(fd, AID_STATSD, AID_STATSD);
+ int result = fchown(fd, AID_STATSD, AID_STATSD);
if (result) {
VLOG("Failed to chown %s to statsd", file);
}
@@ -349,13 +371,10 @@
if (name[0] == '.') continue;
VLOG("file %s", name);
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t uid = result[1];
- int64_t configID = result[2];
-
- sendBroadcast(ConfigKey((int)uid, configID));
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1 || output.mIsHistory) continue;
+ sendBroadcast(ConfigKey((int)output.mUid, output.mConfigId));
}
}
@@ -378,55 +397,58 @@
if (suffixLen <= nameLen &&
strncmp(name + nameLen - suffixLen, suffix.c_str(), suffixLen) == 0) {
// Check again that the file name is parseable.
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1 || output.mIsHistory) continue;
return true;
}
}
return false;
}
-void StorageManager::appendConfigMetricsReport(const ConfigKey& key,
- ProtoOutputStream* proto,
- bool erasa_data) {
+void StorageManager::appendConfigMetricsReport(const ConfigKey& key, ProtoOutputStream* proto,
+ bool erase_data, bool isAdb) {
unique_ptr<DIR, decltype(&closedir)> dir(opendir(STATS_DATA_DIR), closedir);
if (dir == NULL) {
VLOG("Path %s does not exist", STATS_DATA_DIR);
return;
}
- string suffix = StringPrintf("%d_%lld", key.GetUid(), (long long)key.GetId());
-
dirent* de;
while ((de = readdir(dir.get()))) {
char* name = de->d_name;
+ string fileName(name);
if (name[0] == '.') continue;
+ FileName output;
+ parseFileName(name, &output);
- size_t nameLen = strlen(name);
- size_t suffixLen = suffix.length();
- if (suffixLen <= nameLen &&
- strncmp(name + nameLen - suffixLen, suffix.c_str(), suffixLen) == 0) {
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t timestamp = result[0];
- int64_t uid = result[1];
- int64_t configID = result[2];
+ if (output.mTimestampSec == -1 || (output.mIsHistory && !isAdb) ||
+ output.mUid != key.GetUid() || output.mConfigId != key.GetId()) {
+ continue;
+ }
- string file_name = getFilePath(STATS_DATA_DIR, timestamp, uid, configID);
- int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC);
- if (fd != -1) {
- string content;
- if (android::base::ReadFdToString(fd, &content)) {
- proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS,
- content.c_str(), content.size());
- }
- close(fd);
+ auto fullPathName = StringPrintf("%s/%s", STATS_DATA_DIR, fileName.c_str());
+ int fd = open(fullPathName.c_str(), O_RDONLY | O_CLOEXEC);
+ if (fd != -1) {
+ string content;
+ if (android::base::ReadFdToString(fd, &content)) {
+ proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS,
+ content.c_str(), content.size());
}
+ close(fd);
+ } else {
+ ALOGE("file cannot be opened");
+ }
- if (erasa_data) {
- remove(file_name.c_str());
+ if (erase_data) {
+ remove(fullPathName.c_str());
+ } else if (output.mIsHistory && !isAdb) {
+ // This means a real data owner has called to get this data. But the config says it
+ // wants to keep a local history. So now this file must be renamed as a history file.
+ // So that next time, when owner calls getData() again, this data won't be uploaded
+ // again. rename returns 0 on success
+ if (rename(fullPathName.c_str(), (fullPathName + "_history").c_str())) {
+ ALOGE("Failed to rename file %s", fullPathName.c_str());
}
}
}
@@ -458,23 +480,20 @@
while ((de = readdir(dir.get()))) {
char* name = de->d_name;
if (name[0] == '.') continue;
- VLOG("file %s", name);
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t timestamp = result[0];
- int64_t uid = result[1];
- int64_t configID = result[2];
- string file_name = getFilePath(STATS_SERVICE_DIR, timestamp, uid, configID);
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1) continue;
+ string file_name = output.getFullFileName(STATS_SERVICE_DIR);
int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC);
if (fd != -1) {
string content;
if (android::base::ReadFdToString(fd, &content)) {
StatsdConfig config;
if (config.ParseFromString(content)) {
- configsMap[ConfigKey(uid, configID)] = config;
- VLOG("map key uid=%lld|configID=%lld", (long long)uid, (long long)configID);
+ configsMap[ConfigKey(output.mUid, output.mConfigId)] = config;
+ VLOG("map key uid=%lld|configID=%lld", (long long)output.mUid,
+ (long long)output.mConfigId);
}
}
close(fd);
@@ -533,6 +552,30 @@
return false;
}
+void StorageManager::sortFiles(vector<FileInfo>* fileNames) {
+ // Reverse sort to effectively remove from the back (oldest entries).
+ // This will sort files in reverse-chronological order. Local history files have lower
+ // priority than regular data files.
+ sort(fileNames->begin(), fileNames->end(), [](FileInfo& lhs, FileInfo& rhs) {
+ // first consider if the file is a local history
+ if (lhs.mIsHistory && !rhs.mIsHistory) {
+ return false;
+ } else if (rhs.mIsHistory && !lhs.mIsHistory) {
+ return true;
+ }
+
+ // then consider the age.
+ if (lhs.mFileAgeSec < rhs.mFileAgeSec) {
+ return true;
+ } else if (lhs.mFileAgeSec > rhs.mFileAgeSec) {
+ return false;
+ }
+
+ // then good luck.... use string::compare
+ return lhs.mFileName.compare(rhs.mFileName) > 0;
+ });
+}
+
void StorageManager::trimToFit(const char* path) {
unique_ptr<DIR, decltype(&closedir)> dir(opendir(path), closedir);
if (dir == NULL) {
@@ -541,55 +584,46 @@
}
dirent* de;
int totalFileSize = 0;
- vector<string> fileNames;
+ vector<FileInfo> fileNames;
+ auto nowSec = getWallClockSec();
while ((de = readdir(dir.get()))) {
char* name = de->d_name;
if (name[0] == '.') continue;
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t timestamp = result[0];
- int64_t uid = result[1];
- int64_t configID = result[2];
- string file_name = getFilePath(path, timestamp, uid, configID);
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1) continue;
+ string file_name = output.getFullFileName(path);
// Check for timestamp and delete if it's too old.
- long fileAge = getWallClockSec() - timestamp;
- if (fileAge > StatsdStats::kMaxAgeSecond) {
+ long fileAge = nowSec - output.mTimestampSec;
+ if (fileAge > StatsdStats::kMaxAgeSecond ||
+ (output.mIsHistory && fileAge > StatsdStats::kMaxLocalHistoryAgeSecond)) {
deleteFile(file_name.c_str());
+ continue;
}
- fileNames.push_back(file_name);
ifstream file(file_name.c_str(), ifstream::in | ifstream::binary);
+ int fileSize = 0;
if (file.is_open()) {
file.seekg(0, ios::end);
- int fileSize = file.tellg();
+ fileSize = file.tellg();
file.close();
totalFileSize += fileSize;
}
+ fileNames.emplace_back(file_name, output.mIsHistory, fileSize, fileAge);
}
if (fileNames.size() > StatsdStats::kMaxFileNumber ||
totalFileSize > StatsdStats::kMaxFileSize) {
- // Reverse sort to effectively remove from the back (oldest entries).
- // This will sort files in reverse-chronological order.
- sort(fileNames.begin(), fileNames.end(), std::greater<std::string>());
+ sortFiles(&fileNames);
}
// Start removing files from oldest to be under the limit.
while (fileNames.size() > 0 && (fileNames.size() > StatsdStats::kMaxFileNumber ||
totalFileSize > StatsdStats::kMaxFileSize)) {
- string file_name = fileNames.at(fileNames.size() - 1);
- ifstream file(file_name.c_str(), ifstream::in | ifstream::binary);
- if (file.is_open()) {
- file.seekg(0, ios::end);
- int fileSize = file.tellg();
- file.close();
- totalFileSize -= fileSize;
- }
-
- deleteFile(file_name.c_str());
+ totalFileSize -= fileNames.at(fileNames.size() - 1).mFileSizeBytes;
+ deleteFile(fileNames.at(fileNames.size() - 1).mFileName.c_str());
fileNames.pop_back();
}
}
@@ -614,15 +648,13 @@
if (name[0] == '.') {
continue;
}
- int64_t result[3];
- parseFileName(name, result);
- if (result[0] == -1) continue;
- int64_t timestamp = result[0];
- int64_t uid = result[1];
- int64_t configID = result[2];
- dprintf(outFd, "\t #%d, Last updated: %lld, UID: %d, Config ID: %lld", fileCount + 1,
- (long long)timestamp, (int)uid, (long long)configID);
- string file_name = getFilePath(path, timestamp, uid, configID);
+ FileName output;
+ parseFileName(name, &output);
+ if (output.mTimestampSec == -1) continue;
+ dprintf(outFd, "\t #%d, Last updated: %lld, UID: %d, Config ID: %lld, %s", fileCount + 1,
+ (long long)output.mTimestampSec, output.mUid, (long long)output.mConfigId,
+ (output.mIsHistory ? "local history" : ""));
+ string file_name = output.getFullFileName(path);
ifstream file(file_name.c_str(), ifstream::in | ifstream::binary);
if (file.is_open()) {
file.seekg(0, ios::end);
diff --git a/cmds/statsd/src/storage/StorageManager.h b/cmds/statsd/src/storage/StorageManager.h
index dfcea65..69b41c2 100644
--- a/cmds/statsd/src/storage/StorageManager.h
+++ b/cmds/statsd/src/storage/StorageManager.h
@@ -31,6 +31,19 @@
class StorageManager : public virtual RefBase {
public:
+ struct FileInfo {
+ FileInfo(std::string name, bool isHistory, int fileSize, long fileAge)
+ : mFileName(name),
+ mIsHistory(isHistory),
+ mFileSizeBytes(fileSize),
+ mFileAgeSec(fileAge) {
+ }
+ std::string mFileName;
+ bool mIsHistory;
+ int mFileSizeBytes;
+ long mFileAgeSec;
+ };
+
/**
* Writes a given byte array as a file to the specified file path.
*/
@@ -81,10 +94,19 @@
/**
* Appends the ConfigMetricsReport found on disk to the specifid proto
* and, if erase_data, deletes it from disk.
+ *
+ * [isAdb]: if the caller is adb dump. This includes local adb dump or dumpsys by
+ * bugreport or incidentd. When true, we will append any local history data too.
+ *
+ * When
+ * erase_data=true, isAdb=true: append history data to output, remove all data after read
+ * erase_data=false, isAdb=true: append history data to output, keep data after read
+ * erase_data=true, isAdb=false: do not append history data, and remove data after read
+ * erase_data=false, isAdb=false: do not append history data and *rename* all data files to
+ * history files.
*/
- static void appendConfigMetricsReport(const ConfigKey& key,
- ProtoOutputStream* proto,
- bool erase_data);
+ static void appendConfigMetricsReport(const ConfigKey& key, ProtoOutputStream* proto,
+ bool erase_data, bool isAdb);
/**
* Call to load the saved configs from disk.
@@ -115,6 +137,12 @@
*/
static void printStats(int out);
+ static string getDataFileName(long wallClockSec, int uid, int64_t id);
+
+ static string getDataHistoryFileName(long wallClockSec, int uid, int64_t id);
+
+ static void sortFiles(vector<FileInfo>* fileNames);
+
private:
/**
* Prints disk usage statistics about a directory related to statsd.