Add API to let metrics directly drop data without writing to an output.
+ Metrics will do flushIfNeeded() to correctly move the clock and informing
AnomalyTracker the past bucket info, and then clear past buckets.
+ We will still keep the current bucket data for the validity of the future metrics.
Bug: 70571383
Test: statsd_test
Change-Id: Ib13c45574974e7b4e82bd8f305091dc93bda76f5
diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp
index 9b58a14..e98b54d 100644
--- a/cmds/statsd/src/StatsLogProcessor.cpp
+++ b/cmds/statsd/src/StatsLogProcessor.cpp
@@ -330,9 +330,7 @@
mLastByteSizeTimes[key] = timestampNs;
if (totalBytes >
StatsdStats::kMaxMetricsBytesPerConfig) { // Too late. We need to start clearing data.
- // TODO(b/70571383): By 12/15/2017 add API to drop data directly
- ProtoOutputStream proto;
- metricsManager.onDumpReport(timestampNs, &proto);
+ metricsManager.dropData(timestampNs);
StatsdStats::getInstance().noteDataDropped(key);
VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str());
} else if (totalBytes > .9 * StatsdStats::kMaxMetricsBytesPerConfig) {
diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h
index 7a6aa1e..a672ab4 100644
--- a/cmds/statsd/src/StatsLogProcessor.h
+++ b/cmds/statsd/src/StatsLogProcessor.h
@@ -113,7 +113,6 @@
FRIEND_TEST(StatsLogProcessorTest, TestRateLimitByteSize);
FRIEND_TEST(StatsLogProcessorTest, TestRateLimitBroadcast);
FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge);
- FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge);
FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration1);
FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration2);
FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration3);
diff --git a/cmds/statsd/src/metrics/CountMetricProducer.cpp b/cmds/statsd/src/metrics/CountMetricProducer.cpp
index af2e362..671c57f 100644
--- a/cmds/statsd/src/metrics/CountMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/CountMetricProducer.cpp
@@ -151,8 +151,11 @@
protoOutput->end(protoToken);
mPastBuckets.clear();
+}
- // TODO: Clear mDimensionKeyMap once the report is dumped.
+void CountMetricProducer::dropDataLocked(const uint64_t dropTimeNs) {
+ flushIfNeededLocked(dropTimeNs);
+ mPastBuckets.clear();
}
void CountMetricProducer::onConditionChangedLocked(const bool conditionMet,
diff --git a/cmds/statsd/src/metrics/CountMetricProducer.h b/cmds/statsd/src/metrics/CountMetricProducer.h
index 0c4291d..1d8e42b 100644
--- a/cmds/statsd/src/metrics/CountMetricProducer.h
+++ b/cmds/statsd/src/metrics/CountMetricProducer.h
@@ -69,6 +69,8 @@
void dumpStatesLocked(FILE* out, bool verbose) const override{};
+ void dropDataLocked(const uint64_t dropTimeNs) override;
+
// Util function to flush the old packet.
void flushIfNeededLocked(const uint64_t& newEventTime) override;
diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.cpp b/cmds/statsd/src/metrics/DurationMetricProducer.cpp
index 80329c3..ac39662 100644
--- a/cmds/statsd/src/metrics/DurationMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/DurationMetricProducer.cpp
@@ -221,6 +221,11 @@
}
}
+void DurationMetricProducer::dropDataLocked(const uint64_t dropTimeNs) {
+ flushIfNeededLocked(dropTimeNs);
+ mPastBuckets.clear();
+}
+
void DurationMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs,
ProtoOutputStream* protoOutput) {
flushIfNeededLocked(dumpTimeNs);
diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.h b/cmds/statsd/src/metrics/DurationMetricProducer.h
index 73d074f..23408a7 100644
--- a/cmds/statsd/src/metrics/DurationMetricProducer.h
+++ b/cmds/statsd/src/metrics/DurationMetricProducer.h
@@ -74,6 +74,8 @@
void dumpStatesLocked(FILE* out, bool verbose) const override;
+ void dropDataLocked(const uint64_t dropTimeNs) override;
+
// Util function to flush the old packet.
void flushIfNeededLocked(const uint64_t& eventTime);
diff --git a/cmds/statsd/src/metrics/EventMetricProducer.cpp b/cmds/statsd/src/metrics/EventMetricProducer.cpp
index 96d0cfc..fff91f6 100644
--- a/cmds/statsd/src/metrics/EventMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/EventMetricProducer.cpp
@@ -75,6 +75,10 @@
VLOG("~EventMetricProducer() called");
}
+void EventMetricProducer::dropDataLocked(const uint64_t dropTimeNs) {
+ mProto->clear();
+}
+
void EventMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eventTime) {
}
diff --git a/cmds/statsd/src/metrics/EventMetricProducer.h b/cmds/statsd/src/metrics/EventMetricProducer.h
index 3f2c5a5..d6f81fd 100644
--- a/cmds/statsd/src/metrics/EventMetricProducer.h
+++ b/cmds/statsd/src/metrics/EventMetricProducer.h
@@ -55,6 +55,8 @@
// Internal interface to handle sliced condition change.
void onSlicedConditionMayChangeLocked(const uint64_t eventTime) override;
+ void dropDataLocked(const uint64_t dropTimeNs) override;
+
// Internal function to calculate the current used bytes.
size_t byteSizeLocked() const override;
diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp
index 7d09ff9..000c7a7 100644
--- a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp
@@ -353,6 +353,11 @@
}
}
+void GaugeMetricProducer::dropDataLocked(const uint64_t dropTimeNs) {
+ flushIfNeededLocked(dropTimeNs);
+ mPastBuckets.clear();
+}
+
// When a new matched event comes in, we check if event falls into the current
// bucket. If not, flush the old counter to past buckets and initialize the new
// bucket.
diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.h b/cmds/statsd/src/metrics/GaugeMetricProducer.h
index c3ae6ce..ca8dc75 100644
--- a/cmds/statsd/src/metrics/GaugeMetricProducer.h
+++ b/cmds/statsd/src/metrics/GaugeMetricProducer.h
@@ -108,6 +108,8 @@
void dumpStatesLocked(FILE* out, bool verbose) const override{};
+ void dropDataLocked(const uint64_t dropTimeNs) override;
+
// Util function to flush the old packet.
void flushIfNeededLocked(const uint64_t& eventTime) override;
diff --git a/cmds/statsd/src/metrics/MetricProducer.h b/cmds/statsd/src/metrics/MetricProducer.h
index 2bf6241..05b7f87 100644
--- a/cmds/statsd/src/metrics/MetricProducer.h
+++ b/cmds/statsd/src/metrics/MetricProducer.h
@@ -148,6 +148,15 @@
return mMetricId;
}
+ // Let MetricProducer drop in-memory data to save memory.
+ // We still need to keep future data valid and anomaly tracking work, which means we will
+ // have to flush old data, informing anomaly trackers then safely drop old data.
+ // We still keep current bucket data for future metrics' validity.
+ void dropData(const uint64_t dropTimeNs) {
+ std::lock_guard<std::mutex> lock(mMutex);
+ dropDataLocked(dropTimeNs);
+ }
+
protected:
virtual void onConditionChangedLocked(const bool condition, const uint64_t eventTime) = 0;
virtual void onSlicedConditionMayChangeLocked(const uint64_t eventTime) = 0;
@@ -179,6 +188,8 @@
return mStartTimeNs + (mCurrentBucketNum + 1) * mBucketSizeNs;
}
+ virtual void dropDataLocked(const uint64_t dropTimeNs) = 0;
+
const int64_t mMetricId;
const ConfigKey mConfigKey;
diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp
index 4c8a7d8..0693031 100644
--- a/cmds/statsd/src/metrics/MetricsManager.cpp
+++ b/cmds/statsd/src/metrics/MetricsManager.cpp
@@ -173,6 +173,12 @@
}
}
+void MetricsManager::dropData(const uint64_t dropTimeNs) {
+ for (const auto& producer : mAllMetricProducers) {
+ producer->dropData(dropTimeNs);
+ }
+}
+
void MetricsManager::onDumpReport(const uint64_t dumpTimeStampNs, ProtoOutputStream* protoOutput) {
VLOG("=========================Metric Reports Start==========================");
// one StatsLogReport per MetricProduer
diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h
index b50ef4a..a32e037 100644
--- a/cmds/statsd/src/metrics/MetricsManager.h
+++ b/cmds/statsd/src/metrics/MetricsManager.h
@@ -73,6 +73,8 @@
return mLastReportTimeNs;
};
+ virtual void dropData(const uint64_t dropTimeNs);
+
// Config source owner can call onDumpReport() to get all the metrics collected.
virtual void onDumpReport(const uint64_t dumpTimeNs,
android::util::ProtoOutputStream* protoOutput);
diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.cpp b/cmds/statsd/src/metrics/ValueMetricProducer.cpp
index 35fcdc4..f6bba3d 100644
--- a/cmds/statsd/src/metrics/ValueMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/ValueMetricProducer.cpp
@@ -129,6 +129,11 @@
VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId);
}
+void ValueMetricProducer::dropDataLocked(const uint64_t dropTimeNs) {
+ flushIfNeededLocked(dropTimeNs);
+ mPastBuckets.clear();
+}
+
void ValueMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs,
ProtoOutputStream* protoOutput) {
VLOG("metric %lld dump report now...", (long long)mMetricId);
diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.h b/cmds/statsd/src/metrics/ValueMetricProducer.h
index b518f2f..5e42bd2 100644
--- a/cmds/statsd/src/metrics/ValueMetricProducer.h
+++ b/cmds/statsd/src/metrics/ValueMetricProducer.h
@@ -106,6 +106,8 @@
void flushCurrentBucketLocked(const uint64_t& eventTimeNs) override;
+ void dropDataLocked(const uint64_t dropTimeNs) override;
+
const FieldMatcher mValueField;
std::shared_ptr<StatsPullerManager> mStatsPullerManager;
diff --git a/cmds/statsd/tests/StatsLogProcessor_test.cpp b/cmds/statsd/tests/StatsLogProcessor_test.cpp
index 3238b74..efed42e 100644
--- a/cmds/statsd/tests/StatsLogProcessor_test.cpp
+++ b/cmds/statsd/tests/StatsLogProcessor_test.cpp
@@ -51,7 +51,8 @@
}
MOCK_METHOD0(byteSize, size_t());
- MOCK_METHOD2(onDumpReport, void(const uint64_t timeNs, ProtoOutputStream* output));
+
+ MOCK_METHOD1(dropData, void(const uint64_t dropTimeNs));
};
TEST(StatsLogProcessorTest, TestRateLimitByteSize) {
@@ -114,7 +115,7 @@
.Times(1)
.WillRepeatedly(Return(int(StatsdStats::kMaxMetricsBytesPerConfig * 1.2)));
- EXPECT_CALL(mockMetricsManager, onDumpReport(_, _)).Times(1);
+ EXPECT_CALL(mockMetricsManager, dropData(_)).Times(1);
// Expect to call the onDumpReport and skip the broadcast.
p.flushIfNecessaryLocked(1, key, mockMetricsManager);