blob: c33d1ac243bfb34c52a6198eba25248d4413c9c4 [file] [log] [blame]
Chenjie Yub3dda412017-10-24 13:41:59 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Chenjie Yu88588972018-08-03 09:49:22 -070017#define DEBUG false // STOPSHIP if true
Chenjie Yub3dda412017-10-24 13:41:59 -070018#include "Log.h"
19
20#include "ValueMetricProducer.h"
Chenjie Yuc5875052018-03-09 10:13:11 -080021#include "../guardrail/StatsdStats.h"
22#include "../stats_log_util.h"
Chenjie Yub3dda412017-10-24 13:41:59 -070023
Chenjie Yub3dda412017-10-24 13:41:59 -070024#include <limits.h>
25#include <stdlib.h>
26
yrob0378b02017-11-09 20:36:25 -080027using android::util::FIELD_COUNT_REPEATED;
yro2b0f8862017-11-06 14:27:31 -080028using android::util::FIELD_TYPE_BOOL;
Chenjie Yua0f02242018-07-06 16:14:34 -070029using android::util::FIELD_TYPE_DOUBLE;
yro2b0f8862017-11-06 14:27:31 -080030using android::util::FIELD_TYPE_INT32;
31using android::util::FIELD_TYPE_INT64;
32using android::util::FIELD_TYPE_MESSAGE;
Yangster-macd1815dc2017-11-13 21:43:15 -080033using android::util::FIELD_TYPE_STRING;
yro2b0f8862017-11-06 14:27:31 -080034using android::util::ProtoOutputStream;
Yao Chen93fe3a32017-11-02 13:52:59 -070035using std::map;
Chenjie Yub3dda412017-10-24 13:41:59 -070036using std::shared_ptr;
Yao Chen93fe3a32017-11-02 13:52:59 -070037using std::unordered_map;
Chenjie Yub3dda412017-10-24 13:41:59 -070038
39namespace android {
40namespace os {
41namespace statsd {
42
yro2b0f8862017-11-06 14:27:31 -080043// for StatsLogReport
Yangster-mac94e197c2018-01-02 16:03:03 -080044const int FIELD_ID_ID = 1;
yro2b0f8862017-11-06 14:27:31 -080045const int FIELD_ID_VALUE_METRICS = 7;
Yangster-mac9def8e32018-04-17 13:55:51 -070046const int FIELD_ID_TIME_BASE = 9;
47const int FIELD_ID_BUCKET_SIZE = 10;
48const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
Howard Ro9440e092018-12-16 19:15:21 -080049const int FIELD_ID_IS_ACTIVE = 14;
yro2b0f8862017-11-06 14:27:31 -080050// for ValueMetricDataWrapper
51const int FIELD_ID_DATA = 1;
David Chen81245fd2018-04-12 14:33:37 -070052const int FIELD_ID_SKIPPED = 2;
tsaichristineb87ca152019-12-09 15:19:41 -080053// for SkippedBuckets
Yangster-mac9def8e32018-04-17 13:55:51 -070054const int FIELD_ID_SKIPPED_START_MILLIS = 3;
55const int FIELD_ID_SKIPPED_END_MILLIS = 4;
tsaichristineb87ca152019-12-09 15:19:41 -080056const int FIELD_ID_SKIPPED_DROP_EVENT = 5;
57// for DumpEvent Proto
58const int FIELD_ID_BUCKET_DROP_REASON = 1;
59const int FIELD_ID_DROP_TIME = 2;
yro2b0f8862017-11-06 14:27:31 -080060// for ValueMetricData
Yangster-mac468ff042018-01-17 12:26:34 -080061const int FIELD_ID_DIMENSION_IN_WHAT = 1;
Yangster-mac468ff042018-01-17 12:26:34 -080062const int FIELD_ID_BUCKET_INFO = 3;
Yangster-mac9def8e32018-04-17 13:55:51 -070063const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
tsaichristinec876b492019-12-10 13:47:05 -080064const int FIELD_ID_SLICE_BY_STATE = 6;
yro2b0f8862017-11-06 14:27:31 -080065// for ValueBucketInfo
Chenjie Yu32717c32018-10-20 23:54:48 -070066const int FIELD_ID_VALUE_INDEX = 1;
67const int FIELD_ID_VALUE_LONG = 2;
68const int FIELD_ID_VALUE_DOUBLE = 3;
69const int FIELD_ID_VALUES = 9;
Yangster-mac9def8e32018-04-17 13:55:51 -070070const int FIELD_ID_BUCKET_NUM = 4;
71const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
72const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
Yao Chene6cfb142019-04-08 12:00:01 -070073const int FIELD_ID_CONDITION_TRUE_NS = 10;
yro2b0f8862017-11-06 14:27:31 -080074
Chenjie Yuf275f612018-11-30 23:29:06 -080075const Value ZERO_LONG((int64_t)0);
76const Value ZERO_DOUBLE((int64_t)0);
77
Chenjie Yub3dda412017-10-24 13:41:59 -070078// ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently
Chenjie Yuf275f612018-11-30 23:29:06 -080079ValueMetricProducer::ValueMetricProducer(
80 const ConfigKey& key, const ValueMetric& metric, const int conditionIndex,
81 const sp<ConditionWizard>& conditionWizard, const int whatMatcherIndex,
82 const sp<EventMatcherWizard>& matcherWizard, const int pullTagId, const int64_t timeBaseNs,
Ruchir Rastogi21a287b2019-10-02 12:04:33 -070083 const int64_t startTimeNs, const sp<StatsPullerManager>& pullerManager,
84 const unordered_map<int, shared_ptr<Activation>>& eventActivationMap,
tsaichristined21aacf2019-10-07 14:47:38 -070085 const unordered_map<int, vector<shared_ptr<Activation>>>& eventDeactivationMap,
86 const vector<int>& slicedStateAtoms,
87 const unordered_map<int, unordered_map<int, int64_t>>& stateGroupMap)
Ruchir Rastogi21a287b2019-10-02 12:04:33 -070088 : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, conditionWizard,
tsaichristined21aacf2019-10-07 14:47:38 -070089 eventActivationMap, eventDeactivationMap, slicedStateAtoms, stateGroupMap),
Chenjie Yu054ce9c2018-11-12 15:27:29 -080090 mWhatMatcherIndex(whatMatcherIndex),
91 mEventMatcherWizard(matcherWizard),
Chenjie Yue2219202018-06-08 10:07:51 -070092 mPullerManager(pullerManager),
Chenjie Yuc5875052018-03-09 10:13:11 -080093 mPullTagId(pullTagId),
Chenjie Yua0f02242018-07-06 16:14:34 -070094 mIsPulled(pullTagId != -1),
David Chen81245fd2018-04-12 14:33:37 -070095 mMinBucketSizeNs(metric.min_bucket_size_nanos()),
Chenjie Yuc5875052018-03-09 10:13:11 -080096 mDimensionSoftLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) !=
97 StatsdStats::kAtomDimensionKeySizeLimitMap.end()
98 ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).first
99 : StatsdStats::kDimensionKeySizeSoftLimit),
100 mDimensionHardLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) !=
101 StatsdStats::kAtomDimensionKeySizeLimitMap.end()
102 ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).second
Chenjie Yu47234642018-05-14 10:14:16 -0700103 : StatsdStats::kDimensionKeySizeHardLimit),
Chenjie Yua0f02242018-07-06 16:14:34 -0700104 mUseAbsoluteValueOnReset(metric.use_absolute_value_on_reset()),
105 mAggregationType(metric.aggregation_type()),
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700106 mUseDiff(metric.has_use_diff() ? metric.use_diff() : (mIsPulled ? true : false)),
107 mValueDirection(metric.value_direction()),
Chenjie Yuf275f612018-11-30 23:29:06 -0800108 mSkipZeroDiffOutput(metric.skip_zero_diff_output()),
109 mUseZeroDefaultBase(metric.use_zero_default_base()),
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800110 mHasGlobalBase(false),
Jeffrey Huangde3348a2020-05-20 16:06:13 -0700111 mCurrentBucketIsSkipped(false),
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800112 mMaxPullDelayNs(metric.max_pull_delay_sec() > 0 ? metric.max_pull_delay_sec() * NS_PER_SEC
Chenjie Yucd1b7972019-01-16 20:38:15 -0800113 : StatsdStats::kPullMaxDelayNs),
Yao Chene6cfb142019-04-08 12:00:01 -0700114 mSplitBucketForAppUpgrade(metric.split_bucket_for_app_upgrade()),
Ruchir Rastogi21a287b2019-10-02 12:04:33 -0700115 // Condition timer will be set later within the constructor after pulling events
Tej Singhee4495e2019-06-03 18:37:35 -0700116 mConditionTimer(false, timeBaseNs) {
Yangster-macb8144812018-01-04 10:56:23 -0800117 int64_t bucketSizeMills = 0;
118 if (metric.has_bucket()) {
yro59cc24d2018-02-13 20:17:32 -0800119 bucketSizeMills = TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket());
Chenjie Yu6736c892017-11-09 10:50:09 -0800120 } else {
Yangster-macb8144812018-01-04 10:56:23 -0800121 bucketSizeMills = TimeUnitToBucketSizeInMillis(ONE_HOUR);
Chenjie Yu6736c892017-11-09 10:50:09 -0800122 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700123
Yangster-macb8144812018-01-04 10:56:23 -0800124 mBucketSizeNs = bucketSizeMills * 1000000;
Chenjie Yu32717c32018-10-20 23:54:48 -0700125
126 translateFieldMatcher(metric.value_field(), &mFieldMatchers);
127
Yao Chen8a8d16c2018-02-08 14:50:40 -0800128 if (metric.has_dimensions_in_what()) {
129 translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat);
Yangster13fb7e42018-03-07 17:30:49 -0800130 mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what());
tsaichristine90f95bb2019-11-06 17:06:53 -0800131 mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what());
Yao Chen8a8d16c2018-02-08 14:50:40 -0800132 }
133
Yao Chen93fe3a32017-11-02 13:52:59 -0700134 if (metric.links().size() > 0) {
Yao Chen8a8d16c2018-02-08 14:50:40 -0800135 for (const auto& link : metric.links()) {
136 Metric2Condition mc;
137 mc.conditionId = link.condition();
138 translateFieldMatcher(link.fields_in_what(), &mc.metricFields);
139 translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields);
140 mMetric2ConditionLinks.push_back(mc);
141 }
tsaichristine76853372019-08-06 17:17:03 -0700142 mConditionSliced = true;
Yao Chen93fe3a32017-11-02 13:52:59 -0700143 }
Yao Chen8a8d16c2018-02-08 14:50:40 -0800144
tsaichristinec876b492019-12-10 13:47:05 -0800145 for (const auto& stateLink : metric.state_link()) {
146 Metric2State ms;
147 ms.stateAtomId = stateLink.state_atom_id();
148 translateFieldMatcher(stateLink.fields_in_what(), &ms.metricFields);
149 translateFieldMatcher(stateLink.fields_in_state(), &ms.stateFields);
150 mMetric2StateLinks.push_back(ms);
151 }
152
Tej Singh597c7162019-04-17 16:41:45 -0700153 int64_t numBucketsForward = calcBucketsForwardCount(startTimeNs);
154 mCurrentBucketNum += numBucketsForward;
155
Chenjie Yue1361ed2018-07-23 17:33:09 -0700156 flushIfNeededLocked(startTimeNs);
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700157
Chenjie Yua0f02242018-07-06 16:14:34 -0700158 if (mIsPulled) {
Tej Singh3be093b2020-03-04 20:08:38 -0800159 mPullerManager->RegisterReceiver(mPullTagId, mConfigKey, this, getCurrentBucketEndTimeNs(),
Chenjie Yue1361ed2018-07-23 17:33:09 -0700160 mBucketSizeNs);
Yao Chen93fe3a32017-11-02 13:52:59 -0700161 }
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700162
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700163 // Only do this for partial buckets like first bucket. All other buckets should use
Chenjie Yue1361ed2018-07-23 17:33:09 -0700164 // flushIfNeeded to adjust start and end to bucket boundaries.
165 // Adjust start for partial bucket
166 mCurrentBucketStartTimeNs = startTimeNs;
Yao Chene6cfb142019-04-08 12:00:01 -0700167 mConditionTimer.newBucketStart(mCurrentBucketStartTimeNs);
Ruchir Rastogi21a287b2019-10-02 12:04:33 -0700168
Ruchir Rastogi21a287b2019-10-02 12:04:33 -0700169 // Now that activations are processed, start the condition timer if needed.
170 mConditionTimer.onConditionChanged(mIsActive && mCondition == ConditionState::kTrue,
171 mCurrentBucketStartTimeNs);
172
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700173 VLOG("value metric %lld created. bucket size %lld start_time: %lld", (long long)metric.id(),
174 (long long)mBucketSizeNs, (long long)mTimeBaseNs);
Chenjie Yub3dda412017-10-24 13:41:59 -0700175}
176
177ValueMetricProducer::~ValueMetricProducer() {
Yao Chen93fe3a32017-11-02 13:52:59 -0700178 VLOG("~ValueMetricProducer() called");
Chenjie Yua0f02242018-07-06 16:14:34 -0700179 if (mIsPulled) {
Tej Singh3be093b2020-03-04 20:08:38 -0800180 mPullerManager->UnRegisterReceiver(mPullTagId, mConfigKey, this);
Chenjie Yu6736c892017-11-09 10:50:09 -0800181 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700182}
183
tsaichristinec876b492019-12-10 13:47:05 -0800184void ValueMetricProducer::onStateChanged(int64_t eventTimeNs, int32_t atomId,
tsaichristine9f951052020-05-13 14:32:37 -0700185 const HashableDimensionKey& primaryKey,
186 const FieldValue& oldState, const FieldValue& newState) {
tsaichristinec876b492019-12-10 13:47:05 -0800187 VLOG("ValueMetric %lld onStateChanged time %lld, State %d, key %s, %d -> %d",
188 (long long)mMetricId, (long long)eventTimeNs, atomId, primaryKey.toString().c_str(),
tsaichristine9f951052020-05-13 14:32:37 -0700189 oldState.mValue.int_value, newState.mValue.int_value);
tsaichristinec876b492019-12-10 13:47:05 -0800190 // If condition is not true, we do not need to pull for this state change.
191 if (mCondition != ConditionState::kTrue) {
192 return;
193 }
tsaichristine9f951052020-05-13 14:32:37 -0700194
195 // If old and new states are in the same StateGroup, then we do not need to
196 // pull for this state change.
197 FieldValue oldStateCopy = oldState;
198 FieldValue newStateCopy = newState;
199 mapStateValue(atomId, &oldStateCopy);
200 mapStateValue(atomId, &newStateCopy);
201 if (oldStateCopy == newStateCopy) {
202 return;
203 }
204
tsaichristinec876b492019-12-10 13:47:05 -0800205 bool isEventLate = eventTimeNs < mCurrentBucketStartTimeNs;
206 if (isEventLate) {
207 VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
208 (long long)mCurrentBucketStartTimeNs);
209 invalidateCurrentBucket(eventTimeNs, BucketDropReason::EVENT_IN_WRONG_BUCKET);
210 return;
211 }
212 mStateChangePrimaryKey.first = atomId;
213 mStateChangePrimaryKey.second = primaryKey;
214 if (mIsPulled) {
215 pullAndMatchEventsLocked(eventTimeNs);
216 }
217 mStateChangePrimaryKey.first = 0;
218 mStateChangePrimaryKey.second = DEFAULT_DIMENSION_KEY;
219 flushIfNeededLocked(eventTimeNs);
220}
221
Yao Chen427d3722018-03-22 15:21:52 -0700222void ValueMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition,
Yangster-macb142cc82018-03-30 15:22:08 -0700223 const int64_t eventTime) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800224 VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId);
Chenjie Yub3dda412017-10-24 13:41:59 -0700225}
226
Yangster-macb142cc82018-03-30 15:22:08 -0700227void ValueMetricProducer::dropDataLocked(const int64_t dropTimeNs) {
Olivier Gaillard320952b2019-02-06 13:57:24 +0000228 StatsdStats::getInstance().noteBucketDropped(mMetricId);
tsaichristineb7fcf002019-12-06 18:40:47 -0800229
230 // The current partial bucket is not flushed and does not require a pull,
231 // so the data is still valid.
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000232 flushIfNeededLocked(dropTimeNs);
233 clearPastBucketsLocked(dropTimeNs);
Yao Chen06dba5d2018-01-26 13:38:16 -0800234}
235
Yangster-maca802d732018-04-24 07:50:38 -0700236void ValueMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) {
Yangster-maca802d732018-04-24 07:50:38 -0700237 mPastBuckets.clear();
238 mSkippedBuckets.clear();
239}
240
Yangster-macb142cc82018-03-30 15:22:08 -0700241void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
Yangster-mace68f3a52018-04-04 00:01:43 -0700242 const bool include_current_partial_bucket,
Bookatzff71cad2018-09-20 17:17:49 -0700243 const bool erase_data,
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000244 const DumpLatency dumpLatency,
Yangster-mac9def8e32018-04-17 13:55:51 -0700245 std::set<string> *str_set,
Yao Chen288c6002017-12-12 13:43:18 -0800246 ProtoOutputStream* protoOutput) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800247 VLOG("metric %lld dump report now...", (long long)mMetricId);
Yangster-mace68f3a52018-04-04 00:01:43 -0700248 if (include_current_partial_bucket) {
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000249 // For pull metrics, we need to do a pull at bucket boundaries. If we do not do that the
250 // current bucket will have incomplete data and the next will have the wrong snapshot to do
251 // a diff against. If the condition is false, we are fine since the base data is reset and
252 // we are not tracking anything.
253 bool pullNeeded = mIsPulled && mCondition == ConditionState::kTrue;
254 if (pullNeeded) {
255 switch (dumpLatency) {
256 case FAST:
tsaichristineb87ca152019-12-09 15:19:41 -0800257 invalidateCurrentBucket(dumpTimeNs, BucketDropReason::DUMP_REPORT_REQUESTED);
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000258 break;
259 case NO_TIME_CONSTRAINTS:
tsaichristine78b85702019-12-06 12:20:30 -0800260 pullAndMatchEventsLocked(dumpTimeNs);
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000261 break;
262 }
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000263 }
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000264 flushCurrentBucketLocked(dumpTimeNs, dumpTimeNs);
Yangster-mace68f3a52018-04-04 00:01:43 -0700265 }
Yang Lub4722912018-11-15 11:02:03 -0800266 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
Howard Ro07e23ff2018-12-17 17:28:07 -0800267 protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_IS_ACTIVE, isActiveLocked());
Yang Lub4722912018-11-15 11:02:03 -0800268
David Chen81245fd2018-04-12 14:33:37 -0700269 if (mPastBuckets.empty() && mSkippedBuckets.empty()) {
Yangster-mac635b4b32018-01-23 20:17:35 -0800270 return;
271 }
Yangster-mac9def8e32018-04-17 13:55:51 -0700272 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
273 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
274 // Fills the dimension path if not slicing by ALL.
275 if (!mSliceByPositionALL) {
276 if (!mDimensionsInWhat.empty()) {
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700277 uint64_t dimenPathToken =
278 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
Yangster-mac9def8e32018-04-17 13:55:51 -0700279 writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
280 protoOutput->end(dimenPathToken);
281 }
Yangster-mac9def8e32018-04-17 13:55:51 -0700282 }
283
Yi Jin5ee07872018-03-05 18:18:27 -0800284 uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_VALUE_METRICS);
Yao Chen6a8c7992017-11-29 20:02:07 +0000285
tsaichristineb87ca152019-12-09 15:19:41 -0800286 for (const auto& skippedBucket : mSkippedBuckets) {
David Chen81245fd2018-04-12 14:33:37 -0700287 uint64_t wrapperToken =
288 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_SKIPPED);
Yangster-mac9def8e32018-04-17 13:55:51 -0700289 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_START_MILLIS,
tsaichristineb87ca152019-12-09 15:19:41 -0800290 (long long)(NanoToMillis(skippedBucket.bucketStartTimeNs)));
Yangster-mac9def8e32018-04-17 13:55:51 -0700291 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_END_MILLIS,
tsaichristineb87ca152019-12-09 15:19:41 -0800292 (long long)(NanoToMillis(skippedBucket.bucketEndTimeNs)));
293 for (const auto& dropEvent : skippedBucket.dropEvents) {
294 uint64_t dropEventToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
295 FIELD_ID_SKIPPED_DROP_EVENT);
296 protoOutput->write(FIELD_TYPE_INT32 | FIELD_ID_BUCKET_DROP_REASON, dropEvent.reason);
297 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_DROP_TIME,
298 (long long)(NanoToMillis(dropEvent.dropTimeNs)));
299 ;
300 protoOutput->end(dropEventToken);
301 }
David Chen81245fd2018-04-12 14:33:37 -0700302 protoOutput->end(wrapperToken);
303 }
David Chen81245fd2018-04-12 14:33:37 -0700304
Yao Chen93fe3a32017-11-02 13:52:59 -0700305 for (const auto& pair : mPastBuckets) {
Yangster-mac93694462018-01-22 20:49:31 -0800306 const MetricDimensionKey& dimensionKey = pair.first;
Yangster13fb7e42018-03-07 17:30:49 -0800307 VLOG(" dimension key %s", dimensionKey.toString().c_str());
Yi Jin5ee07872018-03-05 18:18:27 -0800308 uint64_t wrapperToken =
Yao Chen288c6002017-12-12 13:43:18 -0800309 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
Chenjie Yub3dda412017-10-24 13:41:59 -0700310
Yangster-mac20877162017-12-22 17:19:39 -0800311 // First fill dimension.
Yangster-mac9def8e32018-04-17 13:55:51 -0700312 if (mSliceByPositionALL) {
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700313 uint64_t dimensionToken =
314 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
Yangster-mac9def8e32018-04-17 13:55:51 -0700315 writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
316 protoOutput->end(dimensionToken);
Yangster-mac9def8e32018-04-17 13:55:51 -0700317 } else {
318 writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
319 FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
Yangster-mac93694462018-01-22 20:49:31 -0800320 }
yro2b0f8862017-11-06 14:27:31 -0800321
tsaichristinec876b492019-12-10 13:47:05 -0800322 // Then fill slice_by_state.
323 for (auto state : dimensionKey.getStateValuesKey().getValues()) {
324 uint64_t stateToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
325 FIELD_ID_SLICE_BY_STATE);
326 writeStateToProto(state, protoOutput);
327 protoOutput->end(stateToken);
328 }
329
yro2b0f8862017-11-06 14:27:31 -0800330 // Then fill bucket_info (ValueBucketInfo).
331 for (const auto& bucket : pair.second) {
Yi Jin5ee07872018-03-05 18:18:27 -0800332 uint64_t bucketInfoToken = protoOutput->start(
Yao Chen288c6002017-12-12 13:43:18 -0800333 FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
Yangster-mac9def8e32018-04-17 13:55:51 -0700334
335 if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
336 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
337 (long long)NanoToMillis(bucket.mBucketStartNs));
338 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
339 (long long)NanoToMillis(bucket.mBucketEndNs));
340 } else {
341 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
342 (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
343 }
Yao Chene6cfb142019-04-08 12:00:01 -0700344 // only write the condition timer value if the metric has a condition.
345 if (mConditionTrackerIndex >= 0) {
346 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_CONDITION_TRUE_NS,
347 (long long)bucket.mConditionTrueNs);
348 }
tsaichristinec876b492019-12-10 13:47:05 -0800349 for (int i = 0; i < (int)bucket.valueIndex.size(); i++) {
Chenjie Yu32717c32018-10-20 23:54:48 -0700350 int index = bucket.valueIndex[i];
351 const Value& value = bucket.values[i];
352 uint64_t valueToken = protoOutput->start(
353 FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_VALUES);
354 protoOutput->write(FIELD_TYPE_INT32 | FIELD_ID_VALUE_INDEX,
355 index);
356 if (value.getType() == LONG) {
357 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_VALUE_LONG,
358 (long long)value.long_value);
359 VLOG("\t bucket [%lld - %lld] value %d: %lld", (long long)bucket.mBucketStartNs,
360 (long long)bucket.mBucketEndNs, index, (long long)value.long_value);
361 } else if (value.getType() == DOUBLE) {
362 protoOutput->write(FIELD_TYPE_DOUBLE | FIELD_ID_VALUE_DOUBLE,
363 value.double_value);
364 VLOG("\t bucket [%lld - %lld] value %d: %.2f", (long long)bucket.mBucketStartNs,
365 (long long)bucket.mBucketEndNs, index, value.double_value);
366 } else {
367 VLOG("Wrong value type for ValueMetric output: %d", value.getType());
368 }
369 protoOutput->end(valueToken);
Chenjie Yua0f02242018-07-06 16:14:34 -0700370 }
Yao Chen288c6002017-12-12 13:43:18 -0800371 protoOutput->end(bucketInfoToken);
yro2b0f8862017-11-06 14:27:31 -0800372 }
Yao Chen288c6002017-12-12 13:43:18 -0800373 protoOutput->end(wrapperToken);
Chenjie Yub3dda412017-10-24 13:41:59 -0700374 }
Yao Chen288c6002017-12-12 13:43:18 -0800375 protoOutput->end(protoToken);
yro2b0f8862017-11-06 14:27:31 -0800376
Yangster-mac94e197c2018-01-02 16:03:03 -0800377 VLOG("metric %lld dump report now...", (long long)mMetricId);
Bookatzff71cad2018-09-20 17:17:49 -0700378 if (erase_data) {
379 mPastBuckets.clear();
380 mSkippedBuckets.clear();
381 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700382}
383
tsaichristineb87ca152019-12-09 15:19:41 -0800384void ValueMetricProducer::invalidateCurrentBucketWithoutResetBase(const int64_t dropTimeNs,
385 const BucketDropReason reason) {
Jeffrey Huangde3348a2020-05-20 16:06:13 -0700386 if (!mCurrentBucketIsSkipped) {
tsaichristineb87ca152019-12-09 15:19:41 -0800387 // Only report to StatsdStats once per invalid bucket.
Olivier Gaillard9a5d3592019-02-05 15:12:39 +0000388 StatsdStats::getInstance().noteInvalidatedBucket(mMetricId);
tsaichristineb87ca152019-12-09 15:19:41 -0800389 }
390
Jeffrey Huangde3348a2020-05-20 16:06:13 -0700391 skipCurrentBucket(dropTimeNs, reason);
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000392}
393
tsaichristineb87ca152019-12-09 15:19:41 -0800394void ValueMetricProducer::invalidateCurrentBucket(const int64_t dropTimeNs,
395 const BucketDropReason reason) {
396 invalidateCurrentBucketWithoutResetBase(dropTimeNs, reason);
Olivier Gaillard9a5d3592019-02-05 15:12:39 +0000397 resetBase();
398}
399
Jeffrey Huangde3348a2020-05-20 16:06:13 -0700400void ValueMetricProducer::skipCurrentBucket(const int64_t dropTimeNs,
401 const BucketDropReason reason) {
402 if (!maxDropEventsReached()) {
403 mCurrentSkippedBucket.dropEvents.emplace_back(buildDropEvent(dropTimeNs, reason));
404 }
405 mCurrentBucketIsSkipped = true;
406}
407
Chenjie Yuf275f612018-11-30 23:29:06 -0800408void ValueMetricProducer::resetBase() {
tsaichristinec876b492019-12-10 13:47:05 -0800409 for (auto& slice : mCurrentBaseInfo) {
410 for (auto& baseInfo : slice.second) {
411 baseInfo.hasBase = false;
412 baseInfo.hasCurrentState = false;
Chenjie Yuf275f612018-11-30 23:29:06 -0800413 }
414 }
415 mHasGlobalBase = false;
416}
417
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700418// Handle active state change. Active state change is treated like a condition change:
419// - drop bucket if active state change event arrives too late
420// - if condition is true, pull data on active state changes
421// - ConditionTimer tracks changes based on AND of condition and active state.
422void ValueMetricProducer::onActiveStateChangedLocked(const int64_t& eventTimeNs) {
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000423 bool isEventTooLate = eventTimeNs < mCurrentBucketStartTimeNs;
tsaichristine90f95bb2019-11-06 17:06:53 -0800424 if (isEventTooLate) {
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700425 // Drop bucket because event arrived too late, ie. we are missing data for this bucket.
tsaichristineb87ca152019-12-09 15:19:41 -0800426 StatsdStats::getInstance().noteLateLogEventSkipped(mMetricId);
427 invalidateCurrentBucket(eventTimeNs, BucketDropReason::EVENT_IN_WRONG_BUCKET);
Yao Chen2794da22017-12-13 16:01:55 -0800428 }
429
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700430 // Call parent method once we've verified the validity of current bucket.
431 MetricProducer::onActiveStateChangedLocked(eventTimeNs);
432
433 if (ConditionState::kTrue != mCondition) {
434 return;
435 }
436
437 // Pull on active state changes.
438 if (!isEventTooLate) {
439 if (mIsPulled) {
tsaichristine78b85702019-12-06 12:20:30 -0800440 pullAndMatchEventsLocked(eventTimeNs);
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700441 }
442 // When active state changes from true to false, clear diff base but don't
443 // reset other counters as we may accumulate more value in the bucket.
444 if (mUseDiff && !mIsActive) {
445 resetBase();
446 }
447 }
448
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700449 flushIfNeededLocked(eventTimeNs);
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700450
451 // Let condition timer know of new active state.
452 mConditionTimer.onConditionChanged(mIsActive, eventTimeNs);
Chenjie Yue1361ed2018-07-23 17:33:09 -0700453}
454
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700455void ValueMetricProducer::onConditionChangedLocked(const bool condition,
456 const int64_t eventTimeNs) {
457 ConditionState newCondition = condition ? ConditionState::kTrue : ConditionState::kFalse;
458 bool isEventTooLate = eventTimeNs < mCurrentBucketStartTimeNs;
459
tsaichristine90f95bb2019-11-06 17:06:53 -0800460 // If the config is not active, skip the event.
461 if (!mIsActive) {
462 mCondition = isEventTooLate ? ConditionState::kUnknown : newCondition;
463 return;
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700464 }
465
tsaichristine90f95bb2019-11-06 17:06:53 -0800466 // If the event arrived late, mark the bucket as invalid and skip the event.
467 if (isEventTooLate) {
468 VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
469 (long long)mCurrentBucketStartTimeNs);
tsaichristineb87ca152019-12-09 15:19:41 -0800470 StatsdStats::getInstance().noteLateLogEventSkipped(mMetricId);
tsaichristine90f95bb2019-11-06 17:06:53 -0800471 StatsdStats::getInstance().noteConditionChangeInNextBucket(mMetricId);
tsaichristineb87ca152019-12-09 15:19:41 -0800472 invalidateCurrentBucket(eventTimeNs, BucketDropReason::EVENT_IN_WRONG_BUCKET);
tsaichristine90f95bb2019-11-06 17:06:53 -0800473 mCondition = ConditionState::kUnknown;
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700474 mConditionTimer.onConditionChanged(mCondition, eventTimeNs);
tsaichristine90f95bb2019-11-06 17:06:53 -0800475 return;
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700476 }
tsaichristine90f95bb2019-11-06 17:06:53 -0800477
478 // If the previous condition was unknown, mark the bucket as invalid
479 // because the bucket will contain partial data. For example, the condition
480 // change might happen close to the end of the bucket and we might miss a
481 // lot of data.
482 //
483 // We still want to pull to set the base.
484 if (mCondition == ConditionState::kUnknown) {
tsaichristineb87ca152019-12-09 15:19:41 -0800485 invalidateCurrentBucket(eventTimeNs, BucketDropReason::CONDITION_UNKNOWN);
tsaichristine90f95bb2019-11-06 17:06:53 -0800486 }
487
488 // Pull and match for the following condition change cases:
489 // unknown/false -> true - condition changed
490 // true -> false - condition changed
491 // true -> true - old condition was true so we can flush the bucket at the
492 // end if needed.
493 //
494 // We don’t need to pull for unknown -> false or false -> false.
495 //
496 // onConditionChangedLocked might happen on bucket boundaries if this is
497 // called before #onDataPulled.
498 if (mIsPulled &&
499 (newCondition == ConditionState::kTrue || mCondition == ConditionState::kTrue)) {
tsaichristine78b85702019-12-06 12:20:30 -0800500 pullAndMatchEventsLocked(eventTimeNs);
tsaichristine90f95bb2019-11-06 17:06:53 -0800501 }
502
503 // For metrics that use diff, when condition changes from true to false,
504 // clear diff base but don't reset other counts because we may accumulate
505 // more value in the bucket.
506 if (mUseDiff &&
507 (mCondition == ConditionState::kTrue && newCondition == ConditionState::kFalse)) {
508 resetBase();
509 }
510
511 // Update condition state after pulling.
512 mCondition = newCondition;
513
514 flushIfNeededLocked(eventTimeNs);
515 mConditionTimer.onConditionChanged(mCondition, eventTimeNs);
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700516}
517
Tej Singh3be093b2020-03-04 20:08:38 -0800518void ValueMetricProducer::prepareFirstBucketLocked() {
519 // Kicks off the puller immediately if condition is true and diff based.
520 if (mIsActive && mIsPulled && mCondition == ConditionState::kTrue && mUseDiff) {
521 pullAndMatchEventsLocked(mCurrentBucketStartTimeNs);
522 }
523}
524
tsaichristine78b85702019-12-06 12:20:30 -0800525void ValueMetricProducer::pullAndMatchEventsLocked(const int64_t timestampNs) {
Chenjie Yue1361ed2018-07-23 17:33:09 -0700526 vector<std::shared_ptr<LogEvent>> allData;
Tej Singh7b975a82020-05-11 11:05:08 -0700527 if (!mPullerManager->Pull(mPullTagId, mConfigKey, timestampNs, &allData)) {
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000528 ALOGE("Stats puller failed for tag: %d at %lld", mPullTagId, (long long)timestampNs);
tsaichristineb87ca152019-12-09 15:19:41 -0800529 invalidateCurrentBucket(timestampNs, BucketDropReason::PULL_FAILED);
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800530 return;
Chenjie Yub3dda412017-10-24 13:41:59 -0700531 }
Olivier Gaillard11203df2019-02-06 13:18:09 +0000532
tsaichristine78b85702019-12-06 12:20:30 -0800533 accumulateEvents(allData, timestampNs, timestampNs);
Olivier Gaillard11203df2019-02-06 13:18:09 +0000534}
535
536int64_t ValueMetricProducer::calcPreviousBucketEndTime(const int64_t currentTimeNs) {
537 return mTimeBaseNs + ((currentTimeNs - mTimeBaseNs) / mBucketSizeNs) * mBucketSizeNs;
538}
539
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000540// By design, statsd pulls data at bucket boundaries using AlarmManager. These pulls are likely
541// to be delayed. Other events like condition changes or app upgrade which are not based on
542// AlarmManager might have arrived earlier and close the bucket.
Olivier Gaillard11203df2019-02-06 13:18:09 +0000543void ValueMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData,
544 bool pullSuccess, int64_t originalPullTimeNs) {
545 std::lock_guard<std::mutex> lock(mMutex);
tsaichristine90f95bb2019-11-06 17:06:53 -0800546 if (mCondition == ConditionState::kTrue) {
547 // If the pull failed, we won't be able to compute a diff.
548 if (!pullSuccess) {
tsaichristineb87ca152019-12-09 15:19:41 -0800549 invalidateCurrentBucket(originalPullTimeNs, BucketDropReason::PULL_FAILED);
tsaichristine90f95bb2019-11-06 17:06:53 -0800550 } else {
551 bool isEventLate = originalPullTimeNs < getCurrentBucketEndTimeNs();
552 if (isEventLate) {
553 // If the event is late, we are in the middle of a bucket. Just
554 // process the data without trying to snap the data to the nearest bucket.
tsaichristine78b85702019-12-06 12:20:30 -0800555 accumulateEvents(allData, originalPullTimeNs, originalPullTimeNs);
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000556 } else {
tsaichristine90f95bb2019-11-06 17:06:53 -0800557 // For scheduled pulled data, the effective event time is snap to the nearest
558 // bucket end. In the case of waking up from a deep sleep state, we will
559 // attribute to the previous bucket end. If the sleep was long but not very
560 // long, we will be in the immediate next bucket. Previous bucket may get a
561 // larger number as we pull at a later time than real bucket end.
562 //
563 // If the sleep was very long, we skip more than one bucket before sleep. In
564 // this case, if the diff base will be cleared and this new data will serve as
565 // new diff base.
566 int64_t bucketEndTime = calcPreviousBucketEndTime(originalPullTimeNs) - 1;
567 StatsdStats::getInstance().noteBucketBoundaryDelayNs(
568 mMetricId, originalPullTimeNs - bucketEndTime);
tsaichristine78b85702019-12-06 12:20:30 -0800569 accumulateEvents(allData, originalPullTimeNs, bucketEndTime);
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000570 }
Olivier Gaillard11203df2019-02-06 13:18:09 +0000571 }
tsaichristine90f95bb2019-11-06 17:06:53 -0800572 }
Olivier Gaillard11203df2019-02-06 13:18:09 +0000573
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000574 // We can probably flush the bucket. Since we used bucketEndTime when calling
575 // #onMatchedLogEventInternalLocked, the current bucket will not have been flushed.
576 flushIfNeededLocked(originalPullTimeNs);
Olivier Gaillard11203df2019-02-06 13:18:09 +0000577}
578
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000579void ValueMetricProducer::accumulateEvents(const std::vector<std::shared_ptr<LogEvent>>& allData,
tsaichristine78b85702019-12-06 12:20:30 -0800580 int64_t originalPullTimeNs, int64_t eventElapsedTimeNs) {
Olivier Gaillard11203df2019-02-06 13:18:09 +0000581 bool isEventLate = eventElapsedTimeNs < mCurrentBucketStartTimeNs;
582 if (isEventLate) {
583 VLOG("Skip bucket end pull due to late arrival: %lld vs %lld",
584 (long long)eventElapsedTimeNs, (long long)mCurrentBucketStartTimeNs);
585 StatsdStats::getInstance().noteLateLogEventSkipped(mMetricId);
tsaichristineb87ca152019-12-09 15:19:41 -0800586 invalidateCurrentBucket(eventElapsedTimeNs, BucketDropReason::EVENT_IN_WRONG_BUCKET);
Olivier Gaillard11203df2019-02-06 13:18:09 +0000587 return;
588 }
589
tsaichristineb87ca152019-12-09 15:19:41 -0800590 const int64_t elapsedRealtimeNs = getElapsedRealtimeNs();
591 const int64_t pullDelayNs = elapsedRealtimeNs - originalPullTimeNs;
Olivier Gaillard9a5d3592019-02-05 15:12:39 +0000592 StatsdStats::getInstance().notePullDelay(mPullTagId, pullDelayNs);
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800593 if (pullDelayNs > mMaxPullDelayNs) {
594 ALOGE("Pull finish too late for atom %d, longer than %lld", mPullTagId,
595 (long long)mMaxPullDelayNs);
596 StatsdStats::getInstance().notePullExceedMaxDelay(mPullTagId);
Olivier Gaillard9a5d3592019-02-05 15:12:39 +0000597 // We are missing one pull from the bucket which means we will not have a complete view of
598 // what's going on.
tsaichristineb87ca152019-12-09 15:19:41 -0800599 invalidateCurrentBucket(eventElapsedTimeNs, BucketDropReason::PULL_DELAYED);
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800600 return;
601 }
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800602
Olivier Gaillard11203df2019-02-06 13:18:09 +0000603 mMatchedMetricDimensionKeys.clear();
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800604 for (const auto& data : allData) {
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800605 LogEvent localCopy = data->makeCopy();
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800606 if (mEventMatcherWizard->matchLogEvent(localCopy, mWhatMatcherIndex) ==
607 MatchingState::kMatched) {
Olivier Gaillard11203df2019-02-06 13:18:09 +0000608 localCopy.setElapsedTimestampNs(eventElapsedTimeNs);
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800609 onMatchedLogEventLocked(mWhatMatcherIndex, localCopy);
610 }
611 }
tsaichristinec876b492019-12-10 13:47:05 -0800612 // If a key that is:
613 // 1. Tracked in mCurrentSlicedBucket and
614 // 2. A superset of the current mStateChangePrimaryKey
615 // was not found in the new pulled data (i.e. not in mMatchedDimensionInWhatKeys)
616 // then we need to reset the base.
Olivier Gaillard11203df2019-02-06 13:18:09 +0000617 for (auto& slice : mCurrentSlicedBucket) {
tsaichristinec876b492019-12-10 13:47:05 -0800618 const auto& whatKey = slice.first.getDimensionKeyInWhat();
619 bool presentInPulledData =
620 mMatchedMetricDimensionKeys.find(whatKey) != mMatchedMetricDimensionKeys.end();
621 if (!presentInPulledData && whatKey.contains(mStateChangePrimaryKey.second)) {
622 auto it = mCurrentBaseInfo.find(whatKey);
623 for (auto& baseInfo : it->second) {
624 baseInfo.hasBase = false;
625 baseInfo.hasCurrentState = false;
Chenjie Yu054ce9c2018-11-12 15:27:29 -0800626 }
Chenjie Yua7259ab2017-12-10 08:31:05 -0800627 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700628 }
Olivier Gaillard11203df2019-02-06 13:18:09 +0000629 mMatchedMetricDimensionKeys.clear();
630 mHasGlobalBase = true;
Olivier Gaillard1e0d8fc2019-02-11 18:08:43 +0000631
632 // If we reach the guardrail, we might have dropped some data which means the bucket is
633 // incomplete.
634 //
635 // The base also needs to be reset. If we do not have the full data, we might
636 // incorrectly compute the diff when mUseZeroDefaultBase is true since an existing key
637 // might be missing from mCurrentSlicedBucket.
638 if (hasReachedGuardRailLimit()) {
tsaichristineb87ca152019-12-09 15:19:41 -0800639 invalidateCurrentBucket(eventElapsedTimeNs, BucketDropReason::DIMENSION_GUARDRAIL_REACHED);
Olivier Gaillard1e0d8fc2019-02-11 18:08:43 +0000640 mCurrentSlicedBucket.clear();
641 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700642}
643
Yangster-maca78d0082018-03-12 12:02:56 -0700644void ValueMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
645 if (mCurrentSlicedBucket.size() == 0) {
646 return;
647 }
648
649 fprintf(out, "ValueMetric %lld dimension size %lu\n", (long long)mMetricId,
650 (unsigned long)mCurrentSlicedBucket.size());
651 if (verbose) {
652 for (const auto& it : mCurrentSlicedBucket) {
Chenjie Yu32717c32018-10-20 23:54:48 -0700653 for (const auto& interval : it.second) {
tsaichristine69000e62019-10-18 17:34:52 -0700654 fprintf(out, "\t(what)%s\t(states)%s (value)%s\n",
655 it.first.getDimensionKeyInWhat().toString().c_str(),
656 it.first.getStateValuesKey().toString().c_str(),
657 interval.value.toString().c_str());
Chenjie Yu32717c32018-10-20 23:54:48 -0700658 }
Yangster-maca78d0082018-03-12 12:02:56 -0700659 }
660 }
661}
662
Olivier Gaillard1e0d8fc2019-02-11 18:08:43 +0000663bool ValueMetricProducer::hasReachedGuardRailLimit() const {
664 return mCurrentSlicedBucket.size() >= mDimensionHardLimit;
665}
666
Yangster-mac93694462018-01-22 20:49:31 -0800667bool ValueMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
Yao Chenb3561512017-11-21 18:07:17 -0800668 // ===========GuardRail==============
669 // 1. Report the tuple count if the tuple count > soft limit
670 if (mCurrentSlicedBucket.find(newKey) != mCurrentSlicedBucket.end()) {
671 return false;
672 }
Chenjie Yuc5875052018-03-09 10:13:11 -0800673 if (mCurrentSlicedBucket.size() > mDimensionSoftLimit - 1) {
Yao Chenb3561512017-11-21 18:07:17 -0800674 size_t newTupleCount = mCurrentSlicedBucket.size() + 1;
Yangster-mac94e197c2018-01-02 16:03:03 -0800675 StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount);
Yao Chenb3561512017-11-21 18:07:17 -0800676 // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
Olivier Gaillard1e0d8fc2019-02-11 18:08:43 +0000677 if (hasReachedGuardRailLimit()) {
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700678 ALOGE("ValueMetric %lld dropping data for dimension key %s", (long long)mMetricId,
679 newKey.toString().c_str());
Misha Wagner1eee2212019-01-22 11:47:11 +0000680 StatsdStats::getInstance().noteHardDimensionLimitReached(mMetricId);
Yao Chenb3561512017-11-21 18:07:17 -0800681 return true;
682 }
683 }
684
685 return false;
686}
687
Chenjie Yudbe5c502018-11-30 23:15:57 -0800688bool ValueMetricProducer::hitFullBucketGuardRailLocked(const MetricDimensionKey& newKey) {
689 // ===========GuardRail==============
690 // 1. Report the tuple count if the tuple count > soft limit
691 if (mCurrentFullBucket.find(newKey) != mCurrentFullBucket.end()) {
692 return false;
693 }
694 if (mCurrentFullBucket.size() > mDimensionSoftLimit - 1) {
695 size_t newTupleCount = mCurrentFullBucket.size() + 1;
696 // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
697 if (newTupleCount > mDimensionHardLimit) {
698 ALOGE("ValueMetric %lld dropping data for full bucket dimension key %s",
699 (long long)mMetricId,
700 newKey.toString().c_str());
701 return true;
702 }
703 }
704
705 return false;
706}
707
Chenjie Yu32717c32018-10-20 23:54:48 -0700708bool getDoubleOrLong(const LogEvent& event, const Matcher& matcher, Value& ret) {
709 for (const FieldValue& value : event.getValues()) {
710 if (value.mField.matches(matcher)) {
711 switch (value.mValue.type) {
712 case INT:
713 ret.setLong(value.mValue.int_value);
714 break;
715 case LONG:
716 ret.setLong(value.mValue.long_value);
717 break;
718 case FLOAT:
719 ret.setDouble(value.mValue.float_value);
720 break;
721 case DOUBLE:
722 ret.setDouble(value.mValue.double_value);
723 break;
724 default:
tsaichristine409468d2019-10-28 11:32:31 -0700725 return false;
Chenjie Yu32717c32018-10-20 23:54:48 -0700726 break;
727 }
728 return true;
729 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700730 }
Chenjie Yu32717c32018-10-20 23:54:48 -0700731 return false;
Chenjie Yua0f02242018-07-06 16:14:34 -0700732}
733
tsaichristinec876b492019-12-10 13:47:05 -0800734void ValueMetricProducer::onMatchedLogEventInternalLocked(
735 const size_t matcherIndex, const MetricDimensionKey& eventKey,
736 const ConditionKey& conditionKey, bool condition, const LogEvent& event,
737 const map<int, HashableDimensionKey>& statePrimaryKeys) {
738 auto whatKey = eventKey.getDimensionKeyInWhat();
739 auto stateKey = eventKey.getStateValuesKey();
740
741 // Skip this event if a state changed occurred for a different primary key.
742 auto it = statePrimaryKeys.find(mStateChangePrimaryKey.first);
743 // Check that both the atom id and the primary key are equal.
744 if (it != statePrimaryKeys.end() && it->second != mStateChangePrimaryKey.second) {
745 VLOG("ValueMetric skip event with primary key %s because state change primary key "
746 "is %s",
747 it->second.toString().c_str(), mStateChangePrimaryKey.second.toString().c_str());
748 return;
749 }
750
Yangster-macb142cc82018-03-30 15:22:08 -0700751 int64_t eventTimeNs = event.GetElapsedTimestampNs();
Yao Chen6a8c7992017-11-29 20:02:07 +0000752 if (eventTimeNs < mCurrentBucketStartTimeNs) {
753 VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
754 (long long)mCurrentBucketStartTimeNs);
755 return;
756 }
tsaichristinec876b492019-12-10 13:47:05 -0800757 mMatchedMetricDimensionKeys.insert(whatKey);
Yao Chen6a8c7992017-11-29 20:02:07 +0000758
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000759 if (!mIsPulled) {
760 // We cannot flush without doing a pull first.
761 flushIfNeededLocked(eventTimeNs);
762 }
Chenjie Yua7259ab2017-12-10 08:31:05 -0800763
Olivier Gaillardfbee9162019-04-11 11:48:01 +0100764 // We should not accumulate the data for pushed metrics when the condition is false.
765 bool shouldSkipForPushMetric = !mIsPulled && !condition;
766 // For pulled metrics, there are two cases:
767 // - to compute diffs, we need to process all the state changes
768 // - for non-diffs metrics, we should ignore the data if the condition wasn't true. If we have a
769 // state change from
770 // + True -> True: we should process the data, it might be a bucket boundary
771 // + True -> False: we als need to process the data.
772 bool shouldSkipForPulledMetric = mIsPulled && !mUseDiff
773 && mCondition != ConditionState::kTrue;
774 if (shouldSkipForPushMetric || shouldSkipForPulledMetric) {
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700775 VLOG("ValueMetric skip event because condition is false");
776 return;
777 }
778
Yangsterf2bee6f2017-11-29 12:01:05 -0800779 if (hitGuardRailLocked(eventKey)) {
Yangster8de69392017-11-27 13:48:29 -0800780 return;
781 }
tsaichristinec876b492019-12-10 13:47:05 -0800782 vector<BaseInfo>& baseInfos = mCurrentBaseInfo[whatKey];
783 if (baseInfos.size() < mFieldMatchers.size()) {
Chenjie Yu32717c32018-10-20 23:54:48 -0700784 VLOG("Resizing number of intervals to %d", (int)mFieldMatchers.size());
tsaichristinec876b492019-12-10 13:47:05 -0800785 baseInfos.resize(mFieldMatchers.size());
786 }
787
788 for (auto baseInfo : baseInfos) {
789 if (!baseInfo.hasCurrentState) {
790 baseInfo.currentState = DEFAULT_DIMENSION_KEY;
791 baseInfo.hasCurrentState = true;
792 }
793 }
794
795 // We need to get the intervals stored with the previous state key so we can
796 // close these value intervals.
797 const auto oldStateKey = baseInfos[0].currentState;
798 vector<Interval>& intervals = mCurrentSlicedBucket[MetricDimensionKey(whatKey, oldStateKey)];
799 if (intervals.size() < mFieldMatchers.size()) {
800 VLOG("Resizing number of intervals to %d", (int)mFieldMatchers.size());
801 intervals.resize(mFieldMatchers.size());
Yangster-maca7fb12d2018-01-03 17:17:20 -0800802 }
Yao Chen6a8c7992017-11-29 20:02:07 +0000803
Misha Wagner26531762019-01-21 14:18:51 +0000804 // We only use anomaly detection under certain cases.
805 // N.B.: The anomaly detection cases were modified in order to fix an issue with value metrics
806 // containing multiple values. We tried to retain all previous behaviour, but we are unsure the
807 // previous behaviour was correct. At the time of the fix, anomaly detection had no owner.
808 // Whoever next works on it should look into the cases where it is triggered in this function.
809 // Discussion here: http://ag/6124370.
810 bool useAnomalyDetection = true;
811
Chenjie Yu32717c32018-10-20 23:54:48 -0700812 for (int i = 0; i < (int)mFieldMatchers.size(); i++) {
813 const Matcher& matcher = mFieldMatchers[i];
tsaichristinec876b492019-12-10 13:47:05 -0800814 BaseInfo& baseInfo = baseInfos[i];
815 Interval& interval = intervals[i];
Chenjie Yu32717c32018-10-20 23:54:48 -0700816 interval.valueIndex = i;
817 Value value;
818 if (!getDoubleOrLong(event, matcher, value)) {
819 VLOG("Failed to get value %d from event %s", i, event.ToString().c_str());
Misha Wagner1eee2212019-01-22 11:47:11 +0000820 StatsdStats::getInstance().noteBadValueType(mMetricId);
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700821 return;
822 }
Chenjie Yudbe5c502018-11-30 23:15:57 -0800823 interval.seenNewData = true;
Chenjie Yuc715b9e2018-10-19 07:52:12 -0700824
Chenjie Yu32717c32018-10-20 23:54:48 -0700825 if (mUseDiff) {
tsaichristinec876b492019-12-10 13:47:05 -0800826 if (!baseInfo.hasBase) {
Chenjie Yuf275f612018-11-30 23:29:06 -0800827 if (mHasGlobalBase && mUseZeroDefaultBase) {
828 // The bucket has global base. This key does not.
829 // Optionally use zero as base.
tsaichristinec876b492019-12-10 13:47:05 -0800830 baseInfo.base = (value.type == LONG ? ZERO_LONG : ZERO_DOUBLE);
831 baseInfo.hasBase = true;
Chenjie Yuf275f612018-11-30 23:29:06 -0800832 } else {
833 // no base. just update base and return.
tsaichristinec876b492019-12-10 13:47:05 -0800834 baseInfo.base = value;
835 baseInfo.hasBase = true;
Misha Wagner26531762019-01-21 14:18:51 +0000836 // If we're missing a base, do not use anomaly detection on incomplete data
837 useAnomalyDetection = false;
tsaichristinec876b492019-12-10 13:47:05 -0800838 // Continue (instead of return) here in order to set baseInfo.base and
839 // baseInfo.hasBase for other baseInfos
Misha Wagner26531762019-01-21 14:18:51 +0000840 continue;
Chenjie Yuf275f612018-11-30 23:29:06 -0800841 }
Chenjie Yu32717c32018-10-20 23:54:48 -0700842 }
tsaichristinec876b492019-12-10 13:47:05 -0800843
Chenjie Yu32717c32018-10-20 23:54:48 -0700844 Value diff;
845 switch (mValueDirection) {
846 case ValueMetric::INCREASING:
tsaichristinec876b492019-12-10 13:47:05 -0800847 if (value >= baseInfo.base) {
848 diff = value - baseInfo.base;
Chenjie Yu32717c32018-10-20 23:54:48 -0700849 } else if (mUseAbsoluteValueOnReset) {
850 diff = value;
851 } else {
852 VLOG("Unexpected decreasing value");
853 StatsdStats::getInstance().notePullDataError(mPullTagId);
tsaichristinec876b492019-12-10 13:47:05 -0800854 baseInfo.base = value;
Misha Wagner26531762019-01-21 14:18:51 +0000855 // If we've got bad data, do not use anomaly detection
856 useAnomalyDetection = false;
857 continue;
Chenjie Yu32717c32018-10-20 23:54:48 -0700858 }
859 break;
860 case ValueMetric::DECREASING:
tsaichristinec876b492019-12-10 13:47:05 -0800861 if (baseInfo.base >= value) {
862 diff = baseInfo.base - value;
Chenjie Yu32717c32018-10-20 23:54:48 -0700863 } else if (mUseAbsoluteValueOnReset) {
864 diff = value;
865 } else {
866 VLOG("Unexpected increasing value");
867 StatsdStats::getInstance().notePullDataError(mPullTagId);
tsaichristinec876b492019-12-10 13:47:05 -0800868 baseInfo.base = value;
Misha Wagner26531762019-01-21 14:18:51 +0000869 // If we've got bad data, do not use anomaly detection
870 useAnomalyDetection = false;
871 continue;
Chenjie Yu32717c32018-10-20 23:54:48 -0700872 }
873 break;
874 case ValueMetric::ANY:
tsaichristinec876b492019-12-10 13:47:05 -0800875 diff = value - baseInfo.base;
Chenjie Yu32717c32018-10-20 23:54:48 -0700876 break;
877 default:
878 break;
879 }
tsaichristinec876b492019-12-10 13:47:05 -0800880 baseInfo.base = value;
Chenjie Yu32717c32018-10-20 23:54:48 -0700881 value = diff;
Yao Chen6a8c7992017-11-29 20:02:07 +0000882 }
Chenjie Yu32717c32018-10-20 23:54:48 -0700883
884 if (interval.hasValue) {
885 switch (mAggregationType) {
886 case ValueMetric::SUM:
887 // for AVG, we add up and take average when flushing the bucket
888 case ValueMetric::AVG:
889 interval.value += value;
890 break;
891 case ValueMetric::MIN:
892 interval.value = std::min(value, interval.value);
893 break;
894 case ValueMetric::MAX:
895 interval.value = std::max(value, interval.value);
896 break;
897 default:
898 break;
899 }
900 } else {
901 interval.value = value;
902 interval.hasValue = true;
903 }
904 interval.sampleSize += 1;
tsaichristinec876b492019-12-10 13:47:05 -0800905 baseInfo.currentState = stateKey;
Yangster8de69392017-11-27 13:48:29 -0800906 }
Bookatzde1b55622017-12-14 18:38:27 -0800907
Misha Wagner26531762019-01-21 14:18:51 +0000908 // Only trigger the tracker if all intervals are correct
909 if (useAnomalyDetection) {
910 // TODO: propgate proper values down stream when anomaly support doubles
tsaichristinec876b492019-12-10 13:47:05 -0800911 long wholeBucketVal = intervals[0].value.long_value;
Misha Wagner26531762019-01-21 14:18:51 +0000912 auto prev = mCurrentFullBucket.find(eventKey);
913 if (prev != mCurrentFullBucket.end()) {
914 wholeBucketVal += prev->second;
915 }
916 for (auto& tracker : mAnomalyTrackers) {
Yao Chen4ce07292019-02-13 13:06:36 -0800917 tracker->detectAndDeclareAnomaly(eventTimeNs, mCurrentBucketNum, mMetricId, eventKey,
918 wholeBucketVal);
Misha Wagner26531762019-01-21 14:18:51 +0000919 }
Bookatzde1b55622017-12-14 18:38:27 -0800920 }
Yangster8de69392017-11-27 13:48:29 -0800921}
922
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000923// For pulled metrics, we always need to make sure we do a pull before flushing the bucket
924// if mCondition is true!
Yangster-macb142cc82018-03-30 15:22:08 -0700925void ValueMetricProducer::flushIfNeededLocked(const int64_t& eventTimeNs) {
926 int64_t currentBucketEndTimeNs = getCurrentBucketEndTimeNs();
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700927 if (eventTimeNs < currentBucketEndTimeNs) {
tsaichristine90f95bb2019-11-06 17:06:53 -0800928 VLOG("eventTime is %lld, less than current bucket end time %lld", (long long)eventTimeNs,
David Chen27785a82018-01-19 17:06:45 -0800929 (long long)(currentBucketEndTimeNs));
Chenjie Yub3dda412017-10-24 13:41:59 -0700930 return;
931 }
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000932 int64_t numBucketsForward = calcBucketsForwardCount(eventTimeNs);
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000933 int64_t nextBucketStartTimeNs = currentBucketEndTimeNs + (numBucketsForward - 1) * mBucketSizeNs;
934 flushCurrentBucketLocked(eventTimeNs, nextBucketStartTimeNs);
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000935}
David Chen27785a82018-01-19 17:06:45 -0800936
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000937int64_t ValueMetricProducer::calcBucketsForwardCount(const int64_t& eventTimeNs) const {
938 int64_t currentBucketEndTimeNs = getCurrentBucketEndTimeNs();
939 if (eventTimeNs < currentBucketEndTimeNs) {
940 return 0;
David Chen27785a82018-01-19 17:06:45 -0800941 }
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000942 return 1 + (eventTimeNs - currentBucketEndTimeNs) / mBucketSizeNs;
David Chen27785a82018-01-19 17:06:45 -0800943}
944
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000945void ValueMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs,
946 const int64_t& nextBucketStartTimeNs) {
Olivier Gaillarde63d9e02019-02-12 14:43:59 +0000947 if (mCondition == ConditionState::kUnknown) {
948 StatsdStats::getInstance().noteBucketUnknownCondition(mMetricId);
949 }
950
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000951 int64_t numBucketsForward = calcBucketsForwardCount(eventTimeNs);
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000952 if (numBucketsForward > 1) {
953 VLOG("Skipping forward %lld buckets", (long long)numBucketsForward);
954 StatsdStats::getInstance().noteSkippedForwardBuckets(mMetricId);
955 // Something went wrong. Maybe the device was sleeping for a long time. It is better
956 // to mark the current bucket as invalid. The last pull might have been successful through.
tsaichristineb87ca152019-12-09 15:19:41 -0800957 invalidateCurrentBucketWithoutResetBase(eventTimeNs,
958 BucketDropReason::MULTIPLE_BUCKETS_SKIPPED);
Olivier Gaillard47a9efc2019-02-22 15:43:31 +0000959 }
960
Chenjie Yub3dda412017-10-24 13:41:59 -0700961 VLOG("finalizing bucket for %ld, dumping %d slices", (long)mCurrentBucketStartTimeNs,
962 (int)mCurrentSlicedBucket.size());
Yangster-macb142cc82018-03-30 15:22:08 -0700963 int64_t fullBucketEndTimeNs = getCurrentBucketEndTimeNs();
Chenjie Yu32717c32018-10-20 23:54:48 -0700964 int64_t bucketEndTime = eventTimeNs < fullBucketEndTimeNs ? eventTimeNs : fullBucketEndTimeNs;
Yao Chene6cfb142019-04-08 12:00:01 -0700965 // Close the current bucket.
966 int64_t conditionTrueDuration = mConditionTimer.newBucketStart(bucketEndTime);
Olivier Gaillard9a5d3592019-02-05 15:12:39 +0000967 bool isBucketLargeEnough = bucketEndTime - mCurrentBucketStartTimeNs >= mMinBucketSizeNs;
tsaichristineb87ca152019-12-09 15:19:41 -0800968 if (!isBucketLargeEnough) {
Jeffrey Huangde3348a2020-05-20 16:06:13 -0700969 skipCurrentBucket(eventTimeNs, BucketDropReason::BUCKET_TOO_SMALL);
tsaichristineb87ca152019-12-09 15:19:41 -0800970 }
Jeffrey Huangde3348a2020-05-20 16:06:13 -0700971 bool bucketHasData = false;
972 if (!mCurrentBucketIsSkipped) {
David Chen81245fd2018-04-12 14:33:37 -0700973 // The current bucket is large enough to keep.
David Chen81245fd2018-04-12 14:33:37 -0700974 for (const auto& slice : mCurrentSlicedBucket) {
Olivier Gaillard9a5d3592019-02-05 15:12:39 +0000975 ValueBucket bucket = buildPartialBucket(bucketEndTime, slice.second);
Yao Chene6cfb142019-04-08 12:00:01 -0700976 bucket.mConditionTrueNs = conditionTrueDuration;
Chenjie Yu32717c32018-10-20 23:54:48 -0700977 // it will auto create new vector of ValuebucketInfo if the key is not found.
978 if (bucket.valueIndex.size() > 0) {
David Chen81245fd2018-04-12 14:33:37 -0700979 auto& bucketList = mPastBuckets[slice.first];
Chenjie Yu32717c32018-10-20 23:54:48 -0700980 bucketList.push_back(bucket);
Jeffrey Huangde3348a2020-05-20 16:06:13 -0700981 bucketHasData = true;
David Chen81245fd2018-04-12 14:33:37 -0700982 }
Chenjie Yuae63b0a2018-04-10 14:59:31 -0700983 }
Jeffrey Huangde3348a2020-05-20 16:06:13 -0700984 }
985
986 if (!bucketHasData && !mCurrentBucketIsSkipped) {
987 skipCurrentBucket(eventTimeNs, BucketDropReason::NO_DATA);
988 }
989
990 if (mCurrentBucketIsSkipped) {
tsaichristine45f703b2020-02-03 10:44:39 -0800991 mCurrentSkippedBucket.bucketStartTimeNs = mCurrentBucketStartTimeNs;
992 mCurrentSkippedBucket.bucketEndTimeNs = bucketEndTime;
tsaichristineb87ca152019-12-09 15:19:41 -0800993 mSkippedBuckets.emplace_back(mCurrentSkippedBucket);
Chenjie Yub3dda412017-10-24 13:41:59 -0700994 }
995
Olivier Gaillardc3719912019-03-15 17:33:40 +0000996 appendToFullBucket(eventTimeNs, fullBucketEndTimeNs);
Olivier Gaillarda8b70112019-02-25 11:24:23 +0000997 initCurrentSlicedBucket(nextBucketStartTimeNs);
Yao Chene6cfb142019-04-08 12:00:01 -0700998 // Update the condition timer again, in case we skipped buckets.
999 mConditionTimer.newBucketStart(nextBucketStartTimeNs);
Olivier Gaillardc3719912019-03-15 17:33:40 +00001000 mCurrentBucketNum += numBucketsForward;
Olivier Gaillard9a5d3592019-02-05 15:12:39 +00001001}
1002
1003ValueBucket ValueMetricProducer::buildPartialBucket(int64_t bucketEndTime,
1004 const std::vector<Interval>& intervals) {
1005 ValueBucket bucket;
1006 bucket.mBucketStartNs = mCurrentBucketStartTimeNs;
1007 bucket.mBucketEndNs = bucketEndTime;
1008 for (const auto& interval : intervals) {
1009 if (interval.hasValue) {
1010 // skip the output if the diff is zero
1011 if (mSkipZeroDiffOutput && mUseDiff && interval.value.isZero()) {
1012 continue;
1013 }
1014 bucket.valueIndex.push_back(interval.valueIndex);
1015 if (mAggregationType != ValueMetric::AVG) {
1016 bucket.values.push_back(interval.value);
1017 } else {
1018 double sum = interval.value.type == LONG ? (double)interval.value.long_value
1019 : interval.value.double_value;
1020 bucket.values.push_back(Value((double)sum / interval.sampleSize));
1021 }
1022 }
1023 }
1024 return bucket;
1025}
1026
Olivier Gaillarda8b70112019-02-25 11:24:23 +00001027void ValueMetricProducer::initCurrentSlicedBucket(int64_t nextBucketStartTimeNs) {
1028 StatsdStats::getInstance().noteBucketCount(mMetricId);
1029 // Cleanup data structure to aggregate values.
Olivier Gaillard9a5d3592019-02-05 15:12:39 +00001030 for (auto it = mCurrentSlicedBucket.begin(); it != mCurrentSlicedBucket.end();) {
1031 bool obsolete = true;
1032 for (auto& interval : it->second) {
1033 interval.hasValue = false;
1034 interval.sampleSize = 0;
1035 if (interval.seenNewData) {
1036 obsolete = false;
1037 }
1038 interval.seenNewData = false;
1039 }
1040
1041 if (obsolete) {
1042 it = mCurrentSlicedBucket.erase(it);
1043 } else {
1044 it++;
1045 }
tsaichristinec876b492019-12-10 13:47:05 -08001046 // TODO: remove mCurrentBaseInfo entries when obsolete
Olivier Gaillard9a5d3592019-02-05 15:12:39 +00001047 }
Olivier Gaillarda8b70112019-02-25 11:24:23 +00001048
Jeffrey Huangde3348a2020-05-20 16:06:13 -07001049 mCurrentBucketIsSkipped = false;
tsaichristineb87ca152019-12-09 15:19:41 -08001050 mCurrentSkippedBucket.reset();
1051
Olivier Gaillarda8b70112019-02-25 11:24:23 +00001052 // If we do not have a global base when the condition is true,
1053 // we will have incomplete bucket for the next bucket.
1054 if (mUseDiff && !mHasGlobalBase && mCondition) {
Jeffrey Huangde3348a2020-05-20 16:06:13 -07001055 mCurrentBucketIsSkipped = false;
Olivier Gaillarda8b70112019-02-25 11:24:23 +00001056 }
1057 mCurrentBucketStartTimeNs = nextBucketStartTimeNs;
1058 VLOG("metric %lld: new bucket start time: %lld", (long long)mMetricId,
1059 (long long)mCurrentBucketStartTimeNs);
Olivier Gaillard9a5d3592019-02-05 15:12:39 +00001060}
1061
1062void ValueMetricProducer::appendToFullBucket(int64_t eventTimeNs, int64_t fullBucketEndTimeNs) {
Olivier Gaillardc3719912019-03-15 17:33:40 +00001063 bool isFullBucketReached = eventTimeNs > fullBucketEndTimeNs;
Jeffrey Huangde3348a2020-05-20 16:06:13 -07001064 if (mCurrentBucketIsSkipped) {
Olivier Gaillardc3719912019-03-15 17:33:40 +00001065 if (isFullBucketReached) {
1066 // If the bucket is invalid, we ignore the full bucket since it contains invalid data.
1067 mCurrentFullBucket.clear();
1068 }
1069 // Current bucket is invalid, we do not add it to the full bucket.
1070 return;
1071 }
1072
1073 if (isFullBucketReached) { // If full bucket, send to anomaly tracker.
David Chen27785a82018-01-19 17:06:45 -08001074 // Accumulate partial buckets with current value and then send to anomaly tracker.
1075 if (mCurrentFullBucket.size() > 0) {
1076 for (const auto& slice : mCurrentSlicedBucket) {
Chenjie Yudbe5c502018-11-30 23:15:57 -08001077 if (hitFullBucketGuardRailLocked(slice.first)) {
1078 continue;
1079 }
Chenjie Yua0f02242018-07-06 16:14:34 -07001080 // TODO: fix this when anomaly can accept double values
Olivier Gaillardc3719912019-03-15 17:33:40 +00001081 auto& interval = slice.second[0];
1082 if (interval.hasValue) {
1083 mCurrentFullBucket[slice.first] += interval.value.long_value;
1084 }
David Chen27785a82018-01-19 17:06:45 -08001085 }
1086 for (const auto& slice : mCurrentFullBucket) {
1087 for (auto& tracker : mAnomalyTrackers) {
1088 if (tracker != nullptr) {
1089 tracker->addPastBucket(slice.first, slice.second, mCurrentBucketNum);
1090 }
1091 }
1092 }
1093 mCurrentFullBucket.clear();
1094 } else {
1095 // Skip aggregating the partial buckets since there's no previous partial bucket.
1096 for (const auto& slice : mCurrentSlicedBucket) {
1097 for (auto& tracker : mAnomalyTrackers) {
1098 if (tracker != nullptr) {
Chenjie Yua0f02242018-07-06 16:14:34 -07001099 // TODO: fix this when anomaly can accept double values
Olivier Gaillardc3719912019-03-15 17:33:40 +00001100 auto& interval = slice.second[0];
1101 if (interval.hasValue) {
1102 tracker->addPastBucket(slice.first, interval.value.long_value,
1103 mCurrentBucketNum);
1104 }
David Chen27785a82018-01-19 17:06:45 -08001105 }
1106 }
1107 }
1108 }
1109 } else {
1110 // Accumulate partial bucket.
1111 for (const auto& slice : mCurrentSlicedBucket) {
Chenjie Yua0f02242018-07-06 16:14:34 -07001112 // TODO: fix this when anomaly can accept double values
Olivier Gaillardc3719912019-03-15 17:33:40 +00001113 auto& interval = slice.second[0];
1114 if (interval.hasValue) {
1115 mCurrentFullBucket[slice.first] += interval.value.long_value;
1116 }
David Chen27785a82018-01-19 17:06:45 -08001117 }
1118 }
Chenjie Yub3dda412017-10-24 13:41:59 -07001119}
1120
Yangsterf2bee6f2017-11-29 12:01:05 -08001121size_t ValueMetricProducer::byteSizeLocked() const {
Yangster-mace2cd6d52017-11-09 20:38:30 -08001122 size_t totalSize = 0;
1123 for (const auto& pair : mPastBuckets) {
1124 totalSize += pair.second.size() * kBucketSize;
1125 }
1126 return totalSize;
yro2b0f8862017-11-06 14:27:31 -08001127}
1128
Chenjie Yub3dda412017-10-24 13:41:59 -07001129} // namespace statsd
1130} // namespace os
Yao Chen93fe3a32017-11-02 13:52:59 -07001131} // namespace android