Add recording of detailed information for freezes and judders
Add judder and freeze events which contain detailed information about
clustered groups of freeze and judder occurrences. This information is
contained in the media.metrics dumpsys for inclusion in bug reports, but
is not enabled by default and must be configured as enabled during a
debugging session or configured as enabled based on device-ids or
similar.
Bug: 234833109
Test: atest VideoRenderQualityTracker_test
Test: play YouTube videos with events programatically enabled
Change-Id: If790eb8444a160bfb7629df7160454ebe710410d
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 94851fb..bdf0894 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -19,12 +19,12 @@
#define LOG_TAG "MediaCodec"
#include <utils/Log.h>
-#include <set>
-#include <random>
-#include <stdlib.h>
-#include <inttypes.h>
-#include <stdlib.h>
#include <dlfcn.h>
+#include <inttypes.h>
+#include <random>
+#include <set>
+#include <stdlib.h>
+#include <string>
#include <C2Buffer.h>
@@ -90,6 +90,8 @@
using aidl::android::media::IResourceManagerClient;
using aidl::android::media::IResourceManagerService;
using aidl::android::media::ClientInfoParcel;
+using FreezeEvent = VideoRenderQualityTracker::FreezeEvent;
+using JudderEvent = VideoRenderQualityTracker::JudderEvent;
// key for media statistics
static const char *kCodecKeyName = "codec";
@@ -212,6 +214,7 @@
static const char *kCodecFramerateContent = "android.media.mediacodec.framerate-content";
static const char *kCodecFramerateDesired = "android.media.mediacodec.framerate-desired";
static const char *kCodecFramerateActual = "android.media.mediacodec.framerate-actual";
+// Freeze
static const char *kCodecFreezeCount = "android.media.mediacodec.freeze-count";
static const char *kCodecFreezeScore = "android.media.mediacodec.freeze-score";
static const char *kCodecFreezeRate = "android.media.mediacodec.freeze-rate";
@@ -226,6 +229,7 @@
"android.media.mediacodec.freeze-distance-ms-histogram";
static const char *kCodecFreezeDistanceMsHistogramBuckets =
"android.media.mediacodec.freeze-distance-ms-histogram-buckets";
+// Judder
static const char *kCodecJudderCount = "android.media.mediacodec.judder-count";
static const char *kCodecJudderScore = "android.media.mediacodec.judder-score";
static const char *kCodecJudderRate = "android.media.mediacodec.judder-rate";
@@ -234,6 +238,32 @@
static const char *kCodecJudderScoreHistogram = "android.media.mediacodec.judder-score-histogram";
static const char *kCodecJudderScoreHistogramBuckets =
"android.media.mediacodec.judder-score-histogram-buckets";
+// Freeze event
+static const char *kCodecFreezeEventCount = "android.media.mediacodec.freeze-event-count";
+static const char *kFreezeEventKeyName = "freeze";
+static const char *kFreezeEventInitialTimeUs = "android.media.mediacodec.freeze.initial-time-us";
+static const char *kFreezeEventDurationMs = "android.media.mediacodec.freeze.duration-ms";
+static const char *kFreezeEventCount = "android.media.mediacodec.freeze.count";
+static const char *kFreezeEventAvgDurationMs = "android.media.mediacodec.freeze.avg-duration-ms";
+static const char *kFreezeEventAvgDistanceMs = "android.media.mediacodec.freeze.avg-distance-ms";
+static const char *kFreezeEventDetailsDurationMs =
+ "android.media.mediacodec.freeze.detail-duration-ms";
+static const char *kFreezeEventDetailsDistanceMs =
+ "android.media.mediacodec.freeze.detail-distance-ms";
+// Judder event
+static const char *kCodecJudderEventCount = "android.media.mediacodec.judder-event-count";
+static const char *kJudderEventKeyName = "judder";
+static const char *kJudderEventInitialTimeUs = "android.media.mediacodec.judder.initial-time-us";
+static const char *kJudderEventDurationMs = "android.media.mediacodec.judder.duration-ms";
+static const char *kJudderEventCount = "android.media.mediacodec.judder.count";
+static const char *kJudderEventAvgScore = "android.media.mediacodec.judder.avg-score";
+static const char *kJudderEventAvgDistanceMs = "android.media.mediacodec.judder.avg-distance-ms";
+static const char *kJudderEventDetailsActualDurationUs =
+ "android.media.mediacodec.judder.detail-actual-duration-us";
+static const char *kJudderEventDetailsContentDurationUs =
+ "android.media.mediacodec.judder.detail-content-duration-us";
+static const char *kJudderEventDetailsDistanceMs =
+ "android.media.mediacodec.judder.detail-distance-ms";
// XXX suppress until we get our representation right
static bool kEmitHistogram = false;
@@ -1169,6 +1199,12 @@
mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogramBuckets,
h.emitBuckets());
}
+ if (m.freezeEventCount != 0) {
+ mediametrics_setInt32(mMetricsHandle, kCodecFreezeEventCount, m.freezeEventCount);
+ }
+ if (m.judderEventCount != 0) {
+ mediametrics_setInt32(mMetricsHandle, kCodecJudderEventCount, m.judderEventCount);
+ }
}
if (mLatencyHist.getCount() != 0 ) {
@@ -1407,6 +1443,53 @@
}
}
+static std::string emitVector(std::vector<int32_t> vector) {
+ std::ostringstream sstr;
+ for (int i = 0; i < vector.size(); ++i) {
+ if (i != 0) {
+ sstr << ',';
+ }
+ sstr << vector[i];
+ }
+ return sstr.str();
+}
+
+static void reportToMediaMetricsIfValid(const FreezeEvent &e) {
+ if (e.valid) {
+ mediametrics_handle_t handle = mediametrics_create(kFreezeEventKeyName);
+ mediametrics_setInt64(handle, kFreezeEventInitialTimeUs, e.initialTimeUs);
+ mediametrics_setInt32(handle, kFreezeEventDurationMs, e.durationMs);
+ mediametrics_setInt64(handle, kFreezeEventCount, e.count);
+ mediametrics_setInt32(handle, kFreezeEventAvgDurationMs, e.sumDurationMs / e.count);
+ mediametrics_setInt32(handle, kFreezeEventAvgDistanceMs, e.sumDistanceMs / e.count);
+ mediametrics_setString(handle, kFreezeEventDetailsDurationMs,
+ emitVector(e.details.durationMs));
+ mediametrics_setString(handle, kFreezeEventDetailsDistanceMs,
+ emitVector(e.details.distanceMs));
+ mediametrics_selfRecord(handle);
+ mediametrics_delete(handle);
+ }
+}
+
+static void reportToMediaMetricsIfValid(const JudderEvent &e) {
+ if (e.valid) {
+ mediametrics_handle_t handle = mediametrics_create(kJudderEventKeyName);
+ mediametrics_setInt64(handle, kJudderEventInitialTimeUs, e.initialTimeUs);
+ mediametrics_setInt32(handle, kJudderEventDurationMs, e.durationMs);
+ mediametrics_setInt64(handle, kJudderEventCount, e.count);
+ mediametrics_setInt32(handle, kJudderEventAvgScore, e.sumScore / e.count);
+ mediametrics_setInt32(handle, kJudderEventAvgDistanceMs, e.sumDistanceMs / e.count);
+ mediametrics_setString(handle, kJudderEventDetailsActualDurationUs,
+ emitVector(e.details.actualRenderDurationUs));
+ mediametrics_setString(handle, kJudderEventDetailsContentDurationUs,
+ emitVector(e.details.contentRenderDurationUs));
+ mediametrics_setString(handle, kJudderEventDetailsDistanceMs,
+ emitVector(e.details.distanceMs));
+ mediametrics_selfRecord(handle);
+ mediametrics_delete(handle);
+ }
+}
+
void MediaCodec::flushMediametrics() {
ALOGD("flushMediametrics");
@@ -1425,6 +1508,10 @@
}
// we no longer have anything pending upload
mMetricsToUpload = false;
+
+ // Freeze and judder events are reported separately
+ reportToMediaMetricsIfValid(mVideoRenderQualityTracker.getAndResetFreezeEvent());
+ reportToMediaMetricsIfValid(mVideoRenderQualityTracker.getAndResetJudderEvent());
}
void MediaCodec::updateLowLatency(const sp<AMessage> &msg) {
@@ -1538,7 +1625,12 @@
// Tunneled frames use INT64_MAX to indicate end-of-stream, so don't report it as a
// rendered frame.
if (!mTunneled || mediaTimeUs != INT64_MAX) {
- mVideoRenderQualityTracker.onFrameRendered(mediaTimeUs, renderTimeNs);
+ FreezeEvent freezeEvent;
+ JudderEvent judderEvent;
+ mVideoRenderQualityTracker.onFrameRendered(mediaTimeUs, renderTimeNs, &freezeEvent,
+ &judderEvent);
+ reportToMediaMetricsIfValid(freezeEvent);
+ reportToMediaMetricsIfValid(judderEvent);
}
}
}
diff --git a/media/libstagefright/VideoRenderQualityTracker.cpp b/media/libstagefright/VideoRenderQualityTracker.cpp
index 239aefc..ac451ce 100644
--- a/media/libstagefright/VideoRenderQualityTracker.cpp
+++ b/media/libstagefright/VideoRenderQualityTracker.cpp
@@ -42,8 +42,10 @@
contentFrameRate = FRAME_RATE_UNDETERMINED;
desiredFrameRate = FRAME_RATE_UNDETERMINED;
actualFrameRate = FRAME_RATE_UNDETERMINED;
+ freezeEventCount = 0;
freezeDurationMsHistogram.clear();
freezeDistanceMsHistogram.clear();
+ judderEventCount = 0;
judderScoreHistogram.clear();
}
@@ -69,11 +71,17 @@
freezeDurationMsHistogramToScore = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
freezeDistanceMsHistogramBuckets = {0, 20, 100, 400, 1000, 2000, 3000, 4000, 8000, 15000, 30000,
60000};
+ freezeEventMax = 0; // enabled only when debugging
+ freezeEventDetailsMax = 20;
+ freezeEventDistanceToleranceMs = 60000; // lump freeze occurrences together when 60s or less
// Judder configuration
judderErrorToleranceUs = 2000;
judderScoreHistogramBuckets = {1, 4, 5, 9, 11, 20, 30, 40, 50, 60, 70, 80};
judderScoreHistogramToScore = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+ judderEventMax = 0; // enabled only when debugging
+ judderEventDetailsMax = 20;
+ judderEventDistanceToleranceMs = 5000; // lump judder occurrences together when 5s or less
}
VideoRenderQualityTracker::VideoRenderQualityTracker() : mConfiguration(Configuration()) {
@@ -139,7 +147,9 @@
mLastContentTimeUs = contentTimeUs;
}
-void VideoRenderQualityTracker::onFrameRendered(int64_t contentTimeUs, int64_t actualRenderTimeNs) {
+void VideoRenderQualityTracker::onFrameRendered(int64_t contentTimeUs, int64_t actualRenderTimeNs,
+ FreezeEvent *freezeEventOut,
+ JudderEvent *judderEventOut) {
if (!mConfiguration.enabled) {
return;
}
@@ -183,10 +193,23 @@
nextExpectedFrame.desiredRenderTimeUs);
}
processMetricsForRenderedFrame(nextExpectedFrame.contentTimeUs,
- nextExpectedFrame.desiredRenderTimeUs, actualRenderTimeUs);
+ nextExpectedFrame.desiredRenderTimeUs, actualRenderTimeUs,
+ freezeEventOut, judderEventOut);
mLastRenderTimeUs = actualRenderTimeUs;
}
+VideoRenderQualityTracker::FreezeEvent VideoRenderQualityTracker::getAndResetFreezeEvent() {
+ FreezeEvent event = std::move(mFreezeEvent);
+ mFreezeEvent.valid = false;
+ return event;
+}
+
+VideoRenderQualityTracker::JudderEvent VideoRenderQualityTracker::getAndResetJudderEvent() {
+ JudderEvent event = std::move(mJudderEvent);
+ mJudderEvent.valid = false;
+ return event;
+}
+
const VideoRenderQualityMetrics &VideoRenderQualityTracker::getMetrics() {
if (!mConfiguration.enabled) {
return mMetrics;
@@ -232,7 +255,10 @@
mLastContentTimeUs = -1;
mLastRenderTimeUs = -1;
mLastFreezeEndTimeUs = -1;
+ mLastJudderEndTimeUs = -1;
mWasPreviousFrameDropped = false;
+ mFreezeEvent.valid = false;
+ mJudderEvent.valid = false;
// Don't worry about tracking frame rendering times from now up until playback catches up to the
// discontinuity. While stuttering or freezing could be found in the next few frames, the impact
@@ -315,7 +341,9 @@
void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTimeUs,
int64_t desiredRenderTimeUs,
- int64_t actualRenderTimeUs) {
+ int64_t actualRenderTimeUs,
+ FreezeEvent *freezeEventOut,
+ JudderEvent *judderEventOut) {
// Capture the timestamp at which the first frame was rendered
if (mMetrics.firstRenderTimeUs == 0) {
mMetrics.firstRenderTimeUs = actualRenderTimeUs;
@@ -338,30 +366,86 @@
// If the previous frame was dropped, there was a freeze if we've already rendered a frame
if (mWasPreviousFrameDropped && mLastRenderTimeUs != -1) {
- processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mMetrics);
+ processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent,
+ mMetrics, mConfiguration);
mLastFreezeEndTimeUs = actualRenderTimeUs;
}
+ maybeCaptureFreezeEvent(actualRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent, mMetrics,
+ mConfiguration, freezeEventOut);
// Judder is computed on the prior video frame, not the current video frame
int64_t judderScore = computePreviousJudderScore(mActualFrameDurationUs,
mContentFrameDurationUs,
mConfiguration);
+ int64_t judderTimeUs = actualRenderTimeUs - mActualFrameDurationUs[0] -
+ mActualFrameDurationUs[1];
if (judderScore != 0) {
- mMetrics.judderScoreHistogram.insert(judderScore);
+ processJudder(judderScore, judderTimeUs, mLastJudderEndTimeUs, mActualFrameDurationUs,
+ mContentFrameDurationUs, mJudderEvent, mMetrics, mConfiguration);
+ mLastJudderEndTimeUs = judderTimeUs + mActualFrameDurationUs[1];
}
+ maybeCaptureJudderEvent(actualRenderTimeUs, mLastJudderEndTimeUs, mJudderEvent, mMetrics,
+ mConfiguration, judderEventOut);
mWasPreviousFrameDropped = false;
}
void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
- int64_t lastFreezeEndTimeUs,
- VideoRenderQualityMetrics &m) {
- int64_t freezeDurationMs = (actualRenderTimeUs - lastRenderTimeUs) / 1000;
- m.freezeDurationMsHistogram.insert(freezeDurationMs);
+ int64_t lastFreezeEndTimeUs, FreezeEvent &e,
+ VideoRenderQualityMetrics &m,
+ const Configuration &c) {
+ int32_t durationMs = int32_t((actualRenderTimeUs - lastRenderTimeUs) / 1000);
+ m.freezeDurationMsHistogram.insert(durationMs);
+ int32_t distanceMs = -1;
if (lastFreezeEndTimeUs != -1) {
- int64_t distanceSinceLastFreezeMs = (lastRenderTimeUs - lastFreezeEndTimeUs) / 1000;
- m.freezeDistanceMsHistogram.insert(distanceSinceLastFreezeMs);
+ // The distance to the last freeze is measured from the end of the last freze to the start
+ // of this freeze.
+ distanceMs = int32_t((lastRenderTimeUs - lastFreezeEndTimeUs) / 1000);
+ m.freezeDistanceMsHistogram.insert(distanceMs);
}
+ if (c.freezeEventMax > 0) {
+ if (e.valid == false) {
+ m.freezeEventCount++;
+ e.valid = true;
+ e.initialTimeUs = lastRenderTimeUs;
+ e.durationMs = 0;
+ e.sumDurationMs = 0;
+ e.sumDistanceMs = 0;
+ e.count = 0;
+ e.details.durationMs.clear();
+ e.details.distanceMs.clear();
+ } else if (distanceMs != -1) {
+ e.durationMs += distanceMs;
+ e.sumDistanceMs += distanceMs;
+ }
+ e.durationMs += durationMs;
+ e.count++;
+ e.sumDurationMs += durationMs;
+ if (e.details.durationMs.size() < c.freezeEventDetailsMax) {
+ e.details.durationMs.push_back(durationMs);
+ e.details.distanceMs.push_back(distanceMs);
+ }
+ }
+}
+
+void VideoRenderQualityTracker::maybeCaptureFreezeEvent(int64_t actualRenderTimeUs,
+ int64_t lastFreezeEndTimeUs, FreezeEvent &e,
+ const VideoRenderQualityMetrics & m,
+ const Configuration &c,
+ FreezeEvent *freezeEventOut) {
+ if (lastFreezeEndTimeUs == -1 || !e.valid) {
+ return;
+ }
+ // Future freeze occurrences are still pulled into the current freeze event if under tolerance
+ int64_t distanceMs = (actualRenderTimeUs - lastFreezeEndTimeUs) / 1000;
+ if (distanceMs < c.freezeEventDistanceToleranceMs) {
+ return;
+ }
+ if (freezeEventOut != nullptr && m.freezeEventCount <= c.freezeEventMax) {
+ *freezeEventOut = std::move(e);
+ }
+ // start recording a new freeze event after pushing the current one back to the caller
+ e.valid = false;
}
int64_t VideoRenderQualityTracker::computePreviousJudderScore(
@@ -406,6 +490,64 @@
return abs(errorUs) / 1000; // error in millis to keep numbers small
}
+void VideoRenderQualityTracker::processJudder(int32_t judderScore, int64_t judderTimeUs,
+ int64_t lastJudderEndTime,
+ const FrameDurationUs &actualDurationUs,
+ const FrameDurationUs &contentDurationUs,
+ JudderEvent &e, VideoRenderQualityMetrics &m,
+ const Configuration &c) {
+ int32_t distanceMs = -1;
+ if (lastJudderEndTime != -1) {
+ distanceMs = int32_t((judderTimeUs - lastJudderEndTime) / 1000);
+ }
+ m.judderScoreHistogram.insert(judderScore);
+ if (c.judderEventMax > 0) {
+ if (!e.valid) {
+ m.judderEventCount++;
+ e.valid = true;
+ e.initialTimeUs = judderTimeUs;
+ e.durationMs = 0;
+ e.sumScore = 0;
+ e.sumDistanceMs = 0;
+ e.count = 0;
+ e.details.contentRenderDurationUs.clear();
+ e.details.actualRenderDurationUs.clear();
+ e.details.distanceMs.clear();
+ } else if (distanceMs != -1) {
+ e.durationMs += distanceMs;
+ e.sumDistanceMs += distanceMs;
+ }
+ e.durationMs += actualDurationUs[1] / 1000;
+ e.count++;
+ e.sumScore += judderScore;
+ if (e.details.contentRenderDurationUs.size() < c.judderEventDetailsMax) {
+ e.details.actualRenderDurationUs.push_back(actualDurationUs[1]);
+ e.details.contentRenderDurationUs.push_back(contentDurationUs[1]);
+ e.details.distanceMs.push_back(distanceMs);
+ }
+ }
+}
+
+void VideoRenderQualityTracker::maybeCaptureJudderEvent(int64_t actualRenderTimeUs,
+ int64_t lastJudderEndTimeUs, JudderEvent &e,
+ const VideoRenderQualityMetrics &m,
+ const Configuration &c,
+ JudderEvent *judderEventOut) {
+ if (lastJudderEndTimeUs == -1 || !e.valid) {
+ return;
+ }
+ // Future judder occurrences are still pulled into the current judder event if under tolerance
+ int64_t distanceMs = (actualRenderTimeUs - lastJudderEndTimeUs) / 1000;
+ if (distanceMs < c.judderEventDistanceToleranceMs) {
+ return;
+ }
+ if (judderEventOut != nullptr && m.judderEventCount <= c.judderEventMax) {
+ *judderEventOut = std::move(e);
+ }
+ // start recording a new judder event after pushing the current one back to the caller
+ e.valid = false;
+}
+
void VideoRenderQualityTracker::configureHistograms(VideoRenderQualityMetrics &m,
const Configuration &c) {
m.freezeDurationMsHistogram.setup(c.freezeDurationMsHistogramBuckets);
diff --git a/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h b/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
index 5cf674f..6c557bf 100644
--- a/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
+++ b/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
@@ -66,6 +66,8 @@
int32_t freezeScore;
// The computed percentage of total playback duration that was frozen.
float freezeRate;
+ // The number of freeze events.
+ int32_t freezeEventCount;
// A histogram of the durations between each freeze.
MediaHistogram<int32_t> freezeDistanceMsHistogram;
@@ -76,6 +78,8 @@
int32_t judderScore;
// The computed percentage of total frames that had judder.
float judderRate;
+ // The number of judder events.
+ int32_t judderEventCount;
};
///////////////////////////////////////////////////////
@@ -135,6 +139,13 @@
std::vector<int64_t> freezeDurationMsHistogramToScore;
// The values used to distribute distances between freezes across a histogram.
std::vector<int32_t> freezeDistanceMsHistogramBuckets;
+ // The maximum number of freeze events to send back to the caller.
+ int64_t freezeEventMax;
+ // The maximum number of detail entries tracked per freeze event.
+ int64_t freezeEventDetailsMax;
+ // The maximum distance in time between two freeze occurrences such that both will be
+ // lumped into the same freeze event.
+ int64_t freezeEventDistanceToleranceMs;
// Judder configuration
//
@@ -145,6 +156,69 @@
// The values used to compare against judder score histogram counts when determining an
// overall score.
std::vector<int32_t> judderScoreHistogramToScore;
+ // The maximum number of judder events to send back to the caller.
+ int64_t judderEventMax;
+ // The maximum number of detail entries tracked per judder event.
+ int64_t judderEventDetailsMax;
+ // The maximum distance in time between two judder occurrences such that both will be
+ // lumped into the same judder event.
+ int64_t judderEventDistanceToleranceMs;
+ };
+
+ struct FreezeEvent {
+ // Details are captured for each freeze up to a limited number. The arrays are guaranteed to
+ // have the same size.
+ struct Details {
+ /// The duration of the freeze.
+ std::vector<int32_t> durationMs;
+ // The distance between the beginning of this freeze and the end of the previous freeze.
+ std::vector<int32_t> distanceMs;
+ };
+ FreezeEvent() : valid(false) {}
+ // Whether or not the data in this structure is valid.
+ bool valid;
+ // The time at which the first freeze for this event was detected.
+ int64_t initialTimeUs;
+ // The total duration from the beginning of the first freeze to the end of the last freeze
+ // in this event.
+ int32_t durationMs;
+ // The number of freezes in this event.
+ int64_t count;
+ // The sum of all durations of all freezes in this event.
+ int64_t sumDurationMs;
+ // The sum of all distances between each freeze in this event.
+ int64_t sumDistanceMs;
+ // Detailed information for the first N freezes in this event.
+ Details details;
+ };
+
+ struct JudderEvent {
+ // Details are captured for each frame judder up to a limited number. The arrays are
+ // guaranteed to have the same size.
+ struct Details {
+ // The actual render duration of the frame for this judder occurrence.
+ std::vector<int32_t> actualRenderDurationUs;
+ // The content render duration of the frame for this judder occurrence.
+ std::vector<int32_t> contentRenderDurationUs;
+ // The distance from this judder occurrence and the previous judder occurrence.
+ std::vector<int32_t> distanceMs;
+ };
+ JudderEvent() : valid(false) {}
+ // Whether or not the data in this structure is valid.
+ bool valid;
+ // The time at which the first judder occurrence for this event was detected.
+ int64_t initialTimeUs;
+ // The total duration from the first judder occurrence to the last judder occurrence in this
+ // event.
+ int32_t durationMs;
+ // The number of judder occurrences in this event.
+ int64_t count;
+ // The sum of all judder scores in this event.
+ int64_t sumScore;
+ // The sum of all distances between each judder occurrence in this event.
+ int64_t sumDistanceMs;
+ // Detailed information for the first N judder occurrences in this event.
+ Details details;
};
VideoRenderQualityTracker();
@@ -164,7 +238,16 @@
void onFrameReleased(int64_t contentTimeUs, int64_t desiredRenderTimeNs);
// Called when the system has detected that the frame has actually been rendered to the display.
- void onFrameRendered(int64_t contentTimeUs, int64_t actualRenderTimeNs);
+ // Returns any freeze events or judder events that were detected.
+ void onFrameRendered(int64_t contentTimeUs, int64_t actualRenderTimeNs,
+ FreezeEvent *freezeEventOut = nullptr,
+ JudderEvent *judderEventOut = nullptr);
+
+ // Gets and resets data for the current freeze event.
+ FreezeEvent getAndResetFreezeEvent();
+
+ // Gets and resets data for the current judder event.
+ JudderEvent getAndResetJudderEvent();
// Retrieve the metrics.
const VideoRenderQualityMetrics &getMetrics();
@@ -233,13 +316,31 @@
// Process a frame freeze.
static void processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
- int64_t lastFreezeEndTimeUs, VideoRenderQualityMetrics &m);
+ int64_t lastFreezeEndTimeUs, FreezeEvent &e,
+ VideoRenderQualityMetrics &m, const Configuration &c);
+
+ // Retrieve a freeze event if an event just finished.
+ static void maybeCaptureFreezeEvent(int64_t actualRenderTimeUs, int64_t lastFreezeEndTimeUs,
+ FreezeEvent &e, const VideoRenderQualityMetrics & m,
+ const Configuration &c, FreezeEvent *freezeEventOut);
// Compute a judder score for the previously-rendered frame.
static int64_t computePreviousJudderScore(const FrameDurationUs &actualRenderDurationUs,
const FrameDurationUs &contentRenderDurationUs,
const Configuration &c);
+ // Process a frame judder.
+ static void processJudder(int32_t judderScore, int64_t judderTimeUs,
+ int64_t lastJudderEndTimeUs,
+ const FrameDurationUs &contentDurationUs,
+ const FrameDurationUs &actualDurationUs, JudderEvent &e,
+ VideoRenderQualityMetrics &m, const Configuration &c);
+
+ // Retrieve a judder event if an event just finished.
+ static void maybeCaptureJudderEvent(int64_t actualRenderTimeUs, int64_t lastJudderEndTimeUs,
+ JudderEvent &e, const VideoRenderQualityMetrics & m,
+ const Configuration &c, JudderEvent *judderEventOut);
+
// Check to see if a discontinuity has occurred by examining the content time and the
// app-desired render time. If so, reset some internal state.
bool resetIfDiscontinuity(int64_t contentTimeUs, int64_t desiredRenderTimeUs);
@@ -252,7 +353,8 @@
// Update the metrics because a rendered frame was detected.
void processMetricsForRenderedFrame(int64_t contentTimeUs, int64_t desiredRenderTimeUs,
- int64_t actualRenderTimeUs);
+ int64_t actualRenderTimeUs,
+ FreezeEvent *freezeEventOut, JudderEvent *judderEventOut);
// Configurable elements of the metrics algorithms.
const Configuration mConfiguration;
@@ -269,12 +371,21 @@
// The most recent timestamp of the first frame rendered after the freeze.
int64_t mLastFreezeEndTimeUs;
- // The previous video frame was dropped.
- bool mWasPreviousFrameDropped;
+ // The most recent timestamp of frame judder.
+ int64_t mLastJudderEndTimeUs;
// The render duration of the playback.
int64_t mRenderDurationMs;
+ // True if the previous frame was dropped.
+ bool mWasPreviousFrameDropped;
+
+ // The freeze event that's currently being tracked.
+ FreezeEvent mFreezeEvent;
+
+ // The judder event that's currently being tracked.
+ JudderEvent mJudderEvent;
+
// Frames skipped at the end of playback shouldn't really be considered skipped, therefore keep
// a list of the frames, and process them as skipped frames the next time a frame is rendered.
std::list<int64_t> mPendingSkippedFrameContentTimeUsList;
diff --git a/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp b/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
index aca31f2..5aac34a 100644
--- a/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
+++ b/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
@@ -26,6 +26,8 @@
using Metrics = VideoRenderQualityMetrics;
using Configuration = VideoRenderQualityTracker::Configuration;
+using FreezeEvent = VideoRenderQualityTracker::FreezeEvent;
+using JudderEvent = VideoRenderQualityTracker::JudderEvent;
static constexpr float FRAME_RATE_UNDETERMINED = VideoRenderQualityMetrics::FRAME_RATE_UNDETERMINED;
static constexpr float FRAME_RATE_24_3_2_PULLDOWN =
@@ -48,7 +50,8 @@
void render(std::initializer_list<T> renderDurationMsList) {
for (auto renderDurationMs : renderDurationMsList) {
mVideoRenderQualityTracker.onFrameReleased(mMediaTimeUs);
- mVideoRenderQualityTracker.onFrameRendered(mMediaTimeUs, mClockTimeNs);
+ mVideoRenderQualityTracker.onFrameRendered(mMediaTimeUs, mClockTimeNs, &mFreezeEvent,
+ &mJudderEvent);
mMediaTimeUs += mContentFrameDurationUs;
mClockTimeNs += int64_t(renderDurationMs * 1000 * 1000);
}
@@ -58,7 +61,8 @@
int64_t durationUs = durationMs < 0 ? mContentFrameDurationUs : durationMs * 1000;
for (int i = 0; i < numFrames; ++i) {
mVideoRenderQualityTracker.onFrameReleased(mMediaTimeUs);
- mVideoRenderQualityTracker.onFrameRendered(mMediaTimeUs, mClockTimeNs);
+ mVideoRenderQualityTracker.onFrameRendered(mMediaTimeUs, mClockTimeNs, &mFreezeEvent,
+ &mJudderEvent);
mMediaTimeUs += mContentFrameDurationUs;
mClockTimeNs += durationUs * 1000;
}
@@ -84,11 +88,25 @@
return mVideoRenderQualityTracker.getMetrics();
}
+ FreezeEvent getAndClearFreezeEvent() {
+ FreezeEvent e = std::move(mFreezeEvent);
+ mFreezeEvent.valid = false;
+ return e;
+ }
+
+ JudderEvent getAndClearJudderEvent() {
+ JudderEvent e = std::move(mJudderEvent);
+ mJudderEvent.valid = false;
+ return e;
+ }
+
private:
VideoRenderQualityTracker mVideoRenderQualityTracker;
int64_t mContentFrameDurationUs;
int64_t mMediaTimeUs;
int64_t mClockTimeNs;
+ VideoRenderQualityTracker::FreezeEvent mFreezeEvent;
+ VideoRenderQualityTracker::JudderEvent mJudderEvent;
};
class VideoRenderQualityTrackerTest : public ::testing::Test {
@@ -511,4 +529,146 @@
}
}
+TEST_F(VideoRenderQualityTrackerTest, capturesFreezeEvents) {
+ Configuration c;
+ c.freezeEventMax = 5;
+ c.freezeEventDetailsMax = 4;
+ c.freezeEventDistanceToleranceMs = 1000;
+ Helper h(20, c);
+ h.render(10);
+ EXPECT_EQ(h.getAndClearFreezeEvent().valid, false);
+ h.drop(3);
+ h.render(1000 / 20); // +1 because it's unclear if the current frame is frozen
+ EXPECT_EQ(h.getAndClearFreezeEvent().valid, false);
+ h.drop(1);
+ h.render(10);
+ EXPECT_EQ(h.getAndClearFreezeEvent().valid, false);
+ h.drop(6);
+ h.render(12);
+ EXPECT_EQ(h.getAndClearFreezeEvent().valid, false);
+ h.drop(10);
+ h.render(1000 / 20 + 1); // +1 because it's unclear if the current frame is frozen
+ EXPECT_EQ(h.getMetrics().freezeEventCount, 1);
+ FreezeEvent e = h.getAndClearFreezeEvent();
+ EXPECT_EQ(e.valid, true); // freeze event
+ // -1 because the last rendered frame is considered frozen
+ EXPECT_EQ(e.initialTimeUs, 9 * 20 * 1000);
+ // only count the last frame of the first group of rendered frames
+ EXPECT_EQ(e.durationMs, (1 + 3 + 1000 / 20 + 1 + 10 + 6 + 12 + 10) * 20);
+ EXPECT_EQ(e.count, 4);
+ // number of dropped frames
+ // +1 because the last rendered frame is considered frozen
+ EXPECT_EQ(e.sumDurationMs, (4 + 2 + 7 + 11) * 20);
+ // number of rendered frames between dropped frames
+ // -1 because the last rendered frame is considered frozen
+ EXPECT_EQ(e.sumDistanceMs, ((1000 / 20) - 1 + 9 + 11) * 20);
+ // +1 for each since the last rendered frame is considered frozen
+ ASSERT_EQ(e.details.durationMs.size(), 4);
+ EXPECT_EQ(e.details.durationMs[0], 4 * 20);
+ EXPECT_EQ(e.details.durationMs[1], 2 * 20);
+ EXPECT_EQ(e.details.durationMs[2], 7 * 20);
+ EXPECT_EQ(e.details.durationMs[3], 11 * 20);
+ // -1 for each since the last rendered frame is considered frozen
+ ASSERT_EQ(e.details.distanceMs.size(), 4);
+ EXPECT_EQ(e.details.distanceMs[0], -1);
+ EXPECT_EQ(e.details.distanceMs[1], 1000 - 20);
+ EXPECT_EQ(e.details.distanceMs[2], 9 * 20);
+ EXPECT_EQ(e.details.distanceMs[3], 11 * 20);
+ int64_t previousEventEndTimeUs = e.initialTimeUs + e.durationMs * 1000;
+ h.drop(1);
+ h.render(4);
+ h.drop(1);
+ h.render(4);
+ h.drop(1);
+ h.render(4);
+ h.drop(1);
+ h.render(4);
+ h.drop(1);
+ h.render(1000 / 20 + 1);
+ EXPECT_EQ(h.getMetrics().freezeEventCount, 2);
+ e = h.getAndClearFreezeEvent();
+ EXPECT_EQ(e.valid, true);
+ // 1000ms tolerance means 1000ms from the end of the last event to the beginning of this event
+ EXPECT_EQ(e.initialTimeUs, previousEventEndTimeUs + 1000 * 1000);
+ EXPECT_EQ(e.count, 5);
+ // 5 freezes captured in the freeze event, but only 4 details are recorded
+ EXPECT_EQ(e.details.durationMs.size(), 4);
+ EXPECT_EQ(e.details.distanceMs.size(), 4);
+ EXPECT_EQ(e.details.distanceMs[0], 1000); // same as the tolerance
+ // The duration across the entire series f freezes is captured, with only 4 details captured
+ // +1 because the first rendered frame is considered frozen (not the 1st dropped frame)
+ EXPECT_EQ(e.durationMs, (1 + 1 + 4 + 1 + 4 + 1 + 4 + 1 + 4 + 1) * 20);
+ // The duration of all 5 freeze events are captured, with only 4 details captured
+ EXPECT_EQ(e.sumDurationMs, (2 + 2 + 2 + 2 + 2) * 20);
+ // The distance of all 5 freeze events are captured, with only 4 details captured
+ EXPECT_EQ(e.sumDistanceMs, (3 + 3 + 3 + 3) * 20);
+ h.drop(1);
+ h.render(1000 / 20 + 1);
+ EXPECT_EQ(h.getMetrics().freezeEventCount, 3);
+ EXPECT_EQ(h.getAndClearFreezeEvent().valid, true);
+ h.drop(1);
+ h.render(1000 / 20 + 1);
+ EXPECT_EQ(h.getMetrics().freezeEventCount, 4);
+ EXPECT_EQ(h.getAndClearFreezeEvent().valid, true);
+ h.drop(1);
+ h.render(1000 / 20 + 1);
+ EXPECT_EQ(h.getMetrics().freezeEventCount, 5);
+ EXPECT_EQ(h.getAndClearFreezeEvent().valid, true);
+ h.drop(1);
+ h.render(1000 / 20 + 1);
+ // The 6th event isn't captured because it exceeds the configured limit
+ EXPECT_EQ(h.getMetrics().freezeEventCount, 6);
+ EXPECT_EQ(h.getAndClearFreezeEvent().valid, false);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, capturesJudderEvents) {
+ Configuration c;
+ c.judderEventMax = 4;
+ c.judderEventDetailsMax = 3;
+ c.judderEventDistanceToleranceMs = 100;
+ Helper h(20, c);
+ h.render({19, 20, 19});
+ EXPECT_EQ(h.getAndClearJudderEvent().valid, false);
+ h.render({15, 19, 20, 19});
+ EXPECT_EQ(h.getAndClearJudderEvent().valid, false);
+ h.render({28, 20, 19});
+ EXPECT_EQ(h.getAndClearJudderEvent().valid, false);
+ h.render({13, 20, 20, 20, 20});
+ EXPECT_EQ(h.getAndClearJudderEvent().valid, false);
+ // Start with judder for the next event at the end of the sequence, because judder is scored
+ // one frame behind, and for combining judder occurrences into events, it's not clear yet if
+ // the current frame has judder or not.
+ h.render({15, 20, 20, 20, 20, 20, 15});
+ JudderEvent e = h.getAndClearJudderEvent();
+ EXPECT_EQ(e.valid, true);
+ EXPECT_EQ(e.initialTimeUs, (19 + 20 + 19) * 1000);
+ EXPECT_EQ(e.durationMs, 15 + 19 + 20 + 19 /**/ + 28 + 20 + 19 /**/ + 13 + 20 * 4 /**/ + 15);
+ EXPECT_EQ(e.count, 4);
+ EXPECT_EQ(e.sumScore, (20 - 15) + (28 - 20) + (20 - 13) + (20 - 15));
+ EXPECT_EQ(e.sumDistanceMs, 19 + 20 + 19 /**/ + 20 + 19 /**/ + 20 * 4);
+ ASSERT_EQ(e.details.actualRenderDurationUs.size(), 3); // 3 details per configured maximum
+ EXPECT_EQ(e.details.actualRenderDurationUs[0], 15 * 1000);
+ EXPECT_EQ(e.details.actualRenderDurationUs[1], 28 * 1000);
+ EXPECT_EQ(e.details.actualRenderDurationUs[2], 13 * 1000);
+ ASSERT_EQ(e.details.contentRenderDurationUs.size(), 3);
+ EXPECT_EQ(e.details.contentRenderDurationUs[0], 20 * 1000);
+ EXPECT_EQ(e.details.contentRenderDurationUs[1], 20 * 1000);
+ EXPECT_EQ(e.details.contentRenderDurationUs[2], 20 * 1000);
+ ASSERT_EQ(e.details.distanceMs.size(), 3);
+ EXPECT_EQ(e.details.distanceMs[0], -1);
+ EXPECT_EQ(e.details.distanceMs[1], 19 + 20 + 19);
+ EXPECT_EQ(e.details.distanceMs[2], 20 + 19);
+ h.render({20, 20, 20, 20, 20, 15});
+ e = h.getAndClearJudderEvent();
+ EXPECT_EQ(e.valid, true);
+ ASSERT_EQ(e.details.distanceMs.size(), 1);
+ EXPECT_EQ(e.details.distanceMs[0], 100); // same as the tolerance
+ h.render({20, 20, 20, 20, 20, 15});
+ EXPECT_EQ(h.getAndClearJudderEvent().valid, true);
+ h.render({20, 20, 20, 20, 20, 15});
+ EXPECT_EQ(h.getAndClearJudderEvent().valid, true);
+ h.render({20, 20, 20, 20, 20, 20});
+ EXPECT_EQ(h.getAndClearJudderEvent().valid, false); // max number of judder events exceeded
+}
+
} // android
diff --git a/services/mediametrics/MediaMetricsService.cpp b/services/mediametrics/MediaMetricsService.cpp
index adb2217..af1372b 100644
--- a/services/mediametrics/MediaMetricsService.cpp
+++ b/services/mediametrics/MediaMetricsService.cpp
@@ -524,6 +524,8 @@
"audiotrack",
// other media
"codec",
+ "freeze",
+ "judder",
"extractor",
"mediadrm",
"mediaparser",