SF: Updating video detection logic. This is V0.
- Remove timestamp detection for FPS, not needed
- Video is now detected via BufferItem.mApi ID
- Make sure Idle timer and media detection don't cancel
each other.
see go/surface-flinger-scheduler for more info
see go/content-fps-detection-in-scheduler to see plans
for improvement.
Test: systrace
Bug: 123956502
Change-Id: I14390a3decd3da8fabf383bd3a2470864b027f7a
diff --git a/services/surfaceflinger/BufferQueueLayer.cpp b/services/surfaceflinger/BufferQueueLayer.cpp
index da34083..e4179ee 100644
--- a/services/surfaceflinger/BufferQueueLayer.cpp
+++ b/services/surfaceflinger/BufferQueueLayer.cpp
@@ -393,9 +393,8 @@
// Add this buffer from our internal queue tracker
{ // Autolock scope
if (mFlinger->mUseSmart90ForVideo) {
- // Report the requested present time to the Scheduler, if the feature is turned on.
- mFlinger->mScheduler->addFramePresentTimeForLayer(item.mTimestamp,
- item.mIsAutoTimestamp, mName.c_str());
+ // Report mApi ID for each layer.
+ mFlinger->mScheduler->addNativeWindowApi(item.mApi);
}
Mutex::Autolock lock(mQueueItemLock);
diff --git a/services/surfaceflinger/Scheduler/Scheduler.cpp b/services/surfaceflinger/Scheduler/Scheduler.cpp
index f3a7f87..0ba6cf9 100644
--- a/services/surfaceflinger/Scheduler/Scheduler.cpp
+++ b/services/surfaceflinger/Scheduler/Scheduler.cpp
@@ -28,6 +28,7 @@
#include <android/hardware/configstore/1.1/ISurfaceFlingerConfigs.h>
#include <configstore/Utils.h>
#include <cutils/properties.h>
+#include <system/window.h>
#include <ui/DisplayStatInfo.h>
#include <utils/Timers.h>
#include <utils/Trace.h>
@@ -296,21 +297,29 @@
mPrimaryDispSync->dump(result);
}
-void Scheduler::addFramePresentTimeForLayer(const nsecs_t framePresentTime, bool isAutoTimestamp,
- const std::string layerName) {
- // This is V1 logic. It calculates the average FPS based on the timestamp frequency
- // regardless of which layer the timestamp came from.
- // For now, the averages and FPS are recorded in the systrace.
- determineTimestampAverage(isAutoTimestamp, framePresentTime);
-
- // This is V2 logic. It calculates the average and median timestamp difference based on the
- // individual layer history. The results are recorded in the systrace.
- determineLayerTimestampStats(layerName, framePresentTime);
+void Scheduler::addNativeWindowApi(int apiId) {
+ std::lock_guard<std::mutex> lock(mWindowApiHistoryLock);
+ mWindowApiHistory[mApiHistoryCounter] = apiId;
+ mApiHistoryCounter++;
+ mApiHistoryCounter = mApiHistoryCounter % scheduler::ARRAY_SIZE;
}
-void Scheduler::incrementFrameCounter() {
- std::lock_guard<std::mutex> lock(mLayerHistoryLock);
- mLayerHistory.incrementCounter();
+void Scheduler::updateFpsBasedOnNativeWindowApi() {
+ int mode;
+ {
+ std::lock_guard<std::mutex> lock(mWindowApiHistoryLock);
+ mode = scheduler::calculate_mode(mWindowApiHistory);
+ }
+ ATRACE_INT("NativeWindowApiMode", mode);
+
+ if (mode == NATIVE_WINDOW_API_MEDIA) {
+ // TODO(b/127365162): These callback names are not accurate anymore. Update.
+ mediaChangeRefreshRate(MediaFeatureState::MEDIA_PLAYING);
+ ATRACE_INT("DetectedVideo", 1);
+ } else {
+ mediaChangeRefreshRate(MediaFeatureState::MEDIA_OFF);
+ ATRACE_INT("DetectedVideo", 0);
+ }
}
void Scheduler::setChangeRefreshRateCallback(
@@ -328,85 +337,6 @@
}
}
-void Scheduler::determineLayerTimestampStats(const std::string layerName,
- const nsecs_t framePresentTime) {
- std::vector<int64_t> differencesMs;
- std::string differencesText = "";
- {
- std::lock_guard<std::mutex> lock(mLayerHistoryLock);
- mLayerHistory.insert(layerName, framePresentTime);
-
- // Traverse through the layer history, and determine the differences in present times.
- nsecs_t newestPresentTime = framePresentTime;
- for (int i = 1; i < mLayerHistory.getSize(); i++) {
- std::unordered_map<std::string, nsecs_t> layers = mLayerHistory.get(i);
- for (auto layer : layers) {
- if (layer.first != layerName) {
- continue;
- }
- int64_t differenceMs = (newestPresentTime - layer.second) / 1000000;
- // Dismiss noise.
- if (differenceMs > 10 && differenceMs < 60) {
- differencesMs.push_back(differenceMs);
- }
- IF_ALOGV() { differencesText += (std::to_string(differenceMs) + " "); }
- newestPresentTime = layer.second;
- }
- }
- }
- ALOGV("Layer %s timestamp intervals: %s", layerName.c_str(), differencesText.c_str());
-
- if (!differencesMs.empty()) {
- // Mean/Average is a good indicator for when 24fps videos are playing, because the frames
- // come in 33, and 49 ms intervals with occasional 41ms.
- const int64_t meanMs = scheduler::calculate_mean(differencesMs);
- const auto tagMean = "TimestampMean_" + layerName;
- ATRACE_INT(tagMean.c_str(), meanMs);
-
- // Mode and median are good indicators for 30 and 60 fps videos, because the majority of
- // frames come in 16, or 33 ms intervals.
- const auto tagMedian = "TimestampMedian_" + layerName;
- ATRACE_INT(tagMedian.c_str(), scheduler::calculate_median(&differencesMs));
-
- const auto tagMode = "TimestampMode_" + layerName;
- ATRACE_INT(tagMode.c_str(), scheduler::calculate_mode(differencesMs));
- }
-}
-
-void Scheduler::determineTimestampAverage(bool isAutoTimestamp, const nsecs_t framePresentTime) {
- ATRACE_INT("AutoTimestamp", isAutoTimestamp);
-
- // Video does not have timestamp automatically set, so we discard timestamps that are
- // coming in from other sources for now.
- if (isAutoTimestamp) {
- return;
- }
- int64_t differenceMs = (framePresentTime - mPreviousFrameTimestamp) / 1000000;
- mPreviousFrameTimestamp = framePresentTime;
-
- if (differenceMs < 10 || differenceMs > 100) {
- // Dismiss noise.
- return;
- }
- ATRACE_INT("TimestampDiff", differenceMs);
-
- mTimeDifferences[mCounter % scheduler::ARRAY_SIZE] = differenceMs;
- mCounter++;
- int64_t mean = scheduler::calculate_mean(mTimeDifferences);
- ATRACE_INT("AutoTimestampMean", mean);
-
- // TODO(b/113612090): This are current numbers from trial and error while running videos
- // from YouTube at 24, 30, and 60 fps.
- if (mean > 14 && mean < 18) {
- ATRACE_INT("MediaFPS", 60);
- } else if (mean > 31 && mean < 34) {
- ATRACE_INT("MediaFPS", 30);
- return;
- } else if (mean > 39 && mean < 42) {
- ATRACE_INT("MediaFPS", 24);
- }
-}
-
void Scheduler::resetIdleTimer() {
if (mIdleTimer) {
mIdleTimer->reset();
@@ -414,21 +344,15 @@
}
void Scheduler::resetTimerCallback() {
- std::lock_guard<std::mutex> lock(mCallbackLock);
- if (mChangeRefreshRateCallback) {
- // We do not notify the applications about config changes when idle timer is reset.
- mChangeRefreshRateCallback(RefreshRateType::PERFORMANCE, ConfigEvent::None);
- ATRACE_INT("ExpiredIdleTimer", 0);
- }
+ // We do not notify the applications about config changes when idle timer is reset.
+ timerChangeRefreshRate(IdleTimerState::RESET);
+ ATRACE_INT("ExpiredIdleTimer", 0);
}
void Scheduler::expiredTimerCallback() {
- std::lock_guard<std::mutex> lock(mCallbackLock);
- if (mChangeRefreshRateCallback) {
- // We do not notify the applications about config changes when idle timer expires.
- mChangeRefreshRateCallback(RefreshRateType::DEFAULT, ConfigEvent::None);
- ATRACE_INT("ExpiredIdleTimer", 1);
- }
+ // We do not notify the applications about config changes when idle timer expires.
+ timerChangeRefreshRate(IdleTimerState::EXPIRED);
+ ATRACE_INT("ExpiredIdleTimer", 1);
}
std::string Scheduler::doDump() {
@@ -437,4 +361,46 @@
return stream.str();
}
+void Scheduler::mediaChangeRefreshRate(MediaFeatureState mediaFeatureState) {
+ // Default playback for media is DEFAULT when media is on.
+ RefreshRateType refreshRateType = RefreshRateType::DEFAULT;
+ ConfigEvent configEvent = ConfigEvent::None;
+
+ {
+ std::lock_guard<std::mutex> lock(mFeatureStateLock);
+ mCurrentMediaFeatureState = mediaFeatureState;
+ // Only switch to PERFORMANCE if idle timer was reset, when turning
+ // media off. If the timer is IDLE, stay at DEFAULT.
+ if (mediaFeatureState == MediaFeatureState::MEDIA_OFF &&
+ mCurrentIdleTimerState == IdleTimerState::RESET) {
+ refreshRateType = RefreshRateType::PERFORMANCE;
+ }
+ }
+ changeRefreshRate(refreshRateType, configEvent);
+}
+
+void Scheduler::timerChangeRefreshRate(IdleTimerState idleTimerState) {
+ RefreshRateType refreshRateType = RefreshRateType::DEFAULT;
+ ConfigEvent configEvent = ConfigEvent::None;
+
+ {
+ std::lock_guard<std::mutex> lock(mFeatureStateLock);
+ mCurrentIdleTimerState = idleTimerState;
+ // Only switch to PERFOMANCE if the idle timer was reset, and media is
+ // not playing. Otherwise, stay at DEFAULT.
+ if (idleTimerState == IdleTimerState::RESET &&
+ mCurrentMediaFeatureState == MediaFeatureState::MEDIA_OFF) {
+ refreshRateType = RefreshRateType::PERFORMANCE;
+ }
+ }
+ changeRefreshRate(refreshRateType, configEvent);
+}
+
+void Scheduler::changeRefreshRate(RefreshRateType refreshRateType, ConfigEvent configEvent) {
+ std::lock_guard<std::mutex> lock(mCallbackLock);
+ if (mChangeRefreshRateCallback) {
+ mChangeRefreshRateCallback(refreshRateType, configEvent);
+ }
+}
+
} // namespace android
diff --git a/services/surfaceflinger/Scheduler/Scheduler.h b/services/surfaceflinger/Scheduler/Scheduler.h
index 5b9759b..2582c93 100644
--- a/services/surfaceflinger/Scheduler/Scheduler.h
+++ b/services/surfaceflinger/Scheduler/Scheduler.h
@@ -141,11 +141,11 @@
void addPresentFence(const std::shared_ptr<FenceTime>& fenceTime);
void setIgnorePresentFences(bool ignore);
nsecs_t expectedPresentTime();
- // Adds the present time for given layer to the history of present times.
- void addFramePresentTimeForLayer(const nsecs_t framePresentTime, bool isAutoTimestamp,
- const std::string layerName);
- // Increments counter in the layer history to indicate that SF has started a new frame.
- void incrementFrameCounter();
+ // apiId indicates the API (NATIVE_WINDOW_API_xxx) that queues the buffer.
+ // TODO(b/123956502): Remove this call with V1 go/content-fps-detection-in-scheduler.
+ void addNativeWindowApi(int apiId);
+ // Updates FPS based on the most occured request for Native Window API.
+ void updateFpsBasedOnNativeWindowApi();
// Callback that gets invoked when Scheduler wants to change the refresh rate.
void setChangeRefreshRateCallback(const ChangeRefreshRateCallback& changeRefreshRateCallback);
@@ -165,17 +165,16 @@
private:
friend class TestableScheduler;
+ // In order to make sure that the features don't override themselves, we need a state machine
+ // to keep track which feature requested the config change.
+ enum class MediaFeatureState { MEDIA_PLAYING, MEDIA_OFF };
+ enum class IdleTimerState { EXPIRED, RESET };
+
// Creates a connection on the given EventThread and forwards the given callbacks.
sp<EventThreadConnection> createConnectionInternal(EventThread*, ResyncCallback&&);
nsecs_t calculateAverage() const;
void updateFrameSkipping(const int64_t skipCount);
- // Collects the statistical mean (average) and median between timestamp
- // intervals for each frame for each layer.
- void determineLayerTimestampStats(const std::string layerName, const nsecs_t framePresentTime);
- // Collects the average difference between timestamps for each frame regardless
- // of which layer the timestamp came from.
- void determineTimestampAverage(bool isAutoTimestamp, const nsecs_t framePresentTime);
// Function that resets the idle timer.
void resetIdleTimer();
// Function that is called when the timer resets.
@@ -184,6 +183,12 @@
void expiredTimerCallback();
// Sets vsync period.
void setVsyncPeriod(const nsecs_t period);
+ // Media feature's function to change the refresh rate.
+ void mediaChangeRefreshRate(MediaFeatureState mediaFeatureState);
+ // Idle timer feature's function to change the refresh rate.
+ void timerChangeRefreshRate(IdleTimerState idleTimerState);
+ // Acquires a lock and calls the ChangeRefreshRateCallback() with given parameters.
+ void changeRefreshRate(RefreshRateType refreshRateType, ConfigEvent configEvent);
// If fences from sync Framework are supported.
const bool mHasSyncFramework;
@@ -217,10 +222,13 @@
std::array<int64_t, scheduler::ARRAY_SIZE> mTimeDifferences{};
size_t mCounter = 0;
- // DetermineLayerTimestampStats is called from BufferQueueLayer::onFrameAvailable which
- // can run on any thread, and cause failure.
- std::mutex mLayerHistoryLock;
- LayerHistory mLayerHistory GUARDED_BY(mLayerHistoryLock);
+ // The following few fields follow native window api bits that come with buffers. If there are
+ // more buffers with NATIVE_WINDOW_API_MEDIA we render at 60Hz, otherwise we render at 90Hz.
+ // There is not dependency on timestamp for V0.
+ // TODO(b/123956502): Remove this when more robust logic for content fps detection is developed.
+ std::mutex mWindowApiHistoryLock;
+ std::array<int, scheduler::ARRAY_SIZE> mWindowApiHistory GUARDED_BY(mWindowApiHistoryLock);
+ int64_t mApiHistoryCounter = 0;
// Timer that records time between requests for next vsync. If the time is higher than a given
// interval, a callback is fired. Set this variable to >0 to use this feature.
@@ -229,6 +237,13 @@
std::mutex mCallbackLock;
ChangeRefreshRateCallback mChangeRefreshRateCallback GUARDED_BY(mCallbackLock);
+
+ // In order to make sure that the features don't override themselves, we need a state machine
+ // to keep track which feature requested the config change.
+ std::mutex mFeatureStateLock;
+ MediaFeatureState mCurrentMediaFeatureState GUARDED_BY(mFeatureStateLock) =
+ MediaFeatureState::MEDIA_OFF;
+ IdleTimerState mCurrentIdleTimerState GUARDED_BY(mFeatureStateLock) = IdleTimerState::RESET;
};
} // namespace android
diff --git a/services/surfaceflinger/Scheduler/SchedulerUtils.cpp b/services/surfaceflinger/Scheduler/SchedulerUtils.cpp
index 191022d..fb5414f 100644
--- a/services/surfaceflinger/Scheduler/SchedulerUtils.cpp
+++ b/services/surfaceflinger/Scheduler/SchedulerUtils.cpp
@@ -34,23 +34,5 @@
return v->at(n);
}
-int64_t calculate_mode(const std::vector<int64_t>& v) {
- if (v.empty()) {
- return 0;
- }
-
- // Create a map with all the counts for the indivicual values in the vector.
- std::unordered_map<int64_t, int64_t> counts;
- for (int64_t value : v) {
- counts[value]++;
- }
-
- // Sort the map, and return the number with the highest count. If two numbers have
- // the same count, first one is returned.
- using ValueType = const decltype(counts)::value_type&;
- const auto compareCounts = [](ValueType l, ValueType r) { return l.second <= r.second; };
- return std::max_element(counts.begin(), counts.end(), compareCounts)->first;
-}
-
} // namespace scheduler
} // namespace android
diff --git a/services/surfaceflinger/Scheduler/SchedulerUtils.h b/services/surfaceflinger/Scheduler/SchedulerUtils.h
index edd23de..9e6e8c7 100644
--- a/services/surfaceflinger/Scheduler/SchedulerUtils.h
+++ b/services/surfaceflinger/Scheduler/SchedulerUtils.h
@@ -18,6 +18,7 @@
#include <cinttypes>
#include <numeric>
+#include <unordered_map>
#include <vector>
namespace android {
@@ -45,7 +46,24 @@
int64_t calculate_median(std::vector<int64_t>* v);
// Calculates the statistical mode in the vector. Return 0 if the vector is empty.
-int64_t calculate_mode(const std::vector<int64_t>& v);
+template <typename T>
+auto calculate_mode(const T& v) {
+ if (v.empty()) {
+ return 0;
+ }
+
+ // Create a map with all the counts for the indivicual values in the vector.
+ std::unordered_map<int64_t, int> counts;
+ for (int64_t value : v) {
+ counts[value]++;
+ }
+
+ // Sort the map, and return the number with the highest count. If two numbers have
+ // the same count, first one is returned.
+ using ValueType = const decltype(counts)::value_type&;
+ const auto compareCounts = [](ValueType l, ValueType r) { return l.second <= r.second; };
+ return static_cast<int>(std::max_element(counts.begin(), counts.end(), compareCounts)->first);
+}
} // namespace scheduler
} // namespace android
\ No newline at end of file
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index 9b0e8c1..ebfcda4 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -1583,15 +1583,16 @@
ATRACE_CALL();
switch (what) {
case MessageQueue::INVALIDATE: {
+ if (mUseSmart90ForVideo) {
+ // This call is made each time SF wakes up and creates a new frame. It is part
+ // of video detection feature.
+ mScheduler->updateFpsBasedOnNativeWindowApi();
+ }
+
if (performSetActiveConfig()) {
break;
}
- if (mUseSmart90ForVideo) {
- // This call is made each time SF wakes up and creates a new frame. It is part
- // of video detection feature.
- mScheduler->incrementFrameCounter();
- }
bool frameMissed = mPreviousPresentFence != Fence::NO_FENCE &&
(mPreviousPresentFence->getStatus() == Fence::Status::Unsignaled);
bool hwcFrameMissed = !mHadClientComposition && frameMissed;
@@ -4744,6 +4745,7 @@
*/
result.append("\nScheduler state:\n");
result.append(mScheduler->doDump() + "\n");
+ StringAppendF(&result, "+ Smart video mode: %s\n\n", mUseSmart90ForVideo ? "on" : "off");
result.append(mRefreshRateStats->doDump() + "\n");
}