SurfaceFlinger: Add TrustedPresentationListener API
TrustedPresentationListener is intended to allow the producer
of a buffer layer to gain a trusted signal on whether and to
what extent a layer is presented. A strawman design would be to
provide the producer details on it's presentation (alpha,
position, scale, final crop, covered region, etc). In the strawman
design the client end would then decide itself whether each of these
parameters were in an acceptable range.
However providing the client feedback on it's per frame position would have
a negative system health impact. Furthermore in some of the target use cases
we can't even be sure the layer of interest is actively producing buffers
and so there may be no existing callback to piggy-back on.
Because of this we use a server side thresholding approach, where the client
expresses some visibility threshold and a time stability constraint. See
SurfaceComposerClient.h comment for details on these thresholds and their
computation.
Bug: 256993331
Test: LayerTrustedPresentationListener_test.cpp
Change-Id: If4bef60dc6b22ce4959c353fa2a19b0994a00f5c
diff --git a/libs/gui/ITransactionCompletedListener.cpp b/libs/gui/ITransactionCompletedListener.cpp
index 2b25b61..985c549 100644
--- a/libs/gui/ITransactionCompletedListener.cpp
+++ b/libs/gui/ITransactionCompletedListener.cpp
@@ -33,7 +33,8 @@
ON_TRANSACTION_COMPLETED = IBinder::FIRST_CALL_TRANSACTION,
ON_RELEASE_BUFFER,
ON_TRANSACTION_QUEUE_STALLED,
- LAST = ON_TRANSACTION_QUEUE_STALLED,
+ ON_TRUSTED_PRESENTATION_CHANGED,
+ LAST = ON_TRUSTED_PRESENTATION_CHANGED,
};
} // Anonymous namespace
@@ -302,6 +303,11 @@
onTransactionQueueStalled)>(Tag::ON_TRANSACTION_QUEUE_STALLED,
reason);
}
+
+ void onTrustedPresentationChanged(int id, bool inTrustedPresentationState) override {
+ callRemoteAsync<decltype(&ITransactionCompletedListener::onTrustedPresentationChanged)>(
+ Tag::ON_TRUSTED_PRESENTATION_CHANGED, id, inTrustedPresentationState);
+ }
};
// Out-of-line virtual method definitions to trigger vtable emission in this translation unit (see
@@ -325,6 +331,9 @@
case Tag::ON_TRANSACTION_QUEUE_STALLED:
return callLocalAsync(data, reply,
&ITransactionCompletedListener::onTransactionQueueStalled);
+ case Tag::ON_TRUSTED_PRESENTATION_CHANGED:
+ return callLocalAsync(data, reply,
+ &ITransactionCompletedListener::onTrustedPresentationChanged);
}
}
diff --git a/libs/gui/LayerState.cpp b/libs/gui/LayerState.cpp
index 59b62fe..43acb16 100644
--- a/libs/gui/LayerState.cpp
+++ b/libs/gui/LayerState.cpp
@@ -185,6 +185,8 @@
if (hasBufferData) {
SAFE_PARCEL(output.writeParcelable, *bufferData);
}
+ SAFE_PARCEL(output.writeParcelable, trustedPresentationThresholds);
+ SAFE_PARCEL(output.writeParcelable, trustedPresentationListener);
return NO_ERROR;
}
@@ -315,6 +317,10 @@
} else {
bufferData = nullptr;
}
+
+ SAFE_PARCEL(input.readParcelable, &trustedPresentationThresholds);
+ SAFE_PARCEL(input.readParcelable, &trustedPresentationListener);
+
return NO_ERROR;
}
@@ -553,6 +559,11 @@
what |= eBufferChanged;
bufferData = other.bufferData;
}
+ if (other.what & eTrustedPresentationInfoChanged) {
+ what |= eTrustedPresentationInfoChanged;
+ trustedPresentationListener = other.trustedPresentationListener;
+ trustedPresentationThresholds = other.trustedPresentationThresholds;
+ }
if (other.what & eDataspaceChanged) {
what |= eDataspaceChanged;
dataspace = other.dataspace;
@@ -998,4 +1009,20 @@
return NO_ERROR;
}
+status_t TrustedPresentationListener::writeToParcel(Parcel* parcel) const {
+ SAFE_PARCEL(parcel->writeStrongBinder, callbackInterface);
+ SAFE_PARCEL(parcel->writeInt32, callbackId);
+ return NO_ERROR;
+}
+
+status_t TrustedPresentationListener::readFromParcel(const Parcel* parcel) {
+ sp<IBinder> tmpBinder = nullptr;
+ SAFE_PARCEL(parcel->readNullableStrongBinder, &tmpBinder);
+ if (tmpBinder) {
+ callbackInterface = checked_interface_cast<ITransactionCompletedListener>(tmpBinder);
+ }
+ SAFE_PARCEL(parcel->readInt32, &callbackId);
+ return NO_ERROR;
+}
+
}; // namespace android
diff --git a/libs/gui/SurfaceComposerClient.cpp b/libs/gui/SurfaceComposerClient.cpp
index 92125ea..732a168 100644
--- a/libs/gui/SurfaceComposerClient.cpp
+++ b/libs/gui/SurfaceComposerClient.cpp
@@ -24,6 +24,7 @@
#include <android/gui/DisplayState.h>
#include <android/gui/ISurfaceComposerClient.h>
#include <android/gui/IWindowInfosListener.h>
+#include <android/gui/TrustedPresentationThresholds.h>
#include <android/os/IInputConstants.h>
#include <gui/TraceUtils.h>
#include <utils/Errors.h>
@@ -63,6 +64,7 @@
using aidl::android::hardware::graphics::common::DisplayDecorationSupport;
using gui::FocusRequest;
using gui::IRegionSamplingListener;
+using gui::TrustedPresentationThresholds;
using gui::WindowInfo;
using gui::WindowInfoHandle;
using gui::WindowInfosListener;
@@ -518,6 +520,45 @@
}
}
+SurfaceComposerClient::PresentationCallbackRAII::PresentationCallbackRAII(
+ TransactionCompletedListener* tcl, int id) {
+ mTcl = tcl;
+ mId = id;
+}
+
+SurfaceComposerClient::PresentationCallbackRAII::~PresentationCallbackRAII() {
+ mTcl->clearTrustedPresentationCallback(mId);
+}
+
+sp<SurfaceComposerClient::PresentationCallbackRAII>
+TransactionCompletedListener::addTrustedPresentationCallback(TrustedPresentationCallback tpc,
+ int id, void* context) {
+ std::scoped_lock<std::mutex> lock(mMutex);
+ mTrustedPresentationCallbacks[id] =
+ std::tuple<TrustedPresentationCallback, void*>(tpc, context);
+ return new SurfaceComposerClient::PresentationCallbackRAII(this, id);
+}
+
+void TransactionCompletedListener::clearTrustedPresentationCallback(int id) {
+ std::scoped_lock<std::mutex> lock(mMutex);
+ mTrustedPresentationCallbacks.erase(id);
+}
+
+void TransactionCompletedListener::onTrustedPresentationChanged(int id,
+ bool presentedWithinThresholds) {
+ TrustedPresentationCallback tpc;
+ void* context;
+ {
+ std::scoped_lock<std::mutex> lock(mMutex);
+ auto it = mTrustedPresentationCallbacks.find(id);
+ if (it == mTrustedPresentationCallbacks.end()) {
+ return;
+ }
+ std::tie(tpc, context) = it->second;
+ }
+ tpc(context, presentedWithinThresholds);
+}
+
// ---------------------------------------------------------------------------
void removeDeadBufferCallback(void* /*context*/, uint64_t graphicBufferId);
@@ -2098,6 +2139,45 @@
t.startTimeNanos = 0;
}
+SurfaceComposerClient::Transaction&
+SurfaceComposerClient::Transaction::setTrustedPresentationCallback(
+ const sp<SurfaceControl>& sc, TrustedPresentationCallback cb,
+ const TrustedPresentationThresholds& thresholds, void* context,
+ sp<SurfaceComposerClient::PresentationCallbackRAII>& outCallbackRef) {
+ auto listener = TransactionCompletedListener::getInstance();
+ outCallbackRef = listener->addTrustedPresentationCallback(cb, sc->getLayerId(), context);
+
+ layer_state_t* s = getLayerState(sc);
+ if (!s) {
+ mStatus = BAD_INDEX;
+ return *this;
+ }
+ s->what |= layer_state_t::eTrustedPresentationInfoChanged;
+ s->trustedPresentationThresholds = thresholds;
+ s->trustedPresentationListener.callbackInterface = TransactionCompletedListener::getIInstance();
+ s->trustedPresentationListener.callbackId = sc->getLayerId();
+
+ return *this;
+}
+
+SurfaceComposerClient::Transaction&
+SurfaceComposerClient::Transaction::clearTrustedPresentationCallback(const sp<SurfaceControl>& sc) {
+ auto listener = TransactionCompletedListener::getInstance();
+ listener->clearTrustedPresentationCallback(sc->getLayerId());
+
+ layer_state_t* s = getLayerState(sc);
+ if (!s) {
+ mStatus = BAD_INDEX;
+ return *this;
+ }
+ s->what |= layer_state_t::eTrustedPresentationInfoChanged;
+ s->trustedPresentationThresholds = TrustedPresentationThresholds();
+ s->trustedPresentationListener.callbackInterface = nullptr;
+ s->trustedPresentationListener.callbackId = -1;
+
+ return *this;
+}
+
// ---------------------------------------------------------------------------
SurfaceComposerClient::SurfaceComposerClient() : mStatus(NO_INIT) {}
diff --git a/libs/gui/aidl/android/gui/TrustedPresentationThresholds.aidl b/libs/gui/aidl/android/gui/TrustedPresentationThresholds.aidl
new file mode 100644
index 0000000..1eea5b4
--- /dev/null
+++ b/libs/gui/aidl/android/gui/TrustedPresentationThresholds.aidl
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.gui;
+
+parcelable TrustedPresentationThresholds {
+ float minAlpha = -1.0f;
+ float minFractionRendered = -1.0f;
+
+ int stabilityRequirementMs = 0;
+}
diff --git a/libs/gui/include/gui/ITransactionCompletedListener.h b/libs/gui/include/gui/ITransactionCompletedListener.h
index 453e8f3..d593f56 100644
--- a/libs/gui/include/gui/ITransactionCompletedListener.h
+++ b/libs/gui/include/gui/ITransactionCompletedListener.h
@@ -196,6 +196,8 @@
uint32_t currentMaxAcquiredBufferCount) = 0;
virtual void onTransactionQueueStalled(const String8& name) = 0;
+
+ virtual void onTrustedPresentationChanged(int id, bool inTrustedPresentationState) = 0;
};
class BnTransactionCompletedListener : public SafeBnInterface<ITransactionCompletedListener> {
diff --git a/libs/gui/include/gui/LayerState.h b/libs/gui/include/gui/LayerState.h
index 45a84f6..17ed2d8 100644
--- a/libs/gui/include/gui/LayerState.h
+++ b/libs/gui/include/gui/LayerState.h
@@ -22,6 +22,7 @@
#include <sys/types.h>
#include <android/gui/IWindowInfosReportedListener.h>
+#include <android/gui/TrustedPresentationThresholds.h>
#include <android/native_window.h>
#include <gui/IGraphicBufferProducer.h>
#include <gui/ITransactionCompletedListener.h>
@@ -56,6 +57,8 @@
using gui::ISurfaceComposerClient;
using gui::LayerMetadata;
+using gui::TrustedPresentationThresholds;
+
struct client_cache_t {
wp<IBinder> token = nullptr;
uint64_t id;
@@ -65,6 +68,19 @@
bool isValid() const { return token != nullptr; }
};
+class TrustedPresentationListener : public Parcelable {
+public:
+ sp<ITransactionCompletedListener> callbackInterface;
+ int callbackId = -1;
+
+ void invoke(bool presentedWithinThresholds) {
+ callbackInterface->onTrustedPresentationChanged(callbackId, presentedWithinThresholds);
+ }
+
+ status_t writeToParcel(Parcel* parcel) const;
+ status_t readFromParcel(const Parcel* parcel);
+};
+
class BufferData : public Parcelable {
public:
virtual ~BufferData() = default;
@@ -148,7 +164,7 @@
enum {
ePositionChanged = 0x00000001,
eLayerChanged = 0x00000002,
- /* unused = 0x00000004, */
+ eTrustedPresentationInfoChanged = 0x00000004,
eAlphaChanged = 0x00000008,
eMatrixChanged = 0x00000010,
eTransparentRegionChanged = 0x00000020,
@@ -339,6 +355,9 @@
gui::DropInputMode dropInputMode;
bool dimmingEnabled;
+
+ TrustedPresentationThresholds trustedPresentationThresholds;
+ TrustedPresentationListener trustedPresentationListener;
};
class ComposerState {
diff --git a/libs/gui/include/gui/SurfaceComposerClient.h b/libs/gui/include/gui/SurfaceComposerClient.h
index 2458a40..6c45b26 100644
--- a/libs/gui/include/gui/SurfaceComposerClient.h
+++ b/libs/gui/include/gui/SurfaceComposerClient.h
@@ -58,6 +58,7 @@
class IGraphicBufferProducer;
class ITunnelModeEnabledListener;
class Region;
+class TransactionCompletedListener;
using gui::DisplayCaptureArgs;
using gui::IRegionSamplingListener;
@@ -106,6 +107,8 @@
const sp<Fence>& /*presentFence*/,
const SurfaceStats& /*stats*/)>;
+using TrustedPresentationCallback = std::function<void(void*, bool)>;
+
// ---------------------------------------------------------------------------
class ReleaseCallbackThread {
@@ -390,6 +393,13 @@
std::unordered_set<sp<SurfaceControl>, SCHash> surfaceControls;
};
+ struct PresentationCallbackRAII : public RefBase {
+ sp<TransactionCompletedListener> mTcl;
+ int mId;
+ PresentationCallbackRAII(TransactionCompletedListener* tcl, int id);
+ virtual ~PresentationCallbackRAII();
+ };
+
class Transaction : public Parcelable {
private:
static sp<IBinder> sApplyToken;
@@ -569,6 +579,59 @@
Transaction& addTransactionCommittedCallback(
TransactionCompletedCallbackTakesContext callback, void* callbackContext);
+ /**
+ * Set a callback to receive feedback about the presentation of a layer.
+ * When the layer is presented according to the passed in Thresholds,
+ * it is said to "enter the state", and receives the callback with true.
+ * When the conditions fall out of thresholds, it is then said to leave the
+ * state.
+ *
+ * There are a few simple thresholds:
+ * minAlpha: Lower bound on computed alpha
+ * minFractionRendered: Lower bounds on fraction of pixels that
+ * were rendered.
+ * stabilityThresholdMs: A time that alpha and fraction rendered
+ * must remain within bounds before we can "enter the state"
+ *
+ * The fraction of pixels rendered is a computation based on scale, crop
+ * and occlusion. The calculation may be somewhat counterintuitive, so we
+ * can work through an example. Imagine we have a layer with a 100x100 buffer
+ * which is occluded by (10x100) pixels on the left, and cropped by (100x10) pixels
+ * on the top. Furthermore imagine this layer is scaled by 0.9 in both dimensions.
+ * (c=crop,o=occluded,b=both,x=none
+ * b c c c
+ * o x x x
+ * o x x x
+ * o x x x
+ *
+ * We first start by computing fr=xscale*yscale=0.9*0.9=0.81, indicating
+ * that "81%" of the pixels were rendered. This corresponds to what was 100
+ * pixels being displayed in 81 pixels. This is somewhat of an abuse of
+ * language, as the information of merged pixels isn't totally lost, but
+ * we err on the conservative side.
+ *
+ * We then repeat a similar process for the crop and covered regions and
+ * accumulate the results: fr = fr * (fractionNotCropped) * (fractionNotCovered)
+ * So for this example we would get 0.9*0.9*0.9*0.9=0.65...
+ *
+ * Notice that this is not completely accurate, as we have double counted
+ * the region marked as b. However we only wanted a "lower bound" and so it
+ * is ok to err in this direction. Selection of the threshold will ultimately
+ * be somewhat arbitrary, and so there are some somewhat arbitrary decisions in
+ * this API as well.
+ *
+ * The caller must keep "PresentationCallbackRAII" alive, or the callback
+ * in SurfaceComposerClient will be unregistered.
+ */
+ Transaction& setTrustedPresentationCallback(const sp<SurfaceControl>& sc,
+ TrustedPresentationCallback callback,
+ const TrustedPresentationThresholds& thresholds,
+ void* context,
+ sp<PresentationCallbackRAII>& outCallbackOwner);
+
+ // Clear local memory in SCC
+ Transaction& clearTrustedPresentationCallback(const sp<SurfaceControl>& sc);
+
// ONLY FOR BLAST ADAPTER
Transaction& notifyProducerDisconnect(const sp<SurfaceControl>& sc);
@@ -795,6 +858,9 @@
std::multimap<int32_t, SurfaceStatsCallbackEntry> mSurfaceStatsListeners;
std::unordered_map<void*, std::function<void(const std::string&)>> mQueueStallListeners;
+ std::unordered_map<int, std::tuple<TrustedPresentationCallback, void*>>
+ mTrustedPresentationCallbacks;
+
public:
static sp<TransactionCompletedListener> getInstance();
static sp<ITransactionCompletedListener> getIInstance();
@@ -814,6 +880,10 @@
void addQueueStallListener(std::function<void(const std::string&)> stallListener, void* id);
void removeQueueStallListener(void *id);
+ sp<SurfaceComposerClient::PresentationCallbackRAII> addTrustedPresentationCallback(
+ TrustedPresentationCallback tpc, int id, void* context);
+ void clearTrustedPresentationCallback(int id);
+
/*
* Adds a jank listener to be informed about SurfaceFlinger's jank classification for a specific
* surface. Jank classifications arrive as part of the transaction callbacks about previous
@@ -844,6 +914,8 @@
void onTransactionQueueStalled(const String8& reason) override;
+ void onTrustedPresentationChanged(int id, bool presentedWithinThresholds) override;
+
private:
ReleaseBufferCallback popReleaseBufferCallbackLocked(const ReleaseCallbackId&);
static sp<TransactionCompletedListener> sInstance;
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index de9ea04..6197e9b 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -250,6 +250,10 @@
if (mHadClonedChild) {
mFlinger->mNumClones--;
}
+ if (hasTrustedPresentationListener()) {
+ mFlinger->mNumTrustedPresentationListeners--;
+ updateTrustedPresentationState(nullptr, -1 /* time_in_ms */, true /* leaveState*/);
+ }
}
// ---------------------------------------------------------------------------
@@ -279,6 +283,7 @@
mRemovedFromDrawingState = true;
mFlinger->mScheduler->deregisterLayer(this);
}
+ updateTrustedPresentationState(nullptr, -1 /* time_in_ms */, true /* leaveState*/);
mFlinger->markLayerPendingRemovalLocked(sp<Layer>::fromExisting(this));
}
@@ -376,6 +381,92 @@
return reduce(mBounds, activeTransparentRegion);
}
+// No early returns.
+void Layer::updateTrustedPresentationState(const DisplayDevice* display, int64_t time_in_ms,
+ bool leaveState) {
+ if (!hasTrustedPresentationListener()) {
+ return;
+ }
+ const bool lastState = mLastComputedTrustedPresentationState;
+ mLastComputedTrustedPresentationState = false;
+
+ if (!leaveState) {
+ const auto outputLayer = findOutputLayerForDisplay(display);
+ if (outputLayer != nullptr) {
+ mLastComputedTrustedPresentationState =
+ computeTrustedPresentationState(mBounds, mSourceBounds,
+ outputLayer->getState().coveredRegion,
+ mScreenBounds, getCompositionState()->alpha,
+ getCompositionState()->geomLayerTransform,
+ mTrustedPresentationThresholds);
+ }
+ }
+ const bool newState = mLastComputedTrustedPresentationState;
+ if (lastState && !newState) {
+ // We were in the trusted presentation state, but now we left it,
+ // emit the callback if needed
+ if (mLastReportedTrustedPresentationState) {
+ mLastReportedTrustedPresentationState = false;
+ mTrustedPresentationListener.invoke(false);
+ }
+ // Reset the timer
+ mEnteredTrustedPresentationStateTime = -1;
+ } else if (!lastState && newState) {
+ // We were not in the trusted presentation state, but we entered it, begin the timer
+ // and make sure this gets called at least once more!
+ mEnteredTrustedPresentationStateTime = time_in_ms;
+ mFlinger->forceFutureUpdate(mTrustedPresentationThresholds.stabilityRequirementMs * 1.5);
+ }
+
+ // Has the timer elapsed, but we are still in the state? Emit a callback if needed
+ if (!mLastReportedTrustedPresentationState && newState &&
+ (time_in_ms - mEnteredTrustedPresentationStateTime >
+ mTrustedPresentationThresholds.stabilityRequirementMs)) {
+ mLastReportedTrustedPresentationState = true;
+ mTrustedPresentationListener.invoke(true);
+ }
+}
+
+/**
+ * See SurfaceComposerClient.h: setTrustedPresentationCallback for discussion
+ * of how the parameters and thresholds are interpreted. The general spirit is
+ * to produce an upper bound on the amount of the buffer which was presented.
+ */
+bool Layer::computeTrustedPresentationState(const FloatRect& bounds, const FloatRect& sourceBounds,
+ const Region& coveredRegion,
+ const FloatRect& screenBounds, float alpha,
+ const ui::Transform& effectiveTransform,
+ const TrustedPresentationThresholds& thresholds) {
+ if (alpha < thresholds.minAlpha) {
+ return false;
+ }
+ if (sourceBounds.getWidth() == 0 || sourceBounds.getHeight() == 0) {
+ return false;
+ }
+ if (screenBounds.getWidth() == 0 || screenBounds.getHeight() == 0) {
+ return false;
+ }
+
+ const float sx = effectiveTransform.dsdx();
+ const float sy = effectiveTransform.dsdy();
+ float fractionRendered = std::min(sx * sy, 1.0f);
+
+ float boundsOverSourceW = bounds.getWidth() / (float)sourceBounds.getWidth();
+ float boundsOverSourceH = bounds.getHeight() / (float)sourceBounds.getHeight();
+ fractionRendered *= boundsOverSourceW * boundsOverSourceH;
+
+ Rect coveredBounds = coveredRegion.bounds();
+ fractionRendered *= (1 -
+ ((coveredBounds.width() / (float)screenBounds.getWidth()) *
+ coveredBounds.height() / (float)screenBounds.getHeight()));
+
+ if (fractionRendered < thresholds.minFractionRendered) {
+ return false;
+ }
+
+ return true;
+}
+
void Layer::computeBounds(FloatRect parentBounds, ui::Transform parentTransform,
float parentShadowRadius) {
const State& s(getDrawingState());
@@ -3988,6 +4079,19 @@
return *this;
}
+void Layer::setTrustedPresentationInfo(TrustedPresentationThresholds const& thresholds,
+ TrustedPresentationListener const& listener) {
+ bool hadTrustedPresentationListener = hasTrustedPresentationListener();
+ mTrustedPresentationListener = listener;
+ mTrustedPresentationThresholds = thresholds;
+ bool haveTrustedPresentationListener = hasTrustedPresentationListener();
+ if (!hadTrustedPresentationListener && haveTrustedPresentationListener) {
+ mFlinger->mNumTrustedPresentationListeners++;
+ } else if (hadTrustedPresentationListener && !haveTrustedPresentationListener) {
+ mFlinger->mNumTrustedPresentationListeners--;
+ }
+}
+
// ---------------------------------------------------------------------------
std::ostream& operator<<(std::ostream& stream, const Layer::FrameRate& rate) {
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index 08a13a3..63894cd 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -527,6 +527,19 @@
uint32_t getTransactionFlags() const { return mTransactionFlags; }
+ static bool computeTrustedPresentationState(const FloatRect& bounds,
+ const FloatRect& sourceBounds,
+ const Region& coveredRegion,
+ const FloatRect& screenBounds, float,
+ const ui::Transform&,
+ const TrustedPresentationThresholds&);
+ void updateTrustedPresentationState(const DisplayDevice* display, int64_t time_in_ms,
+ bool leaveState);
+
+ inline bool hasTrustedPresentationListener() {
+ return mTrustedPresentationListener.callbackInterface != nullptr;
+ }
+
// Sets the masked bits.
void setTransactionFlags(uint32_t mask);
@@ -728,6 +741,9 @@
std::shared_ptr<frametimeline::SurfaceFrame> createSurfaceFrameForBuffer(
const FrameTimelineInfo& info, nsecs_t queueTime, std::string debugName);
+ void setTrustedPresentationInfo(TrustedPresentationThresholds const& thresholds,
+ TrustedPresentationListener const& listener);
+
// Creates a new handle each time, so we only expect
// this to be called once.
sp<IBinder> getHandle();
@@ -883,6 +899,12 @@
// These are only accessed by the main thread or the tracing thread.
State mDrawingState;
+ TrustedPresentationThresholds mTrustedPresentationThresholds;
+ TrustedPresentationListener mTrustedPresentationListener;
+ bool mLastComputedTrustedPresentationState = false;
+ bool mLastReportedTrustedPresentationState = false;
+ int64_t mEnteredTrustedPresentationStateTime = -1;
+
uint32_t mTransactionFlags{0};
// Updated in doTransaction, used to track the last sequence number we
// committed. Currently this is really only used for updating visible
diff --git a/services/surfaceflinger/Scheduler/MessageQueue.cpp b/services/surfaceflinger/Scheduler/MessageQueue.cpp
index e827c12..9b04497 100644
--- a/services/surfaceflinger/Scheduler/MessageQueue.cpp
+++ b/services/surfaceflinger/Scheduler/MessageQueue.cpp
@@ -132,6 +132,10 @@
mLooper->sendMessage(handler, Message());
}
+void MessageQueue::postMessageDelayed(sp<MessageHandler>&& handler, nsecs_t uptimeDelay) {
+ mLooper->sendMessageDelayed(uptimeDelay, handler, Message());
+}
+
void MessageQueue::scheduleConfigure() {
struct ConfigureHandler : MessageHandler {
explicit ConfigureHandler(ICompositor& compositor) : compositor(compositor) {}
diff --git a/services/surfaceflinger/Scheduler/MessageQueue.h b/services/surfaceflinger/Scheduler/MessageQueue.h
index 71f8645..ad0ea72 100644
--- a/services/surfaceflinger/Scheduler/MessageQueue.h
+++ b/services/surfaceflinger/Scheduler/MessageQueue.h
@@ -79,6 +79,7 @@
virtual void setDuration(std::chrono::nanoseconds workDuration) = 0;
virtual void waitMessage() = 0;
virtual void postMessage(sp<MessageHandler>&&) = 0;
+ virtual void postMessageDelayed(sp<MessageHandler>&&, nsecs_t uptimeDelay) = 0;
virtual void scheduleConfigure() = 0;
virtual void scheduleFrame() = 0;
@@ -144,6 +145,7 @@
void waitMessage() override;
void postMessage(sp<MessageHandler>&&) override;
+ void postMessageDelayed(sp<MessageHandler>&&, nsecs_t uptimeDelay) override;
void scheduleConfigure() override;
void scheduleFrame() override;
diff --git a/services/surfaceflinger/Scheduler/Scheduler.h b/services/surfaceflinger/Scheduler/Scheduler.h
index 20221d1..36280e3 100644
--- a/services/surfaceflinger/Scheduler/Scheduler.h
+++ b/services/surfaceflinger/Scheduler/Scheduler.h
@@ -139,6 +139,13 @@
return std::move(future);
}
+ template <typename F, typename T = std::invoke_result_t<F>>
+ [[nodiscard]] std::future<T> scheduleDelayed(F&& f, nsecs_t uptimeDelay) {
+ auto [task, future] = makeTask(std::move(f));
+ postMessageDelayed(std::move(task), uptimeDelay);
+ return std::move(future);
+ }
+
ConnectionHandle createConnection(const char* connectionName, frametimeline::TokenManager*,
std::chrono::nanoseconds workDuration,
std::chrono::nanoseconds readyDuration);
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index 2d8b9c1..41e305e 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -2439,7 +2439,7 @@
scheduleComposite(FrameHint::kNone);
}
- postComposition();
+ postComposition(presentTime);
const bool prevFrameHadClientComposition = mHadClientComposition;
@@ -2553,7 +2553,7 @@
return ui::ROTATION_0;
}
-void SurfaceFlinger::postComposition() {
+void SurfaceFlinger::postComposition(nsecs_t callTime) {
ATRACE_CALL();
ALOGV(__func__);
@@ -2718,6 +2718,17 @@
}
}
+ if (mNumTrustedPresentationListeners > 0) {
+ // We avoid any reverse traversal upwards so this shouldn't be too expensive
+ mDrawingState.traverse([&](Layer* layer) {
+ if (!layer->hasTrustedPresentationListener()) {
+ return;
+ }
+ layer->updateTrustedPresentationState(display, nanoseconds_to_milliseconds(callTime),
+ false);
+ });
+ }
+
// Even though ATRACE_INT64 already checks if tracing is enabled, it doesn't prevent the
// side-effect of getTotalSize(), so we check that again here
if (ATRACE_ENABLED()) {
@@ -4593,6 +4604,11 @@
layer->setFrameTimelineVsyncForBufferlessTransaction(frameTimelineInfo, postTime);
}
+ if (what & layer_state_t::eTrustedPresentationInfoChanged) {
+ layer->setTrustedPresentationInfo(s.trustedPresentationThresholds,
+ s.trustedPresentationListener);
+ }
+
if (layer->setTransactionCompletedListeners(callbackHandles)) flags |= eTraversalNeeded;
// Do not put anything that updates layer state or modifies flags after
// setTransactionCompletedListener
@@ -8118,6 +8134,10 @@
return OK;
}
+void SurfaceFlinger::forceFutureUpdate(int delayInMs) {
+ static_cast<void>(mScheduler->scheduleDelayed([&]() { scheduleRepaint(); }, ms2ns(delayInMs)));
+}
+
} // namespace android
#if defined(__gl_h_)
diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h
index 33f0402..b52dc82 100644
--- a/services/surfaceflinger/SurfaceFlinger.h
+++ b/services/surfaceflinger/SurfaceFlinger.h
@@ -313,6 +313,8 @@
// TODO(b/246793311): Clean up a temporary property
bool mIgnoreHwcPhysicalDisplayOrientation = false;
+ void forceFutureUpdate(int delayInMs);
+
protected:
// We're reference counted, never destroy SurfaceFlinger directly
virtual ~SurfaceFlinger();
@@ -926,7 +928,7 @@
/*
* Compositing
*/
- void postComposition() REQUIRES(kMainThreadContext);
+ void postComposition(nsecs_t callTime) REQUIRES(kMainThreadContext);
/*
* Display management
@@ -1274,6 +1276,8 @@
float mDimmingRatio = -1.f;
std::unique_ptr<renderengine::RenderEngine> mRenderEngine;
+ std::atomic<int> mNumTrustedPresentationListeners = 0;
+
std::unique_ptr<compositionengine::CompositionEngine> mCompositionEngine;
// mMaxRenderTargetSize is only set once in init() so it doesn't need to be protected by
// any mutex.
diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h b/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h
index 81ca659..6b52a3a 100644
--- a/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h
+++ b/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h
@@ -599,7 +599,7 @@
mFlinger->commitTransactions();
mFlinger->flushTransactionQueues(getFuzzedVsyncId(mFdp));
- mFlinger->postComposition();
+ mFlinger->postComposition(systemTime());
}
mFlinger->setTransactionFlags(mFdp.ConsumeIntegral<uint32_t>());
diff --git a/services/surfaceflinger/tests/Android.bp b/services/surfaceflinger/tests/Android.bp
index bd6367d..de47330 100644
--- a/services/surfaceflinger/tests/Android.bp
+++ b/services/surfaceflinger/tests/Android.bp
@@ -43,6 +43,7 @@
"LayerRenderTypeTransaction_test.cpp",
"LayerState_test.cpp",
"LayerTransaction_test.cpp",
+ "LayerTrustedPresentationListener_test.cpp",
"LayerTypeAndRenderTypeTransaction_test.cpp",
"LayerTypeTransaction_test.cpp",
"LayerUpdate_test.cpp",
diff --git a/services/surfaceflinger/tests/LayerTrustedPresentationListener_test.cpp b/services/surfaceflinger/tests/LayerTrustedPresentationListener_test.cpp
new file mode 100644
index 0000000..a843fd1
--- /dev/null
+++ b/services/surfaceflinger/tests/LayerTrustedPresentationListener_test.cpp
@@ -0,0 +1,241 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// TODO(b/129481165): remove the #pragma below and fix conversion issues
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wconversion"
+
+#include <gui/BufferItemConsumer.h>
+#include <ui/Transform.h>
+#include <thread>
+#include "TransactionTestHarnesses.h"
+
+namespace android {
+struct PresentationCallbackHelper {
+ void callbackArrived(bool state) {
+ std::unique_lock l(mMutex);
+ mGotCallback = true;
+ mState = state;
+ mCondition.notify_all();
+ }
+ bool awaitCallback() {
+ std::unique_lock l(mMutex);
+ mGotCallback = false;
+ mCondition.wait_for(l, 5000ms);
+ EXPECT_TRUE(mGotCallback);
+ return mState;
+ }
+
+ bool mState;
+ bool mGotCallback;
+ std::mutex mMutex;
+ std::condition_variable mCondition;
+};
+
+TrustedPresentationThresholds thresh() {
+ TrustedPresentationThresholds thresholds;
+ thresholds.minAlpha = 1.0;
+ thresholds.minFractionRendered = 1.0;
+ thresholds.stabilityRequirementMs = 100;
+ return thresholds;
+}
+
+class LayerTrustedPresentationListenerTest : public LayerTransactionTest {
+public:
+ void SetUp() override {
+ LayerTransactionTest::SetUp();
+ mainLayer = makeLayer();
+ thresholds = thresh();
+ }
+
+ void TearDown() override {
+ LayerTransactionTest::TearDown();
+ mCallback = nullptr;
+ t.reparent(mainLayer, nullptr).apply();
+ mainLayer = nullptr;
+ }
+
+ void thresholdsPrepared() {
+ t.show(mainLayer)
+ .setLayer(mainLayer, INT32_MAX)
+ .setTrustedPresentationCallback(
+ mainLayer,
+ [&](void* context, bool state) {
+ PresentationCallbackHelper* helper =
+ (PresentationCallbackHelper*)context;
+ helper->callbackArrived(state);
+ },
+ thresholds, &pch, mCallback)
+ .setPosition(mainLayer, 100, 100)
+ .apply();
+ }
+
+ sp<SurfaceControl> makeLayer() {
+ sp<SurfaceControl> layer =
+ createLayer("test", 100, 100, ISurfaceComposerClient::eFXSurfaceBufferState,
+ mBlackBgSurface.get());
+ fillBufferLayerColor(layer, Color::RED, 100, 100);
+ return layer;
+ }
+ sp<SurfaceControl> mainLayer;
+ PresentationCallbackHelper pch;
+ SurfaceComposerClient::Transaction t;
+ TrustedPresentationThresholds thresholds;
+ sp<SurfaceComposerClient::PresentationCallbackRAII> mCallback;
+};
+
+// The layer is fully presented with the default test setup.
+TEST_F(LayerTrustedPresentationListenerTest, callback_arrives) {
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+}
+
+// A hidden layer can't be considered presented!
+TEST_F(LayerTrustedPresentationListenerTest, hiding_layer_clears_state) {
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+ t.hide(mainLayer).apply();
+ EXPECT_FALSE(pch.awaitCallback());
+}
+
+// A fully obscured layer can't be considered presented!
+TEST_F(LayerTrustedPresentationListenerTest, obscuring_clears_state) {
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+
+ auto otherLayer = makeLayer();
+ t.show(otherLayer)
+ .setPosition(otherLayer, 100, 100)
+ .setLayer(otherLayer, INT32_MAX)
+ .setLayer(mainLayer, INT32_MAX - 1)
+ .apply();
+ EXPECT_FALSE(pch.awaitCallback());
+}
+
+// Even if the layer obscuring us has an Alpha channel, we are still considered
+// obscured.
+TEST_F(LayerTrustedPresentationListenerTest, obscuring_with_transparency_clears_state) {
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+
+ auto otherLayer = makeLayer();
+ t.show(otherLayer)
+ .setPosition(otherLayer, 100, 100)
+ .setLayer(otherLayer, INT32_MAX)
+ .setFlags(otherLayer, 0, layer_state_t::eLayerOpaque)
+ .setLayer(mainLayer, INT32_MAX - 1)
+ .apply();
+ EXPECT_FALSE(pch.awaitCallback());
+}
+
+// We can't be presented if our alpha is below the threshold.
+TEST_F(LayerTrustedPresentationListenerTest, alpha_below_threshold) {
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+ t.setAlpha(mainLayer, 0.9).apply();
+ EXPECT_FALSE(pch.awaitCallback());
+ t.setAlpha(mainLayer, 1.0).apply();
+ EXPECT_TRUE(pch.awaitCallback());
+}
+
+// Verify that the passed in threshold is actually respected!
+TEST_F(LayerTrustedPresentationListenerTest, alpha_below_other_threshold) {
+ thresholds.minAlpha = 0.8;
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+ t.setAlpha(mainLayer, 0.8).apply();
+ EXPECT_FALSE(pch.awaitCallback());
+ t.setAlpha(mainLayer, 0.9).apply();
+ EXPECT_TRUE(pch.awaitCallback());
+}
+
+// (86*86)/(100*100) = 0.73...so a crop of 86x86 is below the threshold
+// (87*87)/(100*100) = 0.76...so a crop of 87x87 is above the threshold!
+TEST_F(LayerTrustedPresentationListenerTest, crop_below_threshold) {
+ thresholds.minFractionRendered = 0.75;
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+ t.setCrop(mainLayer, Rect(0, 0, 86, 86)).apply();
+ EXPECT_FALSE(pch.awaitCallback());
+ t.setCrop(mainLayer, Rect(0, 0, 87, 87)).apply();
+ EXPECT_TRUE(pch.awaitCallback());
+}
+
+TEST_F(LayerTrustedPresentationListenerTest, scale_below_threshold) {
+ thresholds.minFractionRendered = 0.64;
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+ // 0.8 = sqrt(0.64)
+ t.setMatrix(mainLayer, 0.79, 0, 0, 0.79).apply();
+ EXPECT_FALSE(pch.awaitCallback());
+ t.setMatrix(mainLayer, 0.81, 0, 0, 0.81).apply();
+ EXPECT_TRUE(pch.awaitCallback());
+}
+
+TEST_F(LayerTrustedPresentationListenerTest, obscuring_with_threshold_1) {
+ thresholds.minFractionRendered = 0.75;
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+
+ auto otherLayer = makeLayer();
+ t.show(otherLayer)
+ .setPosition(otherLayer, 100, 100)
+ .setLayer(otherLayer, INT32_MAX)
+ .setLayer(mainLayer, INT32_MAX - 1)
+ .apply();
+ EXPECT_FALSE(pch.awaitCallback());
+ t.setMatrix(otherLayer, 0.49, 0, 0, 0.49).apply();
+ EXPECT_TRUE(pch.awaitCallback());
+ t.setMatrix(otherLayer, 0.51, 0, 0, 0.51).apply();
+ EXPECT_FALSE(pch.awaitCallback());
+}
+
+TEST_F(LayerTrustedPresentationListenerTest, obscuring_with_threshold_2) {
+ thresholds.minFractionRendered = 0.9;
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+
+ auto otherLayer = makeLayer();
+ t.show(otherLayer)
+ .setPosition(otherLayer, 100, 100)
+ .setLayer(otherLayer, INT32_MAX)
+ .setLayer(mainLayer, INT32_MAX - 1)
+ .apply();
+ EXPECT_FALSE(pch.awaitCallback());
+ t.setMatrix(otherLayer, 0.3, 0, 0, 0.3).apply();
+ EXPECT_TRUE(pch.awaitCallback());
+ t.setMatrix(otherLayer, 0.33, 0, 0, 0.33).apply();
+ EXPECT_FALSE(pch.awaitCallback());
+}
+
+TEST_F(LayerTrustedPresentationListenerTest, obscuring_with_alpha) {
+ thresholds.minFractionRendered = 0.9;
+ thresholdsPrepared();
+ EXPECT_TRUE(pch.awaitCallback());
+
+ auto otherLayer = makeLayer();
+ t.show(otherLayer)
+ .setPosition(otherLayer, 100, 100)
+ .setLayer(otherLayer, INT32_MAX)
+ .setLayer(mainLayer, INT32_MAX - 1)
+ .setAlpha(otherLayer, 0.01)
+ .apply();
+ EXPECT_FALSE(pch.awaitCallback());
+ t.setAlpha(otherLayer, 0.0).apply();
+ EXPECT_TRUE(pch.awaitCallback());
+}
+
+} // namespace android