SurfaceFlinger: Add TrustedPresentationListener API
TrustedPresentationListener is intended to allow the producer
of a buffer layer to gain a trusted signal on whether and to
what extent a layer is presented. A strawman design would be to
provide the producer details on it's presentation (alpha,
position, scale, final crop, covered region, etc). In the strawman
design the client end would then decide itself whether each of these
parameters were in an acceptable range.
However providing the client feedback on it's per frame position would have
a negative system health impact. Furthermore in some of the target use cases
we can't even be sure the layer of interest is actively producing buffers
and so there may be no existing callback to piggy-back on.
Because of this we use a server side thresholding approach, where the client
expresses some visibility threshold and a time stability constraint. See
SurfaceComposerClient.h comment for details on these thresholds and their
computation.
Bug: 256993331
Test: LayerTrustedPresentationListener_test.cpp
Change-Id: If4bef60dc6b22ce4959c353fa2a19b0994a00f5c
diff --git a/libs/gui/ITransactionCompletedListener.cpp b/libs/gui/ITransactionCompletedListener.cpp
index 2b25b61..985c549 100644
--- a/libs/gui/ITransactionCompletedListener.cpp
+++ b/libs/gui/ITransactionCompletedListener.cpp
@@ -33,7 +33,8 @@
ON_TRANSACTION_COMPLETED = IBinder::FIRST_CALL_TRANSACTION,
ON_RELEASE_BUFFER,
ON_TRANSACTION_QUEUE_STALLED,
- LAST = ON_TRANSACTION_QUEUE_STALLED,
+ ON_TRUSTED_PRESENTATION_CHANGED,
+ LAST = ON_TRUSTED_PRESENTATION_CHANGED,
};
} // Anonymous namespace
@@ -302,6 +303,11 @@
onTransactionQueueStalled)>(Tag::ON_TRANSACTION_QUEUE_STALLED,
reason);
}
+
+ void onTrustedPresentationChanged(int id, bool inTrustedPresentationState) override {
+ callRemoteAsync<decltype(&ITransactionCompletedListener::onTrustedPresentationChanged)>(
+ Tag::ON_TRUSTED_PRESENTATION_CHANGED, id, inTrustedPresentationState);
+ }
};
// Out-of-line virtual method definitions to trigger vtable emission in this translation unit (see
@@ -325,6 +331,9 @@
case Tag::ON_TRANSACTION_QUEUE_STALLED:
return callLocalAsync(data, reply,
&ITransactionCompletedListener::onTransactionQueueStalled);
+ case Tag::ON_TRUSTED_PRESENTATION_CHANGED:
+ return callLocalAsync(data, reply,
+ &ITransactionCompletedListener::onTrustedPresentationChanged);
}
}
diff --git a/libs/gui/LayerState.cpp b/libs/gui/LayerState.cpp
index 59b62fe..43acb16 100644
--- a/libs/gui/LayerState.cpp
+++ b/libs/gui/LayerState.cpp
@@ -185,6 +185,8 @@
if (hasBufferData) {
SAFE_PARCEL(output.writeParcelable, *bufferData);
}
+ SAFE_PARCEL(output.writeParcelable, trustedPresentationThresholds);
+ SAFE_PARCEL(output.writeParcelable, trustedPresentationListener);
return NO_ERROR;
}
@@ -315,6 +317,10 @@
} else {
bufferData = nullptr;
}
+
+ SAFE_PARCEL(input.readParcelable, &trustedPresentationThresholds);
+ SAFE_PARCEL(input.readParcelable, &trustedPresentationListener);
+
return NO_ERROR;
}
@@ -553,6 +559,11 @@
what |= eBufferChanged;
bufferData = other.bufferData;
}
+ if (other.what & eTrustedPresentationInfoChanged) {
+ what |= eTrustedPresentationInfoChanged;
+ trustedPresentationListener = other.trustedPresentationListener;
+ trustedPresentationThresholds = other.trustedPresentationThresholds;
+ }
if (other.what & eDataspaceChanged) {
what |= eDataspaceChanged;
dataspace = other.dataspace;
@@ -998,4 +1009,20 @@
return NO_ERROR;
}
+status_t TrustedPresentationListener::writeToParcel(Parcel* parcel) const {
+ SAFE_PARCEL(parcel->writeStrongBinder, callbackInterface);
+ SAFE_PARCEL(parcel->writeInt32, callbackId);
+ return NO_ERROR;
+}
+
+status_t TrustedPresentationListener::readFromParcel(const Parcel* parcel) {
+ sp<IBinder> tmpBinder = nullptr;
+ SAFE_PARCEL(parcel->readNullableStrongBinder, &tmpBinder);
+ if (tmpBinder) {
+ callbackInterface = checked_interface_cast<ITransactionCompletedListener>(tmpBinder);
+ }
+ SAFE_PARCEL(parcel->readInt32, &callbackId);
+ return NO_ERROR;
+}
+
}; // namespace android
diff --git a/libs/gui/SurfaceComposerClient.cpp b/libs/gui/SurfaceComposerClient.cpp
index 92125ea..732a168 100644
--- a/libs/gui/SurfaceComposerClient.cpp
+++ b/libs/gui/SurfaceComposerClient.cpp
@@ -24,6 +24,7 @@
#include <android/gui/DisplayState.h>
#include <android/gui/ISurfaceComposerClient.h>
#include <android/gui/IWindowInfosListener.h>
+#include <android/gui/TrustedPresentationThresholds.h>
#include <android/os/IInputConstants.h>
#include <gui/TraceUtils.h>
#include <utils/Errors.h>
@@ -63,6 +64,7 @@
using aidl::android::hardware::graphics::common::DisplayDecorationSupport;
using gui::FocusRequest;
using gui::IRegionSamplingListener;
+using gui::TrustedPresentationThresholds;
using gui::WindowInfo;
using gui::WindowInfoHandle;
using gui::WindowInfosListener;
@@ -518,6 +520,45 @@
}
}
+SurfaceComposerClient::PresentationCallbackRAII::PresentationCallbackRAII(
+ TransactionCompletedListener* tcl, int id) {
+ mTcl = tcl;
+ mId = id;
+}
+
+SurfaceComposerClient::PresentationCallbackRAII::~PresentationCallbackRAII() {
+ mTcl->clearTrustedPresentationCallback(mId);
+}
+
+sp<SurfaceComposerClient::PresentationCallbackRAII>
+TransactionCompletedListener::addTrustedPresentationCallback(TrustedPresentationCallback tpc,
+ int id, void* context) {
+ std::scoped_lock<std::mutex> lock(mMutex);
+ mTrustedPresentationCallbacks[id] =
+ std::tuple<TrustedPresentationCallback, void*>(tpc, context);
+ return new SurfaceComposerClient::PresentationCallbackRAII(this, id);
+}
+
+void TransactionCompletedListener::clearTrustedPresentationCallback(int id) {
+ std::scoped_lock<std::mutex> lock(mMutex);
+ mTrustedPresentationCallbacks.erase(id);
+}
+
+void TransactionCompletedListener::onTrustedPresentationChanged(int id,
+ bool presentedWithinThresholds) {
+ TrustedPresentationCallback tpc;
+ void* context;
+ {
+ std::scoped_lock<std::mutex> lock(mMutex);
+ auto it = mTrustedPresentationCallbacks.find(id);
+ if (it == mTrustedPresentationCallbacks.end()) {
+ return;
+ }
+ std::tie(tpc, context) = it->second;
+ }
+ tpc(context, presentedWithinThresholds);
+}
+
// ---------------------------------------------------------------------------
void removeDeadBufferCallback(void* /*context*/, uint64_t graphicBufferId);
@@ -2098,6 +2139,45 @@
t.startTimeNanos = 0;
}
+SurfaceComposerClient::Transaction&
+SurfaceComposerClient::Transaction::setTrustedPresentationCallback(
+ const sp<SurfaceControl>& sc, TrustedPresentationCallback cb,
+ const TrustedPresentationThresholds& thresholds, void* context,
+ sp<SurfaceComposerClient::PresentationCallbackRAII>& outCallbackRef) {
+ auto listener = TransactionCompletedListener::getInstance();
+ outCallbackRef = listener->addTrustedPresentationCallback(cb, sc->getLayerId(), context);
+
+ layer_state_t* s = getLayerState(sc);
+ if (!s) {
+ mStatus = BAD_INDEX;
+ return *this;
+ }
+ s->what |= layer_state_t::eTrustedPresentationInfoChanged;
+ s->trustedPresentationThresholds = thresholds;
+ s->trustedPresentationListener.callbackInterface = TransactionCompletedListener::getIInstance();
+ s->trustedPresentationListener.callbackId = sc->getLayerId();
+
+ return *this;
+}
+
+SurfaceComposerClient::Transaction&
+SurfaceComposerClient::Transaction::clearTrustedPresentationCallback(const sp<SurfaceControl>& sc) {
+ auto listener = TransactionCompletedListener::getInstance();
+ listener->clearTrustedPresentationCallback(sc->getLayerId());
+
+ layer_state_t* s = getLayerState(sc);
+ if (!s) {
+ mStatus = BAD_INDEX;
+ return *this;
+ }
+ s->what |= layer_state_t::eTrustedPresentationInfoChanged;
+ s->trustedPresentationThresholds = TrustedPresentationThresholds();
+ s->trustedPresentationListener.callbackInterface = nullptr;
+ s->trustedPresentationListener.callbackId = -1;
+
+ return *this;
+}
+
// ---------------------------------------------------------------------------
SurfaceComposerClient::SurfaceComposerClient() : mStatus(NO_INIT) {}
diff --git a/libs/gui/aidl/android/gui/TrustedPresentationThresholds.aidl b/libs/gui/aidl/android/gui/TrustedPresentationThresholds.aidl
new file mode 100644
index 0000000..1eea5b4
--- /dev/null
+++ b/libs/gui/aidl/android/gui/TrustedPresentationThresholds.aidl
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.gui;
+
+parcelable TrustedPresentationThresholds {
+ float minAlpha = -1.0f;
+ float minFractionRendered = -1.0f;
+
+ int stabilityRequirementMs = 0;
+}
diff --git a/libs/gui/include/gui/ITransactionCompletedListener.h b/libs/gui/include/gui/ITransactionCompletedListener.h
index 453e8f3..d593f56 100644
--- a/libs/gui/include/gui/ITransactionCompletedListener.h
+++ b/libs/gui/include/gui/ITransactionCompletedListener.h
@@ -196,6 +196,8 @@
uint32_t currentMaxAcquiredBufferCount) = 0;
virtual void onTransactionQueueStalled(const String8& name) = 0;
+
+ virtual void onTrustedPresentationChanged(int id, bool inTrustedPresentationState) = 0;
};
class BnTransactionCompletedListener : public SafeBnInterface<ITransactionCompletedListener> {
diff --git a/libs/gui/include/gui/LayerState.h b/libs/gui/include/gui/LayerState.h
index 45a84f6..17ed2d8 100644
--- a/libs/gui/include/gui/LayerState.h
+++ b/libs/gui/include/gui/LayerState.h
@@ -22,6 +22,7 @@
#include <sys/types.h>
#include <android/gui/IWindowInfosReportedListener.h>
+#include <android/gui/TrustedPresentationThresholds.h>
#include <android/native_window.h>
#include <gui/IGraphicBufferProducer.h>
#include <gui/ITransactionCompletedListener.h>
@@ -56,6 +57,8 @@
using gui::ISurfaceComposerClient;
using gui::LayerMetadata;
+using gui::TrustedPresentationThresholds;
+
struct client_cache_t {
wp<IBinder> token = nullptr;
uint64_t id;
@@ -65,6 +68,19 @@
bool isValid() const { return token != nullptr; }
};
+class TrustedPresentationListener : public Parcelable {
+public:
+ sp<ITransactionCompletedListener> callbackInterface;
+ int callbackId = -1;
+
+ void invoke(bool presentedWithinThresholds) {
+ callbackInterface->onTrustedPresentationChanged(callbackId, presentedWithinThresholds);
+ }
+
+ status_t writeToParcel(Parcel* parcel) const;
+ status_t readFromParcel(const Parcel* parcel);
+};
+
class BufferData : public Parcelable {
public:
virtual ~BufferData() = default;
@@ -148,7 +164,7 @@
enum {
ePositionChanged = 0x00000001,
eLayerChanged = 0x00000002,
- /* unused = 0x00000004, */
+ eTrustedPresentationInfoChanged = 0x00000004,
eAlphaChanged = 0x00000008,
eMatrixChanged = 0x00000010,
eTransparentRegionChanged = 0x00000020,
@@ -339,6 +355,9 @@
gui::DropInputMode dropInputMode;
bool dimmingEnabled;
+
+ TrustedPresentationThresholds trustedPresentationThresholds;
+ TrustedPresentationListener trustedPresentationListener;
};
class ComposerState {
diff --git a/libs/gui/include/gui/SurfaceComposerClient.h b/libs/gui/include/gui/SurfaceComposerClient.h
index 2458a40..6c45b26 100644
--- a/libs/gui/include/gui/SurfaceComposerClient.h
+++ b/libs/gui/include/gui/SurfaceComposerClient.h
@@ -58,6 +58,7 @@
class IGraphicBufferProducer;
class ITunnelModeEnabledListener;
class Region;
+class TransactionCompletedListener;
using gui::DisplayCaptureArgs;
using gui::IRegionSamplingListener;
@@ -106,6 +107,8 @@
const sp<Fence>& /*presentFence*/,
const SurfaceStats& /*stats*/)>;
+using TrustedPresentationCallback = std::function<void(void*, bool)>;
+
// ---------------------------------------------------------------------------
class ReleaseCallbackThread {
@@ -390,6 +393,13 @@
std::unordered_set<sp<SurfaceControl>, SCHash> surfaceControls;
};
+ struct PresentationCallbackRAII : public RefBase {
+ sp<TransactionCompletedListener> mTcl;
+ int mId;
+ PresentationCallbackRAII(TransactionCompletedListener* tcl, int id);
+ virtual ~PresentationCallbackRAII();
+ };
+
class Transaction : public Parcelable {
private:
static sp<IBinder> sApplyToken;
@@ -569,6 +579,59 @@
Transaction& addTransactionCommittedCallback(
TransactionCompletedCallbackTakesContext callback, void* callbackContext);
+ /**
+ * Set a callback to receive feedback about the presentation of a layer.
+ * When the layer is presented according to the passed in Thresholds,
+ * it is said to "enter the state", and receives the callback with true.
+ * When the conditions fall out of thresholds, it is then said to leave the
+ * state.
+ *
+ * There are a few simple thresholds:
+ * minAlpha: Lower bound on computed alpha
+ * minFractionRendered: Lower bounds on fraction of pixels that
+ * were rendered.
+ * stabilityThresholdMs: A time that alpha and fraction rendered
+ * must remain within bounds before we can "enter the state"
+ *
+ * The fraction of pixels rendered is a computation based on scale, crop
+ * and occlusion. The calculation may be somewhat counterintuitive, so we
+ * can work through an example. Imagine we have a layer with a 100x100 buffer
+ * which is occluded by (10x100) pixels on the left, and cropped by (100x10) pixels
+ * on the top. Furthermore imagine this layer is scaled by 0.9 in both dimensions.
+ * (c=crop,o=occluded,b=both,x=none
+ * b c c c
+ * o x x x
+ * o x x x
+ * o x x x
+ *
+ * We first start by computing fr=xscale*yscale=0.9*0.9=0.81, indicating
+ * that "81%" of the pixels were rendered. This corresponds to what was 100
+ * pixels being displayed in 81 pixels. This is somewhat of an abuse of
+ * language, as the information of merged pixels isn't totally lost, but
+ * we err on the conservative side.
+ *
+ * We then repeat a similar process for the crop and covered regions and
+ * accumulate the results: fr = fr * (fractionNotCropped) * (fractionNotCovered)
+ * So for this example we would get 0.9*0.9*0.9*0.9=0.65...
+ *
+ * Notice that this is not completely accurate, as we have double counted
+ * the region marked as b. However we only wanted a "lower bound" and so it
+ * is ok to err in this direction. Selection of the threshold will ultimately
+ * be somewhat arbitrary, and so there are some somewhat arbitrary decisions in
+ * this API as well.
+ *
+ * The caller must keep "PresentationCallbackRAII" alive, or the callback
+ * in SurfaceComposerClient will be unregistered.
+ */
+ Transaction& setTrustedPresentationCallback(const sp<SurfaceControl>& sc,
+ TrustedPresentationCallback callback,
+ const TrustedPresentationThresholds& thresholds,
+ void* context,
+ sp<PresentationCallbackRAII>& outCallbackOwner);
+
+ // Clear local memory in SCC
+ Transaction& clearTrustedPresentationCallback(const sp<SurfaceControl>& sc);
+
// ONLY FOR BLAST ADAPTER
Transaction& notifyProducerDisconnect(const sp<SurfaceControl>& sc);
@@ -795,6 +858,9 @@
std::multimap<int32_t, SurfaceStatsCallbackEntry> mSurfaceStatsListeners;
std::unordered_map<void*, std::function<void(const std::string&)>> mQueueStallListeners;
+ std::unordered_map<int, std::tuple<TrustedPresentationCallback, void*>>
+ mTrustedPresentationCallbacks;
+
public:
static sp<TransactionCompletedListener> getInstance();
static sp<ITransactionCompletedListener> getIInstance();
@@ -814,6 +880,10 @@
void addQueueStallListener(std::function<void(const std::string&)> stallListener, void* id);
void removeQueueStallListener(void *id);
+ sp<SurfaceComposerClient::PresentationCallbackRAII> addTrustedPresentationCallback(
+ TrustedPresentationCallback tpc, int id, void* context);
+ void clearTrustedPresentationCallback(int id);
+
/*
* Adds a jank listener to be informed about SurfaceFlinger's jank classification for a specific
* surface. Jank classifications arrive as part of the transaction callbacks about previous
@@ -844,6 +914,8 @@
void onTransactionQueueStalled(const String8& reason) override;
+ void onTrustedPresentationChanged(int id, bool presentedWithinThresholds) override;
+
private:
ReleaseBufferCallback popReleaseBufferCallbackLocked(const ReleaseCallbackId&);
static sp<TransactionCompletedListener> sInstance;