Merge "Keep track of pilfered pointer explicitly"
diff --git a/cmds/servicemanager/ServiceManager.cpp b/cmds/servicemanager/ServiceManager.cpp
index cc038ae..695faf8 100644
--- a/cmds/servicemanager/ServiceManager.cpp
+++ b/cmds/servicemanager/ServiceManager.cpp
@@ -718,7 +718,8 @@
if (service.guaranteeClient) {
// we have no record of this client
if (!service.hasClients && !hasClients) {
- sendClientCallbackNotifications(serviceName, true);
+ sendClientCallbackNotifications(serviceName, true,
+ "service is guaranteed to be in use");
}
// guarantee is temporary
@@ -729,34 +730,41 @@
if (isCalledOnInterval) {
if (hasClients && !service.hasClients) {
// client was retrieved in some other way
- sendClientCallbackNotifications(serviceName, true);
+ sendClientCallbackNotifications(serviceName, true, "we now have a record of a client");
}
// there are no more clients, but the callback has not been called yet
if (!hasClients && service.hasClients) {
- sendClientCallbackNotifications(serviceName, false);
+ sendClientCallbackNotifications(serviceName, false,
+ "we now have no record of a client");
}
}
return count;
}
-void ServiceManager::sendClientCallbackNotifications(const std::string& serviceName, bool hasClients) {
+void ServiceManager::sendClientCallbackNotifications(const std::string& serviceName,
+ bool hasClients, const char* context) {
auto serviceIt = mNameToService.find(serviceName);
if (serviceIt == mNameToService.end()) {
- ALOGW("sendClientCallbackNotifications could not find service %s", serviceName.c_str());
+ ALOGW("sendClientCallbackNotifications could not find service %s when %s",
+ serviceName.c_str(), context);
return;
}
Service& service = serviceIt->second;
- CHECK(hasClients != service.hasClients) << "Record shows: " << service.hasClients
- << " so we can't tell clients again that we have client: " << hasClients;
+ CHECK(hasClients != service.hasClients)
+ << "Record shows: " << service.hasClients
+ << " so we can't tell clients again that we have client: " << hasClients
+ << " when: " << context;
- ALOGI("Notifying %s they have clients: %d", serviceName.c_str(), hasClients);
+ ALOGI("Notifying %s they %s have clients when %s", serviceName.c_str(),
+ hasClients ? "do" : "don't", context);
auto ccIt = mNameToClientCallback.find(serviceName);
CHECK(ccIt != mNameToClientCallback.end())
- << "sendClientCallbackNotifications could not find callbacks for service ";
+ << "sendClientCallbackNotifications could not find callbacks for service when "
+ << context;
for (const auto& callback : ccIt->second) {
callback->onClients(service.binder, hasClients);
diff --git a/cmds/servicemanager/ServiceManager.h b/cmds/servicemanager/ServiceManager.h
index b24c11c..f9d4f8f 100644
--- a/cmds/servicemanager/ServiceManager.h
+++ b/cmds/servicemanager/ServiceManager.h
@@ -92,8 +92,9 @@
ServiceCallbackMap::iterator* it,
bool* found);
ssize_t handleServiceClientCallback(const std::string& serviceName, bool isCalledOnInterval);
- // Also updates mHasClients (of what the last callback was)
- void sendClientCallbackNotifications(const std::string& serviceName, bool hasClients);
+ // Also updates mHasClients (of what the last callback was)
+ void sendClientCallbackNotifications(const std::string& serviceName, bool hasClients,
+ const char* context);
// removes a callback from mNameToClientCallback, deleting the entry if the vector is empty
// this updates the iterator to the next location
void removeClientCallback(const wp<IBinder>& who, ClientCallbackMap::iterator* it);
diff --git a/data/etc/input/Android.bp b/data/etc/input/Android.bp
new file mode 100644
index 0000000..90f3c6b
--- /dev/null
+++ b/data/etc/input/Android.bp
@@ -0,0 +1,14 @@
+package {
+ default_applicable_licenses: ["frameworks_native_license"],
+}
+
+filegroup {
+ name: "motion_predictor_model.fb",
+ srcs: ["motion_predictor_model.fb"],
+}
+
+prebuilt_etc {
+ name: "motion_predictor_model_prebuilt",
+ filename_from_src: true,
+ src: "motion_predictor_model.fb",
+}
diff --git a/data/etc/input/motion_predictor_model.fb b/data/etc/input/motion_predictor_model.fb
new file mode 100644
index 0000000..10b3c8b
--- /dev/null
+++ b/data/etc/input/motion_predictor_model.fb
Binary files differ
diff --git a/include/input/MotionPredictor.h b/include/input/MotionPredictor.h
index 045e61b..3fae4e6 100644
--- a/include/input/MotionPredictor.h
+++ b/include/input/MotionPredictor.h
@@ -16,9 +16,15 @@
#pragma once
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+
#include <android-base/thread_annotations.h>
#include <android/sysprop/InputProperties.sysprop.h>
#include <input/Input.h>
+#include <input/TfLiteMotionPredictor.h>
namespace android {
@@ -28,48 +34,51 @@
/**
* Given a set of MotionEvents for the current gesture, predict the motion. The returned MotionEvent
- * contains a set of samples in the future, up to "presentation time + offset".
+ * contains a set of samples in the future.
*
* The typical usage is like this:
*
* MotionPredictor predictor(offset = MY_OFFSET);
- * predictor.setExpectedPresentationTimeNanos(NEXT_PRESENT_TIME);
* predictor.record(DOWN_MOTION_EVENT);
* predictor.record(MOVE_MOTION_EVENT);
- * prediction = predictor.predict();
+ * prediction = predictor.predict(futureTime);
*
- * The presentation time should be set some time before calling .predict(). It could be set before
- * or after the recorded motion events. Must be done on every frame.
- *
- * The resulting motion event will have eventTime <= (NEXT_PRESENT_TIME + MY_OFFSET). It might
- * contain historical data, which are additional samples from the latest recorded MotionEvent's
- * eventTime to the NEXT_PRESENT_TIME + MY_OFFSET.
+ * The resulting motion event will have eventTime <= (futureTime + MY_OFFSET). It might contain
+ * historical data, which are additional samples from the latest recorded MotionEvent's eventTime
+ * to the futureTime + MY_OFFSET.
*
* The offset is used to provide additional flexibility to the caller, in case the default present
* time (typically provided by the choreographer) does not account for some delays, or to simply
- * reduce the aggressiveness of the prediction. Offset can be both positive and negative.
+ * reduce the aggressiveness of the prediction. Offset can be positive or negative.
*/
class MotionPredictor {
public:
/**
* Parameters:
* predictionTimestampOffsetNanos: additional, constant shift to apply to the target
- * presentation time. The prediction will target the time t=(presentationTime +
+ * prediction time. The prediction will target the time t=(prediction time +
* predictionTimestampOffsetNanos).
*
+ * modelPath: filesystem path to a TfLiteMotionPredictorModel flatbuffer, or nullptr to use the
+ * default model path.
+ *
* checkEnableMotionPredition: the function to check whether the prediction should run. Used to
* provide an additional way of turning prediction on and off. Can be toggled at runtime.
*/
- MotionPredictor(nsecs_t predictionTimestampOffsetNanos,
+ MotionPredictor(nsecs_t predictionTimestampOffsetNanos, const char* modelPath = nullptr,
std::function<bool()> checkEnableMotionPrediction = isMotionPredictionEnabled);
void record(const MotionEvent& event);
std::vector<std::unique_ptr<MotionEvent>> predict(nsecs_t timestamp);
bool isPredictionAvailable(int32_t deviceId, int32_t source);
private:
- std::vector<MotionEvent> mEvents;
const nsecs_t mPredictionTimestampOffsetNanos;
const std::function<bool()> mCheckMotionPredictionEnabled;
+
+ std::unique_ptr<TfLiteMotionPredictorModel> mModel;
+ // Buffers/events for each device seen by record().
+ std::unordered_map</*deviceId*/ int32_t, TfLiteMotionPredictorBuffers> mDeviceBuffers;
+ std::unordered_map</*deviceId*/ int32_t, MotionEvent> mLastEvents;
};
} // namespace android
diff --git a/include/input/TfLiteMotionPredictor.h b/include/input/TfLiteMotionPredictor.h
new file mode 100644
index 0000000..ff0f51c
--- /dev/null
+++ b/include/input/TfLiteMotionPredictor.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <span>
+#include <string>
+#include <vector>
+
+#include <tensorflow/lite/core/api/error_reporter.h>
+#include <tensorflow/lite/interpreter.h>
+#include <tensorflow/lite/model.h>
+#include <tensorflow/lite/signature_runner.h>
+
+namespace android {
+
+struct TfLiteMotionPredictorSample {
+ // The untransformed AMOTION_EVENT_AXIS_X and AMOTION_EVENT_AXIS_Y of the sample.
+ struct Point {
+ float x;
+ float y;
+ } position;
+ // The AMOTION_EVENT_AXIS_PRESSURE, _TILT, and _ORIENTATION.
+ float pressure;
+ float tilt;
+ float orientation;
+};
+
+inline TfLiteMotionPredictorSample::Point operator-(const TfLiteMotionPredictorSample::Point& lhs,
+ const TfLiteMotionPredictorSample::Point& rhs) {
+ return {.x = lhs.x - rhs.x, .y = lhs.y - rhs.y};
+}
+
+class TfLiteMotionPredictorModel;
+
+// Buffer storage for a TfLiteMotionPredictorModel.
+class TfLiteMotionPredictorBuffers {
+public:
+ // Creates buffer storage for a model with the given input length.
+ TfLiteMotionPredictorBuffers(size_t inputLength);
+
+ // Adds a motion sample to the buffers.
+ void pushSample(int64_t timestamp, TfLiteMotionPredictorSample sample);
+
+ // Returns true if the buffers are complete enough to generate a prediction.
+ bool isReady() const {
+ // Predictions can't be applied unless there are at least two points to determine
+ // the direction to apply them in.
+ return mAxisFrom && mAxisTo;
+ }
+
+ // Resets all buffers to their initial state.
+ void reset();
+
+ // Copies the buffers to those of a model for prediction.
+ void copyTo(TfLiteMotionPredictorModel& model) const;
+
+ // Returns the current axis of the buffer's samples. Only valid if isReady().
+ TfLiteMotionPredictorSample axisFrom() const { return *mAxisFrom; }
+ TfLiteMotionPredictorSample axisTo() const { return *mAxisTo; }
+
+ // Returns the timestamp of the last sample.
+ int64_t lastTimestamp() const { return mTimestamp; }
+
+private:
+ int64_t mTimestamp = 0;
+
+ std::vector<float> mInputR;
+ std::vector<float> mInputPhi;
+ std::vector<float> mInputPressure;
+ std::vector<float> mInputTilt;
+ std::vector<float> mInputOrientation;
+
+ // The samples defining the current polar axis.
+ std::optional<TfLiteMotionPredictorSample> mAxisFrom;
+ std::optional<TfLiteMotionPredictorSample> mAxisTo;
+};
+
+// A TFLite model for generating motion predictions.
+class TfLiteMotionPredictorModel {
+public:
+ // Creates a model from an encoded Flatbuffer model.
+ static std::unique_ptr<TfLiteMotionPredictorModel> create(const char* modelPath);
+
+ // Returns the length of the model's input buffers.
+ size_t inputLength() const;
+
+ // Executes the model.
+ // Returns true if the model successfully executed and the output tensors can be read.
+ bool invoke();
+
+ // Returns mutable buffers to the input tensors of inputLength() elements.
+ std::span<float> inputR();
+ std::span<float> inputPhi();
+ std::span<float> inputPressure();
+ std::span<float> inputOrientation();
+ std::span<float> inputTilt();
+
+ // Returns immutable buffers to the output tensors of identical length. Only valid after a
+ // successful call to invoke().
+ std::span<const float> outputR() const;
+ std::span<const float> outputPhi() const;
+ std::span<const float> outputPressure() const;
+
+private:
+ explicit TfLiteMotionPredictorModel(std::string model);
+
+ void allocateTensors();
+ void attachInputTensors();
+ void attachOutputTensors();
+
+ TfLiteTensor* mInputR = nullptr;
+ TfLiteTensor* mInputPhi = nullptr;
+ TfLiteTensor* mInputPressure = nullptr;
+ TfLiteTensor* mInputTilt = nullptr;
+ TfLiteTensor* mInputOrientation = nullptr;
+
+ const TfLiteTensor* mOutputR = nullptr;
+ const TfLiteTensor* mOutputPhi = nullptr;
+ const TfLiteTensor* mOutputPressure = nullptr;
+
+ std::string mFlatBuffer;
+ std::unique_ptr<tflite::ErrorReporter> mErrorReporter;
+ std::unique_ptr<tflite::FlatBufferModel> mModel;
+ std::unique_ptr<tflite::Interpreter> mInterpreter;
+ tflite::SignatureRunner* mRunner = nullptr;
+};
+
+} // namespace android
diff --git a/libs/binder/include/binder/ProcessState.h b/libs/binder/include/binder/ProcessState.h
index bad8cb1..471c994 100644
--- a/libs/binder/include/binder/ProcessState.h
+++ b/libs/binder/include/binder/ProcessState.h
@@ -64,6 +64,11 @@
// For main functions - dangerous for libraries to use
status_t setThreadPoolMaxThreadCount(size_t maxThreads);
status_t enableOnewaySpamDetection(bool enable);
+
+ // Set the name of the current thread to look like a threadpool
+ // thread. Typically this is called before joinThreadPool.
+ //
+ // TODO: remove this API, and automatically set it intelligently.
void giveThreadPoolName();
String8 getDriverName();
diff --git a/libs/gui/include/gui/DisplayCaptureArgs.h b/libs/gui/include/gui/DisplayCaptureArgs.h
index ec884cf..c826c17 100644
--- a/libs/gui/include/gui/DisplayCaptureArgs.h
+++ b/libs/gui/include/gui/DisplayCaptureArgs.h
@@ -24,6 +24,7 @@
#include <binder/Parcelable.h>
#include <ui/GraphicTypes.h>
#include <ui/PixelFormat.h>
+#include <ui/Rect.h>
namespace android::gui {
diff --git a/libs/input/Android.bp b/libs/input/Android.bp
index 8f41cc1..83392ec 100644
--- a/libs/input/Android.bp
+++ b/libs/input/Android.bp
@@ -41,6 +41,7 @@
"-Wall",
"-Wextra",
"-Werror",
+ "-Wno-unused-parameter",
],
srcs: [
"Input.cpp",
@@ -52,13 +53,18 @@
"MotionPredictor.cpp",
"PrintTools.cpp",
"PropertyMap.cpp",
+ "TfLiteMotionPredictor.cpp",
"TouchVideoFrame.cpp",
"VelocityControl.cpp",
"VelocityTracker.cpp",
"VirtualKeyMap.cpp",
],
- header_libs: ["jni_headers"],
+ header_libs: [
+ "flatbuffer_headers",
+ "jni_headers",
+ "tensorflow_headers",
+ ],
export_header_lib_headers: ["jni_headers"],
shared_libs: [
@@ -67,6 +73,7 @@
"liblog",
"libPlatformProperties",
"libvintf",
+ "libtflite",
],
static_libs: [
@@ -103,6 +110,10 @@
sanitize: {
misc_undefined: ["integer"],
},
+
+ required: [
+ "motion_predictor_model_prebuilt",
+ ],
},
host: {
shared: {
diff --git a/libs/input/MotionPredictor.cpp b/libs/input/MotionPredictor.cpp
index 0fa0f12..0f889e8 100644
--- a/libs/input/MotionPredictor.cpp
+++ b/libs/input/MotionPredictor.cpp
@@ -18,118 +18,188 @@
#include <input/MotionPredictor.h>
+#include <cinttypes>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include <android-base/strings.h>
+#include <android/input.h>
+#include <log/log.h>
+
+#include <attestation/HmacKeyManager.h>
+#include <input/TfLiteMotionPredictor.h>
+
+namespace android {
+namespace {
+
+const char DEFAULT_MODEL_PATH[] = "/system/etc/motion_predictor_model.fb";
+const int64_t PREDICTION_INTERVAL_NANOS =
+ 12500000 / 3; // TODO(b/266747937): Get this from the model.
+
/**
* Log debug messages about predictions.
* Enable this via "adb shell setprop log.tag.MotionPredictor DEBUG"
*/
-static bool isDebug() {
+bool isDebug() {
return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG, ANDROID_LOG_INFO);
}
-namespace android {
+// Converts a prediction of some polar (r, phi) to Cartesian (x, y) when applied to an axis.
+TfLiteMotionPredictorSample::Point convertPrediction(
+ const TfLiteMotionPredictorSample::Point& axisFrom,
+ const TfLiteMotionPredictorSample::Point& axisTo, float r, float phi) {
+ const TfLiteMotionPredictorSample::Point axis = axisTo - axisFrom;
+ const float axis_phi = std::atan2(axis.y, axis.x);
+ const float x_delta = r * std::cos(axis_phi + phi);
+ const float y_delta = r * std::sin(axis_phi + phi);
+ return {.x = axisTo.x + x_delta, .y = axisTo.y + y_delta};
+}
+
+} // namespace
// --- MotionPredictor ---
-MotionPredictor::MotionPredictor(nsecs_t predictionTimestampOffsetNanos,
+MotionPredictor::MotionPredictor(nsecs_t predictionTimestampOffsetNanos, const char* modelPath,
std::function<bool()> checkMotionPredictionEnabled)
: mPredictionTimestampOffsetNanos(predictionTimestampOffsetNanos),
- mCheckMotionPredictionEnabled(std::move(checkMotionPredictionEnabled)) {}
+ mCheckMotionPredictionEnabled(std::move(checkMotionPredictionEnabled)),
+ mModel(TfLiteMotionPredictorModel::create(modelPath == nullptr ? DEFAULT_MODEL_PATH
+ : modelPath)) {}
void MotionPredictor::record(const MotionEvent& event) {
- mEvents.push_back({});
- mEvents.back().copyFrom(&event, /*keepHistory=*/true);
- if (mEvents.size() > 2) {
- // Just need 2 samples in order to extrapolate
- mEvents.erase(mEvents.begin());
+ if (!isPredictionAvailable(event.getDeviceId(), event.getSource())) {
+ ALOGE("Prediction not supported for device %d's %s source", event.getDeviceId(),
+ inputEventSourceToString(event.getSource()).c_str());
+ return;
}
+
+ TfLiteMotionPredictorBuffers& buffers =
+ mDeviceBuffers.try_emplace(event.getDeviceId(), mModel->inputLength()).first->second;
+
+ const int32_t action = event.getActionMasked();
+ if (action == AMOTION_EVENT_ACTION_UP) {
+ ALOGD_IF(isDebug(), "End of event stream");
+ buffers.reset();
+ return;
+ } else if (action != AMOTION_EVENT_ACTION_DOWN && action != AMOTION_EVENT_ACTION_MOVE) {
+ ALOGD_IF(isDebug(), "Skipping unsupported %s action",
+ MotionEvent::actionToString(action).c_str());
+ return;
+ }
+
+ if (event.getPointerCount() != 1) {
+ ALOGD_IF(isDebug(), "Prediction not supported for multiple pointers");
+ return;
+ }
+
+ const int32_t toolType = event.getPointerProperties(0)->toolType;
+ if (toolType != AMOTION_EVENT_TOOL_TYPE_STYLUS) {
+ ALOGD_IF(isDebug(), "Prediction not supported for non-stylus tool: %s",
+ motionToolTypeToString(toolType));
+ return;
+ }
+
+ for (size_t i = 0; i <= event.getHistorySize(); ++i) {
+ if (event.isResampled(0, i)) {
+ continue;
+ }
+ const PointerCoords* coords = event.getHistoricalRawPointerCoords(0, i);
+ buffers.pushSample(event.getHistoricalEventTime(i),
+ {
+ .position.x = coords->getAxisValue(AMOTION_EVENT_AXIS_X),
+ .position.y = coords->getAxisValue(AMOTION_EVENT_AXIS_Y),
+ .pressure = event.getHistoricalPressure(0, i),
+ .tilt = event.getHistoricalAxisValue(AMOTION_EVENT_AXIS_TILT, 0,
+ i),
+ .orientation = event.getHistoricalOrientation(0, i),
+ });
+ }
+
+ mLastEvents.try_emplace(event.getDeviceId())
+ .first->second.copyFrom(&event, /*keepHistory=*/false);
}
-/**
- * This is an example implementation that should be replaced with the actual prediction.
- * The returned MotionEvent should be similar to the incoming MotionEvent, except for the
- * fields that are predicted:
- *
- * 1) event.getEventTime
- * 2) event.getPointerCoords
- *
- * The returned event should not contain any of the real, existing data. It should only
- * contain the predicted samples.
- */
std::vector<std::unique_ptr<MotionEvent>> MotionPredictor::predict(nsecs_t timestamp) {
- if (mEvents.size() < 2) {
- return {};
- }
+ std::vector<std::unique_ptr<MotionEvent>> predictions;
- const MotionEvent& event = mEvents.back();
- if (!isPredictionAvailable(event.getDeviceId(), event.getSource())) {
- return {};
- }
-
- std::unique_ptr<MotionEvent> prediction = std::make_unique<MotionEvent>();
- std::vector<PointerCoords> futureCoords;
- const nsecs_t futureTime = timestamp + mPredictionTimestampOffsetNanos;
- const nsecs_t currentTime = event.getEventTime();
- const MotionEvent& previous = mEvents.rbegin()[1];
- const nsecs_t oldTime = previous.getEventTime();
- if (currentTime == oldTime) {
- // This can happen if it's an ACTION_POINTER_DOWN event, for example.
- return {}; // prevent division by zero.
- }
-
- for (size_t i = 0; i < event.getPointerCount(); i++) {
- const int32_t pointerId = event.getPointerId(i);
- const PointerCoords* currentPointerCoords = event.getRawPointerCoords(i);
- const float currentX = currentPointerCoords->getAxisValue(AMOTION_EVENT_AXIS_X);
- const float currentY = currentPointerCoords->getAxisValue(AMOTION_EVENT_AXIS_Y);
-
- PointerCoords coords;
- coords.clear();
-
- ssize_t index = previous.findPointerIndex(pointerId);
- if (index >= 0) {
- // We have old data for this pointer. Compute the prediction.
- const PointerCoords* oldPointerCoords = previous.getRawPointerCoords(index);
- const float oldX = oldPointerCoords->getAxisValue(AMOTION_EVENT_AXIS_X);
- const float oldY = oldPointerCoords->getAxisValue(AMOTION_EVENT_AXIS_Y);
-
- // Let's do a linear interpolation while waiting for a real model
- const float scale =
- static_cast<float>(futureTime - currentTime) / (currentTime - oldTime);
- const float futureX = currentX + (currentX - oldX) * scale;
- const float futureY = currentY + (currentY - oldY) * scale;
-
- coords.setAxisValue(AMOTION_EVENT_AXIS_X, futureX);
- coords.setAxisValue(AMOTION_EVENT_AXIS_Y, futureY);
- ALOGD_IF(isDebug(),
- "Prediction by %.1f ms, (%.1f, %.1f), (%.1f, %.1f) --> (%.1f, %.1f)",
- (futureTime - event.getEventTime()) * 1E-6, oldX, oldY, currentX, currentY,
- futureX, futureY);
+ for (const auto& [deviceId, buffer] : mDeviceBuffers) {
+ if (!buffer.isReady()) {
+ continue;
}
- futureCoords.push_back(coords);
+ buffer.copyTo(*mModel);
+ LOG_ALWAYS_FATAL_IF(!mModel->invoke());
+
+ // Read out the predictions.
+ const std::span<const float> predictedR = mModel->outputR();
+ const std::span<const float> predictedPhi = mModel->outputPhi();
+ const std::span<const float> predictedPressure = mModel->outputPressure();
+
+ TfLiteMotionPredictorSample::Point axisFrom = buffer.axisFrom().position;
+ TfLiteMotionPredictorSample::Point axisTo = buffer.axisTo().position;
+
+ if (isDebug()) {
+ ALOGD("deviceId: %d", deviceId);
+ ALOGD("axisFrom: %f, %f", axisFrom.x, axisFrom.y);
+ ALOGD("axisTo: %f, %f", axisTo.x, axisTo.y);
+ ALOGD("mInputR: %s", base::Join(mModel->inputR(), ", ").c_str());
+ ALOGD("mInputPhi: %s", base::Join(mModel->inputPhi(), ", ").c_str());
+ ALOGD("mInputPressure: %s", base::Join(mModel->inputPressure(), ", ").c_str());
+ ALOGD("mInputTilt: %s", base::Join(mModel->inputTilt(), ", ").c_str());
+ ALOGD("mInputOrientation: %s", base::Join(mModel->inputOrientation(), ", ").c_str());
+ ALOGD("predictedR: %s", base::Join(predictedR, ", ").c_str());
+ ALOGD("predictedPhi: %s", base::Join(predictedPhi, ", ").c_str());
+ ALOGD("predictedPressure: %s", base::Join(predictedPressure, ", ").c_str());
+ }
+
+ const MotionEvent& event = mLastEvents[deviceId];
+ bool hasPredictions = false;
+ std::unique_ptr<MotionEvent> prediction = std::make_unique<MotionEvent>();
+ int64_t predictionTime = buffer.lastTimestamp();
+ const int64_t futureTime = timestamp + mPredictionTimestampOffsetNanos;
+
+ for (int i = 0; i < predictedR.size() && predictionTime <= futureTime; ++i) {
+ const TfLiteMotionPredictorSample::Point point =
+ convertPrediction(axisFrom, axisTo, predictedR[i], predictedPhi[i]);
+ // TODO(b/266747654): Stop predictions if confidence is < some threshold.
+
+ ALOGD_IF(isDebug(), "prediction %d: %f, %f", i, point.x, point.y);
+ PointerCoords coords;
+ coords.clear();
+ coords.setAxisValue(AMOTION_EVENT_AXIS_X, point.x);
+ coords.setAxisValue(AMOTION_EVENT_AXIS_Y, point.y);
+ // TODO(b/266747654): Stop predictions if predicted pressure is < some threshold.
+ coords.setAxisValue(AMOTION_EVENT_AXIS_PRESSURE, predictedPressure[i]);
+
+ predictionTime += PREDICTION_INTERVAL_NANOS;
+ if (i == 0) {
+ hasPredictions = true;
+ prediction->initialize(InputEvent::nextId(), event.getDeviceId(), event.getSource(),
+ event.getDisplayId(), INVALID_HMAC,
+ AMOTION_EVENT_ACTION_MOVE, event.getActionButton(),
+ event.getFlags(), event.getEdgeFlags(), event.getMetaState(),
+ event.getButtonState(), event.getClassification(),
+ event.getTransform(), event.getXPrecision(),
+ event.getYPrecision(), event.getRawXCursorPosition(),
+ event.getRawYCursorPosition(), event.getRawTransform(),
+ event.getDownTime(), predictionTime, event.getPointerCount(),
+ event.getPointerProperties(), &coords);
+ } else {
+ prediction->addSample(predictionTime, &coords);
+ }
+
+ axisFrom = axisTo;
+ axisTo = point;
+ }
+ // TODO(b/266747511): Interpolate to futureTime?
+ if (hasPredictions) {
+ predictions.push_back(std::move(prediction));
+ }
}
-
- /**
- * The process of adding samples is different for the first and subsequent samples:
- * 1. Add the first sample via 'initialize' as below
- * 2. Add subsequent samples via 'addSample'
- */
- prediction->initialize(event.getId(), event.getDeviceId(), event.getSource(),
- event.getDisplayId(), event.getHmac(), event.getAction(),
- event.getActionButton(), event.getFlags(), event.getEdgeFlags(),
- event.getMetaState(), event.getButtonState(), event.getClassification(),
- event.getTransform(), event.getXPrecision(), event.getYPrecision(),
- event.getRawXCursorPosition(), event.getRawYCursorPosition(),
- event.getRawTransform(), event.getDownTime(), futureTime,
- event.getPointerCount(), event.getPointerProperties(),
- futureCoords.data());
-
- // To add more predicted samples, use 'addSample':
- prediction->addSample(futureTime + 1, futureCoords.data());
-
- std::vector<std::unique_ptr<MotionEvent>> out;
- out.push_back(std::move(prediction));
- return out;
+ return predictions;
}
bool MotionPredictor::isPredictionAvailable(int32_t /*deviceId*/, int32_t source) {
diff --git a/libs/input/TfLiteMotionPredictor.cpp b/libs/input/TfLiteMotionPredictor.cpp
new file mode 100644
index 0000000..1a337ad
--- /dev/null
+++ b/libs/input/TfLiteMotionPredictor.cpp
@@ -0,0 +1,338 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "TfLiteMotionPredictor"
+#include <input/TfLiteMotionPredictor.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <fstream>
+#include <ios>
+#include <iterator>
+#include <memory>
+#include <span>
+#include <string>
+#include <type_traits>
+#include <utility>
+
+#define ATRACE_TAG ATRACE_TAG_INPUT
+#include <cutils/trace.h>
+#include <log/log.h>
+
+#include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+
+namespace android {
+namespace {
+
+constexpr char SIGNATURE_KEY[] = "serving_default";
+
+// Input tensor names.
+constexpr char INPUT_R[] = "r";
+constexpr char INPUT_PHI[] = "phi";
+constexpr char INPUT_PRESSURE[] = "pressure";
+constexpr char INPUT_TILT[] = "tilt";
+constexpr char INPUT_ORIENTATION[] = "orientation";
+
+// Output tensor names.
+constexpr char OUTPUT_R[] = "r";
+constexpr char OUTPUT_PHI[] = "phi";
+constexpr char OUTPUT_PRESSURE[] = "pressure";
+
+// A TFLite ErrorReporter that logs to logcat.
+class LoggingErrorReporter : public tflite::ErrorReporter {
+public:
+ int Report(const char* format, va_list args) override {
+ return LOG_PRI_VA(ANDROID_LOG_ERROR, LOG_TAG, format, args);
+ }
+};
+
+// Searches a runner for an input tensor.
+TfLiteTensor* findInputTensor(const char* name, tflite::SignatureRunner* runner) {
+ TfLiteTensor* tensor = runner->input_tensor(name);
+ LOG_ALWAYS_FATAL_IF(!tensor, "Failed to find input tensor '%s'", name);
+ return tensor;
+}
+
+// Searches a runner for an output tensor.
+const TfLiteTensor* findOutputTensor(const char* name, tflite::SignatureRunner* runner) {
+ const TfLiteTensor* tensor = runner->output_tensor(name);
+ LOG_ALWAYS_FATAL_IF(!tensor, "Failed to find output tensor '%s'", name);
+ return tensor;
+}
+
+// Returns the buffer for a tensor of type T.
+template <typename T>
+std::span<T> getTensorBuffer(typename std::conditional<std::is_const<T>::value, const TfLiteTensor*,
+ TfLiteTensor*>::type tensor) {
+ LOG_ALWAYS_FATAL_IF(!tensor);
+
+ const TfLiteType type = tflite::typeToTfLiteType<typename std::remove_cv<T>::type>();
+ LOG_ALWAYS_FATAL_IF(tensor->type != type, "Unexpected type for '%s' tensor: %s (expected %s)",
+ tensor->name, TfLiteTypeGetName(tensor->type), TfLiteTypeGetName(type));
+
+ LOG_ALWAYS_FATAL_IF(!tensor->data.data);
+ return {reinterpret_cast<T*>(tensor->data.data),
+ static_cast<typename std::span<T>::index_type>(tensor->bytes / sizeof(T))};
+}
+
+// Verifies that a tensor exists and has an underlying buffer of type T.
+template <typename T>
+void checkTensor(const TfLiteTensor* tensor) {
+ LOG_ALWAYS_FATAL_IF(!tensor);
+
+ const auto buffer = getTensorBuffer<const T>(tensor);
+ LOG_ALWAYS_FATAL_IF(buffer.empty(), "No buffer for tensor '%s'", tensor->name);
+}
+
+} // namespace
+
+TfLiteMotionPredictorBuffers::TfLiteMotionPredictorBuffers(size_t inputLength) {
+ LOG_ALWAYS_FATAL_IF(inputLength == 0, "Buffer input size must be greater than 0");
+ mInputR.resize(inputLength);
+ mInputPhi.resize(inputLength);
+ mInputPressure.resize(inputLength);
+ mInputTilt.resize(inputLength);
+ mInputOrientation.resize(inputLength);
+}
+
+void TfLiteMotionPredictorBuffers::reset() {
+ std::fill(mInputR.begin(), mInputR.end(), 0);
+ std::fill(mInputPhi.begin(), mInputPhi.end(), 0);
+ std::fill(mInputPressure.begin(), mInputPressure.end(), 0);
+ std::fill(mInputTilt.begin(), mInputTilt.end(), 0);
+ std::fill(mInputOrientation.begin(), mInputOrientation.end(), 0);
+ mAxisFrom.reset();
+ mAxisTo.reset();
+}
+
+void TfLiteMotionPredictorBuffers::copyTo(TfLiteMotionPredictorModel& model) const {
+ LOG_ALWAYS_FATAL_IF(mInputR.size() != model.inputLength(),
+ "Buffer length %zu doesn't match model input length %zu", mInputR.size(),
+ model.inputLength());
+ LOG_ALWAYS_FATAL_IF(!isReady(), "Buffers are incomplete");
+
+ std::copy(mInputR.begin(), mInputR.end(), model.inputR().begin());
+ std::copy(mInputPhi.begin(), mInputPhi.end(), model.inputPhi().begin());
+ std::copy(mInputPressure.begin(), mInputPressure.end(), model.inputPressure().begin());
+ std::copy(mInputTilt.begin(), mInputTilt.end(), model.inputTilt().begin());
+ std::copy(mInputOrientation.begin(), mInputOrientation.end(), model.inputOrientation().begin());
+}
+
+void TfLiteMotionPredictorBuffers::pushSample(int64_t timestamp,
+ const TfLiteMotionPredictorSample sample) {
+ // Convert the sample (x, y) into polar (r, φ) based on a reference axis
+ // from the preceding two points (mAxisFrom/mAxisTo).
+
+ mTimestamp = timestamp;
+
+ if (!mAxisTo) { // First point.
+ mAxisTo = sample;
+ return;
+ }
+
+ // Vector from the last point to the current sample point.
+ const TfLiteMotionPredictorSample::Point v = sample.position - mAxisTo->position;
+
+ const float r = std::hypot(v.x, v.y);
+ float phi = 0;
+ float orientation = 0;
+
+ // Ignore the sample if there is no movement. These samples can occur when there's change to a
+ // property other than the coordinates and pollute the input to the model.
+ if (r == 0) {
+ return;
+ }
+
+ if (!mAxisFrom) { // Second point.
+ // We can only determine the distance from the first point, and not any
+ // angle. However, if the second point forms an axis, the orientation can
+ // be transformed relative to that axis.
+ const float axisPhi = std::atan2(v.y, v.x);
+ // A MotionEvent's orientation is measured clockwise from the vertical
+ // axis, but axisPhi is measured counter-clockwise from the horizontal
+ // axis.
+ orientation = M_PI_2 - sample.orientation - axisPhi;
+ } else {
+ const TfLiteMotionPredictorSample::Point axis = mAxisTo->position - mAxisFrom->position;
+ const float axisPhi = std::atan2(axis.y, axis.x);
+ phi = std::atan2(v.y, v.x) - axisPhi;
+
+ if (std::hypot(axis.x, axis.y) > 0) {
+ // See note above.
+ orientation = M_PI_2 - sample.orientation - axisPhi;
+ }
+ }
+
+ // Update the axis for the next point.
+ mAxisFrom = mAxisTo;
+ mAxisTo = sample;
+
+ // Push the current sample onto the end of the input buffers.
+ mInputR.erase(mInputR.begin());
+ mInputPhi.erase(mInputPhi.begin());
+ mInputPressure.erase(mInputPressure.begin());
+ mInputTilt.erase(mInputTilt.begin());
+ mInputOrientation.erase(mInputOrientation.begin());
+
+ mInputR.push_back(r);
+ mInputPhi.push_back(phi);
+ mInputPressure.push_back(sample.pressure);
+ mInputTilt.push_back(sample.tilt);
+ mInputOrientation.push_back(orientation);
+}
+
+std::unique_ptr<TfLiteMotionPredictorModel> TfLiteMotionPredictorModel::create(
+ const char* modelPath) {
+ std::ifstream f(modelPath, std::ios::binary);
+ LOG_ALWAYS_FATAL_IF(!f, "Could not read model from %s", modelPath);
+
+ std::string data;
+ data.assign(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>());
+
+ return std::unique_ptr<TfLiteMotionPredictorModel>(
+ new TfLiteMotionPredictorModel(std::move(data)));
+}
+
+TfLiteMotionPredictorModel::TfLiteMotionPredictorModel(std::string model)
+ : mFlatBuffer(std::move(model)) {
+ mErrorReporter = std::make_unique<LoggingErrorReporter>();
+ mModel = tflite::FlatBufferModel::VerifyAndBuildFromBuffer(mFlatBuffer.data(),
+ mFlatBuffer.length(),
+ /*extra_verifier=*/nullptr,
+ mErrorReporter.get());
+ LOG_ALWAYS_FATAL_IF(!mModel);
+
+ tflite::ops::builtin::BuiltinOpResolver resolver;
+ tflite::InterpreterBuilder builder(*mModel, resolver);
+
+ if (builder(&mInterpreter) != kTfLiteOk || !mInterpreter) {
+ LOG_ALWAYS_FATAL("Failed to build interpreter");
+ }
+
+ mRunner = mInterpreter->GetSignatureRunner(SIGNATURE_KEY);
+ LOG_ALWAYS_FATAL_IF(!mRunner, "Failed to find runner for signature '%s'", SIGNATURE_KEY);
+
+ allocateTensors();
+}
+
+void TfLiteMotionPredictorModel::allocateTensors() {
+ if (mRunner->AllocateTensors() != kTfLiteOk) {
+ LOG_ALWAYS_FATAL("Failed to allocate tensors");
+ }
+
+ attachInputTensors();
+ attachOutputTensors();
+
+ checkTensor<float>(mInputR);
+ checkTensor<float>(mInputPhi);
+ checkTensor<float>(mInputPressure);
+ checkTensor<float>(mInputTilt);
+ checkTensor<float>(mInputOrientation);
+ checkTensor<float>(mOutputR);
+ checkTensor<float>(mOutputPhi);
+ checkTensor<float>(mOutputPressure);
+
+ const auto checkInputTensorSize = [this](const TfLiteTensor* tensor) {
+ const size_t size = getTensorBuffer<const float>(tensor).size();
+ LOG_ALWAYS_FATAL_IF(size != inputLength(),
+ "Tensor '%s' length %zu does not match input length %zu", tensor->name,
+ size, inputLength());
+ };
+
+ checkInputTensorSize(mInputR);
+ checkInputTensorSize(mInputPhi);
+ checkInputTensorSize(mInputPressure);
+ checkInputTensorSize(mInputTilt);
+ checkInputTensorSize(mInputOrientation);
+}
+
+void TfLiteMotionPredictorModel::attachInputTensors() {
+ mInputR = findInputTensor(INPUT_R, mRunner);
+ mInputPhi = findInputTensor(INPUT_PHI, mRunner);
+ mInputPressure = findInputTensor(INPUT_PRESSURE, mRunner);
+ mInputTilt = findInputTensor(INPUT_TILT, mRunner);
+ mInputOrientation = findInputTensor(INPUT_ORIENTATION, mRunner);
+}
+
+void TfLiteMotionPredictorModel::attachOutputTensors() {
+ mOutputR = findOutputTensor(OUTPUT_R, mRunner);
+ mOutputPhi = findOutputTensor(OUTPUT_PHI, mRunner);
+ mOutputPressure = findOutputTensor(OUTPUT_PRESSURE, mRunner);
+}
+
+bool TfLiteMotionPredictorModel::invoke() {
+ ATRACE_BEGIN("TfLiteMotionPredictorModel::invoke");
+ TfLiteStatus result = mRunner->Invoke();
+ ATRACE_END();
+
+ if (result != kTfLiteOk) {
+ return false;
+ }
+
+ // Invoke() might reallocate tensors, so they need to be reattached.
+ attachInputTensors();
+ attachOutputTensors();
+
+ if (outputR().size() != outputPhi().size() || outputR().size() != outputPressure().size()) {
+ LOG_ALWAYS_FATAL("Output size mismatch: (r: %zu, phi: %zu, pressure: %zu)",
+ outputR().size(), outputPhi().size(), outputPressure().size());
+ }
+
+ return true;
+}
+
+size_t TfLiteMotionPredictorModel::inputLength() const {
+ return getTensorBuffer<const float>(mInputR).size();
+}
+
+std::span<float> TfLiteMotionPredictorModel::inputR() {
+ return getTensorBuffer<float>(mInputR);
+}
+
+std::span<float> TfLiteMotionPredictorModel::inputPhi() {
+ return getTensorBuffer<float>(mInputPhi);
+}
+
+std::span<float> TfLiteMotionPredictorModel::inputPressure() {
+ return getTensorBuffer<float>(mInputPressure);
+}
+
+std::span<float> TfLiteMotionPredictorModel::inputTilt() {
+ return getTensorBuffer<float>(mInputTilt);
+}
+
+std::span<float> TfLiteMotionPredictorModel::inputOrientation() {
+ return getTensorBuffer<float>(mInputOrientation);
+}
+
+std::span<const float> TfLiteMotionPredictorModel::outputR() const {
+ return getTensorBuffer<const float>(mOutputR);
+}
+
+std::span<const float> TfLiteMotionPredictorModel::outputPhi() const {
+ return getTensorBuffer<const float>(mOutputPhi);
+}
+
+std::span<const float> TfLiteMotionPredictorModel::outputPressure() const {
+ return getTensorBuffer<const float>(mOutputPressure);
+}
+
+} // namespace android
diff --git a/libs/input/tests/Android.bp b/libs/input/tests/Android.bp
index e2c0860..916a8f2 100644
--- a/libs/input/tests/Android.bp
+++ b/libs/input/tests/Android.bp
@@ -10,6 +10,7 @@
cc_test {
name: "libinput_tests",
+ cpp_std: "c++20",
host_supported: true,
srcs: [
"IdGenerator_test.cpp",
@@ -18,12 +19,18 @@
"InputEvent_test.cpp",
"InputPublisherAndConsumer_test.cpp",
"MotionPredictor_test.cpp",
+ "TfLiteMotionPredictor_test.cpp",
"TouchResampling_test.cpp",
"TouchVideoFrame_test.cpp",
"VelocityTracker_test.cpp",
"VerifiedInputEvent_test.cpp",
],
+ header_libs: [
+ "flatbuffer_headers",
+ "tensorflow_headers",
+ ],
static_libs: [
+ "libgmock",
"libgui_window_info_static",
"libinput",
"libui-types",
@@ -32,6 +39,7 @@
"-Wall",
"-Wextra",
"-Werror",
+ "-Wno-unused-parameter",
],
shared_libs: [
"libbase",
@@ -39,10 +47,14 @@
"libcutils",
"liblog",
"libPlatformProperties",
+ "libtflite",
"libutils",
"libvintf",
],
- data: ["data/*"],
+ data: [
+ "data/*",
+ ":motion_predictor_model.fb",
+ ],
test_options: {
unit_test: true,
},
diff --git a/libs/input/tests/MotionPredictor_test.cpp b/libs/input/tests/MotionPredictor_test.cpp
index d2b59a1..ce87c86 100644
--- a/libs/input/tests/MotionPredictor_test.cpp
+++ b/libs/input/tests/MotionPredictor_test.cpp
@@ -14,17 +14,36 @@
* limitations under the License.
*/
+#include <chrono>
+
+#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <gui/constants.h>
#include <input/Input.h>
#include <input/MotionPredictor.h>
+using namespace std::literals::chrono_literals;
+
namespace android {
+using ::testing::IsEmpty;
+using ::testing::SizeIs;
+using ::testing::UnorderedElementsAre;
+
+const char MODEL_PATH[] =
+#if defined(__ANDROID__)
+ "/system/etc/motion_predictor_model.fb";
+#else
+ "motion_predictor_model.fb";
+#endif
+
constexpr int32_t DOWN = AMOTION_EVENT_ACTION_DOWN;
constexpr int32_t MOVE = AMOTION_EVENT_ACTION_MOVE;
+constexpr int32_t UP = AMOTION_EVENT_ACTION_UP;
+constexpr nsecs_t NSEC_PER_MSEC = 1'000'000;
-static MotionEvent getMotionEvent(int32_t action, float x, float y, nsecs_t eventTime) {
+static MotionEvent getMotionEvent(int32_t action, float x, float y,
+ std::chrono::nanoseconds eventTime, int32_t deviceId = 0) {
MotionEvent event;
constexpr size_t pointerCount = 1;
std::vector<PointerProperties> pointerProperties;
@@ -33,6 +52,7 @@
PointerProperties properties;
properties.clear();
properties.id = i;
+ properties.toolType = AMOTION_EVENT_TOOL_TYPE_STYLUS;
pointerProperties.push_back(properties);
PointerCoords coords;
coords.clear();
@@ -42,73 +62,93 @@
}
ui::Transform identityTransform;
- event.initialize(InputEvent::nextId(), /*deviceId=*/0, AINPUT_SOURCE_STYLUS,
- ADISPLAY_ID_DEFAULT, {0}, action, /*actionButton=*/0, /*flags=*/0,
- AMOTION_EVENT_EDGE_FLAG_NONE, AMETA_NONE, /*buttonState=*/0,
- MotionClassification::NONE, identityTransform, /*xPrecision=*/0.1,
+ event.initialize(InputEvent::nextId(), deviceId, AINPUT_SOURCE_STYLUS, ADISPLAY_ID_DEFAULT, {0},
+ action, /*actionButton=*/0, /*flags=*/0, AMOTION_EVENT_EDGE_FLAG_NONE,
+ AMETA_NONE, /*buttonState=*/0, MotionClassification::NONE, identityTransform,
+ /*xPrecision=*/0.1,
/*yPrecision=*/0.2, /*xCursorPosition=*/280, /*yCursorPosition=*/540,
- identityTransform, /*downTime=*/100, eventTime, pointerCount,
+ identityTransform, /*downTime=*/100, eventTime.count(), pointerCount,
pointerProperties.data(), pointerCoords.data());
return event;
}
-/**
- * A linear motion should be predicted to be linear in the future
- */
-TEST(MotionPredictorTest, LinearPrediction) {
- MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0,
- []() { return true /*enable prediction*/; });
-
- predictor.record(getMotionEvent(DOWN, 0, 1, 0));
- predictor.record(getMotionEvent(MOVE, 1, 3, 10));
- predictor.record(getMotionEvent(MOVE, 2, 5, 20));
- predictor.record(getMotionEvent(MOVE, 3, 7, 30));
- std::vector<std::unique_ptr<MotionEvent>> predicted = predictor.predict(40);
- ASSERT_EQ(1u, predicted.size());
- ASSERT_EQ(predicted[0]->getX(0), 4);
- ASSERT_EQ(predicted[0]->getY(0), 9);
-}
-
-/**
- * A still motion should be predicted to remain still
- */
-TEST(MotionPredictorTest, StationaryPrediction) {
- MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0,
- []() { return true /*enable prediction*/; });
-
- predictor.record(getMotionEvent(DOWN, 0, 1, 0));
- predictor.record(getMotionEvent(MOVE, 0, 1, 10));
- predictor.record(getMotionEvent(MOVE, 0, 1, 20));
- predictor.record(getMotionEvent(MOVE, 0, 1, 30));
- std::vector<std::unique_ptr<MotionEvent>> predicted = predictor.predict(40);
- ASSERT_EQ(1u, predicted.size());
- ASSERT_EQ(predicted[0]->getX(0), 0);
- ASSERT_EQ(predicted[0]->getY(0), 1);
-}
-
TEST(MotionPredictorTest, IsPredictionAvailable) {
- MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0,
+ MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0, MODEL_PATH,
[]() { return true /*enable prediction*/; });
ASSERT_TRUE(predictor.isPredictionAvailable(/*deviceId=*/1, AINPUT_SOURCE_STYLUS));
ASSERT_FALSE(predictor.isPredictionAvailable(/*deviceId=*/1, AINPUT_SOURCE_TOUCHSCREEN));
}
TEST(MotionPredictorTest, Offset) {
- MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/1,
+ MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/1, MODEL_PATH,
[]() { return true /*enable prediction*/; });
- predictor.record(getMotionEvent(DOWN, 0, 1, 30));
- predictor.record(getMotionEvent(MOVE, 0, 1, 35));
- std::vector<std::unique_ptr<MotionEvent>> predicted = predictor.predict(40);
+ predictor.record(getMotionEvent(DOWN, 0, 1, 30ms));
+ predictor.record(getMotionEvent(MOVE, 0, 2, 35ms));
+ std::vector<std::unique_ptr<MotionEvent>> predicted = predictor.predict(40 * NSEC_PER_MSEC);
ASSERT_EQ(1u, predicted.size());
ASSERT_GE(predicted[0]->getEventTime(), 41);
}
+TEST(MotionPredictorTest, FollowsGesture) {
+ MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0, MODEL_PATH,
+ []() { return true /*enable prediction*/; });
+
+ // MOVE without a DOWN is ignored.
+ predictor.record(getMotionEvent(MOVE, 1, 3, 10ms));
+ EXPECT_THAT(predictor.predict(20 * NSEC_PER_MSEC), IsEmpty());
+
+ predictor.record(getMotionEvent(DOWN, 2, 5, 20ms));
+ predictor.record(getMotionEvent(MOVE, 2, 7, 30ms));
+ predictor.record(getMotionEvent(MOVE, 3, 9, 40ms));
+ EXPECT_THAT(predictor.predict(50 * NSEC_PER_MSEC), SizeIs(1));
+
+ predictor.record(getMotionEvent(UP, 4, 11, 50ms));
+ EXPECT_THAT(predictor.predict(20 * NSEC_PER_MSEC), IsEmpty());
+}
+
+TEST(MotionPredictorTest, MultipleDevicesTracked) {
+ MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0, MODEL_PATH,
+ []() { return true /*enable prediction*/; });
+
+ predictor.record(getMotionEvent(DOWN, 1, 3, 0ms, /*deviceId=*/0));
+ predictor.record(getMotionEvent(MOVE, 1, 3, 10ms, /*deviceId=*/0));
+ predictor.record(getMotionEvent(MOVE, 2, 5, 20ms, /*deviceId=*/0));
+ predictor.record(getMotionEvent(MOVE, 3, 7, 30ms, /*deviceId=*/0));
+
+ predictor.record(getMotionEvent(DOWN, 100, 300, 0ms, /*deviceId=*/1));
+ predictor.record(getMotionEvent(MOVE, 100, 300, 10ms, /*deviceId=*/1));
+ predictor.record(getMotionEvent(MOVE, 200, 500, 20ms, /*deviceId=*/1));
+ predictor.record(getMotionEvent(MOVE, 300, 700, 30ms, /*deviceId=*/1));
+
+ {
+ std::vector<std::unique_ptr<MotionEvent>> predicted = predictor.predict(40 * NSEC_PER_MSEC);
+ ASSERT_EQ(2u, predicted.size());
+
+ // Order of the returned vector is not guaranteed.
+ std::vector<int32_t> seenDeviceIds;
+ for (const auto& prediction : predicted) {
+ seenDeviceIds.push_back(prediction->getDeviceId());
+ }
+ EXPECT_THAT(seenDeviceIds, UnorderedElementsAre(0, 1));
+ }
+
+ // End the gesture for device 0.
+ predictor.record(getMotionEvent(UP, 4, 9, 40ms, /*deviceId=*/0));
+ predictor.record(getMotionEvent(MOVE, 400, 900, 40ms, /*deviceId=*/1));
+
+ {
+ std::vector<std::unique_ptr<MotionEvent>> predicted = predictor.predict(40 * NSEC_PER_MSEC);
+ ASSERT_EQ(1u, predicted.size());
+ ASSERT_EQ(predicted[0]->getDeviceId(), 1);
+ }
+}
+
TEST(MotionPredictorTest, FlagDisablesPrediction) {
- MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0,
+ MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0, MODEL_PATH,
[]() { return false /*disable prediction*/; });
- predictor.record(getMotionEvent(DOWN, 0, 1, 30));
- predictor.record(getMotionEvent(MOVE, 0, 1, 35));
- std::vector<std::unique_ptr<MotionEvent>> predicted = predictor.predict(40);
+ predictor.record(getMotionEvent(DOWN, 0, 1, 30ms));
+ predictor.record(getMotionEvent(MOVE, 0, 1, 35ms));
+ std::vector<std::unique_ptr<MotionEvent>> predicted = predictor.predict(40 * NSEC_PER_MSEC);
ASSERT_EQ(0u, predicted.size());
ASSERT_FALSE(predictor.isPredictionAvailable(/*deviceId=*/1, AINPUT_SOURCE_STYLUS));
ASSERT_FALSE(predictor.isPredictionAvailable(/*deviceId=*/1, AINPUT_SOURCE_TOUCHSCREEN));
diff --git a/libs/input/tests/TfLiteMotionPredictor_test.cpp b/libs/input/tests/TfLiteMotionPredictor_test.cpp
new file mode 100644
index 0000000..454f2aa
--- /dev/null
+++ b/libs/input/tests/TfLiteMotionPredictor_test.cpp
@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <cmath>
+#include <fstream>
+#include <ios>
+#include <iterator>
+#include <string>
+
+#include <android-base/file.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include <input/TfLiteMotionPredictor.h>
+
+namespace android {
+namespace {
+
+using ::testing::Each;
+using ::testing::ElementsAre;
+using ::testing::FloatNear;
+
+std::string getModelPath() {
+#if defined(__ANDROID__)
+ return "/system/etc/motion_predictor_model.fb";
+#else
+ return base::GetExecutableDirectory() + "/motion_predictor_model.fb";
+#endif
+}
+
+TEST(TfLiteMotionPredictorTest, BuffersReadiness) {
+ TfLiteMotionPredictorBuffers buffers(/*inputLength=*/5);
+ ASSERT_FALSE(buffers.isReady());
+
+ buffers.pushSample(/*timestamp=*/0, {.position = {.x = 100, .y = 100}});
+ ASSERT_FALSE(buffers.isReady());
+
+ buffers.pushSample(/*timestamp=*/1, {.position = {.x = 100, .y = 100}});
+ ASSERT_FALSE(buffers.isReady());
+
+ // Two samples with distinct positions are required.
+ buffers.pushSample(/*timestamp=*/2, {.position = {.x = 100, .y = 110}});
+ ASSERT_TRUE(buffers.isReady());
+
+ buffers.reset();
+ ASSERT_FALSE(buffers.isReady());
+}
+
+TEST(TfLiteMotionPredictorTest, BuffersRecentData) {
+ TfLiteMotionPredictorBuffers buffers(/*inputLength=*/5);
+
+ buffers.pushSample(/*timestamp=*/1, {.position = {.x = 100, .y = 200}});
+ ASSERT_EQ(buffers.lastTimestamp(), 1);
+
+ buffers.pushSample(/*timestamp=*/2, {.position = {.x = 150, .y = 250}});
+ ASSERT_EQ(buffers.lastTimestamp(), 2);
+ ASSERT_TRUE(buffers.isReady());
+ ASSERT_EQ(buffers.axisFrom().position.x, 100);
+ ASSERT_EQ(buffers.axisFrom().position.y, 200);
+ ASSERT_EQ(buffers.axisTo().position.x, 150);
+ ASSERT_EQ(buffers.axisTo().position.y, 250);
+
+ // Position doesn't change, so neither do the axes.
+ buffers.pushSample(/*timestamp=*/3, {.position = {.x = 150, .y = 250}});
+ ASSERT_EQ(buffers.lastTimestamp(), 3);
+ ASSERT_TRUE(buffers.isReady());
+ ASSERT_EQ(buffers.axisFrom().position.x, 100);
+ ASSERT_EQ(buffers.axisFrom().position.y, 200);
+ ASSERT_EQ(buffers.axisTo().position.x, 150);
+ ASSERT_EQ(buffers.axisTo().position.y, 250);
+
+ buffers.pushSample(/*timestamp=*/4, {.position = {.x = 180, .y = 280}});
+ ASSERT_EQ(buffers.lastTimestamp(), 4);
+ ASSERT_TRUE(buffers.isReady());
+ ASSERT_EQ(buffers.axisFrom().position.x, 150);
+ ASSERT_EQ(buffers.axisFrom().position.y, 250);
+ ASSERT_EQ(buffers.axisTo().position.x, 180);
+ ASSERT_EQ(buffers.axisTo().position.y, 280);
+}
+
+TEST(TfLiteMotionPredictorTest, BuffersCopyTo) {
+ std::unique_ptr<TfLiteMotionPredictorModel> model =
+ TfLiteMotionPredictorModel::create(getModelPath().c_str());
+ TfLiteMotionPredictorBuffers buffers(model->inputLength());
+
+ buffers.pushSample(/*timestamp=*/1,
+ {.position = {.x = 10, .y = 10},
+ .pressure = 0,
+ .orientation = 0,
+ .tilt = 0.2});
+ buffers.pushSample(/*timestamp=*/2,
+ {.position = {.x = 10, .y = 50},
+ .pressure = 0.4,
+ .orientation = M_PI / 4,
+ .tilt = 0.3});
+ buffers.pushSample(/*timestamp=*/3,
+ {.position = {.x = 30, .y = 50},
+ .pressure = 0.5,
+ .orientation = -M_PI / 4,
+ .tilt = 0.4});
+ buffers.pushSample(/*timestamp=*/3,
+ {.position = {.x = 30, .y = 60},
+ .pressure = 0,
+ .orientation = 0,
+ .tilt = 0.5});
+ buffers.copyTo(*model);
+
+ const int zeroPadding = model->inputLength() - 3;
+ ASSERT_GE(zeroPadding, 0);
+
+ EXPECT_THAT(model->inputR().subspan(0, zeroPadding), Each(0));
+ EXPECT_THAT(model->inputPhi().subspan(0, zeroPadding), Each(0));
+ EXPECT_THAT(model->inputPressure().subspan(0, zeroPadding), Each(0));
+ EXPECT_THAT(model->inputTilt().subspan(0, zeroPadding), Each(0));
+ EXPECT_THAT(model->inputOrientation().subspan(0, zeroPadding), Each(0));
+
+ EXPECT_THAT(model->inputR().subspan(zeroPadding), ElementsAre(40, 20, 10));
+ EXPECT_THAT(model->inputPhi().subspan(zeroPadding), ElementsAre(0, -M_PI / 2, M_PI / 2));
+ EXPECT_THAT(model->inputPressure().subspan(zeroPadding), ElementsAre(0.4, 0.5, 0));
+ EXPECT_THAT(model->inputTilt().subspan(zeroPadding), ElementsAre(0.3, 0.4, 0.5));
+ EXPECT_THAT(model->inputOrientation().subspan(zeroPadding),
+ ElementsAre(FloatNear(-M_PI / 4, 1e-5), FloatNear(M_PI / 4, 1e-5),
+ FloatNear(M_PI / 2, 1e-5)));
+}
+
+TEST(TfLiteMotionPredictorTest, ModelInputOutputLength) {
+ std::unique_ptr<TfLiteMotionPredictorModel> model =
+ TfLiteMotionPredictorModel::create(getModelPath().c_str());
+ ASSERT_GT(model->inputLength(), 0u);
+
+ const int inputLength = model->inputLength();
+ ASSERT_EQ(inputLength, model->inputR().size());
+ ASSERT_EQ(inputLength, model->inputPhi().size());
+ ASSERT_EQ(inputLength, model->inputPressure().size());
+ ASSERT_EQ(inputLength, model->inputOrientation().size());
+ ASSERT_EQ(inputLength, model->inputTilt().size());
+
+ ASSERT_TRUE(model->invoke());
+
+ ASSERT_EQ(model->outputR().size(), model->outputPhi().size());
+ ASSERT_EQ(model->outputR().size(), model->outputPressure().size());
+}
+
+TEST(TfLiteMotionPredictorTest, ModelOutput) {
+ std::unique_ptr<TfLiteMotionPredictorModel> model =
+ TfLiteMotionPredictorModel::create(getModelPath().c_str());
+ TfLiteMotionPredictorBuffers buffers(model->inputLength());
+
+ buffers.pushSample(/*timestamp=*/1, {.position = {.x = 100, .y = 200}, .pressure = 0.2});
+ buffers.pushSample(/*timestamp=*/2, {.position = {.x = 150, .y = 250}, .pressure = 0.4});
+ buffers.pushSample(/*timestamp=*/3, {.position = {.x = 180, .y = 280}, .pressure = 0.6});
+ buffers.copyTo(*model);
+
+ ASSERT_TRUE(model->invoke());
+
+ // The actual model output is implementation-defined, but it should at least be non-zero and
+ // non-NaN.
+ const auto is_valid = [](float value) { return !isnan(value) && value != 0; };
+ ASSERT_TRUE(std::all_of(model->outputR().begin(), model->outputR().end(), is_valid));
+ ASSERT_TRUE(std::all_of(model->outputPhi().begin(), model->outputPhi().end(), is_valid));
+ ASSERT_TRUE(
+ std::all_of(model->outputPressure().begin(), model->outputPressure().end(), is_valid));
+}
+
+} // namespace
+} // namespace android
diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegdecoder.h b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegdecoder.h
index d0de48f..419b63d 100644
--- a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegdecoder.h
+++ b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegdecoder.h
@@ -47,7 +47,7 @@
*/
void* getDecompressedImagePtr();
/*
- * Returns the decompressed raw image buffer size. This mgit ethod must be called only after
+ * Returns the decompressed raw image buffer size. This method must be called only after
* calling decompressImage().
*/
size_t getDecompressedImageSize();
@@ -92,10 +92,6 @@
size_t* pWidth, size_t* pHeight,
std::vector<uint8_t>* iccData,
std::vector<uint8_t>* exifData);
- /*
- * Extracts EXIF package and updates the EXIF position / length without decoding the image.
- */
- bool extractEXIF(const void* image, int length);
private:
bool decode(const void* image, int length, bool decodeToRGBA);
diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h
index ae15d24..696be1b 100644
--- a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h
+++ b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h
@@ -321,13 +321,17 @@
jr_compressed_ptr dest);
/*
- * This method is called in the encoding pipeline. It will take the standard 8-bit JPEG image
- * and the compressed recovery map as input, and update the XMP metadata with the end of JPEG
- * marker, and append the compressed gian map after the JPEG.
+ * This method is called in the encoding pipeline. It will take the standard 8-bit JPEG image,
+ * the compressed recovery map and optionally the exif package as inputs, and generate the XMP
+ * metadata, and finally append everything in the order of:
+ * SOI, APP2(EXIF) (if EXIF is from outside), APP2(XMP), primary image, recovery map
+ * Note that EXIF package is only available for encoding API-0 and API-1. For encoding API-2 and
+ * API-3 this parameter is null, but the primary image in JPEG/R may still have EXIF as long as
+ * the input JPEG has EXIF.
*
* @param compressed_jpeg_image compressed 8-bit JPEG image
* @param compress_recovery_map compressed recover map
- * @param exif EXIF package
+ * @param (nullable) exif EXIF package
* @param metadata JPEG/R metadata to encode in XMP of the jpeg
* @param dest compressed JPEGR image
* @return NO_ERROR if calculation succeeds, error code if error occurs.
diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymaputils.h b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymaputils.h
index 8b2672f..c36a363 100644
--- a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymaputils.h
+++ b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymaputils.h
@@ -45,7 +45,6 @@
* @return status of succeed or error code.
*/
status_t Write(jr_compressed_ptr destination, const void* source, size_t length, int &position);
-status_t Write(jr_exif_ptr destination, const void* source, size_t length, int &position);
/*
@@ -105,83 +104,6 @@
* @return XMP metadata in type of string
*/
std::string generateXmp(int secondary_image_length, jpegr_metadata& metadata);
-
-/*
- * Add J R entry to existing exif, or create a new one with J R entry if it's null.
- * EXIF syntax / change:
- * ori:
- * FF E1 - APP1
- * 01 FC - size of APP1 (to be calculated)
- * -----------------------------------------------------
- * 45 78 69 66 00 00 - Exif\0\0 "Exif header"
- * 49 49 2A 00 - TIFF Header
- * 08 00 00 00 - offset to the IFD (image file directory)
- * 06 00 - 6 entries
- * 00 01 - Width Tag
- * 03 00 - 'Short' type
- * 01 00 00 00 - 1 component
- * 00 05 00 00 - image with 0x500
- *--------------------------------------------------------------------------
- * new:
- * FF E1 - APP1
- * 02 08 - new size, equals to old size + EXIF_J_R_ENTRY_LENGTH (12)
- *-----------------------------------------------------
- * 45 78 69 66 00 00 - Exif\0\0 "Exif header"
- * 49 49 2A 00 - TIFF Header
- * 08 00 00 00 - offset to the IFD (image file directory)
- * 07 00 - +1 entry
- * 4A 52 Custom ('J''R') Tag
- * 07 00 - Unknown type
- * 01 00 00 00 - 1 component
- * 00 00 00 00 - empty data
- * 00 01 - Width Tag
- * 03 00 - 'Short' type
- * 01 00 00 00 - 1 component
- * 00 05 00 00 - image with 0x500
- */
-status_t updateExif(jr_exif_ptr exif, jr_exif_ptr dest);
-
-/*
- * Modify offsets in EXIF in place.
- *
- * Each tag has the following structure:
- *
- * 00 01 - Tag
- * 03 00 - data format
- * 01 00 00 00 - number of components
- * 00 05 00 00 - value
- *
- * The value means offset if
- * (1) num_of_components * bytes_per_component > 4 bytes, or
- * (2) tag == 0x8769 (ExifOffset).
- * In both cases, the method will add EXIF_J_R_ENTRY_LENGTH (12) to the offsets.
- */
-void updateExifOffsets(jr_exif_ptr exif, int pos, bool use_big_endian);
-void updateExifOffsets(jr_exif_ptr exif, int pos, int num_entry, bool use_big_endian);
-
-/*
- * Read data from the target position and target length in bytes;
- */
-int readValue(uint8_t* data, int pos, int length, bool use_big_endian);
-
-/*
- * Returns the length of data format in bytes
- *
- * ----------------------------------------------------------------------------------------------
- * | value | 1 | 2 | 3 | 4 |
- * | format | unsigned byte | ascii strings | unsigned short | unsigned long |
- * | bytes/component | 1 | 1 | 2 | 4 |
- * ----------------------------------------------------------------------------------------------
- * | value | 5 | 6 | 7 | 8 |
- * | format |unsigned rational| signed byte | undefined | signed short |
- * | bytes/component | 8 | 1 | 1 | 2 |
- * ----------------------------------------------------------------------------------------------
- * | value | 9 | 10 | 11 | 12 |
- * | format | signed long | signed rational | single float | double float |
- * | bytes/component | 4 | 8 | 4 | 8 |
- * ----------------------------------------------------------------------------------------------
- */
-int findFormatLengthInBytes(int data_format);
}
#endif //ANDROID_JPEGRECOVERYMAP_RECOVERYMAPUTILS_H
diff --git a/libs/jpegrecoverymap/jpegdecoder.cpp b/libs/jpegrecoverymap/jpegdecoder.cpp
index 6fbc6b0..1bf609a 100644
--- a/libs/jpegrecoverymap/jpegdecoder.cpp
+++ b/libs/jpegrecoverymap/jpegdecoder.cpp
@@ -248,60 +248,6 @@
return true;
}
-// TODO (Fyodor/Dichen): merge this method with getCompressedImageParameters() since they have
-// similar functionality. Yet Dichen is not familiar with who's calling
-// getCompressedImageParameters(), looks like it's used by some pending CLs.
-bool JpegDecoder::extractEXIF(const void* image, int length) {
- jpeg_decompress_struct cinfo;
- jpegr_source_mgr mgr(static_cast<const uint8_t*>(image), length);
- jpegrerror_mgr myerr;
-
- cinfo.err = jpeg_std_error(&myerr.pub);
- myerr.pub.error_exit = jpegrerror_exit;
-
- if (setjmp(myerr.setjmp_buffer)) {
- jpeg_destroy_decompress(&cinfo);
- return false;
- }
- jpeg_create_decompress(&cinfo);
-
- jpeg_save_markers(&cinfo, kAPP0Marker, 0xFFFF);
- jpeg_save_markers(&cinfo, kAPP1Marker, 0xFFFF);
- jpeg_save_markers(&cinfo, kAPP2Marker, 0xFFFF);
-
- cinfo.src = &mgr;
- jpeg_read_header(&cinfo, TRUE);
-
- bool exifAppears = false;
- size_t pos = 2; // position after SOI
- for (jpeg_marker_struct* marker = cinfo.marker_list;
- marker && !exifAppears;
- marker = marker->next) {
-
- pos += 4;
- pos += marker->original_length;
-
- if (marker->marker != kAPP1Marker) {
- continue;
- }
-
- const unsigned int len = marker->data_length;
- if (!exifAppears &&
- len > kExifIdCode.size() &&
- !strncmp(reinterpret_cast<const char*>(marker->data),
- kExifIdCode.c_str(),
- kExifIdCode.size())) {
- mEXIFBuffer.resize(len, 0);
- memcpy(static_cast<void*>(mEXIFBuffer.data()), marker->data, len);
- exifAppears = true;
- mExifPos = pos - marker->original_length;
- }
- }
-
- jpeg_destroy_decompress(&cinfo);
- return true;
-}
-
bool JpegDecoder::decompress(jpeg_decompress_struct* cinfo, const uint8_t* dest,
bool isSingleChannel) {
if (isSingleChannel) {
diff --git a/libs/jpegrecoverymap/recoverymap.cpp b/libs/jpegrecoverymap/recoverymap.cpp
index 22289de..30aa846 100644
--- a/libs/jpegrecoverymap/recoverymap.cpp
+++ b/libs/jpegrecoverymap/recoverymap.cpp
@@ -175,18 +175,7 @@
jpeg.data = jpeg_encoder.getCompressedImagePtr();
jpeg.length = jpeg_encoder.getCompressedImageSize();
- jpegr_exif_struct new_exif;
- if (exif == nullptr || exif->data == nullptr) {
- new_exif.length = PSEUDO_EXIF_PACKAGE_LENGTH;
- } else {
- new_exif.length = exif->length + EXIF_J_R_ENTRY_LENGTH;
- }
- new_exif.data = new uint8_t[new_exif.length];
- std::unique_ptr<uint8_t[]> new_exif_data;
- new_exif_data.reset(reinterpret_cast<uint8_t*>(new_exif.data));
- JPEGR_CHECK(updateExif(exif, &new_exif));
-
- JPEGR_CHECK(appendRecoveryMap(&jpeg, &compressed_map, &new_exif, &metadata, dest));
+ JPEGR_CHECK(appendRecoveryMap(&jpeg, &compressed_map, exif, &metadata, dest));
return NO_ERROR;
}
@@ -250,19 +239,7 @@
jpeg.data = jpeg_encoder.getCompressedImagePtr();
jpeg.length = jpeg_encoder.getCompressedImageSize();
- jpegr_exif_struct new_exif;
- if (exif == nullptr || exif->data == nullptr) {
- new_exif.length = PSEUDO_EXIF_PACKAGE_LENGTH;
- } else {
- new_exif.length = exif->length + EXIF_J_R_ENTRY_LENGTH;
- }
-
- new_exif.data = new uint8_t[new_exif.length];
- std::unique_ptr<uint8_t[]> new_exif_data;
- new_exif_data.reset(reinterpret_cast<uint8_t*>(new_exif.data));
- JPEGR_CHECK(updateExif(exif, &new_exif));
-
- JPEGR_CHECK(appendRecoveryMap(&jpeg, &compressed_map, &new_exif, &metadata, dest));
+ JPEGR_CHECK(appendRecoveryMap(&jpeg, &compressed_map, exif, &metadata, dest));
return NO_ERROR;
}
@@ -311,47 +288,7 @@
compressed_map.data = compressed_map_data.get();
JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
- // Extract EXIF from JPEG without decoding.
- JpegDecoder jpeg_decoder;
- if (!jpeg_decoder.extractEXIF(compressed_jpeg_image->data, compressed_jpeg_image->length)) {
- return ERROR_JPEGR_DECODE_ERROR;
- }
-
- // Update exif.
- jpegr_exif_struct exif;
- exif.data = nullptr;
- exif.length = 0;
- jpegr_compressed_struct new_jpeg_image;
- new_jpeg_image.data = nullptr;
- new_jpeg_image.length = 0;
- if (jpeg_decoder.getEXIFPos() != 0) {
- copyJpegWithoutExif(&new_jpeg_image,
- compressed_jpeg_image,
- jpeg_decoder.getEXIFPos(),
- jpeg_decoder.getEXIFSize());
- exif.data = jpeg_decoder.getEXIFPtr();
- exif.length = jpeg_decoder.getEXIFSize();
- }
-
- jpegr_exif_struct new_exif;
- if (exif.data == nullptr) {
- new_exif.length = PSEUDO_EXIF_PACKAGE_LENGTH;
- } else {
- new_exif.length = exif.length + EXIF_J_R_ENTRY_LENGTH;
- }
-
- new_exif.data = new uint8_t[new_exif.length];
- std::unique_ptr<uint8_t[]> new_exif_data;
- new_exif_data.reset(reinterpret_cast<uint8_t*>(new_exif.data));
- JPEGR_CHECK(updateExif(&exif, &new_exif));
-
- JPEGR_CHECK(appendRecoveryMap(
- new_jpeg_image.data == nullptr ? compressed_jpeg_image : &new_jpeg_image,
- &compressed_map, &new_exif, &metadata, dest));
-
- if (new_jpeg_image.data != nullptr) {
- free(new_jpeg_image.data);
- }
+ JPEGR_CHECK(appendRecoveryMap(compressed_jpeg_image, &compressed_map, nullptr, &metadata, dest));
return NO_ERROR;
}
@@ -384,33 +321,6 @@
uncompressed_yuv_420_image.height = jpeg_decoder.getDecompressedImageHeight();
uncompressed_yuv_420_image.colorGamut = compressed_jpeg_image->colorGamut;
- // Update exif.
- jpegr_exif_struct exif;
- exif.data = nullptr;
- exif.length = 0;
- jpegr_compressed_struct new_jpeg_image;
- new_jpeg_image.data = nullptr;
- new_jpeg_image.length = 0;
- if (jpeg_decoder.getEXIFPos() != 0) {
- copyJpegWithoutExif(&new_jpeg_image,
- compressed_jpeg_image,
- jpeg_decoder.getEXIFPos(),
- jpeg_decoder.getEXIFSize());
- exif.data = jpeg_decoder.getEXIFPtr();
- exif.length = jpeg_decoder.getEXIFSize();
- }
-
- jpegr_exif_struct new_exif;
- if (exif.data == nullptr) {
- new_exif.length = PSEUDO_EXIF_PACKAGE_LENGTH;
- } else {
- new_exif.length = exif.length + EXIF_J_R_ENTRY_LENGTH;
- }
- new_exif.data = new uint8_t[new_exif.length];
- std::unique_ptr<uint8_t[]> new_exif_data;
- new_exif_data.reset(reinterpret_cast<uint8_t*>(new_exif.data));
- JPEGR_CHECK(updateExif(&exif, &new_exif));
-
if (uncompressed_p010_image->width != uncompressed_yuv_420_image.width
|| uncompressed_p010_image->height != uncompressed_yuv_420_image.height) {
return ERROR_JPEGR_RESOLUTION_MISMATCH;
@@ -435,13 +345,7 @@
compressed_map.data = compressed_map_data.get();
JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
- JPEGR_CHECK(appendRecoveryMap(
- new_jpeg_image.data == nullptr ? compressed_jpeg_image : &new_jpeg_image,
- &compressed_map, &new_exif, &metadata, dest));
-
- if (new_jpeg_image.data != nullptr) {
- free(new_jpeg_image.data);
- }
+ JPEGR_CHECK(appendRecoveryMap(compressed_jpeg_image, &compressed_map, nullptr, &metadata, dest));
return NO_ERROR;
}
@@ -967,15 +871,20 @@
// JPEG/R structure:
// SOI (ff d8)
+//
+// (Optional, only if EXIF package is from outside)
// APP1 (ff e1)
// 2 bytes of length (2 + length of exif package)
// EXIF package (this includes the first two bytes representing the package length)
-// APP1 (ff e1)
+//
+// (Required, XMP package) APP1 (ff e1)
// 2 bytes of length (2 + 29 + length of xmp package)
// name space ("http://ns.adobe.com/xap/1.0/\0")
// xmp
-// primary image (without the first two bytes (SOI) and without EXIF, may have other packages)
-// secondary image (the recovery map)
+//
+// (Required) primary image (without the first two bytes (SOI), may have other packages)
+//
+// (Required) secondary image (the recovery map)
//
// Metadata versions we are using:
// ECMA TR-98 for JFIF marker
@@ -989,7 +898,6 @@
jr_compressed_ptr dest) {
if (compressed_jpeg_image == nullptr
|| compressed_recovery_map == nullptr
- || exif == nullptr
|| metadata == nullptr
|| dest == nullptr) {
return ERROR_JPEGR_INVALID_NULL_PTR;
@@ -1002,7 +910,7 @@
JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kSOI, 1, pos));
// Write EXIF
- {
+ if (exif != nullptr) {
const int length = 2 + exif->length;
const uint8_t lengthH = ((length >> 8) & 0xff);
const uint8_t lengthL = (length & 0xff);
diff --git a/libs/jpegrecoverymap/recoverymaputils.cpp b/libs/jpegrecoverymap/recoverymaputils.cpp
index d5ad9a5..1617b8b 100644
--- a/libs/jpegrecoverymap/recoverymaputils.cpp
+++ b/libs/jpegrecoverymap/recoverymaputils.cpp
@@ -22,8 +22,6 @@
#include <image_io/xml/xml_handler.h>
#include <image_io/xml/xml_rule.h>
-#include <utils/Log.h>
-
using namespace photos_editing_formats::image_io;
using namespace std;
@@ -55,12 +53,6 @@
return NO_ERROR;
}
-status_t Write(jr_exif_ptr destination, const void* source, size_t length, int &position) {
- memcpy((uint8_t*)destination->data + sizeof(uint8_t) * position, source, length);
- position += length;
- return NO_ERROR;
-}
-
// Extremely simple XML Handler - just searches for interesting elements
class XMPXmlHandler : public XmlHandler {
public:
@@ -344,185 +336,4 @@
return ss.str();
}
-/*
- * Helper function
- * Add J R entry to existing exif, or create a new one with J R entry if it's null.
- */
-status_t updateExif(jr_exif_ptr exif, jr_exif_ptr dest) {
- if (exif == nullptr || exif->data == nullptr) {
- uint8_t data[PSEUDO_EXIF_PACKAGE_LENGTH] = {
- 0x45, 0x78, 0x69, 0x66, 0x00, 0x00,
- 0x49, 0x49, 0x2A, 0x00,
- 0x08, 0x00, 0x00, 0x00,
- 0x01, 0x00,
- 0x4A, 0x52,
- 0x07, 0x00,
- 0x01, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00};
- int pos = 0;
- Write(dest, data, PSEUDO_EXIF_PACKAGE_LENGTH, pos);
- return NO_ERROR;
- }
-
- int num_entry = 0;
- uint8_t num_entry_low = 0;
- uint8_t num_entry_high = 0;
- bool use_big_endian = false;
- if (reinterpret_cast<uint16_t*>(exif->data)[3] == 0x4949) {
- num_entry_low = reinterpret_cast<uint8_t*>(exif->data)[14];
- num_entry_high = reinterpret_cast<uint8_t*>(exif->data)[15];
- } else if (reinterpret_cast<uint16_t*>(exif->data)[3] == 0x4d4d) {
- use_big_endian = true;
- num_entry_high = reinterpret_cast<uint8_t*>(exif->data)[14];
- num_entry_low = reinterpret_cast<uint8_t*>(exif->data)[15];
- } else {
- return ERROR_JPEGR_METADATA_ERROR;
- }
- num_entry = (num_entry_high << 8) | num_entry_low;
- num_entry += 1;
- num_entry_low = num_entry & 0xff;
- num_entry_high = (num_entry >> 8) & 0xff;
-
- int pos = 0;
- Write(dest, (uint8_t*)exif->data, 14, pos);
-
- if (use_big_endian) {
- Write(dest, &num_entry_high, 1, pos);
- Write(dest, &num_entry_low, 1, pos);
- uint8_t data[EXIF_J_R_ENTRY_LENGTH] = {
- 0x4A, 0x52,
- 0x00, 0x07,
- 0x00, 0x00, 0x00, 0x01,
- 0x00, 0x00, 0x00, 0x00};
- Write(dest, data, EXIF_J_R_ENTRY_LENGTH, pos);
- } else {
- Write(dest, &num_entry_low, 1, pos);
- Write(dest, &num_entry_high, 1, pos);
- uint8_t data[EXIF_J_R_ENTRY_LENGTH] = {
- 0x4A, 0x52,
- 0x07, 0x00,
- 0x01, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00};
- Write(dest, data, EXIF_J_R_ENTRY_LENGTH, pos);
- }
-
- Write(dest, (uint8_t*)exif->data + 16, exif->length - 16, pos);
-
- updateExifOffsets(dest,
- 28, // start from the second tag, skip the "JR" tag
- num_entry - 1,
- use_big_endian);
-
- return NO_ERROR;
-}
-
-/*
- * Helper function
- * Modify offsets in EXIF in place.
- */
-void updateExifOffsets(jr_exif_ptr exif, int pos, bool use_big_endian) {
- int num_entry = readValue(reinterpret_cast<uint8_t*>(exif->data), pos, 2, use_big_endian);
- updateExifOffsets(exif, pos + 2, num_entry, use_big_endian);
-}
-
-void updateExifOffsets(jr_exif_ptr exif, int pos, int num_entry, bool use_big_endian) {
- for (int i = 0; i < num_entry; pos += EXIF_J_R_ENTRY_LENGTH, i++) {
- int tag = readValue(reinterpret_cast<uint8_t*>(exif->data), pos, 2, use_big_endian);
- bool need_to_update_offset = false;
- if (tag == 0x8769) {
- need_to_update_offset = true;
- int sub_ifd_offset =
- readValue(reinterpret_cast<uint8_t*>(exif->data), pos + 8, 4, use_big_endian)
- + 6 // "Exif\0\0";
- + EXIF_J_R_ENTRY_LENGTH;
- updateExifOffsets(exif, sub_ifd_offset, use_big_endian);
- } else {
- int data_format =
- readValue(reinterpret_cast<uint8_t*>(exif->data), pos + 2, 2, use_big_endian);
- int num_of_components =
- readValue(reinterpret_cast<uint8_t*>(exif->data), pos + 4, 4, use_big_endian);
- int data_length = findFormatLengthInBytes(data_format) * num_of_components;
- if (data_length > 4) {
- need_to_update_offset = true;
- }
- }
-
- if (!need_to_update_offset) {
- continue;
- }
-
- int offset = readValue(reinterpret_cast<uint8_t*>(exif->data), pos + 8, 4, use_big_endian);
-
- offset += EXIF_J_R_ENTRY_LENGTH;
-
- if (use_big_endian) {
- reinterpret_cast<uint8_t*>(exif->data)[pos + 11] = offset & 0xff;
- reinterpret_cast<uint8_t*>(exif->data)[pos + 10] = (offset >> 8) & 0xff;
- reinterpret_cast<uint8_t*>(exif->data)[pos + 9] = (offset >> 16) & 0xff;
- reinterpret_cast<uint8_t*>(exif->data)[pos + 8] = (offset >> 24) & 0xff;
- } else {
- reinterpret_cast<uint8_t*>(exif->data)[pos + 8] = offset & 0xff;
- reinterpret_cast<uint8_t*>(exif->data)[pos + 9] = (offset >> 8) & 0xff;
- reinterpret_cast<uint8_t*>(exif->data)[pos + 10] = (offset >> 16) & 0xff;
- reinterpret_cast<uint8_t*>(exif->data)[pos + 11] = (offset >> 24) & 0xff;
- }
- }
-}
-
-/*
- * Read data from the target position and target length in bytes;
- */
-int readValue(uint8_t* data, int pos, int length, bool use_big_endian) {
- if (length == 2) {
- if (use_big_endian) {
- return (data[pos] << 8) | data[pos + 1];
- } else {
- return (data[pos + 1] << 8) | data[pos];
- }
- } else if (length == 4) {
- if (use_big_endian) {
- return (data[pos] << 24) | (data[pos + 1] << 16) | (data[pos + 2] << 8) | data[pos + 3];
- } else {
- return (data[pos + 3] << 24) | (data[pos + 2] << 16) | (data[pos + 1] << 8) | data[pos];
- }
- } else {
- // Not support for now.
- ALOGE("Error in readValue(): pos=%d, length=%d", pos, length);
- return -1;
- }
-}
-
-/*
- * Helper function
- * Returns the length of data format in bytes
- */
-int findFormatLengthInBytes(int data_format) {
- switch (data_format) {
- case 1: // unsigned byte
- case 2: // ascii strings
- case 6: // signed byte
- case 7: // undefined
- return 1;
-
- case 3: // unsigned short
- case 8: // signed short
- return 2;
-
- case 4: // unsigned long
- case 9: // signed long
- case 11: // single float
- return 4;
-
- case 5: // unsigned rational
- case 10: // signed rational
- case 12: // double float
- return 8;
-
- default:
- // should not hit here
- ALOGE("Error in findFormatLengthInBytes(): data_format=%d", data_format);
- return -1;
- }
-}
-
-} // namespace android::recoverymap
\ No newline at end of file
+} // namespace android::recoverymap
diff --git a/services/inputflinger/InputReaderBase.cpp b/services/inputflinger/InputReaderBase.cpp
index a864cf8..2450235 100644
--- a/services/inputflinger/InputReaderBase.cpp
+++ b/services/inputflinger/InputReaderBase.cpp
@@ -73,6 +73,9 @@
if (changes & CHANGE_ENABLED_STATE) {
result += "ENABLED_STATE | ";
}
+ if (changes & CHANGE_TOUCHPAD_SETTINGS) {
+ result += "TOUCHPAD_SETTINGS | ";
+ }
if (changes & CHANGE_MUST_REOPEN) {
result += "MUST_REOPEN | ";
}
diff --git a/services/inputflinger/include/InputReaderBase.h b/services/inputflinger/include/InputReaderBase.h
index d55ab28..2173117 100644
--- a/services/inputflinger/include/InputReaderBase.h
+++ b/services/inputflinger/include/InputReaderBase.h
@@ -161,7 +161,7 @@
struct InputReaderConfiguration {
// Describes changes that have occurred.
enum {
- // The pointer speed changed.
+ // The mouse pointer speed changed.
CHANGE_POINTER_SPEED = 1 << 0,
// The pointer gesture control changed.
@@ -200,6 +200,9 @@
// The stylus button reporting configurations has changed.
CHANGE_STYLUS_BUTTON_REPORTING = 1 << 12,
+ // The touchpad settings changed.
+ CHANGE_TOUCHPAD_SETTINGS = 1 << 13,
+
// All devices must be reopened.
CHANGE_MUST_REOPEN = 1 << 31,
};
@@ -309,6 +312,20 @@
// The latest request to enable or disable Pointer Capture.
PointerCaptureRequest pointerCaptureRequest;
+ // The touchpad pointer speed, as a number from -7 (slowest) to 7 (fastest).
+ int32_t touchpadPointerSpeed;
+
+ // True to invert the touchpad scrolling direction, so that moving two fingers downwards on the
+ // touchpad scrolls the content upwards.
+ bool touchpadNaturalScrollingEnabled;
+
+ // True to enable tap-to-click on touchpads.
+ bool touchpadTapToClickEnabled;
+
+ // True to enable a zone on the right-hand side of touchpads where clicks will be turned into
+ // context (a.k.a. "right") clicks.
+ bool touchpadRightClickZoneEnabled;
+
// The set of currently disabled input devices.
std::set<int32_t> disabledDevices;
@@ -337,6 +354,10 @@
pointerGestureZoomSpeedRatio(0.3f),
showTouches(false),
pointerCaptureRequest(),
+ touchpadPointerSpeed(0),
+ touchpadNaturalScrollingEnabled(true),
+ touchpadTapToClickEnabled(true),
+ touchpadRightClickZoneEnabled(false),
stylusButtonMotionEventsEnabled(true) {}
static std::string changesToString(uint32_t changes);
diff --git a/services/inputflinger/reader/mapper/TouchpadInputMapper.cpp b/services/inputflinger/reader/mapper/TouchpadInputMapper.cpp
index b6313a1..9f32311 100644
--- a/services/inputflinger/reader/mapper/TouchpadInputMapper.cpp
+++ b/services/inputflinger/reader/mapper/TouchpadInputMapper.cpp
@@ -146,6 +146,18 @@
}
mGestureConverter.setOrientation(orientation);
}
+ if (!changes || (changes & InputReaderConfiguration::CHANGE_TOUCHPAD_SETTINGS)) {
+ // TODO(b/265798483): load an Android-specific acceleration curve instead of mapping to one
+ // of five ChromeOS curves.
+ const int pointerSensitivity = (config->touchpadPointerSpeed + 7) / 3 + 1;
+ mPropertyProvider.getProperty("Pointer Sensitivity").setIntValues({pointerSensitivity});
+ mPropertyProvider.getProperty("Invert Scrolling")
+ .setBoolValues({config->touchpadNaturalScrollingEnabled});
+ mPropertyProvider.getProperty("Tap Enable")
+ .setBoolValues({config->touchpadTapToClickEnabled});
+ mPropertyProvider.getProperty("Button Right Click Zone Enable")
+ .setBoolValues({config->touchpadRightClickZoneEnabled});
+ }
return {};
}
diff --git a/services/inputflinger/reader/mapper/gestures/GestureConverter.cpp b/services/inputflinger/reader/mapper/gestures/GestureConverter.cpp
index 561b1f8..d636d44 100644
--- a/services/inputflinger/reader/mapper/gestures/GestureConverter.cpp
+++ b/services/inputflinger/reader/mapper/gestures/GestureConverter.cpp
@@ -219,11 +219,11 @@
float deltaY = gesture.details.scroll.dy;
rotateDelta(mOrientation, &deltaX, &deltaY);
- coords.setAxisValue(AMOTION_EVENT_AXIS_X, coords.getAxisValue(AMOTION_EVENT_AXIS_X) - deltaX);
- coords.setAxisValue(AMOTION_EVENT_AXIS_Y, coords.getAxisValue(AMOTION_EVENT_AXIS_Y) - deltaY);
+ coords.setAxisValue(AMOTION_EVENT_AXIS_X, coords.getAxisValue(AMOTION_EVENT_AXIS_X) + deltaX);
+ coords.setAxisValue(AMOTION_EVENT_AXIS_Y, coords.getAxisValue(AMOTION_EVENT_AXIS_Y) + deltaY);
// TODO(b/262876643): set AXIS_GESTURE_{X,Y}_OFFSET.
- coords.setAxisValue(AMOTION_EVENT_AXIS_GESTURE_SCROLL_X_DISTANCE, gesture.details.scroll.dx);
- coords.setAxisValue(AMOTION_EVENT_AXIS_GESTURE_SCROLL_Y_DISTANCE, gesture.details.scroll.dy);
+ coords.setAxisValue(AMOTION_EVENT_AXIS_GESTURE_SCROLL_X_DISTANCE, -gesture.details.scroll.dx);
+ coords.setAxisValue(AMOTION_EVENT_AXIS_GESTURE_SCROLL_Y_DISTANCE, -gesture.details.scroll.dy);
out.push_back(makeMotionArgs(when, readTime, AMOTION_EVENT_ACTION_MOVE, /* actionButton= */ 0,
mButtonState, /* pointerCount= */ 1, mFingerProps.data(),
mFakeFingerCoords.data(), xCursorPosition, yCursorPosition));
diff --git a/services/inputflinger/tests/GestureConverter_test.cpp b/services/inputflinger/tests/GestureConverter_test.cpp
index 36a39bb..9c624ba 100644
--- a/services/inputflinger/tests/GestureConverter_test.cpp
+++ b/services/inputflinger/tests/GestureConverter_test.cpp
@@ -244,7 +244,7 @@
InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID);
GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID);
- Gesture startGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, 10);
+ Gesture startGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, -10);
std::list<NotifyArgs> args = converter.handleGesture(downTime, READ_TIME, startGesture);
ASSERT_EQ(2u, args.size());
@@ -261,7 +261,7 @@
WithMotionClassification(MotionClassification::TWO_FINGER_SWIPE),
WithToolType(AMOTION_EVENT_TOOL_TYPE_FINGER)));
- Gesture continueGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, 5);
+ Gesture continueGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, -5);
args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, continueGesture);
ASSERT_EQ(1u, args.size());
ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()),
@@ -289,7 +289,7 @@
GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID);
converter.setOrientation(ui::ROTATION_90);
- Gesture startGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, 10);
+ Gesture startGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, -10);
std::list<NotifyArgs> args = converter.handleGesture(downTime, READ_TIME, startGesture);
ASSERT_EQ(2u, args.size());
@@ -306,7 +306,7 @@
WithMotionClassification(MotionClassification::TWO_FINGER_SWIPE),
WithToolType(AMOTION_EVENT_TOOL_TYPE_FINGER)));
- Gesture continueGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, 5);
+ Gesture continueGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, -5);
args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, continueGesture);
ASSERT_EQ(1u, args.size());
ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()),
@@ -332,10 +332,10 @@
InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID);
GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID);
- Gesture startGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, 10);
+ Gesture startGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, -10);
std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, startGesture);
- Gesture continueGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, 5);
+ Gesture continueGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, -5);
args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, continueGesture);
Gesture flingGesture(kGestureFling, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 1, 1,
diff --git a/services/surfaceflinger/FrameTimeline/FrameTimeline.cpp b/services/surfaceflinger/FrameTimeline/FrameTimeline.cpp
index 27a099c..925f111 100644
--- a/services/surfaceflinger/FrameTimeline/FrameTimeline.cpp
+++ b/services/surfaceflinger/FrameTimeline/FrameTimeline.cpp
@@ -1095,6 +1095,12 @@
}
void FrameTimeline::DisplayFrame::trace(pid_t surfaceFlingerPid, nsecs_t monoBootOffset) const {
+ if (mSurfaceFrames.empty()) {
+ // We don't want to trace display frames without any surface frames updates as this cannot
+ // be janky
+ return;
+ }
+
if (mToken == FrameTimelineInfo::INVALID_VSYNC_ID) {
// DisplayFrame should not have an invalid token.
ALOGE("Cannot trace DisplayFrame with invalid token");
diff --git a/services/surfaceflinger/FrontEnd/LayerCreationArgs.h b/services/surfaceflinger/FrontEnd/LayerCreationArgs.h
index 7b5a157..9d2aaab 100644
--- a/services/surfaceflinger/FrontEnd/LayerCreationArgs.h
+++ b/services/surfaceflinger/FrontEnd/LayerCreationArgs.h
@@ -18,10 +18,12 @@
#include <binder/Binder.h>
#include <gui/LayerMetadata.h>
+#include <ui/LayerStack.h>
#include <utils/StrongPointer.h>
#include <cstdint>
#include <limits>
#include <optional>
+
constexpr uint32_t UNASSIGNED_LAYER_ID = std::numeric_limits<uint32_t>::max();
namespace android {
@@ -51,6 +53,7 @@
bool addToRoot = true;
wp<IBinder> parentHandle = nullptr;
wp<IBinder> mirrorLayerHandle = nullptr;
+ ui::LayerStack layerStackToMirror = ui::INVALID_LAYER_STACK;
};
} // namespace android::surfaceflinger
diff --git a/services/surfaceflinger/FrontEnd/LayerHierarchy.cpp b/services/surfaceflinger/FrontEnd/LayerHierarchy.cpp
index 678d36b..a4fac1c 100644
--- a/services/surfaceflinger/FrontEnd/LayerHierarchy.cpp
+++ b/services/surfaceflinger/FrontEnd/LayerHierarchy.cpp
@@ -252,8 +252,8 @@
attachToParent(hierarchy);
attachToRelativeParent(hierarchy);
- if (layer->mirrorId != UNASSIGNED_LAYER_ID) {
- LayerHierarchy* mirror = getHierarchyFromId(layer->mirrorId);
+ for (uint32_t mirrorId : layer->mirrorIds) {
+ LayerHierarchy* mirror = getHierarchyFromId(mirrorId);
hierarchy->addChild(mirror, LayerHierarchy::Variant::Mirror);
}
}
@@ -292,14 +292,14 @@
auto it = hierarchy->mChildren.begin();
while (it != hierarchy->mChildren.end()) {
if (it->second == LayerHierarchy::Variant::Mirror) {
- hierarchy->mChildren.erase(it);
- break;
+ it = hierarchy->mChildren.erase(it);
+ } else {
+ it++;
}
- it++;
}
- if (layer->mirrorId != UNASSIGNED_LAYER_ID) {
- hierarchy->addChild(getHierarchyFromId(layer->mirrorId), LayerHierarchy::Variant::Mirror);
+ for (uint32_t mirrorId : layer->mirrorIds) {
+ hierarchy->addChild(getHierarchyFromId(mirrorId), LayerHierarchy::Variant::Mirror);
}
}
diff --git a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp
index 5514c06..547a852 100644
--- a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp
+++ b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp
@@ -44,9 +44,28 @@
layer.parentId = linkLayer(layer.parentId, layer.id);
layer.relativeParentId = linkLayer(layer.relativeParentId, layer.id);
- layer.mirrorId = linkLayer(layer.mirrorId, layer.id);
+ if (layer.layerStackToMirror != ui::INVALID_LAYER_STACK) {
+ // if this layer is mirroring a display, then walk though all the existing root layers
+ // for the layer stack and add them as children to be mirrored.
+ mDisplayMirroringLayers.emplace_back(layer.id);
+ for (auto& rootLayer : mLayers) {
+ if (rootLayer->isRoot() && rootLayer->layerStack == layer.layerStackToMirror) {
+ layer.mirrorIds.emplace_back(rootLayer->id);
+ linkLayer(rootLayer->id, layer.id);
+ }
+ }
+ } else {
+ // Check if we are mirroring a single layer, and if so add it to the list of children
+ // to be mirrored.
+ layer.layerIdToMirror = linkLayer(layer.layerIdToMirror, layer.id);
+ if (layer.layerIdToMirror != UNASSIGNED_LAYER_ID) {
+ layer.mirrorIds.emplace_back(layer.layerIdToMirror);
+ }
+ }
layer.touchCropId = linkLayer(layer.touchCropId, layer.id);
-
+ if (layer.isRoot()) {
+ updateDisplayMirrorLayers(layer);
+ }
mLayers.emplace_back(std::move(newLayer));
}
}
@@ -85,7 +104,14 @@
layer.parentId = unlinkLayer(layer.parentId, layer.id);
layer.relativeParentId = unlinkLayer(layer.relativeParentId, layer.id);
- layer.mirrorId = unlinkLayer(layer.mirrorId, layer.id);
+ if (layer.layerStackToMirror != ui::INVALID_LAYER_STACK) {
+ layer.mirrorIds = unlinkLayers(layer.mirrorIds, layer.id);
+ swapErase(mDisplayMirroringLayers, layer.id);
+ } else {
+ layer.layerIdToMirror = unlinkLayer(layer.layerIdToMirror, layer.id);
+ layer.mirrorIds.clear();
+ }
+
layer.touchCropId = unlinkLayer(layer.touchCropId, layer.id);
auto& references = it->second.references;
@@ -106,8 +132,8 @@
if (linkedLayer->relativeParentId == layer.id) {
linkedLayer->relativeParentId = UNASSIGNED_LAYER_ID;
}
- if (linkedLayer->mirrorId == layer.id) {
- linkedLayer->mirrorId = UNASSIGNED_LAYER_ID;
+ if (swapErase(linkedLayer->mirrorIds, layer.id)) {
+ linkedLayer->changes |= RequestedLayerState::Changes::Mirror;
}
if (linkedLayer->touchCropId == layer.id) {
linkedLayer->touchCropId = UNASSIGNED_LAYER_ID;
@@ -200,6 +226,12 @@
if (oldParentId != layer->parentId) {
unlinkLayer(oldParentId, layer->id);
layer->parentId = linkLayer(layer->parentId, layer->id);
+ if (oldParentId == UNASSIGNED_LAYER_ID) {
+ updateDisplayMirrorLayers(*layer);
+ }
+ }
+ if (layer->what & layer_state_t::eLayerStackChanged && layer->isRoot()) {
+ updateDisplayMirrorLayers(*layer);
}
if (oldRelativeParentId != layer->relativeParentId) {
unlinkLayer(oldRelativeParentId, layer->id);
@@ -308,6 +340,14 @@
return UNASSIGNED_LAYER_ID;
}
+std::vector<uint32_t> LayerLifecycleManager::unlinkLayers(const std::vector<uint32_t>& layerIds,
+ uint32_t linkedLayer) {
+ for (uint32_t layerId : layerIds) {
+ unlinkLayer(layerId, linkedLayer);
+ }
+ return {};
+}
+
std::string LayerLifecycleManager::References::getDebugString() const {
std::string debugInfo = owner.name + "[" + std::to_string(owner.id) + "] refs:";
std::for_each(references.begin(), references.end(),
@@ -329,4 +369,30 @@
mGlobalChanges |= RequestedLayerState::Changes::Hierarchy;
}
+// Some layers mirror the entire display stack. Since we don't have a single root layer per display
+// we have to track all these layers and update what they mirror when the list of root layers
+// on a display changes. This function walks through the list of display mirroring layers
+// and updates its list of layers that its mirroring. This function should be called when a new
+// root layer is added, removed or moved to another display.
+void LayerLifecycleManager::updateDisplayMirrorLayers(RequestedLayerState& rootLayer) {
+ for (uint32_t mirrorLayerId : mDisplayMirroringLayers) {
+ RequestedLayerState* mirrorLayer = getLayerFromId(mirrorLayerId);
+ bool canBeMirrored =
+ rootLayer.isRoot() && rootLayer.layerStack == mirrorLayer->layerStackToMirror;
+ bool currentlyMirrored =
+ std::find(mirrorLayer->mirrorIds.begin(), mirrorLayer->mirrorIds.end(),
+ rootLayer.id) != mirrorLayer->mirrorIds.end();
+
+ if (canBeMirrored && !currentlyMirrored) {
+ mirrorLayer->mirrorIds.emplace_back(rootLayer.id);
+ linkLayer(rootLayer.id, mirrorLayer->id);
+ mirrorLayer->changes |= RequestedLayerState::Changes::Mirror;
+ } else if (!canBeMirrored && currentlyMirrored) {
+ swapErase(mirrorLayer->mirrorIds, rootLayer.id);
+ unlinkLayer(rootLayer.id, mirrorLayer->id);
+ mirrorLayer->changes |= RequestedLayerState::Changes::Mirror;
+ }
+ }
+}
+
} // namespace android::surfaceflinger::frontend
diff --git a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.h b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.h
index 63a7afc..25d27ee 100644
--- a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.h
+++ b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.h
@@ -80,6 +80,9 @@
std::vector<uint32_t>* getLinkedLayersFromId(uint32_t);
uint32_t linkLayer(uint32_t layerId, uint32_t layerToLink);
uint32_t unlinkLayer(uint32_t layerId, uint32_t linkedLayer);
+ std::vector<uint32_t> unlinkLayers(const std::vector<uint32_t>& layerIds, uint32_t linkedLayer);
+
+ void updateDisplayMirrorLayers(RequestedLayerState& rootLayer);
struct References {
// Lifetime tied to mLayers
@@ -90,6 +93,8 @@
std::unordered_map<uint32_t, References> mIdToLayer;
// Listeners are invoked once changes are committed.
std::vector<std::shared_ptr<ILifecycleListener>> mListeners;
+ // Layers that mirror a display stack (see updateDisplayMirrorLayers)
+ std::vector<uint32_t> mDisplayMirroringLayers;
// Aggregation of changes since last commit.
ftl::Flags<RequestedLayerState::Changes> mGlobalChanges;
diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshot.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshot.cpp
index 3a0540c..4e69565 100644
--- a/services/surfaceflinger/FrontEnd/LayerSnapshot.cpp
+++ b/services/surfaceflinger/FrontEnd/LayerSnapshot.cpp
@@ -112,6 +112,10 @@
}
bool LayerSnapshot::getIsVisible() const {
+ if (handleSkipScreenshotFlag & outputFilter.toInternalDisplay) {
+ return false;
+ }
+
if (!hasSomethingToDraw()) {
return false;
}
@@ -125,6 +129,7 @@
std::string LayerSnapshot::getIsVisibleReason() const {
// not visible
+ if (handleSkipScreenshotFlag & outputFilter.toInternalDisplay) return "eLayerSkipScreenshot";
if (!hasSomethingToDraw()) return "!hasSomethingToDraw";
if (invalidTransform) return "invalidTransform";
if (isHiddenByPolicyFromParent) return "hidden by parent or layer flag";
diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshot.h b/services/surfaceflinger/FrontEnd/LayerSnapshot.h
index 4512ade..159410f 100644
--- a/services/surfaceflinger/FrontEnd/LayerSnapshot.h
+++ b/services/surfaceflinger/FrontEnd/LayerSnapshot.h
@@ -84,6 +84,7 @@
gui::GameMode gameMode;
scheduler::LayerInfo::FrameRate frameRate;
ui::Transform::RotationFlags fixedTransformHint;
+ bool handleSkipScreenshotFlag = false;
ChildState childState;
static bool isOpaqueFormat(PixelFormat format);
diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
index cc26591..d0ffe61 100644
--- a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
+++ b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
@@ -714,6 +714,10 @@
snapshot.fixedTransformHint = requested.fixedTransformHint != ui::Transform::ROT_INVALID
? requested.fixedTransformHint
: parentSnapshot.fixedTransformHint;
+ // Display mirrors are always placed in a VirtualDisplay so we never want to capture layers
+ // marked as skip capture
+ snapshot.handleSkipScreenshotFlag = parentSnapshot.handleSkipScreenshotFlag ||
+ (requested.layerStackToMirror != ui::INVALID_LAYER_STACK);
}
if (forceUpdate || requested.changes.get() != 0) {
diff --git a/services/surfaceflinger/FrontEnd/RequestedLayerState.cpp b/services/surfaceflinger/FrontEnd/RequestedLayerState.cpp
index b7fa4f0..d63b126 100644
--- a/services/surfaceflinger/FrontEnd/RequestedLayerState.cpp
+++ b/services/surfaceflinger/FrontEnd/RequestedLayerState.cpp
@@ -45,6 +45,16 @@
return layerId == UNASSIGNED_LAYER_ID ? "none" : std::to_string(layerId);
}
+std::string layerIdsToString(const std::vector<uint32_t>& layerIds) {
+ std::stringstream stream;
+ stream << "{";
+ for (auto layerId : layerIds) {
+ stream << layerId << ",";
+ }
+ stream << "}";
+ return stream.str();
+}
+
} // namespace
RequestedLayerState::RequestedLayerState(const LayerCreationArgs& args)
@@ -64,8 +74,11 @@
if (args.parentHandle != nullptr) {
canBeRoot = false;
}
- mirrorId = LayerHandle::getLayerId(args.mirrorLayerHandle.promote());
- if (mirrorId != UNASSIGNED_LAYER_ID) {
+ layerIdToMirror = LayerHandle::getLayerId(args.mirrorLayerHandle.promote());
+ if (layerIdToMirror != UNASSIGNED_LAYER_ID) {
+ changes |= RequestedLayerState::Changes::Mirror;
+ } else if (args.layerStackToMirror != ui::INVALID_LAYER_STACK) {
+ layerStackToMirror = args.layerStackToMirror;
changes |= RequestedLayerState::Changes::Mirror;
}
@@ -309,7 +322,7 @@
return "[" + std::to_string(id) + "]" + name + ",parent=" + layerIdToString(parentId) +
",relativeParent=" + layerIdToString(relativeParentId) +
",isRelativeOf=" + std::to_string(isRelativeOf) +
- ",mirrorId=" + layerIdToString(mirrorId) +
+ ",mirrorIds=" + layerIdsToString(mirrorIds) +
",handleAlive=" + std::to_string(handleAlive) + ",z=" + std::to_string(z);
}
diff --git a/services/surfaceflinger/FrontEnd/RequestedLayerState.h b/services/surfaceflinger/FrontEnd/RequestedLayerState.h
index 3a16531..6317b95 100644
--- a/services/surfaceflinger/FrontEnd/RequestedLayerState.h
+++ b/services/surfaceflinger/FrontEnd/RequestedLayerState.h
@@ -97,13 +97,15 @@
std::shared_ptr<renderengine::ExternalTexture> externalTexture;
gui::GameMode gameMode;
scheduler::LayerInfo::FrameRate requestedFrameRate;
+ ui::LayerStack layerStackToMirror = ui::INVALID_LAYER_STACK;
+ uint32_t layerIdToMirror = UNASSIGNED_LAYER_ID;
// book keeping states
bool handleAlive = true;
bool isRelativeOf = false;
uint32_t parentId = UNASSIGNED_LAYER_ID;
uint32_t relativeParentId = UNASSIGNED_LAYER_ID;
- uint32_t mirrorId = UNASSIGNED_LAYER_ID;
+ std::vector<uint32_t> mirrorIds{};
uint32_t touchCropId = UNASSIGNED_LAYER_ID;
uint32_t bgColorLayerId = UNASSIGNED_LAYER_ID;
ftl::Flags<RequestedLayerState::Changes> changes;
diff --git a/services/surfaceflinger/FrontEnd/SwapErase.h b/services/surfaceflinger/FrontEnd/SwapErase.h
index f672f99..0061c53 100644
--- a/services/surfaceflinger/FrontEnd/SwapErase.h
+++ b/services/surfaceflinger/FrontEnd/SwapErase.h
@@ -23,12 +23,15 @@
// remove an element from a vector that avoids relocating all the elements after the one
// that is erased.
template <typename T>
-void swapErase(std::vector<T>& vec, const T& value) {
+bool swapErase(std::vector<T>& vec, const T& value) {
+ bool found = false;
auto it = std::find(vec.begin(), vec.end(), value);
if (it != vec.end()) {
std::iter_swap(it, vec.end() - 1);
vec.erase(vec.end() - 1);
+ found = true;
}
+ return found;
}
// Similar to swapErase(std::vector<T>& vec, const T& value) but erases the first element
diff --git a/services/surfaceflinger/Scheduler/MessageQueue.cpp b/services/surfaceflinger/Scheduler/MessageQueue.cpp
index 9b04497..dec8f59 100644
--- a/services/surfaceflinger/Scheduler/MessageQueue.cpp
+++ b/services/surfaceflinger/Scheduler/MessageQueue.cpp
@@ -17,12 +17,12 @@
#define ATRACE_TAG ATRACE_TAG_GRAPHICS
#include <binder/IPCThreadState.h>
-
+#include <gui/DisplayEventReceiver.h>
#include <utils/Log.h>
#include <utils/Timers.h>
#include <utils/threads.h>
-#include <gui/DisplayEventReceiver.h>
+#include <scheduler/interface/ICompositor.h>
#include "EventThread.h"
#include "FrameTimeline.h"
diff --git a/services/surfaceflinger/Scheduler/MessageQueue.h b/services/surfaceflinger/Scheduler/MessageQueue.h
index ad0ea72..0d59337 100644
--- a/services/surfaceflinger/Scheduler/MessageQueue.h
+++ b/services/surfaceflinger/Scheduler/MessageQueue.h
@@ -37,15 +37,7 @@
namespace android {
-struct ICompositor {
- virtual void configure() = 0;
- virtual bool commit(TimePoint frameTime, VsyncId, TimePoint expectedVsyncTime) = 0;
- virtual void composite(TimePoint frameTime, VsyncId) = 0;
- virtual void sample() = 0;
-
-protected:
- ~ICompositor() = default;
-};
+struct ICompositor;
template <typename F>
class Task : public MessageHandler {
diff --git a/services/surfaceflinger/Scheduler/RefreshRateSelector.cpp b/services/surfaceflinger/Scheduler/RefreshRateSelector.cpp
index 1d27cfc..c5b3e14 100644
--- a/services/surfaceflinger/Scheduler/RefreshRateSelector.cpp
+++ b/services/surfaceflinger/Scheduler/RefreshRateSelector.cpp
@@ -929,17 +929,38 @@
RefreshRateOrder refreshRateOrder,
std::optional<DisplayModeId> preferredDisplayModeOpt) const
-> FrameRateRanking {
+ using fps_approx_ops::operator<;
const char* const whence = __func__;
+
+ // find the highest frame rate for each display mode
+ ftl::SmallMap<DisplayModeId, Fps, 8> maxRenderRateForMode;
+ const bool ascending = (refreshRateOrder == RefreshRateOrder::Ascending);
+ if (ascending) {
+ // TODO(b/266481656): Once this bug is fixed, we can remove this workaround and actually
+ // use a lower frame rate when we want Ascending frame rates.
+ for (const auto& frameRateMode : mPrimaryFrameRates) {
+ if (anchorGroupOpt && frameRateMode.modePtr->getGroup() != anchorGroupOpt) {
+ continue;
+ }
+
+ const auto [iter, _] = maxRenderRateForMode.try_emplace(frameRateMode.modePtr->getId(),
+ frameRateMode.fps);
+ if (iter->second < frameRateMode.fps) {
+ iter->second = frameRateMode.fps;
+ }
+ }
+ }
+
std::deque<ScoredFrameRate> ranking;
const auto rankFrameRate = [&](const FrameRateMode& frameRateMode) REQUIRES(mLock) {
- using fps_approx_ops::operator<;
const auto& modePtr = frameRateMode.modePtr;
if (anchorGroupOpt && modePtr->getGroup() != anchorGroupOpt) {
return;
}
const bool ascending = (refreshRateOrder == RefreshRateOrder::Ascending);
- if (ascending && frameRateMode.fps < getMinRefreshRateByPolicyLocked()->getFps()) {
+ const auto id = frameRateMode.modePtr->getId();
+ if (ascending && frameRateMode.fps < *maxRenderRateForMode.get(id)) {
// TODO(b/266481656): Once this bug is fixed, we can remove this workaround and actually
// use a lower frame rate when we want Ascending frame rates.
return;
diff --git a/services/surfaceflinger/Scheduler/Scheduler.cpp b/services/surfaceflinger/Scheduler/Scheduler.cpp
index 1fc1519..1e97f35 100644
--- a/services/surfaceflinger/Scheduler/Scheduler.cpp
+++ b/services/surfaceflinger/Scheduler/Scheduler.cpp
@@ -34,6 +34,8 @@
#include <utils/Trace.h>
#include <FrameTimeline/FrameTimeline.h>
+#include <scheduler/interface/ICompositor.h>
+
#include <algorithm>
#include <cinttypes>
#include <cstdint>
diff --git a/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositionCoverage.h b/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositionCoverage.h
new file mode 100644
index 0000000..3d0f1a9
--- /dev/null
+++ b/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositionCoverage.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+
+#include <ftl/flags.h>
+
+namespace android {
+
+// Whether composition was covered by HWC and/or GPU.
+enum class CompositionCoverage : std::uint8_t {
+ Hwc = 1 << 0,
+
+ // Mutually exclusive: The composition either used the GPU, or reused a buffer that had been
+ // composited on the GPU.
+ Gpu = 1 << 1,
+ GpuReuse = 1 << 2,
+};
+
+using CompositionCoverageFlags = ftl::Flags<CompositionCoverage>;
+
+} // namespace android
diff --git a/services/surfaceflinger/Scheduler/include/scheduler/interface/ICompositor.h b/services/surfaceflinger/Scheduler/include/scheduler/interface/ICompositor.h
new file mode 100644
index 0000000..cc41925
--- /dev/null
+++ b/services/surfaceflinger/Scheduler/include/scheduler/interface/ICompositor.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <scheduler/Time.h>
+#include <scheduler/VsyncId.h>
+
+namespace android {
+
+struct ICompositor {
+ // Configures physical displays, processing hotplug and/or mode setting via the Composer HAL.
+ virtual void configure() = 0;
+
+ // Commits transactions for layers and displays. Returns whether any state has been invalidated,
+ // i.e. whether a frame should be composited for each display.
+ virtual bool commit(TimePoint frameTime, VsyncId, TimePoint expectedVsyncTime) = 0;
+
+ // Composites a frame for each display. CompositionEngine performs GPU and/or HAL composition
+ // via RenderEngine and the Composer HAL, respectively.
+ virtual void composite(TimePoint frameTime, VsyncId) = 0;
+
+ // Samples the composited frame via RegionSamplingThread.
+ virtual void sample() = 0;
+
+protected:
+ ~ICompositor() = default;
+};
+
+} // namespace android
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index bdc57c9..0dc8b05 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -408,15 +408,8 @@
mDebugFlashDelay = base::GetUintProperty("debug.sf.showupdates"s, 0u);
- // DDMS debugging deprecated (b/120782499)
- property_get("debug.sf.ddms", value, "0");
- int debugDdms = atoi(value);
- ALOGI_IF(debugDdms, "DDMS debugging not supported");
-
- property_get("debug.sf.enable_gl_backpressure", value, "1");
- mPropagateBackpressureClientComposition = atoi(value);
- ALOGI_IF(mPropagateBackpressureClientComposition,
- "Enabling backpressure propagation for Client Composition");
+ mBackpressureGpuComposition = base::GetBoolProperty("debug.sf.enable_gl_backpressure"s, true);
+ ALOGI_IF(mBackpressureGpuComposition, "Enabling backpressure for GPU composition");
property_get("ro.surface_flinger.supports_background_blur", value, "0");
bool supportsBlurs = atoi(value);
@@ -2155,11 +2148,11 @@
const Period vsyncPeriod = mScheduler->getVsyncSchedule().period();
const FenceTimePtr& previousPresentFence = getPreviousPresentFence(frameTime, vsyncPeriod);
- // When Backpressure propagation is enabled we want to give a small grace period
+ // When backpressure propagation is enabled, we want to give a small grace period of 1ms
// for the present fence to fire instead of just giving up on this frame to handle cases
// where present fence is just about to get signaled.
- const int graceTimeForPresentFenceMs =
- (mPropagateBackpressureClientComposition || !mHadClientComposition) ? 1 : 0;
+ const int graceTimeForPresentFenceMs = static_cast<int>(
+ mBackpressureGpuComposition || !mCompositionCoverage.test(CompositionCoverage::Gpu));
// Pending frames may trigger backpressure propagation.
const TracedOrdinal<bool> framePending = {"PrevFramePending",
@@ -2182,9 +2175,14 @@
(lastScheduledPresentTime.ns() <
previousPresentTime - frameMissedSlop))};
const TracedOrdinal<bool> hwcFrameMissed = {"PrevHwcFrameMissed",
- mHadDeviceComposition && frameMissed};
+ frameMissed &&
+ mCompositionCoverage.test(
+ CompositionCoverage::Hwc)};
+
const TracedOrdinal<bool> gpuFrameMissed = {"PrevGpuFrameMissed",
- mHadClientComposition && frameMissed};
+ frameMissed &&
+ mCompositionCoverage.test(
+ CompositionCoverage::Gpu)};
if (frameMissed) {
mFrameMissedCount++;
@@ -2222,7 +2220,7 @@
}
if (framePending) {
- if ((hwcFrameMissed && !gpuFrameMissed) || mPropagateBackpressureClientComposition) {
+ if (mBackpressureGpuComposition || (hwcFrameMissed && !gpuFrameMissed)) {
scheduleCommit(FrameHint::kNone);
return false;
}
@@ -2459,29 +2457,43 @@
postComposition(presentTime);
- const bool prevFrameHadClientComposition = mHadClientComposition;
+ const bool hadGpuComposited = mCompositionCoverage.test(CompositionCoverage::Gpu);
+ mCompositionCoverage.clear();
- mHadClientComposition = mHadDeviceComposition = mReusedClientComposition = false;
TimeStats::ClientCompositionRecord clientCompositionRecord;
for (const auto& [_, display] : displays) {
const auto& state = display->getCompositionDisplay()->getState();
- mHadClientComposition |= state.usesClientComposition && !state.reusedClientComposition;
- mHadDeviceComposition |= state.usesDeviceComposition;
- mReusedClientComposition |= state.reusedClientComposition;
+
+ if (state.usesDeviceComposition) {
+ mCompositionCoverage |= CompositionCoverage::Hwc;
+ }
+
+ if (state.reusedClientComposition) {
+ mCompositionCoverage |= CompositionCoverage::GpuReuse;
+ } else if (state.usesClientComposition) {
+ mCompositionCoverage |= CompositionCoverage::Gpu;
+ }
+
clientCompositionRecord.predicted |=
(state.strategyPrediction != CompositionStrategyPredictionState::DISABLED);
clientCompositionRecord.predictionSucceeded |=
(state.strategyPrediction == CompositionStrategyPredictionState::SUCCESS);
}
- clientCompositionRecord.hadClientComposition = mHadClientComposition;
- clientCompositionRecord.reused = mReusedClientComposition;
- clientCompositionRecord.changed = prevFrameHadClientComposition != mHadClientComposition;
+ const bool hasGpuComposited = mCompositionCoverage.test(CompositionCoverage::Gpu);
+
+ clientCompositionRecord.hadClientComposition = hasGpuComposited;
+ clientCompositionRecord.reused = mCompositionCoverage.test(CompositionCoverage::GpuReuse);
+ clientCompositionRecord.changed = hadGpuComposited != hasGpuComposited;
+
mTimeStats->pushCompositionStrategyState(clientCompositionRecord);
- // TODO: b/160583065 Enable skip validation when SF caches all client composition layers
- const bool usedGpuComposition = mHadClientComposition || mReusedClientComposition;
- mScheduler->modulateVsync(&VsyncModulator::onDisplayRefresh, usedGpuComposition);
+ using namespace ftl::flag_operators;
+
+ // TODO(b/160583065): Enable skip validation when SF caches all client composition layers.
+ const bool hasGpuUseOrReuse =
+ mCompositionCoverage.any(CompositionCoverage::Gpu | CompositionCoverage::GpuReuse);
+ mScheduler->modulateVsync(&VsyncModulator::onDisplayRefresh, hasGpuUseOrReuse);
mLayersWithQueuedFrames.clear();
if (mLayerTracingEnabled && mLayerTracing.flagIsSet(LayerTracing::TRACE_COMPOSITION)) {
@@ -3009,15 +3021,15 @@
const auto enableFrameRateOverride = [&] {
using Config = scheduler::RefreshRateSelector::Config;
- if (!sysprop::enable_frame_rate_override(false)) {
+ if (!sysprop::enable_frame_rate_override(true)) {
return Config::FrameRateOverride::Disabled;
}
- if (sysprop::frame_rate_override_for_native_rates(true)) {
+ if (sysprop::frame_rate_override_for_native_rates(false)) {
return Config::FrameRateOverride::AppOverrideNativeRefreshRates;
}
- if (!sysprop::frame_rate_override_global(false)) {
+ if (!sysprop::frame_rate_override_global(true)) {
return Config::FrameRateOverride::AppOverride;
}
diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h
index 207dfe2..ed09c3f 100644
--- a/services/surfaceflinger/SurfaceFlinger.h
+++ b/services/surfaceflinger/SurfaceFlinger.h
@@ -57,6 +57,8 @@
#include <scheduler/PresentLatencyTracker.h>
#include <scheduler/Time.h>
#include <scheduler/TransactionSchedule.h>
+#include <scheduler/interface/CompositionCoverage.h>
+#include <scheduler/interface/ICompositor.h>
#include <ui/FenceResult.h>
#include "Display/DisplayMap.h"
@@ -606,19 +608,9 @@
void onComposerHalVsyncIdle(hal::HWDisplayId) override;
// ICompositor overrides:
-
- // Configures physical displays, processing hotplug and/or mode setting via the Composer HAL.
void configure() override;
-
- // Commits transactions for layers and displays. Returns whether any state has been invalidated,
- // i.e. whether a frame should be composited for each display.
bool commit(TimePoint frameTime, VsyncId, TimePoint expectedVsyncTime) override;
-
- // Composites a frame for each display. CompositionEngine performs GPU and/or HAL composition
- // via RenderEngine and the Composer HAL, respectively.
void composite(TimePoint frameTime, VsyncId) override;
-
- // Samples the composited frame via RegionSamplingThread.
void sample() override;
// ISchedulerCallback overrides:
@@ -1158,17 +1150,6 @@
// Tracks layers that need to update a display's dirty region.
std::vector<sp<Layer>> mLayersPendingRefresh;
- // True if in the previous frame at least one layer was composed via the GPU.
- bool mHadClientComposition = false;
- // True if in the previous frame at least one layer was composed via HW Composer.
- // Note that it is possible for a frame to be composed via both client and device
- // composition, for example in the case of overlays.
- bool mHadDeviceComposition = false;
- // True if in the previous frame, the client composition was skipped by reusing the buffer
- // used in a previous composition. This can happed if the client composition requests
- // did not change.
- bool mReusedClientComposition = false;
-
BootStage mBootStage = BootStage::BOOTLOADER;
struct HotplugEvent {
@@ -1204,7 +1185,7 @@
std::atomic_bool mForceFullDamage = false;
bool mLayerCachingEnabled = false;
- bool mPropagateBackpressureClientComposition = false;
+ bool mBackpressureGpuComposition = false;
LayerTracing mLayerTracing{*this};
bool mLayerTracingEnabled = false;
@@ -1268,6 +1249,9 @@
std::atomic<int> mNumTrustedPresentationListeners = 0;
std::unique_ptr<compositionengine::CompositionEngine> mCompositionEngine;
+
+ CompositionCoverageFlags mCompositionCoverage;
+
// mMaxRenderTargetSize is only set once in init() so it doesn't need to be protected by
// any mutex.
size_t mMaxRenderTargetSize{1};
diff --git a/services/surfaceflinger/tests/Android.bp b/services/surfaceflinger/tests/Android.bp
index de47330..6d12aa7 100644
--- a/services/surfaceflinger/tests/Android.bp
+++ b/services/surfaceflinger/tests/Android.bp
@@ -57,6 +57,7 @@
"SetFrameRateOverride_test.cpp",
"SetGeometry_test.cpp",
"Stress_test.cpp",
+ "TextureFiltering_test.cpp",
"VirtualDisplay_test.cpp",
"WindowInfosListener_test.cpp",
],
diff --git a/services/surfaceflinger/tests/TextureFiltering_test.cpp b/services/surfaceflinger/tests/TextureFiltering_test.cpp
new file mode 100644
index 0000000..e9b1fbb
--- /dev/null
+++ b/services/surfaceflinger/tests/TextureFiltering_test.cpp
@@ -0,0 +1,227 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <android/gui/ISurfaceComposerClient.h>
+#include <gtest/gtest.h>
+#include <gui/DisplayCaptureArgs.h>
+#include <ui/GraphicTypes.h>
+#include <ui/Rect.h>
+
+#include "LayerTransactionTest.h"
+
+namespace android {
+
+bool operator==(const Color& left, const Color& right) {
+ return left.a == right.a && left.r == right.r && left.g == right.g && left.b == right.b;
+}
+
+class TextureFilteringTest : public LayerTransactionTest {
+protected:
+ virtual void SetUp() {
+ LayerTransactionTest::SetUp();
+
+ mParent = createLayer("test-parent", 100, 100,
+ gui::ISurfaceComposerClient::eFXSurfaceContainer);
+ mLayer = createLayer("test-child", 100, 100,
+ gui::ISurfaceComposerClient::eFXSurfaceBufferState, mParent.get());
+ sp<GraphicBuffer> buffer =
+ sp<GraphicBuffer>::make(static_cast<uint32_t>(100), static_cast<uint32_t>(100),
+ PIXEL_FORMAT_RGBA_8888, 1u,
+ BufferUsage::CPU_READ_OFTEN | BufferUsage::CPU_WRITE_OFTEN |
+ BufferUsage::COMPOSER_OVERLAY |
+ BufferUsage::GPU_TEXTURE,
+ "test");
+ TransactionUtils::fillGraphicBufferColor(buffer, Rect{0, 0, 50, 100}, Color::RED);
+ TransactionUtils::fillGraphicBufferColor(buffer, Rect{50, 0, 100, 100}, Color::BLUE);
+ Transaction()
+ .setBuffer(mLayer, buffer)
+ .setDataspace(mLayer, ui::Dataspace::V0_SRGB)
+ .setLayer(mLayer, INT32_MAX)
+ .apply();
+ }
+
+ virtual void TearDown() { LayerTransactionTest::TearDown(); }
+
+ void expectFiltered(Rect redRect, Rect blueRect) {
+ // Check that at least some of the pixels in the red rectangle aren't solid red
+ int redPixels = 0;
+ for (int x = redRect.left; x < redRect.right; x++) {
+ for (int y = redRect.top; y < redRect.bottom; y++) {
+ redPixels += mCapture->getPixelColor(static_cast<uint32_t>(x),
+ static_cast<uint32_t>(y)) == Color::RED;
+ }
+ }
+ ASSERT_LT(redPixels, redRect.getWidth() * redRect.getHeight());
+
+ // Check that at least some of the pixels in the blue rectangle aren't solid blue
+ int bluePixels = 0;
+ for (int x = blueRect.left; x < blueRect.right; x++) {
+ for (int y = blueRect.top; y < blueRect.bottom; y++) {
+ bluePixels += mCapture->getPixelColor(static_cast<uint32_t>(x),
+ static_cast<uint32_t>(y)) == Color::BLUE;
+ }
+ }
+ ASSERT_LT(bluePixels, blueRect.getWidth() * blueRect.getHeight());
+ }
+
+ sp<SurfaceControl> mParent;
+ sp<SurfaceControl> mLayer;
+ std::unique_ptr<ScreenCapture> mCapture;
+};
+
+TEST_F(TextureFilteringTest, NoFiltering) {
+ gui::DisplayCaptureArgs captureArgs;
+ captureArgs.displayToken = mDisplay;
+ captureArgs.width = 100;
+ captureArgs.height = 100;
+ captureArgs.sourceCrop = Rect{100, 100};
+ ScreenCapture::captureDisplay(&mCapture, captureArgs);
+
+ mCapture->expectColor(Rect{0, 0, 50, 100}, Color::RED);
+ mCapture->expectColor(Rect{50, 0, 100, 100}, Color::BLUE);
+}
+
+TEST_F(TextureFilteringTest, BufferCropNoFiltering) {
+ Transaction().setBufferCrop(mLayer, Rect{0, 0, 100, 100}).apply();
+
+ gui::DisplayCaptureArgs captureArgs;
+ captureArgs.displayToken = mDisplay;
+ captureArgs.width = 100;
+ captureArgs.height = 100;
+ captureArgs.sourceCrop = Rect{0, 0, 100, 100};
+ ScreenCapture::captureDisplay(&mCapture, captureArgs);
+
+ mCapture->expectColor(Rect{0, 0, 50, 100}, Color::RED);
+ mCapture->expectColor(Rect{50, 0, 100, 100}, Color::BLUE);
+}
+
+// Expect filtering because the buffer is stretched to the layer's bounds.
+TEST_F(TextureFilteringTest, BufferCropIsFiltered) {
+ Transaction().setBufferCrop(mLayer, Rect{25, 25, 75, 75}).apply();
+
+ gui::DisplayCaptureArgs captureArgs;
+ captureArgs.displayToken = mDisplay;
+ captureArgs.width = 100;
+ captureArgs.height = 100;
+ captureArgs.sourceCrop = Rect{0, 0, 100, 100};
+ ScreenCapture::captureDisplay(&mCapture, captureArgs);
+
+ expectFiltered({0, 0, 50, 100}, {50, 0, 100, 100});
+}
+
+// Expect filtering because the output source crop is stretched to the output buffer's size.
+TEST_F(TextureFilteringTest, OutputSourceCropIsFiltered) {
+ gui::DisplayCaptureArgs captureArgs;
+ captureArgs.displayToken = mDisplay;
+ captureArgs.width = 100;
+ captureArgs.height = 100;
+ captureArgs.sourceCrop = Rect{25, 25, 75, 75};
+ ScreenCapture::captureDisplay(&mCapture, captureArgs);
+
+ expectFiltered({0, 0, 50, 100}, {50, 0, 100, 100});
+}
+
+// Expect filtering because the layer crop and output source crop are stretched to the output
+// buffer's size.
+TEST_F(TextureFilteringTest, LayerCropOutputSourceCropIsFiltered) {
+ Transaction().setCrop(mLayer, Rect{25, 25, 75, 75}).apply();
+
+ gui::DisplayCaptureArgs captureArgs;
+ captureArgs.displayToken = mDisplay;
+ captureArgs.width = 100;
+ captureArgs.height = 100;
+ captureArgs.sourceCrop = Rect{25, 25, 75, 75};
+ ScreenCapture::captureDisplay(&mCapture, captureArgs);
+
+ expectFiltered({0, 0, 50, 100}, {50, 0, 100, 100});
+}
+
+// Expect filtering because the layer is scaled up.
+TEST_F(TextureFilteringTest, LayerCaptureWithScalingIsFiltered) {
+ LayerCaptureArgs captureArgs;
+ captureArgs.layerHandle = mLayer->getHandle();
+ captureArgs.frameScaleX = 2;
+ captureArgs.frameScaleY = 2;
+ ScreenCapture::captureLayers(&mCapture, captureArgs);
+
+ expectFiltered({0, 0, 100, 200}, {100, 0, 200, 200});
+}
+
+// Expect no filtering because the output buffer's size matches the source crop.
+TEST_F(TextureFilteringTest, LayerCaptureOutputSourceCropNoFiltering) {
+ LayerCaptureArgs captureArgs;
+ captureArgs.layerHandle = mLayer->getHandle();
+ captureArgs.sourceCrop = Rect{25, 25, 75, 75};
+ ScreenCapture::captureLayers(&mCapture, captureArgs);
+
+ mCapture->expectColor(Rect{0, 0, 25, 50}, Color::RED);
+ mCapture->expectColor(Rect{25, 0, 50, 50}, Color::BLUE);
+}
+
+// Expect no filtering because the output buffer's size matches the source crop (with a cropped
+// layer).
+TEST_F(TextureFilteringTest, LayerCaptureWithCropNoFiltering) {
+ Transaction().setCrop(mLayer, Rect{10, 10, 90, 90}).apply();
+
+ LayerCaptureArgs captureArgs;
+ captureArgs.layerHandle = mLayer->getHandle();
+ captureArgs.sourceCrop = Rect{25, 25, 75, 75};
+ ScreenCapture::captureLayers(&mCapture, captureArgs);
+
+ mCapture->expectColor(Rect{0, 0, 25, 50}, Color::RED);
+ mCapture->expectColor(Rect{25, 0, 50, 50}, Color::BLUE);
+}
+
+// Expect no filtering because the output source crop and output buffer are the same size.
+TEST_F(TextureFilteringTest, OutputSourceCropDisplayFrameMatchNoFiltering) {
+ // Transaction().setCrop(mLayer, Rect{25, 25, 75, 75}).apply();
+
+ gui::DisplayCaptureArgs captureArgs;
+ captureArgs.displayToken = mDisplay;
+ captureArgs.width = 50;
+ captureArgs.height = 50;
+ captureArgs.sourceCrop = Rect{25, 25, 75, 75};
+ ScreenCapture::captureDisplay(&mCapture, captureArgs);
+
+ mCapture->expectColor(Rect{0, 0, 25, 50}, Color::RED);
+ mCapture->expectColor(Rect{25, 0, 50, 50}, Color::BLUE);
+}
+
+// Expect no filtering because the layer crop shouldn't scale the layer.
+TEST_F(TextureFilteringTest, LayerCropDisplayFrameMatchNoFiltering) {
+ Transaction().setCrop(mLayer, Rect{25, 25, 75, 75}).apply();
+
+ gui::DisplayCaptureArgs captureArgs;
+ captureArgs.displayToken = mDisplay;
+ ScreenCapture::captureDisplay(&mCapture, captureArgs);
+
+ mCapture->expectColor(Rect{25, 25, 50, 75}, Color::RED);
+ mCapture->expectColor(Rect{50, 25, 75, 75}, Color::BLUE);
+}
+
+// Expect no filtering because the parent layer crop shouldn't scale the layer.
+TEST_F(TextureFilteringTest, ParentCropNoFiltering) {
+ Transaction().setCrop(mParent, Rect{25, 25, 75, 75}).apply();
+
+ gui::DisplayCaptureArgs captureArgs;
+ captureArgs.displayToken = mDisplay;
+ ScreenCapture::captureDisplay(&mCapture, captureArgs);
+
+ mCapture->expectColor(Rect{25, 25, 50, 75}, Color::RED);
+ mCapture->expectColor(Rect{50, 25, 75, 75}, Color::BLUE);
+}
+
+} // namespace android
diff --git a/services/surfaceflinger/tests/unittests/FrameTimelineTest.cpp b/services/surfaceflinger/tests/unittests/FrameTimelineTest.cpp
index f47ac6d..abd7789 100644
--- a/services/surfaceflinger/tests/unittests/FrameTimelineTest.cpp
+++ b/services/surfaceflinger/tests/unittests/FrameTimelineTest.cpp
@@ -44,6 +44,17 @@
namespace android::frametimeline {
+static const std::string sLayerNameOne = "layer1";
+static const std::string sLayerNameTwo = "layer2";
+
+constexpr const uid_t sUidOne = 0;
+constexpr pid_t sPidOne = 10;
+constexpr pid_t sPidTwo = 20;
+constexpr int32_t sInputEventId = 5;
+constexpr int32_t sLayerIdOne = 1;
+constexpr int32_t sLayerIdTwo = 2;
+constexpr GameMode sGameMode = GameMode::Unsupported;
+
class FrameTimelineTest : public testing::Test {
public:
FrameTimelineTest() {
@@ -106,6 +117,14 @@
return packets;
}
+ void addEmptySurfaceFrame() {
+ auto surfaceFrame =
+ mFrameTimeline->createSurfaceFrameForToken({}, sPidOne, sUidOne, sLayerIdOne,
+ sLayerNameOne, sLayerNameOne,
+ /*isBuffer*/ false, sGameMode);
+ mFrameTimeline->addSurfaceFrame(std::move(surfaceFrame));
+ }
+
void addEmptyDisplayFrame() {
auto presentFence1 = fenceFactory.createFenceTimeForTest(Fence::NO_FENCE);
// Trigger a flushPresentFence by calling setSfPresent for the next frame
@@ -168,17 +187,6 @@
kStartThreshold};
};
-static const std::string sLayerNameOne = "layer1";
-static const std::string sLayerNameTwo = "layer2";
-
-constexpr const uid_t sUidOne = 0;
-constexpr pid_t sPidOne = 10;
-constexpr pid_t sPidTwo = 20;
-constexpr int32_t sInputEventId = 5;
-constexpr int32_t sLayerIdOne = 1;
-constexpr int32_t sLayerIdTwo = 2;
-constexpr GameMode sGameMode = GameMode::Unsupported;
-
TEST_F(FrameTimelineTest, tokenManagerRemovesStalePredictions) {
int64_t token1 = mTokenManager->generateTokenForPredictions({0, 0, 0});
EXPECT_EQ(getPredictions().size(), 1u);
@@ -1054,6 +1062,9 @@
auto presentFence1 = fenceFactory.createFenceTimeForTest(Fence::NO_FENCE);
tracingSession->StartBlocking();
+
+ // Add an empty surface frame so that display frame would get traced.
+ addEmptySurfaceFrame();
int64_t displayFrameToken1 = mTokenManager->generateTokenForPredictions({10, 30, 30});
// Set up the display frame
@@ -1135,6 +1146,9 @@
// Flush the token so that it would expire
flushTokens();
+ // Add an empty surface frame so that display frame would get traced.
+ addEmptySurfaceFrame();
+
// Set up the display frame
mFrameTimeline->setSfWakeUp(displayFrameToken1, 20, Fps::fromPeriodNsecs(11));
mFrameTimeline->setSfPresent(26, presentFence1);
diff --git a/services/surfaceflinger/tests/unittests/LayerHierarchyTest.cpp b/services/surfaceflinger/tests/unittests/LayerHierarchyTest.cpp
index 783df28..763426a 100644
--- a/services/surfaceflinger/tests/unittests/LayerHierarchyTest.cpp
+++ b/services/surfaceflinger/tests/unittests/LayerHierarchyTest.cpp
@@ -641,4 +641,69 @@
EXPECT_EQ(getTraversalPath(hierarchyBuilder.getOffscreenHierarchy()), expectedTraversalPath);
}
+TEST_F(LayerHierarchyTest, canMirrorDisplay) {
+ LayerHierarchyBuilder hierarchyBuilder(mLifecycleManager.getLayers());
+ setFlags(12, layer_state_t::eLayerSkipScreenshot, layer_state_t::eLayerSkipScreenshot);
+ createDisplayMirrorLayer(3, ui::LayerStack::fromValue(0));
+ setLayerStack(3, 1);
+ UPDATE_AND_VERIFY(hierarchyBuilder);
+
+ std::vector<uint32_t> expected = {3, 1, 11, 111, 12, 121, 122, 1221, 13, 2,
+ 1, 11, 111, 12, 121, 122, 1221, 13, 2};
+ EXPECT_EQ(getTraversalPath(hierarchyBuilder.getHierarchy()), expected);
+ EXPECT_EQ(getTraversalPathInZOrder(hierarchyBuilder.getHierarchy()), expected);
+ expected = {};
+ EXPECT_EQ(getTraversalPath(hierarchyBuilder.getOffscreenHierarchy()), expected);
+}
+
+TEST_F(LayerHierarchyTest, mirrorNonExistingDisplay) {
+ LayerHierarchyBuilder hierarchyBuilder(mLifecycleManager.getLayers());
+ setFlags(12, layer_state_t::eLayerSkipScreenshot, layer_state_t::eLayerSkipScreenshot);
+ createDisplayMirrorLayer(3, ui::LayerStack::fromValue(5));
+ setLayerStack(3, 1);
+ UPDATE_AND_VERIFY(hierarchyBuilder);
+
+ std::vector<uint32_t> expected = {3, 1, 11, 111, 12, 121, 122, 1221, 13, 2};
+ EXPECT_EQ(getTraversalPath(hierarchyBuilder.getHierarchy()), expected);
+ EXPECT_EQ(getTraversalPathInZOrder(hierarchyBuilder.getHierarchy()), expected);
+ expected = {};
+ EXPECT_EQ(getTraversalPath(hierarchyBuilder.getOffscreenHierarchy()), expected);
+}
+
+TEST_F(LayerHierarchyTest, newRootLayerIsMirrored) {
+ LayerHierarchyBuilder hierarchyBuilder(mLifecycleManager.getLayers());
+ setFlags(12, layer_state_t::eLayerSkipScreenshot, layer_state_t::eLayerSkipScreenshot);
+ createDisplayMirrorLayer(3, ui::LayerStack::fromValue(0));
+ setLayerStack(3, 1);
+ UPDATE_AND_VERIFY(hierarchyBuilder);
+
+ createRootLayer(4);
+ UPDATE_AND_VERIFY(hierarchyBuilder);
+
+ std::vector<uint32_t> expected = {3, 1, 11, 111, 12, 121, 122, 1221, 13, 2, 4,
+ 1, 11, 111, 12, 121, 122, 1221, 13, 2, 4};
+ EXPECT_EQ(getTraversalPath(hierarchyBuilder.getHierarchy()), expected);
+ EXPECT_EQ(getTraversalPathInZOrder(hierarchyBuilder.getHierarchy()), expected);
+ expected = {};
+ EXPECT_EQ(getTraversalPath(hierarchyBuilder.getOffscreenHierarchy()), expected);
+}
+
+TEST_F(LayerHierarchyTest, removedRootLayerIsNoLongerMirrored) {
+ LayerHierarchyBuilder hierarchyBuilder(mLifecycleManager.getLayers());
+ setFlags(12, layer_state_t::eLayerSkipScreenshot, layer_state_t::eLayerSkipScreenshot);
+ createDisplayMirrorLayer(3, ui::LayerStack::fromValue(0));
+ setLayerStack(3, 1);
+ UPDATE_AND_VERIFY(hierarchyBuilder);
+
+ reparentLayer(1, UNASSIGNED_LAYER_ID);
+ destroyLayerHandle(1);
+ UPDATE_AND_VERIFY(hierarchyBuilder);
+
+ std::vector<uint32_t> expected = {3, 2, 2};
+ EXPECT_EQ(getTraversalPath(hierarchyBuilder.getHierarchy()), expected);
+ EXPECT_EQ(getTraversalPathInZOrder(hierarchyBuilder.getHierarchy()), expected);
+ expected = {11, 111, 12, 121, 122, 1221, 13};
+ EXPECT_EQ(getTraversalPath(hierarchyBuilder.getOffscreenHierarchy()), expected);
+}
+
} // namespace android::surfaceflinger::frontend
diff --git a/services/surfaceflinger/tests/unittests/LayerHierarchyTest.h b/services/surfaceflinger/tests/unittests/LayerHierarchyTest.h
index 1a82232..852cb91 100644
--- a/services/surfaceflinger/tests/unittests/LayerHierarchyTest.h
+++ b/services/surfaceflinger/tests/unittests/LayerHierarchyTest.h
@@ -60,6 +60,14 @@
return args;
}
+ LayerCreationArgs createDisplayMirrorArgs(uint32_t id, ui::LayerStack layerStack) {
+ LayerCreationArgs args(nullptr, nullptr, "testlayer", 0, {}, std::make_optional(id));
+ args.addToRoot = true;
+ args.parentHandle.clear();
+ args.layerStackToMirror = layerStack;
+ return args;
+ }
+
std::vector<uint32_t> getTraversalPath(const LayerHierarchy& hierarchy) const {
std::vector<uint32_t> layerIds;
hierarchy.traverse([&layerIds = layerIds](const LayerHierarchy& hierarchy,
@@ -90,6 +98,15 @@
mLifecycleManager.addLayers(std::move(layers));
}
+ void createDisplayMirrorLayer(uint32_t id, ui::LayerStack layerStack) {
+ sp<LayerHandle> handle = sp<LayerHandle>::make(id);
+ mHandles[id] = handle;
+ std::vector<std::unique_ptr<RequestedLayerState>> layers;
+ layers.emplace_back(std::make_unique<RequestedLayerState>(
+ createDisplayMirrorArgs(/*id=*/id, layerStack)));
+ mLifecycleManager.addLayers(std::move(layers));
+ }
+
virtual void createLayer(uint32_t id, uint32_t parentId) {
sp<LayerHandle> handle = sp<LayerHandle>::make(id);
mHandles[id] = handle;
diff --git a/services/surfaceflinger/tests/unittests/LayerSnapshotTest.cpp b/services/surfaceflinger/tests/unittests/LayerSnapshotTest.cpp
index e124342..aa6a14e 100644
--- a/services/surfaceflinger/tests/unittests/LayerSnapshotTest.cpp
+++ b/services/surfaceflinger/tests/unittests/LayerSnapshotTest.cpp
@@ -306,4 +306,27 @@
EXPECT_EQ(getSnapshot(1)->frameRate.type, scheduler::LayerInfo::FrameRateCompatibility::NoVote);
}
+// Display Mirroring Tests
+// tree with 3 levels of children
+// ROOT (DISPLAY 0)
+// ├── 1
+// │ ├── 11
+// │ │ └── 111
+// │ ├── 12 (has skip screenshot flag)
+// │ │ ├── 121
+// │ │ └── 122
+// │ │ └── 1221
+// │ └── 13
+// └── 2
+// ROOT (DISPLAY 1)
+// └── 3 (mirrors display 0)
+TEST_F(LayerSnapshotTest, displayMirrorRespects) {
+ setFlags(12, layer_state_t::eLayerSkipScreenshot, layer_state_t::eLayerSkipScreenshot);
+ createDisplayMirrorLayer(3, ui::LayerStack::fromValue(0));
+ setLayerStack(3, 1);
+
+ std::vector<uint32_t> expected = {3, 1, 11, 111, 13, 2, 1, 11, 111, 12, 121, 122, 1221, 13, 2};
+ UPDATE_AND_VERIFY(mSnapshotBuilder, expected);
+}
+
} // namespace android::surfaceflinger::frontend
diff --git a/services/surfaceflinger/tests/unittests/RefreshRateSelectorTest.cpp b/services/surfaceflinger/tests/unittests/RefreshRateSelectorTest.cpp
index 5fddda5..f4d052d 100644
--- a/services/surfaceflinger/tests/unittests/RefreshRateSelectorTest.cpp
+++ b/services/surfaceflinger/tests/unittests/RefreshRateSelectorTest.cpp
@@ -1165,7 +1165,7 @@
case Config::FrameRateOverride::AppOverride:
return {{30_Hz, kMode30}, {60_Hz, kMode60}, {90_Hz, kMode90}};
case Config::FrameRateOverride::Enabled:
- return {{30_Hz, kMode30}, {45_Hz, kMode90}, {60_Hz, kMode60}, {90_Hz, kMode90}};
+ return {{30_Hz, kMode30}, {60_Hz, kMode60}, {90_Hz, kMode90}};
}
}();
ASSERT_EQ(expectedRefreshRates.size(), refreshRates.size());
@@ -1197,7 +1197,7 @@
case Config::FrameRateOverride::AppOverride:
return {{30_Hz, kMode30}, {60_Hz, kMode60}, {90_Hz, kMode90}};
case Config::FrameRateOverride::Enabled:
- return {{30_Hz, kMode30}, {45_Hz, kMode90}, {60_Hz, kMode60}, {90_Hz, kMode90}};
+ return {{30_Hz, kMode30}, {60_Hz, kMode60}, {90_Hz, kMode90}};
}
}();
ASSERT_EQ(expectedRefreshRates.size(), refreshRates.size());
@@ -2983,5 +2983,24 @@
EXPECT_FRAME_RATE_MODE(kMode60, 60_Hz, selector.getBestScoredFrameRate(layers).frameRateMode);
}
+TEST_P(RefreshRateSelectorTest, frameRateIsCappedByPolicy) {
+ if (GetParam() != Config::FrameRateOverride::Enabled) {
+ return;
+ }
+
+ auto selector = createSelector(kModes_60_90, kModeId60);
+
+ constexpr FpsRanges kCappedAt30 = {{60_Hz, 90_Hz}, {30_Hz, 30_Hz}};
+
+ EXPECT_EQ(SetPolicyResult::Changed,
+ selector.setDisplayManagerPolicy(
+ {DisplayModeId(kModeId60), kCappedAt30, kCappedAt30}));
+
+ std::vector<LayerRequirement> layers = {{.weight = 1.f}};
+ layers[0].name = "Test layer";
+ layers[0].vote = LayerVoteType::Min;
+ EXPECT_FRAME_RATE_MODE(kMode60, 30_Hz, selector.getBestScoredFrameRate(layers).frameRateMode);
+}
+
} // namespace
} // namespace android::scheduler
diff --git a/services/surfaceflinger/tests/unittests/TestableScheduler.h b/services/surfaceflinger/tests/unittests/TestableScheduler.h
index 0cbfa63..74885d8 100644
--- a/services/surfaceflinger/tests/unittests/TestableScheduler.h
+++ b/services/surfaceflinger/tests/unittests/TestableScheduler.h
@@ -16,11 +16,12 @@
#pragma once
-#include <Scheduler/Scheduler.h>
#include <ftl/fake_guard.h>
#include <gmock/gmock.h>
#include <gui/ISurfaceComposer.h>
+#include <scheduler/interface/ICompositor.h>
+
#include "Scheduler/EventThread.h"
#include "Scheduler/LayerHistory.h"
#include "Scheduler/Scheduler.h"