Merge "Add support for TFLite motion prediction model."
diff --git a/core/java/android/view/MotionPredictor.java b/core/java/android/view/MotionPredictor.java
index fa86a4c..4d32efe 100644
--- a/core/java/android/view/MotionPredictor.java
+++ b/core/java/android/view/MotionPredictor.java
@@ -22,26 +22,27 @@
import libcore.util.NativeAllocationRegistry;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
/**
* Calculate motion predictions.
*
- * Feed motion events to this class in order to generate the predicted events. The prediction
- * functionality may not be available on all devices. Check if a specific source is supported on a
- * given input device using #isPredictionAvailable.
+ * Feed motion events to this class in order to generate predicted future events. The prediction
+ * functionality may not be available on all devices: check if a specific source is supported on a
+ * given input device using {@link #isPredictionAvailable}.
*
- * Send all of the events that were received from the system here in order to generate complete,
- * accurate predictions. When processing the returned predictions, make sure to consider all of the
- * {@link MotionEvent#getHistoricalAxisValue historical samples}.
+ * Send all of the events that were received from the system to {@link #record} to generate
+ * complete, accurate predictions from {@link #predict}. When processing the returned predictions,
+ * make sure to consider all of the {@link MotionEvent#getHistoricalAxisValue historical samples}.
*/
-// Acts as a pass-through to the native MotionPredictor object.
-// Do not store any state in this Java layer, or add any business logic here. All of the
-// implementation details should go into the native MotionPredictor.
-// The context / resource access must be here rather than in native layer due to the lack of the
-// corresponding native API surface.
public final class MotionPredictor {
+ // This is a pass-through to the native MotionPredictor object (mPtr). Do not store any state or
+ // add any business logic here -- all of the implementation details should go into the native
+ // MotionPredictor (except for accessing the context/resources, which have no corresponding
+ // native API).
+
private static class RegistryHolder {
public static final NativeAllocationRegistry REGISTRY =
NativeAllocationRegistry.createMalloced(
@@ -67,49 +68,63 @@
/**
* Record a movement so that in the future, a prediction for the current gesture can be
- * generated. Ensure to add all motions from the gesture of interest to generate the correct
- * prediction.
+ * generated. Ensure to add all motions from the gesture of interest to generate correct
+ * predictions.
* @param event The received event
*/
public void record(@NonNull MotionEvent event) {
+ if (!isPredictionEnabled()) {
+ return;
+ }
nativeRecord(mPtr, event);
}
/**
- * Get predicted events for all gestures that have been provided to the 'record' function.
+ * Get predicted events for all gestures that have been provided to {@link #record}.
* If events from multiple devices were sent to 'record', this will produce a separate
- * {@link MotionEvent} for each device id. The returned list may be empty if no predictions for
- * any of the added events are available.
+ * {@link MotionEvent} for each device. The returned list may be empty if no predictions for
+ * any of the added events/devices are available.
* Predictions may not reach the requested timestamp if the confidence in the prediction results
* is low.
*
* @param predictionTimeNanos The time that the prediction should target, in the
* {@link android.os.SystemClock#uptimeMillis} time base, but in nanoseconds.
*
- * @return the list of predicted motion events, for each device id. Ensure to check the
- * historical data in addition to the latest ({@link MotionEvent#getX getX()},
- * {@link MotionEvent#getY getY()}) coordinates for smoothest prediction curves. Empty list is
- * returned if predictions are not supported, or not possible for the current set of gestures.
+ * @return A list of predicted motion events, with at most one for each device observed by
+ * {@link #record}. Be sure to check the historical data in addition to the latest
+ * ({@link MotionEvent#getX getX()}, {@link MotionEvent#getY getY()}) coordinates for smooth
+ * prediction curves. An empty list is returned if predictions are not supported, or not
+ * possible for the current set of gestures.
*/
@NonNull
public List<MotionEvent> predict(long predictionTimeNanos) {
+ if (!isPredictionEnabled()) {
+ return Collections.emptyList();
+ }
return Arrays.asList(nativePredict(mPtr, predictionTimeNanos));
}
- /**
- * Check whether this device supports motion predictions for the given source type.
- *
- * @param deviceId The input device id
- * @param source The source of input events
- * @return True if the current device supports predictions, false otherwise.
- */
- public boolean isPredictionAvailable(int deviceId, int source) {
+ private boolean isPredictionEnabled() {
// Device-specific override
if (!mContext.getResources().getBoolean(
com.android.internal.R.bool.config_enableMotionPrediction)) {
return false;
}
- return nativeIsPredictionAvailable(mPtr, deviceId, source);
+ return true;
+ }
+
+ /**
+ * Check whether a device supports motion predictions for a given source type.
+ *
+ * @param deviceId The input device id.
+ * @param source The source of input events.
+ * @return True if the current device supports predictions, false otherwise.
+ *
+ * @see MotionEvent#getDeviceId
+ * @see MotionEvent#getSource
+ */
+ public boolean isPredictionAvailable(int deviceId, int source) {
+ return isPredictionEnabled() && nativeIsPredictionAvailable(mPtr, deviceId, source);
}
private static native long nativeInitialize(int offsetNanos);
diff --git a/core/jni/Android.bp b/core/jni/Android.bp
index d8dffcd..82f414b 100644
--- a/core/jni/Android.bp
+++ b/core/jni/Android.bp
@@ -326,6 +326,7 @@
"libdl",
"libdl_android",
"libtimeinstate",
+ "libtflite",
"server_configurable_flags",
"libimage_io",
"libjpegdecoder",
@@ -343,7 +344,9 @@
header_libs: [
"bionic_libc_platform_headers",
"dnsproxyd_protocol_headers",
+ "flatbuffer_headers",
"libtextclassifier_hash_headers",
+ "tensorflow_headers",
],
runtime_libs: [
"libidmap2",
diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml
index d3aee43..d880296 100644
--- a/core/res/res/values/config.xml
+++ b/core/res/res/values/config.xml
@@ -2383,22 +2383,13 @@
display, this value should be true. -->
<bool name="config_perDisplayFocusEnabled">false</bool>
- <!-- Whether the system enables motion prediction. Only enable this after confirming that the
- model works well on your device. To enable system-based prediction, set this value to true.
- -->
- <bool name="config_enableMotionPrediction">true</bool>
+ <!-- Whether to use the system motion prediction model. Only set this value to true after
+ confirming that the model works well on your device. -->
+ <bool name="config_enableMotionPrediction">false</bool>
<!-- Additional offset to use for motion prediction, in nanoseconds. A positive number indicates
- that the prediction will take place further in the future. For example, suppose a
- MotionEvent arrives with timestamp t=1, and the current expected presentation time is t=2.
- Typically, the prediction will target the presentation time, t=2. If you'd like to make
- prediction more aggressive, you could set the offset to a positive number.
- Setting the offset to 1 here would mean that the prediction will be done for time t=3.
- A negative number may also be provided, to make the prediction less aggressive. In general,
- the offset here should represent some built-in hardware delays that may not be accounted
- for by the "expected present time". See also:
- https://developer.android.com/reference/android/view/
- Choreographer.FrameTimeline#getExpectedPresentationTimeNanos() -->
+ that the prediction will take place further in the future and, in general, should represent
+ some built-in hardware delays that prediction should try to recover. -->
<integer name="config_motionPredictionOffsetNanos">0</integer>
<!-- Whether a software navigation bar should be shown. NOTE: in the future this may be
diff --git a/tests/Input/src/com/android/test/input/MotionPredictorTest.kt b/tests/Input/src/com/android/test/input/MotionPredictorTest.kt
index bc363b0..8b1b06f 100644
--- a/tests/Input/src/com/android/test/input/MotionPredictorTest.kt
+++ b/tests/Input/src/com/android/test/input/MotionPredictorTest.kt
@@ -102,9 +102,9 @@
* a prediction. Here, we send 2 events to the predictor and check the returned event.
* Input:
* t = 0 x = 0 y = 0
- * t = 1 x = 1 y = 2
+ * t = 4 x = 10 y = 20
* Output (expected):
- * t = 3 x = 3 y = 6
+ * t = 12 x = 30 y = 60 ± error
*
* Historical data is ignored for simplicity.
*/
@@ -118,19 +118,20 @@
// ACTION_DOWN t=0 x=0 y=0
predictor.record(downEvent)
- eventTime += Duration.ofMillis(1)
- val moveEvent = getStylusMotionEvent(eventTime, ACTION_MOVE, /*x=*/1f, /*y=*/2f)
+ eventTime += Duration.ofMillis(4)
+ val moveEvent = getStylusMotionEvent(eventTime, ACTION_MOVE, /*x=*/10f, /*y=*/20f)
// ACTION_MOVE t=1 x=1 y=2
predictor.record(moveEvent)
- val predicted = predictor.predict(Duration.ofMillis(2).toNanos())
+ val predicted = predictor.predict(Duration.ofMillis(8).toNanos())
assertEquals(1, predicted.size)
val event = predicted[0]
assertNotNull(event)
- // Prediction will happen for t=3 (2 + 1, since offset is 1 and present time is 2)
- assertEquals(3, event.eventTime)
- assertEquals(3f, event.x, /*delta=*/0.001f)
- assertEquals(6f, event.y, /*delta=*/0.001f)
+ // Prediction will happen for t=12 (since it is the next input interval after the requested
+ // time, 8, plus the model offset, 1).
+ assertEquals(12, event.eventTime)
+ assertEquals(30f, event.x, /*delta=*/5f)
+ assertEquals(60f, event.y, /*delta=*/15f)
}
}