Prediction for motion events
The class MotionPredictor will be used to call into native code to
execute prediction algorithm.
The feature can be enabled by setting the appropriate config on the
device.
To enable the feature:
adb shell setprop persist.input.enable_motion_prediction true
Bug: 210158587
Test: m MotionPrediction && adb install $ANDROID_PRODUCT_OUT/system/app/MotionPrediction/MotionPrediction.apk
Change-Id: Iaee80d99b18da2f788fa8eacf2e14a6b5c6a7088
diff --git a/tests/Input/Android.bp b/tests/Input/Android.bp
index de9bbb6..83893ba 100644
--- a/tests/Input/Android.bp
+++ b/tests/Input/Android.bp
@@ -18,6 +18,7 @@
static_libs: [
"androidx.test.ext.junit",
"androidx.test.rules",
+ "mockito-target-minus-junit4",
"services.core.unboosted",
"testables",
"truth-prebuilt",
diff --git a/tests/Input/src/com/android/test/input/MotionPredictorTest.kt b/tests/Input/src/com/android/test/input/MotionPredictorTest.kt
new file mode 100644
index 0000000..46aad9f
--- /dev/null
+++ b/tests/Input/src/com/android/test/input/MotionPredictorTest.kt
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.android.test.input
+
+import android.content.Context
+import android.content.res.Resources
+import android.os.SystemProperties
+import android.view.InputDevice
+import android.view.MotionEvent
+import android.view.MotionEvent.ACTION_DOWN
+import android.view.MotionEvent.ACTION_MOVE
+import android.view.MotionEvent.PointerCoords
+import android.view.MotionEvent.PointerProperties
+import android.view.MotionPredictor
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.filters.SmallTest
+import androidx.test.platform.app.InstrumentationRegistry
+
+import org.mockito.Mockito.mock
+import org.mockito.Mockito.`when`
+
+import org.junit.After
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNotNull
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+
+import java.time.Duration
+
+private fun getStylusMotionEvent(
+ eventTime: Duration,
+ action: Int,
+ x: Float,
+ y: Float,
+ ): MotionEvent{
+ // One-time: send a DOWN event
+ val pointerCount = 1
+ val properties = arrayOfNulls<MotionEvent.PointerProperties>(pointerCount)
+ val coords = arrayOfNulls<MotionEvent.PointerCoords>(pointerCount)
+
+ for (i in 0 until pointerCount) {
+ properties[i] = PointerProperties()
+ properties[i]!!.id = i
+ properties[i]!!.toolType = MotionEvent.TOOL_TYPE_STYLUS
+ coords[i] = PointerCoords()
+ coords[i]!!.x = x
+ coords[i]!!.y = y
+ }
+
+ return MotionEvent.obtain(/*downTime=*/0, eventTime.toMillis(), action, properties.size,
+ properties, coords, /*metaState=*/0, /*buttonState=*/0,
+ /*xPrecision=*/0f, /*yPrecision=*/0f, /*deviceId=*/0, /*edgeFlags=*/0,
+ InputDevice.SOURCE_STYLUS, /*flags=*/0)
+}
+
+private fun getPredictionContext(offset: Duration, enablePrediction: Boolean): Context {
+ val context = mock(Context::class.java)
+ val resources: Resources = mock(Resources::class.java)
+ `when`(context.getResources()).thenReturn(resources)
+ `when`(resources.getInteger(
+ com.android.internal.R.integer.config_motionPredictionOffsetNanos)).thenReturn(
+ offset.toNanos().toInt())
+ `when`(resources.getBoolean(
+ com.android.internal.R.bool.config_enableMotionPrediction)).thenReturn(enablePrediction)
+ return context
+}
+
+@RunWith(AndroidJUnit4::class)
+@SmallTest
+class MotionPredictorTest {
+ private val instrumentation = InstrumentationRegistry.getInstrumentation()
+ val initialPropertyValue = SystemProperties.get("persist.input.enable_motion_prediction")
+
+ @Before
+ fun setUp() {
+ instrumentation.uiAutomation.executeShellCommand(
+ "setprop persist.input.enable_motion_prediction true")
+ }
+
+ @After
+ fun tearDown() {
+ instrumentation.uiAutomation.executeShellCommand(
+ "setprop persist.input.enable_motion_prediction $initialPropertyValue")
+ }
+
+ /**
+ * In a typical usage, app will send the event to the predictor and then call .predict to draw
+ * a prediction. Here, we send 2 events to the predictor and check the returned event.
+ * Input:
+ * t = 0 x = 0 y = 0
+ * t = 1 x = 1 y = 2
+ * Output (expected):
+ * t = 3 x = 3 y = 6
+ *
+ * Historical data is ignored for simplicity.
+ */
+ @Test
+ fun testPredictedCoordinatesAndTime() {
+ val context = getPredictionContext(
+ /*offset=*/Duration.ofMillis(1), /*enablePrediction=*/true)
+ val predictor = MotionPredictor(context)
+ var eventTime = Duration.ofMillis(0)
+ val downEvent = getStylusMotionEvent(eventTime, ACTION_DOWN, /*x=*/0f, /*y=*/0f)
+ // ACTION_DOWN t=0 x=0 y=0
+ predictor.record(downEvent)
+
+ eventTime += Duration.ofMillis(1)
+ val moveEvent = getStylusMotionEvent(eventTime, ACTION_MOVE, /*x=*/1f, /*y=*/2f)
+ // ACTION_MOVE t=1 x=1 y=2
+ predictor.record(moveEvent)
+
+ val predicted = predictor.predict(Duration.ofMillis(2).toNanos())
+ assertEquals(1, predicted.size)
+ val event = predicted[0]
+ assertNotNull(event)
+
+ // Prediction will happen for t=3 (2 + 1, since offset is 1 and present time is 2)
+ assertEquals(3, event.eventTime)
+ assertEquals(3f, event.x, /*delta=*/0.001f)
+ assertEquals(6f, event.y, /*delta=*/0.001f)
+ }
+}
diff --git a/tests/MotionPrediction/Android.bp b/tests/MotionPrediction/Android.bp
new file mode 100644
index 0000000..b9d01da
--- /dev/null
+++ b/tests/MotionPrediction/Android.bp
@@ -0,0 +1,31 @@
+//
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package {
+ // See: http://go/android-license-faq
+ // A large-scale-change added 'default_applicable_licenses' to import
+ // all of the 'license_kinds' from "frameworks_base_license"
+ // to get the below license kinds:
+ // SPDX-license-identifier-Apache-2.0
+ default_applicable_licenses: ["frameworks_base_license"],
+}
+
+android_app {
+ name: "MotionPrediction",
+ srcs: ["**/*.kt"],
+ platform_apis: true,
+ certificate: "platform",
+}
diff --git a/tests/MotionPrediction/AndroidManifest.xml b/tests/MotionPrediction/AndroidManifest.xml
new file mode 100644
index 0000000..3f8c2f2
--- /dev/null
+++ b/tests/MotionPrediction/AndroidManifest.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+ package="test.motionprediction">
+
+ <application android:allowBackup="false"
+ android:icon="@mipmap/ic_launcher"
+ android:label="@string/app_name"
+ android:supportsRtl="true"
+ android:theme="@style/AppTheme">
+ <activity android:name=".MainActivity"
+ android:exported="true">
+ <intent-filter>
+ <action android:name="android.intent.action.MAIN"/>
+ <category android:name="android.intent.category.LAUNCHER"/>
+ </intent-filter>
+ </activity>
+ </application>
+
+</manifest>
diff --git a/tests/MotionPrediction/OWNERS b/tests/MotionPrediction/OWNERS
new file mode 100644
index 0000000..c88bfe9
--- /dev/null
+++ b/tests/MotionPrediction/OWNERS
@@ -0,0 +1 @@
+include platform/frameworks/base:/INPUT_OWNERS
diff --git a/tests/MotionPrediction/res/layout/activity_main.xml b/tests/MotionPrediction/res/layout/activity_main.xml
new file mode 100644
index 0000000..65dc325
--- /dev/null
+++ b/tests/MotionPrediction/res/layout/activity_main.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+ xmlns:tools="http://schemas.android.com/tools"
+ android:layout_width="match_parent"
+ android:layout_height="match_parent"
+ android:orientation="vertical"
+ android:paddingBottom="@dimen/activity_vertical_margin"
+ android:paddingLeft="@dimen/activity_horizontal_margin"
+ android:paddingRight="@dimen/activity_horizontal_margin"
+ android:paddingTop="@dimen/activity_vertical_margin"
+ tools:context="test.motionprediction.MainActivity">
+
+ <test.motionprediction.DrawingView
+ android:layout_width="match_parent"
+ android:layout_height="match_parent"
+ android:id="@+id/output" />
+
+</LinearLayout>
diff --git a/tests/MotionPrediction/res/mipmap-hdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-hdpi/ic_launcher.png
new file mode 100644
index 0000000..cde69bc
--- /dev/null
+++ b/tests/MotionPrediction/res/mipmap-hdpi/ic_launcher.png
Binary files differ
diff --git a/tests/MotionPrediction/res/mipmap-mdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-mdpi/ic_launcher.png
new file mode 100644
index 0000000..c133a0c
--- /dev/null
+++ b/tests/MotionPrediction/res/mipmap-mdpi/ic_launcher.png
Binary files differ
diff --git a/tests/MotionPrediction/res/mipmap-xhdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-xhdpi/ic_launcher.png
new file mode 100644
index 0000000..bfa42f0
--- /dev/null
+++ b/tests/MotionPrediction/res/mipmap-xhdpi/ic_launcher.png
Binary files differ
diff --git a/tests/MotionPrediction/res/mipmap-xxhdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-xxhdpi/ic_launcher.png
new file mode 100644
index 0000000..324e72c
--- /dev/null
+++ b/tests/MotionPrediction/res/mipmap-xxhdpi/ic_launcher.png
Binary files differ
diff --git a/tests/MotionPrediction/res/mipmap-xxxhdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-xxxhdpi/ic_launcher.png
new file mode 100644
index 0000000..aee44e1
--- /dev/null
+++ b/tests/MotionPrediction/res/mipmap-xxxhdpi/ic_launcher.png
Binary files differ
diff --git a/tests/MotionPrediction/res/values-w820dp/dimens.xml b/tests/MotionPrediction/res/values-w820dp/dimens.xml
new file mode 100644
index 0000000..95669e6
--- /dev/null
+++ b/tests/MotionPrediction/res/values-w820dp/dimens.xml
@@ -0,0 +1,20 @@
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<resources>
+ <!-- Example customization of dimensions originally defined in res/values/dimens.xml
+ (such as screen margins) for screens with more than 820dp of available width. This
+ would include 7" and 10" devices in landscape (~960dp and ~1280dp respectively). -->
+ <dimen name="activity_horizontal_margin">64dp</dimen>
+</resources>
diff --git a/tests/MotionPrediction/res/values/colors.xml b/tests/MotionPrediction/res/values/colors.xml
new file mode 100644
index 0000000..139eb1d
--- /dev/null
+++ b/tests/MotionPrediction/res/values/colors.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<resources>
+ <color name="colorPrimary">#3F51B5</color>
+ <color name="colorPrimaryDark">#303F9F</color>
+ <color name="colorAccent">#FF4081</color>
+</resources>
diff --git a/tests/MotionPrediction/res/values/dimens.xml b/tests/MotionPrediction/res/values/dimens.xml
new file mode 100644
index 0000000..d26136f
--- /dev/null
+++ b/tests/MotionPrediction/res/values/dimens.xml
@@ -0,0 +1,19 @@
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<resources>
+ <!-- Default screen margins, per the Android Design guidelines. -->
+ <dimen name="activity_horizontal_margin">16dp</dimen>
+ <dimen name="activity_vertical_margin">16dp</dimen>
+</resources>
diff --git a/tests/MotionPrediction/res/values/strings.xml b/tests/MotionPrediction/res/values/strings.xml
new file mode 100644
index 0000000..16a2bdf
--- /dev/null
+++ b/tests/MotionPrediction/res/values/strings.xml
@@ -0,0 +1,17 @@
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<resources>
+ <string name="app_name">Motion Prediction</string>
+</resources>
diff --git a/tests/MotionPrediction/res/values/styles.xml b/tests/MotionPrediction/res/values/styles.xml
new file mode 100644
index 0000000..cfb5e3d
--- /dev/null
+++ b/tests/MotionPrediction/res/values/styles.xml
@@ -0,0 +1,23 @@
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<resources>
+ <!-- Base application theme. -->
+ <style name="AppTheme" parent="@android:style/Theme.Material.Light.DarkActionBar">
+ <!-- Customize your theme here. -->
+ <item name="android:colorPrimary">@color/colorPrimary</item>
+ <item name="android:colorPrimaryDark">@color/colorPrimaryDark</item>
+ <item name="android:colorAccent">@color/colorAccent</item>
+ </style>
+</resources>
diff --git a/tests/MotionPrediction/src/test/motionprediction/DrawingView.kt b/tests/MotionPrediction/src/test/motionprediction/DrawingView.kt
new file mode 100644
index 0000000..f529bf7
--- /dev/null
+++ b/tests/MotionPrediction/src/test/motionprediction/DrawingView.kt
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.motionprediction
+
+import android.content.Context
+import android.graphics.Canvas
+import android.graphics.Color
+import android.graphics.Paint
+import android.util.AttributeSet
+import android.view.MotionEvent
+import android.view.MotionEvent.ACTION_DOWN
+import android.view.MotionPredictor
+import android.view.View
+
+import java.util.Vector
+
+private fun drawLine(canvas: Canvas, from: MotionEvent, to: MotionEvent, paint: Paint) {
+ canvas.apply {
+ val x0 = from.getX()
+ val y0 = from.getY()
+ val x1 = to.getX()
+ val y1 = to.getY()
+ // TODO: handle historical data
+ drawLine(x0, y0, x1, y1, paint)
+ }
+}
+
+/**
+ * Draw the current stroke and predicted values
+ */
+class DrawingView(context: Context, attrs: AttributeSet) : View(context, attrs) {
+ private val TAG = "DrawingView"
+
+ val events: MutableMap<Int, Vector<MotionEvent>> = mutableMapOf<Int, Vector<MotionEvent>>()
+
+ var isPredictionAvailable = false
+ private val predictor = MotionPredictor(getContext())
+
+ private var predictionPaint = Paint()
+ private var realPaint = Paint()
+
+ init {
+ setBackgroundColor(Color.WHITE)
+ predictionPaint.color = Color.BLACK
+ predictionPaint.setStrokeWidth(5f)
+ realPaint.color = Color.RED
+ realPaint.setStrokeWidth(5f)
+ }
+
+ private fun addEvent(event: MotionEvent) {
+ if (event.getActionMasked() == ACTION_DOWN) {
+ events.remove(event.deviceId)
+ }
+ var vec = events.getOrPut(event.deviceId) { Vector<MotionEvent>() }
+ vec.add(MotionEvent.obtain(event))
+ predictor.record(event)
+ invalidate()
+ }
+
+ public override fun onTouchEvent(event: MotionEvent): Boolean {
+ isPredictionAvailable = predictor.isPredictionAvailable(event.getDeviceId(),
+ event.getSource())
+ addEvent(event)
+ return true
+ }
+
+ public override fun onDraw(canvas: Canvas) {
+ super.onDraw(canvas)
+ if (!isPredictionAvailable) {
+ canvas.apply {
+ drawRect(0f, 0f, 200f, 200f, realPaint)
+ }
+ }
+
+ var eventTime = 0L
+
+ // Draw real events
+ for ((_, vec) in events ) {
+ for (i in 1 until vec.size) {
+ drawLine(canvas, vec[i - 1], vec[i], realPaint)
+ }
+ eventTime = vec.lastElement().eventTime
+ }
+
+ // Draw predictions. Convert to nanos and hardcode to +20ms into the future
+ val predictionList = predictor.predict(eventTime * 1000000 + 20000000)
+ for (prediction in predictionList) {
+ val realEvents = events.get(prediction.deviceId)!!
+ drawLine(canvas, realEvents[realEvents.size - 1], prediction, predictionPaint)
+ }
+ }
+}
diff --git a/tests/MotionPrediction/src/test/motionprediction/MainActivity.kt b/tests/MotionPrediction/src/test/motionprediction/MainActivity.kt
new file mode 100644
index 0000000..cec2c06
--- /dev/null
+++ b/tests/MotionPrediction/src/test/motionprediction/MainActivity.kt
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.motionprediction
+
+import android.app.Activity
+import android.os.Bundle
+
+class MainActivity : Activity() {
+ val TAG = "MotionPrediction"
+
+ override fun onCreate(savedInstanceState: Bundle?) {
+ super.onCreate(savedInstanceState)
+ setContentView(R.layout.activity_main)
+ }
+}
