Support mouse in VM
1. By requestPointerCapture, make the app exclusively use the mouse event.
2. Add backgroundTouchView because it looks like set onTouch and
onCapturedPointer listeners to the same view doesn't work as expected
3. requestUnbufferedDispatch because of pointer latency
Bug: 325930922
Test: move/click/wheel
Change-Id: I44cdbda923cb18467ab611c342a3ae4e4112403b
diff --git a/java/framework/src/android/system/virtualmachine/VirtualMachine.java b/java/framework/src/android/system/virtualmachine/VirtualMachine.java
index 1076219..2f6e306 100644
--- a/java/framework/src/android/system/virtualmachine/VirtualMachine.java
+++ b/java/framework/src/android/system/virtualmachine/VirtualMachine.java
@@ -164,6 +164,7 @@
private ParcelFileDescriptor mTouchSock;
private ParcelFileDescriptor mKeySock;
+ private ParcelFileDescriptor mMouseSock;
/**
* Status of a virtual machine
@@ -881,6 +882,13 @@
k.pfd = pfds[1];
inputDevices.add(InputDevice.keyboard(k));
}
+ if (vmConfig.getCustomImageConfig().useMouse()) {
+ ParcelFileDescriptor[] pfds = ParcelFileDescriptor.createSocketPair();
+ mMouseSock = pfds[0];
+ InputDevice.Mouse m = new InputDevice.Mouse();
+ m.pfd = pfds[1];
+ inputDevices.add(InputDevice.mouse(m));
+ }
}
rawConfig.inputDevices = inputDevices.toArray(new InputDevice[0]);
@@ -909,6 +917,94 @@
}
/** @hide */
+ public boolean sendMouseEvent(MotionEvent event) {
+ if (mMouseSock == null) {
+ Log.d(TAG, "mMouseSock == null");
+ return false;
+ }
+ // from include/uapi/linux/input-event-codes.h in the kernel.
+ short EV_SYN = 0x00;
+ short EV_REL = 0x02;
+ short EV_KEY = 0x01;
+ short REL_X = 0x00;
+ short REL_Y = 0x01;
+ short SYN_REPORT = 0x00;
+ switch (event.getAction()) {
+ case MotionEvent.ACTION_MOVE:
+ int x = (int) event.getX();
+ int y = (int) event.getY();
+ return writeEventsToSock(
+ mMouseSock,
+ Arrays.asList(
+ new InputEvent(EV_REL, REL_X, x),
+ new InputEvent(EV_REL, REL_Y, y),
+ new InputEvent(EV_SYN, SYN_REPORT, 0)));
+ case MotionEvent.ACTION_BUTTON_PRESS:
+ case MotionEvent.ACTION_BUTTON_RELEASE:
+ short BTN_LEFT = 0x110;
+ short BTN_RIGHT = 0x111;
+ short BTN_MIDDLE = 0x112;
+ short keyCode;
+ switch (event.getActionButton()) {
+ case MotionEvent.BUTTON_PRIMARY:
+ keyCode = BTN_LEFT;
+ break;
+ case MotionEvent.BUTTON_SECONDARY:
+ keyCode = BTN_RIGHT;
+ break;
+ case MotionEvent.BUTTON_TERTIARY:
+ keyCode = BTN_MIDDLE;
+ break;
+ default:
+ Log.d(TAG, event.toString());
+ return false;
+ }
+ return writeEventsToSock(
+ mMouseSock,
+ Arrays.asList(
+ new InputEvent(
+ EV_KEY,
+ keyCode,
+ event.getAction() == MotionEvent.ACTION_BUTTON_PRESS
+ ? 1
+ : 0),
+ new InputEvent(EV_SYN, SYN_REPORT, 0)));
+ case MotionEvent.ACTION_SCROLL:
+ short REL_HWHEEL = 0x06;
+ short REL_WHEEL = 0x08;
+ int scrollX = (int) event.getAxisValue(MotionEvent.AXIS_HSCROLL);
+ int scrollY = (int) event.getAxisValue(MotionEvent.AXIS_VSCROLL);
+ boolean status = true;
+ if (scrollX != 0) {
+ status &=
+ writeEventsToSock(
+ mMouseSock,
+ Arrays.asList(
+ new InputEvent(EV_REL, REL_HWHEEL, scrollX),
+ new InputEvent(EV_SYN, SYN_REPORT, 0)));
+ } else if (scrollY != 0) {
+ status &=
+ writeEventsToSock(
+ mMouseSock,
+ Arrays.asList(
+ new InputEvent(EV_REL, REL_WHEEL, scrollY),
+ new InputEvent(EV_SYN, SYN_REPORT, 0)));
+ } else {
+ Log.d(TAG, event.toString());
+ return false;
+ }
+ return status;
+ case MotionEvent.ACTION_UP:
+ case MotionEvent.ACTION_DOWN:
+ // Ignored because it's handled by ACTION_BUTTON_PRESS and ACTION_BUTTON_RELEASE
+ return true;
+ default:
+ Log.d(TAG, event.toString());
+ return false;
+ }
+ }
+
+ /** @hide */
public boolean sendSingleTouchEvent(MotionEvent event) {
if (mTouchSock == null) {
Log.d(TAG, "mTouchSock == null");
diff --git a/java/framework/src/android/system/virtualmachine/VirtualMachineCustomImageConfig.java b/java/framework/src/android/system/virtualmachine/VirtualMachineCustomImageConfig.java
index 8ec9d2c..2fcad20 100644
--- a/java/framework/src/android/system/virtualmachine/VirtualMachineCustomImageConfig.java
+++ b/java/framework/src/android/system/virtualmachine/VirtualMachineCustomImageConfig.java
@@ -35,6 +35,7 @@
private static final String KEY_DISPLAY_CONFIG = "display_config";
private static final String KEY_TOUCH = "touch";
private static final String KEY_KEYBOARD = "keyboard";
+ private static final String KEY_MOUSE = "mouse";
@Nullable private final String name;
@Nullable private final String kernelPath;
@@ -45,6 +46,7 @@
@Nullable private final DisplayConfig displayConfig;
private final boolean touch;
private final boolean keyboard;
+ private final boolean mouse;
@Nullable
public Disk[] getDisks() {
@@ -84,6 +86,10 @@
return keyboard;
}
+ public boolean useMouse() {
+ return mouse;
+ }
+
/** @hide */
public VirtualMachineCustomImageConfig(
String name,
@@ -94,7 +100,8 @@
Disk[] disks,
DisplayConfig displayConfig,
boolean touch,
- boolean keyboard) {
+ boolean keyboard,
+ boolean mouse) {
this.name = name;
this.kernelPath = kernelPath;
this.initrdPath = initrdPath;
@@ -104,6 +111,7 @@
this.displayConfig = displayConfig;
this.touch = touch;
this.keyboard = keyboard;
+ this.mouse = mouse;
}
static VirtualMachineCustomImageConfig from(PersistableBundle customImageConfigBundle) {
@@ -133,6 +141,7 @@
builder.setDisplayConfig(DisplayConfig.from(displayConfigPb));
builder.useTouch(customImageConfigBundle.getBoolean(KEY_TOUCH));
builder.useKeyboard(customImageConfigBundle.getBoolean(KEY_KEYBOARD));
+ builder.useMouse(customImageConfigBundle.getBoolean(KEY_MOUSE));
return builder.build();
}
@@ -163,6 +172,7 @@
.orElse(null));
pb.putBoolean(KEY_TOUCH, touch);
pb.putBoolean(KEY_KEYBOARD, keyboard);
+ pb.putBoolean(KEY_MOUSE, mouse);
return pb;
}
@@ -213,6 +223,7 @@
private DisplayConfig displayConfig;
private boolean touch;
private boolean keyboard;
+ private boolean mouse;
/** @hide */
public Builder() {}
@@ -272,6 +283,12 @@
}
/** @hide */
+ public Builder useMouse(boolean mouse) {
+ this.mouse = mouse;
+ return this;
+ }
+
+ /** @hide */
public VirtualMachineCustomImageConfig build() {
return new VirtualMachineCustomImageConfig(
this.name,
@@ -282,7 +299,8 @@
this.disks.toArray(new Disk[0]),
displayConfig,
touch,
- keyboard);
+ keyboard,
+ mouse);
}
}
diff --git a/virtualizationmanager/src/aidl.rs b/virtualizationmanager/src/aidl.rs
index a245e11..653281d 100644
--- a/virtualizationmanager/src/aidl.rs
+++ b/virtualizationmanager/src/aidl.rs
@@ -757,6 +757,9 @@
InputDevice::Keyboard(keyboard) => InputDeviceOption::Keyboard(clone_file(
keyboard.pfd.as_ref().ok_or(anyhow!("pfd should have value"))?,
)?),
+ InputDevice::Mouse(mouse) => InputDeviceOption::Mouse(clone_file(
+ mouse.pfd.as_ref().ok_or(anyhow!("pfd should have value"))?,
+ )?),
})
}
/// Given the configuration for a disk image, assembles the `DiskFile` to pass to crosvm.
diff --git a/virtualizationmanager/src/crosvm.rs b/virtualizationmanager/src/crosvm.rs
index b426051..d48ef7b 100644
--- a/virtualizationmanager/src/crosvm.rs
+++ b/virtualizationmanager/src/crosvm.rs
@@ -163,6 +163,7 @@
EvDev(File),
SingleTouch { file: File, width: u32, height: u32, name: Option<String> },
Keyboard(File),
+ Mouse(File),
}
type VfioDevice = Strong<dyn IBoundDevice>;
@@ -996,6 +997,9 @@
InputDeviceOption::Keyboard(file) => {
format!("keyboard[path={}]", add_preserved_fd(&mut preserved_fds, file))
}
+ InputDeviceOption::Mouse(file) => {
+ format!("mouse[path={}]", add_preserved_fd(&mut preserved_fds, file))
+ }
InputDeviceOption::SingleTouch { file, width, height, name } => format!(
"single-touch[path={},width={},height={}{}]",
add_preserved_fd(&mut preserved_fds, file),
diff --git a/virtualizationservice/aidl/android/system/virtualizationservice/InputDevice.aidl b/virtualizationservice/aidl/android/system/virtualizationservice/InputDevice.aidl
index 712d6a9..56c5b6d 100644
--- a/virtualizationservice/aidl/android/system/virtualizationservice/InputDevice.aidl
+++ b/virtualizationservice/aidl/android/system/virtualizationservice/InputDevice.aidl
@@ -34,7 +34,12 @@
parcelable Keyboard {
ParcelFileDescriptor pfd;
}
+ // Mouse input
+ parcelable Mouse {
+ ParcelFileDescriptor pfd;
+ }
SingleTouch singleTouch;
EvDev evDev;
Keyboard keyboard;
+ Mouse mouse;
}
diff --git a/vmlauncher_app/java/com/android/virtualization/vmlauncher/MainActivity.java b/vmlauncher_app/java/com/android/virtualization/vmlauncher/MainActivity.java
index ec0f8e8..10f8bf6 100644
--- a/vmlauncher_app/java/com/android/virtualization/vmlauncher/MainActivity.java
+++ b/vmlauncher_app/java/com/android/virtualization/vmlauncher/MainActivity.java
@@ -35,9 +35,11 @@
import android.system.virtualmachine.VirtualMachineException;
import android.system.virtualmachine.VirtualMachineManager;
import android.view.Display;
+import android.view.InputDevice;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.KeyEvent;
+import android.view.View;
import android.view.WindowManager;
import android.view.WindowInsets;
import android.view.WindowInsetsController;
@@ -129,6 +131,7 @@
customImageConfigBuilder.setDisplayConfig(displayConfigBuilder.build());
customImageConfigBuilder.useTouch(true);
customImageConfigBuilder.useKeyboard(true);
+ customImageConfigBuilder.useMouse(true);
configBuilder.setCustomImageConfig(customImageConfigBuilder.build());
@@ -244,13 +247,22 @@
}
SurfaceView surfaceView = findViewById(R.id.surface_view);
- surfaceView.setOnTouchListener(
+ View backgroundTouchView = findViewById(R.id.background_touch_view);
+ backgroundTouchView.setOnTouchListener(
(v, event) -> {
if (mVirtualMachine == null) {
return false;
}
return mVirtualMachine.sendSingleTouchEvent(event);
});
+ surfaceView.requestUnbufferedDispatch(InputDevice.SOURCE_ANY);
+ surfaceView.setOnCapturedPointerListener(
+ (v, event) -> {
+ if (mVirtualMachine == null) {
+ return false;
+ }
+ return mVirtualMachine.sendMouseEvent(event);
+ });
surfaceView
.getHolder()
.addCallback(
@@ -292,6 +304,16 @@
windowInsetsController.hide(WindowInsets.Type.systemBars());
}
+ @Override
+ public void onWindowFocusChanged(boolean hasFocus) {
+ super.onWindowFocusChanged(hasFocus);
+ if (hasFocus) {
+ SurfaceView surfaceView = findViewById(R.id.surface_view);
+ Log.d(TAG, "requestPointerCapture()");
+ surfaceView.requestPointerCapture();
+ }
+ }
+
@FunctionalInterface
public interface RemoteExceptionCheckedFunction<T> {
void apply(T t) throws RemoteException;
diff --git a/vmlauncher_app/res/layout/activity_main.xml b/vmlauncher_app/res/layout/activity_main.xml
index 5fa2171..c588b19 100644
--- a/vmlauncher_app/res/layout/activity_main.xml
+++ b/vmlauncher_app/res/layout/activity_main.xml
@@ -1,14 +1,24 @@
<?xml version="1.0" encoding="utf-8"?>
-<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
tools:context=".MainActivity">
- <SurfaceView
+ <View
+ android:id="@+id/background_touch_view"
+ android:layout_width="match_parent"
+ android:layout_height="match_parent"
+ />
+ <SurfaceView
android:id="@+id/surface_view"
- android:focusable="true"
- android:layout_width="match_parent"
- android:layout_height="match_parent" />
+ android:layout_width="match_parent"
+ android:layout_height="match_parent"
+ android:focusable="true"
+ android:focusableInTouchMode="true"
+ android:focusedByDefault="true"
+ android:defaultFocusHighlightEnabled="true">
+ <requestFocus />
+ </SurfaceView>
-</LinearLayout>
+</FrameLayout>