Use the cloned window's layer stack space for raw coordinates
Use the cloned window's layer stack (screen) space as the
raw coordinate space for input going to clones. This ensures the raw
coordinates are consistent between the clone and cloned windows while
mirroring.
Flag: com.android.input.flags.use_cloned_screen_coordinates_as_raw
Test: Presubmit
Bug: 377846505
Change-Id: I7370ac64585315d9c754eaa1eb0077c8555fc6a9
diff --git a/libs/gui/WindowInfo.cpp b/libs/gui/WindowInfo.cpp
index 82d2554..6a32d55 100644
--- a/libs/gui/WindowInfo.cpp
+++ b/libs/gui/WindowInfo.cpp
@@ -59,6 +59,32 @@
return out;
}
+status_t writeTransform(android::Parcel* parcel, const ui::Transform& transform) {
+ return parcel->writeFloat(transform.dsdx()) ?:
+ parcel->writeFloat(transform.dtdx()) ?:
+ parcel->writeFloat(transform.tx()) ?:
+ parcel->writeFloat(transform.dtdy()) ?:
+ parcel->writeFloat(transform.dsdy()) ?:
+ parcel->writeFloat(transform.ty());
+}
+
+status_t readTransform(const android::Parcel* parcel, ui::Transform& transform) {
+ float dsdx, dtdx, tx, dtdy, dsdy, ty;
+
+ const status_t status = parcel->readFloat(&dsdx) ?:
+ parcel->readFloat(&dtdx) ?:
+ parcel->readFloat(&tx) ?:
+ parcel->readFloat(&dtdy) ?:
+ parcel->readFloat(&dsdy) ?:
+ parcel->readFloat(&ty);
+ if (status != OK) {
+ return status;
+ }
+
+ transform.set({dsdx, dtdx, tx, dtdy, dsdy, ty, 0, 0, 1});
+ return OK;
+}
+
} // namespace
void WindowInfo::setInputConfig(ftl::Flags<InputConfig> config, bool value) {
@@ -135,12 +161,7 @@
parcel->writeInt32(surfaceInset) ?:
parcel->writeFloat(globalScaleFactor) ?:
parcel->writeFloat(alpha) ?:
- parcel->writeFloat(transform.dsdx()) ?:
- parcel->writeFloat(transform.dtdx()) ?:
- parcel->writeFloat(transform.tx()) ?:
- parcel->writeFloat(transform.dtdy()) ?:
- parcel->writeFloat(transform.dsdy()) ?:
- parcel->writeFloat(transform.ty()) ?:
+ writeTransform(parcel, transform) ?:
parcel->writeInt32(static_cast<int32_t>(touchOcclusionMode)) ?:
parcel->writeInt32(ownerPid.val()) ?:
parcel->writeInt32(ownerUid.val()) ?:
@@ -153,8 +174,12 @@
parcel->writeStrongBinder(touchableRegionCropHandle.promote()) ?:
parcel->writeStrongBinder(windowToken) ?:
parcel->writeStrongBinder(focusTransferTarget) ?:
- parcel->writeBool(canOccludePresentation);
+ parcel->writeBool(canOccludePresentation) ?:
+ parcel->writeBool(cloneLayerStackTransform.has_value());
// clang-format on
+ if (cloneLayerStackTransform) {
+ status = status ?: writeTransform(parcel, *cloneLayerStackTransform);
+ }
return status;
}
@@ -174,10 +199,10 @@
return status;
}
- float dsdx, dtdx, tx, dtdy, dsdy, ty;
int32_t lpFlags, lpType, touchOcclusionModeInt, inputConfigInt, ownerPidInt, ownerUidInt,
displayIdInt;
sp<IBinder> touchableRegionCropHandleSp;
+ bool hasCloneLayerStackTransform = false;
// clang-format off
status = parcel->readInt32(&lpFlags) ?:
@@ -188,12 +213,7 @@
parcel->readInt32(&surfaceInset) ?:
parcel->readFloat(&globalScaleFactor) ?:
parcel->readFloat(&alpha) ?:
- parcel->readFloat(&dsdx) ?:
- parcel->readFloat(&dtdx) ?:
- parcel->readFloat(&tx) ?:
- parcel->readFloat(&dtdy) ?:
- parcel->readFloat(&dsdy) ?:
- parcel->readFloat(&ty) ?:
+ readTransform(parcel, /*byRef*/ transform) ?:
parcel->readInt32(&touchOcclusionModeInt) ?:
parcel->readInt32(&ownerPidInt) ?:
parcel->readInt32(&ownerUidInt) ?:
@@ -206,8 +226,8 @@
parcel->readNullableStrongBinder(&touchableRegionCropHandleSp) ?:
parcel->readNullableStrongBinder(&windowToken) ?:
parcel->readNullableStrongBinder(&focusTransferTarget) ?:
- parcel->readBool(&canOccludePresentation);
-
+ parcel->readBool(&canOccludePresentation)?:
+ parcel->readBool(&hasCloneLayerStackTransform);
// clang-format on
if (status != OK) {
@@ -216,7 +236,6 @@
layoutParamsFlags = ftl::Flags<Flag>(lpFlags);
layoutParamsType = static_cast<Type>(lpType);
- transform.set({dsdx, dtdx, tx, dtdy, dsdy, ty, 0, 0, 1});
touchOcclusionMode = static_cast<TouchOcclusionMode>(touchOcclusionModeInt);
inputConfig = ftl::Flags<InputConfig>(inputConfigInt);
ownerPid = Pid{ownerPidInt};
@@ -224,6 +243,15 @@
touchableRegionCropHandle = touchableRegionCropHandleSp;
displayId = ui::LogicalDisplayId{displayIdInt};
+ cloneLayerStackTransform =
+ hasCloneLayerStackTransform ? std::make_optional<ui::Transform>() : std::nullopt;
+ if (cloneLayerStackTransform) {
+ status = readTransform(parcel, /*byRef*/ *cloneLayerStackTransform);
+ if (status != OK) {
+ return status;
+ }
+ }
+
return OK;
}
diff --git a/libs/gui/include/gui/WindowInfo.h b/libs/gui/include/gui/WindowInfo.h
index eb3be55..b48e696 100644
--- a/libs/gui/include/gui/WindowInfo.h
+++ b/libs/gui/include/gui/WindowInfo.h
@@ -220,9 +220,14 @@
// An alpha of 1.0 means fully opaque and 0.0 means fully transparent.
float alpha;
- // Transform applied to individual windows.
+ // Transform applied to individual windows for input.
+ // Maps display coordinates to the window's input coordinate space.
ui::Transform transform;
+ // Transform applied to get to the layer stack space of the cloned window for input.
+ // Maps display coordinates of the clone window to the layer stack space of the cloned window.
+ std::optional<ui::Transform> cloneLayerStackTransform;
+
/*
* This is filled in by the WM relative to the frame and then translated
* to absolute coordinates by SurfaceFlinger once the frame is computed.
diff --git a/libs/gui/tests/WindowInfo_test.cpp b/libs/gui/tests/WindowInfo_test.cpp
index ce22082..e3f9a07 100644
--- a/libs/gui/tests/WindowInfo_test.cpp
+++ b/libs/gui/tests/WindowInfo_test.cpp
@@ -40,7 +40,18 @@
ASSERT_EQ(OK, i.writeToParcel(&p));
p.setDataPosition(0);
i2.readFromParcel(&p);
- ASSERT_TRUE(i2.token == nullptr);
+ ASSERT_EQ(i2.token, nullptr);
+}
+
+TEST(WindowInfo, ParcellingWithoutCloneTransform) {
+ WindowInfo i, i2;
+ i.cloneLayerStackTransform.reset();
+
+ Parcel p;
+ ASSERT_EQ(OK, i.writeToParcel(&p));
+ p.setDataPosition(0);
+ i2.readFromParcel(&p);
+ ASSERT_EQ(i2.cloneLayerStackTransform, std::nullopt);
}
TEST(WindowInfo, Parcelling) {
@@ -71,6 +82,8 @@
i.applicationInfo.token = new BBinder();
i.applicationInfo.dispatchingTimeoutMillis = 0x12345678ABCD;
i.focusTransferTarget = new BBinder();
+ i.cloneLayerStackTransform = ui::Transform();
+ i.cloneLayerStackTransform->set({5, -1, 100, 4, 0, 40, 0, 0, 1});
Parcel p;
i.writeToParcel(&p);
@@ -100,6 +113,7 @@
ASSERT_EQ(i.touchableRegionCropHandle, i2.touchableRegionCropHandle);
ASSERT_EQ(i.applicationInfo, i2.applicationInfo);
ASSERT_EQ(i.focusTransferTarget, i2.focusTransferTarget);
+ ASSERT_EQ(i.cloneLayerStackTransform, i2.cloneLayerStackTransform);
}
TEST(InputApplicationInfo, Parcelling) {
diff --git a/libs/input/input_flags.aconfig b/libs/input/input_flags.aconfig
index 6cdd249..5da110e 100644
--- a/libs/input/input_flags.aconfig
+++ b/libs/input/input_flags.aconfig
@@ -224,3 +224,13 @@
description: "Allow cursor to transition across multiple connected displays"
bug: "362719483"
}
+
+flag {
+ name: "use_cloned_screen_coordinates_as_raw"
+ namespace: "input"
+ description: "Use the cloned window's layer stack (screen) space as the raw coordinate space for input going to clones"
+ bug: "377846505"
+ metadata {
+ purpose: PURPOSE_BUGFIX
+ }
+}
diff --git a/services/inputflinger/dispatcher/InputDispatcher.cpp b/services/inputflinger/dispatcher/InputDispatcher.cpp
index bfab666..28422d6 100644
--- a/services/inputflinger/dispatcher/InputDispatcher.cpp
+++ b/services/inputflinger/dispatcher/InputDispatcher.cpp
@@ -3013,11 +3013,10 @@
windowHandle->getName().c_str());
return;
}
- inputTargets.push_back(createInputTarget(connection, windowHandle, dispatchMode,
- targetFlags,
- mWindowInfos.getDisplayTransform(
- windowHandle->getInfo()->displayId),
- firstDownTimeInTarget));
+ inputTargets.push_back(
+ createInputTarget(connection, windowHandle, dispatchMode, targetFlags,
+ mWindowInfos.getRawTransform(*windowHandle->getInfo()),
+ firstDownTimeInTarget));
it = inputTargets.end() - 1;
}
@@ -3068,11 +3067,10 @@
windowHandle->getName().c_str());
return;
}
- inputTargets.push_back(createInputTarget(connection, windowHandle, dispatchMode,
- targetFlags,
- mWindowInfos.getDisplayTransform(
- windowHandle->getInfo()->displayId),
- firstDownTimeInTarget));
+ inputTargets.push_back(
+ createInputTarget(connection, windowHandle, dispatchMode, targetFlags,
+ mWindowInfos.getRawTransform(*windowHandle->getInfo()),
+ firstDownTimeInTarget));
it = inputTargets.end() - 1;
}
@@ -3104,7 +3102,8 @@
for (const Monitor& monitor : selectResponsiveMonitorsLocked(monitorsIt->second)) {
InputTarget target{monitor.connection};
// target.firstDownTimeInTarget is not set for global monitors. It is only required in split
- // touch and global monitoring works as intended even without setting firstDownTimeInTarget
+ // touch and global monitoring works as intended even without setting firstDownTimeInTarget.
+ // Since global monitors don't have windows, use the display transform as the raw transform.
target.rawTransform = mWindowInfos.getDisplayTransform(displayId);
target.setDefaultPointerTransform(target.rawTransform);
inputTargets.push_back(target);
@@ -4291,6 +4290,7 @@
motionEntry.downTime, targets);
} else {
targets.emplace_back(fallbackTarget);
+ // Since we don't have a window, use the display transform as the raw transform.
const ui::Transform displayTransform =
mWindowInfos.getDisplayTransform(motionEntry.displayId);
targets.back().rawTransform = displayTransform;
@@ -4376,6 +4376,7 @@
targets);
} else {
targets.emplace_back(connection, targetFlags);
+ // Since we don't have a window, use the display transform as the raw transform.
const ui::Transform displayTransform =
mWindowInfos.getDisplayTransform(motionEntry.displayId);
targets.back().rawTransform = displayTransform;
@@ -5289,6 +5290,16 @@
: kIdentityTransform;
}
+ui::Transform InputDispatcher::DispatcherWindowInfo::getRawTransform(
+ const android::gui::WindowInfo& windowInfo) const {
+ // If the window has a cloneLayerStackTransform, always use it as the transform for the "getRaw"
+ // APIs. If not, fall back to using the DisplayInfo transform of the window's display.
+ return (input_flags::use_cloned_screen_coordinates_as_raw() &&
+ windowInfo.cloneLayerStackTransform)
+ ? *windowInfo.cloneLayerStackTransform
+ : getDisplayTransform(windowInfo.displayId);
+}
+
std::string InputDispatcher::DispatcherWindowInfo::dumpDisplayAndWindowInfo() const {
std::string dump;
if (!mWindowHandlesByDisplay.empty()) {
@@ -6567,9 +6578,8 @@
createInputTarget(connection, windowHandle,
InputTarget::DispatchMode::AS_IS,
dispatchEntry->targetFlags,
- mWindowInfos.getDisplayTransform(
- windowHandle->getInfo()
- ->displayId),
+ mWindowInfos.getRawTransform(
+ *windowHandle->getInfo()),
downTime));
}
}
diff --git a/services/inputflinger/dispatcher/InputDispatcher.h b/services/inputflinger/dispatcher/InputDispatcher.h
index fd550dd..8485b44 100644
--- a/services/inputflinger/dispatcher/InputDispatcher.h
+++ b/services/inputflinger/dispatcher/InputDispatcher.h
@@ -384,6 +384,9 @@
// Get the transform for display, returns Identity-transform if display is missing.
ui::Transform getDisplayTransform(ui::LogicalDisplayId displayId) const;
+ // Get the raw transform to use for motion events going to the given window.
+ ui::Transform getRawTransform(const android::gui::WindowInfo&) const;
+
// Lookup for WindowInfoHandle from token and optionally a display-id. In cases where
// display-id is not provided lookup is done for all displays.
sp<android::gui::WindowInfoHandle> findWindowHandle(
diff --git a/services/inputflinger/tests/InputDispatcher_test.cpp b/services/inputflinger/tests/InputDispatcher_test.cpp
index 3413caa..a5b8e4b 100644
--- a/services/inputflinger/tests/InputDispatcher_test.cpp
+++ b/services/inputflinger/tests/InputDispatcher_test.cpp
@@ -6501,19 +6501,47 @@
ui::LogicalDisplayId::DEFAULT, {PointF{150, 220}}));
firstWindow->assertNoEvents();
- std::unique_ptr<MotionEvent> event = secondWindow->consumeMotionEvent();
- ASSERT_NE(nullptr, event);
- EXPECT_EQ(AMOTION_EVENT_ACTION_DOWN, event->getAction());
+ secondWindow->consumeMotionEvent(
+ AllOf(WithMotionAction(ACTION_DOWN),
+ // Ensure that the events from the "getRaw" API are in logical display
+ // coordinates, which has an x-scale of 2 and y-scale of 4.
+ WithRawCoords(300, 880),
+ // Ensure that the x and y values are in the window's coordinate space.
+ // The left-top of the second window is at (100, 200) in display space, which is
+ // (200, 800) in the logical display space. This will be the origin of the window
+ // space.
+ WithCoords(100, 80)));
+}
- // Ensure that the events from the "getRaw" API are in logical display coordinates.
- EXPECT_EQ(300, event->getRawX(0));
- EXPECT_EQ(880, event->getRawY(0));
+TEST_F(InputDispatcherDisplayProjectionTest, UseCloneLayerStackTransformForRawCoordinates) {
+ SCOPED_FLAG_OVERRIDE(use_cloned_screen_coordinates_as_raw, true);
- // Ensure that the x and y values are in the window's coordinate space.
- // The left-top of the second window is at (100, 200) in display space, which is (200, 800) in
- // the logical display space. This will be the origin of the window space.
- EXPECT_EQ(100, event->getX(0));
- EXPECT_EQ(80, event->getY(0));
+ auto [firstWindow, secondWindow] = setupScaledDisplayScenario();
+
+ const std::array<float, 9> matrix = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 0.0, 0.0, 1.0};
+ ui::Transform secondDisplayTransform;
+ secondDisplayTransform.set(matrix);
+ addDisplayInfo(SECOND_DISPLAY_ID, secondDisplayTransform);
+
+ // When a clone layer stack transform is provided for a window, we should use that as the
+ // "display transform" for input going to that window.
+ sp<FakeWindowHandle> secondWindowClone = secondWindow->clone(SECOND_DISPLAY_ID);
+ secondWindowClone->editInfo()->cloneLayerStackTransform = ui::Transform();
+ secondWindowClone->editInfo()->cloneLayerStackTransform->set(0.5, 0, 0, 0.25);
+ addWindow(secondWindowClone);
+
+ // Touch down on the clone window, and ensure its raw coordinates use
+ // the clone layer stack transform.
+ mDispatcher->notifyMotion(generateMotionArgs(ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN,
+ SECOND_DISPLAY_ID, {PointF{150, 220}}));
+ secondWindowClone->consumeMotionEvent(
+ AllOf(WithMotionAction(ACTION_DOWN),
+ // Ensure the x and y coordinates are in the window's coordinate space.
+ // See previous test case for calculation.
+ WithCoords(100, 80),
+ // Ensure the "getRaw" API uses the clone layer stack transform when it is
+ // provided for the window. It has an x-scale of 0.5 and y-scale of 0.25.
+ WithRawCoords(75, 55)));
}
TEST_F(InputDispatcherDisplayProjectionTest, CancelMotionWithCorrectCoordinates) {
diff --git a/services/surfaceflinger/FrontEnd/LayerHierarchy.h b/services/surfaceflinger/FrontEnd/LayerHierarchy.h
index 47d0041..4fdbae1 100644
--- a/services/surfaceflinger/FrontEnd/LayerHierarchy.h
+++ b/services/surfaceflinger/FrontEnd/LayerHierarchy.h
@@ -102,6 +102,8 @@
// Returns true if the node is a clone.
bool isClone() const { return !mirrorRootIds.empty(); }
+ TraversalPath getClonedFrom() const { return {.id = id, .variant = variant}; }
+
bool operator==(const TraversalPath& other) const {
return id == other.id && mirrorRootIds == other.mirrorRootIds;
}
diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
index 022588d..d0dc065 100644
--- a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
+++ b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
@@ -16,6 +16,8 @@
// #define LOG_NDEBUG 0
#define ATRACE_TAG ATRACE_TAG_GRAPHICS
+#include "FrontEnd/LayerSnapshot.h"
+#include "ui/Transform.h"
#undef LOG_TAG
#define LOG_TAG "SurfaceFlinger"
@@ -1205,13 +1207,27 @@
snapshot.inputInfo.contentSize = {snapshot.croppedBufferSize.getHeight(),
snapshot.croppedBufferSize.getWidth()};
- // If the layer is a clone, we need to crop the input region to cloned root to prevent
- // touches from going outside the cloned area.
+ snapshot.inputInfo.cloneLayerStackTransform.reset();
+
if (path.isClone()) {
snapshot.inputInfo.inputConfig |= InputConfig::CLONE;
// Cloned layers shouldn't handle watch outside since their z order is not determined by
// WM or the client.
snapshot.inputInfo.inputConfig.clear(InputConfig::WATCH_OUTSIDE_TOUCH);
+
+ // Compute the transform that maps the clone's display to the layer stack space of the
+ // cloned window.
+ const LayerSnapshot* clonedSnapshot = getSnapshot(path.getClonedFrom());
+ if (clonedSnapshot != nullptr) {
+ const auto& [clonedInputBounds, s] =
+ getInputBounds(*clonedSnapshot, /*fillParentBounds=*/false);
+ ui::Transform inputToLayer;
+ inputToLayer.set(clonedInputBounds.left, clonedInputBounds.top);
+ const ui::Transform& layerToLayerStack = getInputTransform(*clonedSnapshot);
+ const auto& displayToInput = snapshot.inputInfo.transform;
+ snapshot.inputInfo.cloneLayerStackTransform =
+ layerToLayerStack * inputToLayer * displayToInput;
+ }
}
}