Rotate TouchVideoFrames
When a screen orientation change happens, the touch coordinates are
adjusted in InputReader to accomodate this rotation. When the user is
holding the device, the origin (0, 0) is always at the top left of the
screen.
However, currently, the TouchVideoFrames are not being rotated. This
presents a problem, because the touch coordinates cannot be directly
matched to the heatmap, as received in the HAL.
To account for this, we rotate the touch video frame.
Test: atest libinput_tests inputflinger_tests
Bug: 123241238
Change-Id: Iff45c68b1d2b237d2b1657ed76f50bb23ef8468a
diff --git a/include/input/TouchVideoFrame.h b/include/input/TouchVideoFrame.h
index 566c334..b49c623 100644
--- a/include/input/TouchVideoFrame.h
+++ b/include/input/TouchVideoFrame.h
@@ -19,6 +19,7 @@
#include <stdint.h>
#include <sys/time.h>
+#include <ui/DisplayInfo.h>
#include <vector>
namespace android {
@@ -55,11 +56,23 @@
*/
const struct timeval& getTimestamp() const;
+ /**
+ * Rotate the video frame.
+ * The rotation value is an enum from ui/DisplayInfo.h
+ */
+ void rotate(int32_t orientation);
+
private:
uint32_t mHeight;
uint32_t mWidth;
std::vector<int16_t> mData;
struct timeval mTimestamp;
+
+ /**
+ * Common method for 90 degree and 270 degree rotation
+ */
+ void rotateQuarterTurn(bool clockwise);
+ void rotate180();
};
} // namespace android
diff --git a/libs/input/TouchVideoFrame.cpp b/libs/input/TouchVideoFrame.cpp
index 35cb4a7..8a4298a 100644
--- a/libs/input/TouchVideoFrame.cpp
+++ b/libs/input/TouchVideoFrame.cpp
@@ -39,4 +39,62 @@
const struct timeval& TouchVideoFrame::getTimestamp() const { return mTimestamp; }
+void TouchVideoFrame::rotate(int32_t orientation) {
+ switch (orientation) {
+ case DISPLAY_ORIENTATION_90:
+ rotateQuarterTurn(true /*clockwise*/);
+ break;
+ case DISPLAY_ORIENTATION_180:
+ rotate180();
+ break;
+ case DISPLAY_ORIENTATION_270:
+ rotateQuarterTurn(false /*clockwise*/);
+ break;
+ }
+}
+
+/**
+ * Rotate once clockwise by a quarter turn === rotate 90 degrees
+ * Rotate once counterclockwise by a quarter turn === rotate 270 degrees
+ * For a clockwise rotation:
+ * An element at position (i, j) is rotated to (j, height - i - 1)
+ * For a counterclockwise rotation:
+ * An element at position (i, j) is rotated to (width - j - 1, i)
+ */
+void TouchVideoFrame::rotateQuarterTurn(bool clockwise) {
+ std::vector<int16_t> rotated(mData.size());
+ for (size_t i = 0; i < mHeight; i++) {
+ for (size_t j = 0; j < mWidth; j++) {
+ size_t iRotated, jRotated;
+ if (clockwise) {
+ iRotated = j;
+ jRotated = mHeight - i - 1;
+ } else {
+ iRotated = mWidth - j - 1;
+ jRotated = i;
+ }
+ size_t indexRotated = iRotated * mHeight + jRotated;
+ rotated[indexRotated] = mData[i * mWidth + j];
+ }
+ }
+ mData = std::move(rotated);
+ std::swap(mHeight, mWidth);
+}
+
+/**
+ * An element at position (i, j) is rotated to (height - i - 1, width - j - 1)
+ * This is equivalent to moving element [i] to position [height * width - i - 1]
+ * Since element at [height * width - i - 1] would move to position [i],
+ * we can just swap elements [i] and [height * width - i - 1].
+ */
+void TouchVideoFrame::rotate180() {
+ if (mData.size() == 0) {
+ return;
+ }
+ // Just need to swap elements i and (height * width - 1 - i)
+ for (size_t i = 0; i < mData.size() / 2; i++) {
+ std::swap(mData[i], mData[mHeight * mWidth - 1 - i]);
+ }
+}
+
} // namespace android
diff --git a/libs/input/tests/TouchVideoFrame_test.cpp b/libs/input/tests/TouchVideoFrame_test.cpp
index 3c1c7f3..815424e 100644
--- a/libs/input/tests/TouchVideoFrame_test.cpp
+++ b/libs/input/tests/TouchVideoFrame_test.cpp
@@ -67,5 +67,130 @@
ASSERT_FALSE(frame == changedTimestampFrame);
}
+// --- Rotate 90 degrees ---
+
+TEST(TouchVideoFrame, Rotate90_0x0) {
+ TouchVideoFrame frame(0, 0, {}, TIMESTAMP);
+ TouchVideoFrame frameRotated(0, 0, {}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_90);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate90_1x1) {
+ TouchVideoFrame frame(1, 1, {1}, TIMESTAMP);
+ TouchVideoFrame frameRotated(1, 1, {1}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_90);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate90_2x2) {
+ TouchVideoFrame frame(2, 2, {1, 2, 3, 4}, TIMESTAMP);
+ TouchVideoFrame frameRotated(2, 2, {3, 1, 4, 2}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_90);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate90_3x2) {
+ TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ TouchVideoFrame frameRotated(2, 3, {5, 3, 1, 6, 4, 2}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_90);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate90_3x2_4times) {
+ TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ TouchVideoFrame frameOriginal(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_90);
+ frame.rotate(DISPLAY_ORIENTATION_90);
+ frame.rotate(DISPLAY_ORIENTATION_90);
+ frame.rotate(DISPLAY_ORIENTATION_90);
+ ASSERT_EQ(frame, frameOriginal);
+}
+
+// --- Rotate 180 degrees ---
+
+TEST(TouchVideoFrame, Rotate180_0x0) {
+ TouchVideoFrame frame(0, 0, {}, TIMESTAMP);
+ TouchVideoFrame frameRotated(0, 0, {}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_180);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate180_1x1) {
+ TouchVideoFrame frame(1, 1, {1}, TIMESTAMP);
+ TouchVideoFrame frameRotated(1, 1, {1}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_180);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate180_2x2) {
+ TouchVideoFrame frame(2, 2, {1, 2, 3, 4}, TIMESTAMP);
+ TouchVideoFrame frameRotated(2, 2, {4, 3, 2, 1}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_180);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate180_3x2) {
+ TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ TouchVideoFrame frameRotated(3, 2, {6, 5, 4, 3, 2, 1}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_180);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate180_3x2_2times) {
+ TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ TouchVideoFrame frameOriginal(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_180);
+ frame.rotate(DISPLAY_ORIENTATION_180);
+ ASSERT_EQ(frame, frameOriginal);
+}
+
+TEST(TouchVideoFrame, Rotate180_3x3) {
+ TouchVideoFrame frame(3, 3, {1, 2, 3, 4, 5, 6, 7, 8, 9}, TIMESTAMP);
+ TouchVideoFrame frameRotated(3, 3, {9, 8, 7, 6, 5, 4, 3, 2, 1}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_180);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+// --- Rotate 270 degrees ---
+
+TEST(TouchVideoFrame, Rotate270_0x0) {
+ TouchVideoFrame frame(0, 0, {}, TIMESTAMP);
+ TouchVideoFrame frameRotated(0, 0, {}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_270);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate270_1x1) {
+ TouchVideoFrame frame(1, 1, {1}, TIMESTAMP);
+ TouchVideoFrame frameRotated(1, 1, {1}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_270);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate270_2x2) {
+ TouchVideoFrame frame(2, 2, {1, 2, 3, 4}, TIMESTAMP);
+ TouchVideoFrame frameRotated(2, 2, {2, 4, 1, 3}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_270);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate270_3x2) {
+ TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ TouchVideoFrame frameRotated(2, 3, {2, 4, 6, 1, 3, 5}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_270);
+ ASSERT_EQ(frame, frameRotated);
+}
+
+TEST(TouchVideoFrame, Rotate270_3x2_4times) {
+ TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ TouchVideoFrame frameOriginal(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);
+ frame.rotate(DISPLAY_ORIENTATION_270);
+ frame.rotate(DISPLAY_ORIENTATION_270);
+ frame.rotate(DISPLAY_ORIENTATION_270);
+ frame.rotate(DISPLAY_ORIENTATION_270);
+ ASSERT_EQ(frame, frameOriginal);
+}
+
} // namespace test
} // namespace android
diff --git a/services/inputflinger/InputReader.cpp b/services/inputflinger/InputReader.cpp
index 1a1ae21..21ba029 100644
--- a/services/inputflinger/InputReader.cpp
+++ b/services/inputflinger/InputReader.cpp
@@ -6565,6 +6565,8 @@
const int32_t displayId = getAssociatedDisplay().value_or(ADISPLAY_ID_NONE);
const int32_t deviceId = getDeviceId();
std::vector<TouchVideoFrame> frames = mDevice->getEventHub()->getVideoFrames(deviceId);
+ std::for_each(frames.begin(), frames.end(),
+ [this](TouchVideoFrame& frame) { frame.rotate(this->mSurfaceOrientation); });
NotifyMotionArgs args(mContext->getNextSequenceNum(), when, deviceId,
source, displayId, policyFlags,
action, actionButton, flags, metaState, buttonState, MotionClassification::NONE,
diff --git a/services/inputflinger/tests/InputReader_test.cpp b/services/inputflinger/tests/InputReader_test.cpp
index a080744..80a55f1 100644
--- a/services/inputflinger/tests/InputReader_test.cpp
+++ b/services/inputflinger/tests/InputReader_test.cpp
@@ -6453,4 +6453,52 @@
ASSERT_EQ(std::vector<TouchVideoFrame>(), motionArgs.videoFrames);
}
+TEST_F(MultiTouchInputMapperTest, VideoFrames_AreRotated) {
+ MultiTouchInputMapper* mapper = new MultiTouchInputMapper(mDevice);
+ prepareAxes(POSITION);
+ addConfigurationProperty("touch.deviceType", "touchScreen");
+ addMapperAndConfigure(mapper);
+ // Unrotated video frame
+ TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, {1, 2});
+ NotifyMotionArgs motionArgs;
+
+ // Test all 4 orientations
+ for (int32_t orientation : {DISPLAY_ORIENTATION_0, DISPLAY_ORIENTATION_90,
+ DISPLAY_ORIENTATION_180, DISPLAY_ORIENTATION_270}) {
+ SCOPED_TRACE("Orientation " + StringPrintf("%i", orientation));
+ clearViewports();
+ prepareDisplay(orientation);
+ std::vector<TouchVideoFrame> frames{frame};
+ mFakeEventHub->setVideoFrames({{mDevice->getId(), frames}});
+ processPosition(mapper, 100, 200);
+ processSync(mapper);
+ ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs));
+ frames[0].rotate(orientation);
+ ASSERT_EQ(frames, motionArgs.videoFrames);
+ }
+}
+
+TEST_F(MultiTouchInputMapperTest, VideoFrames_MultipleFramesAreRotated) {
+ MultiTouchInputMapper* mapper = new MultiTouchInputMapper(mDevice);
+ prepareAxes(POSITION);
+ addConfigurationProperty("touch.deviceType", "touchScreen");
+ addMapperAndConfigure(mapper);
+ // Unrotated video frames. There's no rule that they must all have the same dimensions,
+ // so mix these.
+ TouchVideoFrame frame1(3, 2, {1, 2, 3, 4, 5, 6}, {1, 2});
+ TouchVideoFrame frame2(3, 3, {0, 1, 2, 3, 4, 5, 6, 7, 8}, {1, 3});
+ TouchVideoFrame frame3(2, 2, {10, 20, 10, 0}, {1, 4});
+ std::vector<TouchVideoFrame> frames{frame1, frame2, frame3};
+ NotifyMotionArgs motionArgs;
+
+ prepareDisplay(DISPLAY_ORIENTATION_90);
+ mFakeEventHub->setVideoFrames({{mDevice->getId(), frames}});
+ processPosition(mapper, 100, 200);
+ processSync(mapper);
+ ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs));
+ std::for_each(frames.begin(), frames.end(),
+ [](TouchVideoFrame& frame) { frame.rotate(DISPLAY_ORIENTATION_90); });
+ ASSERT_EQ(frames, motionArgs.videoFrames);
+}
+
} // namespace android