Differentiate fused and unfused external styluses
An external stylus can be "fused" with touch data if it reports
pointer-specific data independently from the touch device. The only such
data we currently support is pressure. Thus, a fused external stylus
will reports pressure data.
If a fused stylus is connected, we withold all touches for up to 72
milliseconds to wait for pressure data from the stylus. If we get
pressure data within this time, we assume that the first pointer that
went down is actually the stylus, and fuse the pressure information from
the stylus to the pointer.
If an external stylus does not report pressure, it is an "unfused"
stylus.
When such an external stylus is connected, we don't withold any touches.
We still report button presses from these styluses through the touch
device.
DD: go/android-stylus-buttons
Bug: 246394583
Test: atest inputflinger_tests
Change-Id: I3d687a10630019756170e7e5e5f5d1902eb96e36
diff --git a/services/inputflinger/reader/include/StylusState.h b/services/inputflinger/reader/include/StylusState.h
index 8d14d3c..ff15e0c 100644
--- a/services/inputflinger/reader/include/StylusState.h
+++ b/services/inputflinger/reader/include/StylusState.h
@@ -24,27 +24,19 @@
struct StylusState {
/* Time the stylus event was received. */
- nsecs_t when;
- /* Pressure as reported by the stylus, normalized to the range [0, 1.0]. */
- float pressure;
+ nsecs_t when{};
+ /*
+ * Pressure as reported by the stylus if supported, normalized to the range [0, 1.0].
+ * The presence of a pressure value indicates that the stylus is able to tell whether it is
+ * touching the display.
+ */
+ std::optional<float> pressure{};
/* The state of the stylus buttons as a bitfield (e.g. AMOTION_EVENT_BUTTON_SECONDARY). */
- uint32_t buttons;
+ uint32_t buttons{};
/* Which tool type the stylus is currently using (e.g. AMOTION_EVENT_TOOL_TYPE_ERASER). */
- int32_t toolType;
+ int32_t toolType{AMOTION_EVENT_TOOL_TYPE_UNKNOWN};
- void copyFrom(const StylusState& other) {
- when = other.when;
- pressure = other.pressure;
- buttons = other.buttons;
- toolType = other.toolType;
- }
-
- void clear() {
- when = LLONG_MAX;
- pressure = 0.f;
- buttons = 0;
- toolType = AMOTION_EVENT_TOOL_TYPE_UNKNOWN;
- }
+ void clear() { *this = StylusState{}; }
};
} // namespace android
diff --git a/services/inputflinger/reader/mapper/ExternalStylusInputMapper.cpp b/services/inputflinger/reader/mapper/ExternalStylusInputMapper.cpp
index 56fc5fa..2809939 100644
--- a/services/inputflinger/reader/mapper/ExternalStylusInputMapper.cpp
+++ b/services/inputflinger/reader/mapper/ExternalStylusInputMapper.cpp
@@ -32,8 +32,10 @@
void ExternalStylusInputMapper::populateDeviceInfo(InputDeviceInfo* info) {
InputMapper::populateDeviceInfo(info);
- info->addMotionRange(AMOTION_EVENT_AXIS_PRESSURE, AINPUT_SOURCE_STYLUS, 0.0f, 1.0f, 0.0f, 0.0f,
- 0.0f);
+ if (mRawPressureAxis.valid) {
+ info->addMotionRange(AMOTION_EVENT_AXIS_PRESSURE, AINPUT_SOURCE_STYLUS, 0.0f, 1.0f, 0.0f,
+ 0.0f, 0.0f);
+ }
}
void ExternalStylusInputMapper::dump(std::string& dump) {
@@ -79,13 +81,12 @@
mStylusState.toolType = AMOTION_EVENT_TOOL_TYPE_STYLUS;
}
- int32_t pressure = mSingleTouchMotionAccumulator.getAbsolutePressure();
if (mRawPressureAxis.valid) {
- mStylusState.pressure = float(pressure) / mRawPressureAxis.maxValue;
- } else if (mTouchButtonAccumulator.isToolActive()) {
- mStylusState.pressure = 1.0f;
- } else {
- mStylusState.pressure = 0.0f;
+ auto rawPressure = static_cast<float>(mSingleTouchMotionAccumulator.getAbsolutePressure());
+ mStylusState.pressure = (rawPressure - mRawPressureAxis.minValue) /
+ static_cast<float>(mRawPressureAxis.maxValue - mRawPressureAxis.minValue);
+ } else if (mTouchButtonAccumulator.hasButtonTouch()) {
+ mStylusState.pressure = mTouchButtonAccumulator.isHovering() ? 0.0f : 1.0f;
}
mStylusState.buttons = mTouchButtonAccumulator.getButtonState();
diff --git a/services/inputflinger/reader/mapper/InputMapper.cpp b/services/inputflinger/reader/mapper/InputMapper.cpp
index 844afe0..8e3539c 100644
--- a/services/inputflinger/reader/mapper/InputMapper.cpp
+++ b/services/inputflinger/reader/mapper/InputMapper.cpp
@@ -19,6 +19,7 @@
#include "InputMapper.h"
#include "InputDevice.h"
+#include "input/PrintTools.h"
namespace android {
@@ -129,7 +130,7 @@
void InputMapper::dumpStylusState(std::string& dump, const StylusState& state) {
dump += StringPrintf(INDENT4 "When: %" PRId64 "\n", state.when);
- dump += StringPrintf(INDENT4 "Pressure: %f\n", state.pressure);
+ dump += StringPrintf(INDENT4 "Pressure: %s\n", toString(state.pressure).c_str());
dump += StringPrintf(INDENT4 "Button State: 0x%08x\n", state.buttons);
dump += StringPrintf(INDENT4 "Tool Type: %" PRId32 "\n", state.toolType);
}
diff --git a/services/inputflinger/reader/mapper/TouchInputMapper.cpp b/services/inputflinger/reader/mapper/TouchInputMapper.cpp
index d447368..f8d6cf9 100644
--- a/services/inputflinger/reader/mapper/TouchInputMapper.cpp
+++ b/services/inputflinger/reader/mapper/TouchInputMapper.cpp
@@ -1705,22 +1705,24 @@
void TouchInputMapper::applyExternalStylusTouchState(nsecs_t when) {
CookedPointerData& currentPointerData = mCurrentCookedState.cookedPointerData;
const CookedPointerData& lastPointerData = mLastCookedState.cookedPointerData;
+ if (!mFusedStylusPointerId || !currentPointerData.isTouching(*mFusedStylusPointerId)) {
+ return;
+ }
- if (mFusedStylusPointerId && currentPointerData.isTouching(*mFusedStylusPointerId)) {
- float pressure = mExternalStylusState.pressure;
- if (pressure == 0.0f && lastPointerData.isTouching(*mFusedStylusPointerId)) {
- const PointerCoords& coords =
- lastPointerData.pointerCoordsForId(*mFusedStylusPointerId);
- pressure = coords.getAxisValue(AMOTION_EVENT_AXIS_PRESSURE);
- }
- PointerCoords& coords = currentPointerData.editPointerCoordsWithId(*mFusedStylusPointerId);
- coords.setAxisValue(AMOTION_EVENT_AXIS_PRESSURE, pressure);
+ float pressure = lastPointerData.isTouching(*mFusedStylusPointerId)
+ ? lastPointerData.pointerCoordsForId(*mFusedStylusPointerId)
+ .getAxisValue(AMOTION_EVENT_AXIS_PRESSURE)
+ : 0.f;
+ if (mExternalStylusState.pressure && *mExternalStylusState.pressure > 0.f) {
+ pressure = *mExternalStylusState.pressure;
+ }
+ PointerCoords& coords = currentPointerData.editPointerCoordsWithId(*mFusedStylusPointerId);
+ coords.setAxisValue(AMOTION_EVENT_AXIS_PRESSURE, pressure);
+ if (mExternalStylusState.toolType != AMOTION_EVENT_TOOL_TYPE_UNKNOWN) {
PointerProperties& properties =
currentPointerData.editPointerPropertiesWithId(*mFusedStylusPointerId);
- if (mExternalStylusState.toolType != AMOTION_EVENT_TOOL_TYPE_UNKNOWN) {
- properties.toolType = mExternalStylusState.toolType;
- }
+ properties.toolType = mExternalStylusState.toolType;
}
}
@@ -1729,36 +1731,48 @@
return false;
}
- const bool initialDown = mLastRawState.rawPointerData.pointerCount == 0 &&
- state.rawPointerData.pointerCount != 0;
- if (initialDown) {
- if (mExternalStylusState.pressure != 0.0f) {
- ALOGD_IF(DEBUG_STYLUS_FUSION, "Have both stylus and touch data, beginning fusion");
- mFusedStylusPointerId = state.rawPointerData.touchingIdBits.firstMarkedBit();
- } else if (timeout) {
- ALOGD_IF(DEBUG_STYLUS_FUSION, "Timeout expired, assuming touch is not a stylus.");
- mFusedStylusPointerId.reset();
- mExternalStylusFusionTimeout = LLONG_MAX;
- } else {
- if (mExternalStylusFusionTimeout == LLONG_MAX) {
- mExternalStylusFusionTimeout = state.when + EXTERNAL_STYLUS_DATA_TIMEOUT;
- }
- ALOGD_IF(DEBUG_STYLUS_FUSION,
- "No stylus data but stylus is connected, requesting timeout (%" PRId64 "ms)",
- mExternalStylusFusionTimeout);
- getContext()->requestTimeoutAtTime(mExternalStylusFusionTimeout);
- return true;
- }
- }
-
// Check if the stylus pointer has gone up.
if (mFusedStylusPointerId &&
!state.rawPointerData.touchingIdBits.hasBit(*mFusedStylusPointerId)) {
ALOGD_IF(DEBUG_STYLUS_FUSION, "Stylus pointer is going up");
mFusedStylusPointerId.reset();
+ return false;
}
- return false;
+ const bool initialDown = mLastRawState.rawPointerData.pointerCount == 0 &&
+ state.rawPointerData.pointerCount != 0;
+ if (!initialDown) {
+ return false;
+ }
+
+ if (!mExternalStylusState.pressure) {
+ ALOGD_IF(DEBUG_STYLUS_FUSION, "Stylus does not support pressure, no pointer fusion needed");
+ return false;
+ }
+
+ if (*mExternalStylusState.pressure != 0.0f) {
+ ALOGD_IF(DEBUG_STYLUS_FUSION, "Have both stylus and touch data, beginning fusion");
+ mFusedStylusPointerId = state.rawPointerData.touchingIdBits.firstMarkedBit();
+ return false;
+ }
+
+ if (timeout) {
+ ALOGD_IF(DEBUG_STYLUS_FUSION, "Timeout expired, assuming touch is not a stylus.");
+ mFusedStylusPointerId.reset();
+ mExternalStylusFusionTimeout = LLONG_MAX;
+ return false;
+ }
+
+ // We are waiting for the external stylus to report a pressure value. Withhold touches from
+ // being processed until we either get pressure data or timeout.
+ if (mExternalStylusFusionTimeout == LLONG_MAX) {
+ mExternalStylusFusionTimeout = state.when + EXTERNAL_STYLUS_DATA_TIMEOUT;
+ }
+ ALOGD_IF(DEBUG_STYLUS_FUSION,
+ "No stylus data but stylus is connected, requesting timeout (%" PRId64 "ms)",
+ mExternalStylusFusionTimeout);
+ getContext()->requestTimeoutAtTime(mExternalStylusFusionTimeout);
+ return true;
}
std::list<NotifyArgs> TouchInputMapper::timeoutExpired(nsecs_t when) {
@@ -1782,7 +1796,7 @@
std::list<NotifyArgs> TouchInputMapper::updateExternalStylusState(const StylusState& state) {
std::list<NotifyArgs> out;
const bool buttonsChanged = mExternalStylusState.buttons != state.buttons;
- mExternalStylusState.copyFrom(state);
+ mExternalStylusState = state;
if (mFusedStylusPointerId || mExternalStylusFusionTimeout != LLONG_MAX || buttonsChanged) {
// The following three cases are handled here:
// - We're in the middle of a fused stream of data;
diff --git a/services/inputflinger/reader/mapper/TouchInputMapper.h b/services/inputflinger/reader/mapper/TouchInputMapper.h
index d2faf3f..85af1f7 100644
--- a/services/inputflinger/reader/mapper/TouchInputMapper.h
+++ b/services/inputflinger/reader/mapper/TouchInputMapper.h
@@ -795,6 +795,8 @@
[[nodiscard]] std::list<NotifyArgs> abortPointerSimple(nsecs_t when, nsecs_t readTime,
uint32_t policyFlags);
+ // Attempts to assign a pointer id to the external stylus. Returns true if the state should be
+ // withheld from further processing while waiting for data from the stylus.
bool assignExternalStylusId(const RawState& state, bool timeout);
void applyExternalStylusButtonState(nsecs_t when);
void applyExternalStylusTouchState(nsecs_t when);
diff --git a/services/inputflinger/reader/mapper/accumulator/TouchButtonAccumulator.cpp b/services/inputflinger/reader/mapper/accumulator/TouchButtonAccumulator.cpp
index 1891205..bc23a8e 100644
--- a/services/inputflinger/reader/mapper/accumulator/TouchButtonAccumulator.cpp
+++ b/services/inputflinger/reader/mapper/accumulator/TouchButtonAccumulator.cpp
@@ -167,4 +167,8 @@
return mHaveStylus;
}
+bool TouchButtonAccumulator::hasButtonTouch() const {
+ return mHaveBtnTouch;
+}
+
} // namespace android
diff --git a/services/inputflinger/reader/mapper/accumulator/TouchButtonAccumulator.h b/services/inputflinger/reader/mapper/accumulator/TouchButtonAccumulator.h
index 65b0a62..c2de23c 100644
--- a/services/inputflinger/reader/mapper/accumulator/TouchButtonAccumulator.h
+++ b/services/inputflinger/reader/mapper/accumulator/TouchButtonAccumulator.h
@@ -40,6 +40,7 @@
bool isToolActive() const;
bool isHovering() const;
bool hasStylus() const;
+ bool hasButtonTouch() const;
private:
bool mHaveBtnTouch{};
diff --git a/services/inputflinger/tests/InputReader_test.cpp b/services/inputflinger/tests/InputReader_test.cpp
index fe36d42..3ce03f3 100644
--- a/services/inputflinger/tests/InputReader_test.cpp
+++ b/services/inputflinger/tests/InputReader_test.cpp
@@ -3156,7 +3156,10 @@
// Set a pressure value of 0 on the stylus. It doesn't generate any events.
const auto& RAW_PRESSURE_MAX = UinputExternalStylusWithPressure::RAW_PRESSURE_MAX;
+ // Send a non-zero value first to prevent the kernel from consuming the zero event.
+ stylus->setPressure(100);
stylus->setPressure(0);
+ ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasNotCalled());
// Start a finger gesture. The touch device will withhold generating any touches for
// up to 72 milliseconds while waiting for pressure data from the external stylus.
@@ -3203,6 +3206,45 @@
ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyKeyWasNotCalled());
}
+TEST_F(ExternalStylusIntegrationTest, UnfusedExternalStylus) {
+ const Point centerPoint = mDevice->getCenterPoint();
+
+ // Create an external stylus device that does not support pressure. It should not affect any
+ // touch pointers.
+ std::unique_ptr<UinputExternalStylus> stylus = createUinputDevice<UinputExternalStylus>();
+ ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertInputDevicesChanged());
+ ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyConfigurationChangedWasCalled());
+ const auto stylusInfo = findDeviceByName(stylus->getName());
+ ASSERT_TRUE(stylusInfo);
+
+ ASSERT_EQ(AINPUT_SOURCE_STYLUS | AINPUT_SOURCE_KEYBOARD, stylusInfo->getSources());
+
+ const auto touchscreenId = mDeviceInfo.getId();
+
+ // Start a finger gesture and ensure a finger pointer is generated for it, without waiting for
+ // pressure data from the external stylus.
+ mDevice->sendSlot(FIRST_SLOT);
+ mDevice->sendTrackingId(FIRST_TRACKING_ID);
+ mDevice->sendToolType(MT_TOOL_FINGER);
+ mDevice->sendDown(centerPoint);
+ auto waitUntil = std::chrono::system_clock::now() +
+ std::chrono::milliseconds(ns2ms(EXTERNAL_STYLUS_DATA_TIMEOUT));
+ mDevice->sendSync();
+ ASSERT_NO_FATAL_FAILURE(
+ mTestListener
+ ->assertNotifyMotionWasCalled(AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN),
+ WithToolType(
+ AMOTION_EVENT_TOOL_TYPE_FINGER),
+ WithButtonState(0),
+ WithDeviceId(touchscreenId),
+ WithPressure(1.f)),
+ waitUntil));
+
+ // The external stylus did not generate any events.
+ ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasNotCalled());
+ ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyKeyWasNotCalled());
+}
+
// --- InputDeviceTest ---
class InputDeviceTest : public testing::Test {
protected:
diff --git a/services/inputflinger/tests/TestInputListener.cpp b/services/inputflinger/tests/TestInputListener.cpp
index a1299ee..2801072 100644
--- a/services/inputflinger/tests/TestInputListener.cpp
+++ b/services/inputflinger/tests/TestInputListener.cpp
@@ -69,16 +69,18 @@
ASSERT_NO_FATAL_FAILURE(assertNotCalled<NotifyKeyArgs>("notifyKey() should not be called."));
}
-void TestInputListener::assertNotifyMotionWasCalled(NotifyMotionArgs* outEventArgs) {
+void TestInputListener::assertNotifyMotionWasCalled(NotifyMotionArgs* outEventArgs,
+ std::optional<TimePoint> waitUntil) {
ASSERT_NO_FATAL_FAILURE(
assertCalled<NotifyMotionArgs>(outEventArgs,
- "Expected notifyMotion() to have been called."));
+ "Expected notifyMotion() to have been called.",
+ waitUntil));
}
void TestInputListener::assertNotifyMotionWasCalled(
- const ::testing::Matcher<NotifyMotionArgs>& matcher) {
+ const ::testing::Matcher<NotifyMotionArgs>& matcher, std::optional<TimePoint> waitUntil) {
NotifyMotionArgs outEventArgs;
- ASSERT_NO_FATAL_FAILURE(assertNotifyMotionWasCalled(&outEventArgs));
+ ASSERT_NO_FATAL_FAILURE(assertNotifyMotionWasCalled(&outEventArgs, waitUntil));
ASSERT_THAT(outEventArgs, matcher);
}
@@ -119,15 +121,18 @@
}
template <class NotifyArgsType>
-void TestInputListener::assertCalled(NotifyArgsType* outEventArgs, std::string message) {
+void TestInputListener::assertCalled(NotifyArgsType* outEventArgs, std::string message,
+ std::optional<TimePoint> waitUntil) {
std::unique_lock<std::mutex> lock(mLock);
base::ScopedLockAssertion assumeLocked(mLock);
std::vector<NotifyArgsType>& queue = std::get<std::vector<NotifyArgsType>>(mQueues);
if (queue.empty()) {
- const bool eventReceived =
- mCondition.wait_for(lock, mEventHappenedTimeout,
- [&queue]() REQUIRES(mLock) { return !queue.empty(); });
+ const auto time =
+ waitUntil.value_or(std::chrono::system_clock::now() + mEventHappenedTimeout);
+ const bool eventReceived = mCondition.wait_until(lock, time, [&queue]() REQUIRES(mLock) {
+ return !queue.empty();
+ });
if (!eventReceived) {
FAIL() << "Timed out waiting for event: " << message.c_str();
}
diff --git a/services/inputflinger/tests/TestInputListener.h b/services/inputflinger/tests/TestInputListener.h
index c53f8e0..9665f70 100644
--- a/services/inputflinger/tests/TestInputListener.h
+++ b/services/inputflinger/tests/TestInputListener.h
@@ -50,9 +50,11 @@
void assertNotifyKeyWasNotCalled();
- void assertNotifyMotionWasCalled(NotifyMotionArgs* outEventArgs = nullptr);
+ void assertNotifyMotionWasCalled(NotifyMotionArgs* outEventArgs = nullptr,
+ std::optional<TimePoint> waitUntil = {});
- void assertNotifyMotionWasCalled(const ::testing::Matcher<NotifyMotionArgs>& matcher);
+ void assertNotifyMotionWasCalled(const ::testing::Matcher<NotifyMotionArgs>& matcher,
+ std::optional<TimePoint> waitUntil = {});
void assertNotifyMotionWasNotCalled(std::optional<TimePoint> waitUntil = {});
@@ -65,7 +67,8 @@
private:
template <class NotifyArgsType>
- void assertCalled(NotifyArgsType* outEventArgs, std::string message);
+ void assertCalled(NotifyArgsType* outEventArgs, std::string message,
+ std::optional<TimePoint> waitUntil = {});
template <class NotifyArgsType>
void assertNotCalled(std::string message, std::optional<TimePoint> timeout = {});