blob: e2a0bc26f697b763f47c9b02bd154311b38a07a3 [file] [log] [blame]
Siarhei Vishniakou473174e2017-12-27 16:44:42 -08001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef _UI_INPUT_CLASSIFIER_H
18#define _UI_INPUT_CLASSIFIER_H
19
Siarhei Vishniakou61291d42019-02-11 18:13:20 -080020#include <android-base/thread_annotations.h>
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -080021#include <future>
Siarhei Vishniakou473174e2017-12-27 16:44:42 -080022#include <thread>
Siarhei Vishniakou16523972020-03-04 17:48:39 -080023#include <unordered_map>
Siarhei Vishniakou473174e2017-12-27 16:44:42 -080024
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -080025#include <aidl/android/hardware/input/processor/IInputProcessor.h>
Siarhei Vishniakou473174e2017-12-27 16:44:42 -080026#include "BlockingQueue.h"
27#include "InputListener.h"
Siarhei Vishniakou473174e2017-12-27 16:44:42 -080028namespace android {
29
30enum class ClassifierEventType : uint8_t {
31 MOTION = 0,
32 DEVICE_RESET = 1,
33 HAL_RESET = 2,
34 EXIT = 3,
35};
36
37struct ClassifierEvent {
38 ClassifierEventType type;
39 std::unique_ptr<NotifyArgs> args;
40
41 ClassifierEvent(ClassifierEventType type, std::unique_ptr<NotifyArgs> args);
42 ClassifierEvent(std::unique_ptr<NotifyMotionArgs> args);
43 ClassifierEvent(std::unique_ptr<NotifyDeviceResetArgs> args);
44 ClassifierEvent(ClassifierEvent&& other);
45 ClassifierEvent& operator=(ClassifierEvent&& other);
46
47 // Convenience function to create a HAL_RESET event
48 static ClassifierEvent createHalResetEvent();
49 // Convenience function to create an EXIT event
50 static ClassifierEvent createExitEvent();
51
52 std::optional<int32_t> getDeviceId() const;
53};
54
55// --- Interfaces ---
56
57/**
58 * Interface for adding a MotionClassification to NotifyMotionArgs.
59 *
60 * To implement, override the classify function.
61 */
62class MotionClassifierInterface {
63public:
64 MotionClassifierInterface() { }
65 virtual ~MotionClassifierInterface() { }
66 /**
67 * Based on the motion event described by NotifyMotionArgs,
68 * provide a MotionClassification for the current gesture.
69 */
70 virtual MotionClassification classify(const NotifyMotionArgs& args) = 0;
Siarhei Vishniakou15b66d12019-02-04 14:27:29 -080071 /**
72 * Reset all internal HAL state.
73 */
Siarhei Vishniakou473174e2017-12-27 16:44:42 -080074 virtual void reset() = 0;
Siarhei Vishniakou15b66d12019-02-04 14:27:29 -080075 /**
76 * Reset HAL state for a specific device.
77 */
Siarhei Vishniakou473174e2017-12-27 16:44:42 -080078 virtual void reset(const NotifyDeviceResetArgs& args) = 0;
Siarhei Vishniakoua028c442019-02-04 14:33:23 -080079
80 /**
81 * Dump the state of the motion classifier
82 */
83 virtual void dump(std::string& dump) = 0;
Siarhei Vishniakou473174e2017-12-27 16:44:42 -080084};
85
86/**
87 * Base interface for an InputListener stage.
88 * Provides classification to events.
89 */
Siarhei Vishniakou18050092021-09-01 13:32:49 -070090class InputClassifierInterface : public InputListenerInterface {
Siarhei Vishniakoua028c442019-02-04 14:33:23 -080091public:
Siarhei Vishniakouc9ac19e2020-03-19 11:55:01 -070092 virtual void setMotionClassifierEnabled(bool enabled) = 0;
Siarhei Vishniakoua028c442019-02-04 14:33:23 -080093 /**
94 * Dump the state of the input classifier.
95 * This method may be called on any thread (usually by the input manager).
96 */
97 virtual void dump(std::string& dump) = 0;
Siarhei Vishniakou18050092021-09-01 13:32:49 -070098
Siarhei Vishniakou473174e2017-12-27 16:44:42 -080099 InputClassifierInterface() { }
100 virtual ~InputClassifierInterface() { }
101};
102
103// --- Implementations ---
104
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800105class ScopedDeathRecipient {
106public:
107 explicit ScopedDeathRecipient(AIBinder_DeathRecipient_onBinderDied onBinderDied, void* cookie);
108 ScopedDeathRecipient(const ScopedDeathRecipient&) = delete;
109 ScopedDeathRecipient& operator=(ScopedDeathRecipient const&) = delete;
110 void linkToDeath(AIBinder* binder);
111 ~ScopedDeathRecipient();
112
113private:
114 AIBinder_DeathRecipient* mRecipient;
115 void* mCookie;
116};
117
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800118/**
119 * Implementation of MotionClassifierInterface that calls the InputClassifier HAL
120 * in order to determine the classification for the current gesture.
121 *
122 * The InputClassifier HAL may keep track of the entire gesture in order to determine
123 * the classification, and may be hardware-specific. It may use the data in
124 * NotifyMotionArgs::videoFrames field to drive the classification decisions.
125 * The HAL is called from a separate thread.
126 */
Greg Kaiser04b3a052019-01-29 07:10:14 -0800127class MotionClassifier final : public MotionClassifierInterface {
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800128public:
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800129 /*
130 * Create an instance of MotionClassifier.
131 * The death recipient, if provided, will be subscribed to the HAL death.
132 * The death recipient could be used to destroy MotionClassifier.
133 *
134 * This function should be called asynchronously, because getService takes a long time.
Siarhei Vishniakou15b66d12019-02-04 14:27:29 -0800135 */
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800136 static std::unique_ptr<MotionClassifierInterface> create(
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800137 std::shared_ptr<aidl::android::hardware::input::processor::IInputProcessor> service);
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800138
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800139 ~MotionClassifier();
Siarhei Vishniakou4bdbb6a2019-04-11 09:42:09 -0700140
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800141 /**
142 * Classifies events asynchronously; that is, it doesn't block events on a classification,
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800143 * but instead sends them over to the classifier HAL. After a classification of a specific
144 * event is determined, MotionClassifier then marks the next event in the stream with this
145 * classification.
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800146 *
147 * Therefore, it is acceptable to have the classifications be delayed by 1-2 events
148 * in a particular gesture.
149 */
150 virtual MotionClassification classify(const NotifyMotionArgs& args) override;
151 virtual void reset() override;
152 virtual void reset(const NotifyDeviceResetArgs& args) override;
153
Siarhei Vishniakoua028c442019-02-04 14:33:23 -0800154 virtual void dump(std::string& dump) override;
155
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800156private:
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800157 friend class MotionClassifierTest; // to create MotionClassifier with a test HAL implementation
158 explicit MotionClassifier(
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800159 std::shared_ptr<aidl::android::hardware::input::processor::IInputProcessor> service);
Siarhei Vishniakou4bdbb6a2019-04-11 09:42:09 -0700160
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800161 // The events that need to be sent to the HAL.
162 BlockingQueue<ClassifierEvent> mEvents;
163 /**
Siarhei Vishniakou15b66d12019-02-04 14:27:29 -0800164 * Add an event to the queue mEvents.
165 */
166 void enqueueEvent(ClassifierEvent&& event);
167 /**
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800168 * Thread that will communicate with InputClassifier HAL.
169 * This should be the only thread that communicates with InputClassifier HAL,
170 * because this thread is allowed to block on the HAL calls.
171 */
172 std::thread mHalThread;
173 /**
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800174 * Process events and call the InputClassifier HAL
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800175 */
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800176 void processEvents();
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800177 /**
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800178 * Access to the InputProcessor HAL. May be null if init() hasn't completed yet.
Siarhei Vishniakou4bdbb6a2019-04-11 09:42:09 -0700179 * When init() successfully completes, mService is guaranteed to remain non-null and to not
180 * change its value until MotionClassifier is destroyed.
181 * This variable is *not* guarded by mLock in the InputClassifier thread, because
182 * that thread knows exactly when this variable is initialized.
183 * When accessed in any other thread, mService is checked for nullness with a lock.
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800184 */
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800185 std::shared_ptr<aidl::android::hardware::input::processor::IInputProcessor> mService;
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800186 std::mutex mLock;
187 /**
188 * Per-device input classifications. Should only be accessed using the
189 * getClassification / setClassification methods.
190 */
191 std::unordered_map<int32_t /*deviceId*/, MotionClassification>
Siarhei Vishniakou61291d42019-02-11 18:13:20 -0800192 mClassifications GUARDED_BY(mLock);
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800193 /**
194 * Set the current classification for a given device.
195 */
196 void setClassification(int32_t deviceId, MotionClassification classification);
197 /**
198 * Get the current classification for a given device.
199 */
200 MotionClassification getClassification(int32_t deviceId);
201 void updateClassification(int32_t deviceId, nsecs_t eventTime,
Siarhei Vishniakou61291d42019-02-11 18:13:20 -0800202 MotionClassification classification);
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800203 /**
204 * Clear all current classifications
205 */
206 void clearClassifications();
207 /**
208 * Per-device times when the last ACTION_DOWN was received.
209 * Used to reject late classifications that do not belong to the current gesture.
210 *
211 * Accessed indirectly by both InputClassifier thread and the thread that receives notifyMotion.
212 */
Siarhei Vishniakou61291d42019-02-11 18:13:20 -0800213 std::unordered_map<int32_t /*deviceId*/, nsecs_t /*downTime*/> mLastDownTimes GUARDED_BY(mLock);
214
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800215 void updateLastDownTime(int32_t deviceId, nsecs_t downTime);
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800216
Siarhei Vishniakoue3021d72020-02-28 15:25:41 -0800217 void clearDeviceState(int32_t deviceId);
218
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800219 /**
220 * Exit the InputClassifier HAL thread.
221 * Useful for tests to ensure proper cleanup.
222 */
223 void requestExit();
Siarhei Vishniakou4bdbb6a2019-04-11 09:42:09 -0700224 /**
225 * Return string status of mService
226 */
227 const char* getServiceStatus() REQUIRES(mLock);
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800228};
229
230/**
231 * Implementation of the InputClassifierInterface.
232 * Represents a separate stage of input processing. All of the input events go through this stage.
233 * Acts as a passthrough for all input events except for motion events.
234 * The events of motion type are sent to MotionClassifier.
235 */
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800236class InputClassifier : public InputClassifierInterface {
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800237public:
Siarhei Vishniakou18050092021-09-01 13:32:49 -0700238 explicit InputClassifier(InputListenerInterface& listener);
Siarhei Vishniakou45bb0882019-02-04 14:25:28 -0800239
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800240 void notifyConfigurationChanged(const NotifyConfigurationChangedArgs* args) override;
241 void notifyKey(const NotifyKeyArgs* args) override;
242 void notifyMotion(const NotifyMotionArgs* args) override;
243 void notifySwitch(const NotifySwitchArgs* args) override;
244 void notifySensor(const NotifySensorArgs* args) override;
245 void notifyVibratorState(const NotifyVibratorStateArgs* args) override;
246 void notifyDeviceReset(const NotifyDeviceResetArgs* args) override;
Prabir Pradhan7e186182020-11-10 13:56:45 -0800247 void notifyPointerCaptureChanged(const NotifyPointerCaptureChangedArgs* args) override;
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800248
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800249 void dump(std::string& dump) override;
Siarhei Vishniakoua028c442019-02-04 14:33:23 -0800250
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800251 ~InputClassifier();
252
Siarhei Vishniakouc9ac19e2020-03-19 11:55:01 -0700253 // Called from InputManager
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800254 void setMotionClassifierEnabled(bool enabled) override;
Siarhei Vishniakouc9ac19e2020-03-19 11:55:01 -0700255
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800256private:
Siarhei Vishniakou15b66d12019-02-04 14:27:29 -0800257 // Protect access to mMotionClassifier, since it may become null via a hidl callback
258 std::mutex mLock;
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800259 // The next stage to pass input events to
Siarhei Vishniakou18050092021-09-01 13:32:49 -0700260 InputListenerInterface& mListener;
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800261
262 std::unique_ptr<MotionClassifierInterface> mMotionClassifier GUARDED_BY(mLock);
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800263 std::future<void> mInitializeMotionClassifier GUARDED_BY(mLock);
264
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800265 /**
266 * Set the value of mMotionClassifier.
267 * This is called from 2 different threads:
268 * 1) mInitializeMotionClassifierThread, when we have constructed a MotionClassifier
269 * 2) A binder thread of the HalDeathRecipient, which is created when HAL dies. This would cause
270 * mMotionClassifier to become nullptr.
271 */
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800272 void setMotionClassifierLocked(std::unique_ptr<MotionClassifierInterface> motionClassifier)
273 REQUIRES(mLock);
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800274
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800275 static void onBinderDied(void* cookie);
Siarhei Vishniakou16523972020-03-04 17:48:39 -0800276
Siarhei Vishniakou34d6fef2022-02-01 19:03:45 -0800277 std::unique_ptr<ScopedDeathRecipient> mHalDeathRecipient GUARDED_BY(mLock);
Siarhei Vishniakou473174e2017-12-27 16:44:42 -0800278};
279
280} // namespace android
Greg Kaiser04b3a052019-01-29 07:10:14 -0800281#endif