Add TouchVideoDevice
TouchVideoDevice will represent a video device that reports heatmaps.
The heatmaps are strength of the signal at each point of the
touchscreen, taken at a particular time.
The only purpose of TouchVideoDevice is to read TouchVideoFrames.
Test: this is a partial cherry-pick of a standalone binary that was used
to pull heatmaps. Integration tested by observing the action of
InputClassifier HAL.
Bug: 111480215
Change-Id: Iba1b38114b156d4debb5966b803f3df6cdb1b91c
diff --git a/services/inputflinger/Android.bp b/services/inputflinger/Android.bp
index 1fbc6bf..0d8d34f 100644
--- a/services/inputflinger/Android.bp
+++ b/services/inputflinger/Android.bp
@@ -62,6 +62,7 @@
"EventHub.cpp",
"InputReader.cpp",
"InputReaderFactory.cpp",
+ "TouchVideoDevice.cpp",
],
shared_libs: [
diff --git a/services/inputflinger/EventHub.cpp b/services/inputflinger/EventHub.cpp
index a3ecebc..904580f 100644
--- a/services/inputflinger/EventHub.cpp
+++ b/services/inputflinger/EventHub.cpp
@@ -1003,14 +1003,18 @@
} while (nWrite == -1 && errno == EINTR);
if (nWrite != 1 && errno != EAGAIN) {
- ALOGW("Could not write wake signal, errno=%d", errno);
+ ALOGW("Could not write wake signal: %s", strerror(errno));
}
}
void EventHub::scanDevicesLocked() {
- status_t res = scanDirLocked(DEVICE_PATH);
- if(res < 0) {
- ALOGE("scan dir failed for %s\n", DEVICE_PATH);
+ status_t result = scanDirLocked(DEVICE_PATH);
+ if(result < 0) {
+ ALOGE("scan dir failed for %s", DEVICE_PATH);
+ }
+ result = scanVideoDirLocked(VIDEO_DEVICE_PATH);
+ if (result != OK) {
+ ALOGE("scan video dir failed for %s", VIDEO_DEVICE_PATH);
}
if (mDevices.indexOfKey(VIRTUAL_KEYBOARD_ID) < 0) {
createVirtualKeyboardLocked();
@@ -1396,6 +1400,17 @@
toString(usingClockIoctl));
}
+void EventHub::openVideoDeviceLocked(const std::string& devicePath) {
+ std::unique_ptr<TouchVideoDevice> videoDevice = TouchVideoDevice::create(devicePath);
+ if (!videoDevice) {
+ ALOGE("Could not create touch video device for %s. Ignoring", devicePath.c_str());
+ return;
+ }
+ ALOGI("Adding video device %s to list of unattached video devices",
+ videoDevice->getName().c_str());
+ mUnattachedVideoDevices.push_back(std::move(videoDevice));
+}
+
bool EventHub::isDeviceEnabled(int32_t deviceId) {
AutoMutex _l(mLock);
Device* device = getDeviceLocked(deviceId);
@@ -1575,24 +1590,31 @@
return NAME_NOT_FOUND;
}
-status_t EventHub::closeDeviceByPathLocked(const char *devicePath) {
+void EventHub::closeDeviceByPathLocked(const char *devicePath) {
Device* device = getDeviceByPathLocked(devicePath);
if (device) {
closeDeviceLocked(device);
- return 0;
+ return;
}
ALOGV("Remove device: %s not found, device may already have been removed.", devicePath);
- return -1;
+}
+
+void EventHub::closeVideoDeviceByPathLocked(const std::string& devicePath) {
+ mUnattachedVideoDevices.erase(std::remove_if(mUnattachedVideoDevices.begin(),
+ mUnattachedVideoDevices.end(), [&devicePath](
+ const std::unique_ptr<TouchVideoDevice>& videoDevice) {
+ return videoDevice->getPath() == devicePath; }), mUnattachedVideoDevices.end());
}
void EventHub::closeAllDevicesLocked() {
+ mUnattachedVideoDevices.clear();
while (mDevices.size() > 0) {
closeDeviceLocked(mDevices.valueAt(mDevices.size() - 1));
}
}
void EventHub::closeDeviceLocked(Device* device) {
- ALOGI("Removed device: path=%s name=%s id=%d fd=%d classes=0x%x\n",
+ ALOGI("Removed device: path=%s name=%s id=%d fd=%d classes=0x%x",
device->path.c_str(), device->identifier.name.c_str(), device->id,
device->fd, device->classes);
@@ -1670,7 +1692,12 @@
else if (event->wd == mVideoWd) {
if (isV4lTouchNode(event->name)) {
std::string filename = StringPrintf("%s/%s", VIDEO_DEVICE_PATH, event->name);
- ALOGV("Received an inotify event for a video device %s", filename.c_str());
+ if (event->mask & IN_CREATE) {
+ openVideoDeviceLocked(filename);
+ } else {
+ ALOGI("Removing video device '%s' due to inotify event", filename.c_str());
+ closeVideoDeviceByPathLocked(filename);
+ }
}
}
else {
@@ -1708,6 +1735,30 @@
return 0;
}
+/**
+ * Look for all dirname/v4l-touch* devices, and open them.
+ */
+status_t EventHub::scanVideoDirLocked(const std::string& dirname)
+{
+ DIR* dir;
+ struct dirent* de;
+ dir = opendir(dirname.c_str());
+ if(!dir) {
+ ALOGE("Could not open video directory %s", dirname.c_str());
+ return BAD_VALUE;
+ }
+
+ while((de = readdir(dir))) {
+ const char* name = de->d_name;
+ if (isV4lTouchNode(name)) {
+ ALOGI("Found touch video device %s", name);
+ openVideoDeviceLocked(dirname + "/" + name);
+ }
+ }
+ closedir(dir);
+ return OK;
+}
+
void EventHub::requestReopenDevices() {
ALOGV("requestReopenDevices() called");
@@ -1754,6 +1805,14 @@
dump += StringPrintf(INDENT3 "HaveKeyboardLayoutOverlay: %s\n",
toString(device->overlayKeyMap != nullptr));
}
+
+ dump += INDENT "Unattached video devices:\n";
+ for (const std::unique_ptr<TouchVideoDevice>& videoDevice : mUnattachedVideoDevices) {
+ dump += INDENT2 + videoDevice->dump() + "\n";
+ }
+ if (mUnattachedVideoDevices.empty()) {
+ dump += INDENT2 "<none>\n";
+ }
} // release lock
}
diff --git a/services/inputflinger/TouchVideoDevice.cpp b/services/inputflinger/TouchVideoDevice.cpp
new file mode 100644
index 0000000..ad70ccc
--- /dev/null
+++ b/services/inputflinger/TouchVideoDevice.cpp
@@ -0,0 +1,247 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "TouchVideoDevice.h"
+
+#define LOG_TAG "TouchVideoDevice"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <linux/videodev2.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <iostream>
+
+#include <android-base/stringprintf.h>
+#include <android-base/unique_fd.h>
+#include <log/log.h>
+
+using android::base::StringPrintf;
+using android::base::unique_fd;
+
+namespace android {
+
+TouchVideoDevice::TouchVideoDevice(int fd, std::string&& name, std::string&& devicePath,
+ uint32_t width, uint32_t height,
+ const std::array<const int16_t*, NUM_BUFFERS>& readLocations) :
+ mFd(fd), mName(std::move(name)), mPath(std::move(devicePath)),
+ mWidth(width), mHeight(height),
+ mReadLocations(readLocations) {
+ mFrames.reserve(MAX_QUEUE_SIZE);
+};
+
+std::unique_ptr<TouchVideoDevice> TouchVideoDevice::create(std::string devicePath) {
+ unique_fd fd(open(devicePath.c_str(), O_RDWR | O_NONBLOCK));
+ if (fd.get() == INVALID_FD) {
+ ALOGE("Could not open video device %s: %s", devicePath.c_str(), strerror(errno));
+ return nullptr;
+ }
+
+ struct v4l2_capability cap;
+ int result = ioctl(fd.get(), VIDIOC_QUERYCAP, &cap);
+ if (result == -1) {
+ ALOGE("VIDIOC_QUERYCAP failed: %s", strerror(errno));
+ return nullptr;
+ }
+ if (!(cap.capabilities & V4L2_CAP_TOUCH)) {
+ ALOGE("Capability V4L2_CAP_TOUCH is not present, can't use device for heatmap data. "
+ "Make sure device specifies V4L2_CAP_TOUCH");
+ return nullptr;
+ }
+ ALOGI("Opening video device: driver = %s, card = %s, bus_info = %s, version = %i",
+ cap.driver, cap.card, cap.bus_info, cap.version);
+ std::string name = reinterpret_cast<const char*>(cap.card);
+
+ struct v4l2_input v4l2_input_struct;
+ result = ioctl(fd.get(), VIDIOC_ENUMINPUT, &v4l2_input_struct);
+ if (result == -1) {
+ ALOGE("VIDIOC_ENUMINPUT failed: %s", strerror(errno));
+ return nullptr;
+ }
+
+ if (v4l2_input_struct.type != V4L2_INPUT_TYPE_TOUCH) {
+ ALOGE("Video device does not provide touch data. "
+ "Make sure device specifies V4L2_INPUT_TYPE_TOUCH.");
+ return nullptr;
+ }
+
+ struct v4l2_format v4l2_fmt;
+ v4l2_fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+ result = ioctl(fd.get(), VIDIOC_G_FMT, &v4l2_fmt);
+ if (result == -1) {
+ ALOGE("VIDIOC_G_FMT failed: %s", strerror(errno));
+ return nullptr;
+ }
+ const uint32_t width = v4l2_fmt.fmt.pix.width;
+ const uint32_t height = v4l2_fmt.fmt.pix.height;
+ ALOGI("Frame dimensions: width = %" PRIu32 " height = %" PRIu32, width, height);
+
+ struct v4l2_requestbuffers req;
+ req.count = NUM_BUFFERS;
+ req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+ req.memory = V4L2_MEMORY_MMAP;
+ result = ioctl(fd.get(), VIDIOC_REQBUFS, &req);
+ if (result == -1) {
+ ALOGE("VIDIOC_REQBUFS failed: %s", strerror(errno));
+ return nullptr;
+ }
+ if (req.count != NUM_BUFFERS) {
+ ALOGE("Requested %zu buffers, but driver responded with count=%i", NUM_BUFFERS, req.count);
+ return nullptr;
+ }
+
+ struct v4l2_buffer buf = {};
+ buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+ buf.memory = V4L2_MEMORY_MMAP;
+ std::array<const int16_t*, NUM_BUFFERS> readLocations;
+ for (size_t i = 0; i < NUM_BUFFERS; i++) {
+ buf.index = i;
+ result = ioctl(fd.get(), VIDIOC_QUERYBUF, &buf);
+ if (result == -1) {
+ ALOGE("VIDIOC_QUERYBUF failed: %s", strerror(errno));
+ return nullptr;
+ }
+ if (buf.length != width * height * sizeof(int16_t)) {
+ ALOGE("Unexpected value of buf.length = %i (offset = %" PRIu32 ")",
+ buf.length, buf.m.offset);
+ return nullptr;
+ }
+
+ readLocations[i] = static_cast<const int16_t*>(mmap(nullptr /* start anywhere */,
+ buf.length, PROT_READ /* required */, MAP_SHARED /* recommended */,
+ fd.get(), buf.m.offset));
+ if (readLocations[i] == MAP_FAILED) {
+ ALOGE("%s: map failed: %s", __func__, strerror(errno));
+ return nullptr;
+ }
+ }
+
+ enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+ result = ioctl(fd.get(), VIDIOC_STREAMON, &type);
+ if (result == -1) {
+ ALOGE("VIDIOC_STREAMON failed: %s", strerror(errno));
+ return nullptr;
+ }
+
+ for (size_t i = 0; i < NUM_BUFFERS; i++) {
+ buf.index = i;
+ result = ioctl(fd.get(), VIDIOC_QBUF, &buf);
+ if (result == -1) {
+ ALOGE("VIDIOC_QBUF failed for buffer %zu: %s", i, strerror(errno));
+ return nullptr;
+ }
+ }
+ // Using 'new' to access a non-public constructor.
+ return std::unique_ptr<TouchVideoDevice>(new TouchVideoDevice(
+ fd.release(), std::move(name), std::move(devicePath), width, height, readLocations));
+}
+
+size_t TouchVideoDevice::readAndQueueFrames() {
+ std::vector<TouchVideoFrame> frames = readFrames();
+ const size_t numFrames = frames.size();
+ if (numFrames == 0) {
+ // Likely an error occurred
+ return 0;
+ }
+ // Concatenate the vectors, then clip up to maximum size allowed
+ mFrames.insert(mFrames.end(), std::make_move_iterator(frames.begin()),
+ std::make_move_iterator(frames.end()));
+ if (mFrames.size() > MAX_QUEUE_SIZE) {
+ ALOGE("More than %zu frames have been accumulated. Dropping %zu frames", MAX_QUEUE_SIZE,
+ mFrames.size() - MAX_QUEUE_SIZE);
+ mFrames.erase(mFrames.begin(), mFrames.end() - MAX_QUEUE_SIZE);
+ }
+ return numFrames;
+}
+
+std::vector<TouchVideoFrame> TouchVideoDevice::consumeFrames() {
+ std::vector<TouchVideoFrame> frames = std::move(mFrames);
+ mFrames = {};
+ return frames;
+}
+
+std::optional<TouchVideoFrame> TouchVideoDevice::readFrame() {
+ struct v4l2_buffer buf = {};
+ buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+ buf.memory = V4L2_MEMORY_MMAP;
+ int result = ioctl(mFd.get(), VIDIOC_DQBUF, &buf);
+ if (result == -1) {
+ // EAGAIN means we've reached the end of the read buffer, so it's expected.
+ if (errno != EAGAIN) {
+ ALOGE("VIDIOC_DQBUF failed: %s", strerror(errno));
+ }
+ return std::nullopt;
+ }
+ if ((buf.flags & V4L2_BUF_FLAG_TIMESTAMP_MASK) != V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC) {
+ // We use CLOCK_MONOTONIC for input events, so if the clocks don't match,
+ // we can't compare timestamps. Just log a warning, since this is a driver issue
+ ALOGW("The timestamp %ld.%ld was not acquired using CLOCK_MONOTONIC",
+ buf.timestamp.tv_sec, buf.timestamp.tv_usec);
+ }
+ std::vector<int16_t> data(mWidth * mHeight);
+ const int16_t* readFrom = mReadLocations[buf.index];
+ std::copy(readFrom, readFrom + mWidth * mHeight, data.begin());
+ TouchVideoFrame frame(mWidth, mHeight, std::move(data), buf.timestamp);
+
+ result = ioctl(mFd.get(), VIDIOC_QBUF, &buf);
+ if (result == -1) {
+ ALOGE("VIDIOC_QBUF failed: %s", strerror(errno));
+ }
+ return std::make_optional(std::move(frame));
+}
+
+/*
+ * This function should not be called unless buffer is ready! This must be checked with
+ * select, poll, epoll, or some other similar api first.
+ * The oldest frame will be at the beginning of the array.
+ */
+std::vector<TouchVideoFrame> TouchVideoDevice::readFrames() {
+ std::vector<TouchVideoFrame> frames;
+ while (true) {
+ std::optional<TouchVideoFrame> frame = readFrame();
+ if (!frame) {
+ break;
+ }
+ frames.push_back(std::move(*frame));
+ }
+ return frames;
+}
+
+TouchVideoDevice::~TouchVideoDevice() {
+ enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+ int result = ioctl(mFd.get(), VIDIOC_STREAMOFF, &type);
+ if (result == -1) {
+ ALOGE("VIDIOC_STREAMOFF failed: %s", strerror(errno));
+ }
+ for (const int16_t* buffer : mReadLocations) {
+ void* bufferAddress = static_cast<void*>(const_cast<int16_t*>(buffer));
+ result = munmap(bufferAddress, mWidth * mHeight * sizeof(int16_t));
+ if (result == -1) {
+ ALOGE("%s: Couldn't unmap: [%s]", __func__, strerror(errno));
+ }
+ }
+}
+
+std::string TouchVideoDevice::dump() const {
+ return StringPrintf("Video device %s (%s) : width=%" PRIu32 ", height=%" PRIu32
+ ", fd=%i, hasValidFd=%s",
+ mName.c_str(), mPath.c_str(), mWidth, mHeight, mFd.get(),
+ hasValidFd() ? "true" : "false");
+}
+
+} // namespace android
diff --git a/services/inputflinger/include/EventHub.h b/services/inputflinger/include/EventHub.h
index 1ddb978..744b2a3 100644
--- a/services/inputflinger/include/EventHub.h
+++ b/services/inputflinger/include/EventHub.h
@@ -37,6 +37,8 @@
#include <linux/input.h>
#include <sys/epoll.h>
+#include "TouchVideoDevice.h"
+
/* Convenience constants. */
#define BTN_FIRST 0x100 // first button code
@@ -378,11 +380,13 @@
};
status_t openDeviceLocked(const char *devicePath);
+ void openVideoDeviceLocked(const std::string& devicePath);
void createVirtualKeyboardLocked();
void addDeviceLocked(Device* device);
void assignDescriptorLocked(InputDeviceIdentifier& identifier);
- status_t closeDeviceByPathLocked(const char *devicePath);
+ void closeDeviceByPathLocked(const char *devicePath);
+ void closeVideoDeviceByPathLocked(const std::string& devicePath);
void closeDeviceLocked(Device* device);
void closeAllDevicesLocked();
@@ -397,6 +401,7 @@
status_t unregisterDeviceFromEpollLocked(Device* device);
status_t scanDirLocked(const char *dirname);
+ status_t scanVideoDirLocked(const std::string& dirname);
void scanDevicesLocked();
status_t readNotifyLocked();
@@ -438,6 +443,14 @@
BitSet32 mControllerNumbers;
KeyedVector<int32_t, Device*> mDevices;
+ /**
+ * Video devices that report touchscreen heatmap, but have not (yet) been paired
+ * with a specific input device. Video device discovery is independent from input device
+ * discovery, so the two types of devices could be found in any order.
+ * Ideally, video devices in this queue do not have an open fd, or at least aren't
+ * actively streaming.
+ */
+ std::vector<std::unique_ptr<TouchVideoDevice>> mUnattachedVideoDevices;
Device *mOpeningDevices;
Device *mClosingDevices;
diff --git a/services/inputflinger/include/TouchVideoDevice.h b/services/inputflinger/include/TouchVideoDevice.h
new file mode 100644
index 0000000..7ff2653
--- /dev/null
+++ b/services/inputflinger/include/TouchVideoDevice.h
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _INPUTFLINGER_TOUCH_VIDEO_DEVICE_H
+#define _INPUTFLINGER_TOUCH_VIDEO_DEVICE_H
+
+#include <array>
+#include <android-base/unique_fd.h>
+#include <input/TouchVideoFrame.h>
+#include <optional>
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+namespace android {
+
+/**
+ * Represents a video device that uses v4l2 api to report touch heatmap data.
+ */
+class TouchVideoDevice {
+public:
+ /**
+ * Create a new TouchVideoDevice for the path provided.
+ * Return nullptr upon failure.
+ */
+ static std::unique_ptr<TouchVideoDevice> create(std::string devicePath);
+ ~TouchVideoDevice();
+
+ bool hasValidFd() const { return mFd.get() != INVALID_FD; }
+ /**
+ * Obtain the file descriptor associated with this video device.
+ * Could be used for adding to epoll.
+ */
+ int getFd() const { return mFd.get(); }
+ /**
+ * Get the name of this video device.
+ */
+ const std::string& getName() const { return mName; }
+ /**
+ * Get the file path of this video device.
+ */
+ const std::string& getPath() const { return mPath; }
+ /**
+ * Get the width of the heatmap frame
+ */
+ uint32_t getWidth() const { return mWidth; }
+ /**
+ * Get the height of the heatmap frame
+ */
+ uint32_t getHeight() const { return mHeight; }
+ /**
+ * Direct read of the frame. Stores the frame into internal buffer.
+ * Return the number of frames that were successfully read.
+ *
+ * This function should not be called unless buffer is ready!
+ * This must be checked with select, poll, epoll, or similar api first.
+ * If epoll indicates that there is data ready to read, but this function
+ * returns zero, then it is likely an error occurred.
+ */
+ size_t readAndQueueFrames();
+ /**
+ * Return all of the queued frames, and erase them from the local buffer.
+ */
+ std::vector<TouchVideoFrame> consumeFrames();
+ /**
+ * Get string representation of this video device.
+ */
+ std::string dump() const;
+
+private:
+ android::base::unique_fd mFd;
+ std::string mName;
+ std::string mPath;
+
+ uint32_t mWidth;
+ uint32_t mHeight;
+
+ static constexpr int INVALID_FD = -1;
+ /**
+ * How many buffers to request for heatmap.
+ * The kernel driver will be allocating these buffers for us,
+ * and will provide memory locations to read these from.
+ */
+ static constexpr size_t NUM_BUFFERS = 3;
+ std::array<const int16_t*, NUM_BUFFERS> mReadLocations;
+ /**
+ * How many buffers to keep for the internal queue. When the internal buffer
+ * exceeds this capacity, oldest frames will be dropped.
+ */
+ static constexpr size_t MAX_QUEUE_SIZE = 10;
+ std::vector<TouchVideoFrame> mFrames;
+
+ /**
+ * The constructor is private because opening a v4l2 device requires many checks.
+ * To get a new TouchVideoDevice, use 'create' instead.
+ */
+ explicit TouchVideoDevice(int fd, std::string&& name, std::string&& devicePath,
+ uint32_t width, uint32_t height,
+ const std::array<const int16_t*, NUM_BUFFERS>& readLocations);
+ /**
+ * Read all currently available frames.
+ */
+ std::vector<TouchVideoFrame> readFrames();
+ /**
+ * Read a single frame. May return nullopt if no data is currently available for reading.
+ */
+ std::optional<TouchVideoFrame> readFrame();
+};
+} // namespace android
+#endif //_INPUTFLINGER_TOUCH_VIDEO_DEVICE_H