adb: Use kernel aio for functionfs.
This method works around the downsides of
ENDPOINT_ALLOC, namely that it is not affected
by memory fragmentation and it uses an upstream
interface.
Also add libasyncio to provide the necessary syscalls
to both adb and mtp.
Add some small optimizations to file_sync.
Bug: 37916658
Test: run adb push/pull
Change-Id: If3b3be02b5e2d4f9cffec1b8ddc02a5768a51a1f
diff --git a/adb/daemon/usb.cpp b/adb/daemon/usb.cpp
index 0f92282..87ed3db 100644
--- a/adb/daemon/usb.cpp
+++ b/adb/daemon/usb.cpp
@@ -26,6 +26,7 @@
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
+#include <sys/mman.h>
#include <sys/types.h>
#include <unistd.h>
@@ -49,16 +50,11 @@
#define MAX_PACKET_SIZE_HS 512
#define MAX_PACKET_SIZE_SS 1024
-// Kernels before 3.3 have a 16KiB transfer limit That limit was replaced
-// with a 16MiB global limit in 3.3, but each URB submitted required a
-// contiguous kernel allocation, so you would get ENOMEM if you tried to
-// send something larger than the biggest available contiguous kernel
-// memory region. Large contiguous allocations could be unreliable
-// on a device kernel that has been running for a while fragmenting its
-// memory so we start with a larger allocation, and shrink the amount if
-// necessary.
#define USB_FFS_BULK_SIZE 16384
+// Number of buffers needed to fit MAX_PAYLOAD, with an extra for ZLPs.
+#define USB_FFS_NUM_BUFS ((MAX_PAYLOAD / USB_FFS_BULK_SIZE) + 1)
+
#define cpu_to_le16(x) htole16(x)
#define cpu_to_le32(x) htole32(x)
@@ -234,6 +230,26 @@
},
};
+static void aio_block_init(aio_block* aiob) {
+ aiob->iocb.resize(USB_FFS_NUM_BUFS);
+ aiob->iocbs.resize(USB_FFS_NUM_BUFS);
+ aiob->events.resize(USB_FFS_NUM_BUFS);
+ aiob->num_submitted = 0;
+ for (unsigned i = 0; i < USB_FFS_NUM_BUFS; i++) {
+ aiob->iocbs[i] = &aiob->iocb[i];
+ }
+}
+
+static int getMaxPacketSize(int ffs_fd) {
+ usb_endpoint_descriptor desc;
+ if (ioctl(ffs_fd, FUNCTIONFS_ENDPOINT_DESC, reinterpret_cast<unsigned long>(&desc))) {
+ D("[ could not get endpoint descriptor! (%d) ]", errno);
+ return MAX_PACKET_SIZE_HS;
+ } else {
+ return desc.wMaxPacketSize;
+ }
+}
+
bool init_functionfs(struct usb_handle* h) {
LOG(INFO) << "initializing functionfs";
@@ -301,6 +317,14 @@
goto err;
}
+ if (io_setup(USB_FFS_NUM_BUFS, &h->read_aiob.ctx) ||
+ io_setup(USB_FFS_NUM_BUFS, &h->write_aiob.ctx)) {
+ D("[ aio: got error on io_setup (%d) ]", errno);
+ }
+
+ h->read_aiob.fd = h->bulk_out;
+ h->write_aiob.fd = h->bulk_in;
+
h->max_rw = MAX_PAYLOAD;
while (h->max_rw >= USB_FFS_BULK_SIZE && retries < ENDPOINT_ALLOC_RETRIES) {
int ret_in = ioctl(h->bulk_in, FUNCTIONFS_ENDPOINT_ALLOC, static_cast<__u32>(h->max_rw));
@@ -410,6 +434,65 @@
return 0;
}
+static int usb_ffs_do_aio(usb_handle* h, const void* data, int len, bool read) {
+ aio_block* aiob = read ? &h->read_aiob : &h->write_aiob;
+ bool zero_packet = false;
+
+ int num_bufs = len / USB_FFS_BULK_SIZE + (len % USB_FFS_BULK_SIZE == 0 ? 0 : 1);
+ const char* cur_data = reinterpret_cast<const char*>(data);
+ int packet_size = getMaxPacketSize(aiob->fd);
+
+ if (posix_madvise(const_cast<void*>(data), len, POSIX_MADV_SEQUENTIAL | POSIX_MADV_WILLNEED) <
+ 0) {
+ D("[ Failed to madvise: %d ]", errno);
+ }
+
+ for (int i = 0; i < num_bufs; i++) {
+ int buf_len = std::min(len, USB_FFS_BULK_SIZE);
+ io_prep(&aiob->iocb[i], aiob->fd, cur_data, buf_len, 0, read);
+
+ len -= buf_len;
+ cur_data += buf_len;
+
+ if (len == 0 && buf_len % packet_size == 0 && read) {
+ // adb does not expect the device to send a zero packet after data transfer,
+ // but the host *does* send a zero packet for the device to read.
+ zero_packet = true;
+ }
+ }
+ if (zero_packet) {
+ io_prep(&aiob->iocb[num_bufs], aiob->fd, reinterpret_cast<const void*>(cur_data),
+ packet_size, 0, read);
+ num_bufs += 1;
+ }
+
+ if (io_submit(aiob->ctx, num_bufs, aiob->iocbs.data()) < num_bufs) {
+ D("[ aio: got error submitting %s (%d) ]", read ? "read" : "write", errno);
+ return -1;
+ }
+ if (TEMP_FAILURE_RETRY(
+ io_getevents(aiob->ctx, num_bufs, num_bufs, aiob->events.data(), nullptr)) < num_bufs) {
+ D("[ aio: got error waiting %s (%d) ]", read ? "read" : "write", errno);
+ return -1;
+ }
+ for (int i = 0; i < num_bufs; i++) {
+ if (aiob->events[i].res < 0) {
+ errno = aiob->events[i].res;
+ D("[ aio: got error event on %s (%d) ]", read ? "read" : "write", errno);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int usb_ffs_aio_read(usb_handle* h, void* data, int len) {
+ return usb_ffs_do_aio(h, data, len, true);
+}
+
+static int usb_ffs_aio_write(usb_handle* h, const void* data, int len) {
+ return usb_ffs_do_aio(h, data, len, false);
+}
+
static void usb_ffs_kick(usb_handle* h) {
int err;
@@ -438,6 +521,9 @@
h->kicked = false;
adb_close(h->bulk_out);
adb_close(h->bulk_in);
+ io_destroy(h->read_aiob.ctx);
+ io_destroy(h->write_aiob.ctx);
+
// Notify usb_adb_open_thread to open a new connection.
h->lock.lock();
h->open_new_connection = true;
@@ -450,8 +536,17 @@
usb_handle* h = new usb_handle();
- h->write = usb_ffs_write;
- h->read = usb_ffs_read;
+ if (android::base::GetBoolProperty("sys.usb.ffs.aio_compat", false)) {
+ // Devices on older kernels (< 3.18) will not have aio support for ffs
+ // unless backported. Fall back on the non-aio functions instead.
+ h->write = usb_ffs_write;
+ h->read = usb_ffs_read;
+ } else {
+ h->write = usb_ffs_aio_write;
+ h->read = usb_ffs_aio_read;
+ aio_block_init(&h->read_aiob);
+ aio_block_init(&h->write_aiob);
+ }
h->kick = usb_ffs_kick;
h->close = usb_ffs_close;